diff --git a/Gemfile.lock b/Gemfile.lock index 1a59976afe2a49b0ef2201bad6dadfd90de07de9..1299adfc7f89b54a3958383f0a07096cd4d82dac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -76,7 +76,7 @@ GEM annotate (3.1.1) activerecord (>= 3.2, < 7.0) rake (>= 10.4, < 14.0) - autoprefixer-rails (10.3.3.0) + autoprefixer-rails (10.4.0.0) execjs (~> 2) aws-eventstream (1.2.0) aws-partitions (1.543.0) @@ -123,7 +123,7 @@ GEM xpath (~> 3.2) childprocess (4.1.0) concurrent-ruby (1.1.9) - countries (4.1.2) + countries (4.1.3) i18n_data (~> 0.15.0) sixarm_ruby_unaccent (~> 1.1) country_select (6.0.0) @@ -222,18 +222,18 @@ GEM kamifusen (1.10.6) image_processing rails - kaminari (1.2.1) + kaminari (1.2.2) activesupport (>= 4.1.0) - kaminari-actionview (= 1.2.1) - kaminari-activerecord (= 1.2.1) - kaminari-core (= 1.2.1) - kaminari-actionview (1.2.1) + kaminari-actionview (= 1.2.2) + kaminari-activerecord (= 1.2.2) + kaminari-core (= 1.2.2) + kaminari-actionview (1.2.2) actionview - kaminari-core (= 1.2.1) - kaminari-activerecord (1.2.1) + kaminari-core (= 1.2.2) + kaminari-activerecord (1.2.2) activerecord - kaminari-core (= 1.2.1) - kaminari-core (1.2.1) + kaminari-core (= 1.2.2) + kaminari-core (1.2.2) listen (3.7.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) @@ -405,7 +405,7 @@ GEM websocket-extensions (0.1.5) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.5.1) + zeitwerk (2.5.3) PLATFORMS ruby diff --git a/app/controllers/admin/application_controller.rb b/app/controllers/admin/application_controller.rb index 0f004a7487b42dd693e61e3ee6c28f88bf0394a4..59994e6fe5383ceda3bfda8196c3aae99186baa4 100644 --- a/app/controllers/admin/application_controller.rb +++ b/app/controllers/admin/application_controller.rb @@ -3,6 +3,7 @@ class Admin::ApplicationController < ApplicationController before_action :authenticate_user! around_action :switch_locale + after_action :push_to_github protected diff --git a/app/models/communication/website.rb b/app/models/communication/website.rb index 2553b6d9debb529c1fdf0e9fbb680f77b9176cbb..89fa30249840ecb6136f6b9692bccebe4524c357 100644 --- a/app/models/communication/website.rb +++ b/app/models/communication/website.rb @@ -27,9 +27,11 @@ # fk_rails_... (university_id => universities.id) # class Communication::Website < ApplicationRecord - include Communication::Website::WithBatchPublication + include Communication::Website::WithGit include Communication::Website::WithCategories - include Communication::Website::WithPublishableObjects + +# include Communication::Website::WithBatchPublication +# include Communication::Website::WithPublishableObjects belongs_to :university belongs_to :about, polymorphic: true, optional: true diff --git a/app/models/communication/website/github_file.rb b/app/models/communication/website/github_file.rb index a915bd9c2d109edcfeb1f541087686cb94c72aef..a8d720c97b222695ed7317ea6a7d55a5b825e68b 100644 --- a/app/models/communication/website/github_file.rb +++ b/app/models/communication/website/github_file.rb @@ -26,6 +26,10 @@ class Communication::Website::GithubFile < ApplicationRecord after_destroy :remove_from_github + def needs_sync? + false + end + def publish return unless valid_for_publication? && github.valid? add_to_batch(github) diff --git a/app/models/communication/website/page.rb b/app/models/communication/website/page.rb index 3763dd68e34b8a501dd8c853b9fe409a54774a3f..9df86f987ad1d2eff03ad5be4d270981a162c9b8 100644 --- a/app/models/communication/website/page.rb +++ b/app/models/communication/website/page.rb @@ -40,6 +40,7 @@ class Communication::Website::Page < ApplicationRecord include Communication::Website::WithMedia include WithGithubFiles + include WithGitSync include WithMenuItemTarget include WithSlug # We override slug_unavailable? method include WithTree diff --git a/app/models/communication/website/with_git.rb b/app/models/communication/website/with_git.rb new file mode 100644 index 0000000000000000000000000000000000000000..d0a7b9471a2df9e456f1a5c60a7b12c0063e14e5 --- /dev/null +++ b/app/models/communication/website/with_git.rb @@ -0,0 +1,25 @@ +module Communication::Website::WithGit + extend ActiveSupport::Concern + + included do + after_commit :push_to_git + end + + def sync_file(github_file) + if github_file.needs_sync? + repository.add_to_batch github_file + touch! + end + end + + def push_to_git + repository.sync! if repository.needs_sync? + end + handle_asynchronously :push_to_git + + protected + + def repository + @repository ||= Git::Repository.new self + end +end diff --git a/app/models/concerns/with_git_sync.rb b/app/models/concerns/with_git_sync.rb new file mode 100644 index 0000000000000000000000000000000000000000..115071e413e9eee7d54c49c70acde32efbcef1fe --- /dev/null +++ b/app/models/concerns/with_git_sync.rb @@ -0,0 +1,21 @@ +module WithWithGitSync + extend ActiveSupport::Concern + + included do + after_save :add_to_git_batch + end + + protected + + def list_of_websites + respond_to?(:websites) ? websites : [website] + end + + def add_to_git_batch + list_of_websites.each do |website| + file = Git::File.new + file.path = github_path_generated + file.previous_path + end + end +end diff --git a/app/models/concerns/with_github_files.rb b/app/models/concerns/with_github_files.rb index f9762bb6f958267361dda84807ffe818b766795d..f3fb6e65af45bbd154971dc8dd22f1d7785497ff 100644 --- a/app/models/concerns/with_github_files.rb +++ b/app/models/concerns/with_github_files.rb @@ -10,23 +10,23 @@ module WithGithubFiles dependent: :destroy after_save :create_github_files - after_save_commit :publish_github_files, unless: :skip_github_publication - after_save_commit :unpublish_github_files, if: :should_unpublish_github_files? + # after_save_commit :publish_github_files, unless: :skip_github_publication + # after_save_commit :unpublish_github_files, if: :should_unpublish_github_files? end - def force_publish! - publish_github_files - end + # def force_publish! + # publish_github_files + # end def github_path_generated "content/#{self.class.name.demodulize.pluralize.underscore}/#{self.slug}/_index.html" end - def to_static(github_file) + def to_static ApplicationController.render( template: "admin/#{self.class.name.underscore.pluralize}/static", layout: false, - assigns: { self.class.name.demodulize.underscore => self, github_file: github_file } + assigns: { self.class.name.demodulize.underscore => self } ) end diff --git a/app/services/git/providers/github.rb b/app/services/git/providers/github.rb new file mode 100644 index 0000000000000000000000000000000000000000..a987033614e982ec8258dc4ae48e4a1020e3f2da --- /dev/null +++ b/app/services/git/providers/github.rb @@ -0,0 +1,126 @@ +class Git::Providers::Github + + def add_to_batch( path: nil, + previous_path: nil, + data:) + @batch ||= [] + file = find_in_tree previous_path + if file.nil? # New file + @batch << { + path: path, + mode: '100644', # https://docs.github.com/en/rest/reference/git#create-a-tree + type: 'blob', + content: data + } + elsif previous_path != path || file_sha(previous_path) != local_file_sha(data) + # Different path or content + @batch << { + path: previous_path, + mode: file[:mode], + type: file[:type], + sha: nil + } + @batch << { + path: path, + mode: file[:mode], + type: file[:type], + content: data + } + end + end + + def commit_batch(commit_message) + unless @batch.empty? + new_tree = client.create_tree repository, @batch, base_tree: tree[:sha] + commit = client.create_commit repository, commit_message, new_tree[:sha], branch_sha + client.update_branch repository, default_branch, commit[:sha] + end + @tree = nil + true + end + + def remove(path, commit_message) + client.delete_contents repository, path, commit_message, file_sha(path) + true + rescue + false + end + + def read_file_at(path) + data = client.content repository, path: path + Base64.decode64 data.content + rescue + '' + end + + protected + + def client + @client ||= Octokit::Client.new access_token: access_token + end + + # https://medium.com/@obodley/renaming-a-file-using-the-git-api-fed1e6f04188 + def move_file(from, to) + file = find_in_tree from + return if file.nil? + content = [{ + path: from, + mode: file[:mode], + type: file[:type], + sha: nil + }, + { + path: to, + mode: file[:mode], + type: file[:type], + sha: file[:sha] + }] + new_tree = client.create_tree repository, content, base_tree: tree[:sha] + message = "Move #{from} to #{to}" + commit = client.create_commit repository, message, new_tree[:sha], branch_sha + client.update_branch repository, default_branch, commit[:sha] + @tree = nil + true + rescue + false + end + + def file_sha(path) + begin + content = client.content repository, path: path + sha = content[:sha] + rescue + sha = nil + end + sha + end + + def local_file_sha(data) + # Git SHA-1 is calculated from the String "blob <length>\x00<contents>" + # Source: https://alblue.bandlem.com/2011/08/git-tip-of-week-objects.html + OpenSSL::Digest::SHA1.hexdigest "blob #{data.bytesize}\x00#{data}" + end + + def default_branch + @default_branch ||= client.repo(repository)[:default_branch] + end + + def branch_sha + @branch_sha ||= client.branch(repository, default_branch)[:commit][:sha] + end + + def tree + @tree ||= client.tree repository, branch_sha, recursive: true + end + + def find_in_tree(path) + tree[:tree].each do |file| + return file if path == file[:path] + end + nil + end + + def tmp_directory + "tmp/github/#{repository}" + end +end diff --git a/app/services/github.rb b/app/services/git/repository.rb similarity index 84% rename from app/services/github.rb rename to app/services/git/repository.rb index f218accbfce3e8b5cceed266ced462e33a8192a9..0c55992aa7a93151f7d75a8210f8bb364216c23e 100644 --- a/app/services/github.rb +++ b/app/services/git/repository.rb @@ -1,39 +1,33 @@ -class Github - attr_reader :website, :access_token, :repository - - def self.with_website(website) - new website - end +class Git::Repository + attr_reader :website def initialize(website) @website = website - @access_token = website&.access_token - @repository = website&.repository + end + + def access_token + @access_token ||= website&.access_token + end + + def repository + @repository ||= website&.repository + end + + def provider + @provider ||= Git::Providers::Github.new + end + + def files + @files ||= [] end def valid? repository.present? && access_token.present? end - def publish(path: nil, - previous_path: nil, - commit: nil, - data:) - local_path = "#{ tmp_directory }/#{ path }" - Pathname(local_path).dirname.mkpath - File.write local_path, data - return if repository.blank? - if !previous_path.blank? && path != previous_path - move_file previous_path, path - end - client.create_contents repository, - path, - commit, - file: local_path, - sha: file_sha(path) - true - rescue => e - false + def push(commit_message: nil) + return unless files.any? + # TODO add files to batch and commit end def add_to_batch( path: nil, diff --git a/docs/websites/export.md b/docs/websites/export.md index a7bb28a667f6952c7362fde84af8ba9392632946..209dfde27b40cd42735713883eb95f61542344a8 100644 --- a/docs/websites/export.md +++ b/docs/websites/export.md @@ -1,5 +1,49 @@ # Export +## Contexte + +Chaque website peut avoir un repository git. +Tous les objets de ce website doivent être synchronisés sur le repository. +Les publications doivent se font en asynchrone parce qu'elles peuvent être longues. + + +Certains objets peuvent appartenir à plusieurs websites, donc plusieurs repositories, comme par exemple les programs. +Certains objets ont des dépendances, par exemple les pages enfants, les auteurs ou les catégories. + + +Les fichiers renommés doivent être déplacés sur git. +Les fichiers supprimés ou dépubliés doivent être supprimés sur git. +Il faut veiller à limiter le nombre de commits, et éviter les commits vides. + +## Architecture + +Les git::providers permettent de dialoguer avec les services comme Github et Gitlab. +Le git::repository sert de façade et abstrait le provider. + + +Chaque objet publiable utilise un objet active record Communication::Website::GitFile qui garde la trace du dernier chemin et du SHA. + +## Flux + +Lors de l'enregistrement d'un objet, il faut : +- créer éventuellement ses git_files (1 pour chaque website) +- envoyer les git_files aux repositories (add_to_batch) +- modifier ses dépendances, qui créent leur git_files pour chaque repository +- envoyer les git_files des dépendances aux repositories respectifs (add_to_batch) +- pour chaque website, si au moins un fichier a été ajouté : + - déclencher une modification (touch), qui génère une action asynchrone : + - pour chaque file : + - générer le fichier statique + - calculer le SHA + - comparer au SHA stocké + - needs_sync si SHA différent ou path différent + - si au moins un needs_sync : + - créer un commit pour tout ça + - push + - mettre à jour les previous_path et les SHA des git_files + +## Code + Tout objet qui doit être exporté sur un ou plusieurs websites doit : - avoir une méthode `website` ou `websites` - inclure le concern `WithGithubFiles`