From df2a814b5c65f31cb7b4ccb10354dc3da12e976f Mon Sep 17 00:00:00 2001 From: Arnaud Levy <contact@arnaudlevy.com> Date: Mon, 3 Jan 2022 21:55:16 +0100 Subject: [PATCH] page can be saved --- .../admin/application_controller.rb | 1 - .../communication/website/pages_controller.rb | 2 + app/models/administration/member.rb | 2 +- app/models/communication/website.rb | 67 +++---- app/models/communication/website/category.rb | 5 +- app/models/communication/website/git_file.rb | 120 ++++++++++++ .../communication/website/github_file.rb | 132 ------------- app/models/communication/website/home.rb | 7 +- app/models/communication/website/menu.rb | 5 +- app/models/communication/website/page.rb | 8 +- app/models/communication/website/post.rb | 7 +- app/models/communication/website/with_git.rb | 25 --- .../website/with_git_repository.rb | 7 + app/models/concerns/with_git.rb | 47 +++++ app/models/concerns/with_git_sync.rb | 21 -- app/models/concerns/with_github_files.rb | 90 --------- .../website => concerns}/with_media.rb | 2 +- app/models/education/program.rb | 7 +- app/models/education/school.rb | 7 +- app/models/research/journal.rb | 7 +- app/models/research/journal/article.rb | 2 +- app/models/research/journal/volume.rb | 2 +- app/services/git/providers/github.rb | 26 --- app/services/git/repository.rb | 179 ++++++------------ .../communication/website/pages/show.html.erb | 2 +- db/migrate/20220103162509_rename_git_files.rb | 5 + db/migrate/20220103174641_change_git_file.rb | 8 + db/schema.rb | 13 +- docs/websites/export.md | 65 ++++++- docs/websites/prototype.md | 16 +- 30 files changed, 375 insertions(+), 512 deletions(-) create mode 100644 app/models/communication/website/git_file.rb delete mode 100644 app/models/communication/website/github_file.rb delete mode 100644 app/models/communication/website/with_git.rb create mode 100644 app/models/communication/website/with_git_repository.rb create mode 100644 app/models/concerns/with_git.rb delete mode 100644 app/models/concerns/with_git_sync.rb delete mode 100644 app/models/concerns/with_github_files.rb rename app/models/{communication/website => concerns}/with_media.rb (94%) create mode 100644 db/migrate/20220103162509_rename_git_files.rb create mode 100644 db/migrate/20220103174641_change_git_file.rb diff --git a/app/controllers/admin/application_controller.rb b/app/controllers/admin/application_controller.rb index 59994e6fe..0f004a748 100644 --- a/app/controllers/admin/application_controller.rb +++ b/app/controllers/admin/application_controller.rb @@ -3,7 +3,6 @@ class Admin::ApplicationController < ApplicationController before_action :authenticate_user! around_action :switch_locale - after_action :push_to_github protected diff --git a/app/controllers/admin/communication/website/pages_controller.rb b/app/controllers/admin/communication/website/pages_controller.rb index 17fb1954a..4fe68ac63 100644 --- a/app/controllers/admin/communication/website/pages_controller.rb +++ b/app/controllers/admin/communication/website/pages_controller.rb @@ -52,6 +52,7 @@ class Admin::Communication::Website::PagesController < Admin::Communication::Web def create @page.website = @website if @page.save + @page.sync_with_git redirect_to admin_communication_website_page_path(@page), notice: t('admin.successfully_created_html', model: @page.to_s) else breadcrumb @@ -61,6 +62,7 @@ class Admin::Communication::Website::PagesController < Admin::Communication::Web def update if @page.update(page_params) + @page.sync_with_git redirect_to admin_communication_website_page_path(@page), notice: t('admin.successfully_updated_html', model: @page.to_s) else breadcrumb diff --git a/app/models/administration/member.rb b/app/models/administration/member.rb index bf3afeffa..7ffa80d32 100644 --- a/app/models/administration/member.rb +++ b/app/models/administration/member.rb @@ -28,7 +28,7 @@ # fk_rails_... (user_id => users.id) # class Administration::Member < ApplicationRecord - include WithGithubFiles + include WithGit include WithSlug has_rich_text :biography diff --git a/app/models/communication/website.rb b/app/models/communication/website.rb index 89fa30249..9fafe6d66 100644 --- a/app/models/communication/website.rb +++ b/app/models/communication/website.rb @@ -27,38 +27,37 @@ # fk_rails_... (university_id => universities.id) # class Communication::Website < ApplicationRecord - include Communication::Website::WithGit - include Communication::Website::WithCategories - -# include Communication::Website::WithBatchPublication -# include Communication::Website::WithPublishableObjects - - belongs_to :university - belongs_to :about, polymorphic: true, optional: true - has_one :home, - class_name: 'Communication::Website::Home', - foreign_key: :communication_website_id, - dependent: :destroy - has_many :pages, - foreign_key: :communication_website_id, - dependent: :destroy - has_many :posts, - foreign_key: :communication_website_id, - dependent: :destroy - has_many :categories, - class_name: 'Communication::Website::Category', - foreign_key: :communication_website_id, - dependent: :destroy - has_many :menus, - class_name: 'Communication::Website::Menu', - foreign_key: :communication_website_id, - dependent: :destroy - has_one :imported_website, - class_name: 'Communication::Website::Imported::Website', - dependent: :destroy - has_many :github_files, - class_name: 'Communication::Website::GithubFile', - dependent: :destroy + include WithGitRepository + include WithCategories + + belongs_to :university + belongs_to :about, + polymorphic: true, + optional: true + has_one :home, + class_name: 'Communication::Website::Home', + foreign_key: :communication_website_id, + dependent: :destroy + has_many :pages, + foreign_key: :communication_website_id, + dependent: :destroy + has_many :posts, + foreign_key: :communication_website_id, + dependent: :destroy + has_many :categories, + class_name: 'Communication::Website::Category', + foreign_key: :communication_website_id, + dependent: :destroy + has_many :menus, + class_name: 'Communication::Website::Menu', + foreign_key: :communication_website_id, + dependent: :destroy + has_one :imported_website, + class_name: 'Communication::Website::Imported::Website', + dependent: :destroy + has_many :git_files, + class_name: 'Communication::Website::GitFile', + dependent: :destroy after_create :create_home after_save :publish_about_object, if: :saved_change_to_about_id? @@ -121,10 +120,6 @@ class Communication::Website < ApplicationRecord build_home(university_id: university_id).save end - def github - @github ||= Github.with_website self - end - def about_school? about_type == 'Education::School' end diff --git a/app/models/communication/website/category.rb b/app/models/communication/website/category.rb index 79a489d5c..b0a3bae4e 100644 --- a/app/models/communication/website/category.rb +++ b/app/models/communication/website/category.rb @@ -32,7 +32,7 @@ # fk_rails_... (university_id => universities.id) # class Communication::Website::Category < ApplicationRecord - include WithGithubFiles + include WithGit include WithMenuItemTarget include WithSlug # We override slug_unavailable? method include WithTree @@ -79,8 +79,7 @@ class Communication::Website::Category < ApplicationRecord "#{name}" end - # Override from WithGithubFiles - def github_path_generated + def git_path_static "content/categories/#{path}/_index.html".gsub(/\/+/, '/') end diff --git a/app/models/communication/website/git_file.rb b/app/models/communication/website/git_file.rb new file mode 100644 index 000000000..ebf1afb61 --- /dev/null +++ b/app/models/communication/website/git_file.rb @@ -0,0 +1,120 @@ +# == Schema Information +# +# Table name: communication_website_git_files +# +# id :uuid not null, primary key +# about_type :string not null +# identifier :string default("static") +# previous_path :string +# previous_sha :string +# created_at :datetime not null +# updated_at :datetime not null +# about_id :uuid not null +# website_id :uuid not null +# +# Indexes +# +# index_communication_website_git_files_on_website_id (website_id) +# index_communication_website_github_files_on_about (about_type,about_id) +# +# Foreign Keys +# +# fk_rails_... (website_id => communication_websites.id) +# +class Communication::Website::GitFile < ApplicationRecord + belongs_to :website, class_name: 'Communication::Website' + belongs_to :about, polymorphic: true + + def synced? + previous_path == path && previous_sha == sha + end + + def path + about.send "git_path_#{identifier}" + end + + def sha + # Git SHA-1 is calculated from the String "blob <length>\x00<contents>" + # Source: https://alblue.bandlem.com/2011/08/git-tip-of-week-objects.html + data = to_s + OpenSSL::Digest::SHA1.hexdigest "blob #{data.bytesize}\x00#{data}" + end + + def to_s + ApplicationController.render( + template: "admin/#{about.class.name.underscore.pluralize}/#{identifier}", + layout: false, + assigns: { about.class.name.demodulize.downcase => about } + ) + end + + protected + + # def add_media_to_batch(github) + # return unless manifest_data[:has_media] && about.respond_to?(:active_storage_blobs) + # about.active_storage_blobs.each { |blob| add_blob_to_batch(github, blob) } + # end + # + # def add_blob_to_batch(github, blob) + # github.add_to_batch github_blob_params(blob) + # end + # + # def remove_from_github + # return unless github.valid? + # github.remove github_path, github_remove_commit_message + # remove_media_from_github + # end + # + # def remove_media_from_github + # return unless manifest_data[:with_media] && about.respond_to?(:active_storage_blobs) + # about.active_storage_blobs.each { |blob| remove_blob_from_github(blob) } + # end + # + # def remove_blob_from_github(blob) + # github.remove github_blob_path(blob), github_blob_remove_commit_message + # end + # + # def github_params + # { + # path: manifest_data[:generated_path].call(self), + # previous_path: github_path, + # data: manifest_data[:data].call(self) + # } + # end + # + # def github_blob_params(blob) + # blob.analyze unless blob.analyzed? + # { + # path: github_blob_path(blob), + # data: ApplicationController.render( + # template: 'active_storage/blobs/static', + # layout: false, + # assigns: { blob: blob } + # ) + # } + # end + # + # def github_blob_path(blob) + # "data/media/#{blob.id[0..1]}/#{blob.id}.yml" + # end + # + # def github_commit_message + # "[#{about.class.name.demodulize} - #{manifest_identifier}] Save #{about.to_s}" + # end + # + # def github_remove_commit_message + # "[#{about.class.name.demodulize} - #{manifest_identifier}] Remove #{about.to_s}" + # end + # + # def github_blob_remove_commit_message(blob) + # "[Medium] Remove ##{blob.id}" + # end + # + # def valid_for_publication? + # if about.respond_to?(:published) + # about.published? + # else + # true + # end + # end +end diff --git a/app/models/communication/website/github_file.rb b/app/models/communication/website/github_file.rb deleted file mode 100644 index a8d720c97..000000000 --- a/app/models/communication/website/github_file.rb +++ /dev/null @@ -1,132 +0,0 @@ -# == Schema Information -# -# Table name: communication_website_github_files -# -# id :uuid not null, primary key -# about_type :string not null -# github_path :string -# manifest_identifier :string -# created_at :datetime not null -# updated_at :datetime not null -# about_id :uuid not null -# website_id :uuid not null -# -# Indexes -# -# index_communication_website_github_files_on_about (about_type,about_id) -# index_communication_website_github_files_on_website_id (website_id) -# -# Foreign Keys -# -# fk_rails_... (website_id => communication_websites.id) -# -class Communication::Website::GithubFile < ApplicationRecord - belongs_to :website, class_name: 'Communication::Website' - belongs_to :about, polymorphic: true - - after_destroy :remove_from_github - - def needs_sync? - false - end - - def publish - return unless valid_for_publication? && github.valid? - add_to_batch(github) - if github.commit_batch(github_commit_message) - update_column :github_path, manifest_data[:generated_path].call(self) - end - end - handle_asynchronously :publish, queue: 'default' - - def unpublish - remove_from_github - end - handle_asynchronously :unpublish, queue: 'default' - - def add_to_batch(github) - return unless valid_for_publication? - github.add_to_batch github_params - add_media_to_batch(github) - end - - def manifest_data - @manifest_data ||= about.github_manifest.detect { |item| - item[:identifier] == manifest_identifier - } - end - - protected - - def add_media_to_batch(github) - return unless manifest_data[:has_media] && about.respond_to?(:active_storage_blobs) - about.active_storage_blobs.each { |blob| add_blob_to_batch(github, blob) } - end - - def add_blob_to_batch(github, blob) - github.add_to_batch github_blob_params(blob) - end - - def remove_from_github - return unless github.valid? - github.remove github_path, github_remove_commit_message - remove_media_from_github - end - - def remove_media_from_github - return unless manifest_data[:with_media] && about.respond_to?(:active_storage_blobs) - about.active_storage_blobs.each { |blob| remove_blob_from_github(blob) } - end - - def remove_blob_from_github(blob) - github.remove github_blob_path(blob), github_blob_remove_commit_message - end - - def github - @github ||= Github.with_website(website) - end - - def github_params - { - path: manifest_data[:generated_path].call(self), - previous_path: github_path, - data: manifest_data[:data].call(self) - } - end - - def github_blob_params(blob) - blob.analyze unless blob.analyzed? - { - path: github_blob_path(blob), - data: ApplicationController.render( - template: 'active_storage/blobs/static', - layout: false, - assigns: { blob: blob } - ) - } - end - - def github_blob_path(blob) - "data/media/#{blob.id[0..1]}/#{blob.id}.yml" - end - - def github_commit_message - "[#{about.class.name.demodulize} - #{manifest_identifier}] Save #{about.to_s}" - end - - def github_remove_commit_message - "[#{about.class.name.demodulize} - #{manifest_identifier}] Remove #{about.to_s}" - end - - def github_blob_remove_commit_message(blob) - "[Medium] Remove ##{blob.id}" - end - - def valid_for_publication? - if about.respond_to?(:published) - about.published? - else - true - end - end -end diff --git a/app/models/communication/website/home.rb b/app/models/communication/website/home.rb index 54d68f7aa..23370016c 100644 --- a/app/models/communication/website/home.rb +++ b/app/models/communication/website/home.rb @@ -21,8 +21,8 @@ # fk_rails_... (university_id => universities.id) # class Communication::Website::Home < ApplicationRecord - include Communication::Website::WithMedia - include WithGithubFiles + include WithGit + include WithMedia belongs_to :university belongs_to :website, foreign_key: :communication_website_id @@ -42,8 +42,7 @@ class Communication::Website::Home < ApplicationRecord ) end - # Override from WithGithubFiles - def github_path_generated + def git_path_static 'content/_index.html' end diff --git a/app/models/communication/website/menu.rb b/app/models/communication/website/menu.rb index 4a8ea972f..e66a5a16d 100644 --- a/app/models/communication/website/menu.rb +++ b/app/models/communication/website/menu.rb @@ -22,7 +22,7 @@ # fk_rails_... (university_id => universities.id) # class Communication::Website::Menu < ApplicationRecord - include WithGithubFiles + include WithGit belongs_to :university belongs_to :website, foreign_key: :communication_website_id @@ -37,8 +37,7 @@ class Communication::Website::Menu < ApplicationRecord "#{title}" end - # Override from WithGithubFiles - def github_path_generated + def git_path_static "data/menus/#{identifier}.yml" end diff --git a/app/models/communication/website/page.rb b/app/models/communication/website/page.rb index 9df86f987..6ff766a5d 100644 --- a/app/models/communication/website/page.rb +++ b/app/models/communication/website/page.rb @@ -38,9 +38,8 @@ # class Communication::Website::Page < ApplicationRecord - include Communication::Website::WithMedia - include WithGithubFiles - include WithGitSync + include WithGit + include WithMedia include WithMenuItemTarget include WithSlug # We override slug_unavailable? method include WithTree @@ -72,8 +71,7 @@ class Communication::Website::Page < ApplicationRecord scope :ordered, -> { order(:position) } scope :recent, -> { order(updated_at: :desc).limit(5) } - # Override from WithGithubFiles - def github_path_generated + def git_path_static "content/pages/#{path}/_index.html".gsub(/\/+/, '/') end diff --git a/app/models/communication/website/post.rb b/app/models/communication/website/post.rb index 3158b2887..290c08f4e 100644 --- a/app/models/communication/website/post.rb +++ b/app/models/communication/website/post.rb @@ -31,8 +31,8 @@ # fk_rails_... (university_id => universities.id) # class Communication::Website::Post < ApplicationRecord - include Communication::Website::WithMedia - include WithGithubFiles + include WithGit + include WithMedia include WithMenuItemTarget include WithSlug # We override slug_unavailable? method @@ -67,8 +67,7 @@ class Communication::Website::Post < ApplicationRecord "/#{website.posts_github_directory}/#{published_at.strftime "%Y/%m/%d"}/#{slug}/" end - # Override from WithGithubFiles - def github_path_generated + def git_path_static "content/posts/#{published_at.year}/#{published_at.strftime "%Y-%m-%d"}-#{slug}.html" end diff --git a/app/models/communication/website/with_git.rb b/app/models/communication/website/with_git.rb deleted file mode 100644 index d0a7b9471..000000000 --- a/app/models/communication/website/with_git.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Communication::Website::WithGit - extend ActiveSupport::Concern - - included do - after_commit :push_to_git - end - - def sync_file(github_file) - if github_file.needs_sync? - repository.add_to_batch github_file - touch! - end - end - - def push_to_git - repository.sync! if repository.needs_sync? - end - handle_asynchronously :push_to_git - - protected - - def repository - @repository ||= Git::Repository.new self - end -end diff --git a/app/models/communication/website/with_git_repository.rb b/app/models/communication/website/with_git_repository.rb new file mode 100644 index 000000000..9317c1ba5 --- /dev/null +++ b/app/models/communication/website/with_git_repository.rb @@ -0,0 +1,7 @@ +module Communication::Website::WithGitRepository + extend ActiveSupport::Concern + + def git_repository + @git_repository ||= Git::Repository.new self + end +end diff --git a/app/models/concerns/with_git.rb b/app/models/concerns/with_git.rb new file mode 100644 index 000000000..f078c1bc0 --- /dev/null +++ b/app/models/concerns/with_git.rb @@ -0,0 +1,47 @@ +module WithGit + extend ActiveSupport::Concern + + included do + has_many :git_files, + class_name: "Communication::Website::GitFile", + as: :about, + dependent: :destroy + end + + def sync_with_git + websites.each do |website| + identifiers.each do |identifier| + git_file = git_files.where(website: website, about: self, identifier: identifier).first_or_create + website.git_repository.add_git_file git_file + end + website.git_repository.sync! + end + end + handle_asynchronously :sync_with_git + + + def git_path_static + "" + end + + # Overridden for multiple files generation + def identifiers + [:static] + end + + # Overridden if websites relation exists + def websites + [website] + end + + protected + + def sync_git_files + websites.each do |website| + identifiers.each do |identifier| + git_file = git_files.where(website: website, about: self, identifier: identifier).first_or_create + website.sync_file git_file + end + end + end +end diff --git a/app/models/concerns/with_git_sync.rb b/app/models/concerns/with_git_sync.rb deleted file mode 100644 index 115071e41..000000000 --- a/app/models/concerns/with_git_sync.rb +++ /dev/null @@ -1,21 +0,0 @@ -module WithWithGitSync - extend ActiveSupport::Concern - - included do - after_save :add_to_git_batch - end - - protected - - def list_of_websites - respond_to?(:websites) ? websites : [website] - end - - def add_to_git_batch - list_of_websites.each do |website| - file = Git::File.new - file.path = github_path_generated - file.previous_path - end - end -end diff --git a/app/models/concerns/with_github_files.rb b/app/models/concerns/with_github_files.rb deleted file mode 100644 index f3fb6e65a..000000000 --- a/app/models/concerns/with_github_files.rb +++ /dev/null @@ -1,90 +0,0 @@ -module WithGithubFiles - extend ActiveSupport::Concern - - included do - attr_accessor :skip_github_publication - - has_many :github_files, - class_name: "Communication::Website::GithubFile", - as: :about, - dependent: :destroy - - after_save :create_github_files - # after_save_commit :publish_github_files, unless: :skip_github_publication - # after_save_commit :unpublish_github_files, if: :should_unpublish_github_files? - end - - # def force_publish! - # publish_github_files - # end - - def github_path_generated - "content/#{self.class.name.demodulize.pluralize.underscore}/#{self.slug}/_index.html" - end - - def to_static - ApplicationController.render( - template: "admin/#{self.class.name.underscore.pluralize}/static", - layout: false, - assigns: { self.class.name.demodulize.underscore => self } - ) - end - - def github_manifest - [ - { - identifier: "primary", - generated_path: -> (github_file) { github_path_generated }, - data: -> (github_file) { to_static(github_file) }, - has_media: true - } - ] - end - - def list_of_websites - respond_to?(:websites) ? websites : [website] - end - - protected - - def create_github_files - list_of_websites.each do |website| - github_manifest.each do |manifest_item| - github_files.where(website: website, manifest_identifier: manifest_item[:identifier]).first_or_create - end - end - end - - def publish_github_files - if respond_to?(:descendents) - publish_github_files_with_descendents - else - list_of_websites.each do |website| - github_manifest.each do |manifest_item| - github_file = github_files.where(website: website, manifest_identifier: manifest_item[:identifier]).first_or_create - github_file.publish - end - end - end - end - - def publish_github_files_with_descendents - target_objects = [self, descendents].flatten - list_of_websites.each do |current_website| - github = Github.with_website current_website - github.send_batch_to_website(target_objects, message: "[#{self.class.name.demodulize}] Save #{to_s} & descendents") - end - end - - def unpublish_github_files - list_of_websites.each do |current_website| - github_manifest.each do |manifest_item| - github_files.find_by(website: current_website, manifest_identifier: manifest_item[:identifier])&.unpublish - end - end - end - - def should_unpublish_github_files? - respond_to?(:published?) && saved_change_to_published? && !published? - end -end diff --git a/app/models/communication/website/with_media.rb b/app/models/concerns/with_media.rb similarity index 94% rename from app/models/communication/website/with_media.rb rename to app/models/concerns/with_media.rb index c96a5bcc0..250aaa59f 100644 --- a/app/models/communication/website/with_media.rb +++ b/app/models/concerns/with_media.rb @@ -1,4 +1,4 @@ -module Communication::Website::WithMedia +module WithMedia extend ActiveSupport::Concern def active_storage_blobs diff --git a/app/models/education/program.rb b/app/models/education/program.rb index ceef499a7..998df27c4 100644 --- a/app/models/education/program.rb +++ b/app/models/education/program.rb @@ -30,12 +30,12 @@ # fk_rails_... (university_id => universities.id) # class Education::Program < ApplicationRecord - include WithGithubFiles + include WithGit + include WithMedia include WithMenuItemTarget include WithSlug include WithTree include WithInheritance - include Communication::Website::WithMedia rich_text_areas_with_inheritance :accessibility, :contacts, @@ -107,8 +107,7 @@ class Education::Program < ApplicationRecord best_image end - # Override from WithGithubFiles - def github_path_generated + def git_path_static "content/programs/#{path}/_index.html".gsub(/\/+/, '/') end diff --git a/app/models/education/school.rb b/app/models/education/school.rb index 724b69e2a..84660f756 100644 --- a/app/models/education/school.rb +++ b/app/models/education/school.rb @@ -24,7 +24,7 @@ # fk_rails_... (university_id => universities.id) # class Education::School < ApplicationRecord - include WithGithubFiles + include WithGit belongs_to :university has_many :websites, class_name: 'Communication::Website', as: :about @@ -43,9 +43,8 @@ class Education::School < ApplicationRecord "#{name}" end - def github_path_generated - # Override from WithGithubFiles - "_data/school.yml" + def git_path_static + "data/school.yml" end def to_static(github_file) diff --git a/app/models/research/journal.rb b/app/models/research/journal.rb index f05a836dc..0a695ba4d 100644 --- a/app/models/research/journal.rb +++ b/app/models/research/journal.rb @@ -21,7 +21,7 @@ # fk_rails_... (university_id => universities.id) # class Research::Journal < ApplicationRecord - include WithGithubFiles + include WithGit belongs_to :university has_many :websites, class_name: 'Communication::Website', as: :about @@ -34,9 +34,8 @@ class Research::Journal < ApplicationRecord "#{title}" end - def github_path_generated - # Override from WithGithubFiles - "_data/journal.yml" + def git_path_static + "data/journal.yml" end def to_static(github_file) diff --git a/app/models/research/journal/article.rb b/app/models/research/journal/article.rb index 981887685..479903a0a 100644 --- a/app/models/research/journal/article.rb +++ b/app/models/research/journal/article.rb @@ -32,7 +32,7 @@ # fk_rails_... (updated_by_id => users.id) # class Research::Journal::Article < ApplicationRecord - include WithGithubFiles + include WithGit has_rich_text :text has_one_attached :pdf diff --git a/app/models/research/journal/volume.rb b/app/models/research/journal/volume.rb index e57f5edac..b00038a2a 100644 --- a/app/models/research/journal/volume.rb +++ b/app/models/research/journal/volume.rb @@ -26,7 +26,7 @@ # fk_rails_... (university_id => universities.id) # class Research::Journal::Volume < ApplicationRecord - include WithGithubFiles + include WithGit has_one_attached_deletable :cover diff --git a/app/services/git/providers/github.rb b/app/services/git/providers/github.rb index a98703361..dc67bcbf2 100644 --- a/app/services/git/providers/github.rb +++ b/app/services/git/providers/github.rb @@ -59,32 +59,6 @@ class Git::Providers::Github @client ||= Octokit::Client.new access_token: access_token end - # https://medium.com/@obodley/renaming-a-file-using-the-git-api-fed1e6f04188 - def move_file(from, to) - file = find_in_tree from - return if file.nil? - content = [{ - path: from, - mode: file[:mode], - type: file[:type], - sha: nil - }, - { - path: to, - mode: file[:mode], - type: file[:type], - sha: file[:sha] - }] - new_tree = client.create_tree repository, content, base_tree: tree[:sha] - message = "Move #{from} to #{to}" - commit = client.create_commit repository, message, new_tree[:sha], branch_sha - client.update_branch repository, default_branch, commit[:sha] - @tree = nil - true - rescue - false - end - def file_sha(path) begin content = client.content repository, path: path diff --git a/app/services/git/repository.rb b/app/services/git/repository.rb index 0c55992aa..2c48cb8ff 100644 --- a/app/services/git/repository.rb +++ b/app/services/git/repository.rb @@ -1,10 +1,28 @@ class Git::Repository - attr_reader :website + attr_reader :website, :commit_message def initialize(website) @website = website end + def add_git_file(git_file) + @commit_message = "[#{ git_file.about.class.name }] Save #{ git_file.about }" if git_files.empty? + git_files << git_file + end + + def sync! + return unless valid? + return if git_files.empty? + sync_git_files + mark_as_synced if commit_batch + end + + protected + + def client + @client ||= Octokit::Client.new access_token: access_token + end + def access_token @access_token ||= website&.access_token end @@ -17,40 +35,59 @@ class Git::Repository @provider ||= Git::Providers::Github.new end - def files - @files ||= [] + def git_files + @git_files ||= [] + end + + def batch + @batch ||= [] + end + + def default_branch + @default_branch ||= client.repo(repository)[:default_branch] + end + + def branch_sha + @branch_sha ||= client.branch(repository, default_branch)[:commit][:sha] + end + + def tree + @tree ||= client.tree repository, branch_sha, recursive: true end def valid? repository.present? && access_token.present? end - def push(commit_message: nil) - return unless files.any? - # TODO add files to batch and commit + def sync_git_files + git_files.each do |git_file| + next if git_file.synced? + add_to_batch path: git_file.path, + previous_path: git_file.previous_path, + data: git_file.to_s + end end def add_to_batch( path: nil, previous_path: nil, data:) - @batch ||= [] file = find_in_tree previous_path if file.nil? # New file - @batch << { + batch << { path: path, mode: '100644', # https://docs.github.com/en/rest/reference/git#create-a-tree type: 'blob', content: data } - elsif previous_path != path || file_sha(previous_path) != local_file_sha(data) + elsif previous_path != path || git_sha(previous_path) != sha(data) # Different path or content - @batch << { + batch << { path: previous_path, mode: file[:mode], type: file[:type], sha: nil } - @batch << { + batch << { path: path, mode: file[:mode], type: file[:type], @@ -59,105 +96,21 @@ class Git::Repository end end - def commit_batch(commit_message) - unless @batch.empty? - new_tree = client.create_tree repository, @batch, base_tree: tree[:sha] - commit = client.create_commit repository, commit_message, new_tree[:sha], branch_sha - client.update_branch repository, default_branch, commit[:sha] - end - @tree = nil - true - end - - def remove(path, commit_message) - client.delete_contents repository, path, commit_message, file_sha(path) + def commit_batch + return if batch.empty? + new_tree = client.create_tree repository, batch, base_tree: tree[:sha] + commit = client.create_commit repository, commit_message, new_tree[:sha], branch_sha + client.update_branch repository, default_branch, commit[:sha] true - rescue - false - end - - def read_file_at(path) - data = client.content repository, path: path - Base64.decode64 data.content - rescue - '' end - def send_batch_to_website(objects, message: 'Batch objects') - return unless valid? - - github_files = [] - objects.each do |object| - next unless object.list_of_websites.include? website - object.github_manifest.each do |manifest_item| - github_file = object.github_files.where(website: website, manifest_identifier: manifest_item[:identifier]).first_or_create - github_files << github_file - github_file.add_to_batch(self) - end - end - - if commit_batch(message) - github_files.each do |github_file| - github_file.update_column :github_path, github_file.manifest_data[:generated_path].call(github_file) - end + def mark_as_synced + git_files.each do |git_file| + git_file.update_columns previous_path: git_file.path, previous_sha: git_file.sha end end - handle_asynchronously :send_batch_to_website, queue: 'default' - - protected - - def pages - list = client.contents repository, path: '_pages' - list.map do |hash| - page_with_id(hash[:name]) - end - end - - def page_with_id(id) - path = "_pages/#{id}" - data = client.content repository, path: path - raw = Base64.decode64 data.content - parsed = FrontMatterParser::Parser.new(:md).call(raw) - page = Communication::Website::Page.new - page.id = id - page.title = parsed.front_matter['title'] - page.permalink = parsed.front_matter['permalink'] - page.content = parsed.content - page.raw = raw - page - end - - def client - @client ||= Octokit::Client.new access_token: access_token - end - - # https://medium.com/@obodley/renaming-a-file-using-the-git-api-fed1e6f04188 - def move_file(from, to) - file = find_in_tree from - return if file.nil? - content = [{ - path: from, - mode: file[:mode], - type: file[:type], - sha: nil - }, - { - path: to, - mode: file[:mode], - type: file[:type], - sha: file[:sha] - }] - new_tree = client.create_tree repository, content, base_tree: tree[:sha] - message = "Move #{from} to #{to}" - commit = client.create_commit repository, message, new_tree[:sha], branch_sha - client.update_branch repository, default_branch, commit[:sha] - @tree = nil - true - rescue - false - end - def file_sha(path) + def git_sha(path) begin content = client.content repository, path: path sha = content[:sha] @@ -167,32 +120,16 @@ class Git::Repository sha end - def local_file_sha(data) + def sha(data) # Git SHA-1 is calculated from the String "blob <length>\x00<contents>" # Source: https://alblue.bandlem.com/2011/08/git-tip-of-week-objects.html OpenSSL::Digest::SHA1.hexdigest "blob #{data.bytesize}\x00#{data}" end - def default_branch - @default_branch ||= client.repo(repository)[:default_branch] - end - - def branch_sha - @branch_sha ||= client.branch(repository, default_branch)[:commit][:sha] - end - - def tree - @tree ||= client.tree repository, branch_sha, recursive: true - end - def find_in_tree(path) tree[:tree].each do |file| return file if path == file[:path] end nil end - - def tmp_directory - "tmp/github/#{repository}" - end end diff --git a/app/views/admin/communication/website/pages/show.html.erb b/app/views/admin/communication/website/pages/show.html.erb index 3487ac7eb..b42d6c952 100644 --- a/app/views/admin/communication/website/pages/show.html.erb +++ b/app/views/admin/communication/website/pages/show.html.erb @@ -78,6 +78,7 @@ <% end %> <% content_for :action_bar_left do %> + <%= destroy_link @page %> <%= link_to t('communication.website.force_publication'), publish_admin_communication_website_page_path(@page), method: :post, @@ -86,5 +87,4 @@ <% content_for :action_bar_right do %> <%= edit_link @page %> - <%= destroy_link @page %> <% end %> diff --git a/db/migrate/20220103162509_rename_git_files.rb b/db/migrate/20220103162509_rename_git_files.rb new file mode 100644 index 000000000..a3eed8bca --- /dev/null +++ b/db/migrate/20220103162509_rename_git_files.rb @@ -0,0 +1,5 @@ +class RenameGitFiles < ActiveRecord::Migration[6.1] + def change + rename_table :communication_website_github_files, :communication_website_git_files + end +end diff --git a/db/migrate/20220103174641_change_git_file.rb b/db/migrate/20220103174641_change_git_file.rb new file mode 100644 index 000000000..68962316b --- /dev/null +++ b/db/migrate/20220103174641_change_git_file.rb @@ -0,0 +1,8 @@ +class ChangeGitFile < ActiveRecord::Migration[6.1] + def change + rename_column :communication_website_git_files, :github_path, :previous_path + remove_column :communication_website_git_files, :manifest_identifier + add_column :communication_website_git_files, :previous_sha, :string + add_column :communication_website_git_files, :identifier, :string, default: 'static' + end +end diff --git a/db/schema.rb b/db/schema.rb index c72e6d804..ecbca4813 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_12_24_090935) do +ActiveRecord::Schema.define(version: 2022_01_03_174641) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" @@ -123,16 +123,17 @@ ActiveRecord::Schema.define(version: 2021_12_24_090935) do t.index ["communication_website_post_id", "communication_website_category_id"], name: "post_category" end - create_table "communication_website_github_files", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "github_path" + create_table "communication_website_git_files", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "previous_path" t.string "about_type", null: false t.uuid "about_id", null: false t.uuid "website_id", null: false t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false - t.string "manifest_identifier" + t.string "previous_sha" + t.string "identifier", default: "static" t.index ["about_type", "about_id"], name: "index_communication_website_github_files_on_about" - t.index ["website_id"], name: "index_communication_website_github_files_on_website_id" + t.index ["website_id"], name: "index_communication_website_git_files_on_website_id" end create_table "communication_website_homes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| @@ -552,7 +553,7 @@ ActiveRecord::Schema.define(version: 2021_12_24_090935) do add_foreign_key "communication_website_categories", "communication_websites" add_foreign_key "communication_website_categories", "education_programs", column: "program_id" add_foreign_key "communication_website_categories", "universities" - add_foreign_key "communication_website_github_files", "communication_websites", column: "website_id" + add_foreign_key "communication_website_git_files", "communication_websites", column: "website_id" add_foreign_key "communication_website_homes", "communication_websites" add_foreign_key "communication_website_homes", "universities" add_foreign_key "communication_website_imported_authors", "administration_members", column: "author_id" diff --git a/docs/websites/export.md b/docs/websites/export.md index 209dfde27..a4c71a4ac 100644 --- a/docs/websites/export.md +++ b/docs/websites/export.md @@ -25,9 +25,11 @@ Chaque objet publiable utilise un objet active record Communication::Website::Gi ## Flux -Lors de l'enregistrement d'un objet, il faut : -- créer éventuellement ses git_files (1 pour chaque website) -- envoyer les git_files aux repositories (add_to_batch) +### Version 1 + +Lors de l'enregistrement d'un objet, il faut, pour chaque website : +- créer éventuellement le git_file (1 pour chaque website) +- envoyer le git_file (add_to_batch) - modifier ses dépendances, qui créent leur git_files pour chaque repository - envoyer les git_files des dépendances aux repositories respectifs (add_to_batch) - pour chaque website, si au moins un fichier a été ajouté : @@ -42,16 +44,59 @@ Lors de l'enregistrement d'un objet, il faut : - push - mettre à jour les previous_path et les SHA des git_files +Ce flux cause un problème majeur : tout ce qui est analysé disparaît en asynchrone + +### Version 2 + +Après l'enregistrement d'un objet, il faut, pour chaque website, lancer une tâche asynchrone de synchronisation. +Cette tâche est lancée par les controllers, et intégrée dans le partial `WithGit`. +``` +def create + @page.website = @website + if @page.save + @page.sync_with_git + ... + end +end + +def update + if @page.update(page_params) + @page.sync_with_git + ... + end +end +``` + ## Code -Tout objet qui doit être exporté sur un ou plusieurs websites doit : - - avoir une méthode `website` ou `websites` - - inclure le concern `WithGithubFiles` +### Website::WithRepository + +Le website a un trait WithRepository qui gère son rapport avec le repository Git, quel que soit le provider (Github, Gitlab...). + +### Objets exportables vers Git + +Tous les objets qui doivent être exportés vers Git : +- doivent utiliser le partial `WithGit`, qui gère l'export vers les repositories des objets et de leurs dépendances +- doivent présenter une méthode `websites`, éventuellement avec un seul website dans un tableau +- peuvent intégrer le concern `WithMedia` s'il utilise des médias (`featured_image` et/ou images dans des rich texts) +- peuvent présenter une méthode `static_files` qui liste les identifiants des git_files à générer, pour les objets qui créent plusieurs fichiers + +### GitFile +La responsabilité de la synchronisation repose sur Communication::Website::GitFile, notamment : +- le fichier doit-il être synchronisé ? +- le fichier doit-il être créé ? +- le fichier doit-il être déplacé ? +- le fichier doit-il être supprimé ? -S'il possède des médias (`featured_image` et/ou images dans des rich texts), il doit inclure le concern `Communication::Website::WithMedia` -Le concern `WithGithubFiles` ajoute un manifest à l'objet qui permet de définir les fichiers exportés côté GitHub pour celui-ci. +Pour cela, le git_file dispose des propriétés suivantes : +- previous_path (le chemin à la dernière sauvegarde, nil si pas encore créé, ou détruit) +- previous_sha (le hash de la précédente version, utile pour savoir si le fichier a changé) +- identifier (l'identifiant du fichier à créer, `static` par défaut, pour les objets créant plusieurs fichiers) -Quand l'objet est sauvegardé, on se base sur le(s) websites et ce manifest pour créer et publier des objets `Communication::Website::GithubFile`. Ces derniers permettent de garder la trace du chemin actuel d'un fichier distant dans le cas où celui-ci viendrait à être déplacé (changement de slug, etc.). -Ces fichiers servent également dans le cas où on souhaite republier manuellement une partie d'un site (exemple : tous les posts), la méthode `Communication::Website#publish_posts!` peut tout grouper en un batch. +Et pour générer les fichiers, il dispose des méthodes : +- to_s (pour générer le fichier statique à jour) +- sha (pour calculer le hash du fichier à jour) +- path (pour générer le chemin à jour) +- synced? (pour savoir s'il faut regénérer ou pas) diff --git a/docs/websites/prototype.md b/docs/websites/prototype.md index 1408eaaff..f051d83fc 100644 --- a/docs/websites/prototype.md +++ b/docs/websites/prototype.md @@ -8,17 +8,17 @@ - [x] Gestion volume admin - [x] Gestion article admin - [x] Gestion site -- [ ] Définir about (éventuellement à la main) -- [ ] Création du repo github -- [ ] Copie du template -- [ ] Lecture d'une collection depuis github +- [x] Définir about (éventuellement à la main) +- [ ] Création du repo github -> pas nécessaire, fait par le dev +- [ ] Copie du template -> pas nécessaire, fait par le dev +- [ ] Lecture d'une collection depuis github -> finalement descendant depuis la DB - [x] Ecriture d'un fichier dans github -- [ ] Hébergement Netlify ou autre +- [x] Hébergement Netlify ou autre ### Points à vérifier - [x] Lecture écriture Github -- [ ] Performance avec un backend Github -- [ ] Relations avec les auteurs et les droits (doublon DB, autre...) -- [ ] Articulation avec les objets en DB (volumes et articles par ex.) +- [x] Performance avec un backend Github -> mauvaise -> DB +- [x] Relations avec les auteurs et les droits (doublon DB, autre...) -> full DB +- [x] Articulation avec les objets en DB (volumes et articles par ex.) - [x] Gestion des mises à jour des templates -> utiliser un template Github -- GitLab