From c645b94ff8c053b5a42ac447f3370efa02651036 Mon Sep 17 00:00:00 2001
From: Arnaud Levy <contact@arnaudlevy.com>
Date: Mon, 25 Oct 2021 08:45:49 +0200
Subject: [PATCH] batches
---
.../communication/website/imported/page.rb | 1 +
.../communication/website/imported/website.rb | 10 +++
app/models/communication/website/page.rb | 24 +++----
app/models/communication/website/post.rb | 10 ++-
app/models/concerns/with_github.rb | 13 ++--
app/services/github.rb | 69 ++++++++++++++-----
.../communication/websites/show.html.erb | 8 +--
app/views/admin/layouts/application.html.erb | 2 +-
...to_communication_website_imported_pages.rb | 5 ++
db/schema.rb | 3 +-
10 files changed, 94 insertions(+), 51 deletions(-)
create mode 100644 db/migrate/20211025062028_add_index_to_communication_website_imported_pages.rb
diff --git a/app/models/communication/website/imported/page.rb b/app/models/communication/website/imported/page.rb
index 86bd82880..595e88b64 100644
--- a/app/models/communication/website/imported/page.rb
+++ b/app/models/communication/website/imported/page.rb
@@ -23,6 +23,7 @@
# Indexes
#
# idx_communication_website_imported_pages_on_featured_medium_id (featured_medium_id)
+# index_communication_website_imported_pages_on_identifier (identifier)
# index_communication_website_imported_pages_on_page_id (page_id)
# index_communication_website_imported_pages_on_university_id (university_id)
# index_communication_website_imported_pages_on_website_id (website_id)
diff --git a/app/models/communication/website/imported/website.rb b/app/models/communication/website/imported/website.rb
index 882be65aa..dff61d59b 100644
--- a/app/models/communication/website/imported/website.rb
+++ b/app/models/communication/website/imported/website.rb
@@ -52,6 +52,7 @@ class Communication::Website::Imported::Website < ApplicationRecord
end
def sync_pages
+ Communication::Website::Page.skip_callback(:save, :after, :publish_to_github)
wordpress.pages.each do |data|
page = pages.where(university: university, identifier: data['id']).first_or_initialize
page.data = data
@@ -66,6 +67,15 @@ class Communication::Website::Imported::Website < ApplicationRecord
generated_page.parent = parent.page
generated_page.save
end
+ # Batch update all changes (1 query only, good for github API limits)
+ github = Github.with_site website
+ website.pages.find_each do |page|
+ github.add_to_batch path: page.github_path_generated,
+ previous_path: page.github_path,
+ data: page.to_jekyll
+ end
+ github.commit_batch '[Page] Batch update from import'
+ Communication::Website::Page.set_callback(:save, :after, :publish_to_github)
end
def sync_posts
diff --git a/app/models/communication/website/page.rb b/app/models/communication/website/page.rb
index a1900da43..b8b94fb27 100644
--- a/app/models/communication/website/page.rb
+++ b/app/models/communication/website/page.rb
@@ -64,6 +64,18 @@ class Communication::Website::Page < ApplicationRecord
children.any?
end
+ def github_path_generated
+ "_pages/#{path}/index.html".gsub('//', '/')
+ end
+
+ def to_jekyll
+ ApplicationController.render(
+ template: 'admin/communication/website/pages/jekyll',
+ layout: false,
+ assigns: { page: self }
+ )
+ end
+
def to_s
"#{ title }"
end
@@ -77,16 +89,4 @@ class Communication::Website::Page < ApplicationRecord
def update_children_paths
children.each(&:save)
end
-
- def github_path_generated
- "_pages/#{path}/index.html".gsub('//', '/')
- end
-
- def to_jekyll
- ApplicationController.render(
- template: 'admin/communication/website/pages/jekyll',
- layout: false,
- assigns: { page: self }
- )
- end
end
diff --git a/app/models/communication/website/post.rb b/app/models/communication/website/post.rb
index 88e15c76f..9e25cdd85 100644
--- a/app/models/communication/website/post.rb
+++ b/app/models/communication/website/post.rb
@@ -46,12 +46,6 @@ class Communication::Website::Post < ApplicationRecord
validates :title, presence: true
- def to_s
- "#{title}"
- end
-
- protected
-
def github_path_generated
"_posts/#{published_at.strftime "%Y/%m"}/#{published_at.strftime "%Y-%m-%d"}-#{slug}.html"
end
@@ -63,4 +57,8 @@ class Communication::Website::Post < ApplicationRecord
assigns: { post: self }
)
end
+
+ def to_s
+ "#{title}"
+ end
end
diff --git a/app/models/concerns/with_github.rb b/app/models/concerns/with_github.rb
index 9fe689123..7aa18027e 100644
--- a/app/models/concerns/with_github.rb
+++ b/app/models/concerns/with_github.rb
@@ -28,11 +28,12 @@ module WithGithub
end
def publish_to_github
- github.publish path: github_path_generated,
- previous_path: github_path,
- commit: github_commit_message,
- data: to_jekyll
- update_column :github_path, github_path_generated
+ if github.publish(path: github_path_generated,
+ previous_path: github_path,
+ commit: github_commit_message,
+ data: to_jekyll)
+ update_column :github_path, github_path_generated
+ end
end
- handle_asynchronously :publish_to_github
+ handle_asynchronously :publish_to_github, queue: 'default'
end
diff --git a/app/services/github.rb b/app/services/github.rb
index ce50a1d6e..dd068ed3f 100644
--- a/app/services/github.rb
+++ b/app/services/github.rb
@@ -26,8 +26,9 @@ class Github
commit,
file: local_path,
sha: file_sha(path)
+ true
rescue
- # byebug
+ false
end
def send_file(attachment, path)
@@ -39,8 +40,44 @@ class Github
commit_message,
attachment.download,
sha: file_sha(path)
+ true
rescue
- # byebug
+ false
+ end
+
+ def add_to_batch( path: nil,
+ previous_path: nil,
+ data:)
+ @batch ||= []
+ file = find_in_tree previous_path
+ if file.nil? # New file
+ @batch << {
+ path: path,
+ mode: '100644', # https://docs.github.com/en/rest/reference/git#create-a-tree
+ type: 'blob',
+ content: data
+ }
+ else # Existing file
+ @batch << {
+ path: previous_path,
+ mode: file[:mode],
+ type: file[:type],
+ sha: nil
+ }
+ @batch << {
+ path: path,
+ mode: file[:mode],
+ type: file[:type],
+ content: data
+ }
+ end
+ end
+
+ def commit_batch(commit_message)
+ new_tree = client.create_tree repository, @batch, base_tree: tree[:sha]
+ commit = client.create_commit repository, commit_message, new_tree[:sha], branch_sha
+ client.update_branch repository, default_branch, commit[:sha]
+ @tree = nil
end
def read_file_at(path)
@@ -50,6 +87,8 @@ class Github
''
end
+ protected
+
def pages
list = client.contents repository, path: '_pages'
list.map do |hash|
@@ -75,8 +114,6 @@ class Github
@client ||= Octokit::Client.new access_token: access_token
end
- protected
-
# https://medium.com/@obodley/renaming-a-file-using-the-git-api-fed1e6f04188
def move_file(from, to)
file = find_in_tree from
@@ -96,11 +133,11 @@ class Github
new_tree = client.create_tree repository, content, base_tree: tree[:sha]
message = "Move #{from} to #{to}"
commit = client.create_commit repository, message, new_tree[:sha], branch_sha
- [:main, :master].each do |branch|
- client.update_branch repository, branch, commit[:sha]
- end
+ client.update_branch repository, default_branch, commit[:sha]
+ @tree = nil
+ true
rescue
- ''
+ false
end
def file_sha(path)
@@ -113,18 +150,12 @@ class Github
sha
end
+ def default_branch
+ @default_branch ||= client.repo(repository)[:default_branch]
+ end
+
def branch_sha
- unless @branch_sha
- [:main, :master].each do |branch|
- begin
- # Crashes if branch does not exist
- response = client.branch repository, branch
- @branch_sha = response['commit']['sha']
- end
- break if @branch_sha
- end
- end
- @branch_sha
+ @branch_sha ||= client.branch(repository, default_branch)[:commit][:sha]
end
def tree
diff --git a/app/views/admin/communication/websites/show.html.erb b/app/views/admin/communication/websites/show.html.erb
index 50efde949..2a7724826 100644
--- a/app/views/admin/communication/websites/show.html.erb
+++ b/app/views/admin/communication/websites/show.html.erb
@@ -1,14 +1,10 @@
<% content_for :title, @website %>
<% content_for :title_right do %>
- <%= link_to @website.domain_url, @website.domain_url, target: :_blank %>
-<% end %>
-
-
-<p>
+ <%= link_to @website.domain_url, @website.domain_url, target: :_blank %><br>
<%= I18n.t("activerecord.attributes.communication/website.about_#{@website.about_type}") %>
<%= link_to @website.about, [:admin, @website.about] unless @website.about.nil? %>
-</p>
+<% end %>
<div class="card mt-5">
<div class="card-header">
diff --git a/app/views/admin/layouts/application.html.erb b/app/views/admin/layouts/application.html.erb
index 003a54d20..94b603dc8 100644
--- a/app/views/admin/layouts/application.html.erb
+++ b/app/views/admin/layouts/application.html.erb
@@ -31,7 +31,7 @@
<%= render 'admin/application/top' %>
<main class="content">
<div class="container-fluid p-0">
- <p class="float-end pt-2"><%= yield :title_right %></p>
+ <p class="float-end text-end pt-1"><%= yield :title_right %></p>
<h1><%= yield :title %></h1>
<%= yield %>
</div>
diff --git a/db/migrate/20211025062028_add_index_to_communication_website_imported_pages.rb b/db/migrate/20211025062028_add_index_to_communication_website_imported_pages.rb
new file mode 100644
index 000000000..fa7fd412a
--- /dev/null
+++ b/db/migrate/20211025062028_add_index_to_communication_website_imported_pages.rb
@@ -0,0 +1,5 @@
+class AddIndexToCommunicationWebsiteImportedPages < ActiveRecord::Migration[6.1]
+ def change
+ add_index :communication_website_imported_pages, :identifier
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 3449b82fb..a46467ee3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2021_10_23_153416) do
+ActiveRecord::Schema.define(version: 2021_10_25_062028) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
@@ -112,6 +112,7 @@ ActiveRecord::Schema.define(version: 2021_10_23_153416) do
t.jsonb "data"
t.uuid "featured_medium_id"
t.index ["featured_medium_id"], name: "idx_communication_website_imported_pages_on_featured_medium_id"
+ t.index ["identifier"], name: "index_communication_website_imported_pages_on_identifier"
t.index ["page_id"], name: "index_communication_website_imported_pages_on_page_id"
t.index ["university_id"], name: "index_communication_website_imported_pages_on_university_id"
t.index ["website_id"], name: "index_communication_website_imported_pages_on_website_id"
--
GitLab