From c1baff9b862f56f2d3ab1fa439f7c85e13872a39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Gaya?= <sebastien.gaya@gmail.com>
Date: Tue, 19 Oct 2021 14:08:39 +0200
Subject: [PATCH] imported media
---
Gemfile | 1 +
Gemfile.lock | 1 +
.../communication/websites_controller.rb | 2 +
.../communication/website/imported/medium.rb | 66 +++++++++++++++++++
.../communication/website/imported/website.rb | 11 ++++
app/models/research/journal/article.rb | 2 +-
app/services/wordpress.rb | 4 ++
.../communication/websites/import.html.erb | 36 ++++++++++
config/locales/communication/en.yml | 5 ++
config/locales/communication/fr.yml | 5 ++
...te_communication_website_imported_media.rb | 15 +++++
...to_communication_website_imported_media.rb | 5 ++
db/schema.rb | 21 +++++-
.../communication/website/imported/media.yml | 44 +++++++++++++
.../website/imported/medium_test.rb | 33 ++++++++++
15 files changed, 248 insertions(+), 3 deletions(-)
create mode 100644 app/models/communication/website/imported/medium.rb
create mode 100644 db/migrate/20211019091847_create_communication_website_imported_media.rb
create mode 100644 db/migrate/20211019102112_add_filename_to_communication_website_imported_media.rb
create mode 100644 test/fixtures/communication/website/imported/media.yml
create mode 100644 test/models/communication/website/imported/medium_test.rb
diff --git a/Gemfile b/Gemfile
index e9225d991..1f264b24e 100644
--- a/Gemfile
+++ b/Gemfile
@@ -8,6 +8,7 @@ gem 'pg', '~> 1.1'
gem 'aws-sdk-s3'
gem 'puma'
gem 'image_processing'
+gem 'mini_magick'
gem 'delayed_job_active_record'
gem 'delayed_job_web'
gem 'bootsnap', '>= 1.4.4', require: false
diff --git a/Gemfile.lock b/Gemfile.lock
index aded56c9f..54f6ebefd 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -401,6 +401,7 @@ DEPENDENCIES
kamifusen
kaminari
listen (~> 3.3)
+ mini_magick
octokit
pg (~> 1.1)
puma
diff --git a/app/controllers/admin/communication/websites_controller.rb b/app/controllers/admin/communication/websites_controller.rb
index 261658aac..fbe9b765d 100644
--- a/app/controllers/admin/communication/websites_controller.rb
+++ b/app/controllers/admin/communication/websites_controller.rb
@@ -22,6 +22,8 @@ class Admin::Communication::WebsitesController < Admin::Communication::Applicati
@imported_website = @website.imported_website
@imported_pages = @imported_website.pages.page params[:pages_page]
@imported_posts = @imported_website.posts.page params[:posts_page]
+ @imported_media = @imported_website.media.with_attached_file.page params[:media_page]
+ @imported_media_total_size = @imported_website.media.joins(file_attachment: :blob).sum(:byte_size)
breadcrumb
add_breadcrumb Communication::Website::Imported::Website.model_name.human
end
diff --git a/app/models/communication/website/imported/medium.rb b/app/models/communication/website/imported/medium.rb
new file mode 100644
index 000000000..e3d3ae8c0
--- /dev/null
+++ b/app/models/communication/website/imported/medium.rb
@@ -0,0 +1,66 @@
+# == Schema Information
+#
+# Table name: communication_website_imported_media
+#
+# id :uuid not null, primary key
+# data :jsonb
+# file_url :text
+# identifier :string
+# remote_created_at :datetime
+# remote_updated_at :datetime
+# created_at :datetime not null
+# updated_at :datetime not null
+# university_id :uuid not null
+# website_id :uuid not null
+#
+# Indexes
+#
+# index_communication_website_imported_media_on_university_id (university_id)
+# index_communication_website_imported_media_on_website_id (website_id)
+#
+# Foreign Keys
+#
+# fk_rails_... (university_id => universities.id)
+# fk_rails_... (website_id => communication_website_imported_websites.id)
+#
+class Communication::Website::Imported::Medium < ApplicationRecord
+ belongs_to :university
+ belongs_to :website,
+ class_name: 'Communication::Website::Imported::Website'
+
+ has_one_attached :file
+
+ after_commit :download_file_from_file_url, on: [:create, :update], if: :saved_change_to_file_url
+
+ def data=(value)
+ super value
+ escaped_source_url = Addressable::URI.parse(value['source_url']).display_uri.to_s
+ self.file_url = escaped_source_url
+ self.filename = File.basename(URI(escaped_source_url).path)
+ self.remote_created_at = DateTime.parse(value['date_gmt'])
+ self.remote_updated_at = DateTime.parse(value['modified_gmt'])
+ end
+
+ protected
+
+ def download_file_from_file_url
+ uri = URI(file_url)
+ http = Net::HTTP.new(uri.host, uri.port)
+ http.use_ssl = true
+ # IUT Bordeaux Montaigne pb with certificate
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ request = Net::HTTP::Get.new(uri.request_uri)
+ response = http.request(request)
+ tempfile = Tempfile.open("Osuny-ImportedMedium-#{SecureRandom.hex}", Dir.tmpdir)
+ begin
+ tempfile.binmode
+ tempfile.write(response.body)
+ tempfile.flush
+ tempfile.rewind
+ file.attach(io: tempfile, filename: filename, content_type: data['mime_type'])
+ ensure
+ tempfile.close!
+ end
+ end
+ handle_asynchronously :download_file_from_file_url, queue: 'default'
+end
diff --git a/app/models/communication/website/imported/website.rb b/app/models/communication/website/imported/website.rb
index 962cd135a..612c30377 100644
--- a/app/models/communication/website/imported/website.rb
+++ b/app/models/communication/website/imported/website.rb
@@ -23,12 +23,15 @@ class Communication::Website::Imported::Website < ApplicationRecord
belongs_to :university
belongs_to :website,
class_name: 'Communication::Website'
+ has_many :media,
+ class_name: 'Communication::Website::Imported::Medium'
has_many :pages,
class_name: 'Communication::Website::Imported::Page'
has_many :posts,
class_name: 'Communication::Website::Imported::Post'
def run!
+ sync_media
sync_pages
sync_posts
end
@@ -39,6 +42,14 @@ class Communication::Website::Imported::Website < ApplicationRecord
@wordpress ||= Wordpress.new website.domain_url
end
+ def sync_media
+ wordpress.media.each do |data|
+ medium = media.where(university: university, identifier: data['id']).first_or_create
+ medium.data = data
+ medium.save
+ end
+ end
+
def sync_pages
wordpress.pages.each do |data|
page = pages.where(university: university, identifier: data['id']).first_or_create
diff --git a/app/models/research/journal/article.rb b/app/models/research/journal/article.rb
index 20a415c65..0410c664d 100644
--- a/app/models/research/journal/article.rb
+++ b/app/models/research/journal/article.rb
@@ -10,7 +10,7 @@
# text :text
# title :string
# created_at :datetime not null
-# updated_at :date not null
+# updated_at :datetime not null
# research_journal_id :uuid not null
# research_journal_volume_id :uuid
# university_id :uuid not null
diff --git a/app/services/wordpress.rb b/app/services/wordpress.rb
index b30fa53f7..1d9fc1833 100644
--- a/app/services/wordpress.rb
+++ b/app/services/wordpress.rb
@@ -25,6 +25,10 @@ class Wordpress
load "#{domain}/wp-json/wp/v2/pages"
end
+ def media
+ load "#{domain}/wp-json/wp/v2/media"
+ end
+
protected
def load(url)
diff --git a/app/views/admin/communication/websites/import.html.erb b/app/views/admin/communication/websites/import.html.erb
index a3ba7cf48..086d4f676 100644
--- a/app/views/admin/communication/websites/import.html.erb
+++ b/app/views/admin/communication/websites/import.html.erb
@@ -67,3 +67,39 @@
<%= paginate @imported_pages, param_name: :pages_page, theme: 'bootstrap-5' %>
</div>
</div>
+
+<div class="card mt-5">
+ <div class="card-header">
+ <h2><%= @imported_media.total_count %> media (<%= number_to_human_size(@imported_media_total_size) %>)</h2>
+ </div>
+ <table class="<%= table_classes %>">
+ <thead>
+ <tr>
+ <th><%= Communication::Website::Imported::Medium.human_attribute_name('filename') %></th>
+ <th><%= t('communication.website.imported.media.file_size') %></th>
+ <th class="text-end" width="150"> </th>
+ </tr>
+ </thead>
+ <tbody>
+ <% @imported_media.each do |medium| %>
+ <tr>
+ <td><%= medium.filename %></td>
+ <td><%= number_to_human_size(medium.file.blob.byte_size) if medium.file.attached? %></td>
+ <td class="text-end">
+ <% if medium.file.attached? %>
+ <%= link_to t('show'),
+ url_for(medium.file),
+ class: button_classes,
+ target: :blank %>
+ <% else %>
+ <%= t('communication.website.imported.media.not_imported_yet') %>
+ <% end %>
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+ </table>
+ <div class="card-footer">
+ <%= paginate @imported_media, param_name: :media_page, theme: 'bootstrap-5' %>
+ </div>
+</div>
diff --git a/config/locales/communication/en.yml b/config/locales/communication/en.yml
index 9623e9237..ad4de977c 100644
--- a/config/locales/communication/en.yml
+++ b/config/locales/communication/en.yml
@@ -3,6 +3,9 @@ en:
website:
imported:
launch: Launch import
+ media:
+ file_size: File size
+ not_imported_yet: Not imported yet
refresh: Refresh import
show: Show import
pending: Import in progress
@@ -33,6 +36,8 @@ en:
about_: Nothing (independent website)
about_Research::Journal: Journal website
about_School: School website
+ communication/website/imported/medium:
+ filename: Filename
communication/website/page:
title: Title
description: Description (SEO)
diff --git a/config/locales/communication/fr.yml b/config/locales/communication/fr.yml
index f8c2eb904..a487c22d3 100644
--- a/config/locales/communication/fr.yml
+++ b/config/locales/communication/fr.yml
@@ -3,6 +3,9 @@ fr:
website:
imported:
launch: Importer le site
+ media:
+ file_size: Taille du fichier
+ not_imported_yet: Non importé pour le moment
refresh: Relancer l'import
show: Voir l'import
pending: Import en cours
@@ -33,6 +36,8 @@ fr:
about_: Aucun sujet (site indépendant)
about_Research::Journal: Site de revue scientifique
about_School: Site d'école
+ communication/website/imported/medium:
+ filename: Nom du fichier
communication/website/page:
title: Titre
description: Description (SEO)
diff --git a/db/migrate/20211019091847_create_communication_website_imported_media.rb b/db/migrate/20211019091847_create_communication_website_imported_media.rb
new file mode 100644
index 000000000..b20ab7281
--- /dev/null
+++ b/db/migrate/20211019091847_create_communication_website_imported_media.rb
@@ -0,0 +1,15 @@
+class CreateCommunicationWebsiteImportedMedia < ActiveRecord::Migration[6.1]
+ def change
+ create_table :communication_website_imported_media, id: :uuid do |t|
+ t.string :identifier
+ t.jsonb :data
+ t.text :file_url
+ t.datetime :remote_created_at
+ t.datetime :remote_updated_at
+ t.references :university, null: false, foreign_key: true, type: :uuid
+ t.references :website, null: false, foreign_key: { to_table: :communication_website_imported_websites }, type: :uuid
+
+ t.timestamps
+ end
+ end
+end
diff --git a/db/migrate/20211019102112_add_filename_to_communication_website_imported_media.rb b/db/migrate/20211019102112_add_filename_to_communication_website_imported_media.rb
new file mode 100644
index 000000000..7233345e6
--- /dev/null
+++ b/db/migrate/20211019102112_add_filename_to_communication_website_imported_media.rb
@@ -0,0 +1,5 @@
+class AddFilenameToCommunicationWebsiteImportedMedia < ActiveRecord::Migration[6.1]
+ def change
+ add_column :communication_website_imported_media, :filename, :string
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 683a249f5..7b3b61778 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2021_10_19_092503) do
+ActiveRecord::Schema.define(version: 2021_10_19_102112) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
@@ -68,6 +68,21 @@ ActiveRecord::Schema.define(version: 2021_10_19_092503) do
t.index ["criterion_id"], name: "index_administration_qualiopi_indicators_on_criterion_id"
end
+ create_table "communication_website_imported_media", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+ t.string "identifier"
+ t.jsonb "data"
+ t.text "file_url"
+ t.datetime "remote_created_at"
+ t.datetime "remote_updated_at"
+ t.uuid "university_id", null: false
+ t.uuid "website_id", null: false
+ t.datetime "created_at", precision: 6, null: false
+ t.datetime "updated_at", precision: 6, null: false
+ t.string "filename"
+ t.index ["university_id"], name: "index_communication_website_imported_media_on_university_id"
+ t.index ["website_id"], name: "index_communication_website_imported_media_on_website_id"
+ end
+
create_table "communication_website_imported_pages", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
t.uuid "university_id", null: false
t.uuid "website_id", null: false
@@ -220,7 +235,7 @@ ActiveRecord::Schema.define(version: 2021_10_19_092503) do
t.uuid "research_journal_id", null: false
t.uuid "research_journal_volume_id"
t.datetime "created_at", precision: 6, null: false
- t.date "updated_at", null: false
+ t.datetime "updated_at", precision: 6, null: false
t.uuid "updated_by_id"
t.text "abstract"
t.text "references"
@@ -335,6 +350,8 @@ ActiveRecord::Schema.define(version: 2021_10_19_092503) do
add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id"
add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id"
add_foreign_key "administration_qualiopi_indicators", "administration_qualiopi_criterions", column: "criterion_id"
+ add_foreign_key "communication_website_imported_media", "communication_website_imported_websites", column: "website_id"
+ add_foreign_key "communication_website_imported_media", "universities"
add_foreign_key "communication_website_imported_pages", "communication_website_imported_websites", column: "website_id"
add_foreign_key "communication_website_imported_pages", "communication_website_pages", column: "page_id"
add_foreign_key "communication_website_imported_pages", "universities"
diff --git a/test/fixtures/communication/website/imported/media.yml b/test/fixtures/communication/website/imported/media.yml
new file mode 100644
index 000000000..3414bff4c
--- /dev/null
+++ b/test/fixtures/communication/website/imported/media.yml
@@ -0,0 +1,44 @@
+# == Schema Information
+#
+# Table name: communication_website_imported_media
+#
+# id :uuid not null, primary key
+# data :jsonb
+# file_url :text
+# filename :string
+# identifier :string
+# remote_created_at :datetime
+# remote_updated_at :datetime
+# created_at :datetime not null
+# updated_at :datetime not null
+# university_id :uuid not null
+# website_id :uuid not null
+#
+# Indexes
+#
+# index_communication_website_imported_media_on_university_id (university_id)
+# index_communication_website_imported_media_on_website_id (website_id)
+#
+# Foreign Keys
+#
+# fk_rails_... (university_id => universities.id)
+# fk_rails_... (website_id => communication_website_imported_websites.id)
+#
+
+one:
+ university: one
+ identifier: MyString
+ website: one
+ data:
+ remote_created_at: 2021-10-19 11:18:47
+ remote_updated_at: 2021-10-19 11:18:47
+ file_url: MyText
+
+two:
+ university: two
+ identifier: MyString
+ website: two
+ data:
+ remote_created_at: 2021-10-19 11:18:47
+ remote_updated_at: 2021-10-19 11:18:47
+ file_url: MyText
diff --git a/test/models/communication/website/imported/medium_test.rb b/test/models/communication/website/imported/medium_test.rb
new file mode 100644
index 000000000..c084dcee9
--- /dev/null
+++ b/test/models/communication/website/imported/medium_test.rb
@@ -0,0 +1,33 @@
+# == Schema Information
+#
+# Table name: communication_website_imported_media
+#
+# id :uuid not null, primary key
+# data :jsonb
+# file_url :text
+# filename :string
+# identifier :string
+# remote_created_at :datetime
+# remote_updated_at :datetime
+# created_at :datetime not null
+# updated_at :datetime not null
+# university_id :uuid not null
+# website_id :uuid not null
+#
+# Indexes
+#
+# index_communication_website_imported_media_on_university_id (university_id)
+# index_communication_website_imported_media_on_website_id (website_id)
+#
+# Foreign Keys
+#
+# fk_rails_... (university_id => universities.id)
+# fk_rails_... (website_id => communication_website_imported_websites.id)
+#
+require "test_helper"
+
+class Communication::Website::Imported::MediumTest < ActiveSupport::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
--
GitLab