diff --git a/Gemfile b/Gemfile index 101cd25..e7e5de3 100644 --- a/Gemfile +++ b/Gemfile @@ -100,3 +100,5 @@ gem "addressable", "~> 2.8" gem "aws-sdk-s3", "~> 1.149" gem "opensearch-ruby", "~> 3.3" + +gem "good_job", "~> 3.99" diff --git a/Gemfile.lock b/Gemfile.lock index baef1c8..b5c0cb6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -131,6 +131,8 @@ GEM ruby2_keywords ed25519 (1.3.0) erubi (1.12.0) + et-orbi (1.2.11) + tzinfo exifr (1.4.0) faraday (2.9.0) faraday-net_http (>= 2.0, < 3.2) @@ -139,12 +141,22 @@ GEM ffi (1.16.3) filesize (0.2.0) fspath (3.1.2) + fugit (1.11.0) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) github_webhook (1.4.2) activesupport (>= 4) rack (>= 1.3) railties (>= 4) globalid (1.2.1) activesupport (>= 6.1) + good_job (3.99.1) + activejob (>= 6.0.0) + activerecord (>= 6.0.0) + concurrent-ruby (>= 1.0.2) + fugit (>= 1.1) + railties (>= 6.0.0) + thor (>= 0.14.1) httparty (0.21.0) mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) @@ -215,6 +227,7 @@ GEM public_suffix (5.0.3) puma (6.4.0) nio4r (~> 2.0) + raabro (1.4.0) racc (1.7.3) rack (3.0.8) rack-session (2.0.0) @@ -349,6 +362,7 @@ DEPENDENCIES ed25519 (~> 1.3) filesize (~> 0.2.0) github_webhook (~> 1.4) + good_job (~> 3.99) httparty (~> 0.21.0) image_optim (~> 0.31.3) jbuilder diff --git a/Procfile b/Procfile index 5b316b9..39a81b5 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ server: bin/rails server -p 3000 -b 0.0.0.0 cron: bundler exec whenever --set environment="$RAILS_ENV" --update-crontab && crond -f +jobs: bundle exec good_job start diff --git a/Procfile.dev b/Procfile.dev index d2dd46e..0f37947 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,3 +1,4 @@ server: bin/rails server -p 3000 -b 0.0.0.0 assets: yarn build --watch cron: bundler exec whenever --set environment="$RAILS_ENV" --update-crontab && crond -f +jobs: bundle exec good_job start diff --git a/app/channels/application_cable/channel.rb b/app/channels/application_cable/channel.rb deleted file mode 100644 index 9aec230..0000000 --- a/app/channels/application_cable/channel.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module ApplicationCable - class Channel < ActionCable::Channel::Base - end -end diff --git a/app/channels/application_cable/connection.rb b/app/channels/application_cable/connection.rb deleted file mode 100644 index 8d6c2a1..0000000 --- a/app/channels/application_cable/connection.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module ApplicationCable - class Connection < ActionCable::Connection::Base - end -end diff --git a/app/controllers/admin/exceptions_controller.rb b/app/controllers/admin/exceptions_controller.rb new file mode 100644 index 0000000..164c5ae --- /dev/null +++ b/app/controllers/admin/exceptions_controller.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Admin + class ExceptionsController < ApplicationController + def index + @pagy, @exceptions = pagy(ExceptionLog.order(id: :desc), items: 25) + end + + def show + @exception = ExceptionLog.find(params[:id]) + end + end +end diff --git a/app/controllers/exceptions_controller.rb b/app/controllers/exceptions_controller.rb deleted file mode 100644 index 340e4b9..0000000 --- a/app/controllers/exceptions_controller.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -class ExceptionsController < ApplicationController - def index - @pagy, @exceptions = pagy(ExceptionLog.order(id: :desc), items: 25) - end - - def show - @exception = ExceptionLog.find(params[:id]) - end -end diff --git a/app/javascript/styles/exceptions.scss b/app/javascript/styles/admin/exceptions.scss similarity index 81% rename from app/javascript/styles/exceptions.scss rename to app/javascript/styles/admin/exceptions.scss index 20c8bf8..0949977 100644 --- a/app/javascript/styles/exceptions.scss +++ b/app/javascript/styles/admin/exceptions.scss @@ -1,4 +1,4 @@ -body.c-exceptions { +body.c-admin-exceptions { background-color: #2C2F33; color: #FFFDD0; diff --git a/app/javascript/styles/application.scss b/app/javascript/styles/application.scss index 57f8cd5..45113da 100644 --- a/app/javascript/styles/application.scss +++ b/app/javascript/styles/application.scss @@ -6,7 +6,7 @@ @import "yiff.media/home"; @import "yiff.rest/home"; @import "yiff.rocks/home"; -@import "exceptions"; +@import "admin/exceptions"; div#notice { padding: 0.25em; diff --git a/app/views/exceptions/index.html.erb b/app/views/admin/exceptions/index.html.erb similarity index 93% rename from app/views/exceptions/index.html.erb rename to app/views/admin/exceptions/index.html.erb index bd99576..cc095be 100644 --- a/app/views/exceptions/index.html.erb +++ b/app/views/admin/exceptions/index.html.erb @@ -17,7 +17,7 @@ <%= exception.version.present? ? link_to(exception.version, GitHelper.commit_url(exception.version)) : "Unknown" %> <%= exception.extra_params.dig("params", "controller") %>/<%= exception.extra_params.dig("params", "action") %> <%= truncate(exception.message, length: 50) %> - <%= link_to("View", exception_path(exception)) %> + <%= link_to("View", admin_exception_path(exception)) %> <% end %> diff --git a/app/views/exceptions/show.html.erb b/app/views/admin/exceptions/show.html.erb similarity index 100% rename from app/views/exceptions/show.html.erb rename to app/views/admin/exceptions/show.html.erb diff --git a/config/application.rb b/config/application.rb index b4e8bca..3139e50 100644 --- a/config/application.rb +++ b/config/application.rb @@ -35,6 +35,7 @@ module Websites config.action_controller.action_on_unpermitted_parameters = :raise config.action_dispatch.default_headers.clear + config.active_job.queue_adapter = :good_job config.middleware.insert_before(0, Middleware::DevHost) if Rails.env.development? config.middleware.insert_before(ActionDispatch::Static, Middleware::CustomStatic, { diff --git a/config/default_config.rb b/config/default_config.rb index b351963..9c3afc3 100644 --- a/config/default_config.rb +++ b/config/default_config.rb @@ -278,7 +278,7 @@ module Websites def e621_apikey end - def exceptions_domain + def admin_domain end def enable_analytics diff --git a/config/routes/other_routes.rb b/config/routes/other_routes.rb index a05ec12..385ea6f 100644 --- a/config/routes/other_routes.rb +++ b/config/routes/other_routes.rb @@ -1,10 +1,14 @@ # frozen_string_literal: true module OtherRoutes + ADMIN_DOMAIN = "admin.#{Websites.config.admin_domain}".freeze def self.extended(router) router.instance_exec do - constraints(DomainConstraint.new(Websites.config.exceptions_domain, "exceptions")) do - resources(:exceptions, only: %i[index show], path: "") + constraints(DomainConstraint.new(Websites.config.admin_domain, "admin")) do + namespace(:admin, path: "") do + resources(:exceptions, only: %i[index show]) + mount(GoodJob::Engine, at: "/jobs") + end end end end diff --git a/db/migrate/20240806104838_create_good_jobs.rb b/db/migrate/20240806104838_create_good_jobs.rb new file mode 100644 index 0000000..1880740 --- /dev/null +++ b/db/migrate/20240806104838_create_good_jobs.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +class CreateGoodJobs < ActiveRecord::Migration[7.1] + def change + # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support + # enable_extension 'pgcrypto' + + create_table(:good_jobs, id: :uuid) do |t| + t.text(:queue_name) + t.integer(:priority) + t.jsonb(:serialized_params) + t.datetime(:scheduled_at) + t.datetime(:performed_at) + t.datetime(:finished_at) + t.text(:error) + + t.timestamps + + t.uuid(:active_job_id) + t.text(:concurrency_key) + t.text(:cron_key) + t.uuid(:retried_good_job_id) + t.datetime(:cron_at) + + t.uuid(:batch_id) + t.uuid(:batch_callback_id) + + t.boolean(:is_discrete) # rubocop:disable Rails/ThreeStateBooleanColumn + t.integer(:executions_count) + t.text(:job_class) + t.integer(:error_event, limit: 2) + t.text(:labels, array: true) + t.uuid(:locked_by_id) + t.datetime(:locked_at) + end + + create_table(:good_job_batches, id: :uuid) do |t| + t.timestamps + t.text(:description) + t.jsonb(:serialized_properties) + t.text(:on_finish) + t.text(:on_success) + t.text(:on_discard) + t.text(:callback_queue_name) + t.integer(:callback_priority) + t.datetime(:enqueued_at) + t.datetime(:discarded_at) + t.datetime(:finished_at) + end + + create_table(:good_job_executions, id: :uuid) do |t| + t.timestamps + + t.uuid(:active_job_id, null: false) + t.text(:job_class) + t.text(:queue_name) + t.jsonb(:serialized_params) + t.datetime(:scheduled_at) + t.datetime(:finished_at) + t.text(:error) + t.integer(:error_event, limit: 2) + t.text(:error_backtrace, array: true) + t.uuid(:process_id) + t.interval(:duration) + end + + create_table(:good_job_processes, id: :uuid) do |t| + t.timestamps + t.jsonb(:state) + t.integer(:lock_type, limit: 2) + end + + create_table(:good_job_settings, id: :uuid) do |t| + t.timestamps + t.text(:key) + t.jsonb(:value) + t.index(:key, unique: true) + end + + add_index(:good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: :index_good_jobs_on_scheduled_at) + add_index(:good_jobs, %i[queue_name scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at) + add_index(:good_jobs, %i[active_job_id created_at], name: :index_good_jobs_on_active_job_id_and_created_at) + add_index(:good_jobs, :concurrency_key, where: "(finished_at IS NULL)", name: :index_good_jobs_on_concurrency_key_when_unfinished) + add_index(:good_jobs, %i[cron_key created_at], where: "(cron_key IS NOT NULL)", name: :index_good_jobs_on_cron_key_and_created_at_cond) + add_index(:good_jobs, %i[cron_key cron_at], where: "(cron_key IS NOT NULL)", unique: true, name: :index_good_jobs_on_cron_key_and_cron_at_cond) + add_index(:good_jobs, [:finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", name: :index_good_jobs_jobs_on_finished_at) + add_index(:good_jobs, %i[priority created_at], order: { priority: "DESC NULLS LAST", created_at: :asc }, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished) + add_index(:good_jobs, %i[priority created_at], order: { priority: "ASC NULLS LAST", created_at: :asc }, + where: "finished_at IS NULL", name: :index_good_job_jobs_for_candidate_lookup) + add_index(:good_jobs, [:batch_id], where: "batch_id IS NOT NULL") + add_index(:good_jobs, [:batch_callback_id], where: "batch_callback_id IS NOT NULL") + add_index(:good_jobs, :labels, using: :gin, where: "(labels IS NOT NULL)", name: :index_good_jobs_on_labels) + + add_index(:good_job_executions, %i[active_job_id created_at], name: :index_good_job_executions_on_active_job_id_and_created_at) + add_index(:good_jobs, %i[priority scheduled_at], order: { priority: "ASC NULLS LAST", scheduled_at: :asc }, + where: "finished_at IS NULL AND locked_by_id IS NULL", name: :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked) + add_index(:good_jobs, :locked_by_id, + where: "locked_by_id IS NOT NULL", name: "index_good_jobs_on_locked_by_id") + add_index(:good_job_executions, %i[process_id created_at], name: :index_good_job_executions_on_process_id_and_created_at) + end +end diff --git a/db/schema.rb b/db/schema.rb index 0ea81af..d994c2d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_06_04_105520) do +ActiveRecord::Schema[7.1].define(version: 2024_08_06_104838) do create_schema "e621" # These are extensions that must be enabled in order to support this database @@ -147,6 +147,94 @@ ActiveRecord::Schema[7.1].define(version: 2024_06_04_105520) do t.datetime "updated_at", null: false end + create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "description" + t.jsonb "serialized_properties" + t.text "on_finish" + t.text "on_success" + t.text "on_discard" + t.text "callback_queue_name" + t.integer "callback_priority" + t.datetime "enqueued_at" + t.datetime "discarded_at" + t.datetime "finished_at" + end + + create_table "good_job_executions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "active_job_id", null: false + t.text "job_class" + t.text "queue_name" + t.jsonb "serialized_params" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.text "error" + t.integer "error_event", limit: 2 + t.text "error_backtrace", array: true + t.uuid "process_id" + t.interval "duration" + t.index ["active_job_id", "created_at"], name: "index_good_job_executions_on_active_job_id_and_created_at" + t.index ["process_id", "created_at"], name: "index_good_job_executions_on_process_id_and_created_at" + end + + create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.jsonb "state" + t.integer "lock_type", limit: 2 + end + + create_table "good_job_settings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "key" + t.jsonb "value" + t.index ["key"], name: "index_good_job_settings_on_key", unique: true + end + + create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.text "queue_name" + t.integer "priority" + t.jsonb "serialized_params" + t.datetime "scheduled_at" + t.datetime "performed_at" + t.datetime "finished_at" + t.text "error" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "active_job_id" + t.text "concurrency_key" + t.text "cron_key" + t.uuid "retried_good_job_id" + t.datetime "cron_at" + t.uuid "batch_id" + t.uuid "batch_callback_id" + t.boolean "is_discrete" + t.integer "executions_count" + t.text "job_class" + t.integer "error_event", limit: 2 + t.text "labels", array: true + t.uuid "locked_by_id" + t.datetime "locked_at" + t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" + t.index ["batch_callback_id"], name: "index_good_jobs_on_batch_callback_id", where: "(batch_callback_id IS NOT NULL)" + t.index ["batch_id"], name: "index_good_jobs_on_batch_id", where: "(batch_id IS NOT NULL)" + t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" + t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at_cond", where: "(cron_key IS NOT NULL)" + t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at_cond", unique: true, where: "(cron_key IS NOT NULL)" + t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" + t.index ["labels"], name: "index_good_jobs_on_labels", where: "(labels IS NOT NULL)", using: :gin + t.index ["locked_by_id"], name: "index_good_jobs_on_locked_by_id", where: "(locked_by_id IS NOT NULL)" + t.index ["priority", "created_at"], name: "index_good_job_jobs_for_candidate_lookup", where: "(finished_at IS NULL)" + t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" + t.index ["priority", "scheduled_at"], name: "index_good_jobs_on_priority_scheduled_at_unfinished_unlocked", where: "((finished_at IS NULL) AND (locked_by_id IS NULL))" + t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" + t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" + end + create_table "short_urls", force: :cascade do |t| t.bigint "creator_id", null: false t.bigint "api_key_id" diff --git a/lib/middleware/dev_host.rb b/lib/middleware/dev_host.rb index 8bf6876..9ebf236 100644 --- a/lib/middleware/dev_host.rb +++ b/lib/middleware/dev_host.rb @@ -2,7 +2,7 @@ module Middleware class DevHost - DEFAULT_HOST = "v2.yiff.rest" + DEFAULT_HOST = "admin.furry.computer" def initialize(app) @app = app end