From b1c702e3cdec6eb4c6adb844ee3d7cca7a2d1cca Mon Sep 17 00:00:00 2001 From: Donovan Daniels Date: Mon, 6 May 2024 02:47:53 -0500 Subject: [PATCH] Add image management ui poorly tested but it worked well enough, I'm sure I'll be patching bugs over the next few weeks Also remove turbo because it sucks Also changed the way we handle hosts in dev --- .gitignore | 1 + Dockerfile | 3 - Gemfile | 11 +- Gemfile.lock | 28 ++++- app/controllers/application_controller.rb | 3 +- .../api_v2/application_controller.rb | 24 ++++ .../yiff_rest/api_v2/images_controller.rb | 77 ++++++++++++ .../yiff_rest/api_v2/manage_controller.rb | 38 ++++++ .../yiff_rest/api_v2_controller.rb | 8 +- .../yiff_rest/discord_controller.rb | 13 +- app/helpers/application_helper.rb | 21 ++++ app/javascript/application.ts | 1 - app/javascript/controllers/notice.ts | 16 ++- app/javascript/styles/application.scss | 4 + app/javascript/styles/yiff.rest/home.scss | 1 + app/javascript/styles/yiff.rest/v2.scss | 87 ++++++++++++++ app/jobs/e621_thumbnail_job.rb | 9 +- app/logical/api_category.rb | 64 ++++++++++ app/logical/api_image_upload_service.rb | 58 +++++++++ app/logical/cloudflare_service.rb | 15 +++ app/logical/file_download.rb | 104 ++++++++++++++++ app/logical/requests/e621.rb | 42 ++++--- app/logical/storage_manager.rb | 6 - app/logical/storage_manager/bunny.rb | 2 +- app/logical/storage_manager/local.rb | 28 +++-- app/logical/storage_manager/s3.rb | 47 ++++++++ app/models/api_image.rb | 113 +++++++++++++++++- app/models/api_user.rb | 5 +- app/models/application_record.rb | 1 + app/models/e621_thumbnail.rb | 8 +- app/views/layouts/application.html.erb | 6 +- app/views/static/_backtrace.html.erb | 11 ++ app/views/static/error.html.erb | 22 ++-- .../yiff_rest/api_v2/images/edit.html.erb | 16 +++ .../yiff_rest/api_v2/images/index.html.erb | 65 ++++++++++ .../yiff_rest/api_v2/images/new.html.erb | 19 +++ .../yiff_rest/api_v2/manage/_nav.html.erb | 52 ++++++++ .../yiff_rest/api_v2/manage/index.html.erb | 40 +++++++ config/default_config.rb | 55 +++++++-- config/initializers/pagy.rb | 4 + config/routes/domain_constraint.rb | 12 +- config/routes/yiff_rest_routes.rb | 10 +- ...240504225048_drop_api_images_created_by.rb | 7 ++ db/schema.rb | 3 +- docker-compose.yml | 1 + lib/middleware/dev_host.rb | 11 +- package.json | 1 - yarn.lock | 18 --- 48 files changed, 1077 insertions(+), 114 deletions(-) create mode 100644 app/controllers/yiff_rest/api_v2/application_controller.rb create mode 100644 app/controllers/yiff_rest/api_v2/images_controller.rb create mode 100644 app/controllers/yiff_rest/api_v2/manage_controller.rb create mode 100644 app/javascript/styles/yiff.rest/v2.scss create mode 100644 app/logical/api_category.rb create mode 100644 app/logical/api_image_upload_service.rb create mode 100644 app/logical/cloudflare_service.rb create mode 100644 app/logical/file_download.rb delete mode 100644 app/logical/storage_manager.rb create mode 100644 app/logical/storage_manager/s3.rb create mode 100644 app/views/static/_backtrace.html.erb create mode 100644 app/views/yiff_rest/api_v2/images/edit.html.erb create mode 100644 app/views/yiff_rest/api_v2/images/index.html.erb create mode 100644 app/views/yiff_rest/api_v2/images/new.html.erb create mode 100644 app/views/yiff_rest/api_v2/manage/_nav.html.erb create mode 100644 app/views/yiff_rest/api_v2/manage/index.html.erb create mode 100644 config/initializers/pagy.rb create mode 100644 db/migrate/20240504225048_drop_api_images_created_by.rb diff --git a/.gitignore b/.gitignore index bc45c3f..552f561 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ # Ignore master key for decrypting credentials and more. /config/master.key +/config/local_config.rb /app/assets/builds/* !/app/assets/builds/.keep diff --git a/Dockerfile b/Dockerfile index 66748c9..9f2f555 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,4 @@ COPY --from=node-builder /root/.cache/node /root/.cache/node COPY --from=node-builder /app/node_modules node_modules COPY --from=ruby-builder /usr/local/bundle /usr/local/bundle -# Stop bin/rails console from offering autocomplete -RUN echo "IRB.conf[:USE_AUTOCOMPLETE] = false" > ~/.irbrc - CMD ["foreman", "start", "-f", "Procfile.dev"] diff --git a/Gemfile b/Gemfile index 160e3fd..a0aa477 100644 --- a/Gemfile +++ b/Gemfile @@ -21,9 +21,6 @@ gem "puma", ">= 5.0" # Bundle and transpile JavaScript [https://github.com/rails/jsbundling-rails] gem "jsbundling-rails" -# Hotwire's SPA-like page accelerator [https://turbo.hotwired.dev] -gem "turbo-rails" - # Hotwire's modest JavaScript framework [https://stimulus.hotwired.dev] gem "stimulus-rails" @@ -93,3 +90,11 @@ gem "ruby-vips", "~> 2.2" gem "simple_form", "~> 5.3" gem "responders", "~> 3.1" + +gem "pagy", "~> 8.3" + +gem "retriable", "~> 3.1" + +gem "addressable", "~> 2.8" + +gem "aws-sdk-s3", "~> 1.149" diff --git a/Gemfile.lock b/Gemfile.lock index f034ca2..b490686 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -77,6 +77,22 @@ GEM addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) ast (2.4.2) + aws-eventstream (1.3.0) + aws-partitions (1.924.0) + aws-sdk-core (3.194.1) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.651.0) + aws-sigv4 (~> 1.8) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.80.0) + aws-sdk-core (~> 3, >= 3.193.0) + aws-sigv4 (~> 1.1) + aws-sdk-s3 (1.149.0) + aws-sdk-core (~> 3, >= 3.194.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.8) + aws-sigv4 (1.8.0) + aws-eventstream (~> 1, >= 1.0.2) base64 (0.2.0) better_html (2.0.2) actionview (>= 6.0) @@ -145,6 +161,7 @@ GEM jbuilder (2.11.5) actionview (>= 5.0.0) activesupport (>= 5.0.0) + jmespath (1.6.2) jsbundling-rails (1.2.1) railties (>= 6.0.0) json (2.6.3) @@ -176,6 +193,7 @@ GEM nio4r (2.5.9) nokogiri (1.15.4-x86_64-linux) racc (~> 1.4) + pagy (8.3.0) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) @@ -241,6 +259,7 @@ GEM responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) + retriable (3.1.2) rexml (3.2.6) rubocop (1.57.2) json (~> 2.3) @@ -287,10 +306,6 @@ GEM stringio (3.0.8) thor (1.3.0) timeout (0.4.1) - turbo-rails (1.5.0) - actionpack (>= 6.0.0) - activejob (>= 6.0.0) - railties (>= 6.0.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.5.0) @@ -314,6 +329,8 @@ PLATFORMS x86_64-linux DEPENDENCIES + addressable (~> 2.8) + aws-sdk-s3 (~> 1.149) bootsnap capybara debug @@ -325,12 +342,14 @@ DEPENDENCIES image_optim (~> 0.31.3) jbuilder jsbundling-rails + pagy (~> 8.3) pg (~> 1.1) puma (>= 5.0) rails (~> 7.1.1) redis (~> 5.0) request_store (~> 1.5) responders (~> 3.1) + retriable (~> 3.1) rubocop (~> 1.57) rubocop-erb (~> 0.3.0) rubocop-rails (~> 2.22) @@ -340,7 +359,6 @@ DEPENDENCIES sprockets-rails stimulus-rails timeout (~> 0.4.1) - turbo-rails tzinfo-data web-console whenever (~> 1.0) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 30ad64a..121d2df 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -196,7 +196,7 @@ class ApplicationController < ActionController::Base @backtrace = Rails.backtrace_cleaner.clean(@exception.backtrace) format = :html unless format.in?(%i[html json]) - @message = "An unexpected error occurred." if Rails.env.production? && message == exception.message + @message = "An unexpected error occurred." if !(Rails.env.development? || CurrentUser.is_admin?) && message == exception.message WebLogger.log_exception(@exception, expected: @expected) log = ExceptionLog.add(exception, request) unless @expected @@ -253,4 +253,5 @@ class ApplicationController < ActionController::Base include CommonRoutes include RenderMethods include SearchMethods + include Pagy::Backend end diff --git a/app/controllers/yiff_rest/api_v2/application_controller.rb b/app/controllers/yiff_rest/api_v2/application_controller.rb new file mode 100644 index 0000000..1662664 --- /dev/null +++ b/app/controllers/yiff_rest/api_v2/application_controller.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module YiffRest + module APIV2 + class ApplicationController < ::YiffRest::ApplicationController + protected + + def site_domain + YiffRestRoutes::V2_DOMAIN + end + + def validate_discord + redirect_to(Websites.config.yiffyapi_discord_redirect("manage_images"), allow_other_host: true) if session[:discord_user].blank? + end + + def prepare_user + name = session.dig("discord_user", "global_name") || "#{session.dig('discord_user', 'username')}#{@session.dig('discord_user', 'discriminator')}" + CurrentUser.user = APIUser.find_or_create_by(id: session.dig("discord_user", "id")) + CurrentUser.update!(name: name, discord_data: session["discord_user"]) + CurrentUser.update_avatar(session.dig("discord_user", "avatar")) + end + end + end +end diff --git a/app/controllers/yiff_rest/api_v2/images_controller.rb b/app/controllers/yiff_rest/api_v2/images_controller.rb new file mode 100644 index 0000000..c9793d2 --- /dev/null +++ b/app/controllers/yiff_rest/api_v2/images_controller.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module YiffRest + module APIV2 + class ImagesController < ApplicationController + before_action :validate_discord + before_action :prepare_user + before_action :admin_only + + def index + @sp = search_images_params + @pagy, @images = pagy(APIImage.search(@sp).order(created_at: :desc), size: [1, 2, 2, 1]) + end + + def new + @image = APIImage.new(create_params) + @categories = APIImage.categories + end + + def edit + @image = APIImage.find(params[:id]) + @categories = APIImage.categories + end + + def create + @service = APIImageUploadService.new(create_params) + @image = @service.start! + @categories = APIImage.categories + if @image.invalid? + flash.now[:alert] = @image.errors.full_messages.join(", ") + return render(:new) + end + redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image added") + end + + def update + @image = APIImage.find(params[:id]) + @image.update(update_params) + @categories = APIImage.categories + if @image.invalid? + flash.now[:alert] = @image.errors.full_messages.join(", ") + return render(:edit) + end + redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image updated") + end + + def destroy + @image = APIImage.find(params[:id]) + @image.destroy + redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image deleted") + end + + private + + def load_category + @category = APIImage.categories.find { |c| c.db == params[:manage_id] } + raise(ActiveRecord::RecordNotFound) if @category.blank? + end + + def site_title + "YiffyAPI V2 - Manage Images" + end + + def search_images_params + permit_search_params(%i[category]) + end + + def create_params + params.fetch(:api_image, {}).permit(:category, :original_url, :sources_string, :artists_string, :file) + end + + def update_params + params.fetch(:api_image, {}).permit(:category, :sources_string, :artists_string) + end + end + end +end diff --git a/app/controllers/yiff_rest/api_v2/manage_controller.rb b/app/controllers/yiff_rest/api_v2/manage_controller.rb new file mode 100644 index 0000000..648cb45 --- /dev/null +++ b/app/controllers/yiff_rest/api_v2/manage_controller.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module YiffRest + module APIV2 + class ManageController < ApplicationController + before_action :validate_discord, except: %i[logout] + before_action :prepare_user, except: %i[logout] + + def index + @categories = APIImage.categories + end + + def show + @category = APIImage.categories.find { |c| c.db == params[:id] } + raise(ActiveRecord::RecordNotFound) if @category.blank? + @count = APIImage.cached_count(params[:id]) + @pagy, @images = pagy(APIImage.where(category: params[:id]).search(search_params).order(created_at: :desc), size: [1, 2, 2, 1]) + end + + def sync + APIImage.sync_all + redirect_to(yiff_rest_api_v2_manage_index_path, notice: "Images will soon be synced.") + end + + def logout + session.delete("discord_user") + CurrentUser.user = nil + redirect_to(yiff_rest_root_path, notice: "You have been logged out.") + end + + private + + def site_title + "YiffyAPI V2 - Manage Images" + end + end + end +end diff --git a/app/controllers/yiff_rest/api_v2_controller.rb b/app/controllers/yiff_rest/api_v2_controller.rb index 84399a7..78d5376 100644 --- a/app/controllers/yiff_rest/api_v2_controller.rb +++ b/app/controllers/yiff_rest/api_v2_controller.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module YiffRest - class APIV2Controller < ApplicationController + class APIV2Controller < APIV2::ApplicationController include ::ApplicationController::ReadonlyMethods before_action :handle_ratelimit, except: %i[robots] before_action :validate_images_access, only: %i[index] @@ -143,12 +143,8 @@ module YiffRest private - def site_domain - YiffRestRoutes::V2_DOMAIN - end - def api_categories - categories = APIImage.categories.pluck(:db) + categories = APIImage.categories.map(&:db) [*categories, "chris"] # gay little polar cutie end diff --git a/app/controllers/yiff_rest/discord_controller.rb b/app/controllers/yiff_rest/discord_controller.rb index 176aa23..9f71914 100644 --- a/app/controllers/yiff_rest/discord_controller.rb +++ b/app/controllers/yiff_rest/discord_controller.rb @@ -2,7 +2,7 @@ module YiffRest class DiscordController < ApplicationController - before_action :handle_oauth, only: %i[count_servers flags apikey] + before_action :handle_oauth, except: %i[index interactions] def index end @@ -14,10 +14,15 @@ module YiffRest end def apikey - return redirect_to("http://websites4.containers.local:3000/apikeys?domain=yiff.rest") if Rails.env.development? + return redirect_to("http://websites4.containers.local:3000/apikeys?domain=yiff.rest", allow_other_host: true) if Rails.env.development? redirect_to("https://yiff.rest/apikeys", allow_other_host: true) end + def manage_images + return redirect_to("http://websites4.containers.local:3000/manage?domain=v2.yiff.rest", allow_other_host: true) if Rails.env.development? + redirect_to("https://v2.yiff.rest/manage", allow_other_host: true) + end + def interactions begin verify_request! @@ -81,8 +86,10 @@ module YiffRest public_flags: DiscordConstants::UserFlags.parse_flags(user["public_flags"]).map(&:title), all_flags: DiscordConstants::UserFlags.parse_flags(user["flags"] - user["public_flags"]).map(&:title), } - when "apikey" + when "apikey", "manage_images" session[:discord_user] = res.get_authorization["user"].slice("id", "username", "discriminator", "global_name", "avatar") + else + raise("Unknown action: #{params[:action]}") end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 60c6bd6..e6a6d41 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true module ApplicationHelper + include Pagy::Frontend + def page_title return content_for(:page_title) if content_for?(:page_title) return site_title if defined?(site_title) @@ -33,4 +35,23 @@ module ApplicationHelper return css if request.path == path "" end + + def time_tag(content, time) + datetime = time.strftime("%Y-%m-%dT%H:%M%:z") + tag.time(content || datetime, datetime: datetime, title: time.to_fs) + end + + def compact_time(time) + time_tag(time.strftime("%Y-%m-%d %H:%M:%S"), time) + end + + def error_messages_for(instance_name) + instance = instance_variable_get("@#{instance_name}") + + if instance&.errors&.any? + %(
Error: #{instance.__send__(:errors).full_messages.join(', ')}
).html_safe + else + "" + end + end end diff --git a/app/javascript/application.ts b/app/javascript/application.ts index 63a5845..89fec31 100644 --- a/app/javascript/application.ts +++ b/app/javascript/application.ts @@ -1,6 +1,5 @@ /// // Entry point for the build script in your package.json -import "@hotwired/turbo-rails"; import "bootstrap"; import "bootstrap/dist/css/bootstrap.min.css"; import jQuery from "jquery"; diff --git a/app/javascript/controllers/notice.ts b/app/javascript/controllers/notice.ts index 1d08f30..1eeccee 100644 --- a/app/javascript/controllers/notice.ts +++ b/app/javascript/controllers/notice.ts @@ -3,9 +3,21 @@ import { Controller } from "@hotwired/stimulus"; export default class extends Controller { static override targets = ["notice"]; declare noticeTarget: HTMLElement; + timeout: number | undefined; - close(event: KeyboardEvent) { - event.preventDefault(); + // automatically close + override connect() { + this.timeout = setTimeout(() => this._close(), 3500); + } + + _close() { $(this.noticeTarget).fadeOut("fast"); } + + close(event: KeyboardEvent) { + if (this.timeout) { + clearTimeout(this.timeout); + } + event.preventDefault(); + } } diff --git a/app/javascript/styles/application.scss b/app/javascript/styles/application.scss index 6ef3cb8..4750858 100644 --- a/app/javascript/styles/application.scss +++ b/app/javascript/styles/application.scss @@ -47,3 +47,7 @@ body.c-application { text-align: center; } } + +.backtrace-line { + list-style: none; +} diff --git a/app/javascript/styles/yiff.rest/home.scss b/app/javascript/styles/yiff.rest/home.scss index 461a08b..2da8bed 100644 --- a/app/javascript/styles/yiff.rest/home.scss +++ b/app/javascript/styles/yiff.rest/home.scss @@ -1,6 +1,7 @@ @import "apikeys"; @import "discord"; @import "state"; +@import "v2"; body.s-yiff-rest { background-color: #2C2F33; diff --git a/app/javascript/styles/yiff.rest/v2.scss b/app/javascript/styles/yiff.rest/v2.scss new file mode 100644 index 0000000..1a32f21 --- /dev/null +++ b/app/javascript/styles/yiff.rest/v2.scss @@ -0,0 +1,87 @@ +body.s-v2-yiff-rest { + background-color: #2c2f33; + color: #fffdd0; + + a { + color: #fff; + text-decoration: none; + } + + // Bootstrap does not have built in dark pagination + .pagy-bootstrap { + $gray-100: #f8f9fa; + $gray-600: #6c757d; + $gray-800: #343a40; + $pagination-focus-outline: 0; + $focus-ring-width: 0.25rem; + $focus-ring-opacity: 0.25; + $blue: #0d6efd; + $white: #fff; + $primary: $blue; + $focus-ring-color: rgba($primary, $focus-ring-opacity); + $focus-ring-blur: 0; + $focus-ring-box-shadow: 0 0 $focus-ring-blur $focus-ring-width + $focus-ring-color; + $pagination-focus-box-shadow: $focus-ring-box-shadow; + + $pagination-dark-color: $gray-100; + $pagination-dark-bg: $gray-800; + $pagination-dark-border-color: $gray-600; + + $pagination-dark-hover-color: $pagination-dark-color; + $pagination-dark-hover-bg: $gray-600; + $pagination-dark-hover-border-color: $pagination-dark-border-color; + $component-active-color: $white; + $component-active-bg: $primary; + $pagination-active-color: $component-active-color; + $pagination-active-bg: $component-active-bg; + $pagination-active-border-color: $component-active-bg; + + .page-link { + color: $pagination-dark-color; + background-color: $pagination-dark-bg; + border-color: $pagination-dark-border-color; + + &:hover { + color: $pagination-dark-hover-color; + background-color: $pagination-dark-hover-bg; + border-color: $pagination-dark-hover-border-color; + } + + &:focus { + outline: $pagination-focus-outline; + box-shadow: $pagination-focus-box-shadow; + } + } + + .page-item { + &.active .page-link { + color: $pagination-active-color; + background-color: $pagination-active-bg; + border-color: $pagination-active-border-color; + } + + &.disabled .page-link { + background-color: $pagination-dark-bg; + border-color: $pagination-dark-border-color; + } + } + } + + .category-list, .category-image-list { + a { + color: #ede660; + } + + ul.details { + padding: 0; + + li { + list-style: none; + ul { + padding-left: 1rem; + } + } + } + } +} diff --git a/app/jobs/e621_thumbnail_job.rb b/app/jobs/e621_thumbnail_job.rb index 8984fd2..f06027f 100644 --- a/app/jobs/e621_thumbnail_job.rb +++ b/app/jobs/e621_thumbnail_job.rb @@ -8,11 +8,10 @@ class E621ThumbnailJob < ApplicationJob outfile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .#{entry.filetype}]) cutfile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .cut.webm]) palettefile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .palette.png]) - Timeout.timeout(1000) do + Timeout.timeout(120) do infile.binmode - HTTParty.get("https://static1.e621.net/data/#{entry.stripped_md5[0..1]}/#{entry.stripped_md5[2..3]}/#{entry.stripped_md5}.webm", stream_body: true) do |fragment| - infile.write(fragment) - end + io = FileDownload.new("https://static1.e621.net/data/#{entry.stripped_md5[0..1]}/#{entry.stripped_md5[2..3]}/#{entry.stripped_md5}.webm").download! + IO.copy_stream(io, infile) duration = `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 #{infile.path}`.to_f offset = duration > 10 ? rand(0..(duration - 10)) : duration / 2 if entry.filetype == "gif" @@ -23,7 +22,7 @@ class E621ThumbnailJob < ApplicationJob `ffmpeg -y -i #{infile.path} -ss #{offset} -vframes 1 #{outfile.path} 1>&2` end ImageOptim.new.optimize_image!(outfile.path) - StorageManager::E621Thumbnails.upload("#{entry.stripped_md5}.#{entry.filetype}", outfile.read) + Websites.config.e621_thumbnails_storage.upload("/#{entry.stripped_md5}.#{entry.filetype}", outfile) entry.update!(status: "complete") execute_webhook(entry, title: "Thumbnail Generated (#{entry.filetype})") end diff --git a/app/logical/api_category.rb b/app/logical/api_category.rb new file mode 100644 index 0000000..ce10715 --- /dev/null +++ b/app/logical/api_category.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +class APICategory + attr_reader :name, :db, :sfw + + SOURCES = [ + %w[e621.net E621], + %w[furaffinity.net FurAffinity], + %w[inkbunny.net InkBunny], + %w[deviantart.com DeviantArt], + %w[twitter.com Twitter], + ].freeze + + def initialize(name, db, sfw) + @name = name + @db = db + @sfw = sfw + end + + def count + APIImage.cached_count(db) + end + + def real_count + APIImage.where(category: db).count + end + + def unsourced_count + APIImage.where(category: db, sources: []).count + end + + def sources + r = APIImage.joins("CROSS JOIN unnest(sources) AS source").where(category: db).distinct + other = r + list = [] + + APICategory::SOURCES.each do |value, name| + count = r.where("source ILIKE ?", "%#{value}%").count + list << { name: name, count: count } + other = other.where("source NOT ILIKE ?", "%#{value}%") + end + list << { name: "Unsourced", count: unsourced_count } + # noinspection RubyMismatchedArgumentType + list << { name: "Other", count: other.count } + list + end + + def created_at + APIImage.where(category: db).minimum(:created_at) + end + + def updated_at + APIImage.where(category: db).maximum(:updated_at) + end + + def as_json(*) + { + name: name, + db: db, + sfw: sfw, + count: count, + } + end +end diff --git a/app/logical/api_image_upload_service.rb b/app/logical/api_image_upload_service.rb new file mode 100644 index 0000000..e1e8b3d --- /dev/null +++ b/app/logical/api_image_upload_service.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +class APIImageUploadService + class UploadError < StandardError; end + attr_reader :params, :image, :upload + + def initialize(params) + @params = params + end + + def start! + file = params.delete(:file) + APIImage.transaction do + @image = APIImage.new(**params, creator: CurrentUser.user) + begin + @image.file = get_file(@image, file: file) + process_file(@image, @image.file) + @image.save + return @image if @image.invalid? + store_file(@image, @image.file) + @image + rescue UploadError, ActiveRecord::RecordInvalid => e + @image.exception = e + @image + end + end + end + + def get_file(image, file: nil) + return file if file.present? + raise(UploadError, "No file or source URL provided") if image.original_url.blank? + + download = FileDownload.new(image.original_url) + download.download! + end + + def process_file(image, file) + mime, ext = @image.file_header_info(file.path) + image.mime_type = mime + image.file_ext = ext + image.file_size = file.size + image.id = Digest::MD5.file(file.path).hexdigest + + width, height = calculate_dimensions(file.path) + image.width = width + image.height = height + image.validate!(:file) + end + + def store_file(image, file) + Websites.config.yiffy2_storage.put(image.path, file) + end + + def calculate_dimensions(file_path) + image = Vips::Image.new_from_file(file_path) + [image.width, image.height] + end +end diff --git a/app/logical/cloudflare_service.rb b/app/logical/cloudflare_service.rb new file mode 100644 index 0000000..46f4401 --- /dev/null +++ b/app/logical/cloudflare_service.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module CloudflareService + def self.ips + text, code = Cache.fetch("cloudflare_ips", expires_in: 24.hours) do + resp = HTTParty.get("https://api.cloudflare.com/client/v4/ips", Websites.config.httparty_options) + [resp.body, resp.code] + end + return [] if code != 200 + + json = JSON.parse(text, symbolize_names: true) + ips = json[:result][:ipv4_cidrs] + json[:result][:ipv6_cidrs] + ips.map { |ip| IPAddr.new(ip) } + end +end diff --git a/app/logical/file_download.rb b/app/logical/file_download.rb new file mode 100644 index 0000000..d153433 --- /dev/null +++ b/app/logical/file_download.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +class FileDownload + include ActiveModel::Validations + class Error < StandardError; end + + RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError].freeze + MAX_SIZE = 50.megabytes + + attr_reader :url + + validate :validate_url + + def initialize(url) + begin + unencoded = Addressable::URI.unencode(url) + escaped = Addressable::URI.escape(unencoded) + @url = Addressable::URI.parse(escaped) + rescue Addressable::URI::InvalidURIError + @url = nil + end + validate! + end + + def size + res = HTTParty.head(uncached_url, **httparty_options, timeout: 3) + + if res.success? + res.content_length + else + raise(HTTParty::ResponseError, res) + end + end + + def download!(tries: 3, **) + Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do + http_get_streaming(uncached_url, **) + end + end + + def validate_url + errors.add(:base, "URL must not be blank") if url.blank? + errors.add(:base, "'#{url}' is not a valid url") if url.host.blank? + errors.add(:base, "'#{url}' is not a valid url. Did you mean 'https://#{url}'?") unless url.scheme.in?(%w[http https]) + end + + def http_get_streaming(url, file: Tempfile.new(binmode: true), max_size: MAX_SIZE) + size = 0 + + res = HTTParty.get(url, httparty_options) do |chunk| + next if [301, 302].include?(chunk.code) + + size += chunk.size + raise(Error, "File is too large (max size: #{max_size})") if size > max_size && max_size > 0 + + file.write(chunk) + end + + if res.success? + file.rewind + file + else + raise("HTTP error code: #{res.code} #{res.message}") + end + end + + # Prevent Cloudflare from potentially mangling the image + def uncached_url + return file_url unless is_cloudflare?(file_url) + + url = file_url.dup + url.query_values = url.query_values.to_h.merge(nc: Time.now) + url + end + + alias file_url url + + def httparty_options + { + timeout: 10, + stream_body: true, + connection_adapter: ValidatingConnectionAdapter, + }.deep_merge(Websites.config.httparty_options) + end + + def is_cloudflare?(url) + ip_addr = IPAddr.new(Resolv.getaddress(url.hostname)) + CloudflareService.ips.any? { |subnet| subnet.include?(ip_addr) } + end +end + +# Hook into HTTParty to validate the IP before following redirects. +# https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter +class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter + def self.call(uri, options) + ip_addr = IPAddr.new(Resolv.getaddress(uri.hostname)) + raise(FileDownload::Error, "Downloads from #{ip_addr} are not allowed") if ip_blocked?(ip_addr) + super + end + + def self.ip_blocked?(ip_addr) + ip_addr.private? || ip_addr.loopback? || ip_addr.link_local? + end +end diff --git a/app/logical/requests/e621.rb b/app/logical/requests/e621.rb index ce74ba5..6ac6d9f 100644 --- a/app/logical/requests/e621.rb +++ b/app/logical/requests/e621.rb @@ -7,7 +7,9 @@ module Requests def status r = self.class.get("/posts.json?limit=0", { + **Websites.config.httparty_options, headers: { + **Websites.config.http_headers, "User-Agent" => "E621Status/1.0.0 (https://status.e621.ws; \"donovan_dmc\")", }, timeout: 5, @@ -33,11 +35,7 @@ module Requests path = "/" path = "/posts/#{id}.json" if id path = "/posts.json?md5=#{md5}" if md5 - r = self.class.get(path, { - headers: { - "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")", - }, - }) + r = self.class.get(path, Websites.config.httparty_options) return nil if r.code != 200 JSON.parse(r.body)["post"] end @@ -49,20 +47,30 @@ module Requests path += "tags=md5:#{md5s.join(',')}%20limit:100" if md5s path += "%20status:#{status}" if path.include?("tags=") && !status.nil? path += "tags=status:#{status}" if !path.include?("tags=") && !status.nil? - r = self.class.get(path, { - headers: { - "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")", - }, - }) + Rails.logger.info("Fetching posts from e621: #{path}") if Rails.env.development? + r = self.class.get(path, Websites.config.httparty_options) JSON.parse(r.body)["posts"] || [] end + def get_all_posts(ids: [], md5s: []) + posts = [] + ids.each_slice(100) do |slice| + posts += get_posts(ids: slice, status: "any") + end + md5s -= posts.map { |p| p["file"]["md5"] } + md5s.each_slice(100) do |slice| + posts += get_posts(md5s: slice, status: "any") + end + missing = { + ids: ids - posts.pluck("id"), + md5s: md5s - posts.map { |p| p["file"]["md5"] }, + } + + { posts: posts, missing: missing } + end + def find_replacement(md5:) - r = self.class.get("/post_replacements.json?search[md5]=#{md5}", { - headers: { - "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")", - }, - }) + r = self.class.get("/post_replacements.json?search[md5]=#{md5}", Websites.config.httparty_options) JSON.parse(r.body)&.first end @@ -77,5 +85,9 @@ module Requests def self.get_posts(**) new.get_posts(**) end + + def self.get_all_posts(**) + new.get_all_posts(**) + end end end diff --git a/app/logical/storage_manager.rb b/app/logical/storage_manager.rb deleted file mode 100644 index 380d6f1..0000000 --- a/app/logical/storage_manager.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module StorageManager - E621Thumbnails = Local.new(base_url: Websites.config.e621_thumbnails_base_url, base_path: Websites.config.e621_thumbnails_base_path) - # E621Thumbnails = Bunny.new(base_url: Websites.config.e621_thumbnails_base_url, access_key: Websites.config.e621_thumbnails_access_key, storage_zone_name: Websites.config.e621_thumbnails_storage_zone_name) -end diff --git a/app/logical/storage_manager/bunny.rb b/app/logical/storage_manager/bunny.rb index cd22eb5..d9c78e2 100644 --- a/app/logical/storage_manager/bunny.rb +++ b/app/logical/storage_manager/bunny.rb @@ -23,7 +23,7 @@ module StorageManager end def url_for(entry) - "#{base_url}/#{entry.stripped_md5}.#{entry.filetype}" + "#{base_url}#{entry.path}" end end end diff --git a/app/logical/storage_manager/local.rb b/app/logical/storage_manager/local.rb index 0dc2346..c7b6569 100644 --- a/app/logical/storage_manager/local.rb +++ b/app/logical/storage_manager/local.rb @@ -2,6 +2,7 @@ module StorageManager class Local + DEFAULT_PERMISSIONS = 0o644 attr_reader :base_url, :base_path def initialize(base_url:, base_path:) @@ -11,28 +12,41 @@ module StorageManager def delete(path) return unless exists?(path) - File.delete("#{base_path}/#{path}") + File.delete("#{base_path}#{path}") end def exists?(path) - File.exist?("#{base_path}/#{path}") + File.exist?("#{base_path}#{path}") end def get(path) - File.read("#{base_path}/#{path}") + File.open("#{base_path}#{path}", "r", binmode: true) end - def put(path, body) - File.write("#{base_path}/#{path}", body) + def put(path, io) + temp = "#{base_path}#{path}-#{SecureRandom.uuid}.tmp" + + FileUtils.mkdir_p(File.dirname(temp)) + io.rewind + bytes_copied = IO.copy_stream(io, temp) + raise("store failed: #{bytes_copied}/#{io.size} bytes copied") if bytes_copied != io.size + + FileUtils.chmod(DEFAULT_PERMISSIONS, temp) + File.rename(temp, "#{base_path}#{path}") + rescue StandardError => e + FileUtils.rm_f(temp) + raise(e) + ensure + FileUtils.rm_f(temp) if temp end def upload(path, body) put(path, body) - "#{base_url}/#{path}" + "#{base_url}#{path}" end def url_for(entry) - "#{base_url}/#{entry.stripped_md5}.#{entry.filetype}" + "#{base_url}#{entry.path}" end end end diff --git a/app/logical/storage_manager/s3.rb b/app/logical/storage_manager/s3.rb new file mode 100644 index 0000000..241f3bf --- /dev/null +++ b/app/logical/storage_manager/s3.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module StorageManager + class S3 + attr_reader :base_url + + def initialize(endpoint:, access_key_id:, secret_access_key:, bucket:, base_url:) + @s3 = Aws::S3::Resource.new( + region: "weur", + endpoint: endpoint, + credentials: Aws::Credentials.new(access_key_id, secret_access_key), + ).bucket(bucket) + @base_url = base_url + end + + def delete(path) + return unless exists?(path) + @s3.object(trim(path)).delete + end + + def exists?(path) + @s3.object(trim(path)).exists? + end + + def get(path) + @s3.object(trim(path)).get.body + end + + def put(path, io) + @s3.object(trim(path)).put(body: io, content_type: Marcel::MimeType.for(io)) + end + + def upload(path, body) + put(path, body) + "#{base_url}#{path}" + end + + def url_for(entry) + "#{base_url}#{entry.path}" + end + + def trim(path) + # R2 for some reason doesn't trim preceding slashes + path.sub(%r{^/}, "") + end + end +end diff --git a/app/models/api_image.rb b/app/models/api_image.rb index e4d6212..d73b33a 100644 --- a/app/models/api_image.rb +++ b/app/models/api_image.rb @@ -1,10 +1,58 @@ # frozen_string_literal: true class APIImage < ApplicationRecord - CDN_URL = "https://v2.yiff.media/" - belongs_to_creator + attr_accessor :file, :exception + + validates :category, presence: true, inclusion: { in: -> { APIImage.categories.map(&:db) } } + validates :id, uniqueness: true, on: :file + validate on: :file do |image| + ext, mime = file_header_info(file.path) + image.errors.add(:file, "type is invalid (#{mime})") if ext == mime + end + validate do |image| + image.errors.add(:base, exception.message) if exception + end + + after_create :invalidate_cache + after_update :update_files, if: :saved_change_to_category? + after_destroy :delete_files + + def delete_files + Websites.config.yiffy2_storage.delete(path) + invalidate_cache + end + + def update_files + file = Websites.config.yiffy2_storage.get(path_before_last_save) + Websites.config.yiffy2_storage.put(path, file) + Websites.config.yiffy2_storage.delete(path_before_last_save) + invalidate_cache + end + + def file_header_info(file_path) + File.open(file_path) do |bin| + mime_type = Marcel::MimeType.for(bin) + ext = case mime_type + when "image/jpeg" + "jpg" + when "image/gif" + "gif" + when "image/png" + "png" + else + mime_type + end + [mime_type, ext] + end + end + + def invalidate_cache + Cache.redis.del("yiffy2:images:#{category_before_last_save}") + Cache.redis.del("yiffy2:images:#{category}") + end + module SearchMethods def random(category, limit, size_limit = nil) q = where(category: category) @@ -27,14 +75,40 @@ class APIImage < ApplicationRecord id.gsub("-", "") end + alias stripped_md5 md5 + + def path_before_last_save + "/#{category_before_last_save.tr('.', '/')}/#{md5}.#{file_ext}" + end + + def path + "/#{category.tr('.', '/')}/#{md5}.#{file_ext}" + end + def url - "#{CDN_URL}#{category.gsub('.', '/')}/#{md5}.#{file_ext}" + Websites.config.yiffy2_storage.url_for(self) end def short_url ShortUrl.override(md5, url).shorturl end + def sources_string + sources.join("\n") + end + + def sources_string=(str) + self.sources = str.split("\n").map(&:strip) + end + + def artists_string + artists.join("\n") + end + + def artists_string=(str) + self.artists = str.split("\n").map(&:strip) + end + def serializable_hash(*) { artists: artists, @@ -61,9 +135,8 @@ class APIImage < ApplicationRecord **nsfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") }, } - # noinspection RubyMismatchedArgumentType - sfw.map { |c| { name: titles[c], db: c, sfw: true } } - .concat(nsfw.map { |c| { name: titles[c], db: c, sfw: false } }) + sfw.map { |cat| APICategory.new(titles[cat], cat, true) } + .concat(nsfw.map { |cat| APICategory.new(titles[cat], cat, false) }) end def self.category_title(db) @@ -96,4 +169,32 @@ class APIImage < ApplicationRecord state: "total", },] end + + def self.sync_all + sync_e621 + end + + # TODO + def self.sync_e621 + images = joins("CROSS JOIN unnest(sources) AS source").where("source ILIKE ?", "%e621.net%").distinct.limit(300) + ids = [] + md5s = [] + images.each do |img| + img.sources.each do |source| + next unless source.include?("e621.net") + id = source.split("/").last + ids << id.to_i if id.to_i.to_s == id + end + md5s << img.md5 + end + Requests::E621.get_all_posts(ids: ids, md5s: md5s) => { posts:, missing: } + Rails.logger.info("Failed to find #{missing[:ids].length} posts") unless missing[:ids].empty? + mismatch = images.select { |img| posts.none? { |post| post["file"]["md5"] == img.md5 } } + Rails.logger.info("Found #{mismatch.length} mismatched images") unless mismatch.empty? + posts + end + + def sync + APIImageSyncJob.perform_later(self) + end end diff --git a/app/models/api_user.rb b/app/models/api_user.rb index d61a5cd..e10f4c5 100644 --- a/app/models/api_user.rb +++ b/app/models/api_user.rb @@ -67,8 +67,8 @@ class APIUser < ApplicationRecord end def update_avatar(hash) - return if hash == last_avatar_hash && avatar.attached? - return if Cache.fetch("avatar_update:#{id}") + return false if hash == last_avatar_hash && avatar.attached? + return false if Cache.fetch("avatar_update:#{id}") Cache.write("avatar_update:#{id}", "1", expires_in: 1.day) avatar.purge url = "https://yiff.rest/Blep.png" @@ -76,6 +76,7 @@ class APIUser < ApplicationRecord image = URI.open(url) # rubocop:disable Security/Open avatar.attach(io: image, filename: "#{hash}.webp") update!(last_avatar_hash: hash) + true end def can_create_apikey? diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 8468393..78fc6f3 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -143,6 +143,7 @@ class ApplicationRecord < ActiveRecord::Base q = all q = q.attribute_matches(:id, params[:id]) + q = q.attribute_matches(:category, params[:category]) q = q.attribute_matches(:created_at, params[:created_at]) if attribute_names.include?("created_at") q = q.attribute_matches(:updated_at, params[:updated_at]) if attribute_names.include?("updated_at") diff --git a/app/models/e621_thumbnail.rb b/app/models/e621_thumbnail.rb index 02cba06..bd49a95 100644 --- a/app/models/e621_thumbnail.rb +++ b/app/models/e621_thumbnail.rb @@ -27,11 +27,15 @@ class E621Thumbnail < ApplicationRecord end def delete_files! - StorageManager::E621Thumbnails.delete("#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}") + Websites.config.e621_thumbnails_storage.delete("/#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}") end def url - StorageManager::E621Thumbnails.url_for(self) + Websites.config.e621_thumbnails_storage.url_for(self) + end + + def path + "/#{stripped_md5}.#{filetype}" end end diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index a2ba0d3..30934c1 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -1,4 +1,4 @@ -<% console if Rails.env.development? %> +<%# console if Rails.env.development? %> @@ -11,8 +11,8 @@ <%= yield :html_head %> <% end %> - <%= stylesheet_link_tag "application", "data-turbo-track": "reload" %> - <%= javascript_include_tag "application", "data-turbo-track": "reload" %> + <%= stylesheet_link_tag "application" %> + <%= javascript_include_tag "application" %> diff --git a/app/views/static/_backtrace.html.erb b/app/views/static/_backtrace.html.erb new file mode 100644 index 0000000..0e45671 --- /dev/null +++ b/app/views/static/_backtrace.html.erb @@ -0,0 +1,11 @@ +<%# locals: (backtrace:) -%> + + diff --git a/app/views/static/error.html.erb b/app/views/static/error.html.erb index b605baa..b9dd6d6 100644 --- a/app/views/static/error.html.erb +++ b/app/views/static/error.html.erb @@ -1,12 +1,16 @@ -<% content_for(:page_title) do %> - Error +<% if CurrentUser.user.try(:is_admin?) && @exception.present? %> +

<%= @exception.class.to_s %> exception raised

+ +

<%= @message || @exception.message.dup.force_encoding("utf-8") %>

+

Log ID: <%= @log_code || "(none)" %>

+ <%= render "static/backtrace", backtrace: @exception.backtrace %> +<% elsif @message %> +

<%= @message %>

+

Log ID: <%= @log_code || "(none)" %>

+<% else %> +

An error happened but there are no details provided.

<% end %> -

<%= @message %>

-<% if Rails.env.production? %> - <% if @log_code.present? %> -

Log Code: <%= @log_code %>

- <% end %> -<% else %> - <%= @backtrace %> +<% content_for(:page_title) do %> + Unexpected Error <% end %> diff --git a/app/views/yiff_rest/api_v2/images/edit.html.erb b/app/views/yiff_rest/api_v2/images/edit.html.erb new file mode 100644 index 0000000..e5632af --- /dev/null +++ b/app/views/yiff_rest/api_v2/images/edit.html.erb @@ -0,0 +1,16 @@ +<% content_for(:page_title) do %> + YiffyAPI - Edit Image +<% end %> + +<%= render "yiff_rest/api_v2/manage/nav" %> + +
+
+ <%= simple_form_for(@image, url: yiff_rest_api_v2_manage_image_path(manage_id: params[:manage_id], id: params[:id]), method: :patch) do |f| %> + <%= f.input(:sources_string, as: :text, label: "Sources") %> + <%= f.input(:artists_string, as: :text, label: "Artists") %> + <%= f.input(:category, as: :select, collection: @categories.map { |cat| [cat.name, cat.db] }) %> + <%= f.button(:submit, "Update Image", name: nil) %> + <% end %> +
+
diff --git a/app/views/yiff_rest/api_v2/images/index.html.erb b/app/views/yiff_rest/api_v2/images/index.html.erb new file mode 100644 index 0000000..a338c35 --- /dev/null +++ b/app/views/yiff_rest/api_v2/images/index.html.erb @@ -0,0 +1,65 @@ +<%= render "yiff_rest/api_v2/manage/nav" %> + +
+ + + + + + + + + + + + <% @images.each do |img| %> + + + + + + + + <% end %> + +
ArtistsDetailsSources
+ <%= link_to(img.url, target: "_blank", rel: "noopener") do %> + <%= image_tag(img.url, class: "img-fluid img-thumbnail") %> + <% end %> + <%= img.artists.join(", ") %> +
    +
  • Resolution: <%= img.width %>x<%= img.height %>
    • +
  • MD5: <%= img.md5 %>
    • +
  • Type: <%= img.file_ext %>
    • +
  • Size: <%= number_to_human_size(img.file_size) %>
    • +
  • Creator: <%= img.creator.name %>
    • +
  • Created At: <%= compact_time(img.created_at) %>
    • +
  • Updated At: <%= compact_time(img.updated_at) %>
    • + <% unless @sp[:category].present? %> +
  • Category: <%= img.category %>
    • + <% end %> +
  • + Upload Type: + <% if img.original_url.present? %> + <%= link_to "url".html_safe, img.original_url %> + <% else %> + file + <% end %> +
    • +
+ 		+ <% img.sources.each do |source| %> + <%= link_to source, source %>
+ <% end %> + 		+ <%= link_to "Edit", edit_yiff_rest_api_v2_manage_image_path(manage_id: params[:id], id: img.id) %> | + <%= link_to "Delete", yiff_rest_api_v2_manage_image_path(manage_id: params[:id], id: img.id), method: :delete %> +
+
+ +
+
+ <%== pagy_bootstrap_nav(@pagy) if @pagy.pages > 1 %> +
+
+ diff --git a/app/views/yiff_rest/api_v2/images/new.html.erb b/app/views/yiff_rest/api_v2/images/new.html.erb new file mode 100644 index 0000000..4141775 --- /dev/null +++ b/app/views/yiff_rest/api_v2/images/new.html.erb @@ -0,0 +1,19 @@ +<% content_for(:page_title) do %> + YiffyAPI - Upload Image +<% end %> + +<%= render "yiff_rest/api_v2/manage/nav" %> + + +
+
+ <%= simple_form_for(@image, url: yiff_rest_api_v2_manage_images_path, method: :post) do |f| %> + <%= f.input(:file, as: :file) %> + <%= f.input(:original_url, label: "File URL") %> + <%= f.input(:sources_string, as: :text, label: "Sources") %> + <%= f.input(:artists_string, as: :text, label: "Artists") %> + <%= f.input(:category, as: :select, input_html: { value: params.dig(:api_image, :category) }, collection: @categories.map { |cat| [cat.name, cat.db] }) %> + <%= f.button(:submit, "Upload Image", name: nil) %> + <% end %> +
+
diff --git a/app/views/yiff_rest/api_v2/manage/_nav.html.erb b/app/views/yiff_rest/api_v2/manage/_nav.html.erb new file mode 100644 index 0000000..6485143 --- /dev/null +++ b/app/views/yiff_rest/api_v2/manage/_nav.html.erb @@ -0,0 +1,52 @@ + diff --git a/app/views/yiff_rest/api_v2/manage/index.html.erb b/app/views/yiff_rest/api_v2/manage/index.html.erb new file mode 100644 index 0000000..1e59d81 --- /dev/null +++ b/app/views/yiff_rest/api_v2/manage/index.html.erb @@ -0,0 +1,40 @@ +<%= render "yiff_rest/api_v2/manage/nav" %> + + + + + + + + + + + <% @categories.each do |category| %> + <% total = category.real_count %> + + + + + + <% end %> + +
Category NameDetails
<%= category.name %> +
    +
  • ID: <%= category.db %>
    • +
  • Count: <%= total %> (cache: <%= category.count %>)
    • +
  • Created At: <%= compact_time(category.created_at) %>
    • +
  • Updated At: <%= compact_time(category.updated_at) %>
    • +
  • SFW: <%= category.sfw ? "Yes" : "No" %>
    • +
  • + Sources: +
      + <% category.sources.each do |source| %> +
    • <%= source[:name] %>: <%= source[:count] %> + <%# no point in having a percentage since they won't add up to 100 %> + <%# ((source[:count].to_f / total.to_f) * 100).round(0) %> +
      • + <% end %> +
    +
    • +
+ 		<%= link_to "Manage", yiff_rest_api_v2_manage_images_path(search: { category: category.db }) %>
diff --git a/config/default_config.rb b/config/default_config.rb index 9037ccf..6498c18 100644 --- a/config/default_config.rb +++ b/config/default_config.rb @@ -107,14 +107,6 @@ module Websites def e621_thumbnails_storage_zone_name end - def e621_thumbnails_base_url - "https://thumbs.yiff.media" - end - - def e621_thumbnails_base_path - "/data/e621-thumbnails" - end - def e621_thumbnails_webhook # Requests::DiscordWebhook.new(id: "", token: "") end @@ -211,6 +203,53 @@ module Websites ].join("; "), } end + + def http_headers + { + "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")", + } + end + + def httparty_options + { + timeout: 10, + open_timout: 5, + headers: http_headers, + } + end + + def yiffy2_cdn_url + # return "http://yiffy2.local" if Rails.env.development? + "https://v2.yiff.media" + end + + def yiffy2_bucket_name + end + + def yiffy2_access_key_id + end + + def yiffy2_secret_access_key + end + + def yiffy2_s3_endpoint + end + + def e621_thumbnails_storage + # Bunny.new(base_url: Websites.config.e621_thumbnails_base_url, access_key: Websites.config.e621_thumbnails_access_key, storage_zone_name: Websites.config.e621_thumbnails_storage_zone_name) + StorageManager::Local.new(base_url: "https://thumbs.yiff.media", base_path: "/data/e621-thumbnails") + end + + def yiffy2_storage + return StorageManager::Local.new(base_url: yiffy2_cdn_url, base_path: "/data/yiffy2") if Rails.env.development? + StorageManager::S3.new( + endpoint: yiffy2_s3_endpoint, + access_key_id: yiffy2_access_key_id, + secret_access_key: yiffy2_secret_access_key, + bucket: yiffy2_bucket_name, + base_url: yiffy2_cdn_url, + ) + end end class EnvironmentConfiguration diff --git a/config/initializers/pagy.rb b/config/initializers/pagy.rb new file mode 100644 index 0000000..0f46236 --- /dev/null +++ b/config/initializers/pagy.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require "pagy/extras/bootstrap" +Pagy::DEFAULT[:items] = 20 diff --git a/config/routes/domain_constraint.rb b/config/routes/domain_constraint.rb index 6a8bf6a..1aa88bb 100644 --- a/config/routes/domain_constraint.rb +++ b/config/routes/domain_constraint.rb @@ -11,17 +11,25 @@ class DomainConstraint end def matches?(request) - Rails.logger.info("Domain: #{@domain}; Subdomain: #{@subdomain}; Current Host: #{request.domain}; Current Subdomains: #{request.subdomain}; Matches (Domain): #{domain_matches?(request)}; Matches (Subdomain): #{subdomain_matches?(request)}") if Rails.env.development? + if Rails.env.development? + domain = request.env["websites.dev_domain"].presence || request.domain + subdomains = request.env["websites.dev_subdomains"].presence || request.subdomains + Rails.logger.info("Domain: #{@domain}; Subdomain: #{@subdomain}; Current Host: #{domain}; Current Subdomains: #{subdomains}; Matches (Domain): #{domain_matches?(request)}; Matches (Subdomain): #{subdomain_matches?(request)}") + end subdomain_matches?(request) && domain_matches?(request) end private def domain_matches?(request) - request.domain == @domain + (Rails.env.development? && request.env["websites.dev_domain"] == @domain) || request.domain == @domain end def subdomain_matches?(request) + if Rails.env.development? && !request.env["websites.dev_subdomains"].nil? + return true if @subdomain.nil? && request.env["websites.dev_subdomains"].blank? + return true if request.env["websites.dev_subdomains"] == @subdomain + end (@subdomain.nil? && request.subdomain.blank?) || request.subdomain == @subdomain end end diff --git a/config/routes/yiff_rest_routes.rb b/config/routes/yiff_rest_routes.rb index 4e8a0f5..71545c3 100644 --- a/config/routes/yiff_rest_routes.rb +++ b/config/routes/yiff_rest_routes.rb @@ -47,6 +47,13 @@ module YiffRestRoutes constraints(DomainConstraint.new(DOMAIN, V2)) do namespace(:api_v2, path: "") do + resources(:manage, constraints: { id: /[a-z0-9\-.]+/i }, only: %i[index]) do + collection do + get(:logout) + put(:sync) + resources(:images, as: "manage_images") + end + end get("/", to: redirect("https://yiff.rest")) get(:robots, constraints: { format: "txt" }) get(:state, to: redirect("https://state.yiff.rest")) @@ -56,7 +63,7 @@ module YiffRestRoutes get("/categories/(*category)", action: :category, as: :api_v2_category, constraints: { category: /[a-z0-9\-.]+/i }) get("/images/:id", action: :image, as: :api_v2_image, constraints: { id: /[a-f0-9]{32}/ }) post(:bulk, constraints: { format: "json" }, defaults: { format: :json }) - get("/(*category)", action: :index, as: :api_v2_images) + get("/(*category)", action: :index, as: :api_v2_images, constraints: { category: /(?!rails).*/ }) end end @@ -65,6 +72,7 @@ module YiffRestRoutes get(:count_servers) get(:flags) get(:apikey) + get(:manage_images) post(:interactions) end diff --git a/db/migrate/20240504225048_drop_api_images_created_by.rb b/db/migrate/20240504225048_drop_api_images_created_by.rb new file mode 100644 index 0000000..0b2e293 --- /dev/null +++ b/db/migrate/20240504225048_drop_api_images_created_by.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class DropAPIImagesCreatedBy < ActiveRecord::Migration[7.1] + def change + remove_column(:api_images, :created_by, :string, null: false) + end +end diff --git a/db/schema.rb b/db/schema.rb index 29c4269..80c3d78 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_01_02_073042) do +ActiveRecord::Schema[7.1].define(version: 2024_05_04_225048) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -51,7 +51,6 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_02_073042) do t.integer "height", null: false t.string "mime_type", null: false t.string "category", null: false - t.string "created_by", null: false t.string "original_url" t.string "file_ext", null: false t.integer "file_size", null: false diff --git a/docker-compose.yml b/docker-compose.yml index a08e17f..b5ac1b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - .:/app - e621_thumbnail_data:/data/e621-thumbnails - oceanic_docs_data:/data/oceanic-docs + - /var/www/yiffy2:/data/yiffy2 tmpfs: - /app/tmp/pids environment: diff --git a/lib/middleware/dev_host.rb b/lib/middleware/dev_host.rb index 34c8e62..8bf6876 100644 --- a/lib/middleware/dev_host.rb +++ b/lib/middleware/dev_host.rb @@ -1,14 +1,21 @@ +# frozen_string_literal: true + module Middleware class DevHost + DEFAULT_HOST = "v2.yiff.rest" def initialize(app) @app = app end def call(env) request = Rack::Request.new(env) - domain = request.params["domain"] + domain = request.params["domain"].presence || DEFAULT_HOST - env["HTTP_HOST"] = domain if Rails.env.development? && domain + if Rails.env.development? && domain + env["websites.dev_host"] = domain + env["websites.dev_domain"] = domain.split(".").last(2).join(".") + env["websites.dev_subdomains"] = domain.split(".")[0..-3].join(".") + end @app.call(env) end diff --git a/package.json b/package.json index 8592b08..58ddd2e 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "packageManager": "yarn@1.22.19", "dependencies": { "@hotwired/stimulus": "^3.2.2", - "@hotwired/turbo-rails": "^7.3.0", "@popperjs/core": "^2.11.8", "@rails/ujs": "^7.1.2", "bootstrap": "^5.3.2", diff --git a/yarn.lock b/yarn.lock index 3a7cdb6..84167da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -192,19 +192,6 @@ resolved "https://registry.yarnpkg.com/@hotwired/stimulus/-/stimulus-3.2.2.tgz#071aab59c600fed95b97939e605ff261a4251608" integrity sha512-eGeIqNOQpXoPAIP7tC1+1Yc1yl1xnwYqg+3mzqxyrbE5pg5YFBZcA6YoTiByJB6DKAEsiWtl6tjTJS4IYtbB7A== -"@hotwired/turbo-rails@^7.3.0": - version "7.3.0" - resolved "https://registry.yarnpkg.com/@hotwired/turbo-rails/-/turbo-rails-7.3.0.tgz#422c21752509f3edcd6c7b2725bbe9e157815f51" - integrity sha512-fvhO64vp/a2UVQ3jue9WTc2JisMv9XilIC7ViZmXAREVwiQ2S4UC7Go8f9A1j4Xu7DBI6SbFdqILk5ImqVoqyA== - dependencies: - "@hotwired/turbo" "^7.3.0" - "@rails/actioncable" "^7.0" - -"@hotwired/turbo@^7.3.0": - version "7.3.0" - resolved "https://registry.yarnpkg.com/@hotwired/turbo/-/turbo-7.3.0.tgz#2226000fff1aabda9fd9587474565c9929dbf15d" - integrity sha512-Dcu+NaSvHLT7EjrDrkEmH4qET2ZJZ5IcCWmNXxNQTBwlnE5tBZfN6WxZ842n5cHV52DH/AKNirbPBtcEXDLW4g== - "@humanwhocodes/config-array@^0.11.13": version "0.11.13" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297" @@ -268,11 +255,6 @@ resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.8.tgz#6b79032e760a0899cd4204710beede972a3a185f" integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A== -"@rails/actioncable@^7.0": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@rails/actioncable/-/actioncable-7.1.1.tgz#e8c49769d41f35a4473133c259cc98adc04dddf8" - integrity sha512-ZRJ9rdwFQQjRbtgJnweY0/4UQyxN6ojEGRdib0JkjnuIciv+4ok/aAeZmBJqNreTMaBqS0eHyA9hCArwN58opg== - "@rails/ujs@^7.1.2": version "7.1.2" resolved "https://registry.yarnpkg.com/@rails/ujs/-/ujs-7.1.2.tgz#ea903bcc0224e17156015d995b6f1b83e27d64b2"