diff --git a/.gitignore b/.gitignore index bc45c3f..552f561 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ # Ignore master key for decrypting credentials and more. /config/master.key +/config/local_config.rb /app/assets/builds/* !/app/assets/builds/.keep diff --git a/Dockerfile b/Dockerfile index 66748c9..9f2f555 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,4 @@ COPY --from=node-builder /root/.cache/node /root/.cache/node COPY --from=node-builder /app/node_modules node_modules COPY --from=ruby-builder /usr/local/bundle /usr/local/bundle -# Stop bin/rails console from offering autocomplete -RUN echo "IRB.conf[:USE_AUTOCOMPLETE] = false" > ~/.irbrc - CMD ["foreman", "start", "-f", "Procfile.dev"] diff --git a/Gemfile b/Gemfile index 160e3fd..a0aa477 100644 --- a/Gemfile +++ b/Gemfile @@ -21,9 +21,6 @@ gem "puma", ">= 5.0" # Bundle and transpile JavaScript [https://github.com/rails/jsbundling-rails] gem "jsbundling-rails" -# Hotwire's SPA-like page accelerator [https://turbo.hotwired.dev] -gem "turbo-rails" - # Hotwire's modest JavaScript framework [https://stimulus.hotwired.dev] gem "stimulus-rails" @@ -93,3 +90,11 @@ gem "ruby-vips", "~> 2.2" gem "simple_form", "~> 5.3" gem "responders", "~> 3.1" + +gem "pagy", "~> 8.3" + +gem "retriable", "~> 3.1" + +gem "addressable", "~> 2.8" + +gem "aws-sdk-s3", "~> 1.149" diff --git a/Gemfile.lock b/Gemfile.lock index f034ca2..b490686 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -77,6 +77,22 @@ GEM addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) ast (2.4.2) + aws-eventstream (1.3.0) + aws-partitions (1.924.0) + aws-sdk-core (3.194.1) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.651.0) + aws-sigv4 (~> 1.8) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.80.0) + aws-sdk-core (~> 3, >= 3.193.0) + aws-sigv4 (~> 1.1) + aws-sdk-s3 (1.149.0) + aws-sdk-core (~> 3, >= 3.194.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.8) + aws-sigv4 (1.8.0) + aws-eventstream (~> 1, >= 1.0.2) base64 (0.2.0) better_html (2.0.2) actionview (>= 6.0) @@ -145,6 +161,7 @@ GEM jbuilder (2.11.5) actionview (>= 5.0.0) activesupport (>= 5.0.0) + jmespath (1.6.2) jsbundling-rails (1.2.1) railties (>= 6.0.0) json (2.6.3) @@ -176,6 +193,7 @@ GEM nio4r (2.5.9) nokogiri (1.15.4-x86_64-linux) racc (~> 1.4) + pagy (8.3.0) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) @@ -241,6 +259,7 @@ GEM responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) + retriable (3.1.2) rexml (3.2.6) rubocop (1.57.2) json (~> 2.3) @@ -287,10 +306,6 @@ GEM stringio (3.0.8) thor (1.3.0) timeout (0.4.1) - turbo-rails (1.5.0) - actionpack (>= 6.0.0) - activejob (>= 6.0.0) - railties (>= 6.0.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.5.0) @@ -314,6 +329,8 @@ PLATFORMS x86_64-linux DEPENDENCIES + addressable (~> 2.8) + aws-sdk-s3 (~> 1.149) bootsnap capybara debug @@ -325,12 +342,14 @@ DEPENDENCIES image_optim (~> 0.31.3) jbuilder jsbundling-rails + pagy (~> 8.3) pg (~> 1.1) puma (>= 5.0) rails (~> 7.1.1) redis (~> 5.0) request_store (~> 1.5) responders (~> 3.1) + retriable (~> 3.1) rubocop (~> 1.57) rubocop-erb (~> 0.3.0) rubocop-rails (~> 2.22) @@ -340,7 +359,6 @@ DEPENDENCIES sprockets-rails stimulus-rails timeout (~> 0.4.1) - turbo-rails tzinfo-data web-console whenever (~> 1.0) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 30ad64a..121d2df 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -196,7 +196,7 @@ class ApplicationController < ActionController::Base @backtrace = Rails.backtrace_cleaner.clean(@exception.backtrace) format = :html unless format.in?(%i[html json]) - @message = "An unexpected error occurred." if Rails.env.production? && message == exception.message + @message = "An unexpected error occurred." if !(Rails.env.development? || CurrentUser.is_admin?) && message == exception.message WebLogger.log_exception(@exception, expected: @expected) log = ExceptionLog.add(exception, request) unless @expected @@ -253,4 +253,5 @@ class ApplicationController < ActionController::Base include CommonRoutes include RenderMethods include SearchMethods + include Pagy::Backend end diff --git a/app/controllers/yiff_rest/api_v2/application_controller.rb b/app/controllers/yiff_rest/api_v2/application_controller.rb new file mode 100644 index 0000000..1662664 --- /dev/null +++ b/app/controllers/yiff_rest/api_v2/application_controller.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module YiffRest + module APIV2 + class ApplicationController < ::YiffRest::ApplicationController + protected + + def site_domain + YiffRestRoutes::V2_DOMAIN + end + + def validate_discord + redirect_to(Websites.config.yiffyapi_discord_redirect("manage_images"), allow_other_host: true) if session[:discord_user].blank? + end + + def prepare_user + name = session.dig("discord_user", "global_name") || "#{session.dig('discord_user', 'username')}#{@session.dig('discord_user', 'discriminator')}" + CurrentUser.user = APIUser.find_or_create_by(id: session.dig("discord_user", "id")) + CurrentUser.update!(name: name, discord_data: session["discord_user"]) + CurrentUser.update_avatar(session.dig("discord_user", "avatar")) + end + end + end +end diff --git a/app/controllers/yiff_rest/api_v2/images_controller.rb b/app/controllers/yiff_rest/api_v2/images_controller.rb new file mode 100644 index 0000000..c9793d2 --- /dev/null +++ b/app/controllers/yiff_rest/api_v2/images_controller.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module YiffRest + module APIV2 + class ImagesController < ApplicationController + before_action :validate_discord + before_action :prepare_user + before_action :admin_only + + def index + @sp = search_images_params + @pagy, @images = pagy(APIImage.search(@sp).order(created_at: :desc), size: [1, 2, 2, 1]) + end + + def new + @image = APIImage.new(create_params) + @categories = APIImage.categories + end + + def edit + @image = APIImage.find(params[:id]) + @categories = APIImage.categories + end + + def create + @service = APIImageUploadService.new(create_params) + @image = @service.start! + @categories = APIImage.categories + if @image.invalid? + flash.now[:alert] = @image.errors.full_messages.join(", ") + return render(:new) + end + redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image added") + end + + def update + @image = APIImage.find(params[:id]) + @image.update(update_params) + @categories = APIImage.categories + if @image.invalid? + flash.now[:alert] = @image.errors.full_messages.join(", ") + return render(:edit) + end + redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image updated") + end + + def destroy + @image = APIImage.find(params[:id]) + @image.destroy + redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image deleted") + end + + private + + def load_category + @category = APIImage.categories.find { |c| c.db == params[:manage_id] } + raise(ActiveRecord::RecordNotFound) if @category.blank? + end + + def site_title + "YiffyAPI V2 - Manage Images" + end + + def search_images_params + permit_search_params(%i[category]) + end + + def create_params + params.fetch(:api_image, {}).permit(:category, :original_url, :sources_string, :artists_string, :file) + end + + def update_params + params.fetch(:api_image, {}).permit(:category, :sources_string, :artists_string) + end + end + end +end diff --git a/app/controllers/yiff_rest/api_v2/manage_controller.rb b/app/controllers/yiff_rest/api_v2/manage_controller.rb new file mode 100644 index 0000000..648cb45 --- /dev/null +++ b/app/controllers/yiff_rest/api_v2/manage_controller.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module YiffRest + module APIV2 + class ManageController < ApplicationController + before_action :validate_discord, except: %i[logout] + before_action :prepare_user, except: %i[logout] + + def index + @categories = APIImage.categories + end + + def show + @category = APIImage.categories.find { |c| c.db == params[:id] } + raise(ActiveRecord::RecordNotFound) if @category.blank? + @count = APIImage.cached_count(params[:id]) + @pagy, @images = pagy(APIImage.where(category: params[:id]).search(search_params).order(created_at: :desc), size: [1, 2, 2, 1]) + end + + def sync + APIImage.sync_all + redirect_to(yiff_rest_api_v2_manage_index_path, notice: "Images will soon be synced.") + end + + def logout + session.delete("discord_user") + CurrentUser.user = nil + redirect_to(yiff_rest_root_path, notice: "You have been logged out.") + end + + private + + def site_title + "YiffyAPI V2 - Manage Images" + end + end + end +end diff --git a/app/controllers/yiff_rest/api_v2_controller.rb b/app/controllers/yiff_rest/api_v2_controller.rb index 84399a7..78d5376 100644 --- a/app/controllers/yiff_rest/api_v2_controller.rb +++ b/app/controllers/yiff_rest/api_v2_controller.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module YiffRest - class APIV2Controller < ApplicationController + class APIV2Controller < APIV2::ApplicationController include ::ApplicationController::ReadonlyMethods before_action :handle_ratelimit, except: %i[robots] before_action :validate_images_access, only: %i[index] @@ -143,12 +143,8 @@ module YiffRest private - def site_domain - YiffRestRoutes::V2_DOMAIN - end - def api_categories - categories = APIImage.categories.pluck(:db) + categories = APIImage.categories.map(&:db) [*categories, "chris"] # gay little polar cutie end diff --git a/app/controllers/yiff_rest/discord_controller.rb b/app/controllers/yiff_rest/discord_controller.rb index 176aa23..9f71914 100644 --- a/app/controllers/yiff_rest/discord_controller.rb +++ b/app/controllers/yiff_rest/discord_controller.rb @@ -2,7 +2,7 @@ module YiffRest class DiscordController < ApplicationController - before_action :handle_oauth, only: %i[count_servers flags apikey] + before_action :handle_oauth, except: %i[index interactions] def index end @@ -14,10 +14,15 @@ module YiffRest end def apikey - return redirect_to("http://websites4.containers.local:3000/apikeys?domain=yiff.rest") if Rails.env.development? + return redirect_to("http://websites4.containers.local:3000/apikeys?domain=yiff.rest", allow_other_host: true) if Rails.env.development? redirect_to("https://yiff.rest/apikeys", allow_other_host: true) end + def manage_images + return redirect_to("http://websites4.containers.local:3000/manage?domain=v2.yiff.rest", allow_other_host: true) if Rails.env.development? + redirect_to("https://v2.yiff.rest/manage", allow_other_host: true) + end + def interactions begin verify_request! @@ -81,8 +86,10 @@ module YiffRest public_flags: DiscordConstants::UserFlags.parse_flags(user["public_flags"]).map(&:title), all_flags: DiscordConstants::UserFlags.parse_flags(user["flags"] - user["public_flags"]).map(&:title), } - when "apikey" + when "apikey", "manage_images" session[:discord_user] = res.get_authorization["user"].slice("id", "username", "discriminator", "global_name", "avatar") + else + raise("Unknown action: #{params[:action]}") end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 60c6bd6..e6a6d41 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true module ApplicationHelper + include Pagy::Frontend + def page_title return content_for(:page_title) if content_for?(:page_title) return site_title if defined?(site_title) @@ -33,4 +35,23 @@ module ApplicationHelper return css if request.path == path "" end + + def time_tag(content, time) + datetime = time.strftime("%Y-%m-%dT%H:%M%:z") + tag.time(content || datetime, datetime: datetime, title: time.to_fs) + end + + def compact_time(time) + time_tag(time.strftime("%Y-%m-%d %H:%M:%S"), time) + end + + def error_messages_for(instance_name) + instance = instance_variable_get("@#{instance_name}") + + if instance&.errors&.any? + %(
Error: #{instance.__send__(:errors).full_messages.join(', ')}
).html_safe + else + "" + end + end end diff --git a/app/javascript/application.ts b/app/javascript/application.ts index 63a5845..89fec31 100644 --- a/app/javascript/application.ts +++ b/app/javascript/application.ts @@ -1,6 +1,5 @@ /// // Entry point for the build script in your package.json -import "@hotwired/turbo-rails"; import "bootstrap"; import "bootstrap/dist/css/bootstrap.min.css"; import jQuery from "jquery"; diff --git a/app/javascript/controllers/notice.ts b/app/javascript/controllers/notice.ts index 1d08f30..1eeccee 100644 --- a/app/javascript/controllers/notice.ts +++ b/app/javascript/controllers/notice.ts @@ -3,9 +3,21 @@ import { Controller } from "@hotwired/stimulus"; export default class extends Controller { static override targets = ["notice"]; declare noticeTarget: HTMLElement; + timeout: number | undefined; - close(event: KeyboardEvent) { - event.preventDefault(); + // automatically close + override connect() { + this.timeout = setTimeout(() => this._close(), 3500); + } + + _close() { $(this.noticeTarget).fadeOut("fast"); } + + close(event: KeyboardEvent) { + if (this.timeout) { + clearTimeout(this.timeout); + } + event.preventDefault(); + } } diff --git a/app/javascript/styles/application.scss b/app/javascript/styles/application.scss index 6ef3cb8..4750858 100644 --- a/app/javascript/styles/application.scss +++ b/app/javascript/styles/application.scss @@ -47,3 +47,7 @@ body.c-application { text-align: center; } } + +.backtrace-line { + list-style: none; +} diff --git a/app/javascript/styles/yiff.rest/home.scss b/app/javascript/styles/yiff.rest/home.scss index 461a08b..2da8bed 100644 --- a/app/javascript/styles/yiff.rest/home.scss +++ b/app/javascript/styles/yiff.rest/home.scss @@ -1,6 +1,7 @@ @import "apikeys"; @import "discord"; @import "state"; +@import "v2"; body.s-yiff-rest { background-color: #2C2F33; diff --git a/app/javascript/styles/yiff.rest/v2.scss b/app/javascript/styles/yiff.rest/v2.scss new file mode 100644 index 0000000..1a32f21 --- /dev/null +++ b/app/javascript/styles/yiff.rest/v2.scss @@ -0,0 +1,87 @@ +body.s-v2-yiff-rest { + background-color: #2c2f33; + color: #fffdd0; + + a { + color: #fff; + text-decoration: none; + } + + // Bootstrap does not have built in dark pagination + .pagy-bootstrap { + $gray-100: #f8f9fa; + $gray-600: #6c757d; + $gray-800: #343a40; + $pagination-focus-outline: 0; + $focus-ring-width: 0.25rem; + $focus-ring-opacity: 0.25; + $blue: #0d6efd; + $white: #fff; + $primary: $blue; + $focus-ring-color: rgba($primary, $focus-ring-opacity); + $focus-ring-blur: 0; + $focus-ring-box-shadow: 0 0 $focus-ring-blur $focus-ring-width + $focus-ring-color; + $pagination-focus-box-shadow: $focus-ring-box-shadow; + + $pagination-dark-color: $gray-100; + $pagination-dark-bg: $gray-800; + $pagination-dark-border-color: $gray-600; + + $pagination-dark-hover-color: $pagination-dark-color; + $pagination-dark-hover-bg: $gray-600; + $pagination-dark-hover-border-color: $pagination-dark-border-color; + $component-active-color: $white; + $component-active-bg: $primary; + $pagination-active-color: $component-active-color; + $pagination-active-bg: $component-active-bg; + $pagination-active-border-color: $component-active-bg; + + .page-link { + color: $pagination-dark-color; + background-color: $pagination-dark-bg; + border-color: $pagination-dark-border-color; + + &:hover { + color: $pagination-dark-hover-color; + background-color: $pagination-dark-hover-bg; + border-color: $pagination-dark-hover-border-color; + } + + &:focus { + outline: $pagination-focus-outline; + box-shadow: $pagination-focus-box-shadow; + } + } + + .page-item { + &.active .page-link { + color: $pagination-active-color; + background-color: $pagination-active-bg; + border-color: $pagination-active-border-color; + } + + &.disabled .page-link { + background-color: $pagination-dark-bg; + border-color: $pagination-dark-border-color; + } + } + } + + .category-list, .category-image-list { + a { + color: #ede660; + } + + ul.details { + padding: 0; + + li { + list-style: none; + ul { + padding-left: 1rem; + } + } + } + } +} diff --git a/app/jobs/e621_thumbnail_job.rb b/app/jobs/e621_thumbnail_job.rb index 8984fd2..f06027f 100644 --- a/app/jobs/e621_thumbnail_job.rb +++ b/app/jobs/e621_thumbnail_job.rb @@ -8,11 +8,10 @@ class E621ThumbnailJob < ApplicationJob outfile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .#{entry.filetype}]) cutfile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .cut.webm]) palettefile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .palette.png]) - Timeout.timeout(1000) do + Timeout.timeout(120) do infile.binmode - HTTParty.get("https://static1.e621.net/data/#{entry.stripped_md5[0..1]}/#{entry.stripped_md5[2..3]}/#{entry.stripped_md5}.webm", stream_body: true) do |fragment| - infile.write(fragment) - end + io = FileDownload.new("https://static1.e621.net/data/#{entry.stripped_md5[0..1]}/#{entry.stripped_md5[2..3]}/#{entry.stripped_md5}.webm").download! + IO.copy_stream(io, infile) duration = `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 #{infile.path}`.to_f offset = duration > 10 ? rand(0..(duration - 10)) : duration / 2 if entry.filetype == "gif" @@ -23,7 +22,7 @@ class E621ThumbnailJob < ApplicationJob `ffmpeg -y -i #{infile.path} -ss #{offset} -vframes 1 #{outfile.path} 1>&2` end ImageOptim.new.optimize_image!(outfile.path) - StorageManager::E621Thumbnails.upload("#{entry.stripped_md5}.#{entry.filetype}", outfile.read) + Websites.config.e621_thumbnails_storage.upload("/#{entry.stripped_md5}.#{entry.filetype}", outfile) entry.update!(status: "complete") execute_webhook(entry, title: "Thumbnail Generated (#{entry.filetype})") end diff --git a/app/logical/api_category.rb b/app/logical/api_category.rb new file mode 100644 index 0000000..ce10715 --- /dev/null +++ b/app/logical/api_category.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +class APICategory + attr_reader :name, :db, :sfw + + SOURCES = [ + %w[e621.net E621], + %w[furaffinity.net FurAffinity], + %w[inkbunny.net InkBunny], + %w[deviantart.com DeviantArt], + %w[twitter.com Twitter], + ].freeze + + def initialize(name, db, sfw) + @name = name + @db = db + @sfw = sfw + end + + def count + APIImage.cached_count(db) + end + + def real_count + APIImage.where(category: db).count + end + + def unsourced_count + APIImage.where(category: db, sources: []).count + end + + def sources + r = APIImage.joins("CROSS JOIN unnest(sources) AS source").where(category: db).distinct + other = r + list = [] + + APICategory::SOURCES.each do |value, name| + count = r.where("source ILIKE ?", "%#{value}%").count + list << { name: name, count: count } + other = other.where("source NOT ILIKE ?", "%#{value}%") + end + list << { name: "Unsourced", count: unsourced_count } + # noinspection RubyMismatchedArgumentType + list << { name: "Other", count: other.count } + list + end + + def created_at + APIImage.where(category: db).minimum(:created_at) + end + + def updated_at + APIImage.where(category: db).maximum(:updated_at) + end + + def as_json(*) + { + name: name, + db: db, + sfw: sfw, + count: count, + } + end +end diff --git a/app/logical/api_image_upload_service.rb b/app/logical/api_image_upload_service.rb new file mode 100644 index 0000000..e1e8b3d --- /dev/null +++ b/app/logical/api_image_upload_service.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +class APIImageUploadService + class UploadError < StandardError; end + attr_reader :params, :image, :upload + + def initialize(params) + @params = params + end + + def start! + file = params.delete(:file) + APIImage.transaction do + @image = APIImage.new(**params, creator: CurrentUser.user) + begin + @image.file = get_file(@image, file: file) + process_file(@image, @image.file) + @image.save + return @image if @image.invalid? + store_file(@image, @image.file) + @image + rescue UploadError, ActiveRecord::RecordInvalid => e + @image.exception = e + @image + end + end + end + + def get_file(image, file: nil) + return file if file.present? + raise(UploadError, "No file or source URL provided") if image.original_url.blank? + + download = FileDownload.new(image.original_url) + download.download! + end + + def process_file(image, file) + mime, ext = @image.file_header_info(file.path) + image.mime_type = mime + image.file_ext = ext + image.file_size = file.size + image.id = Digest::MD5.file(file.path).hexdigest + + width, height = calculate_dimensions(file.path) + image.width = width + image.height = height + image.validate!(:file) + end + + def store_file(image, file) + Websites.config.yiffy2_storage.put(image.path, file) + end + + def calculate_dimensions(file_path) + image = Vips::Image.new_from_file(file_path) + [image.width, image.height] + end +end diff --git a/app/logical/cloudflare_service.rb b/app/logical/cloudflare_service.rb new file mode 100644 index 0000000..46f4401 --- /dev/null +++ b/app/logical/cloudflare_service.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module CloudflareService + def self.ips + text, code = Cache.fetch("cloudflare_ips", expires_in: 24.hours) do + resp = HTTParty.get("https://api.cloudflare.com/client/v4/ips", Websites.config.httparty_options) + [resp.body, resp.code] + end + return [] if code != 200 + + json = JSON.parse(text, symbolize_names: true) + ips = json[:result][:ipv4_cidrs] + json[:result][:ipv6_cidrs] + ips.map { |ip| IPAddr.new(ip) } + end +end diff --git a/app/logical/file_download.rb b/app/logical/file_download.rb new file mode 100644 index 0000000..d153433 --- /dev/null +++ b/app/logical/file_download.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +class FileDownload + include ActiveModel::Validations + class Error < StandardError; end + + RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError].freeze + MAX_SIZE = 50.megabytes + + attr_reader :url + + validate :validate_url + + def initialize(url) + begin + unencoded = Addressable::URI.unencode(url) + escaped = Addressable::URI.escape(unencoded) + @url = Addressable::URI.parse(escaped) + rescue Addressable::URI::InvalidURIError + @url = nil + end + validate! + end + + def size + res = HTTParty.head(uncached_url, **httparty_options, timeout: 3) + + if res.success? + res.content_length + else + raise(HTTParty::ResponseError, res) + end + end + + def download!(tries: 3, **) + Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do + http_get_streaming(uncached_url, **) + end + end + + def validate_url + errors.add(:base, "URL must not be blank") if url.blank? + errors.add(:base, "'#{url}' is not a valid url") if url.host.blank? + errors.add(:base, "'#{url}' is not a valid url. Did you mean 'https://#{url}'?") unless url.scheme.in?(%w[http https]) + end + + def http_get_streaming(url, file: Tempfile.new(binmode: true), max_size: MAX_SIZE) + size = 0 + + res = HTTParty.get(url, httparty_options) do |chunk| + next if [301, 302].include?(chunk.code) + + size += chunk.size + raise(Error, "File is too large (max size: #{max_size})") if size > max_size && max_size > 0 + + file.write(chunk) + end + + if res.success? + file.rewind + file + else + raise("HTTP error code: #{res.code} #{res.message}") + end + end + + # Prevent Cloudflare from potentially mangling the image + def uncached_url + return file_url unless is_cloudflare?(file_url) + + url = file_url.dup + url.query_values = url.query_values.to_h.merge(nc: Time.now) + url + end + + alias file_url url + + def httparty_options + { + timeout: 10, + stream_body: true, + connection_adapter: ValidatingConnectionAdapter, + }.deep_merge(Websites.config.httparty_options) + end + + def is_cloudflare?(url) + ip_addr = IPAddr.new(Resolv.getaddress(url.hostname)) + CloudflareService.ips.any? { |subnet| subnet.include?(ip_addr) } + end +end + +# Hook into HTTParty to validate the IP before following redirects. +# https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter +class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter + def self.call(uri, options) + ip_addr = IPAddr.new(Resolv.getaddress(uri.hostname)) + raise(FileDownload::Error, "Downloads from #{ip_addr} are not allowed") if ip_blocked?(ip_addr) + super + end + + def self.ip_blocked?(ip_addr) + ip_addr.private? || ip_addr.loopback? || ip_addr.link_local? + end +end diff --git a/app/logical/requests/e621.rb b/app/logical/requests/e621.rb index ce74ba5..6ac6d9f 100644 --- a/app/logical/requests/e621.rb +++ b/app/logical/requests/e621.rb @@ -7,7 +7,9 @@ module Requests def status r = self.class.get("/posts.json?limit=0", { + **Websites.config.httparty_options, headers: { + **Websites.config.http_headers, "User-Agent" => "E621Status/1.0.0 (https://status.e621.ws; \"donovan_dmc\")", }, timeout: 5, @@ -33,11 +35,7 @@ module Requests path = "/" path = "/posts/#{id}.json" if id path = "/posts.json?md5=#{md5}" if md5 - r = self.class.get(path, { - headers: { - "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")", - }, - }) + r = self.class.get(path, Websites.config.httparty_options) return nil if r.code != 200 JSON.parse(r.body)["post"] end @@ -49,20 +47,30 @@ module Requests path += "tags=md5:#{md5s.join(',')}%20limit:100" if md5s path += "%20status:#{status}" if path.include?("tags=") && !status.nil? path += "tags=status:#{status}" if !path.include?("tags=") && !status.nil? - r = self.class.get(path, { - headers: { - "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")", - }, - }) + Rails.logger.info("Fetching posts from e621: #{path}") if Rails.env.development? + r = self.class.get(path, Websites.config.httparty_options) JSON.parse(r.body)["posts"] || [] end + def get_all_posts(ids: [], md5s: []) + posts = [] + ids.each_slice(100) do |slice| + posts += get_posts(ids: slice, status: "any") + end + md5s -= posts.map { |p| p["file"]["md5"] } + md5s.each_slice(100) do |slice| + posts += get_posts(md5s: slice, status: "any") + end + missing = { + ids: ids - posts.pluck("id"), + md5s: md5s - posts.map { |p| p["file"]["md5"] }, + } + + { posts: posts, missing: missing } + end + def find_replacement(md5:) - r = self.class.get("/post_replacements.json?search[md5]=#{md5}", { - headers: { - "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")", - }, - }) + r = self.class.get("/post_replacements.json?search[md5]=#{md5}", Websites.config.httparty_options) JSON.parse(r.body)&.first end @@ -77,5 +85,9 @@ module Requests def self.get_posts(**) new.get_posts(**) end + + def self.get_all_posts(**) + new.get_all_posts(**) + end end end diff --git a/app/logical/storage_manager.rb b/app/logical/storage_manager.rb deleted file mode 100644 index 380d6f1..0000000 --- a/app/logical/storage_manager.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module StorageManager - E621Thumbnails = Local.new(base_url: Websites.config.e621_thumbnails_base_url, base_path: Websites.config.e621_thumbnails_base_path) - # E621Thumbnails = Bunny.new(base_url: Websites.config.e621_thumbnails_base_url, access_key: Websites.config.e621_thumbnails_access_key, storage_zone_name: Websites.config.e621_thumbnails_storage_zone_name) -end diff --git a/app/logical/storage_manager/bunny.rb b/app/logical/storage_manager/bunny.rb index cd22eb5..d9c78e2 100644 --- a/app/logical/storage_manager/bunny.rb +++ b/app/logical/storage_manager/bunny.rb @@ -23,7 +23,7 @@ module StorageManager end def url_for(entry) - "#{base_url}/#{entry.stripped_md5}.#{entry.filetype}" + "#{base_url}#{entry.path}" end end end diff --git a/app/logical/storage_manager/local.rb b/app/logical/storage_manager/local.rb index 0dc2346..c7b6569 100644 --- a/app/logical/storage_manager/local.rb +++ b/app/logical/storage_manager/local.rb @@ -2,6 +2,7 @@ module StorageManager class Local + DEFAULT_PERMISSIONS = 0o644 attr_reader :base_url, :base_path def initialize(base_url:, base_path:) @@ -11,28 +12,41 @@ module StorageManager def delete(path) return unless exists?(path) - File.delete("#{base_path}/#{path}") + File.delete("#{base_path}#{path}") end def exists?(path) - File.exist?("#{base_path}/#{path}") + File.exist?("#{base_path}#{path}") end def get(path) - File.read("#{base_path}/#{path}") + File.open("#{base_path}#{path}", "r", binmode: true) end - def put(path, body) - File.write("#{base_path}/#{path}", body) + def put(path, io) + temp = "#{base_path}#{path}-#{SecureRandom.uuid}.tmp" + + FileUtils.mkdir_p(File.dirname(temp)) + io.rewind + bytes_copied = IO.copy_stream(io, temp) + raise("store failed: #{bytes_copied}/#{io.size} bytes copied") if bytes_copied != io.size + + FileUtils.chmod(DEFAULT_PERMISSIONS, temp) + File.rename(temp, "#{base_path}#{path}") + rescue StandardError => e + FileUtils.rm_f(temp) + raise(e) + ensure + FileUtils.rm_f(temp) if temp end def upload(path, body) put(path, body) - "#{base_url}/#{path}" + "#{base_url}#{path}" end def url_for(entry) - "#{base_url}/#{entry.stripped_md5}.#{entry.filetype}" + "#{base_url}#{entry.path}" end end end diff --git a/app/logical/storage_manager/s3.rb b/app/logical/storage_manager/s3.rb new file mode 100644 index 0000000..241f3bf --- /dev/null +++ b/app/logical/storage_manager/s3.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module StorageManager + class S3 + attr_reader :base_url + + def initialize(endpoint:, access_key_id:, secret_access_key:, bucket:, base_url:) + @s3 = Aws::S3::Resource.new( + region: "weur", + endpoint: endpoint, + credentials: Aws::Credentials.new(access_key_id, secret_access_key), + ).bucket(bucket) + @base_url = base_url + end + + def delete(path) + return unless exists?(path) + @s3.object(trim(path)).delete + end + + def exists?(path) + @s3.object(trim(path)).exists? + end + + def get(path) + @s3.object(trim(path)).get.body + end + + def put(path, io) + @s3.object(trim(path)).put(body: io, content_type: Marcel::MimeType.for(io)) + end + + def upload(path, body) + put(path, body) + "#{base_url}#{path}" + end + + def url_for(entry) + "#{base_url}#{entry.path}" + end + + def trim(path) + # R2 for some reason doesn't trim preceding slashes + path.sub(%r{^/}, "") + end + end +end diff --git a/app/models/api_image.rb b/app/models/api_image.rb index e4d6212..d73b33a 100644 --- a/app/models/api_image.rb +++ b/app/models/api_image.rb @@ -1,10 +1,58 @@ # frozen_string_literal: true class APIImage < ApplicationRecord - CDN_URL = "https://v2.yiff.media/" - belongs_to_creator + attr_accessor :file, :exception + + validates :category, presence: true, inclusion: { in: -> { APIImage.categories.map(&:db) } } + validates :id, uniqueness: true, on: :file + validate on: :file do |image| + ext, mime = file_header_info(file.path) + image.errors.add(:file, "type is invalid (#{mime})") if ext == mime + end + validate do |image| + image.errors.add(:base, exception.message) if exception + end + + after_create :invalidate_cache + after_update :update_files, if: :saved_change_to_category? + after_destroy :delete_files + + def delete_files + Websites.config.yiffy2_storage.delete(path) + invalidate_cache + end + + def update_files + file = Websites.config.yiffy2_storage.get(path_before_last_save) + Websites.config.yiffy2_storage.put(path, file) + Websites.config.yiffy2_storage.delete(path_before_last_save) + invalidate_cache + end + + def file_header_info(file_path) + File.open(file_path) do |bin| + mime_type = Marcel::MimeType.for(bin) + ext = case mime_type + when "image/jpeg" + "jpg" + when "image/gif" + "gif" + when "image/png" + "png" + else + mime_type + end + [mime_type, ext] + end + end + + def invalidate_cache + Cache.redis.del("yiffy2:images:#{category_before_last_save}") + Cache.redis.del("yiffy2:images:#{category}") + end + module SearchMethods def random(category, limit, size_limit = nil) q = where(category: category) @@ -27,14 +75,40 @@ class APIImage < ApplicationRecord id.gsub("-", "") end + alias stripped_md5 md5 + + def path_before_last_save + "/#{category_before_last_save.tr('.', '/')}/#{md5}.#{file_ext}" + end + + def path + "/#{category.tr('.', '/')}/#{md5}.#{file_ext}" + end + def url - "#{CDN_URL}#{category.gsub('.', '/')}/#{md5}.#{file_ext}" + Websites.config.yiffy2_storage.url_for(self) end def short_url ShortUrl.override(md5, url).shorturl end + def sources_string + sources.join("\n") + end + + def sources_string=(str) + self.sources = str.split("\n").map(&:strip) + end + + def artists_string + artists.join("\n") + end + + def artists_string=(str) + self.artists = str.split("\n").map(&:strip) + end + def serializable_hash(*) { artists: artists, @@ -61,9 +135,8 @@ class APIImage < ApplicationRecord **nsfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") }, } - # noinspection RubyMismatchedArgumentType - sfw.map { |c| { name: titles[c], db: c, sfw: true } } - .concat(nsfw.map { |c| { name: titles[c], db: c, sfw: false } }) + sfw.map { |cat| APICategory.new(titles[cat], cat, true) } + .concat(nsfw.map { |cat| APICategory.new(titles[cat], cat, false) }) end def self.category_title(db) @@ -96,4 +169,32 @@ class APIImage < ApplicationRecord state: "total", },] end + + def self.sync_all + sync_e621 + end + + # TODO + def self.sync_e621 + images = joins("CROSS JOIN unnest(sources) AS source").where("source ILIKE ?", "%e621.net%").distinct.limit(300) + ids = [] + md5s = [] + images.each do |img| + img.sources.each do |source| + next unless source.include?("e621.net") + id = source.split("/").last + ids << id.to_i if id.to_i.to_s == id + end + md5s << img.md5 + end + Requests::E621.get_all_posts(ids: ids, md5s: md5s) => { posts:, missing: } + Rails.logger.info("Failed to find #{missing[:ids].length} posts") unless missing[:ids].empty? + mismatch = images.select { |img| posts.none? { |post| post["file"]["md5"] == img.md5 } } + Rails.logger.info("Found #{mismatch.length} mismatched images") unless mismatch.empty? + posts + end + + def sync + APIImageSyncJob.perform_later(self) + end end diff --git a/app/models/api_user.rb b/app/models/api_user.rb index d61a5cd..e10f4c5 100644 --- a/app/models/api_user.rb +++ b/app/models/api_user.rb @@ -67,8 +67,8 @@ class APIUser < ApplicationRecord end def update_avatar(hash) - return if hash == last_avatar_hash && avatar.attached? - return if Cache.fetch("avatar_update:#{id}") + return false if hash == last_avatar_hash && avatar.attached? + return false if Cache.fetch("avatar_update:#{id}") Cache.write("avatar_update:#{id}", "1", expires_in: 1.day) avatar.purge url = "https://yiff.rest/Blep.png" @@ -76,6 +76,7 @@ class APIUser < ApplicationRecord image = URI.open(url) # rubocop:disable Security/Open avatar.attach(io: image, filename: "#{hash}.webp") update!(last_avatar_hash: hash) + true end def can_create_apikey? diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 8468393..78fc6f3 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -143,6 +143,7 @@ class ApplicationRecord < ActiveRecord::Base q = all q = q.attribute_matches(:id, params[:id]) + q = q.attribute_matches(:category, params[:category]) q = q.attribute_matches(:created_at, params[:created_at]) if attribute_names.include?("created_at") q = q.attribute_matches(:updated_at, params[:updated_at]) if attribute_names.include?("updated_at") diff --git a/app/models/e621_thumbnail.rb b/app/models/e621_thumbnail.rb index 02cba06..bd49a95 100644 --- a/app/models/e621_thumbnail.rb +++ b/app/models/e621_thumbnail.rb @@ -27,11 +27,15 @@ class E621Thumbnail < ApplicationRecord end def delete_files! - StorageManager::E621Thumbnails.delete("#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}") + Websites.config.e621_thumbnails_storage.delete("/#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}") end def url - StorageManager::E621Thumbnails.url_for(self) + Websites.config.e621_thumbnails_storage.url_for(self) + end + + def path + "/#{stripped_md5}.#{filetype}" end end diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index a2ba0d3..30934c1 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -1,4 +1,4 @@ -<% console if Rails.env.development? %> +<%# console if Rails.env.development? %> @@ -11,8 +11,8 @@ <%= yield :html_head %> <% end %> - <%= stylesheet_link_tag "application", "data-turbo-track": "reload" %> - <%= javascript_include_tag "application", "data-turbo-track": "reload" %> + <%= stylesheet_link_tag "application" %> + <%= javascript_include_tag "application" %> diff --git a/app/views/static/_backtrace.html.erb b/app/views/static/_backtrace.html.erb new file mode 100644 index 0000000..0e45671 --- /dev/null +++ b/app/views/static/_backtrace.html.erb @@ -0,0 +1,11 @@ +<%# locals: (backtrace:) -%> + + diff --git a/app/views/static/error.html.erb b/app/views/static/error.html.erb index b605baa..b9dd6d6 100644 --- a/app/views/static/error.html.erb +++ b/app/views/static/error.html.erb @@ -1,12 +1,16 @@ -<% content_for(:page_title) do %> - Error +<% if CurrentUser.user.try(:is_admin?) && @exception.present? %> +

<%= @exception.class.to_s %> exception raised

+ +

<%= @message || @exception.message.dup.force_encoding("utf-8") %>

+

Log ID: <%= @log_code || "(none)" %>

+ <%= render "static/backtrace", backtrace: @exception.backtrace %> +<% elsif @message %> +

<%= @message %>

+

Log ID: <%= @log_code || "(none)" %>

+<% else %> +

An error happened but there are no details provided.

<% end %> -

<%= @message %>

-<% if Rails.env.production? %> - <% if @log_code.present? %> -

Log Code: <%= @log_code %>

- <% end %> -<% else %> - <%= @backtrace %> +<% content_for(:page_title) do %> + Unexpected Error <% end %> diff --git a/app/views/yiff_rest/api_v2/images/edit.html.erb b/app/views/yiff_rest/api_v2/images/edit.html.erb new file mode 100644 index 0000000..e5632af --- /dev/null +++ b/app/views/yiff_rest/api_v2/images/edit.html.erb @@ -0,0 +1,16 @@ +<% content_for(:page_title) do %> + YiffyAPI - Edit Image +<% end %> + +<%= render "yiff_rest/api_v2/manage/nav" %> + +
+
+ <%= simple_form_for(@image, url: yiff_rest_api_v2_manage_image_path(manage_id: params[:manage_id], id: params[:id]), method: :patch) do |f| %> + <%= f.input(:sources_string, as: :text, label: "Sources") %> + <%= f.input(:artists_string, as: :text, label: "Artists") %> + <%= f.input(:category, as: :select, collection: @categories.map { |cat| [cat.name, cat.db] }) %> + <%= f.button(:submit, "Update Image", name: nil) %> + <% end %> +
+
diff --git a/app/views/yiff_rest/api_v2/images/index.html.erb b/app/views/yiff_rest/api_v2/images/index.html.erb new file mode 100644 index 0000000..a338c35 --- /dev/null +++ b/app/views/yiff_rest/api_v2/images/index.html.erb @@ -0,0 +1,65 @@ +<%= render "yiff_rest/api_v2/manage/nav" %> + +
+ + + + + + + + + + + + <% @images.each do |img| %> + + + + + + + + <% end %> + +
ArtistsDetailsSources
+ <%= link_to(img.url, target: "_blank", rel: "noopener") do %> + <%= image_tag(img.url, class: "img-fluid img-thumbnail") %> + <% end %> + <%= img.artists.join(", ") %> +
    +
  • Resolution: <%= img.width %>x<%= img.height %>
    • +
  • MD5: <%= img.md5 %>
    • +
  • Type: <%= img.file_ext %>
    • +
  • Size: <%= number_to_human_size(img.file_size) %>
    • +
  • Creator: <%= img.creator.name %>
    • +
  • Created At: <%= compact_time(img.created_at) %>
    • +
  • Updated At: <%= compact_time(img.updated_at) %>
    • + <% unless @sp[:category].present? %> +
  • Category: <%= img.category %>
    • + <% end %> +
  • + Upload Type: + <% if img.original_url.present? %> + <%= link_to "url".html_safe, img.original_url %> + <% else %> + file + <% end %> +
    • +
+ 		+ <% img.sources.each do |source| %> + <%= link_to source, source %>
+ <% end %> + 		+ <%= link_to "Edit", edit_yiff_rest_api_v2_manage_image_path(manage_id: params[:id], id: img.id) %> | + <%= link_to "Delete", yiff_rest_api_v2_manage_image_path(manage_id: params[:id], id: img.id), method: :delete %> +
+
+ + + diff --git a/app/views/yiff_rest/api_v2/images/new.html.erb b/app/views/yiff_rest/api_v2/images/new.html.erb new file mode 100644 index 0000000..4141775 --- /dev/null +++ b/app/views/yiff_rest/api_v2/images/new.html.erb @@ -0,0 +1,19 @@ +<% content_for(:page_title) do %> + YiffyAPI - Upload Image +<% end %> + +<%= render "yiff_rest/api_v2/manage/nav" %> + + +
+
+ <%= simple_form_for(@image, url: yiff_rest_api_v2_manage_images_path, method: :post) do |f| %> + <%= f.input(:file, as: :file) %> + <%= f.input(:original_url, label: "File URL") %> + <%= f.input(:sources_string, as: :text, label: "Sources") %> + <%= f.input(:artists_string, as: :text, label: "Artists") %> + <%= f.input(:category, as: :select, input_html: { value: params.dig(:api_image, :category) }, collection: @categories.map { |cat| [cat.name, cat.db] }) %> + <%= f.button(:submit, "Upload Image", name: nil) %> + <% end %> +
+
diff --git a/app/views/yiff_rest/api_v2/manage/_nav.html.erb b/app/views/yiff_rest/api_v2/manage/_nav.html.erb new file mode 100644 index 0000000..6485143 --- /dev/null +++ b/app/views/yiff_rest/api_v2/manage/_nav.html.erb @@ -0,0 +1,52 @@ + diff --git a/app/views/yiff_rest/api_v2/manage/index.html.erb b/app/views/yiff_rest/api_v2/manage/index.html.erb new file mode 100644 index 0000000..1e59d81 --- /dev/null +++ b/app/views/yiff_rest/api_v2/manage/index.html.erb @@ -0,0 +1,40 @@ +<%= render "yiff_rest/api_v2/manage/nav" %> + + + + + + + + + + + <% @categories.each do |category| %> + <% total = category.real_count %> + + + + + + <% end %> + +
Category NameDetails
<%= category.name %> +
    +
  • ID: <%= category.db %>
    • +
  • Count: <%= total %> (cache: <%= category.count %>)
    • +
  • Created At: <%= compact_time(category.created_at) %>
    • +
  • Updated At: <%= compact_time(category.updated_at) %>
    • +
  • SFW: <%= category.sfw ? "Yes" : "No" %>
    • +
  • + Sources: +
      + <% category.sources.each do |source| %> +
    • <%= source[:name] %>: <%= source[:count] %> + <%# no point in having a percentage since they won't add up to 100 %> + <%# ((source[:count].to_f / total.to_f) * 100).round(0) %> +
      • + <% end %> +
    +
    • +
+ 		<%= link_to "Manage", yiff_rest_api_v2_manage_images_path(search: { category: category.db }) %>
diff --git a/config/default_config.rb b/config/default_config.rb index 9037ccf..6498c18 100644 --- a/config/default_config.rb +++ b/config/default_config.rb @@ -107,14 +107,6 @@ module Websites def e621_thumbnails_storage_zone_name end - def e621_thumbnails_base_url - "https://thumbs.yiff.media" - end - - def e621_thumbnails_base_path - "/data/e621-thumbnails" - end - def e621_thumbnails_webhook # Requests::DiscordWebhook.new(id: "", token: "") end @@ -211,6 +203,53 @@ module Websites ].join("; "), } end + + def http_headers + { + "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")", + } + end + + def httparty_options + { + timeout: 10, + open_timout: 5, + headers: http_headers, + } + end + + def yiffy2_cdn_url + # return "http://yiffy2.local" if Rails.env.development? + "https://v2.yiff.media" + end + + def yiffy2_bucket_name + end + + def yiffy2_access_key_id + end + + def yiffy2_secret_access_key + end + + def yiffy2_s3_endpoint + end + + def e621_thumbnails_storage + # Bunny.new(base_url: Websites.config.e621_thumbnails_base_url, access_key: Websites.config.e621_thumbnails_access_key, storage_zone_name: Websites.config.e621_thumbnails_storage_zone_name) + StorageManager::Local.new(base_url: "https://thumbs.yiff.media", base_path: "/data/e621-thumbnails") + end + + def yiffy2_storage + return StorageManager::Local.new(base_url: yiffy2_cdn_url, base_path: "/data/yiffy2") if Rails.env.development? + StorageManager::S3.new( + endpoint: yiffy2_s3_endpoint, + access_key_id: yiffy2_access_key_id, + secret_access_key: yiffy2_secret_access_key, + bucket: yiffy2_bucket_name, + base_url: yiffy2_cdn_url, + ) + end end class EnvironmentConfiguration diff --git a/config/initializers/pagy.rb b/config/initializers/pagy.rb new file mode 100644 index 0000000..0f46236 --- /dev/null +++ b/config/initializers/pagy.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require "pagy/extras/bootstrap" +Pagy::DEFAULT[:items] = 20 diff --git a/config/routes/domain_constraint.rb b/config/routes/domain_constraint.rb index 6a8bf6a..1aa88bb 100644 --- a/config/routes/domain_constraint.rb +++ b/config/routes/domain_constraint.rb @@ -11,17 +11,25 @@ class DomainConstraint end def matches?(request) - Rails.logger.info("Domain: #{@domain}; Subdomain: #{@subdomain}; Current Host: #{request.domain}; Current Subdomains: #{request.subdomain}; Matches (Domain): #{domain_matches?(request)}; Matches (Subdomain): #{subdomain_matches?(request)}") if Rails.env.development? + if Rails.env.development? + domain = request.env["websites.dev_domain"].presence || request.domain + subdomains = request.env["websites.dev_subdomains"].presence || request.subdomains + Rails.logger.info("Domain: #{@domain}; Subdomain: #{@subdomain}; Current Host: #{domain}; Current Subdomains: #{subdomains}; Matches (Domain): #{domain_matches?(request)}; Matches (Subdomain): #{subdomain_matches?(request)}") + end subdomain_matches?(request) && domain_matches?(request) end private def domain_matches?(request) - request.domain == @domain + (Rails.env.development? && request.env["websites.dev_domain"] == @domain) || request.domain == @domain end def subdomain_matches?(request) + if Rails.env.development? && !request.env["websites.dev_subdomains"].nil? + return true if @subdomain.nil? && request.env["websites.dev_subdomains"].blank? + return true if request.env["websites.dev_subdomains"] == @subdomain + end (@subdomain.nil? && request.subdomain.blank?) || request.subdomain == @subdomain end end diff --git a/config/routes/yiff_rest_routes.rb b/config/routes/yiff_rest_routes.rb index 4e8a0f5..71545c3 100644 --- a/config/routes/yiff_rest_routes.rb +++ b/config/routes/yiff_rest_routes.rb @@ -47,6 +47,13 @@ module YiffRestRoutes constraints(DomainConstraint.new(DOMAIN, V2)) do namespace(:api_v2, path: "") do + resources(:manage, constraints: { id: /[a-z0-9\-.]+/i }, only: %i[index]) do + collection do + get(:logout) + put(:sync) + resources(:images, as: "manage_images") + end + end get("/", to: redirect("https://yiff.rest")) get(:robots, constraints: { format: "txt" }) get(:state, to: redirect("https://state.yiff.rest")) @@ -56,7 +63,7 @@ module YiffRestRoutes get("/categories/(*category)", action: :category, as: :api_v2_category, constraints: { category: /[a-z0-9\-.]+/i }) get("/images/:id", action: :image, as: :api_v2_image, constraints: { id: /[a-f0-9]{32}/ }) post(:bulk, constraints: { format: "json" }, defaults: { format: :json }) - get("/(*category)", action: :index, as: :api_v2_images) + get("/(*category)", action: :index, as: :api_v2_images, constraints: { category: /(?!rails).*/ }) end end @@ -65,6 +72,7 @@ module YiffRestRoutes get(:count_servers) get(:flags) get(:apikey) + get(:manage_images) post(:interactions) end diff --git a/db/migrate/20240504225048_drop_api_images_created_by.rb b/db/migrate/20240504225048_drop_api_images_created_by.rb new file mode 100644 index 0000000..0b2e293 --- /dev/null +++ b/db/migrate/20240504225048_drop_api_images_created_by.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class DropAPIImagesCreatedBy < ActiveRecord::Migration[7.1] + def change + remove_column(:api_images, :created_by, :string, null: false) + end +end diff --git a/db/schema.rb b/db/schema.rb index 29c4269..80c3d78 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_01_02_073042) do +ActiveRecord::Schema[7.1].define(version: 2024_05_04_225048) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -51,7 +51,6 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_02_073042) do t.integer "height", null: false t.string "mime_type", null: false t.string "category", null: false - t.string "created_by", null: false t.string "original_url" t.string "file_ext", null: false t.integer "file_size", null: false diff --git a/docker-compose.yml b/docker-compose.yml index a08e17f..b5ac1b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - .:/app - e621_thumbnail_data:/data/e621-thumbnails - oceanic_docs_data:/data/oceanic-docs + - /var/www/yiffy2:/data/yiffy2 tmpfs: - /app/tmp/pids environment: diff --git a/lib/middleware/dev_host.rb b/lib/middleware/dev_host.rb index 34c8e62..8bf6876 100644 --- a/lib/middleware/dev_host.rb +++ b/lib/middleware/dev_host.rb @@ -1,14 +1,21 @@ +# frozen_string_literal: true + module Middleware class DevHost + DEFAULT_HOST = "v2.yiff.rest" def initialize(app) @app = app end def call(env) request = Rack::Request.new(env) - domain = request.params["domain"] + domain = request.params["domain"].presence || DEFAULT_HOST - env["HTTP_HOST"] = domain if Rails.env.development? && domain + if Rails.env.development? && domain + env["websites.dev_host"] = domain + env["websites.dev_domain"] = domain.split(".").last(2).join(".") + env["websites.dev_subdomains"] = domain.split(".")[0..-3].join(".") + end @app.call(env) end diff --git a/package.json b/package.json index 8592b08..58ddd2e 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "packageManager": "yarn@1.22.19", "dependencies": { "@hotwired/stimulus": "^3.2.2", - "@hotwired/turbo-rails": "^7.3.0", "@popperjs/core": "^2.11.8", "@rails/ujs": "^7.1.2", "bootstrap": "^5.3.2", diff --git a/yarn.lock b/yarn.lock index 3a7cdb6..84167da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -192,19 +192,6 @@ resolved "https://registry.yarnpkg.com/@hotwired/stimulus/-/stimulus-3.2.2.tgz#071aab59c600fed95b97939e605ff261a4251608" integrity sha512-eGeIqNOQpXoPAIP7tC1+1Yc1yl1xnwYqg+3mzqxyrbE5pg5YFBZcA6YoTiByJB6DKAEsiWtl6tjTJS4IYtbB7A== -"@hotwired/turbo-rails@^7.3.0": - version "7.3.0" - resolved "https://registry.yarnpkg.com/@hotwired/turbo-rails/-/turbo-rails-7.3.0.tgz#422c21752509f3edcd6c7b2725bbe9e157815f51" - integrity sha512-fvhO64vp/a2UVQ3jue9WTc2JisMv9XilIC7ViZmXAREVwiQ2S4UC7Go8f9A1j4Xu7DBI6SbFdqILk5ImqVoqyA== - dependencies: - "@hotwired/turbo" "^7.3.0" - "@rails/actioncable" "^7.0" - -"@hotwired/turbo@^7.3.0": - version "7.3.0" - resolved "https://registry.yarnpkg.com/@hotwired/turbo/-/turbo-7.3.0.tgz#2226000fff1aabda9fd9587474565c9929dbf15d" - integrity sha512-Dcu+NaSvHLT7EjrDrkEmH4qET2ZJZ5IcCWmNXxNQTBwlnE5tBZfN6WxZ842n5cHV52DH/AKNirbPBtcEXDLW4g== - "@humanwhocodes/config-array@^0.11.13": version "0.11.13" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297" @@ -268,11 +255,6 @@ resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.8.tgz#6b79032e760a0899cd4204710beede972a3a185f" integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A== -"@rails/actioncable@^7.0": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@rails/actioncable/-/actioncable-7.1.1.tgz#e8c49769d41f35a4473133c259cc98adc04dddf8" - integrity sha512-ZRJ9rdwFQQjRbtgJnweY0/4UQyxN6ojEGRdib0JkjnuIciv+4ok/aAeZmBJqNreTMaBqS0eHyA9hCArwN58opg== - "@rails/ujs@^7.1.2": version "7.1.2" resolved "https://registry.yarnpkg.com/@rails/ujs/-/ujs-7.1.2.tgz#ea903bcc0224e17156015d995b6f1b83e27d64b2"