Add image management ui

poorly tested but it worked well enough, I'm sure I'll be patching bugs over the next few weeks
Also remove turbo because it sucks
Also changed the way we handle hosts in dev

.gitignore

@@ -34,6 +34,7 @@
 
 # Ignore master key for decrypting credentials and more.
 /config/master.key
+/config/local_config.rb
 
 /app/assets/builds/*
 !/app/assets/builds/.keep

Dockerfile

@@ -40,7 +40,4 @@ COPY --from=node-builder /root/.cache/node /root/.cache/node
 COPY --from=node-builder /app/node_modules node_modules
 COPY --from=ruby-builder /usr/local/bundle /usr/local/bundle
 
-# Stop bin/rails console from offering autocomplete
-RUN echo "IRB.conf[:USE_AUTOCOMPLETE] = false" > ~/.irbrc
-
 CMD ["foreman", "start", "-f", "Procfile.dev"]

Gemfile

@@ -21,9 +21,6 @@ gem "puma", ">= 5.0"
 # Bundle and transpile JavaScript [https://github.com/rails/jsbundling-rails]
 gem "jsbundling-rails"
 
-# Hotwire's SPA-like page accelerator [https://turbo.hotwired.dev]
-gem "turbo-rails"
-
 # Hotwire's modest JavaScript framework [https://stimulus.hotwired.dev]
 gem "stimulus-rails"
 
@@ -93,3 +90,11 @@ gem "ruby-vips", "~> 2.2"
 gem "simple_form", "~> 5.3"
 
 gem "responders", "~> 3.1"
+
+gem "pagy", "~> 8.3"
+
+gem "retriable", "~> 3.1"
+
+gem "addressable", "~> 2.8"
+
+gem "aws-sdk-s3", "~> 1.149"

Gemfile.lock

@@ -77,6 +77,22 @@ GEM
     addressable (2.8.5)
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.924.0)
+    aws-sdk-core (3.194.1)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.8)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.80.0)
+      aws-sdk-core (~> 3, >= 3.193.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.149.0)
+      aws-sdk-core (~> 3, >= 3.194.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.8)
+    aws-sigv4 (1.8.0)
+      aws-eventstream (~> 1, >= 1.0.2)
     base64 (0.2.0)
     better_html (2.0.2)
       actionview (>= 6.0)
@@ -145,6 +161,7 @@ GEM
     jbuilder (2.11.5)
       actionview (>= 5.0.0)
       activesupport (>= 5.0.0)
+    jmespath (1.6.2)
     jsbundling-rails (1.2.1)
       railties (>= 6.0.0)
     json (2.6.3)
@@ -176,6 +193,7 @@ GEM
     nio4r (2.5.9)
     nokogiri (1.15.4-x86_64-linux)
       racc (~> 1.4)
+    pagy (8.3.0)
     parallel (1.23.0)
     parser (3.2.2.4)
       ast (~> 2.4.1)
@@ -241,6 +259,7 @@ GEM
     responders (3.1.1)
       actionpack (>= 5.2)
       railties (>= 5.2)
+    retriable (3.1.2)
     rexml (3.2.6)
     rubocop (1.57.2)
       json (~> 2.3)
@@ -287,10 +306,6 @@ GEM
     stringio (3.0.8)
     thor (1.3.0)
     timeout (0.4.1)
-    turbo-rails (1.5.0)
-      actionpack (>= 6.0.0)
-      activejob (>= 6.0.0)
-      railties (>= 6.0.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
@@ -314,6 +329,8 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  addressable (~> 2.8)
+  aws-sdk-s3 (~> 1.149)
   bootsnap
   capybara
   debug
@@ -325,12 +342,14 @@ DEPENDENCIES
   image_optim (~> 0.31.3)
   jbuilder
   jsbundling-rails
+  pagy (~> 8.3)
   pg (~> 1.1)
   puma (>= 5.0)
   rails (~> 7.1.1)
   redis (~> 5.0)
   request_store (~> 1.5)
   responders (~> 3.1)
+  retriable (~> 3.1)
   rubocop (~> 1.57)
   rubocop-erb (~> 0.3.0)
   rubocop-rails (~> 2.22)
@@ -340,7 +359,6 @@ DEPENDENCIES
   sprockets-rails
   stimulus-rails
   timeout (~> 0.4.1)
-  turbo-rails
   tzinfo-data
   web-console
   whenever (~> 1.0)

app/controllers/application_controller.rb

@@ -196,7 +196,7 @@ class ApplicationController < ActionController::Base
       @backtrace = Rails.backtrace_cleaner.clean(@exception.backtrace)
       format = :html unless format.in?(%i[html json])
 
-      @message = "An unexpected error occurred." if Rails.env.production? && message == exception.message
+      @message = "An unexpected error occurred." if !(Rails.env.development? || CurrentUser.is_admin?) && message == exception.message
 
       WebLogger.log_exception(@exception, expected: @expected)
       log = ExceptionLog.add(exception, request) unless @expected
@@ -253,4 +253,5 @@ class ApplicationController < ActionController::Base
   include CommonRoutes
   include RenderMethods
   include SearchMethods
+  include Pagy::Backend
 end

app/controllers/yiff_rest/api_v2/application_controller.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module YiffRest
+  module APIV2
+    class ApplicationController < ::YiffRest::ApplicationController
+      protected
+
+      def site_domain
+        YiffRestRoutes::V2_DOMAIN
+      end
+
+      def validate_discord
+        redirect_to(Websites.config.yiffyapi_discord_redirect("manage_images"), allow_other_host: true) if session[:discord_user].blank?
+      end
+
+      def prepare_user
+        name = session.dig("discord_user", "global_name") || "#{session.dig('discord_user', 'username')}#{@session.dig('discord_user', 'discriminator')}"
+        CurrentUser.user = APIUser.find_or_create_by(id: session.dig("discord_user", "id"))
+        CurrentUser.update!(name: name, discord_data: session["discord_user"])
+        CurrentUser.update_avatar(session.dig("discord_user", "avatar"))
+      end
+    end
+  end
+end

app/controllers/yiff_rest/api_v2/images_controller.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module YiffRest
+  module APIV2
+    class ImagesController < ApplicationController
+      before_action :validate_discord
+      before_action :prepare_user
+      before_action :admin_only
+
+      def index
+        @sp = search_images_params
+        @pagy, @images = pagy(APIImage.search(@sp).order(created_at: :desc), size: [1, 2, 2, 1])
+      end
+
+      def new
+        @image = APIImage.new(create_params)
+        @categories = APIImage.categories
+      end
+
+      def edit
+        @image = APIImage.find(params[:id])
+        @categories = APIImage.categories
+      end
+
+      def create
+        @service = APIImageUploadService.new(create_params)
+        @image = @service.start!
+        @categories = APIImage.categories
+        if @image.invalid?
+          flash.now[:alert] = @image.errors.full_messages.join(", ")
+          return render(:new)
+        end
+        redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image added")
+      end
+
+      def update
+        @image = APIImage.find(params[:id])
+        @image.update(update_params)
+        @categories = APIImage.categories
+        if @image.invalid?
+          flash.now[:alert] = @image.errors.full_messages.join(", ")
+          return render(:edit)
+        end
+        redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image updated")
+      end
+
+      def destroy
+        @image = APIImage.find(params[:id])
+        @image.destroy
+        redirect_to(yiff_rest_api_v2_manage_images_path(search: { category: @image.category }), notice: "Image deleted")
+      end
+
+      private
+
+      def load_category
+        @category = APIImage.categories.find { |c| c.db == params[:manage_id] }
+        raise(ActiveRecord::RecordNotFound) if @category.blank?
+      end
+
+      def site_title
+        "YiffyAPI V2 - Manage Images"
+      end
+
+      def search_images_params
+        permit_search_params(%i[category])
+      end
+
+      def create_params
+        params.fetch(:api_image, {}).permit(:category, :original_url, :sources_string, :artists_string, :file)
+      end
+
+      def update_params
+        params.fetch(:api_image, {}).permit(:category, :sources_string, :artists_string)
+      end
+    end
+  end
+end

app/controllers/yiff_rest/api_v2/manage_controller.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module YiffRest
+  module APIV2
+    class ManageController < ApplicationController
+      before_action :validate_discord, except: %i[logout]
+      before_action :prepare_user, except: %i[logout]
+
+      def index
+        @categories = APIImage.categories
+      end
+
+      def show
+        @category = APIImage.categories.find { |c| c.db == params[:id] }
+        raise(ActiveRecord::RecordNotFound) if @category.blank?
+        @count = APIImage.cached_count(params[:id])
+        @pagy, @images = pagy(APIImage.where(category: params[:id]).search(search_params).order(created_at: :desc), size: [1, 2, 2, 1])
+      end
+
+      def sync
+        APIImage.sync_all
+        redirect_to(yiff_rest_api_v2_manage_index_path, notice: "Images will soon be synced.")
+      end
+
+      def logout
+        session.delete("discord_user")
+        CurrentUser.user = nil
+        redirect_to(yiff_rest_root_path, notice: "You have been logged out.")
+      end
+
+      private
+
+      def site_title
+        "YiffyAPI V2 - Manage Images"
+      end
+    end
+  end
+end

app/controllers/yiff_rest/api_v2_controller.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module YiffRest
-  class APIV2Controller < ApplicationController
+  class APIV2Controller < APIV2::ApplicationController
     include ::ApplicationController::ReadonlyMethods
     before_action :handle_ratelimit, except: %i[robots]
     before_action :validate_images_access, only: %i[index]
@@ -143,12 +143,8 @@ module YiffRest
 
     private
 
-    def site_domain
-      YiffRestRoutes::V2_DOMAIN
-    end
-
     def api_categories
-      categories = APIImage.categories.pluck(:db)
+      categories = APIImage.categories.map(&:db)
       [*categories, "chris"] # gay little polar cutie
     end
 

app/controllers/yiff_rest/discord_controller.rb

@@ -2,7 +2,7 @@
 
 module YiffRest
   class DiscordController < ApplicationController
-    before_action :handle_oauth, only: %i[count_servers flags apikey]
+    before_action :handle_oauth, except: %i[index interactions]
 
     def index
     end
@@ -14,10 +14,15 @@ module YiffRest
     end
 
     def apikey
-      return redirect_to("http://websites4.containers.local:3000/apikeys?domain=yiff.rest") if Rails.env.development?
+      return redirect_to("http://websites4.containers.local:3000/apikeys?domain=yiff.rest", allow_other_host: true) if Rails.env.development?
       redirect_to("https://yiff.rest/apikeys", allow_other_host: true)
     end
 
+    def manage_images
+      return redirect_to("http://websites4.containers.local:3000/manage?domain=v2.yiff.rest", allow_other_host: true) if Rails.env.development?
+      redirect_to("https://v2.yiff.rest/manage", allow_other_host: true)
+    end
+
     def interactions
       begin
         verify_request!
@@ -81,8 +86,10 @@ module YiffRest
           public_flags:  DiscordConstants::UserFlags.parse_flags(user["public_flags"]).map(&:title),
           all_flags:     DiscordConstants::UserFlags.parse_flags(user["flags"] - user["public_flags"]).map(&:title),
         }
-      when "apikey"
+      when "apikey", "manage_images"
         session[:discord_user] = res.get_authorization["user"].slice("id", "username", "discriminator", "global_name", "avatar")
+      else
+        raise("Unknown action: #{params[:action]}")
       end
     end
 

app/helpers/application_helper.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module ApplicationHelper
+  include Pagy::Frontend
+
   def page_title
     return content_for(:page_title) if content_for?(:page_title)
     return site_title if defined?(site_title)
@@ -33,4 +35,23 @@ module ApplicationHelper
     return css if request.path == path
     ""
   end
+
+  def time_tag(content, time)
+    datetime = time.strftime("%Y-%m-%dT%H:%M%:z")
+    tag.time(content || datetime, datetime: datetime, title: time.to_fs)
+  end
+
+  def compact_time(time)
+    time_tag(time.strftime("%Y-%m-%d %H:%M:%S"), time)
+  end
+
+  def error_messages_for(instance_name)
+    instance = instance_variable_get("@#{instance_name}")
+
+    if instance&.errors&.any?
+      %(<div class="error-messages ui-state-error ui-corner-all"><strong>Error</strong>: #{instance.__send__(:errors).full_messages.join(', ')}</div>).html_safe
+    else
+      ""
+    end
+  end
 end

app/javascript/application.ts

@@ -1,6 +1,5 @@
 /// <reference path="./@types/global.d.ts" />
 // Entry point for the build script in your package.json
-import "@hotwired/turbo-rails";
 import "bootstrap";
 import "bootstrap/dist/css/bootstrap.min.css";
 import jQuery from "jquery";

app/javascript/controllers/notice.ts

@@ -3,9 +3,21 @@ import { Controller } from "@hotwired/stimulus";
 export default class extends Controller {
     static override targets = ["notice"];
     declare noticeTarget: HTMLElement;
+    timeout: number | undefined;
 
-    close(event: KeyboardEvent) {
+    // automatically close
-        event.preventDefault();
+    override connect() {
+        this.timeout = setTimeout(() => this._close(), 3500);
+    }
+
+    _close() {
         $(this.noticeTarget).fadeOut("fast");
     }
+
+    close(event: KeyboardEvent) {
+        if (this.timeout) {
+            clearTimeout(this.timeout);
+        }
+        event.preventDefault();
+    }
 }

app/javascript/styles/application.scss

@@ -47,3 +47,7 @@ body.c-application {
     text-align: center;
   }
 }
+
+.backtrace-line {
+  list-style: none;
+}

app/javascript/styles/yiff.rest/home.scss

@@ -1,6 +1,7 @@
 @import "apikeys";
 @import "discord";
 @import "state";
+@import "v2";
 
 body.s-yiff-rest {
   background-color: #2C2F33;

app/javascript/styles/yiff.rest/v2.scss

@@ -0,0 +1,87 @@
+body.s-v2-yiff-rest {
+  background-color: #2c2f33;
+  color: #fffdd0;
+
+  a {
+    color: #fff;
+    text-decoration: none;
+  }
+
+  // Bootstrap does not have built in dark pagination
+  .pagy-bootstrap {
+    $gray-100: #f8f9fa;
+    $gray-600: #6c757d;
+    $gray-800: #343a40;
+    $pagination-focus-outline: 0;
+    $focus-ring-width: 0.25rem;
+    $focus-ring-opacity: 0.25;
+    $blue: #0d6efd;
+    $white: #fff;
+    $primary: $blue;
+    $focus-ring-color: rgba($primary, $focus-ring-opacity);
+    $focus-ring-blur: 0;
+    $focus-ring-box-shadow: 0 0 $focus-ring-blur $focus-ring-width
+    $focus-ring-color;
+    $pagination-focus-box-shadow: $focus-ring-box-shadow;
+
+    $pagination-dark-color: $gray-100;
+    $pagination-dark-bg: $gray-800;
+    $pagination-dark-border-color: $gray-600;
+
+    $pagination-dark-hover-color: $pagination-dark-color;
+    $pagination-dark-hover-bg: $gray-600;
+    $pagination-dark-hover-border-color: $pagination-dark-border-color;
+    $component-active-color: $white;
+    $component-active-bg: $primary;
+    $pagination-active-color: $component-active-color;
+    $pagination-active-bg: $component-active-bg;
+    $pagination-active-border-color: $component-active-bg;
+
+    .page-link {
+      color: $pagination-dark-color;
+      background-color: $pagination-dark-bg;
+      border-color: $pagination-dark-border-color;
+
+      &:hover {
+        color: $pagination-dark-hover-color;
+        background-color: $pagination-dark-hover-bg;
+        border-color: $pagination-dark-hover-border-color;
+      }
+
+      &:focus {
+        outline: $pagination-focus-outline;
+        box-shadow: $pagination-focus-box-shadow;
+      }
+    }
+
+    .page-item {
+      &.active .page-link {
+        color: $pagination-active-color;
+        background-color: $pagination-active-bg;
+        border-color: $pagination-active-border-color;
+      }
+
+      &.disabled .page-link {
+        background-color: $pagination-dark-bg;
+        border-color: $pagination-dark-border-color;
+      }
+    }
+  }
+
+  .category-list, .category-image-list {
+    a {
+      color: #ede660;
+    }
+
+    ul.details {
+      padding: 0;
+
+      li {
+        list-style: none;
+        ul {
+          padding-left: 1rem;
+        }
+      }
+    }
+  }
+}

app/jobs/e621_thumbnail_job.rb

@@ -8,11 +8,10 @@ class E621ThumbnailJob < ApplicationJob
     outfile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .#{entry.filetype}])
     cutfile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .cut.webm])
     palettefile = Tempfile.new(%W[e621-thumbnail-#{entry.stripped_md5} .palette.png])
-    Timeout.timeout(1000) do
+    Timeout.timeout(120) do
       infile.binmode
-      HTTParty.get("https://static1.e621.net/data/#{entry.stripped_md5[0..1]}/#{entry.stripped_md5[2..3]}/#{entry.stripped_md5}.webm", stream_body: true) do |fragment|
+      io = FileDownload.new("https://static1.e621.net/data/#{entry.stripped_md5[0..1]}/#{entry.stripped_md5[2..3]}/#{entry.stripped_md5}.webm").download!
-        infile.write(fragment)
+      IO.copy_stream(io, infile)
-      end
       duration = `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 #{infile.path}`.to_f
       offset = duration > 10 ? rand(0..(duration - 10)) : duration / 2
       if entry.filetype == "gif"
@@ -23,7 +22,7 @@ class E621ThumbnailJob < ApplicationJob
         `ffmpeg -y -i #{infile.path} -ss #{offset} -vframes 1 #{outfile.path} 1>&2`
       end
       ImageOptim.new.optimize_image!(outfile.path)
-      StorageManager::E621Thumbnails.upload("#{entry.stripped_md5}.#{entry.filetype}", outfile.read)
+      Websites.config.e621_thumbnails_storage.upload("/#{entry.stripped_md5}.#{entry.filetype}", outfile)
       entry.update!(status: "complete")
       execute_webhook(entry, title: "Thumbnail Generated (#{entry.filetype})")
     end

app/logical/api_category.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+class APICategory
+  attr_reader :name, :db, :sfw
+
+  SOURCES = [
+    %w[e621.net E621],
+    %w[furaffinity.net FurAffinity],
+    %w[inkbunny.net InkBunny],
+    %w[deviantart.com DeviantArt],
+    %w[twitter.com Twitter],
+  ].freeze
+
+  def initialize(name, db, sfw)
+    @name = name
+    @db = db
+    @sfw = sfw
+  end
+
+  def count
+    APIImage.cached_count(db)
+  end
+
+  def real_count
+    APIImage.where(category: db).count
+  end
+
+  def unsourced_count
+    APIImage.where(category: db, sources: []).count
+  end
+
+  def sources
+    r = APIImage.joins("CROSS JOIN unnest(sources) AS source").where(category: db).distinct
+    other = r
+    list = []
+
+    APICategory::SOURCES.each do |value, name|
+      count = r.where("source ILIKE ?", "%#{value}%").count
+      list << { name: name, count: count }
+      other = other.where("source NOT ILIKE ?", "%#{value}%")
+    end
+    list << { name: "Unsourced", count: unsourced_count }
+    # noinspection RubyMismatchedArgumentType
+    list << { name: "Other", count: other.count }
+    list
+  end
+
+  def created_at
+    APIImage.where(category: db).minimum(:created_at)
+  end
+
+  def updated_at
+    APIImage.where(category: db).maximum(:updated_at)
+  end
+
+  def as_json(*)
+    {
+      name:  name,
+      db:    db,
+      sfw:   sfw,
+      count: count,
+    }
+  end
+end

app/logical/api_image_upload_service.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class APIImageUploadService
+  class UploadError < StandardError; end
+  attr_reader :params, :image, :upload
+
+  def initialize(params)
+    @params = params
+  end
+
+  def start!
+    file = params.delete(:file)
+    APIImage.transaction do
+      @image = APIImage.new(**params, creator: CurrentUser.user)
+      begin
+        @image.file = get_file(@image, file: file)
+        process_file(@image, @image.file)
+        @image.save
+        return @image if @image.invalid?
+        store_file(@image, @image.file)
+        @image
+      rescue UploadError, ActiveRecord::RecordInvalid => e
+        @image.exception = e
+        @image
+      end
+    end
+  end
+
+  def get_file(image, file: nil)
+    return file if file.present?
+    raise(UploadError, "No file or source URL provided") if image.original_url.blank?
+
+    download = FileDownload.new(image.original_url)
+    download.download!
+  end
+
+  def process_file(image, file)
+    mime, ext = @image.file_header_info(file.path)
+    image.mime_type = mime
+    image.file_ext = ext
+    image.file_size = file.size
+    image.id = Digest::MD5.file(file.path).hexdigest
+
+    width, height = calculate_dimensions(file.path)
+    image.width = width
+    image.height = height
+    image.validate!(:file)
+  end
+
+  def store_file(image, file)
+    Websites.config.yiffy2_storage.put(image.path, file)
+  end
+
+  def calculate_dimensions(file_path)
+    image = Vips::Image.new_from_file(file_path)
+    [image.width, image.height]
+  end
+end

app/logical/cloudflare_service.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module CloudflareService
+  def self.ips
+    text, code = Cache.fetch("cloudflare_ips", expires_in: 24.hours) do
+      resp = HTTParty.get("https://api.cloudflare.com/client/v4/ips", Websites.config.httparty_options)
+      [resp.body, resp.code]
+    end
+    return [] if code != 200
+
+    json = JSON.parse(text, symbolize_names: true)
+    ips = json[:result][:ipv4_cidrs] + json[:result][:ipv6_cidrs]
+    ips.map { |ip| IPAddr.new(ip) }
+  end
+end

app/logical/file_download.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+class FileDownload
+  include ActiveModel::Validations
+  class Error < StandardError; end
+
+  RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError].freeze
+  MAX_SIZE = 50.megabytes
+
+  attr_reader :url
+
+  validate :validate_url
+
+  def initialize(url)
+    begin
+      unencoded = Addressable::URI.unencode(url)
+      escaped = Addressable::URI.escape(unencoded)
+      @url = Addressable::URI.parse(escaped)
+    rescue Addressable::URI::InvalidURIError
+      @url = nil
+    end
+    validate!
+  end
+
+  def size
+    res = HTTParty.head(uncached_url, **httparty_options, timeout: 3)
+
+    if res.success?
+      res.content_length
+    else
+      raise(HTTParty::ResponseError, res)
+    end
+  end
+
+  def download!(tries: 3, **)
+    Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do
+      http_get_streaming(uncached_url, **)
+    end
+  end
+
+  def validate_url
+    errors.add(:base, "URL must not be blank") if url.blank?
+    errors.add(:base, "'#{url}' is not a valid url") if url.host.blank?
+    errors.add(:base, "'#{url}' is not a valid url. Did you mean 'https://#{url}'?") unless url.scheme.in?(%w[http https])
+  end
+
+  def http_get_streaming(url, file: Tempfile.new(binmode: true), max_size: MAX_SIZE)
+    size = 0
+
+    res = HTTParty.get(url, httparty_options) do |chunk|
+      next if [301, 302].include?(chunk.code)
+
+      size += chunk.size
+      raise(Error, "File is too large (max size: #{max_size})") if size > max_size && max_size > 0
+
+      file.write(chunk)
+    end
+
+    if res.success?
+      file.rewind
+      file
+    else
+      raise("HTTP error code: #{res.code} #{res.message}")
+    end
+  end
+
+  # Prevent Cloudflare from potentially mangling the image
+  def uncached_url
+    return file_url unless is_cloudflare?(file_url)
+
+    url = file_url.dup
+    url.query_values = url.query_values.to_h.merge(nc: Time.now)
+    url
+  end
+
+  alias file_url url
+
+  def httparty_options
+    {
+      timeout:            10,
+      stream_body:        true,
+      connection_adapter: ValidatingConnectionAdapter,
+    }.deep_merge(Websites.config.httparty_options)
+  end
+
+  def is_cloudflare?(url)
+    ip_addr = IPAddr.new(Resolv.getaddress(url.hostname))
+    CloudflareService.ips.any? { |subnet| subnet.include?(ip_addr) }
+  end
+end
+
+# Hook into HTTParty to validate the IP before following redirects.
+# https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter
+class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter
+  def self.call(uri, options)
+    ip_addr = IPAddr.new(Resolv.getaddress(uri.hostname))
+    raise(FileDownload::Error, "Downloads from #{ip_addr} are not allowed") if ip_blocked?(ip_addr)
+    super
+  end
+
+  def self.ip_blocked?(ip_addr)
+    ip_addr.private? || ip_addr.loopback? || ip_addr.link_local?
+  end
+end

app/logical/requests/e621.rb

@@ -7,7 +7,9 @@ module Requests
 
     def status
       r = self.class.get("/posts.json?limit=0", {
+        **Websites.config.httparty_options,
         headers: {
+          **Websites.config.http_headers,
           "User-Agent" => "E621Status/1.0.0 (https://status.e621.ws; \"donovan_dmc\")",
         },
         timeout: 5,
@@ -33,11 +35,7 @@ module Requests
       path = "/"
       path = "/posts/#{id}.json" if id
       path = "/posts.json?md5=#{md5}" if md5
-      r = self.class.get(path, {
+      r = self.class.get(path, Websites.config.httparty_options)
-        headers: {
-          "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")",
-        },
-      })
       return nil if r.code != 200
       JSON.parse(r.body)["post"]
     end
@@ -49,20 +47,30 @@ module Requests
       path += "tags=md5:#{md5s.join(',')}%20limit:100" if md5s
       path += "%20status:#{status}" if path.include?("tags=") && !status.nil?
       path += "tags=status:#{status}" if !path.include?("tags=") && !status.nil?
-      r = self.class.get(path, {
+      Rails.logger.info("Fetching posts from e621: #{path}") if Rails.env.development?
-        headers: {
+      r = self.class.get(path, Websites.config.httparty_options)
-          "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")",
-        },
-      })
       JSON.parse(r.body)["posts"] || []
     end
 
+    def get_all_posts(ids: [], md5s: [])
+      posts = []
+      ids.each_slice(100) do |slice|
+        posts += get_posts(ids: slice, status: "any")
+      end
+      md5s -= posts.map { |p| p["file"]["md5"] }
+      md5s.each_slice(100) do |slice|
+        posts += get_posts(md5s: slice, status: "any")
+      end
+      missing = {
+        ids:  ids - posts.pluck("id"),
+        md5s: md5s - posts.map { |p| p["file"]["md5"] },
+      }
+
+      { posts: posts, missing: missing }
+    end
+
     def find_replacement(md5:)
-      r = self.class.get("/post_replacements.json?search[md5]=#{md5}", {
+      r = self.class.get("/post_replacements.json?search[md5]=#{md5}", Websites.config.httparty_options)
-        headers: {
-          "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")",
-        },
-      })
       JSON.parse(r.body)&.first
     end
 
@@ -77,5 +85,9 @@ module Requests
     def self.get_posts(**)
       new.get_posts(**)
     end
+
+    def self.get_all_posts(**)
+      new.get_all_posts(**)
+    end
   end
 end

app/logical/storage_manager.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-module StorageManager
-  E621Thumbnails = Local.new(base_url: Websites.config.e621_thumbnails_base_url, base_path: Websites.config.e621_thumbnails_base_path)
-  # E621Thumbnails = Bunny.new(base_url: Websites.config.e621_thumbnails_base_url, access_key: Websites.config.e621_thumbnails_access_key, storage_zone_name: Websites.config.e621_thumbnails_storage_zone_name)
-end

app/logical/storage_manager/bunny.rb

@@ -23,7 +23,7 @@ module StorageManager
     end
 
     def url_for(entry)
-      "#{base_url}/#{entry.stripped_md5}.#{entry.filetype}"
+      "#{base_url}#{entry.path}"
     end
   end
 end

app/logical/storage_manager/local.rb

@@ -2,6 +2,7 @@
 
 module StorageManager
   class Local
+    DEFAULT_PERMISSIONS = 0o644
     attr_reader :base_url, :base_path
 
     def initialize(base_url:, base_path:)
@@ -11,28 +12,41 @@ module StorageManager
 
     def delete(path)
       return unless exists?(path)
-      File.delete("#{base_path}/#{path}")
+      File.delete("#{base_path}#{path}")
     end
 
     def exists?(path)
-      File.exist?("#{base_path}/#{path}")
+      File.exist?("#{base_path}#{path}")
     end
 
     def get(path)
-      File.read("#{base_path}/#{path}")
+      File.open("#{base_path}#{path}", "r", binmode: true)
     end
 
-    def put(path, body)
+    def put(path, io)
-      File.write("#{base_path}/#{path}", body)
+      temp = "#{base_path}#{path}-#{SecureRandom.uuid}.tmp"
+
+      FileUtils.mkdir_p(File.dirname(temp))
+      io.rewind
+      bytes_copied = IO.copy_stream(io, temp)
+      raise("store failed: #{bytes_copied}/#{io.size} bytes copied") if bytes_copied != io.size
+
+      FileUtils.chmod(DEFAULT_PERMISSIONS, temp)
+      File.rename(temp, "#{base_path}#{path}")
+    rescue StandardError => e
+      FileUtils.rm_f(temp)
+      raise(e)
+    ensure
+      FileUtils.rm_f(temp) if temp
     end
 
     def upload(path, body)
       put(path, body)
-      "#{base_url}/#{path}"
+      "#{base_url}#{path}"
     end
 
     def url_for(entry)
-      "#{base_url}/#{entry.stripped_md5}.#{entry.filetype}"
+      "#{base_url}#{entry.path}"
     end
   end
 end

app/logical/storage_manager/s3.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module StorageManager
+  class S3
+    attr_reader :base_url
+
+    def initialize(endpoint:, access_key_id:, secret_access_key:, bucket:, base_url:)
+      @s3 = Aws::S3::Resource.new(
+        region:      "weur",
+        endpoint:    endpoint,
+        credentials: Aws::Credentials.new(access_key_id, secret_access_key),
+      ).bucket(bucket)
+      @base_url = base_url
+    end
+
+    def delete(path)
+      return unless exists?(path)
+      @s3.object(trim(path)).delete
+    end
+
+    def exists?(path)
+      @s3.object(trim(path)).exists?
+    end
+
+    def get(path)
+      @s3.object(trim(path)).get.body
+    end
+
+    def put(path, io)
+      @s3.object(trim(path)).put(body: io, content_type: Marcel::MimeType.for(io))
+    end
+
+    def upload(path, body)
+      put(path, body)
+      "#{base_url}#{path}"
+    end
+
+    def url_for(entry)
+      "#{base_url}#{entry.path}"
+    end
+
+    def trim(path)
+      # R2 for some reason doesn't trim preceding slashes
+      path.sub(%r{^/}, "")
+    end
+  end
+end

app/models/api_image.rb

@@ -1,10 +1,58 @@
 # frozen_string_literal: true
 
 class APIImage < ApplicationRecord
-  CDN_URL = "https://v2.yiff.media/"
-
   belongs_to_creator
 
+  attr_accessor :file, :exception
+
+  validates :category, presence: true, inclusion: { in: -> { APIImage.categories.map(&:db) } }
+  validates :id, uniqueness: true, on: :file
+  validate on: :file do |image|
+    ext, mime = file_header_info(file.path)
+    image.errors.add(:file, "type is invalid (#{mime})") if ext == mime
+  end
+  validate do |image|
+    image.errors.add(:base, exception.message) if exception
+  end
+
+  after_create :invalidate_cache
+  after_update :update_files, if: :saved_change_to_category?
+  after_destroy :delete_files
+
+  def delete_files
+    Websites.config.yiffy2_storage.delete(path)
+    invalidate_cache
+  end
+
+  def update_files
+    file = Websites.config.yiffy2_storage.get(path_before_last_save)
+    Websites.config.yiffy2_storage.put(path, file)
+    Websites.config.yiffy2_storage.delete(path_before_last_save)
+    invalidate_cache
+  end
+
+  def file_header_info(file_path)
+    File.open(file_path) do |bin|
+      mime_type = Marcel::MimeType.for(bin)
+      ext = case mime_type
+            when "image/jpeg"
+              "jpg"
+            when "image/gif"
+              "gif"
+            when "image/png"
+              "png"
+            else
+              mime_type
+            end
+      [mime_type, ext]
+    end
+  end
+
+  def invalidate_cache
+    Cache.redis.del("yiffy2:images:#{category_before_last_save}")
+    Cache.redis.del("yiffy2:images:#{category}")
+  end
+
   module SearchMethods
     def random(category, limit, size_limit = nil)
       q = where(category: category)
@@ -27,14 +75,40 @@ class APIImage < ApplicationRecord
     id.gsub("-", "")
   end
 
+  alias stripped_md5 md5
+
+  def path_before_last_save
+    "/#{category_before_last_save.tr('.', '/')}/#{md5}.#{file_ext}"
+  end
+
+  def path
+    "/#{category.tr('.', '/')}/#{md5}.#{file_ext}"
+  end
+
   def url
-    "#{CDN_URL}#{category.gsub('.', '/')}/#{md5}.#{file_ext}"
+    Websites.config.yiffy2_storage.url_for(self)
   end
 
   def short_url
     ShortUrl.override(md5, url).shorturl
   end
 
+  def sources_string
+    sources.join("\n")
+  end
+
+  def sources_string=(str)
+    self.sources = str.split("\n").map(&:strip)
+  end
+
+  def artists_string
+    artists.join("\n")
+  end
+
+  def artists_string=(str)
+    self.artists = str.split("\n").map(&:strip)
+  end
+
   def serializable_hash(*)
     {
       artists:   artists,
@@ -61,9 +135,8 @@ class APIImage < ApplicationRecord
       **nsfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") },
     }
 
-    # noinspection RubyMismatchedArgumentType
+    sfw.map { |cat| APICategory.new(titles[cat], cat, true) }
-    sfw.map { |c| { name: titles[c], db: c, sfw: true } }
+       .concat(nsfw.map { |cat| APICategory.new(titles[cat], cat, false) })
-       .concat(nsfw.map { |c| { name: titles[c], db: c, sfw: false } })
   end
 
   def self.category_title(db)
@@ -96,4 +169,32 @@ class APIImage < ApplicationRecord
       state:    "total",
     },]
   end
+
+  def self.sync_all
+    sync_e621
+  end
+
+  # TODO
+  def self.sync_e621
+    images = joins("CROSS JOIN unnest(sources) AS source").where("source ILIKE ?", "%e621.net%").distinct.limit(300)
+    ids = []
+    md5s = []
+    images.each do |img|
+      img.sources.each do |source|
+        next unless source.include?("e621.net")
+        id = source.split("/").last
+        ids << id.to_i if id.to_i.to_s == id
+      end
+      md5s << img.md5
+    end
+    Requests::E621.get_all_posts(ids: ids, md5s: md5s) => { posts:, missing: }
+    Rails.logger.info("Failed to find #{missing[:ids].length} posts") unless missing[:ids].empty?
+    mismatch = images.select { |img| posts.none? { |post| post["file"]["md5"] == img.md5 } }
+    Rails.logger.info("Found #{mismatch.length} mismatched images") unless mismatch.empty?
+    posts
+  end
+
+  def sync
+    APIImageSyncJob.perform_later(self)
+  end
 end

app/models/api_user.rb

@@ -67,8 +67,8 @@ class APIUser < ApplicationRecord
   end
 
   def update_avatar(hash)
-    return if hash == last_avatar_hash && avatar.attached?
+    return false if hash == last_avatar_hash && avatar.attached?
-    return if Cache.fetch("avatar_update:#{id}")
+    return false if Cache.fetch("avatar_update:#{id}")
     Cache.write("avatar_update:#{id}", "1", expires_in: 1.day)
     avatar.purge
     url = "https://yiff.rest/Blep.png"
@@ -76,6 +76,7 @@ class APIUser < ApplicationRecord
     image = URI.open(url) # rubocop:disable Security/Open
     avatar.attach(io: image, filename: "#{hash}.webp")
     update!(last_avatar_hash: hash)
+    true
   end
 
   def can_create_apikey?

app/models/application_record.rb

@@ -143,6 +143,7 @@ class ApplicationRecord < ActiveRecord::Base
 
       q = all
       q = q.attribute_matches(:id, params[:id])
+      q = q.attribute_matches(:category, params[:category])
       q = q.attribute_matches(:created_at, params[:created_at]) if attribute_names.include?("created_at")
       q = q.attribute_matches(:updated_at, params[:updated_at]) if attribute_names.include?("updated_at")
 

app/models/e621_thumbnail.rb

@@ -27,11 +27,15 @@ class E621Thumbnail < ApplicationRecord
     end
 
     def delete_files!
-      StorageManager::E621Thumbnails.delete("#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}")
+      Websites.config.e621_thumbnails_storage.delete("/#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}")
     end
 
     def url
-      StorageManager::E621Thumbnails.url_for(self)
+      Websites.config.e621_thumbnails_storage.url_for(self)
+    end
+
+    def path
+      "/#{stripped_md5}.#{filetype}"
     end
   end
 

app/views/layouts/application.html.erb

@@ -1,4 +1,4 @@
-<% console if Rails.env.development? %>
+<%# console if Rails.env.development? %>
 <!DOCTYPE html>
 <html lang="en">
   <head>
@@ -11,8 +11,8 @@
       <%= yield :html_head %>
     <% end %>
 
-    <%= stylesheet_link_tag "application", "data-turbo-track": "reload" %>
+    <%= stylesheet_link_tag "application" %>
-    <%= javascript_include_tag "application", "data-turbo-track": "reload" %>
+    <%= javascript_include_tag "application" %>
   </head>
 
   <body class="<%= body_class %>">

app/views/static/_backtrace.html.erb

@@ -0,0 +1,11 @@
+<%# locals: (backtrace:) -%>
+
+<ul class="backtrace">
+  <% if defined?(exception) %>
+    <li><%= exception.class.to_s %> exception raised</li>
+  <% end %>
+
+  <% Rails.backtrace_cleaner.clean(backtrace).each do |b| %>
+    <li class="backtrace-line"><%= b %></li>
+  <% end %>
+</ul>

app/views/static/error.html.erb

@@ -1,12 +1,16 @@
-<% content_for(:page_title) do %>
+<% if CurrentUser.user.try(:is_admin?) && @exception.present? %>
-  Error
+  <h1><%= @exception.class.to_s %> exception raised</h1>
+
+  <p><%= @message || @exception.message.dup.force_encoding("utf-8") %></p>
+  <p>Log ID: <%= @log_code || "(none)" %></p>
+  <%= render "static/backtrace", backtrace: @exception.backtrace %>
+<% elsif @message %>
+  <p><%= @message %></p>
+  <p>Log ID: <%= @log_code || "(none)" %></p>
+<% else %>
+  <p>An error happened but there are no details provided.</p>
 <% end %>
 
-<h1><%= @message %></h1>
+<% content_for(:page_title) do %>
-<% if Rails.env.production? %>
+  Unexpected Error
-  <% if @log_code.present? %>
-    <h3>Log Code: <%= @log_code %></h3>
-  <% end %>
-<% else %>
-  <%= @backtrace %>
 <% end %>

app/views/yiff_rest/api_v2/images/edit.html.erb

@@ -0,0 +1,16 @@
+<% content_for(:page_title) do %>
+  YiffyAPI - Edit Image
+<% end %>
+
+<%= render "yiff_rest/api_v2/manage/nav" %>
+
+<div class="w-100">
+  <div style="width: 40%; margin-left: 30%;">
+    <%= simple_form_for(@image, url: yiff_rest_api_v2_manage_image_path(manage_id: params[:manage_id], id: params[:id]), method: :patch) do |f| %>
+      <%= f.input(:sources_string, as: :text, label: "Sources") %>
+      <%= f.input(:artists_string, as: :text, label: "Artists") %>
+      <%= f.input(:category, as: :select, collection: @categories.map { |cat| [cat.name, cat.db] }) %>
+      <%= f.button(:submit, "Update Image", name: nil) %>
+    <% end %>
+  </div>
+</div>

app/views/yiff_rest/api_v2/images/index.html.erb

@@ -0,0 +1,65 @@
+<%= render "yiff_rest/api_v2/manage/nav" %>
+
+<div class="category-image-list pt-2" style="padding-bottom: 70px;">
+  <table class="table table-striped table-dark">
+    <thead>
+    <tr>
+      <th style="width: 12%;"></th>
+      <th style="width: 10%;">Artists</th>
+      <th style="width: 25%;">Details</th>
+      <th style="width: 45%;">Sources</th>
+      <th style="width: 8%"></th>
+    </tr>
+    </thead>
+    <tbody>
+    <% @images.each do |img| %>
+      <tr>
+        <td style="width: 12%">
+          <%= link_to(img.url, target: "_blank", rel: "noopener") do %>
+            <%= image_tag(img.url, class: "img-fluid img-thumbnail") %>
+          <% end %>
+        </td>
+        <td><%= img.artists.join(", ") %></td>
+        <td>
+          <ul class="details">
+            <li>Resolution: <b><%= img.width %>x<%= img.height %></b></li>
+            <li>MD5: <b><%= img.md5 %></b></li>
+            <li>Type: <b><%= img.file_ext %></b></li>
+            <li>Size: <b><%= number_to_human_size(img.file_size) %></b></li>
+            <li>Creator: <b><%= img.creator.name %></b></li>
+            <li>Created At: <b><%= compact_time(img.created_at) %></b></li>
+            <li>Updated At: <b><%= compact_time(img.updated_at) %></b></li>
+            <% unless @sp[:category].present? %>
+              <li>Category: <b><%= img.category %></b></li>
+            <% end %>
+            <li>
+              Upload Type:
+              <% if img.original_url.present? %>
+                <%= link_to "<b>url</b>".html_safe, img.original_url %>
+              <% else %>
+                <b>file</b>
+              <% end %>
+            </li>
+          </ul>
+        </td>
+        <td>
+          <% img.sources.each do |source| %>
+            <%= link_to source, source %><br>
+          <% end %>
+        </td>
+        <td>
+          <%= link_to "Edit", edit_yiff_rest_api_v2_manage_image_path(manage_id: params[:id], id: img.id) %> |
+          <%= link_to "Delete", yiff_rest_api_v2_manage_image_path(manage_id: params[:id], id: img.id), method: :delete %>
+        </td>
+      </tr>
+    <% end %>
+    </tbody>
+  </table>
+</div>
+
+<footer class="bg-dark pt-3 fixed-bottom">
+  <div class=" d-flex align-items-center justify-content-center">
+    <%== pagy_bootstrap_nav(@pagy) if @pagy.pages > 1 %>
+  </div>
+</footer>
+

app/views/yiff_rest/api_v2/images/new.html.erb

@@ -0,0 +1,19 @@
+<% content_for(:page_title) do %>
+  YiffyAPI - Upload Image
+<% end %>
+
+<%= render "yiff_rest/api_v2/manage/nav" %>
+
+
+<div class="w-100">
+  <div style="width: 40%; margin-left: 30%;">
+    <%= simple_form_for(@image, url: yiff_rest_api_v2_manage_images_path, method: :post) do |f| %>
+      <%= f.input(:file, as: :file) %>
+      <%= f.input(:original_url, label: "File URL") %>
+      <%= f.input(:sources_string, as: :text, label: "Sources") %>
+      <%= f.input(:artists_string, as: :text, label: "Artists") %>
+      <%= f.input(:category, as: :select, input_html: { value: params.dig(:api_image, :category) }, collection: @categories.map { |cat| [cat.name, cat.db] }) %>
+      <%= f.button(:submit, "Upload Image", name: nil) %>
+    <% end %>
+  </div>
+</div>

app/views/yiff_rest/api_v2/manage/_nav.html.erb

@@ -0,0 +1,52 @@
+<nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+  <div class="container-fluid">
+    <a class="navbar-brand" href="#">
+      <%= image_tag("#{assets_path}/Blep.png", width: "30", height: "30", class: "rounded-4") %>
+    </a>
+    <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    <div class="collapse navbar-collapse" id="navbarSupportedContent">
+      <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+        <li class="nav-item">
+          <%= link_to "Home", yiff_rest_api_v2_manage_index_path, class: "nav-link#{if_active(' active', path: yiff_rest_api_v2_manage_index_path)}" %>
+        </li>
+        <% if params[:controller] == "yiff_rest/api_v2/manage" %>
+        <% elsif params[:controller] == "yiff_rest/api_v2/images" %>
+         <% if params[:action] == "index" %>
+            <li class="nav-item">
+              <%= link_to "Upload Image", new_yiff_rest_api_v2_manage_image_path(api_image: ({ category: @sp[:category] } if @sp[:category])), class: "nav-link#{if_active(' active', path: new_yiff_rest_api_v2_manage_image_path(manage_id: params[:id]))}" %>
+            </li>
+          <% else %>
+            <% category = params.dig(:api_image, :category) if params[:action] == "new" %>
+            <% category = params.dig(:search, :category) if params[:action] == "index" %>
+            <% category = @image.category if params[:action] == "edit" %>
+            <% if category %>
+              <li class="nav-item">
+                <%= link_to "Back To Category", yiff_rest_api_v2_manage_images_path(search: { category: category }), class: "nav-link" %>
+              </li>
+            <% end %>
+          <% end %>
+        <% end %>
+      </ul>
+      <div class="d-flex">
+        <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+          <li class="nav-item dropdown">
+            <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+              <%= CurrentUser.name %>
+              <% if CurrentUser.is_manager? %>
+                (<%= CurrentUser.level_name %>)
+              <% end %>
+            </a>
+            <ul class="dropdown-menu dropdown-menu-dark">
+              <li><%= link_to "Log Out", logout_yiff_rest_api_v2_manage_index_path, class: "dropdown-item" %></li>
+            </ul>
+          </li>
+        </ul>
+        <% if CurrentUser.avatar.attached? %>
+          <%= image_tag(CurrentUser.avatar, width: 30, height: 30, class: "rounded-4", alt: "Discord User Icon") %>
+        <% end %>
+      </div>
+    </div>
+  </div>
+</nav>

app/views/yiff_rest/api_v2/manage/index.html.erb

@@ -0,0 +1,40 @@
+<%= render "yiff_rest/api_v2/manage/nav" %>
+
+<table class="category-list table table-striped table-dark">
+  <thead>
+    <tr>
+      <th style="width: 70%;">Category Name</th>
+      <th style="width: 20%;">Details</th>
+      <th style="width: 10%;"></th>
+    </tr>
+  </thead>
+  <tbody>
+    <% @categories.each do |category| %>
+      <% total = category.real_count %>
+      <tr>
+        <td><%= category.name %></td>
+        <td>
+          <ul class="details">
+            <li>ID: <b><%= category.db %></b></li>
+            <li>Count: <b><%= total %></b> (cache: <b><%= category.count %>)</b></li>
+            <li>Created At: <b><%= compact_time(category.created_at) %></b></li>
+            <li>Updated At: <b><%= compact_time(category.updated_at) %></b></li>
+            <li>SFW: <b><%= category.sfw ? "Yes" : "No" %></b></li>
+            <li>
+              Sources:
+              <ul>
+                <% category.sources.each do |source| %>
+                  <li><%= source[:name] %>: <b><%= source[:count] %></b>
+                    <%# no point in having a percentage since they won't add up to 100 %>
+                    <%# ((source[:count].to_f / total.to_f) * 100).round(0) %>
+                  </li>
+                <% end %>
+              </ul>
+            </li>
+          </ul>
+        </td>
+        <td><%= link_to "Manage", yiff_rest_api_v2_manage_images_path(search: { category: category.db }) %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>

config/default_config.rb

@@ -107,14 +107,6 @@ module Websites
     def e621_thumbnails_storage_zone_name
     end
 
-    def e621_thumbnails_base_url
-      "https://thumbs.yiff.media"
-    end
-
-    def e621_thumbnails_base_path
-      "/data/e621-thumbnails"
-    end
-
     def e621_thumbnails_webhook
       # Requests::DiscordWebhook.new(id: "", token: "")
     end
@@ -211,6 +203,53 @@ module Websites
         ].join("; "),
       }
     end
+
+    def http_headers
+      {
+        "User-Agent" => "Websites/4.0.0 (https://github.com/DonovanDMC/Websites; \"donovan_dmc\")",
+      }
+    end
+
+    def httparty_options
+      {
+        timeout:     10,
+        open_timout: 5,
+        headers:     http_headers,
+      }
+    end
+
+    def yiffy2_cdn_url
+      # return "http://yiffy2.local" if Rails.env.development?
+      "https://v2.yiff.media"
+    end
+
+    def yiffy2_bucket_name
+    end
+
+    def yiffy2_access_key_id
+    end
+
+    def yiffy2_secret_access_key
+    end
+
+    def yiffy2_s3_endpoint
+    end
+
+    def e621_thumbnails_storage
+      # Bunny.new(base_url: Websites.config.e621_thumbnails_base_url, access_key: Websites.config.e621_thumbnails_access_key, storage_zone_name: Websites.config.e621_thumbnails_storage_zone_name)
+      StorageManager::Local.new(base_url: "https://thumbs.yiff.media", base_path: "/data/e621-thumbnails")
+    end
+
+    def yiffy2_storage
+      return StorageManager::Local.new(base_url: yiffy2_cdn_url, base_path: "/data/yiffy2") if Rails.env.development?
+      StorageManager::S3.new(
+        endpoint:          yiffy2_s3_endpoint,
+        access_key_id:     yiffy2_access_key_id,
+        secret_access_key: yiffy2_secret_access_key,
+        bucket:            yiffy2_bucket_name,
+        base_url:          yiffy2_cdn_url,
+      )
+    end
   end
 
   class EnvironmentConfiguration

config/initializers/pagy.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "pagy/extras/bootstrap"
+Pagy::DEFAULT[:items] = 20

config/routes/domain_constraint.rb

@@ -11,17 +11,25 @@ class DomainConstraint
   end
 
   def matches?(request)
-    Rails.logger.info("Domain: #{@domain}; Subdomain: #{@subdomain}; Current Host: #{request.domain}; Current Subdomains: #{request.subdomain}; Matches (Domain): #{domain_matches?(request)}; Matches (Subdomain): #{subdomain_matches?(request)}") if Rails.env.development?
+    if Rails.env.development?
+      domain = request.env["websites.dev_domain"].presence || request.domain
+      subdomains = request.env["websites.dev_subdomains"].presence || request.subdomains
+      Rails.logger.info("Domain: #{@domain}; Subdomain: #{@subdomain}; Current Host: #{domain}; Current Subdomains: #{subdomains}; Matches (Domain): #{domain_matches?(request)}; Matches (Subdomain): #{subdomain_matches?(request)}")
+    end
     subdomain_matches?(request) && domain_matches?(request)
   end
 
   private
 
   def domain_matches?(request)
-    request.domain == @domain
+    (Rails.env.development? && request.env["websites.dev_domain"] == @domain) || request.domain == @domain
   end
 
   def subdomain_matches?(request)
+    if Rails.env.development? && !request.env["websites.dev_subdomains"].nil?
+      return true if @subdomain.nil? && request.env["websites.dev_subdomains"].blank?
+      return true if request.env["websites.dev_subdomains"] == @subdomain
+    end
     (@subdomain.nil? && request.subdomain.blank?) || request.subdomain == @subdomain
   end
 end

config/routes/yiff_rest_routes.rb

@@ -47,6 +47,13 @@ module YiffRestRoutes
 
         constraints(DomainConstraint.new(DOMAIN, V2)) do
           namespace(:api_v2, path: "") do
+            resources(:manage, constraints: { id: /[a-z0-9\-.]+/i }, only: %i[index]) do
+              collection do
+                get(:logout)
+                put(:sync)
+                resources(:images, as: "manage_images")
+              end
+            end
             get("/", to: redirect("https://yiff.rest"))
             get(:robots, constraints: { format: "txt" })
             get(:state, to: redirect("https://state.yiff.rest"))
@@ -56,7 +63,7 @@ module YiffRestRoutes
             get("/categories/(*category)", action: :category, as: :api_v2_category, constraints: { category: /[a-z0-9\-.]+/i })
             get("/images/:id", action: :image, as: :api_v2_image, constraints: { id: /[a-f0-9]{32}/ })
             post(:bulk, constraints: { format: "json" }, defaults: { format: :json })
-            get("/(*category)", action: :index, as: :api_v2_images)
+            get("/(*category)", action: :index, as: :api_v2_images, constraints: { category: /(?!rails).*/ })
           end
         end
 
@@ -65,6 +72,7 @@ module YiffRestRoutes
             get(:count_servers)
             get(:flags)
             get(:apikey)
+            get(:manage_images)
             post(:interactions)
           end
 

db/migrate/20240504225048_drop_api_images_created_by.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class DropAPIImagesCreatedBy < ActiveRecord::Migration[7.1]
+  def change
+    remove_column(:api_images, :created_by, :string, null: false)
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2024_01_02_073042) do
+ActiveRecord::Schema[7.1].define(version: 2024_05_04_225048) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -51,7 +51,6 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_02_073042) do
     t.integer "height", null: false
     t.string "mime_type", null: false
     t.string "category", null: false
-    t.string "created_by", null: false
     t.string "original_url"
     t.string "file_ext", null: false
     t.integer "file_size", null: false

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - .:/app
       - e621_thumbnail_data:/data/e621-thumbnails
       - oceanic_docs_data:/data/oceanic-docs
+      - /var/www/yiffy2:/data/yiffy2
     tmpfs:
       - /app/tmp/pids
     environment:

lib/middleware/dev_host.rb

@@ -1,14 +1,21 @@
+# frozen_string_literal: true
+
 module Middleware
   class DevHost
+    DEFAULT_HOST = "v2.yiff.rest"
     def initialize(app)
       @app = app
     end
 
     def call(env)
       request = Rack::Request.new(env)
-      domain = request.params["domain"]
+      domain = request.params["domain"].presence || DEFAULT_HOST
 
-      env["HTTP_HOST"] = domain if Rails.env.development? && domain
+      if Rails.env.development? && domain
+        env["websites.dev_host"] = domain
+        env["websites.dev_domain"] = domain.split(".").last(2).join(".")
+        env["websites.dev_subdomains"] = domain.split(".")[0..-3].join(".")
+      end
 
       @app.call(env)
     end

package.json

@@ -3,7 +3,6 @@
   "packageManager": "yarn@1.22.19",
   "dependencies": {
     "@hotwired/stimulus": "^3.2.2",
-    "@hotwired/turbo-rails": "^7.3.0",
     "@popperjs/core": "^2.11.8",
     "@rails/ujs": "^7.1.2",
     "bootstrap": "^5.3.2",

yarn.lock

@@ -192,19 +192,6 @@
   resolved "https://registry.yarnpkg.com/@hotwired/stimulus/-/stimulus-3.2.2.tgz#071aab59c600fed95b97939e605ff261a4251608"
   integrity sha512-eGeIqNOQpXoPAIP7tC1+1Yc1yl1xnwYqg+3mzqxyrbE5pg5YFBZcA6YoTiByJB6DKAEsiWtl6tjTJS4IYtbB7A==
 
-"@hotwired/turbo-rails@^7.3.0":
-  version "7.3.0"
-  resolved "https://registry.yarnpkg.com/@hotwired/turbo-rails/-/turbo-rails-7.3.0.tgz#422c21752509f3edcd6c7b2725bbe9e157815f51"
-  integrity sha512-fvhO64vp/a2UVQ3jue9WTc2JisMv9XilIC7ViZmXAREVwiQ2S4UC7Go8f9A1j4Xu7DBI6SbFdqILk5ImqVoqyA==
-  dependencies:
-    "@hotwired/turbo" "^7.3.0"
-    "@rails/actioncable" "^7.0"
-
-"@hotwired/turbo@^7.3.0":
-  version "7.3.0"
-  resolved "https://registry.yarnpkg.com/@hotwired/turbo/-/turbo-7.3.0.tgz#2226000fff1aabda9fd9587474565c9929dbf15d"
-  integrity sha512-Dcu+NaSvHLT7EjrDrkEmH4qET2ZJZ5IcCWmNXxNQTBwlnE5tBZfN6WxZ842n5cHV52DH/AKNirbPBtcEXDLW4g==
-
 "@humanwhocodes/config-array@^0.11.13":
   version "0.11.13"
   resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297"
@@ -268,11 +255,6 @@
   resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.8.tgz#6b79032e760a0899cd4204710beede972a3a185f"
   integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==
 
-"@rails/actioncable@^7.0":
-  version "7.1.1"
-  resolved "https://registry.yarnpkg.com/@rails/actioncable/-/actioncable-7.1.1.tgz#e8c49769d41f35a4473133c259cc98adc04dddf8"
-  integrity sha512-ZRJ9rdwFQQjRbtgJnweY0/4UQyxN6ojEGRdib0JkjnuIciv+4ok/aAeZmBJqNreTMaBqS0eHyA9hCArwN58opg==
-
 "@rails/ujs@^7.1.2":
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/@rails/ujs/-/ujs-7.1.2.tgz#ea903bcc0224e17156015d995b6f1b83e27d64b2"