diff --git a/app/controllers/yiff_rest/api_v2/images_controller.rb b/app/controllers/yiff_rest/api_v2/images_controller.rb index cccd787..60c05e9 100644 --- a/app/controllers/yiff_rest/api_v2/images_controller.rb +++ b/app/controllers/yiff_rest/api_v2/images_controller.rb @@ -6,10 +6,10 @@ module YiffRest before_action :validate_discord before_action :prepare_user before_action :admin_only + before_action :load_image, only: %i[edit update destroy delete_with_reason toggle_viewable convert update_cache] def index @sp = search_images_params - @sp[:id] ||= @sp[:md5] @pagy, @images = pagy(APIImage.search(@sp).order(created_at: :desc), size: [1, 2, 2, 1]) end @@ -57,6 +57,26 @@ module YiffRest @image = APIImage.find(params[:id]) end + def toggle_viewable + @image.update(is_viewable: !@image.is_viewable?) + redirect_back(fallback_location: yiff_rest_api_v2_manage_images_path, notice: "Image is #{@image.is_viewable? ? 'now' : 'no longer'} visible.") + end + + def convert + return render_expected_error(400, "Image is already external") if @image.is_external? + return if request.get? + + url = params.dig(:api_image, :external_url) + @image.convert_to_external(url) + redirect_to(yiff_rest_api_v2_manage_images_path(search: { id: @image.id }), notice: "Image concerted") + end + + def update_cache + return render_expected_error(400, "Image is not external") unless @image.is_external? + @image.external_api_image.update_cache + redirect_back(fallback_location: yiff_rest_api_v2_manage_images_path, notice: "Cache will be updated soon") + end + def iqdb @sp = {} end @@ -84,12 +104,16 @@ module YiffRest raise(ActiveRecord::RecordNotFound) if @category.blank? end + def load_image + @image = APIImage.find(params[:id]) + end + def site_title "YiffyAPI V2 - Manage Images" end def search_images_params - permit_search_params(%i[category md5 original_url artist]) + permit_search_params(%i[category md5 original_url artist viewable external]) end def search_iqdb_params diff --git a/app/controllers/yiff_rest/api_v2_controller.rb b/app/controllers/yiff_rest/api_v2_controller.rb index 45b6cef..3677be1 100644 --- a/app/controllers/yiff_rest/api_v2_controller.rb +++ b/app/controllers/yiff_rest/api_v2_controller.rb @@ -83,7 +83,7 @@ module YiffRest end def image - image = APIImage.find_by(id: params[:id].gsub("-", "")) + image = APIImage.search({ md5: params[:id].gsub("-", "") }).first return render_error(YiffyAPIErrorCodes::IMAGES_NOT_FOUND, error: "Image not found.") unless image render(json: { success: true, diff --git a/app/javascript/styles/yiff.rest/v2.scss b/app/javascript/styles/yiff.rest/v2.scss index 3e014bb..429ea33 100644 --- a/app/javascript/styles/yiff.rest/v2.scss +++ b/app/javascript/styles/yiff.rest/v2.scss @@ -23,4 +23,12 @@ body.s-v2-yiff-rest { } } } + + .red-text { + color: #fe6a64; + } + + .green-text { + color: #4caf50; + } } diff --git a/app/jobs/update_external_cache_job.rb b/app/jobs/update_external_cache_job.rb new file mode 100644 index 0000000..a55f062 --- /dev/null +++ b/app/jobs/update_external_cache_job.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class UpdateExternalCacheJob < ApplicationJob + def perform(image) + image.update_cache! + end +end diff --git a/app/logical/iqdb.rb b/app/logical/iqdb.rb index 6047eaa..6fbd57d 100644 --- a/app/logical/iqdb.rb +++ b/app/logical/iqdb.rb @@ -22,18 +22,31 @@ module Iqdb raise(Error, "This service is temporarily unavailable. Please try again later.") end - def update_image(image) - Websites.config.yiffy2_storage.get(image.path) do |file| - Tempfile.open("yiffy2-#{image.md5}") do |tempfile| - file.binmode - tempfile.binmode - tempfile.write(file.read) + def get_image(image, &) + Tempfile.open("yiffy2-#{image.md5}") do |tempfile| + tempfile.binmode + if image.is_external? && image.is_viewable? + HTTParty.get(image.external_api_image.file_url, Websites.config.httparty_options.merge(stream_body: true)) do |fragment| + tempfile.write(fragment) + end tempfile.rewind - thumb = generate_thumbnail(tempfile.path) - raise(Error, "failed to generate thumb for #{image.iqdb_id}") unless thumb - response = make_request("/images/#{image.iqdb_id}", :post, get_channels_data(thumb)) - raise(Error, "iqdb request failed") if response.status != 200 + else + Websites.config.yiffy2_storage.get(image.path) do |file| + file.binmode + tempfile.write(file.read) + tempfile.rewind + end end + yield(tempfile) + end + end + + def update_image(image) + get_image(image) do |tempfile| + thumb = generate_thumbnail(tempfile.path) + raise(Error, "failed to generate thumb for #{image.iqdb_id}") unless thumb + response = make_request("/images/#{image.iqdb_id}", :post, get_channels_data(thumb)) + raise(Error, "iqdb request failed") if response.status != 200 end end @@ -48,8 +61,8 @@ module Iqdb end def query_image(image, score_cutoff) - Websites.config.yiffy2_storage.get(image.path) do |file| - query_file(file, score_cutoff) + get_image(image) do |tempfile| + query_file(tempfile, score_cutoff) end end diff --git a/app/logical/requests/femboy_fans.rb b/app/logical/requests/femboy_fans.rb new file mode 100644 index 0000000..6142b3a --- /dev/null +++ b/app/logical/requests/femboy_fans.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Requests + class FemboyFans + include HTTParty + base_uri "https://femboy.fan" + + def options + opt = Websites.config.httparty_options + opt[:headers]["Authorization"] = "Basic #{Base64.strict_encode64("#{Websites.config.femboyfans_username}:#{Websites.config.femboyfans_apikey}")}" if Websites.config.femboyfans_username.present? && Websites.config.femboyfans_apikey.present? + opt + end + + def get_post(id: nil, md5: nil) + raise(ArgumentError, "id or md5 must be given") if id.nil? && md5.nil? + path = "/" + path = "/posts/#{id}.json" if id + path = "/posts.json?md5=#{md5}" if md5 + r = self.class.get(path, options) + return nil if r.code != 200 + JSON.parse(r.body) + end + + def get_posts_by_tags(tags:, limit: 100, page: 1) + path = "/posts.json?tags=#{tags}&limit=#{limit}&page=#{page}" + r = self.class.get(path, options) + JSON.parse(r.body) || [] + end + + def get_all_posts_by_tags(tags:) + posts = [] + page = 1 + loop do + posts += p = get_posts_by_tags(tags: tags, limit: 320, page: page) + break if p.length < 320 + page += 1 + end + posts + end + + def get_posts(ids: nil, md5s: nil, status: nil) + raise(ArgumentError, "ids or md5s must be given") if ids.nil? && md5s.nil? + path = "/posts.json?" + path += "tags=id:#{ids.join(',')}%20limit:100" if ids + path += "tags=md5:#{md5s.join(',')}%20limit:100" if md5s + path += "%20status:#{status}" if path.include?("tags=") && !status.nil? + path += "tags=status:#{status}" if !path.include?("tags=") && !status.nil? + Rails.logger.info("Fetching posts from femboyfans: #{path}") if Rails.env.development? + r = self.class.get(path, options) + JSON.parse(r.body) || [] + end + + def get_all_posts(ids: [], md5s: []) + posts = [] + ids.each_slice(100) do |slice| + posts += get_posts(ids: slice, status: "any") + end + md5s -= posts.map { |p| p["file"]["md5"] } + md5s.each_slice(100) do |slice| + posts += get_posts(md5s: slice, status: "any") + end + missing = { + ids: ids - posts.pluck("id"), + md5s: md5s - posts.map { |p| p["file"]["md5"] }, + } + + { posts: posts, missing: missing } + end + + def find_replacement(md5:) + r = self.class.get("/posts/replacements.json?search[md5]=#{md5}", options) + JSON.parse(r.body)&.first + end + + def name_to_id(name) + Cache.fetch("name_to_id:#{name.downcase}", expires_in: 1.day) do + r = self.class.get("/users.json?search[name_matches]=#{name}", options) + JSON.parse(r.body)&.first&.dig("id") + end + end + + def self.status + new.status + end + + def self.get_post(...) + new.get_post(...) + end + + def self.get_posts(...) + new.get_posts(...) + end + + def self.get_all_posts(...) + new.get_all_posts(...) + end + + def self.get_posts_by_tags(...) + new.get_posts_by_tags(...) + end + + def self.get_all_posts_by_tags(...) + new.get_all_posts_by_tags(...) + end + + def self.find_replacement(...) + new.find_replacement(...) + end + + def self.name_to_id(...) + new.name_to_id(...) + end + end +end diff --git a/app/models/api_image.rb b/app/models/api_image.rb index 7147136..95f99f2 100644 --- a/app/models/api_image.rb +++ b/app/models/api_image.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class APIImage < ApplicationRecord + class Error < StandardError; end belongs_to_creator attr_accessor :file, :exception, :deletion_reason, :no_webhook_messages @@ -25,19 +26,35 @@ class APIImage < ApplicationRecord after_destroy :remove_iqdb after_destroy :send_deleted, unless: :no_webhook_messages + has_one :external_api_image, dependent: :destroy + scope :viewable, -> { where(is_viewable: true) } + scope :unviewable, -> { where(is_viewable: false) } + scope :external, -> { joins(:external_api_image).where.not("external_api_images.id": nil) } + scope :internal, -> { left_outer_joins(:external_api_image).where("external_api_images.id": nil) } + + def is_external? + external_api_image.present? + end + def delete_files + return if is_external? + delete_files! + end + + def delete_files! + invalidate_cache Websites.config.yiffy2_storage.delete(path) Websites.config.yiffy2_backup_storage.delete(path) - invalidate_cache end def update_files + invalidate_cache + return if is_external? file = Websites.config.yiffy2_storage.get(path_before_last_save) Websites.config.yiffy2_storage.put(path, file) Websites.config.yiffy2_storage.delete(path_before_last_save) Websites.config.yiffy2_backup_storage.put(path, file) Websites.config.yiffy2_backup_storage.delete(path_before_last_save) - invalidate_cache end def file_header_info(file_path) @@ -64,7 +81,7 @@ class APIImage < ApplicationRecord module SearchMethods def random(category, limit, size_limit = nil) - q = where(category: category) + q = viewable.where(category: category) q = q.where(file_size: ..size_limit) if size_limit q.ids.sample(limit).map(&method(:find)) end @@ -85,7 +102,15 @@ class APIImage < ApplicationRecord q = super q = q.attribute_matches(:category, params[:category]) q = q.attribute_matches(:original_url, params[:original_url]) + q = q.attribute_matches(:is_viewable, params[:viewable]) q = q.artists_like(params[:artist]) if params[:artist].present? + if params[:md5].present? # rubocop:disable Style/IfUnlessModifier + q = q.attribute_matches(:id, params[:md5]).or(q.where(id: ExternalAPIImage.where("cached_data->'file'->>'md5' = ?", params[:md5]).select("api_image_id"))) + end + if params[:external].present? + q = q.external if params[:external].truthy? + q = q.internal if params[:external].falsy? + end q.order(created_at: :desc) end end @@ -134,7 +159,29 @@ class APIImage < ApplicationRecord self.artists = str.split("\n").map(&:strip) end + def format_external(*) + img = external_api_image + { + artists: img.artists, + sources: img.sources, + width: img.width, + height: img.height, + url: img.file_url, + type: Marcel::EXTENSIONS[img.ext], + name: "#{img.md5}.#{img.ext}", + id: md5, + md5: img.md5, + ext: img.ext, + size: img.size, + reportURL: img.report_url, + shortURL: short_url, + external: true, + viewable: is_viewable?, + } + end + def serializable_hash(*) + return format_external(*) if is_external? { artists: artists, sources: sources, @@ -144,13 +191,44 @@ class APIImage < ApplicationRecord type: mime_type, name: "#{md5}.#{file_ext}", id: md5, + md5: md5, ext: file_ext, size: file_size, reportURL: nil, shortURL: short_url, + external: false, + viewable: is_viewable?, } end + def convert_to_external(url) + transaction do + uri = URI.parse(url) + case uri.host + when "femboy.fan" + type = "femboyfans" + if %r{^/posts/(\d+)} =~ uri.path + external_id = $1.to_i + else + raise(Error, "Failed to parse external id from #{url}") + end + when "e621.net" + type = "e621" + if %r{^/posts/(\d+)} =~ uri.path + external_id = $1.to_i + else + raise(Error, "Failed to parse external id from #{url}") + end + else + raise(Error, "Attempted to convert to unknown external: #{url}") + end + ExternalAPIImage.create!(api_image: self, external_id: external_id, site: type) + update_columns(artists: [], sources: [], width: 0, height: 0, mime_type: "external", file_ext: "external", file_size: 0) + delete_files! + send_converted + end + end + def self.categories sfw = %w[animals.birb animals.blep animals.dikdik furry.boop furry.cuddle furry.flop furry.fursuit furry.hold furry.howl furry.hug furry.kiss furry.lick furry.propose] nsfw = %w[furry.butts furry.bulge furry.yiff.andromorph furry.yiff.gay furry.yiff.gynomorph furry.yiff.lesbian furry.yiff.straight] @@ -231,6 +309,7 @@ class APIImage < ApplicationRecord RED_TICK = "<:RedTick:1235058898549870724>" def execute(content) + return unless Rails.env.production? Websites.config.yiffyapi_image_logs_webhook.execute({ embeds: [content], }) @@ -283,6 +362,7 @@ class APIImage < ApplicationRecord changes = [] check_change(:artists, changes) check_change(:category, changes) + check_change(:is_viewable, changes) if sources != sources_before_last_save diff = [] @@ -311,6 +391,19 @@ class APIImage < ApplicationRecord }) end + def send_converted + execute({ + title: "Image Converted", + description: <<~DESC, + ID: `#{md5}` + Blame: #{blame} + Converted to external image: #{external_api_image.url} + DESC + color: YELLOW, + timestamp: Time.now.iso8601, + }) + end + def blame if CurrentUser.user.present? "<@#{CurrentUser.id}> (`#{CurrentUser.name}`)" @@ -327,20 +420,28 @@ class APIImage < ApplicationRecord include WebhookMethods def is_image? - %w[png jpg].include?(file_ext) + ext = file_ext + ext = external_api_image.ext if is_external? + %w[png jpg].include?(ext) end def is_gif? - file_ext == "gif" + ext = file_ext + ext = external_api_image.ext if is_external? + ext == "gif" + end + + def is_available? + !is_external? || !external_api_image.is_deleted? end def update_iqdb - return unless is_image? + return unless is_available? && is_image? IqdbUpdateJob.perform_later(id) end def update_iqdb! - return unless is_image? + return unless is_available? && is_image? IqdbUpdateJob.perform_now(id) end diff --git a/app/models/external_api_image.rb b/app/models/external_api_image.rb new file mode 100644 index 0000000..d2fda1d --- /dev/null +++ b/app/models/external_api_image.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +class ExternalAPIImage < ApplicationRecord + class Error < StandardError; end + CACHE_TIME = 2.days + IGNORED_ARTISTS = %w[avoid_posting conditional_dnp sound_warning epilepsy_warning].freeze + + belongs_to :api_image + + validates :site, inclusion: { in: %w[e621 femboyfans] } + + enum site: { + "e621" => "e621", + "femboyfans" => "femboyfans", + } + + module DataMethods + def url + case site + when "e621" + "https://e621.net/posts/#{external_id}" + when "femboyfans" + "https://femboy.fan/posts/#{external_id}" + else + data_error("format url") + end + end + + def file_url + return nil if is_deleted? + case site + when "e621", "femboyfans" + cache.dig("file", "url") + else + data_error("format file_url") + end + end + + def width + case site + when "e621", "femboyfans" + cache.dig("file", "width") + else + data_error("format width") + end + end + + def height + case site + when "e621", "femboyfans" + cache.dig("file", "height") + else + data_error("format height") + end + end + + def size + case site + when "e621", "femboyfans" + cache.dig("file", "size") + else + data_error("format size") + end + end + + def ext + case site + when "e621", "femboyfans" + cache.dig("file", "ext") + else + data_error("format ext") + end + end + + def md5 + case site + when "e621", "femboyfans" + cache.dig("file", "md5") + else + data_error("format md5") + end + end + + def is_deleted? + case site + when "e621", "femboyfans" + cache.dig("flags", "deleted") + else + false + end + end + + def artists + case site + when "e621", "femboyfans" + cache.dig("tags", "artist") - IGNORED_ARTISTS + else + data_error("format artists") + end + end + + def sources + case site + when "e621", "femboyfans" + [url] + cache["sources"].select { |s| s.start_with?("http") } + else + data_error("format sources") + end + end + + def report_url + case site + when "e621", "femboyfans" + url + end + end + + def data_error(text) + raise(Error, "Don't know how to #{text} for ExternalApiImage #{id}: #{site}") + end + end + + module CacheMethods + def cache + update_cache! if cached_data.blank? + update_cache if cache_expired? + cached_data + end + + def cache_expired? + cache_last_updated_at < CACHE_TIME.ago + end + + def update_cache + UpdateExternalCacheJob.perform_later(self) + end + + def update_cache! + case site + when "e621" + data = Requests::E621.get_post(id: external_id) + when "femboyfans" + data = Requests::FemboyFans.get_post(id: external_id) + else + data_error("update cache") + end + update(cached_data: data, cache_last_updated_at: Time.now) + if is_deleted? + api_image.remove_iqdb + else + api_image.update_iqdb + end + api_image.update(is_viewable: !is_deleted?) + end + end + + include CacheMethods + include DataMethods +end diff --git a/app/views/yiff_rest/api_v2/images/_external_image.html.erb b/app/views/yiff_rest/api_v2/images/_external_image.html.erb new file mode 100644 index 0000000..ee8ee47 --- /dev/null +++ b/app/views/yiff_rest/api_v2/images/_external_image.html.erb @@ -0,0 +1,46 @@ +<%# locals: (image:, external:) -%> + +