# frozen_string_literal: true class APIImage < ApplicationRecord class Error < StandardError; end belongs_to_creator attr_accessor :file, :exception, :deletion_reason, :no_webhook_messages validates :category, presence: true, inclusion: { in: -> { APIImage.categories.map(&:db) } } validates :id, uniqueness: true, on: :file validate on: :file do |image| ext, mime = file_header_info(file.path) image.errors.add(:file, "type is invalid (#{mime})") if ext == mime end validate do |image| image.errors.add(:base, exception.message) if exception end after_create :invalidate_cache after_create :send_created, unless: :no_webhook_messages after_create :initialize_short_url after_create :update_iqdb after_update :update_files, if: :saved_change_to_category? after_update :send_updated, unless: :no_webhook_messages after_destroy :delete_files after_destroy :remove_iqdb after_destroy :send_deleted, unless: :no_webhook_messages has_one :external_api_image, dependent: :destroy scope :viewable, -> { where(is_viewable: true) } scope :unviewable, -> { where(is_viewable: false) } scope :external, -> { joins(:external_api_image).where.not("external_api_images.id": nil) } scope :internal, -> { left_outer_joins(:external_api_image).where("external_api_images.id": nil) } def is_external? external_api_image.present? end def delete_files return if is_external? delete_files! end def delete_files! invalidate_cache Websites.config.yiffy2_storage.delete(path) Websites.config.yiffy2_backup_storage.delete(path) end def update_files invalidate_cache return if is_external? file = Websites.config.yiffy2_storage.get(path_before_last_save) Websites.config.yiffy2_storage.put(path, file) Websites.config.yiffy2_storage.delete(path_before_last_save) Websites.config.yiffy2_backup_storage.put(path, file) Websites.config.yiffy2_backup_storage.delete(path_before_last_save) end def file_header_info(file_path) File.open(file_path) do |bin| mime_type = Marcel::MimeType.for(bin) ext = case mime_type when "image/jpeg" "jpg" when "image/gif" "gif" when "image/png" "png" else mime_type end [mime_type, ext] end end def invalidate_cache Cache.redis.del("yiffy2:images:#{category_before_last_save}") Cache.redis.del("yiffy2:images:#{category}") end module SearchMethods def random(category, limit, size_limit = nil) q = viewable.where(category: category) q = q.where(file_size: ..size_limit) if size_limit q.ids.sample(limit).map(&method(:find)) end def bulk(category_map, size_limit = nil) data = {} category_map.each do |category, limit| data[category] = random(category, limit, size_limit) end data end def artists_like(name) where(id: APIImage.from("unnest(artists) AS artist").where("artist LIKE ?", name.to_escaped_for_sql_like)) end def external_artists_like(name) where(id: ExternalAPIImage.from("external_api_images, jsonb_array_elements_text(cached_data->'tags'->'artist') AS artist").where("artist LIKE ?", name.to_escaped_for_sql_like).select("api_image_id")) end def search(params) q = super q = q.attribute_matches(:category, params[:category]) q = q.attribute_matches(:original_url, params[:original_url]) q = q.attribute_matches(:is_viewable, params[:viewable]) if params[:artist].present? # rubocop:disable Style/IfUnlessModifier q = q.artists_like(params[:artist]).or(external_artists_like(params[:artist])) end if params[:md5].present? # rubocop:disable Style/IfUnlessModifier q = q.attribute_matches(:id, params[:md5]).or(q.where(id: ExternalAPIImage.where("cached_data->'file'->>'md5' = ?", params[:md5]).select("api_image_id"))) end if params[:external].present? q = q.external if params[:external].truthy? q = q.internal if params[:external].falsy? end q.order(created_at: :desc) end end extend SearchMethods def md5 id.gsub("-", "") end alias stripped_md5 md5 def path_before_last_save "/#{category_before_last_save.tr('.', '/')}/#{md5}.#{file_ext}" end def path "/#{category.tr('.', '/')}/#{md5}.#{file_ext}" end def url Websites.config.yiffy2_storage.url_for(self) end def short_url ShortUrl.override(md5, url).shorturl end def initialize_short_url ShortUrl.override(md5, url) end def sources_string sources.join("\n") end def sources_string=(str) self.sources = str.split("\n").map(&:strip) end def artists_string artists.join("\n") end def artists_string=(str) self.artists = str.split("\n").map(&:strip) end def format_external(*) img = external_api_image { artists: img.artists, sources: img.sources, width: img.width, height: img.height, url: img.file_url, type: Marcel::EXTENSIONS[img.ext], name: "#{img.md5}.#{img.ext}", id: md5, md5: img.md5, ext: img.ext, size: img.size, reportURL: img.report_url, shortURL: short_url, external: true, viewable: is_viewable?, seen: seen, } end def serializable_hash(*) return format_external(*) if is_external? { artists: artists, sources: sources, width: width, height: height, url: url, type: mime_type, name: "#{md5}.#{file_ext}", id: md5, md5: md5, ext: file_ext, size: file_size, reportURL: nil, shortURL: short_url, external: false, viewable: is_viewable?, seen: seen, } end def convert_to_external(url) transaction do uri = URI.parse(url) case uri.host when "femboy.fan" type = "femboyfans" if %r{^/posts/(\d+)} =~ uri.path external_id = $1.to_i else raise(Error, "Failed to parse external id from #{url}") end when "e621.net" type = "e621" if %r{^/posts/(\d+)} =~ uri.path external_id = $1.to_i else raise(Error, "Failed to parse external id from #{url}") end else raise(Error, "Attempted to convert to unknown external: #{url}") end ExternalAPIImage.create!(api_image: self, external_id: external_id, site: type) delete_files! update_columns(artists: [], sources: [], width: 0, height: 0, mime_type: "external", file_ext: "external", file_size: 0) end send_converted reload_external_api_image external_api_image end def self.categories sfw = %w[animals.birb animals.blep animals.dikdik furry.boop furry.cuddle furry.flop furry.fursuit furry.hold furry.howl furry.hug furry.kiss furry.lick furry.propose] nsfw = %w[furry.butts furry.bulge furry.yiff.andromorph furry.yiff.gay furry.yiff.gynomorph furry.yiff.lesbian furry.yiff.straight] titles = { **sfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") }, "animals.dikdik" => "Animals > Dik Dik", **nsfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") }, } sfw.map { |cat| APICategory.new(titles[cat], cat, true) } .concat(nsfw.map { |cat| APICategory.new(titles[cat], cat, false) }) end def self.category_title(db) categories.find { |k| k == db }.try(:db) || db.split(".").map(&:capitalize).join(" > ") end def self.cached_count(category) count = Cache.redis.get("yiffy2:images:#{category}") if count.nil? count = APIImage.where(category: category).count # images aren't changing so we really don't need any expiry, but I still want an expiry just in case Cache.redis.set("yiffy2:images:#{category}", count.to_s, ex: 60 * 60 * 24 * 7) end count.to_i end def self.state data = group(:category).count.map do |k, v| { count: v, name: category_title(k), category: k, state: v < 5 ? "red" : v < 20 ? "yellow" : "green", } end [*data, { count: data.pluck(:count).reduce(:+), name: "Total", category: "total", state: "total", },] end def self.sync_all sync_e621 end # TODO def self.sync_e621 images = joins("CROSS JOIN unnest(sources) AS source").where("source ILIKE ?", "%e621.net%").distinct.limit(300) ids = [] md5s = [] images.each do |img| img.sources.each do |source| next unless source.include?("e621.net") id = source.split("/").last ids << id.to_i if id.to_i.to_s == id end md5s << img.md5 end Requests::E621.get_all_posts(ids: ids, md5s: md5s) => { posts:, missing: } Rails.logger.info("Failed to find #{missing[:ids].length} posts") unless missing[:ids].empty? mismatch = images.select { |img| posts.none? { |post| post["file"]["md5"] == img.md5 } } Rails.logger.info("Found #{mismatch.length} mismatched images") unless mismatch.empty? posts end def sync APIImageSyncJob.perform_later(self) end module WebhookMethods GREEN = 0x008000 YELLOW = 0xFFA500 RED = 0xFF0000 GREEN_TICK = "<:GreenTick:1235058920762904576>" RED_TICK = "<:RedTick:1235058898549870724>" def execute(content) return unless Rails.env.production? Websites.config.yiffyapi_image_logs_webhook.execute({ embeds: [content], }) end def send_created execute({ title: "Image Uploaded", description: <<~DESC, ID: `#{md5}` Category: `#{category}` Artists: `#{artists.join(', ').presence || '[NONE]'}` Sources: #{sources.map { |s| "* #{s}" }.join("\n").presence || '[NONE]'} Upload Type: #{original_url.present? ? "[url](#{original_url})" : 'file'} Uploaded By: <@#{creator.id}> (`#{creator.name}`) Created At: Updated At: DESC color: GREEN, timestamp: Time.now.iso8601, }) end def send_deleted execute({ title: "Image Deleted", description: <<~DESC, ID: `#{md5}` Category: `#{category}` Artists: `#{artists.join(', ').presence || '[NONE]'}}` Sources: #{sources.map { |s| "* #{s}" }.join("\n").presence || '[NONE]'}} Upload Type: #{original_url.present? ? "[url](#{original_url})" : 'file'} Uploaded By: <@#{creator.id}> (`#{creator.name}`) Reason: #{deletion_reason || 'None Provided'} Created At: Updated At: DESC color: RED, timestamp: Time.now.iso8601, }) end def check_change(attr, changes) changes << "#{attr.to_s.titleize}: **#{send("#{attr}_before_last_save")}** -> **#{send(attr)}**" if send("saved_change_to_#{attr}?") end def send_updated changes = [] check_change(:artists, changes) check_change(:category, changes) check_change(:is_viewable, changes) if sources != sources_before_last_save diff = [] sources_before_last_save.each do |source| diff << "- #{source}" unless sources.include?(source) end sources.each do |source| diff << "+ #{source}" unless sources_before_last_save.include?(source) end changes << "Sources:\n```diff\n#{diff.join("\n")}\n```" end return if changes.empty? changes << "Blame: #{blame}" execute({ title: "Image Updated", description: <<~DESC, ID: `#{md5}` #{changes.join("\n")} DESC color: YELLOW, timestamp: Time.now.iso8601, }) end def send_converted execute({ title: "Image Converted", description: <<~DESC, ID: `#{md5}` Blame: #{blame} Converted to external image: #{external_api_image.url} DESC color: YELLOW, timestamp: Time.now.iso8601, }) end def blame if CurrentUser.user.present? "<@#{CurrentUser.id}> (`#{CurrentUser.name}`)" elsif Rails.const_defined?("Console") "**Console**" elsif CurrentUser.ip_addr "**#{CurrentUser.ip_addr}**" else "**Unknown**" end end end include WebhookMethods def is_image? ext = file_ext ext = external_api_image.ext if is_external? %w[png jpg].include?(ext) end def is_gif? ext = file_ext ext = external_api_image.ext if is_external? ext == "gif" end def is_available? !is_external? || !external_api_image.is_deleted? end def update_iqdb return unless is_available? && is_image? IqdbUpdateJob.perform_later(id) end def update_iqdb! return unless is_available? && is_image? IqdbUpdateJob.perform_now(id) end def remove_iqdb return unless is_image? IqdbRemoveJob.perform_later(iqdb_id) end def remove_iqdb! return unless is_image? IqdbRemoveJob.perform_now(iqdb_id) end def seen Cache.fetch("img:#{md5}", expires_in: 1.minute) do (Cache.redis.get("yiffy2:stats:image:#{md5}").presence || 0).to_i end end def self.all_seen Cache.fetch("img:all_seen", expires_in: 1.minutes) do keys = all.map { |img| "yiffy2:stats:image:#{img.md5}" } values = Cache.redis.mget(*keys) results = {} keys.each_with_index do |key, index| val = values.at(index) next if val.nil? results[key[-32..]] = val.to_i end results end end def self.create_external!(type, id, category) site = ExternalAPIImage::SITES.find { |s| s.value == type } raise(Error, "Invalid site: #{type}") if site.blank? img = create!(width: 0, height: 0, mime_type: "external", file_ext: "external", file_size: 0, category: category) img.convert_to_external(format(site.format, id)) end end