495 lines
14 KiB
Ruby
495 lines
14 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
class APIImage < ApplicationRecord
|
|
class Error < StandardError; end
|
|
belongs_to_creator
|
|
|
|
attr_accessor :file, :exception, :deletion_reason, :no_webhook_messages
|
|
|
|
validates :category, presence: true, inclusion: { in: -> { APIImage.categories.map(&:db) } }
|
|
validates :id, uniqueness: true, on: :file
|
|
validate on: :file do |image|
|
|
ext, mime = file_header_info(file.path)
|
|
image.errors.add(:file, "type is invalid (#{mime})") if ext == mime
|
|
end
|
|
validate do |image|
|
|
image.errors.add(:base, exception.message) if exception
|
|
end
|
|
|
|
after_create :invalidate_cache
|
|
after_create :send_created, unless: :no_webhook_messages
|
|
after_create :initialize_short_url
|
|
after_create :update_iqdb
|
|
after_update :update_files, if: :saved_change_to_category?
|
|
after_update :send_updated, unless: :no_webhook_messages
|
|
after_destroy :delete_files
|
|
after_destroy :remove_iqdb
|
|
after_destroy :send_deleted, unless: :no_webhook_messages
|
|
|
|
has_one :external_api_image, dependent: :destroy
|
|
scope :viewable, -> { where(is_viewable: true) }
|
|
scope :unviewable, -> { where(is_viewable: false) }
|
|
scope :external, -> { joins(:external_api_image).where.not("external_api_images.id": nil) }
|
|
scope :internal, -> { left_outer_joins(:external_api_image).where("external_api_images.id": nil) }
|
|
|
|
def is_external?
|
|
external_api_image.present?
|
|
end
|
|
|
|
def delete_files
|
|
return if is_external?
|
|
delete_files!
|
|
end
|
|
|
|
def delete_files!
|
|
invalidate_cache
|
|
Websites.config.yiffy2_storage.delete(path)
|
|
Websites.config.yiffy2_backup_storage.delete(path)
|
|
end
|
|
|
|
def update_files
|
|
invalidate_cache
|
|
return if is_external?
|
|
file = Websites.config.yiffy2_storage.get(path_before_last_save)
|
|
Websites.config.yiffy2_storage.put(path, file)
|
|
Websites.config.yiffy2_storage.delete(path_before_last_save)
|
|
Websites.config.yiffy2_backup_storage.put(path, file)
|
|
Websites.config.yiffy2_backup_storage.delete(path_before_last_save)
|
|
end
|
|
|
|
def file_header_info(file_path)
|
|
File.open(file_path) do |bin|
|
|
mime_type = Marcel::MimeType.for(bin)
|
|
ext = case mime_type
|
|
when "image/jpeg"
|
|
"jpg"
|
|
when "image/gif"
|
|
"gif"
|
|
when "image/png"
|
|
"png"
|
|
else
|
|
mime_type
|
|
end
|
|
[mime_type, ext]
|
|
end
|
|
end
|
|
|
|
def invalidate_cache
|
|
Cache.redis.del("yiffy2:images:#{category_before_last_save}")
|
|
Cache.redis.del("yiffy2:images:#{category}")
|
|
end
|
|
|
|
module SearchMethods
|
|
def random(category, limit, size_limit = nil)
|
|
q = viewable.where(category: category)
|
|
q = q.where(file_size: ..size_limit) if size_limit
|
|
q.ids.sample(limit).map(&method(:find))
|
|
end
|
|
|
|
def bulk(category_map, size_limit = nil)
|
|
data = {}
|
|
category_map.each do |category, limit|
|
|
data[category] = random(category, limit, size_limit)
|
|
end
|
|
data
|
|
end
|
|
|
|
def artists_like(name)
|
|
where(id: APIImage.from("unnest(artists) AS artist").where("artist LIKE ?", name.to_escaped_for_sql_like))
|
|
end
|
|
|
|
def external_artists_like(name)
|
|
where(id: ExternalAPIImage.from("external_api_images, jsonb_array_elements_text(cached_data->'tags'->'artist') AS artist").where("artist LIKE ?", name.to_escaped_for_sql_like).select("api_image_id"))
|
|
end
|
|
|
|
def search(params)
|
|
q = super
|
|
q = q.attribute_matches(:category, params[:category])
|
|
q = q.attribute_matches(:original_url, params[:original_url])
|
|
q = q.attribute_matches(:is_viewable, params[:viewable])
|
|
if params[:artist].present? # rubocop:disable Style/IfUnlessModifier
|
|
q = q.artists_like(params[:artist]).or(external_artists_like(params[:artist]))
|
|
end
|
|
if params[:md5].present? # rubocop:disable Style/IfUnlessModifier
|
|
q = q.attribute_matches(:id, params[:md5]).or(q.where(id: ExternalAPIImage.where("cached_data->'file'->>'md5' = ?", params[:md5]).select("api_image_id")))
|
|
end
|
|
if params[:external].present?
|
|
q = q.external if params[:external].truthy?
|
|
q = q.internal if params[:external].falsy?
|
|
end
|
|
q.order(created_at: :desc)
|
|
end
|
|
end
|
|
|
|
extend SearchMethods
|
|
|
|
def md5
|
|
id.gsub("-", "")
|
|
end
|
|
|
|
alias stripped_md5 md5
|
|
|
|
def path_before_last_save
|
|
"/#{category_before_last_save.tr('.', '/')}/#{md5}.#{file_ext}"
|
|
end
|
|
|
|
def path
|
|
"/#{category.tr('.', '/')}/#{md5}.#{file_ext}"
|
|
end
|
|
|
|
def url
|
|
Websites.config.yiffy2_storage.url_for(self)
|
|
end
|
|
|
|
def short_url
|
|
ShortUrl.override(md5, url).shorturl
|
|
end
|
|
|
|
def initialize_short_url
|
|
ShortUrl.override(md5, url)
|
|
end
|
|
|
|
def sources_string
|
|
sources.join("\n")
|
|
end
|
|
|
|
def sources_string=(str)
|
|
self.sources = str.split("\n").map(&:strip)
|
|
end
|
|
|
|
def artists_string
|
|
artists.join("\n")
|
|
end
|
|
|
|
def artists_string=(str)
|
|
self.artists = str.split("\n").map(&:strip)
|
|
end
|
|
|
|
def format_external(*)
|
|
img = external_api_image
|
|
{
|
|
artists: img.artists,
|
|
sources: img.sources,
|
|
width: img.width,
|
|
height: img.height,
|
|
url: img.file_url,
|
|
type: Marcel::EXTENSIONS[img.ext],
|
|
name: "#{img.md5}.#{img.ext}",
|
|
id: md5,
|
|
md5: img.md5,
|
|
ext: img.ext,
|
|
size: img.size,
|
|
reportURL: img.report_url,
|
|
shortURL: short_url,
|
|
external: true,
|
|
viewable: is_viewable?,
|
|
seen: seen,
|
|
}
|
|
end
|
|
|
|
def serializable_hash(*)
|
|
return format_external(*) if is_external?
|
|
{
|
|
artists: artists,
|
|
sources: sources,
|
|
width: width,
|
|
height: height,
|
|
url: url,
|
|
type: mime_type,
|
|
name: "#{md5}.#{file_ext}",
|
|
id: md5,
|
|
md5: md5,
|
|
ext: file_ext,
|
|
size: file_size,
|
|
reportURL: nil,
|
|
shortURL: short_url,
|
|
external: false,
|
|
viewable: is_viewable?,
|
|
seen: seen,
|
|
}
|
|
end
|
|
|
|
def convert_to_external(url)
|
|
transaction do
|
|
uri = URI.parse(url)
|
|
case uri.host
|
|
when "femboy.fan"
|
|
type = "femboyfans"
|
|
if %r{^/posts/(\d+)} =~ uri.path
|
|
external_id = $1.to_i
|
|
else
|
|
raise(Error, "Failed to parse external id from #{url}")
|
|
end
|
|
when "e621.net"
|
|
type = "e621"
|
|
if %r{^/posts/(\d+)} =~ uri.path
|
|
external_id = $1.to_i
|
|
else
|
|
raise(Error, "Failed to parse external id from #{url}")
|
|
end
|
|
else
|
|
raise(Error, "Attempted to convert to unknown external: #{url}")
|
|
end
|
|
ExternalAPIImage.create!(api_image: self, external_id: external_id, site: type)
|
|
delete_files!
|
|
update_columns(artists: [], sources: [], width: 0, height: 0, mime_type: "external", file_ext: "external", file_size: 0)
|
|
end
|
|
send_converted
|
|
reload_external_api_image
|
|
external_api_image
|
|
end
|
|
|
|
def self.categories
|
|
sfw = %w[animals.birb animals.blep animals.dikdik furry.boop furry.cuddle furry.flop furry.fursuit furry.hold furry.howl furry.hug furry.kiss furry.lick furry.propose]
|
|
nsfw = %w[furry.butts furry.bulge furry.yiff.andromorph furry.yiff.gay furry.yiff.gynomorph furry.yiff.lesbian furry.yiff.straight]
|
|
titles = {
|
|
**sfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") },
|
|
"animals.dikdik" => "Animals > Dik Dik",
|
|
**nsfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") },
|
|
}
|
|
|
|
sfw.map { |cat| APICategory.new(titles[cat], cat, true) }
|
|
.concat(nsfw.map { |cat| APICategory.new(titles[cat], cat, false) })
|
|
end
|
|
|
|
def self.category_title(db)
|
|
categories.find { |k| k == db }.try(:db) || db.split(".").map(&:capitalize).join(" > ")
|
|
end
|
|
|
|
def self.cached_count(category)
|
|
count = Cache.redis.get("yiffy2:images:#{category}")
|
|
if count.nil?
|
|
count = APIImage.where(category: category).count
|
|
# images aren't changing so we really don't need any expiry, but I still want an expiry just in case
|
|
Cache.redis.set("yiffy2:images:#{category}", count.to_s, ex: 60 * 60 * 24 * 7)
|
|
end
|
|
count.to_i
|
|
end
|
|
|
|
def self.state
|
|
data = group(:category).count.map do |k, v|
|
|
{
|
|
count: v,
|
|
name: category_title(k),
|
|
category: k,
|
|
state: v < 5 ? "red" : v < 20 ? "yellow" : "green",
|
|
}
|
|
end
|
|
[*data, {
|
|
count: data.pluck(:count).reduce(:+),
|
|
name: "Total",
|
|
category: "total",
|
|
state: "total",
|
|
},]
|
|
end
|
|
|
|
def self.sync_all
|
|
sync_e621
|
|
end
|
|
|
|
# TODO
|
|
def self.sync_e621
|
|
images = joins("CROSS JOIN unnest(sources) AS source").where("source ILIKE ?", "%e621.net%").distinct.limit(300)
|
|
ids = []
|
|
md5s = []
|
|
images.each do |img|
|
|
img.sources.each do |source|
|
|
next unless source.include?("e621.net")
|
|
id = source.split("/").last
|
|
ids << id.to_i if id.to_i.to_s == id
|
|
end
|
|
md5s << img.md5
|
|
end
|
|
Requests::E621.get_all_posts(ids: ids, md5s: md5s) => { posts:, missing: }
|
|
Rails.logger.info("Failed to find #{missing[:ids].length} posts") unless missing[:ids].empty?
|
|
mismatch = images.select { |img| posts.none? { |post| post["file"]["md5"] == img.md5 } }
|
|
Rails.logger.info("Found #{mismatch.length} mismatched images") unless mismatch.empty?
|
|
posts
|
|
end
|
|
|
|
def sync
|
|
APIImageSyncJob.perform_later(self)
|
|
end
|
|
|
|
module WebhookMethods
|
|
GREEN = 0x008000
|
|
YELLOW = 0xFFA500
|
|
RED = 0xFF0000
|
|
GREEN_TICK = "<:GreenTick:1235058920762904576>"
|
|
RED_TICK = "<:RedTick:1235058898549870724>"
|
|
|
|
def execute(content)
|
|
return unless Rails.env.production?
|
|
Websites.config.yiffyapi_image_logs_webhook.execute({
|
|
embeds: [content],
|
|
})
|
|
end
|
|
|
|
def send_created
|
|
execute({
|
|
title: "Image Uploaded",
|
|
description: <<~DESC,
|
|
ID: `#{md5}`
|
|
Category: `#{category}`
|
|
Artists: `#{artists.join(', ').presence || '[NONE]'}`
|
|
Sources:
|
|
#{sources.map { |s| "* #{s}" }.join("\n").presence || '[NONE]'}
|
|
Upload Type: #{original_url.present? ? "[url](#{original_url})" : 'file'}
|
|
Uploaded By: <@#{creator.id}> (`#{creator.name}`)
|
|
Created At: <t:#{created_at.to_i}>
|
|
Updated At: <t:#{updated_at.to_i}>
|
|
DESC
|
|
color: GREEN,
|
|
timestamp: Time.now.iso8601,
|
|
})
|
|
end
|
|
|
|
def send_deleted
|
|
execute({
|
|
title: "Image Deleted",
|
|
description: <<~DESC,
|
|
ID: `#{md5}`
|
|
Category: `#{category}`
|
|
Artists: `#{artists.join(', ').presence || '[NONE]'}}`
|
|
Sources:
|
|
#{sources.map { |s| "* #{s}" }.join("\n").presence || '[NONE]'}}
|
|
Upload Type: #{original_url.present? ? "[url](#{original_url})" : 'file'}
|
|
Uploaded By: <@#{creator.id}> (`#{creator.name}`)
|
|
Reason: #{deletion_reason || 'None Provided'}
|
|
Created At: <t:#{created_at.to_i}>
|
|
Updated At: <t:#{updated_at.to_i}>
|
|
DESC
|
|
color: RED,
|
|
timestamp: Time.now.iso8601,
|
|
})
|
|
end
|
|
|
|
def check_change(attr, changes)
|
|
changes << "#{attr.to_s.titleize}: **#{send("#{attr}_before_last_save")}** -> **#{send(attr)}**" if send("saved_change_to_#{attr}?")
|
|
end
|
|
|
|
def send_updated
|
|
changes = []
|
|
check_change(:artists, changes)
|
|
check_change(:category, changes)
|
|
check_change(:is_viewable, changes)
|
|
|
|
if sources != sources_before_last_save
|
|
diff = []
|
|
sources_before_last_save.each do |source|
|
|
diff << "- #{source}" unless sources.include?(source)
|
|
end
|
|
|
|
sources.each do |source|
|
|
diff << "+ #{source}" unless sources_before_last_save.include?(source)
|
|
end
|
|
|
|
changes << "Sources:\n```diff\n#{diff.join("\n")}\n```"
|
|
end
|
|
|
|
return if changes.empty?
|
|
|
|
changes << "Blame: #{blame}"
|
|
execute({
|
|
title: "Image Updated",
|
|
description: <<~DESC,
|
|
ID: `#{md5}`
|
|
#{changes.join("\n")}
|
|
DESC
|
|
color: YELLOW,
|
|
timestamp: Time.now.iso8601,
|
|
})
|
|
end
|
|
|
|
def send_converted
|
|
execute({
|
|
title: "Image Converted",
|
|
description: <<~DESC,
|
|
ID: `#{md5}`
|
|
Blame: #{blame}
|
|
Converted to external image: #{external_api_image.url}
|
|
DESC
|
|
color: YELLOW,
|
|
timestamp: Time.now.iso8601,
|
|
})
|
|
end
|
|
|
|
def blame
|
|
if CurrentUser.user.present?
|
|
"<@#{CurrentUser.id}> (`#{CurrentUser.name}`)"
|
|
elsif Rails.const_defined?("Console")
|
|
"**Console**"
|
|
elsif CurrentUser.ip_addr
|
|
"**#{CurrentUser.ip_addr}**"
|
|
else
|
|
"**Unknown**"
|
|
end
|
|
end
|
|
end
|
|
|
|
include WebhookMethods
|
|
|
|
def is_image?
|
|
ext = file_ext
|
|
ext = external_api_image.ext if is_external?
|
|
%w[png jpg].include?(ext)
|
|
end
|
|
|
|
def is_gif?
|
|
ext = file_ext
|
|
ext = external_api_image.ext if is_external?
|
|
ext == "gif"
|
|
end
|
|
|
|
def is_available?
|
|
!is_external? || !external_api_image.is_deleted?
|
|
end
|
|
|
|
def update_iqdb
|
|
return unless is_available? && is_image?
|
|
IqdbUpdateJob.perform_later(id)
|
|
end
|
|
|
|
def update_iqdb!
|
|
return unless is_available? && is_image?
|
|
IqdbUpdateJob.perform_now(id)
|
|
end
|
|
|
|
def remove_iqdb
|
|
return unless is_image?
|
|
IqdbRemoveJob.perform_later(iqdb_id)
|
|
end
|
|
|
|
def remove_iqdb!
|
|
return unless is_image?
|
|
IqdbRemoveJob.perform_now(iqdb_id)
|
|
end
|
|
|
|
def seen
|
|
Cache.fetch("img:#{md5}", expires_in: 1.minute) do
|
|
(Cache.redis.get("yiffy2:stats:image:#{md5}").presence || 0).to_i
|
|
end
|
|
end
|
|
|
|
def self.all_seen
|
|
Cache.fetch("img:all_seen", expires_in: 1.minutes) do
|
|
keys = all.map { |img| "yiffy2:stats:image:#{img.md5}" }
|
|
values = Cache.redis.mget(*keys)
|
|
results = {}
|
|
keys.each_with_index do |key, index|
|
|
val = values.at(index)
|
|
next if val.nil?
|
|
results[key[-32..]] = val.to_i
|
|
end
|
|
results
|
|
end
|
|
end
|
|
|
|
def self.create_external!(type, id, category)
|
|
site = ExternalAPIImage::SITES.find { |s| s.value == type }
|
|
raise(Error, "Invalid site: #{type}") if site.blank?
|
|
img = create!(width: 0, height: 0, mime_type: "external", file_ext: "external", file_size: 0, category: category)
|
|
img.convert_to_external(format(site.format, id))
|
|
end
|
|
end
|