Websites/app/models/api_image.rb

489 lines
14 KiB
Ruby
Raw Normal View History

2024-05-03 03:04:43 +00:00
# frozen_string_literal: true
class APIImage < ApplicationRecord
class Error < StandardError; end
2024-05-03 03:04:43 +00:00
belongs_to_creator
2024-05-07 06:57:00 +00:00
attr_accessor :file, :exception, :deletion_reason, :no_webhook_messages
validates :category, presence: true, inclusion: { in: -> { APIImage.categories.map(&:db) } }
validates :id, uniqueness: true, on: :file
validate on: :file do |image|
ext, mime = file_header_info(file.path)
image.errors.add(:file, "type is invalid (#{mime})") if ext == mime
end
validate do |image|
image.errors.add(:base, exception.message) if exception
end
after_create :invalidate_cache
2024-05-07 06:57:00 +00:00
after_create :send_created, unless: :no_webhook_messages
after_create :initialize_short_url
2024-10-13 06:04:55 +00:00
after_create :update_iqdb
after_update :update_files, if: :saved_change_to_category?
2024-05-07 06:57:00 +00:00
after_update :send_updated, unless: :no_webhook_messages
after_destroy :delete_files
2024-10-13 06:04:55 +00:00
after_destroy :remove_iqdb
2024-05-07 06:57:00 +00:00
after_destroy :send_deleted, unless: :no_webhook_messages
has_one :external_api_image, dependent: :destroy
scope :viewable, -> { where(is_viewable: true) }
scope :unviewable, -> { where(is_viewable: false) }
scope :external, -> { joins(:external_api_image).where.not("external_api_images.id": nil) }
scope :internal, -> { left_outer_joins(:external_api_image).where("external_api_images.id": nil) }
def is_external?
external_api_image.present?
end
def delete_files
return if is_external?
delete_files!
end
def delete_files!
invalidate_cache
Websites.config.yiffy2_storage.delete(path)
Websites.config.yiffy2_backup_storage.delete(path)
end
def update_files
invalidate_cache
return if is_external?
file = Websites.config.yiffy2_storage.get(path_before_last_save)
Websites.config.yiffy2_storage.put(path, file)
Websites.config.yiffy2_storage.delete(path_before_last_save)
Websites.config.yiffy2_backup_storage.put(path, file)
Websites.config.yiffy2_backup_storage.delete(path_before_last_save)
end
2024-06-11 07:20:34 +00:00
def file_header_info(file_path)
File.open(file_path) do |bin|
mime_type = Marcel::MimeType.for(bin)
ext = case mime_type
when "image/jpeg"
"jpg"
when "image/gif"
"gif"
when "image/png"
"png"
else
mime_type
end
[mime_type, ext]
end
end
def invalidate_cache
Cache.redis.del("yiffy2:images:#{category_before_last_save}")
Cache.redis.del("yiffy2:images:#{category}")
end
2024-05-03 03:04:43 +00:00
module SearchMethods
def random(category, limit, size_limit = nil)
q = viewable.where(category: category)
2024-05-03 03:04:43 +00:00
q = q.where(file_size: ..size_limit) if size_limit
q.ids.sample(limit).map(&method(:find))
end
def bulk(category_map, size_limit = nil)
data = {}
category_map.each do |category, limit|
data[category] = random(category, limit, size_limit)
end
data
end
2024-05-07 06:57:00 +00:00
2024-10-08 12:39:28 +00:00
def artists_like(name)
where(id: APIImage.from("unnest(artists) AS artist").where("artist LIKE ?", name.to_escaped_for_sql_like))
end
2024-05-07 06:57:00 +00:00
def search(params)
q = super
q = q.attribute_matches(:category, params[:category])
q = q.attribute_matches(:original_url, params[:original_url])
q = q.attribute_matches(:is_viewable, params[:viewable])
2024-10-08 12:39:28 +00:00
q = q.artists_like(params[:artist]) if params[:artist].present?
if params[:md5].present? # rubocop:disable Style/IfUnlessModifier
q = q.attribute_matches(:id, params[:md5]).or(q.where(id: ExternalAPIImage.where("cached_data->'file'->>'md5' = ?", params[:md5]).select("api_image_id")))
end
if params[:external].present?
q = q.external if params[:external].truthy?
q = q.internal if params[:external].falsy?
end
2024-05-07 06:57:00 +00:00
q.order(created_at: :desc)
end
2024-05-03 03:04:43 +00:00
end
extend SearchMethods
def md5
id.gsub("-", "")
end
alias stripped_md5 md5
def path_before_last_save
"/#{category_before_last_save.tr('.', '/')}/#{md5}.#{file_ext}"
end
def path
"/#{category.tr('.', '/')}/#{md5}.#{file_ext}"
end
2024-05-03 03:04:43 +00:00
def url
Websites.config.yiffy2_storage.url_for(self)
2024-05-03 03:04:43 +00:00
end
def short_url
ShortUrl.override(md5, url).shorturl
end
def initialize_short_url
ShortUrl.override(md5, url)
end
def sources_string
sources.join("\n")
end
def sources_string=(str)
self.sources = str.split("\n").map(&:strip)
end
def artists_string
artists.join("\n")
end
def artists_string=(str)
self.artists = str.split("\n").map(&:strip)
end
def format_external(*)
img = external_api_image
{
artists: img.artists,
sources: img.sources,
width: img.width,
height: img.height,
url: img.file_url,
type: Marcel::EXTENSIONS[img.ext],
name: "#{img.md5}.#{img.ext}",
id: md5,
md5: img.md5,
ext: img.ext,
size: img.size,
reportURL: img.report_url,
shortURL: short_url,
external: true,
viewable: is_viewable?,
seen: seen,
}
end
2024-05-03 03:04:43 +00:00
def serializable_hash(*)
return format_external(*) if is_external?
2024-05-03 03:04:43 +00:00
{
artists: artists,
sources: sources,
width: width,
height: height,
url: url,
type: mime_type,
name: "#{md5}.#{file_ext}",
id: md5,
md5: md5,
2024-05-03 03:04:43 +00:00
ext: file_ext,
size: file_size,
reportURL: nil,
shortURL: short_url,
external: false,
viewable: is_viewable?,
seen: seen,
2024-05-03 03:04:43 +00:00
}
end
def convert_to_external(url)
transaction do
uri = URI.parse(url)
case uri.host
when "femboy.fan"
type = "femboyfans"
if %r{^/posts/(\d+)} =~ uri.path
external_id = $1.to_i
else
raise(Error, "Failed to parse external id from #{url}")
end
when "e621.net"
type = "e621"
if %r{^/posts/(\d+)} =~ uri.path
external_id = $1.to_i
else
raise(Error, "Failed to parse external id from #{url}")
end
else
raise(Error, "Attempted to convert to unknown external: #{url}")
end
ExternalAPIImage.create!(api_image: self, external_id: external_id, site: type)
delete_files!
2024-10-21 23:09:37 +00:00
update_columns(artists: [], sources: [], width: 0, height: 0, mime_type: "external", file_ext: "external", file_size: 0)
end
2024-10-22 00:16:52 +00:00
send_converted
reload_external_api_image
external_api_image
end
2024-05-03 03:04:43 +00:00
def self.categories
sfw = %w[animals.birb animals.blep animals.dikdik furry.boop furry.cuddle furry.flop furry.fursuit furry.hold furry.howl furry.hug furry.kiss furry.lick furry.propose]
nsfw = %w[furry.butts furry.bulge furry.yiff.andromorph furry.yiff.gay furry.yiff.gynomorph furry.yiff.lesbian furry.yiff.straight]
titles = {
**sfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") },
"animals.dikdik" => "Animals > Dik Dik",
**nsfw.index_with { |c| c.split(".").map(&:capitalize).join(" > ") },
}
sfw.map { |cat| APICategory.new(titles[cat], cat, true) }
.concat(nsfw.map { |cat| APICategory.new(titles[cat], cat, false) })
2024-05-03 03:04:43 +00:00
end
def self.category_title(db)
categories.find { |k| k == db }.try(:db) || db.split(".").map(&:capitalize).join(" > ")
end
def self.cached_count(category)
count = Cache.redis.get("yiffy2:images:#{category}")
if count.nil?
count = APIImage.where(category: category).count
# images aren't changing so we really don't need any expiry, but I still want an expiry just in case
Cache.redis.set("yiffy2:images:#{category}", count.to_s, ex: 60 * 60 * 24 * 7)
end
count.to_i
end
def self.state
data = group(:category).count.map do |k, v|
{
count: v,
name: category_title(k),
category: k,
state: v < 5 ? "red" : v < 20 ? "yellow" : "green",
}
end
[*data, {
count: data.pluck(:count).reduce(:+),
name: "Total",
category: "total",
state: "total",
},]
end
def self.sync_all
sync_e621
end
# TODO
def self.sync_e621
images = joins("CROSS JOIN unnest(sources) AS source").where("source ILIKE ?", "%e621.net%").distinct.limit(300)
ids = []
md5s = []
images.each do |img|
img.sources.each do |source|
next unless source.include?("e621.net")
id = source.split("/").last
ids << id.to_i if id.to_i.to_s == id
end
md5s << img.md5
end
Requests::E621.get_all_posts(ids: ids, md5s: md5s) => { posts:, missing: }
Rails.logger.info("Failed to find #{missing[:ids].length} posts") unless missing[:ids].empty?
mismatch = images.select { |img| posts.none? { |post| post["file"]["md5"] == img.md5 } }
Rails.logger.info("Found #{mismatch.length} mismatched images") unless mismatch.empty?
posts
end
def sync
APIImageSyncJob.perform_later(self)
end
2024-05-07 06:57:00 +00:00
module WebhookMethods
GREEN = 0x008000
YELLOW = 0xFFA500
RED = 0xFF0000
GREEN_TICK = "<:GreenTick:1235058920762904576>"
RED_TICK = "<:RedTick:1235058898549870724>"
def execute(content)
return unless Rails.env.production?
2024-05-07 06:57:00 +00:00
Websites.config.yiffyapi_image_logs_webhook.execute({
embeds: [content],
})
end
def send_created
execute({
title: "Image Uploaded",
description: <<~DESC,
ID: `#{md5}`
2024-05-07 07:54:31 +00:00
Category: `#{category}`
2024-10-22 00:43:38 +00:00
Artists: `#{artists.join(', ').presence || '[NONE]'}`
2024-05-07 06:57:00 +00:00
Sources:
2024-10-22 00:43:38 +00:00
#{sources.map { |s| "* #{s}" }.join("\n").presence || '[NONE]'}
2024-05-07 06:57:00 +00:00
Upload Type: #{original_url.present? ? "[url](#{original_url})" : 'file'}
Uploaded By: <@#{creator.id}> (`#{creator.name}`)
Created At: <t:#{created_at.to_i}>
Updated At: <t:#{updated_at.to_i}>
DESC
color: GREEN,
timestamp: Time.now.iso8601,
})
end
def send_deleted
execute({
title: "Image Deleted",
description: <<~DESC,
ID: `#{md5}`
2024-05-07 07:54:31 +00:00
Category: `#{category}`
2024-10-22 00:43:38 +00:00
Artists: `#{artists.join(', ').presence || '[NONE]'}}`
2024-05-07 06:57:00 +00:00
Sources:
2024-10-22 00:43:38 +00:00
#{sources.map { |s| "* #{s}" }.join("\n").presence || '[NONE]'}}
2024-05-07 06:57:00 +00:00
Upload Type: #{original_url.present? ? "[url](#{original_url})" : 'file'}
Uploaded By: <@#{creator.id}> (`#{creator.name}`)
Reason: #{deletion_reason || 'None Provided'}
Created At: <t:#{created_at.to_i}>
Updated At: <t:#{updated_at.to_i}>
DESC
color: RED,
timestamp: Time.now.iso8601,
})
end
def check_change(attr, changes)
changes << "#{attr.to_s.titleize}: **#{send("#{attr}_before_last_save")}** -> **#{send(attr)}**" if send("saved_change_to_#{attr}?")
end
def send_updated
changes = []
check_change(:artists, changes)
check_change(:category, changes)
check_change(:is_viewable, changes)
2024-05-07 06:57:00 +00:00
if sources != sources_before_last_save
diff = []
sources_before_last_save.each do |source|
diff << "- #{source}" unless sources.include?(source)
end
sources.each do |source|
diff << "+ #{source}" unless sources_before_last_save.include?(source)
end
changes << "Sources:\n```diff\n#{diff.join("\n")}\n```"
end
return if changes.empty?
changes << "Blame: #{blame}"
execute({
title: "Image Updated",
description: <<~DESC,
ID: `#{md5}`
#{changes.join("\n")}
DESC
color: YELLOW,
timestamp: Time.now.iso8601,
})
end
def send_converted
execute({
title: "Image Converted",
description: <<~DESC,
ID: `#{md5}`
Blame: #{blame}
Converted to external image: #{external_api_image.url}
DESC
color: YELLOW,
timestamp: Time.now.iso8601,
})
end
2024-05-07 06:57:00 +00:00
def blame
if CurrentUser.user.present?
"<@#{CurrentUser.id}> (`#{CurrentUser.name}`)"
elsif Rails.const_defined?("Console")
"**Console**"
elsif CurrentUser.ip_addr
"**#{CurrentUser.ip_addr}**"
else
"**Unknown**"
end
end
end
include WebhookMethods
2024-10-13 06:04:55 +00:00
def is_image?
ext = file_ext
ext = external_api_image.ext if is_external?
%w[png jpg].include?(ext)
2024-10-13 06:04:55 +00:00
end
def is_gif?
ext = file_ext
ext = external_api_image.ext if is_external?
ext == "gif"
end
def is_available?
!is_external? || !external_api_image.is_deleted?
2024-10-13 06:04:55 +00:00
end
def update_iqdb
return unless is_available? && is_image?
2024-10-13 06:04:55 +00:00
IqdbUpdateJob.perform_later(id)
end
def update_iqdb!
return unless is_available? && is_image?
2024-10-13 06:04:55 +00:00
IqdbUpdateJob.perform_now(id)
end
def remove_iqdb
return unless is_image?
IqdbRemoveJob.perform_later(iqdb_id)
end
def remove_iqdb!
return unless is_image?
IqdbRemoveJob.perform_now(iqdb_id)
end
2024-10-22 00:16:52 +00:00
def seen
Cache.fetch("img:#{md5}", expires_in: 1.minute) do
(Cache.redis.get("yiffy2:stats:image:#{md5}").presence || 0).to_i
end
end
def self.all_seen
2024-10-24 06:38:42 +00:00
Cache.fetch("img:all_seen", expires_in: 1.minutes) do
keys = all.map { |img| "yiffy2:stats:image:#{img.md5}" }
values = Cache.redis.mget(*keys)
results = {}
keys.each_with_index do |key, index|
val = values.at(index)
next if val.nil?
results[key[-32..]] = val.to_i
end
results
end
end
2024-10-22 00:16:52 +00:00
def self.create_external!(type, id, category)
site = ExternalAPIImage::SITES.find { |s| s.value == type }
raise(Error, "Invalid site: #{type}") if site.blank?
img = create!(width: 0, height: 0, mime_type: "external", file_ext: "external", file_size: 0, category: category)
img.convert_to_external(format(site.format, id))
end
2024-05-03 03:04:43 +00:00
end