2024-05-03 03:04:43 +00:00
# frozen_string_literal: true
class APIImage < ApplicationRecord
2024-10-21 22:58:53 +00:00
class Error < StandardError ; end
2024-05-03 03:04:43 +00:00
belongs_to_creator
2024-05-07 06:57:00 +00:00
attr_accessor :file , :exception , :deletion_reason , :no_webhook_messages
2024-05-06 07:47:53 +00:00
validates :category , presence : true , inclusion : { in : - > { APIImage . categories . map ( & :db ) } }
validates :id , uniqueness : true , on : :file
validate on : :file do | image |
ext , mime = file_header_info ( file . path )
image . errors . add ( :file , " type is invalid ( #{ mime } ) " ) if ext == mime
end
validate do | image |
image . errors . add ( :base , exception . message ) if exception
end
after_create :invalidate_cache
2024-05-07 06:57:00 +00:00
after_create :send_created , unless : :no_webhook_messages
2024-05-09 02:33:45 +00:00
after_create :initialize_short_url
2024-10-13 06:04:55 +00:00
after_create :update_iqdb
2024-05-06 07:47:53 +00:00
after_update :update_files , if : :saved_change_to_category?
2024-05-07 06:57:00 +00:00
after_update :send_updated , unless : :no_webhook_messages
2024-05-06 07:47:53 +00:00
after_destroy :delete_files
2024-10-13 06:04:55 +00:00
after_destroy :remove_iqdb
2024-05-07 06:57:00 +00:00
after_destroy :send_deleted , unless : :no_webhook_messages
2024-10-21 22:58:53 +00:00
has_one :external_api_image , dependent : :destroy
scope :viewable , - > { where ( is_viewable : true ) }
scope :unviewable , - > { where ( is_viewable : false ) }
scope :external , - > { joins ( :external_api_image ) . where . not ( " external_api_images.id " : nil ) }
scope :internal , - > { left_outer_joins ( :external_api_image ) . where ( " external_api_images.id " : nil ) }
def is_external?
external_api_image . present?
end
2024-05-06 07:47:53 +00:00
def delete_files
2024-10-21 22:58:53 +00:00
return if is_external?
delete_files!
end
def delete_files!
invalidate_cache
2024-05-06 07:47:53 +00:00
Websites . config . yiffy2_storage . delete ( path )
2024-06-29 07:13:04 +00:00
Websites . config . yiffy2_backup_storage . delete ( path )
2024-05-06 07:47:53 +00:00
end
def update_files
2024-10-21 22:58:53 +00:00
invalidate_cache
return if is_external?
2024-05-06 07:47:53 +00:00
file = Websites . config . yiffy2_storage . get ( path_before_last_save )
Websites . config . yiffy2_storage . put ( path , file )
Websites . config . yiffy2_storage . delete ( path_before_last_save )
2024-06-29 07:13:04 +00:00
Websites . config . yiffy2_backup_storage . put ( path , file )
Websites . config . yiffy2_backup_storage . delete ( path_before_last_save )
2024-05-06 07:47:53 +00:00
end
2024-06-11 07:20:34 +00:00
def file_header_info ( file_path )
2024-05-06 07:47:53 +00:00
File . open ( file_path ) do | bin |
mime_type = Marcel :: MimeType . for ( bin )
ext = case mime_type
when " image/jpeg "
" jpg "
when " image/gif "
" gif "
when " image/png "
" png "
else
mime_type
end
[ mime_type , ext ]
end
end
def invalidate_cache
Cache . redis . del ( " yiffy2:images: #{ category_before_last_save } " )
Cache . redis . del ( " yiffy2:images: #{ category } " )
end
2024-05-03 03:04:43 +00:00
module SearchMethods
def random ( category , limit , size_limit = nil )
2024-10-21 22:58:53 +00:00
q = viewable . where ( category : category )
2024-05-03 03:04:43 +00:00
q = q . where ( file_size : .. size_limit ) if size_limit
q . ids . sample ( limit ) . map ( & method ( :find ) )
end
def bulk ( category_map , size_limit = nil )
data = { }
category_map . each do | category , limit |
data [ category ] = random ( category , limit , size_limit )
end
data
end
2024-05-07 06:57:00 +00:00
2024-10-08 12:39:28 +00:00
def artists_like ( name )
where ( id : APIImage . from ( " unnest(artists) AS artist " ) . where ( " artist LIKE ? " , name . to_escaped_for_sql_like ) )
end
2024-10-24 09:10:07 +00:00
def external_artists_like ( name )
where ( id : ExternalAPIImage . from ( " external_api_images, jsonb_array_elements_text(cached_data->'tags'->'artist') AS artist " ) . where ( " artist LIKE ? " , name . to_escaped_for_sql_like ) . select ( " api_image_id " ) )
end
2024-05-07 06:57:00 +00:00
def search ( params )
q = super
q = q . attribute_matches ( :category , params [ :category ] )
q = q . attribute_matches ( :original_url , params [ :original_url ] )
2024-10-21 22:58:53 +00:00
q = q . attribute_matches ( :is_viewable , params [ :viewable ] )
2024-10-24 09:10:07 +00:00
if params [ :artist ] . present? # rubocop:disable Style/IfUnlessModifier
q = q . artists_like ( params [ :artist ] ) . or ( external_artists_like ( params [ :artist ] ) )
end
2024-10-21 22:58:53 +00:00
if params [ :md5 ] . present? # rubocop:disable Style/IfUnlessModifier
q = q . attribute_matches ( :id , params [ :md5 ] ) . or ( q . where ( id : ExternalAPIImage . where ( " cached_data->'file'->>'md5' = ? " , params [ :md5 ] ) . select ( " api_image_id " ) ) )
end
if params [ :external ] . present?
q = q . external if params [ :external ] . truthy?
q = q . internal if params [ :external ] . falsy?
end
2024-05-07 06:57:00 +00:00
q . order ( created_at : :desc )
end
2024-05-03 03:04:43 +00:00
end
extend SearchMethods
def md5
id . gsub ( " - " , " " )
end
2024-05-06 07:47:53 +00:00
alias stripped_md5 md5
def path_before_last_save
" / #{ category_before_last_save . tr ( '.' , '/' ) } / #{ md5 } . #{ file_ext } "
end
def path
" / #{ category . tr ( '.' , '/' ) } / #{ md5 } . #{ file_ext } "
end
2024-05-03 03:04:43 +00:00
def url
2024-05-06 07:47:53 +00:00
Websites . config . yiffy2_storage . url_for ( self )
2024-05-03 03:04:43 +00:00
end
def short_url
ShortUrl . override ( md5 , url ) . shorturl
end
2024-05-09 02:33:45 +00:00
def initialize_short_url
ShortUrl . override ( md5 , url )
end
2024-05-06 07:47:53 +00:00
def sources_string
sources . join ( " \n " )
end
def sources_string = ( str )
self . sources = str . split ( " \n " ) . map ( & :strip )
end
def artists_string
artists . join ( " \n " )
end
def artists_string = ( str )
self . artists = str . split ( " \n " ) . map ( & :strip )
end
2024-10-21 22:58:53 +00:00
def format_external ( * )
img = external_api_image
{
artists : img . artists ,
sources : img . sources ,
width : img . width ,
height : img . height ,
url : img . file_url ,
type : Marcel :: EXTENSIONS [ img . ext ] ,
name : " #{ img . md5 } . #{ img . ext } " ,
id : md5 ,
md5 : img . md5 ,
ext : img . ext ,
size : img . size ,
reportURL : img . report_url ,
shortURL : short_url ,
external : true ,
viewable : is_viewable? ,
2024-10-24 06:36:13 +00:00
seen : seen ,
2024-10-21 22:58:53 +00:00
}
end
2024-05-03 03:04:43 +00:00
def serializable_hash ( * )
2024-10-21 22:58:53 +00:00
return format_external ( * ) if is_external?
2024-05-03 03:04:43 +00:00
{
artists : artists ,
sources : sources ,
width : width ,
height : height ,
url : url ,
type : mime_type ,
name : " #{ md5 } . #{ file_ext } " ,
id : md5 ,
2024-10-21 22:58:53 +00:00
md5 : md5 ,
2024-05-03 03:04:43 +00:00
ext : file_ext ,
size : file_size ,
reportURL : nil ,
shortURL : short_url ,
2024-10-21 22:58:53 +00:00
external : false ,
viewable : is_viewable? ,
2024-10-24 06:36:13 +00:00
seen : seen ,
2024-05-03 03:04:43 +00:00
}
end
2024-10-21 22:58:53 +00:00
def convert_to_external ( url )
transaction do
uri = URI . parse ( url )
case uri . host
when " femboy.fan "
type = " femboyfans "
if %r{ ^/posts/( \ d+) } =~ uri . path
external_id = $1 . to_i
else
raise ( Error , " Failed to parse external id from #{ url } " )
end
when " e621.net "
type = " e621 "
if %r{ ^/posts/( \ d+) } =~ uri . path
external_id = $1 . to_i
else
raise ( Error , " Failed to parse external id from #{ url } " )
end
else
raise ( Error , " Attempted to convert to unknown external: #{ url } " )
end
ExternalAPIImage . create! ( api_image : self , external_id : external_id , site : type )
delete_files!
2024-10-21 23:09:37 +00:00
update_columns ( artists : [ ] , sources : [ ] , width : 0 , height : 0 , mime_type : " external " , file_ext : " external " , file_size : 0 )
2024-10-21 22:58:53 +00:00
end
2024-10-22 00:16:52 +00:00
send_converted
reload_external_api_image
external_api_image
2024-10-21 22:58:53 +00:00
end
2024-05-03 03:04:43 +00:00
def self . categories
sfw = %w[ animals.birb animals.blep animals.dikdik furry.boop furry.cuddle furry.flop furry.fursuit furry.hold furry.howl furry.hug furry.kiss furry.lick furry.propose ]
nsfw = %w[ furry.butts furry.bulge furry.yiff.andromorph furry.yiff.gay furry.yiff.gynomorph furry.yiff.lesbian furry.yiff.straight ]
titles = {
** sfw . index_with { | c | c . split ( " . " ) . map ( & :capitalize ) . join ( " > " ) } ,
" animals.dikdik " = > " Animals > Dik Dik " ,
** nsfw . index_with { | c | c . split ( " . " ) . map ( & :capitalize ) . join ( " > " ) } ,
}
2024-05-06 07:47:53 +00:00
sfw . map { | cat | APICategory . new ( titles [ cat ] , cat , true ) }
. concat ( nsfw . map { | cat | APICategory . new ( titles [ cat ] , cat , false ) } )
2024-05-03 03:04:43 +00:00
end
def self . category_title ( db )
categories . find { | k | k == db } . try ( :db ) || db . split ( " . " ) . map ( & :capitalize ) . join ( " > " )
end
def self . cached_count ( category )
count = Cache . redis . get ( " yiffy2:images: #{ category } " )
if count . nil?
count = APIImage . where ( category : category ) . count
# images aren't changing so we really don't need any expiry, but I still want an expiry just in case
Cache . redis . set ( " yiffy2:images: #{ category } " , count . to_s , ex : 60 * 60 * 24 * 7 )
end
count . to_i
end
def self . state
data = group ( :category ) . count . map do | k , v |
{
count : v ,
name : category_title ( k ) ,
category : k ,
state : v < 5 ? " red " : v < 20 ? " yellow " : " green " ,
}
end
[ * data , {
count : data . pluck ( :count ) . reduce ( :+ ) ,
name : " Total " ,
category : " total " ,
state : " total " ,
} , ]
end
2024-05-06 07:47:53 +00:00
def self . sync_all
sync_e621
end
# TODO
def self . sync_e621
images = joins ( " CROSS JOIN unnest(sources) AS source " ) . where ( " source ILIKE ? " , " %e621.net% " ) . distinct . limit ( 300 )
ids = [ ]
md5s = [ ]
images . each do | img |
img . sources . each do | source |
next unless source . include? ( " e621.net " )
id = source . split ( " / " ) . last
ids << id . to_i if id . to_i . to_s == id
end
md5s << img . md5
end
Requests :: E621 . get_all_posts ( ids : ids , md5s : md5s ) = > { posts : , missing : }
Rails . logger . info ( " Failed to find #{ missing [ :ids ] . length } posts " ) unless missing [ :ids ] . empty?
mismatch = images . select { | img | posts . none? { | post | post [ " file " ] [ " md5 " ] == img . md5 } }
Rails . logger . info ( " Found #{ mismatch . length } mismatched images " ) unless mismatch . empty?
posts
end
def sync
APIImageSyncJob . perform_later ( self )
end
2024-05-07 06:57:00 +00:00
module WebhookMethods
GREEN = 0x008000
YELLOW = 0xFFA500
RED = 0xFF0000
GREEN_TICK = " <:GreenTick:1235058920762904576> "
RED_TICK = " <:RedTick:1235058898549870724> "
def execute ( content )
2024-10-21 22:58:53 +00:00
return unless Rails . env . production?
2024-05-07 06:57:00 +00:00
Websites . config . yiffyapi_image_logs_webhook . execute ( {
embeds : [ content ] ,
} )
end
def send_created
execute ( {
title : " Image Uploaded " ,
description : << ~ DESC ,
ID : ` #{ md5 } `
2024-05-07 07:54:31 +00:00
Category : ` #{ category } `
2024-10-22 00:43:38 +00:00
Artists : ` #{ artists . join ( ', ' ) . presence || '[NONE]' } `
2024-05-07 06:57:00 +00:00
Sources :
2024-10-22 00:43:38 +00:00
#{sources.map { |s| "* #{s}" }.join("\n").presence || '[NONE]'}
2024-05-07 06:57:00 +00:00
Upload Type : #{original_url.present? ? "[url](#{original_url})" : 'file'}
Uploaded By : < @ #{creator.id}> (`#{creator.name}`)
Created At : < t : #{created_at.to_i}>
Updated At : < t : #{updated_at.to_i}>
DESC
color : GREEN ,
timestamp : Time . now . iso8601 ,
} )
end
def send_deleted
execute ( {
title : " Image Deleted " ,
description : << ~ DESC ,
ID : ` #{ md5 } `
2024-05-07 07:54:31 +00:00
Category : ` #{ category } `
2024-10-22 00:43:38 +00:00
Artists : ` #{ artists . join ( ', ' ) . presence || '[NONE]' } } `
2024-05-07 06:57:00 +00:00
Sources :
2024-10-22 00:43:38 +00:00
#{sources.map { |s| "* #{s}" }.join("\n").presence || '[NONE]'}}
2024-05-07 06:57:00 +00:00
Upload Type : #{original_url.present? ? "[url](#{original_url})" : 'file'}
Uploaded By : < @ #{creator.id}> (`#{creator.name}`)
Reason : #{deletion_reason || 'None Provided'}
Created At : < t : #{created_at.to_i}>
Updated At : < t : #{updated_at.to_i}>
DESC
color : RED ,
timestamp : Time . now . iso8601 ,
} )
end
def check_change ( attr , changes )
changes << " #{ attr . to_s . titleize } : ** #{ send ( " #{ attr } _before_last_save " ) } ** -> ** #{ send ( attr ) } ** " if send ( " saved_change_to_ #{ attr } ? " )
end
def send_updated
changes = [ ]
check_change ( :artists , changes )
check_change ( :category , changes )
2024-10-21 22:58:53 +00:00
check_change ( :is_viewable , changes )
2024-05-07 06:57:00 +00:00
if sources != sources_before_last_save
diff = [ ]
sources_before_last_save . each do | source |
diff << " - #{ source } " unless sources . include? ( source )
end
sources . each do | source |
diff << " + #{ source } " unless sources_before_last_save . include? ( source )
end
changes << " Sources: \n ```diff \n #{ diff . join ( " \n " ) } \n ``` "
end
return if changes . empty?
changes << " Blame: #{ blame } "
execute ( {
title : " Image Updated " ,
description : << ~ DESC ,
ID : ` #{ md5 } `
#{changes.join("\n")}
DESC
color : YELLOW ,
timestamp : Time . now . iso8601 ,
} )
end
2024-10-21 22:58:53 +00:00
def send_converted
execute ( {
title : " Image Converted " ,
description : << ~ DESC ,
ID : ` #{ md5 } `
Blame : #{blame}
Converted to external image : #{external_api_image.url}
DESC
color : YELLOW ,
timestamp : Time . now . iso8601 ,
} )
end
2024-05-07 06:57:00 +00:00
def blame
if CurrentUser . user . present?
" <@ #{ CurrentUser . id } > (` #{ CurrentUser . name } `) "
elsif Rails . const_defined? ( " Console " )
" **Console** "
elsif CurrentUser . ip_addr
" ** #{ CurrentUser . ip_addr } ** "
else
" **Unknown** "
end
end
end
include WebhookMethods
2024-10-13 06:04:55 +00:00
def is_image?
2024-10-21 22:58:53 +00:00
ext = file_ext
ext = external_api_image . ext if is_external?
%w[ png jpg ] . include? ( ext )
2024-10-13 06:04:55 +00:00
end
def is_gif?
2024-10-21 22:58:53 +00:00
ext = file_ext
ext = external_api_image . ext if is_external?
ext == " gif "
end
def is_available?
! is_external? || ! external_api_image . is_deleted?
2024-10-13 06:04:55 +00:00
end
def update_iqdb
2024-10-21 22:58:53 +00:00
return unless is_available? && is_image?
2024-10-13 06:04:55 +00:00
IqdbUpdateJob . perform_later ( id )
end
def update_iqdb!
2024-10-21 22:58:53 +00:00
return unless is_available? && is_image?
2024-10-13 06:04:55 +00:00
IqdbUpdateJob . perform_now ( id )
end
def remove_iqdb
return unless is_image?
IqdbRemoveJob . perform_later ( iqdb_id )
end
def remove_iqdb!
return unless is_image?
IqdbRemoveJob . perform_now ( iqdb_id )
end
2024-10-22 00:16:52 +00:00
2024-10-24 06:36:13 +00:00
def seen
Cache . fetch ( " img: #{ md5 } " , expires_in : 1 . minute ) do
( Cache . redis . get ( " yiffy2:stats:image: #{ md5 } " ) . presence || 0 ) . to_i
end
end
def self . all_seen
2024-10-24 06:38:42 +00:00
Cache . fetch ( " img:all_seen " , expires_in : 1 . minutes ) do
2024-10-24 06:36:13 +00:00
keys = all . map { | img | " yiffy2:stats:image: #{ img . md5 } " }
values = Cache . redis . mget ( * keys )
results = { }
keys . each_with_index do | key , index |
val = values . at ( index )
next if val . nil?
results [ key [ - 32 .. ] ] = val . to_i
end
results
end
end
2024-10-22 00:16:52 +00:00
def self . create_external! ( type , id , category )
site = ExternalAPIImage :: SITES . find { | s | s . value == type }
raise ( Error , " Invalid site: #{ type } " ) if site . blank?
img = create! ( width : 0 , height : 0 , mime_type : " external " , file_ext : " external " , file_size : 0 , category : category )
img . convert_to_external ( format ( site . format , id ) )
end
2024-05-03 03:04:43 +00:00
end