40 lines
961 B
Ruby
40 lines
961 B
Ruby
|
# frozen_string_literal: true
|
||
|
|
|
||
|
|
class E621ExportDownloadJob < ApplicationJob
|
||
|
|
queue_as :default
|
||
|
|
|
||
|
|
def perform(type)
|
||
|
|
date = Time.zone.now.strftime("%Y-%m-%d")
|
||
|
|
url = "https://e621.net/db_export/#{type}-#{date}.csv.gz"
|
||
|
|
file = Tempfile.new(%W[e621-export-#{type}-#{date} .csv.gz])
|
||
|
|
file.binmode
|
||
|
|
res = HTTParty.get(url, httparty_options) do |chunk|
|
||
|
|
next if [301, 302].include?(chunk.code)
|
||
|
|
|
||
|
|
file.write(chunk)
|
||
|
|
end
|
||
|
|
unless res.success?
|
||
|
|
file.close
|
||
|
|
file.delete
|
||
|
|
raise("HTTP error code: #{res.code} #{res.message}")
|
||
|
|
end
|
||
|
|
|
||
|
|
file.rewind
|
||
|
|
gz = Zlib::GzipReader.new(file)
|
||
|
|
csv = Tempfile.new(%W[e621-export-#{date} .csv])
|
||
|
|
csv.binmode
|
||
|
|
csv.write(gz.read)
|
||
|
|
gz.close
|
||
|
|
file.close
|
||
|
|
file.delete
|
||
|
|
TableImportJob.perform_now("e621.#{type}", csv.path)
|
||
|
|
end
|
||
|
|
|
||
|
|
def httparty_options
|
||
|
|
{
|
||
|
|
timeout: 300,
|
||
|
|
stream_body: true,
|
||
|
|
}.deep_merge(Websites.config.httparty_options)
|
||
|
|
end
|
||
|
|
end
|