Websites/app/jobs/e621_export_download_job.rb

# frozen_string_literal: true

class E621ExportDownloadJob < ApplicationJob
  queue_as :exports

  def perform(type)
    date = Time.zone.now.strftime("%Y-%m-%d")
    url = "https://e621.net/db_export/#{type}-#{date}.csv.gz"
    file = Tempfile.new(%W[e621-export-#{type}-#{date} .csv.gz])
    file.binmode
    res = HTTParty.get(url, httparty_options) do |chunk|
      next if [301, 302].include?(chunk.code)

      file.write(chunk)
    end
    unless res.success?
      file.close
      file.delete
      raise("HTTP error code: #{res.code} #{res.message}")
    end

    file.rewind
    gz = Zlib::GzipReader.new(file)
    csv = Tempfile.new(%W[e621-export-#{date} .csv])
    csv.binmode
    csv.write(gz.read)
    gz.close
    file.close
    file.delete
    TableImportJob.perform_now("e621.#{type}", csv.path)
  end

  def httparty_options
    {
      timeout:     300,
      stream_body: true,
    }.deep_merge(Websites.config.httparty_options)
  end
end