# frozen_string_literal: true class E621ExportDownloadJob < ApplicationJob queue_as :exports def perform(type) date = Time.zone.now.strftime("%Y-%m-%d") url = "https://e621.net/db_export/#{type}-#{date}.csv.gz" file = Tempfile.new(%W[e621-export-#{type}-#{date} .csv.gz]) file.binmode res = HTTParty.get(url, httparty_options) do |chunk| next if [301, 302].include?(chunk.code) file.write(chunk) end unless res.success? file.close file.delete raise("HTTP error code: #{res.code} #{res.message}") end file.rewind gz = Zlib::GzipReader.new(file) csv = Tempfile.new(%W[e621-export-#{date} .csv]) csv.binmode csv.write(gz.read) gz.close file.close file.delete TableImportJob.perform_now("e621.#{type}", csv.path) end def httparty_options { timeout: 300, stream_body: true, }.deep_merge(Websites.config.httparty_options) end end