From f84eef5e5ffecc1c85cbeae53d96de6543cbcf38 Mon Sep 17 00:00:00 2001 From: Donovan Daniels Date: Sat, 29 Jun 2024 02:13:04 -0500 Subject: [PATCH] Add git storage manager Yes, I'm aware this is dumb No, I don't care --- Dockerfile | 3 +- app/jobs/e621_thumbnail_job.rb | 1 + app/logical/api_image_upload_service.rb | 1 + app/logical/storage_manager/git.rb | 28 +++++ app/logical/storage_manager/null.rb | 29 +++++ app/models/api_image.rb | 3 + app/models/e621_thumbnail.rb | 1 + config/default_config.rb | 18 +-- docker-compose.prod.yml | 140 ------------------------ docker-compose.yml | 3 +- 10 files changed, 76 insertions(+), 151 deletions(-) create mode 100644 app/logical/storage_manager/git.rb create mode 100644 app/logical/storage_manager/null.rb delete mode 100644 docker-compose.prod.yml diff --git a/Dockerfile b/Dockerfile index 9f2f555..1967c89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,8 @@ FROM ruby:3.2.2-alpine3.18 RUN apk --no-cache add ffmpeg vips gifsicle \ postgresql14-client \ - git jemalloc tzdata + git git-lfs jemalloc tzdata \ + openssh-keygen WORKDIR /app diff --git a/app/jobs/e621_thumbnail_job.rb b/app/jobs/e621_thumbnail_job.rb index b9ecada..0f008ef 100644 --- a/app/jobs/e621_thumbnail_job.rb +++ b/app/jobs/e621_thumbnail_job.rb @@ -23,6 +23,7 @@ class E621ThumbnailJob < ApplicationJob end ImageOptim.new.optimize_image!(outfile.path) Websites.config.e621_thumbnails_storage.upload("/#{entry.stripped_md5}.#{entry.filetype}", outfile) + Websites.config.e621_thumbnails_backup_storage.put("/#{entry.stripped_md5}.#{entry.filetype}", outfile) entry.update!(status: "complete") execute_webhook(entry, title: "Thumbnail Generated (#{entry.filetype})") end diff --git a/app/logical/api_image_upload_service.rb b/app/logical/api_image_upload_service.rb index e1e8b3d..3ec6a5f 100644 --- a/app/logical/api_image_upload_service.rb +++ b/app/logical/api_image_upload_service.rb @@ -49,6 +49,7 @@ class APIImageUploadService def store_file(image, file) Websites.config.yiffy2_storage.put(image.path, file) + Websites.config.yiffy2_backup_storage.put(image.path, file) end def calculate_dimensions(file_path) diff --git a/app/logical/storage_manager/git.rb b/app/logical/storage_manager/git.rb new file mode 100644 index 0000000..b6d97c5 --- /dev/null +++ b/app/logical/storage_manager/git.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module StorageManager + class Git < StorageManager::Local + def delete(path) + return unless exists?(path) + super + system!("git add #{base_path}#{path}") + system!("git commit -m \"Remove #{path[1..]}\"") + system!("git push") + end + + def put(path, io) + super + if exists?(path) + system!("git add #{base_path}#{path}") + system!("git commit -m \"Add #{path[1..]}\"") + system!("git push") + end + end + + private + + def system!(*) + system(*, exception: true, chdir: base_path) + end + end +end diff --git a/app/logical/storage_manager/null.rb b/app/logical/storage_manager/null.rb new file mode 100644 index 0000000..a2f01f4 --- /dev/null +++ b/app/logical/storage_manager/null.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module StorageManager + class Null + def delete(_path) + # noop + end + + def exists?(_path) + false + end + + def get(_path) + nil + end + + def put(_path, _io) + # noop + end + + def upload(_path, _body) + nil + end + + def url_for(_entry) + nil + end + end +end diff --git a/app/models/api_image.rb b/app/models/api_image.rb index c761bb6..273567e 100644 --- a/app/models/api_image.rb +++ b/app/models/api_image.rb @@ -25,6 +25,7 @@ class APIImage < ApplicationRecord def delete_files Websites.config.yiffy2_storage.delete(path) + Websites.config.yiffy2_backup_storage.delete(path) invalidate_cache end @@ -32,6 +33,8 @@ class APIImage < ApplicationRecord file = Websites.config.yiffy2_storage.get(path_before_last_save) Websites.config.yiffy2_storage.put(path, file) Websites.config.yiffy2_storage.delete(path_before_last_save) + Websites.config.yiffy2_backup_storage.put(path, file) + Websites.config.yiffy2_backup_storage.delete(path_before_last_save) invalidate_cache end diff --git a/app/models/e621_thumbnail.rb b/app/models/e621_thumbnail.rb index bd49a95..5f84d51 100644 --- a/app/models/e621_thumbnail.rb +++ b/app/models/e621_thumbnail.rb @@ -28,6 +28,7 @@ class E621Thumbnail < ApplicationRecord def delete_files! Websites.config.e621_thumbnails_storage.delete("/#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}") + Websites.config.e621_thumbnails_backup_storage.delete("/#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}") end def url diff --git a/config/default_config.rb b/config/default_config.rb index 08b7063..5c9aa45 100644 --- a/config/default_config.rb +++ b/config/default_config.rb @@ -244,19 +244,19 @@ module Websites end def e621_thumbnails_storage - # Bunny.new(base_url: Websites.config.e621_thumbnails_base_url, access_key: Websites.config.e621_thumbnails_access_key, storage_zone_name: Websites.config.e621_thumbnails_storage_zone_name) StorageManager::Local.new(base_url: "https://thumbs.yiff.media", base_path: "/data/e621-thumbnails") end + def e621_thumbnails_backup_storage + StorageManager::Null.new + end + def yiffy2_storage - return StorageManager::Local.new(base_url: yiffy2_cdn_url, base_path: "/data/yiffy2") if Rails.env.development? - StorageManager::S3.new( - endpoint: yiffy2_s3_endpoint, - access_key_id: yiffy2_access_key_id, - secret_access_key: yiffy2_secret_access_key, - bucket: yiffy2_bucket_name, - base_url: yiffy2_cdn_url, - ) + StorageManager::Local.new(base_url: yiffy2_cdn_url, base_path: "/data/yiffy2") + end + + def yiffy2_backup_storage + StorageManager::Null.new end def e621_username diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml deleted file mode 100644 index d4a80ee..0000000 --- a/docker-compose.prod.yml +++ /dev/null @@ -1,140 +0,0 @@ -x-environment: &common-env - WEBSITES_READONLY: "${READONLY:-0}" - -services: - websites: - build: - dockerfile: Dockerfile - context: . - image: websites4 - command: foreman start -f Procfile - restart: unless-stopped - volumes: - - .:/app - - /var/www/e621-thumbnails:/data/e621-thumbnails - - /var/www/oceanic-docs:/data/oceanic-docs - tmpfs: - - /app/tmp/pids - environment: - <<: *common-env - RAILS_ENV: production - depends_on: - postgres: - condition: service_started - redis: - condition: service_started - labels: - - "hostname=websites4.containers.local" - tty: true - - postgres: - image: postgres:14-alpine - volumes: - - ./data/postgres:/var/lib/postgresql/data - restart: unless-stopped - environment: - - POSTGRES_USER=websites - - POSTGRES_DB=websites - - POSTGRES_HOST_AUTH_METHOD=trust - healthcheck: - interval: 5s - timeout: 2s - test: pg_isready -U websites - hostname: postgres.websites4.containers.local - labels: - - "hostname=postgres.websites4.containers.local" - networks: - - default - - redis: - image: redis:alpine - command: redis-server --save 10 1 --loglevel warning - volumes: - - ./data/redis:/data - restart: unless-stopped - healthcheck: - test: redis-cli ping - interval: 10s - timeout: 5s - hostname: redis.websites4.containers.local - labels: - - "hostname=redis.websites4.containers.local" - networks: - - default - - opensearch: - image: opensearchproject/opensearch:2.13.0 - environment: - - discovery.type=single-node - - logger.level=WARN - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true - - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g - volumes: - - ./data/opensearch:/usr/share/opensearch/data - init: true - healthcheck: - interval: 5s - timeout: 2s - retries: 12 - test: curl "opensearch:9200/_cluster/health?wait_for_status=yellow&timeout=2s" - hostname: opensearch.websites4.containers.local - labels: - - "hostname=opensearch.websites4.containers.local" - networks: - - default - - imgen: - image: ghcr.io/donovandmc/imgen - init: true - volumes: - - ./docker/imgen:/app - restart: always - healthcheck: - interval: 10s - timeout: 2s - test: lsof -i :3621 || exit 1 - depends_on: - rethinkdb: - condition: service_started - redis: - condition: service_healthy - deploy: - resources: - limits: - memory: 256M - cpus: "1" - environment: - <<: *common-env - hostname: imgen.websites4.containers.local - labels: - - "hostname=imgen.websites4.containers.local" - - rethinkdb: - image: rethinkdb - command: rethinkdb --bind all -n rdb - volumes: - - ./data/rethink:/data - deploy: - resources: - limits: - memory: 256M - reservations: - memory: 64M - restart: always - hostname: rethinkdb.websites4.containers.local - labels: - - "hostname=rethinkdb.websites4.containers.local" - networks: - - default - -networks: - default: - name: websites4 - driver: bridge - ipam: - driver: default - config: - - subnet: 172.19.3.64/27 - driver_opts: - com.docker.network.bridge.name: br-websites diff --git a/docker-compose.yml b/docker-compose.yml index dffdc08..3d09c67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - .:/app - e621_thumbnail_data:/data/e621-thumbnails - oceanic_docs_data:/data/oceanic-docs - - /var/www/yiffy2:/data/yiffy2 + - yiffy2_image_data:/data/yiffy2 tmpfs: - /app/tmp/pids environment: @@ -142,5 +142,6 @@ volumes: redis_data: opensearch_data: e621_thumbnail_data: + yiffy2_image_data: oceanic_docs_data: rethinkdb_data: