Add git storage manager

Yes, I'm aware this is dumb
No, I don't care

Dockerfile

@@ -19,7 +19,8 @@ FROM ruby:3.2.2-alpine3.18
 
 RUN apk --no-cache add ffmpeg vips gifsicle \
   postgresql14-client \
-  git jemalloc tzdata
+  git git-lfs jemalloc tzdata \
+  openssh-keygen
 
 WORKDIR /app
 

app/jobs/e621_thumbnail_job.rb

@@ -23,6 +23,7 @@ class E621ThumbnailJob < ApplicationJob
       end
       ImageOptim.new.optimize_image!(outfile.path)
       Websites.config.e621_thumbnails_storage.upload("/#{entry.stripped_md5}.#{entry.filetype}", outfile)
+      Websites.config.e621_thumbnails_backup_storage.put("/#{entry.stripped_md5}.#{entry.filetype}", outfile)
       entry.update!(status: "complete")
       execute_webhook(entry, title: "Thumbnail Generated (#{entry.filetype})")
     end

app/logical/api_image_upload_service.rb

@@ -49,6 +49,7 @@ class APIImageUploadService
 
   def store_file(image, file)
     Websites.config.yiffy2_storage.put(image.path, file)
+    Websites.config.yiffy2_backup_storage.put(image.path, file)
   end
 
   def calculate_dimensions(file_path)

app/logical/storage_manager/git.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module StorageManager
+  class Git < StorageManager::Local
+    def delete(path)
+      return unless exists?(path)
+      super
+      system!("git add #{base_path}#{path}")
+      system!("git commit -m \"Remove #{path[1..]}\"")
+      system!("git push")
+    end
+
+    def put(path, io)
+      super
+      if exists?(path)
+        system!("git add #{base_path}#{path}")
+        system!("git commit -m \"Add #{path[1..]}\"")
+        system!("git push")
+      end
+    end
+
+    private
+
+    def system!(*)
+      system(*, exception: true, chdir: base_path)
+    end
+  end
+end

app/logical/storage_manager/null.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module StorageManager
+  class Null
+    def delete(_path)
+      # noop
+    end
+
+    def exists?(_path)
+      false
+    end
+
+    def get(_path)
+      nil
+    end
+
+    def put(_path, _io)
+      # noop
+    end
+
+    def upload(_path, _body)
+      nil
+    end
+
+    def url_for(_entry)
+      nil
+    end
+  end
+end

app/models/api_image.rb

@@ -25,6 +25,7 @@ class APIImage < ApplicationRecord
 
   def delete_files
     Websites.config.yiffy2_storage.delete(path)
+    Websites.config.yiffy2_backup_storage.delete(path)
     invalidate_cache
   end
 
@@ -32,6 +33,8 @@ class APIImage < ApplicationRecord
     file = Websites.config.yiffy2_storage.get(path_before_last_save)
     Websites.config.yiffy2_storage.put(path, file)
     Websites.config.yiffy2_storage.delete(path_before_last_save)
+    Websites.config.yiffy2_backup_storage.put(path, file)
+    Websites.config.yiffy2_backup_storage.delete(path_before_last_save)
     invalidate_cache
   end
 

app/models/e621_thumbnail.rb

@@ -28,6 +28,7 @@ class E621Thumbnail < ApplicationRecord
 
     def delete_files!
       Websites.config.e621_thumbnails_storage.delete("/#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}")
+      Websites.config.e621_thumbnails_backup_storage.delete("/#{stripped_md5}.#{filetype}") if StorageManager::E621Thumbnails.exists?("#{stripped_md5}.#{filetype}")
     end
 
     def url

config/default_config.rb

@@ -244,19 +244,19 @@ module Websites
     end
 
     def e621_thumbnails_storage
-      # Bunny.new(base_url: Websites.config.e621_thumbnails_base_url, access_key: Websites.config.e621_thumbnails_access_key, storage_zone_name: Websites.config.e621_thumbnails_storage_zone_name)
       StorageManager::Local.new(base_url: "https://thumbs.yiff.media", base_path: "/data/e621-thumbnails")
     end
 
+    def e621_thumbnails_backup_storage
+      StorageManager::Null.new
+    end
+
     def yiffy2_storage
-      return StorageManager::Local.new(base_url: yiffy2_cdn_url, base_path: "/data/yiffy2") if Rails.env.development?
+      StorageManager::Local.new(base_url: yiffy2_cdn_url, base_path: "/data/yiffy2")
-      StorageManager::S3.new(
+    end
-        endpoint:          yiffy2_s3_endpoint,
+
-        access_key_id:     yiffy2_access_key_id,
+    def yiffy2_backup_storage
-        secret_access_key: yiffy2_secret_access_key,
+      StorageManager::Null.new
-        bucket:            yiffy2_bucket_name,
-        base_url:          yiffy2_cdn_url,
-      )
     end
 
     def e621_username

docker-compose.prod.yml

@@ -1,140 +0,0 @@
-x-environment: &common-env
-  WEBSITES_READONLY: "${READONLY:-0}"
-
-services:
-  websites:
-    build:
-      dockerfile: Dockerfile
-      context: .
-    image: websites4
-    command: foreman start -f Procfile
-    restart: unless-stopped
-    volumes:
-      - .:/app
-      - /var/www/e621-thumbnails:/data/e621-thumbnails
-      - /var/www/oceanic-docs:/data/oceanic-docs
-    tmpfs:
-      - /app/tmp/pids
-    environment:
-      <<: *common-env
-      RAILS_ENV: production
-    depends_on:
-      postgres:
-        condition: service_started
-      redis:
-        condition: service_started
-    labels:
-      - "hostname=websites4.containers.local"
-    tty: true
-
-  postgres:
-    image: postgres:14-alpine
-    volumes:
-      - ./data/postgres:/var/lib/postgresql/data
-    restart: unless-stopped
-    environment:
-      - POSTGRES_USER=websites
-      - POSTGRES_DB=websites
-      - POSTGRES_HOST_AUTH_METHOD=trust
-    healthcheck:
-      interval: 5s
-      timeout: 2s
-      test: pg_isready -U websites
-    hostname: postgres.websites4.containers.local
-    labels:
-      - "hostname=postgres.websites4.containers.local"
-    networks:
-      - default
-
-  redis:
-    image: redis:alpine
-    command: redis-server --save 10 1 --loglevel warning
-    volumes:
-      - ./data/redis:/data
-    restart: unless-stopped
-    healthcheck:
-      test: redis-cli ping
-      interval: 10s
-      timeout: 5s
-    hostname: redis.websites4.containers.local
-    labels:
-      - "hostname=redis.websites4.containers.local"
-    networks:
-      - default
-
-  opensearch:
-    image: opensearchproject/opensearch:2.13.0
-    environment:
-      - discovery.type=single-node
-      - logger.level=WARN
-      - DISABLE_SECURITY_PLUGIN=true
-      - DISABLE_INSTALL_DEMO_CONFIG=true
-      - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
-    volumes:
-      - ./data/opensearch:/usr/share/opensearch/data
-    init: true
-    healthcheck:
-      interval: 5s
-      timeout: 2s
-      retries: 12
-      test: curl "opensearch:9200/_cluster/health?wait_for_status=yellow&timeout=2s"
-    hostname: opensearch.websites4.containers.local
-    labels:
-      - "hostname=opensearch.websites4.containers.local"
-    networks:
-      - default
-
-  imgen:
-    image: ghcr.io/donovandmc/imgen
-    init: true
-    volumes:
-      - ./docker/imgen:/app
-    restart: always
-    healthcheck:
-      interval: 10s
-      timeout: 2s
-      test: lsof -i :3621 || exit 1
-    depends_on:
-      rethinkdb:
-        condition: service_started
-      redis:
-        condition: service_healthy
-    deploy:
-      resources:
-        limits:
-          memory: 256M
-          cpus: "1"
-    environment:
-      <<: *common-env
-    hostname: imgen.websites4.containers.local
-    labels:
-      - "hostname=imgen.websites4.containers.local"
-
-  rethinkdb:
-    image: rethinkdb
-    command: rethinkdb --bind all -n rdb
-    volumes:
-      - ./data/rethink:/data
-    deploy:
-      resources:
-        limits:
-          memory: 256M
-        reservations:
-          memory: 64M
-    restart: always
-    hostname: rethinkdb.websites4.containers.local
-    labels:
-      - "hostname=rethinkdb.websites4.containers.local"
-    networks:
-      - default
-
-networks:
-  default:
-    name: websites4
-    driver: bridge
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.19.3.64/27
-    driver_opts:
-      com.docker.network.bridge.name: br-websites

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - .:/app
       - e621_thumbnail_data:/data/e621-thumbnails
       - oceanic_docs_data:/data/oceanic-docs
-      - /var/www/yiffy2:/data/yiffy2
+      - yiffy2_image_data:/data/yiffy2
     tmpfs:
       - /app/tmp/pids
     environment:
@@ -142,5 +142,6 @@ volumes:
   redis_data:
   opensearch_data:
   e621_thumbnail_data:
+  yiffy2_image_data:
   oceanic_docs_data:
   rethinkdb_data: