Add docker-compose setup for self-hosting the backend (#68)

Closes storytold/artcraft#1301 Adds `build/selfhost/`, a Docker Compose setup that runs `storyteller-web` with MySQL 8.4, Redis, Elasticsearch and SeaweedFS (S3-compatible storage). - A `migrate` service applies `_database/sql/migrations` in order and records them in `__diesel_schema_migrations`, so reruns only apply new migrations. - `storage-init` creates the three buckets the server expects. - The server image is built from `build/service_cpu.Dockerfile`. - `.env.example` lists every variable the server requires at startup, with placeholders for third-party providers. SeaweedFS is used because MinIO no longer publishes Docker images. **Tested locally (Docker Desktop on Windows):** image build, all 285 migrations (plus a rerun), `GET /_status` healthy, model listing, account creation and login, and an image upload stored in the bucket. **Not included:** background workers, Elasticsearch index setup.
2026-11-02 18:09:35 +00:00 · 2026-10-12 04:21:50 +05:30
parent 2893b6f713
commit 25f38b79b8
3 changed files with 299 additions and 0 deletions
@@ -0,0 +1,59 @@
+# Copy this file to `.env` and change the values.
+# `.env` is ignored by git; never commit real keys.
+
+# ----- Local infrastructure (used by docker-compose.yml) -----
+
+MYSQL_ROOT_PASSWORD=change-me-root
+MYSQL_PASSWORD=change-me
+
+# Credentials for the local S3-compatible storage (SeaweedFS).
+S3_ACCESS_KEY=artcraft
+S3_SECRET_KEY=change-me-please
+
+# ----- storyteller-web settings -----
+
+# Cookies are served over plain HTTP locally.
+COOKIE_SECURE=false
+COOKIE_DOMAIN=localhost
+COOKIE_SECRET=change-me
+SORT_KEY_SECRET=change-me
+WEBSITE_HOMEPAGE_REDIRECT=http://localhost:4201
+
+ENABLE_PAGING=false
+NUM_WORKERS=4
+
+# ----- Third-party providers -----
+# storyteller-web requires these to be set at startup. Placeholders let the server
+# start; features backed by a provider only work once you add a real key.
+
+FAL_API_KEY=unset
+FAL_WEBHOOK_URL=http://localhost:12345/v1/webhooks/fal
+GMICLOUD_API_KEY=unset
+GROK_API_KEY=unset
+OPENAI_API_KEY=unset
+WORLDLABS_API_KEY=unset
+BEEBLE_API_KEY=unset
+BEEBLE_WEBHOOK_URL=http://localhost:12345/v1/webhooks/beeble
+SEEDANCE2PRO_COOKIES=unset
+SEEDANCE2PRO_WHITELIST_COOKIES=unset
+SEEDANCE2PRO_BYTEPLUS_ULTRA_COOKIES=unset
+
+# Email (Resend)
+RESEND_API_KEY=unset
+
+# ----- Stripe (billing) -----
+
+STRIPE_SECRET_KEY=unset
+STRIPE_SECRET_WEBHOOK_SIGNING_KEY=unset
+STRIPE_FAKEYOU_ACCOUNT_ID=unset
+STRIPE_PORTAL_DEFAULT_CONFIG_ID=unset
+STRIPE_CHECKOUT_SUCCESS_URL_PATH=/checkout_success
+STRIPE_CHECKOUT_CANCEL_URL_PATH=/checkout_cancel
+STRIPE_PORTAL_RETURN_URL_PATH=/portal_success
+
+STRIPE_ARTCRAFT_ACCOUNT_ID=unset
+STRIPE_ARTCRAFT_SECRET_KEY=unset
+STRIPE_ARTCRAFT_SECRET_WEBHOOK_KEY=unset
+STRIPE_ARTCRAFT_CHECKOUT_SUCCESS_URL=http://localhost:4201/checkout_success
+STRIPE_ARTCRAFT_CHECKOUT_CANCEL_URL=http://localhost:4201/checkout_cancel
+STRIPE_ARTCRAFT_PORTAL_RETURN_URL=http://localhost:4201/portal_closed
@@ -0,0 +1,77 @@
+# Self-hosting with Docker Compose
+
+This directory runs the ArtCraft backend (`storyteller-web`) and the services it
+depends on with Docker Compose.
+
+| Service           | Image                    | Purpose                                     |
+|-------------------|--------------------------|---------------------------------------------|
+| `storyteller-web` | built from this repo     | HTTP API, on `http://localhost:12345`       |
+| `mysql`           | `mysql:8.4`              | Accounts, media metadata, jobs, billing     |
+| `migrate`         | `mysql:8.4`              | Applies `_database/sql/migrations`, exits   |
+| `redis`           | `redis:7`                | Caching, rate limiting, job progress        |
+| `elasticsearch`   | `elasticsearch:8.15.3`   | Search                                      |
+| `seaweedfs`       | `chrislusf/seaweedfs`    | S3-compatible storage for media             |
+| `storage-init`    | `chrislusf/seaweedfs`    | Creates the storage buckets, exits          |
+
+Only the API port (`12345`) is published to the host.
+
+## Requirements
+
+- Docker with Docker Compose v2.23 or newer.
+- About 8 GB of memory for Docker and 20 GB of free disk. The first build
+  compiles the Rust workspace in release mode and can take a long time.
+
+## Usage
+
+From this directory:
+
+```bash
+cp .env.example .env    # then edit the passwords and secrets
+docker compose up --build -d
+```
+
+Check that the server is up:
+
+```bash
+curl http://localhost:12345/_status
+```
+
+Stop everything with `docker compose down`. Add `-v` to also delete the
+database, search index and stored media.
+
+## Configuration
+
+Settings live in `.env`, which git ignores. `docker-compose.yml` sets the
+connection settings for MySQL, Redis, Elasticsearch and storage, so you only
+need to change passwords and secrets.
+
+`storyteller-web` requires credentials for every third-party provider (FAL,
+OpenAI, Stripe, Resend and others) to be present at startup. `.env.example`
+sets them to the placeholder `unset`, which lets the server start. Features
+that call a provider only work after you add a real key for it.
+
+Provider webhooks, such as `FAL_WEBHOOK_URL`, must be reachable from the
+internet for asynchronous generation results to arrive.
+
+## Database migrations
+
+The `migrate` service applies every migration in `_database/sql/migrations` in
+order and records each one in `__diesel_schema_migrations`, the table the diesel
+CLI uses. It runs on every `docker compose up` and only applies migrations that
+have not run yet.
+
+## Connecting the web app
+
+To point the browser app at this server, run it from `frontend` as described in
+the repository README:
+
+```bash
+VITE_USE_LOCAL_API=true npx nx dev artcraft-webapp
+```
+
+## Limitations
+
+- Only the HTTP API is included. Background workers (provider polling, video
+  thumbnails, email, analytics) are not started.
+- Elasticsearch starts with no indexes. Index definitions are in
+  `_database/elasticsearch/index_definitions`.
@@ -0,0 +1,163 @@
+# Self-hosted ArtCraft backend (storyteller-web) with its dependencies.
+#
+# Usage (from this directory):
+#   cp .env.example .env
+#   docker compose up --build
+#
+# See README.md in this directory for details.
+
+name: artcraft-selfhost
+
+services:
+  mysql:
+    image: mysql:8.4
+    restart: unless-stopped
+    environment:
+      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
+      MYSQL_DATABASE: storyteller
+      MYSQL_USER: storyteller
+      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
+    volumes:
+      - mysql-data:/var/lib/mysql
+    healthcheck:
+      # NB: Use TCP (127.0.0.1), not the socket, so the check fails while the image's
+      # temporary init server (which has networking disabled) is still running.
+      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-uroot", "-p${MYSQL_ROOT_PASSWORD}", "--silent"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+
+  # Applies the SQL migrations in `_database/sql/migrations` in order.
+  # Applied versions are recorded in `__diesel_schema_migrations` (the same table
+  # the diesel CLI uses), so re-running is safe and only applies new migrations.
+  migrate:
+    image: mysql:8.4
+    depends_on:
+      mysql:
+        condition: service_healthy
+    environment:
+      MYSQL_PWD: ${MYSQL_ROOT_PASSWORD}
+    volumes:
+      - ../../_database/sql/migrations:/migrations:ro
+    entrypoint: ["bash", "-c"]
+    command:
+      - |
+        set -euo pipefail
+        run_sql() { mysql -h mysql -u root storyteller "$$@"; }
+        run_sql -e "CREATE TABLE IF NOT EXISTS __diesel_schema_migrations (
+          version VARCHAR(50) PRIMARY KEY NOT NULL,
+          run_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)"
+        for dir in /migrations/*/; do
+          name=$$(basename "$$dir")
+          version=$$(echo "$${name%%_*}" | tr -d '-')
+          if [ -n "$$(run_sql -N -e "SELECT 1 FROM __diesel_schema_migrations WHERE version = '$$version'")" ]; then
+            continue
+          fi
+          echo "Applying migration $$name"
+          run_sql < "$$dir/up.sql"
+          run_sql -e "INSERT INTO __diesel_schema_migrations (version) VALUES ('$$version')"
+        done
+        echo "Migrations complete."
+
+  redis:
+    image: redis:7
+    restart: unless-stopped
+    volumes:
+      - redis-data:/data
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3
+    restart: unless-stopped
+    environment:
+      discovery.type: single-node
+      xpack.security.enabled: "false"
+      ES_JAVA_OPTS: -Xms512m -Xmx512m
+    volumes:
+      - elasticsearch-data:/usr/share/elasticsearch/data
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fs http://localhost:9200/_cluster/health || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+
+  # S3-compatible object storage (SeaweedFS) for uploaded and generated media.
+  seaweedfs:
+    image: chrislusf/seaweedfs:3.97
+    restart: unless-stopped
+    command: server -dir=/data -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json
+    configs:
+      - source: seaweedfs-s3
+        target: /etc/seaweedfs/s3.json
+    volumes:
+      - seaweedfs-data:/data
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+
+  # Creates the buckets storyteller-web expects.
+  storage-init:
+    image: chrislusf/seaweedfs:3.97
+    depends_on:
+      seaweedfs:
+        condition: service_healthy
+    entrypoint: ["sh", "-c"]
+    command:
+      - |
+        set -e
+        for bucket in artcraft-private artcraft-public artcraft-public-gc; do
+          echo "s3.bucket.create -name $$bucket" | weed shell -master=seaweedfs:9333
+        done
+
+  storyteller-web:
+    build:
+      context: ../..
+      dockerfile: build/service_cpu.Dockerfile
+    image: artcraft-services:local
+    restart: unless-stopped
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      storage-init:
+        condition: service_completed_successfully
+      redis:
+        condition: service_started
+      elasticsearch:
+        condition: service_healthy
+    env_file: .env
+    environment:
+      SERVER_ENVIRONMENT: development
+      BIND_ADDRESS: 0.0.0.0:12345
+      MYSQL_URL: mysql://storyteller:${MYSQL_PASSWORD}@mysql:3306/storyteller
+      REDIS_0_URL: redis://redis:6379/0
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+      S3_COMPATIBLE_ENDPOINT_URL: http://seaweedfs:8333
+      ACCESS_KEY: ${S3_ACCESS_KEY}
+      SECRET_KEY: ${S3_SECRET_KEY}
+      REGION_NAME: us-east-1
+      W2L_PRIVATE_DOWNLOAD_BUCKET_NAME: artcraft-private
+      W2L_PUBLIC_DOWNLOAD_BUCKET_NAME: artcraft-public
+      GC_ENABLED_PUBLIC_BUCKET_NAME: artcraft-public-gc
+      AUDIO_UPLOADS_BUCKET_ROOT: /user_uploaded_audio
+    ports:
+      - "12345:12345"
+
+configs:
+  seaweedfs-s3:
+    content: |
+      {
+        "identities": [
+          {
+            "name": "artcraft",
+            "credentials": [{ "accessKey": "${S3_ACCESS_KEY}", "secretKey": "${S3_SECRET_KEY}" }],
+            "actions": ["Admin", "Read", "Write", "List", "Tagging"]
+          }
+        ]
+      }
+
+volumes:
+  mysql-data:
+  redis-data:
+  elasticsearch-data:
+  seaweedfs-data: