From 25f38b79b8cef0c07b3df9400a718799e145c02e Mon Sep 17 00:00:00 2001 From: Navaras P <69687392+NavarasP@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:21:50 +0530 Subject: [PATCH] Add docker-compose setup for self-hosting the backend (#68) Closes storytold/artcraft#1301 Adds `build/selfhost/`, a Docker Compose setup that runs `storyteller-web` with MySQL 8.4, Redis, Elasticsearch and SeaweedFS (S3-compatible storage). - A `migrate` service applies `_database/sql/migrations` in order and records them in `__diesel_schema_migrations`, so reruns only apply new migrations. - `storage-init` creates the three buckets the server expects. - The server image is built from `build/service_cpu.Dockerfile`. - `.env.example` lists every variable the server requires at startup, with placeholders for third-party providers. SeaweedFS is used because MinIO no longer publishes Docker images. **Tested locally (Docker Desktop on Windows):** image build, all 285 migrations (plus a rerun), `GET /_status` healthy, model listing, account creation and login, and an image upload stored in the bucket. **Not included:** background workers, Elasticsearch index setup. --- build/selfhost/.env.example | 59 +++++++++++ build/selfhost/README.md | 77 ++++++++++++++ build/selfhost/docker-compose.yml | 163 ++++++++++++++++++++++++++++++ 3 files changed, 299 insertions(+) create mode 100644 build/selfhost/.env.example create mode 100644 build/selfhost/README.md create mode 100644 build/selfhost/docker-compose.yml diff --git a/build/selfhost/.env.example b/build/selfhost/.env.example new file mode 100644 index 000000000..77c5891ea --- /dev/null +++ b/build/selfhost/.env.example @@ -0,0 +1,59 @@ +# Copy this file to `.env` and change the values. +# `.env` is ignored by git; never commit real keys. + +# ----- Local infrastructure (used by docker-compose.yml) ----- + +MYSQL_ROOT_PASSWORD=change-me-root +MYSQL_PASSWORD=change-me + +# Credentials for the local S3-compatible storage (SeaweedFS). +S3_ACCESS_KEY=artcraft +S3_SECRET_KEY=change-me-please + +# ----- storyteller-web settings ----- + +# Cookies are served over plain HTTP locally. +COOKIE_SECURE=false +COOKIE_DOMAIN=localhost +COOKIE_SECRET=change-me +SORT_KEY_SECRET=change-me +WEBSITE_HOMEPAGE_REDIRECT=http://localhost:4201 + +ENABLE_PAGING=false +NUM_WORKERS=4 + +# ----- Third-party providers ----- +# storyteller-web requires these to be set at startup. Placeholders let the server +# start; features backed by a provider only work once you add a real key. + +FAL_API_KEY=unset +FAL_WEBHOOK_URL=http://localhost:12345/v1/webhooks/fal +GMICLOUD_API_KEY=unset +GROK_API_KEY=unset +OPENAI_API_KEY=unset +WORLDLABS_API_KEY=unset +BEEBLE_API_KEY=unset +BEEBLE_WEBHOOK_URL=http://localhost:12345/v1/webhooks/beeble +SEEDANCE2PRO_COOKIES=unset +SEEDANCE2PRO_WHITELIST_COOKIES=unset +SEEDANCE2PRO_BYTEPLUS_ULTRA_COOKIES=unset + +# Email (Resend) +RESEND_API_KEY=unset + +# ----- Stripe (billing) ----- + +STRIPE_SECRET_KEY=unset +STRIPE_SECRET_WEBHOOK_SIGNING_KEY=unset +STRIPE_FAKEYOU_ACCOUNT_ID=unset +STRIPE_PORTAL_DEFAULT_CONFIG_ID=unset +STRIPE_CHECKOUT_SUCCESS_URL_PATH=/checkout_success +STRIPE_CHECKOUT_CANCEL_URL_PATH=/checkout_cancel +STRIPE_PORTAL_RETURN_URL_PATH=/portal_success + +STRIPE_ARTCRAFT_ACCOUNT_ID=unset +STRIPE_ARTCRAFT_SECRET_KEY=unset +STRIPE_ARTCRAFT_SECRET_WEBHOOK_KEY=unset +STRIPE_ARTCRAFT_CHECKOUT_SUCCESS_URL=http://localhost:4201/checkout_success +STRIPE_ARTCRAFT_CHECKOUT_CANCEL_URL=http://localhost:4201/checkout_cancel +STRIPE_ARTCRAFT_PORTAL_RETURN_URL=http://localhost:4201/portal_closed diff --git a/build/selfhost/README.md b/build/selfhost/README.md new file mode 100644 index 000000000..b9f7b13d5 --- /dev/null +++ b/build/selfhost/README.md @@ -0,0 +1,77 @@ +# Self-hosting with Docker Compose + +This directory runs the ArtCraft backend (`storyteller-web`) and the services it +depends on with Docker Compose. + +| Service | Image | Purpose | +|-------------------|--------------------------|---------------------------------------------| +| `storyteller-web` | built from this repo | HTTP API, on `http://localhost:12345` | +| `mysql` | `mysql:8.4` | Accounts, media metadata, jobs, billing | +| `migrate` | `mysql:8.4` | Applies `_database/sql/migrations`, exits | +| `redis` | `redis:7` | Caching, rate limiting, job progress | +| `elasticsearch` | `elasticsearch:8.15.3` | Search | +| `seaweedfs` | `chrislusf/seaweedfs` | S3-compatible storage for media | +| `storage-init` | `chrislusf/seaweedfs` | Creates the storage buckets, exits | + +Only the API port (`12345`) is published to the host. + +## Requirements + +- Docker with Docker Compose v2.23 or newer. +- About 8 GB of memory for Docker and 20 GB of free disk. The first build + compiles the Rust workspace in release mode and can take a long time. + +## Usage + +From this directory: + +```bash +cp .env.example .env # then edit the passwords and secrets +docker compose up --build -d +``` + +Check that the server is up: + +```bash +curl http://localhost:12345/_status +``` + +Stop everything with `docker compose down`. Add `-v` to also delete the +database, search index and stored media. + +## Configuration + +Settings live in `.env`, which git ignores. `docker-compose.yml` sets the +connection settings for MySQL, Redis, Elasticsearch and storage, so you only +need to change passwords and secrets. + +`storyteller-web` requires credentials for every third-party provider (FAL, +OpenAI, Stripe, Resend and others) to be present at startup. `.env.example` +sets them to the placeholder `unset`, which lets the server start. Features +that call a provider only work after you add a real key for it. + +Provider webhooks, such as `FAL_WEBHOOK_URL`, must be reachable from the +internet for asynchronous generation results to arrive. + +## Database migrations + +The `migrate` service applies every migration in `_database/sql/migrations` in +order and records each one in `__diesel_schema_migrations`, the table the diesel +CLI uses. It runs on every `docker compose up` and only applies migrations that +have not run yet. + +## Connecting the web app + +To point the browser app at this server, run it from `frontend` as described in +the repository README: + +```bash +VITE_USE_LOCAL_API=true npx nx dev artcraft-webapp +``` + +## Limitations + +- Only the HTTP API is included. Background workers (provider polling, video + thumbnails, email, analytics) are not started. +- Elasticsearch starts with no indexes. Index definitions are in + `_database/elasticsearch/index_definitions`. diff --git a/build/selfhost/docker-compose.yml b/build/selfhost/docker-compose.yml new file mode 100644 index 000000000..048c501d7 --- /dev/null +++ b/build/selfhost/docker-compose.yml @@ -0,0 +1,163 @@ +# Self-hosted ArtCraft backend (storyteller-web) with its dependencies. +# +# Usage (from this directory): +# cp .env.example .env +# docker compose up --build +# +# See README.md in this directory for details. + +name: artcraft-selfhost + +services: + mysql: + image: mysql:8.4 + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: storyteller + MYSQL_USER: storyteller + MYSQL_PASSWORD: ${MYSQL_PASSWORD} + volumes: + - mysql-data:/var/lib/mysql + healthcheck: + # NB: Use TCP (127.0.0.1), not the socket, so the check fails while the image's + # temporary init server (which has networking disabled) is still running. + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-uroot", "-p${MYSQL_ROOT_PASSWORD}", "--silent"] + interval: 5s + timeout: 5s + retries: 30 + + # Applies the SQL migrations in `_database/sql/migrations` in order. + # Applied versions are recorded in `__diesel_schema_migrations` (the same table + # the diesel CLI uses), so re-running is safe and only applies new migrations. + migrate: + image: mysql:8.4 + depends_on: + mysql: + condition: service_healthy + environment: + MYSQL_PWD: ${MYSQL_ROOT_PASSWORD} + volumes: + - ../../_database/sql/migrations:/migrations:ro + entrypoint: ["bash", "-c"] + command: + - | + set -euo pipefail + run_sql() { mysql -h mysql -u root storyteller "$$@"; } + run_sql -e "CREATE TABLE IF NOT EXISTS __diesel_schema_migrations ( + version VARCHAR(50) PRIMARY KEY NOT NULL, + run_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)" + for dir in /migrations/*/; do + name=$$(basename "$$dir") + version=$$(echo "$${name%%_*}" | tr -d '-') + if [ -n "$$(run_sql -N -e "SELECT 1 FROM __diesel_schema_migrations WHERE version = '$$version'")" ]; then + continue + fi + echo "Applying migration $$name" + run_sql < "$$dir/up.sql" + run_sql -e "INSERT INTO __diesel_schema_migrations (version) VALUES ('$$version')" + done + echo "Migrations complete." + + redis: + image: redis:7 + restart: unless-stopped + volumes: + - redis-data:/data + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3 + restart: unless-stopped + environment: + discovery.type: single-node + xpack.security.enabled: "false" + ES_JAVA_OPTS: -Xms512m -Xmx512m + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + healthcheck: + test: ["CMD-SHELL", "curl -fs http://localhost:9200/_cluster/health || exit 1"] + interval: 10s + timeout: 5s + retries: 30 + + # S3-compatible object storage (SeaweedFS) for uploaded and generated media. + seaweedfs: + image: chrislusf/seaweedfs:3.97 + restart: unless-stopped + command: server -dir=/data -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json + configs: + - source: seaweedfs-s3 + target: /etc/seaweedfs/s3.json + volumes: + - seaweedfs-data:/data + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz || exit 1"] + interval: 5s + timeout: 5s + retries: 30 + + # Creates the buckets storyteller-web expects. + storage-init: + image: chrislusf/seaweedfs:3.97 + depends_on: + seaweedfs: + condition: service_healthy + entrypoint: ["sh", "-c"] + command: + - | + set -e + for bucket in artcraft-private artcraft-public artcraft-public-gc; do + echo "s3.bucket.create -name $$bucket" | weed shell -master=seaweedfs:9333 + done + + storyteller-web: + build: + context: ../.. + dockerfile: build/service_cpu.Dockerfile + image: artcraft-services:local + restart: unless-stopped + depends_on: + migrate: + condition: service_completed_successfully + storage-init: + condition: service_completed_successfully + redis: + condition: service_started + elasticsearch: + condition: service_healthy + env_file: .env + environment: + SERVER_ENVIRONMENT: development + BIND_ADDRESS: 0.0.0.0:12345 + MYSQL_URL: mysql://storyteller:${MYSQL_PASSWORD}@mysql:3306/storyteller + REDIS_0_URL: redis://redis:6379/0 + ELASTICSEARCH_URL: http://elasticsearch:9200 + S3_COMPATIBLE_ENDPOINT_URL: http://seaweedfs:8333 + ACCESS_KEY: ${S3_ACCESS_KEY} + SECRET_KEY: ${S3_SECRET_KEY} + REGION_NAME: us-east-1 + W2L_PRIVATE_DOWNLOAD_BUCKET_NAME: artcraft-private + W2L_PUBLIC_DOWNLOAD_BUCKET_NAME: artcraft-public + GC_ENABLED_PUBLIC_BUCKET_NAME: artcraft-public-gc + AUDIO_UPLOADS_BUCKET_ROOT: /user_uploaded_audio + ports: + - "12345:12345" + +configs: + seaweedfs-s3: + content: | + { + "identities": [ + { + "name": "artcraft", + "credentials": [{ "accessKey": "${S3_ACCESS_KEY}", "secretKey": "${S3_SECRET_KEY}" }], + "actions": ["Admin", "Read", "Write", "List", "Tagging"] + } + ] + } + +volumes: + mysql-data: + redis-data: + elasticsearch-data: + seaweedfs-data: