diff --git a/README.md b/README.md index fb76b5c..e220f98 100644 --- a/README.md +++ b/README.md @@ -39,37 +39,41 @@ Generate a reusable auth key from https://login.tailscale.com/admin/settings/key 4. Override sensitive values (`CHANGE_ME`) in the environment variables section: - `TS_AUTHKEY` - Tailscale auth key (reusable recommended) - `DB_PASSWORD` - PostgreSQL password + - `S3_ACCESS_KEY` - Garage S3 access key + - `S3_SECRET_KEY` - Garage S3 secret key 5. Deploy ## Access Once deployed: https://prefect.taila5ad8.ts.net -## Architecture +## Services -``` -┌─────────────────────────────────────────────────────────────┐ -│ Tailscale Network │ -├─────────────────────────────────────────────────────────────┤ -│ │ -│ ┌─────────────┐ ┌─────────────┐ ┌──────────────┐ │ -│ │ Tailscale │────▶│ Prefect │────▶│ PostgreSQL │ │ -│ │ Sidecar │ │ Server │ │ (external) │ │ -│ │ :443 │ │ :4200 │ │ │ │ -│ └─────────────┘ └──────┬──────┘ └──────────────┘ │ -│ │ │ -│ ┌──────▼──────┐ │ -│ │ Redis │ │ -│ │ (messaging)│ │ -│ └─────────────┘ │ -│ │ -│ ┌─────────────┐ │ -│ │ Prefect │ │ -│ │ Services │ │ -│ │ (background)│ │ -│ └─────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────┘ +| Service | Description | Image | +|---------|-------------|-------| +| `tailscale` | HTTPS ingress via Tailscale | `tailscale/tailscale` | +| `redis` | Messaging broker | `redis:7-alpine` | +| `prefect-server` | API + UI | `prefecthq/prefect:3-latest` | +| `prefect-services` | Background services | `prefecthq/prefect:3-latest` | +| `prefect-worker-pg-backup` | Worker for PostgreSQL backups | `gitea.arnodo.fr/damien/prefect-worker-pg-backup` | + +## Work Pools + +The `prefect-worker-pg-backup` service automatically creates and listens to the `pg-backup-pool` work pool (type: process). + +To deploy a flow to this pool: + +```python +from prefect import flow + +@flow +def my_backup_flow(): + ... + +my_backup_flow.deploy( + name="my-backup", + work_pool_name="pg-backup-pool" +) ``` ## Directory Structure @@ -79,13 +83,3 @@ Once deployed: https://prefect.taila5ad8.ts.net └── tailscale/ └── serve-config.json # Tailscale HTTPS configuration ``` - -## Post-Deployment: Create Work Pool - -After the server is running, create a work pool for your workflows: - -```bash -export PREFECT_API_URL=https://prefect.taila5ad8.ts.net/api -prefect work-pool create backup-pool --type process -``` - diff --git a/docker-compose.yml b/docker-compose.yml index f2a4ea6..8610a93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,6 +70,23 @@ services: command: prefect server services start restart: unless-stopped + # === PREFECT WORKER - PostgreSQL Backup === + prefect-worker-pg-backup: + image: gitea.arnodo.fr/damien/prefect-worker-pg-backup:latest + container_name: prefect-worker-pg-backup + network_mode: service:tailscale + depends_on: + - prefect-server + environment: + # Prefect API connection (via Tailscale) + - PREFECT_API_URL=http://localhost:4200/api + # S3 credentials for Garage + - AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY} + - AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY} + - AWS_ENDPOINT_URL=${S3_ENDPOINT_URL} + command: prefect worker start --pool pg-backup-pool --type process + restart: unless-stopped + volumes: tailscale-state: redis-data: diff --git a/stack.env b/stack.env index 06df7ef..dbe9b6c 100644 --- a/stack.env +++ b/stack.env @@ -1,9 +1,9 @@ -# ============================================================================== +# ============================================================================= # Prefect Stack Environment Variables -# ============================================================================== +# ============================================================================= # This file is used by Portainer when deploying from Git repository. # Sensitive values (marked CHANGE_ME) must be set in Portainer UI. -# ============================================================================== +# ============================================================================= # Tailscale TS_AUTHKEY=CHANGE_ME @@ -13,3 +13,8 @@ DB_HOST=postgresql.taila5ad8.ts.net DB_PORT=5432 DB_USER=prefect DB_PASSWORD=CHANGE_ME + +# S3 Storage (Garage) - for backup worker +S3_ACCESS_KEY=CHANGE_ME +S3_SECRET_KEY=CHANGE_ME +S3_ENDPOINT_URL=https://s3.taila5ad8.ts.net