From 6a924a0360d67566485b021c003eb55e40d3edaf Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Sat, 31 Jan 2026 11:39:13 +0000 Subject: [PATCH 1/4] feat: add prefect worker for PostgreSQL backups --- docker-compose.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index f2a4ea6..8610a93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,6 +70,23 @@ services: command: prefect server services start restart: unless-stopped + # === PREFECT WORKER - PostgreSQL Backup === + prefect-worker-pg-backup: + image: gitea.arnodo.fr/damien/prefect-worker-pg-backup:latest + container_name: prefect-worker-pg-backup + network_mode: service:tailscale + depends_on: + - prefect-server + environment: + # Prefect API connection (via Tailscale) + - PREFECT_API_URL=http://localhost:4200/api + # S3 credentials for Garage + - AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY} + - AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY} + - AWS_ENDPOINT_URL=${S3_ENDPOINT_URL} + command: prefect worker start --pool pg-backup-pool --type process + restart: unless-stopped + volumes: tailscale-state: redis-data: -- 2.53.0 From c8e67e7bbbc445431388254f52925b164a1d6f54 Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Sat, 31 Jan 2026 11:39:21 +0000 Subject: [PATCH 2/4] feat: add S3 credentials for backup worker --- stack.env | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/stack.env b/stack.env index 06df7ef..dbe9b6c 100644 --- a/stack.env +++ b/stack.env @@ -1,9 +1,9 @@ -# ============================================================================== +# ============================================================================= # Prefect Stack Environment Variables -# ============================================================================== +# ============================================================================= # This file is used by Portainer when deploying from Git repository. # Sensitive values (marked CHANGE_ME) must be set in Portainer UI. -# ============================================================================== +# ============================================================================= # Tailscale TS_AUTHKEY=CHANGE_ME @@ -13,3 +13,8 @@ DB_HOST=postgresql.taila5ad8.ts.net DB_PORT=5432 DB_USER=prefect DB_PASSWORD=CHANGE_ME + +# S3 Storage (Garage) - for backup worker +S3_ACCESS_KEY=CHANGE_ME +S3_SECRET_KEY=CHANGE_ME +S3_ENDPOINT_URL=https://s3.taila5ad8.ts.net -- 2.53.0 From 18d2c59f7ca4399fddb12a454a31f171fd24853b Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Sat, 31 Jan 2026 11:39:49 +0000 Subject: [PATCH 3/4] docs: update README with worker architecture --- README.md | 90 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index fb76b5c..4029466 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ Generate a reusable auth key from https://login.tailscale.com/admin/settings/key 4. Override sensitive values (`CHANGE_ME`) in the environment variables section: - `TS_AUTHKEY` - Tailscale auth key (reusable recommended) - `DB_PASSWORD` - PostgreSQL password + - `S3_ACCESS_KEY` - Garage S3 access key + - `S3_SECRET_KEY` - Garage S3 secret key 5. Deploy ## Access @@ -48,28 +50,62 @@ Once deployed: https://prefect.taila5ad8.ts.net ## Architecture ``` -┌─────────────────────────────────────────────────────────────┐ -│ Tailscale Network │ -├─────────────────────────────────────────────────────────────┤ -│ │ -│ ┌─────────────┐ ┌─────────────┐ ┌──────────────┐ │ -│ │ Tailscale │────▶│ Prefect │────▶│ PostgreSQL │ │ -│ │ Sidecar │ │ Server │ │ (external) │ │ -│ │ :443 │ │ :4200 │ │ │ │ -│ └─────────────┘ └──────┬──────┘ └──────────────┘ │ -│ │ │ -│ ┌──────▼──────┐ │ -│ │ Redis │ │ -│ │ (messaging)│ │ -│ └─────────────┘ │ -│ │ -│ ┌─────────────┐ │ -│ │ Prefect │ │ -│ │ Services │ │ -│ │ (background)│ │ -│ └─────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────┘ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Tailscale Network │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌─────────────┐ ┌────────────────┐ │ +│ │ Tailscale │◀───▶│ Prefect │◀───▶│ PostgreSQL │ │ +│ │ Sidecar │ │ Server │ │ (external) │ │ +│ │ :443 │ │ :4200 │ │ │ │ +│ └──────────────┘ └──────┬──────┘ └────────────────┘ │ +│ ▲ │ │ +│ │ ┌─────▼─────┐ │ +│ │ │ Redis │ │ +│ │ │(messaging)│ │ +│ │ └───────────┘ │ +│ │ ▲ │ +│ │ ┌─────┴─────┐ │ +│ │ │ Prefect │ │ +│ │ │ Services │ │ +│ │ │(background)│ │ +│ │ └───────────┘ │ +│ │ │ +│ │ ┌─────────────────┐ ┌─────────────┐ │ +│ └─────────────▶│ Prefect Worker │◀───▶│ Garage │ │ +│ │ (pg-backup-pool)│ │ (S3) │ │ +│ └─────────────────┘ └─────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +## Services + +| Service | Description | Image | +|---------|-------------|-------| +| `tailscale` | HTTPS ingress via Tailscale | `tailscale/tailscale` | +| `redis` | Messaging broker | `redis:7-alpine` | +| `prefect-server` | API + UI | `prefecthq/prefect:3-latest` | +| `prefect-services` | Background services | `prefecthq/prefect:3-latest` | +| `prefect-worker-pg-backup` | Worker for PostgreSQL backups | `gitea.arnodo.fr/damien/prefect-worker-pg-backup` | + +## Work Pools + +The `prefect-worker-pg-backup` service automatically creates and listens to the `pg-backup-pool` work pool (type: process). + +To deploy a flow to this pool: + +```python +from prefect import flow + +@flow +def my_backup_flow(): + ... + +my_backup_flow.deploy( + name="my-backup", + work_pool_name="pg-backup-pool" +) ``` ## Directory Structure @@ -79,13 +115,3 @@ Once deployed: https://prefect.taila5ad8.ts.net └── tailscale/ └── serve-config.json # Tailscale HTTPS configuration ``` - -## Post-Deployment: Create Work Pool - -After the server is running, create a work pool for your workflows: - -```bash -export PREFECT_API_URL=https://prefect.taila5ad8.ts.net/api -prefect work-pool create backup-pool --type process -``` - -- 2.53.0 From 99c59cc9a326953a430de8dc13acdc647541a005 Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Sat, 31 Jan 2026 12:09:02 +0000 Subject: [PATCH 4/4] Update README.md --- README.md | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/README.md b/README.md index 4029466..e220f98 100644 --- a/README.md +++ b/README.md @@ -47,38 +47,6 @@ Generate a reusable auth key from https://login.tailscale.com/admin/settings/key Once deployed: https://prefect.taila5ad8.ts.net -## Architecture - -``` -┌─────────────────────────────────────────────────────────────────────────┐ -│ Tailscale Network │ -├─────────────────────────────────────────────────────────────────────────┤ -│ │ -│ ┌──────────────┐ ┌─────────────┐ ┌────────────────┐ │ -│ │ Tailscale │◀───▶│ Prefect │◀───▶│ PostgreSQL │ │ -│ │ Sidecar │ │ Server │ │ (external) │ │ -│ │ :443 │ │ :4200 │ │ │ │ -│ └──────────────┘ └──────┬──────┘ └────────────────┘ │ -│ ▲ │ │ -│ │ ┌─────▼─────┐ │ -│ │ │ Redis │ │ -│ │ │(messaging)│ │ -│ │ └───────────┘ │ -│ │ ▲ │ -│ │ ┌─────┴─────┐ │ -│ │ │ Prefect │ │ -│ │ │ Services │ │ -│ │ │(background)│ │ -│ │ └───────────┘ │ -│ │ │ -│ │ ┌─────────────────┐ ┌─────────────┐ │ -│ └─────────────▶│ Prefect Worker │◀───▶│ Garage │ │ -│ │ (pg-backup-pool)│ │ (S3) │ │ -│ └─────────────────┘ └─────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────────────┘ -``` - ## Services | Service | Description | Image | -- 2.53.0