From beb84cf11d421db4cfb77f77866fab36bae68d1c Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Sat, 31 Jan 2026 15:05:01 +0000 Subject: [PATCH 1/4] refactor: remove worker service, use Prefect Blocks for secrets --- docker-compose.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7043ae9..e0259db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,23 +70,6 @@ services: command: prefect server services start restart: unless-stopped - # === PREFECT WORKER - PostgreSQL Backup === - prefect-worker-pg-backup: - image: gitea.arnodo.fr/damien/prefect-worker-pg-backup:latest - container_name: prefect-worker-pg-backup - network_mode: service:tailscale - depends_on: - - prefect-server - environment: - # Prefect API connection (via Tailscale) - - PREFECT_API_URL=http://localhost:4200/api - # S3 credentials for Garage - - AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY} - - AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY} - - AWS_ENDPOINT_URL=${S3_ENDPOINT_URL} - command: prefect worker start --pool pg-backup-pool --type process - restart: unless-stopped - volumes: tailscale-state: redis-data: -- 2.53.0 From 7bf75f85e19c7553ebc213bb524569d53ea23588 Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Sat, 31 Jan 2026 15:05:21 +0000 Subject: [PATCH 2/4] docs: update README - remove S3 vars, document Blocks for secrets --- README.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8519d00..1bdd9b6 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,6 @@ In the stack configuration, add the following environment variables: | `DB_PORT` | PostgreSQL port | `5432` | | `DB_USER` | Database user | `prefect` | | `DB_PASSWORD` | Database password | *secret* | -| `S3_ACCESS_KEY` | Garage S3 access key | *secret* | -| `S3_SECRET_KEY` | Garage S3 secret key | *secret* | -| `S3_ENDPOINT_URL` | Garage S3 endpoint | `https://s3.taila5ad8.ts.net` | > **Tip**: Use Komodo's secret variables (marked with 🔒) for sensitive values. @@ -82,25 +79,34 @@ Once deployed: https://prefect.taila5ad8.ts.net | `redis` | Messaging broker | `redis:7-alpine` | | `prefect-server` | API + UI | `prefecthq/prefect:3-latest` | | `prefect-services` | Background services | `prefecthq/prefect:3-latest` | -| `prefect-worker-pg-backup` | Worker for PostgreSQL backups | `gitea.arnodo.fr/damien/prefect-worker-pg-backup` | -## Work Pools +## Workers & Secrets -The `prefect-worker-pg-backup` service automatically creates and listens to the `pg-backup-pool` work pool (type: process). +Workers are deployed separately with their own docker-compose. Flow-specific secrets (S3 credentials, API keys, etc.) should be managed via **Prefect Blocks**, not environment variables. -To deploy a flow to this pool: +### Creating a Block (example with S3/Garage) + +```python +from prefect_aws import AwsCredentials + +creds = AwsCredentials( + aws_access_key_id="xxx", + aws_secret_access_key="xxx", + aws_endpoint_url="https://s3.taila5ad8.ts.net" +) +creds.save("garage-credentials") +``` + +### Using in a flow ```python from prefect import flow +from prefect_aws import AwsCredentials @flow def my_backup_flow(): - ... - -my_backup_flow.deploy( - name="my-backup", - work_pool_name="pg-backup-pool" -) + creds = AwsCredentials.load("garage-credentials") + # use creds... ``` ## Repository Structure -- 2.53.0 From 115a690d8f0591c8ac02c06792516bb29f004b72 Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Sat, 31 Jan 2026 15:07:55 +0000 Subject: [PATCH 3/4] refactor: keep worker, remove S3 secrets (use Prefect Blocks instead) --- docker-compose.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index e0259db..5c476ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,6 +70,18 @@ services: command: prefect server services start restart: unless-stopped + # === PREFECT WORKER - PostgreSQL Backup === + prefect-worker-pg-backup: + image: gitea.arnodo.fr/damien/prefect-worker-pg-backup:latest + container_name: prefect-worker-pg-backup + network_mode: service:tailscale + depends_on: + - prefect-server + environment: + - PREFECT_API_URL=http://localhost:4200/api + command: prefect worker start --pool pg-backup-pool --type process + restart: unless-stopped + volumes: tailscale-state: redis-data: -- 2.53.0 From 6fa6ba549666945329ba446e5ee8a9e5dd8ea3b0 Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Sat, 31 Jan 2026 15:08:29 +0000 Subject: [PATCH 4/4] docs: update README - keep worker, use Blocks for flow secrets --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1bdd9b6..d87b8dd 100644 --- a/README.md +++ b/README.md @@ -79,10 +79,15 @@ Once deployed: https://prefect.taila5ad8.ts.net | `redis` | Messaging broker | `redis:7-alpine` | | `prefect-server` | API + UI | `prefecthq/prefect:3-latest` | | `prefect-services` | Background services | `prefecthq/prefect:3-latest` | +| `prefect-worker-pg-backup` | Worker for PostgreSQL backups | `gitea.arnodo.fr/damien/prefect-worker-pg-backup` | -## Workers & Secrets +## Work Pools -Workers are deployed separately with their own docker-compose. Flow-specific secrets (S3 credentials, API keys, etc.) should be managed via **Prefect Blocks**, not environment variables. +The `prefect-worker-pg-backup` service automatically creates and listens to the `pg-backup-pool` work pool (type: process). + +## Secrets Management + +Flow-specific secrets (S3 credentials, database passwords, API keys, etc.) should be managed via **Prefect Blocks**, not environment variables in the compose file. ### Creating a Block (example with S3/Garage) @@ -107,6 +112,11 @@ from prefect_aws import AwsCredentials def my_backup_flow(): creds = AwsCredentials.load("garage-credentials") # use creds... + +my_backup_flow.deploy( + name="my-backup", + work_pool_name="pg-backup-pool" +) ``` ## Repository Structure -- 2.53.0