Merge pull request 'feat: add Prefect worker for PostgreSQL backups' (#1) from feature/add-pg-backup-worker into main

Reviewed-on: #1

README.md

@@ -39,37 +39,41 @@ Generate a reusable auth key from https://login.tailscale.com/admin/settings/key
 4. Override sensitive values (`CHANGE_ME`) in the environment variables section:
    - `TS_AUTHKEY` - Tailscale auth key (reusable recommended)
    - `DB_PASSWORD` - PostgreSQL password
+   - `S3_ACCESS_KEY` - Garage S3 access key
+   - `S3_SECRET_KEY` - Garage S3 secret key
 5. Deploy
 
 ## Access
 
 Once deployed: https://prefect.taila5ad8.ts.net
 
-## Architecture
+## Services
 
-```
+| Service | Description | Image |
-┌─────────────────────────────────────────────────────────────┐
+|---------|-------------|-------|
-│                     Tailscale Network                       │
+| `tailscale` | HTTPS ingress via Tailscale | `tailscale/tailscale` |
-├─────────────────────────────────────────────────────────────┤
+| `redis` | Messaging broker | `redis:7-alpine` |
-│                                                             │
+| `prefect-server` | API + UI | `prefecthq/prefect:3-latest` |
-│  ┌─────────────┐     ┌─────────────┐     ┌──────────────┐  │
+| `prefect-services` | Background services | `prefecthq/prefect:3-latest` |
-│  │  Tailscale  │────▶│   Prefect   │────▶│  PostgreSQL  │  │
+| `prefect-worker-pg-backup` | Worker for PostgreSQL backups | `gitea.arnodo.fr/damien/prefect-worker-pg-backup` |
-│  │   Sidecar   │     │   Server    │     │  (external)  │  │
+
-│  │   :443      │     │   :4200     │     │              │  │
+## Work Pools
-│  └─────────────┘     └──────┬──────┘     └──────────────┘  │
+
-│                             │                               │
+The `prefect-worker-pg-backup` service automatically creates and listens to the `pg-backup-pool` work pool (type: process).
-│                      ┌──────▼──────┐                       │
+
-│                      │    Redis    │                       │
+To deploy a flow to this pool:
-│                      │  (messaging)│                       │
+
-│                      └─────────────┘                       │
+```python
-│                                                             │
+from prefect import flow
-│                      ┌─────────────┐                       │
+
-│                      │   Prefect   │                       │
+@flow
-│                      │  Services   │                       │
+def my_backup_flow():
-│                      │ (background)│                       │
+    ...
-│                      └─────────────┘                       │
+
-│                                                             │
+my_backup_flow.deploy(
-└─────────────────────────────────────────────────────────────┘
+    name="my-backup",
+    work_pool_name="pg-backup-pool"
+)
 ```
 
 ## Directory Structure
@@ -79,13 +83,3 @@ Once deployed: https://prefect.taila5ad8.ts.net
 └── tailscale/
     └── serve-config.json     # Tailscale HTTPS configuration
 ```
-
-## Post-Deployment: Create Work Pool
-
-After the server is running, create a work pool for your workflows:
-
-```bash
-export PREFECT_API_URL=https://prefect.taila5ad8.ts.net/api
-prefect work-pool create backup-pool --type process
-```
-

docker-compose.yml

@@ -70,6 +70,23 @@ services:
     command: prefect server services start
     restart: unless-stopped
 
+  # === PREFECT WORKER - PostgreSQL Backup ===
+  prefect-worker-pg-backup:
+    image: gitea.arnodo.fr/damien/prefect-worker-pg-backup:latest
+    container_name: prefect-worker-pg-backup
+    network_mode: service:tailscale
+    depends_on:
+      - prefect-server
+    environment:
+      # Prefect API connection (via Tailscale)
+      - PREFECT_API_URL=http://localhost:4200/api
+      # S3 credentials for Garage
+      - AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY}
+      - AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY}
+      - AWS_ENDPOINT_URL=${S3_ENDPOINT_URL}
+    command: prefect worker start --pool pg-backup-pool --type process
+    restart: unless-stopped
+
 volumes:
   tailscale-state:
   redis-data:

stack.env

@@ -1,9 +1,9 @@
-# ==============================================================================
+# =============================================================================
 # Prefect Stack Environment Variables
-# ==============================================================================
+# =============================================================================
 # This file is used by Portainer when deploying from Git repository.
 # Sensitive values (marked CHANGE_ME) must be set in Portainer UI.
-# ==============================================================================
+# =============================================================================
 
 # Tailscale
 TS_AUTHKEY=CHANGE_ME
@@ -13,3 +13,8 @@ DB_HOST=postgresql.taila5ad8.ts.net
 DB_PORT=5432
 DB_USER=prefect
 DB_PASSWORD=CHANGE_ME
+
+# S3 Storage (Garage) - for backup worker
+S3_ACCESS_KEY=CHANGE_ME
+S3_SECRET_KEY=CHANGE_ME
+S3_ENDPOINT_URL=https://s3.taila5ad8.ts.net