Compare commits
10 Commits
b739f9177b
...
refactor/r
| Author | SHA1 | Date | |
|---|---|---|---|
| 6fa6ba5496 | |||
| 115a690d8f | |||
| 7bf75f85e1 | |||
| beb84cf11d | |||
| 0493b81108 | |||
| 2ed2bbb15b | |||
| 2411e6ef26 | |||
| 3d37355b67 | |||
| 99611f7c7e | |||
| ddd4224f92 |
98
README.md
98
README.md
@@ -1,6 +1,6 @@
|
|||||||
# Prefect Deployment
|
# Prefect Deployment
|
||||||
|
|
||||||
GitOps deployment for Prefect workflow orchestration with Tailscale HTTPS access.
|
GitOps deployment for Prefect workflow orchestration with Tailscale HTTPS access via Komodo.
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
@@ -13,35 +13,59 @@ CREATE DATABASE prefect;
|
|||||||
CREATE USER prefect WITH PASSWORD 'your-secure-password';
|
CREATE USER prefect WITH PASSWORD 'your-secure-password';
|
||||||
GRANT ALL PRIVILEGES ON DATABASE prefect TO prefect;
|
GRANT ALL PRIVILEGES ON DATABASE prefect TO prefect;
|
||||||
|
|
||||||
-- PostgreSQL 15+ requires:
|
|
||||||
\c prefect
|
\c prefect
|
||||||
|
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||||
GRANT ALL ON SCHEMA public TO prefect;
|
GRANT ALL ON SCHEMA public TO prefect;
|
||||||
```
|
```
|
||||||
|
|
||||||
### Host Configuration
|
|
||||||
|
|
||||||
Download configuration files to `/opt/prefect` (first time only):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo mkdir -p /opt/prefect/tailscale
|
|
||||||
sudo curl -o /opt/prefect/tailscale/serve-config.json https://gitea.arnodo.fr/Damien/prefect-deployment/raw/branch/main/serve-config.json
|
|
||||||
```
|
|
||||||
|
|
||||||
### Tailscale Auth Key
|
### Tailscale Auth Key
|
||||||
|
|
||||||
Generate a reusable auth key from https://login.tailscale.com/admin/settings/keys
|
Generate a reusable auth key from https://login.tailscale.com/admin/settings/keys
|
||||||
|
|
||||||
## Deployment
|
## Deployment with Komodo
|
||||||
|
|
||||||
1. Create a new stack in Portainer
|
### 1. Add Git Provider (if using private repo)
|
||||||
2. Select "Repository" and point to this repository
|
|
||||||
3. Portainer will load `stack.env` automatically
|
In Komodo UI: Settings → Git Providers → Add your Gitea instance credentials.
|
||||||
4. Override sensitive values (`CHANGE_ME`) in the environment variables section:
|
|
||||||
- `TS_AUTHKEY` - Tailscale auth key (reusable recommended)
|
### 2. Create Stack
|
||||||
- `DB_PASSWORD` - PostgreSQL password
|
|
||||||
- `S3_ACCESS_KEY` - Garage S3 access key
|
1. Navigate to **Stacks** → **New Stack**
|
||||||
- `S3_SECRET_KEY` - Garage S3 secret key
|
2. Configure:
|
||||||
5. Deploy
|
- **Name**: `prefect`
|
||||||
|
- **Server**: Select your target server
|
||||||
|
- **Source**: Git Repo
|
||||||
|
- **Git Provider**: `gitea.arnodo.fr` (or your provider)
|
||||||
|
- **Repo**: `Damien/prefect-deployment`
|
||||||
|
- **Branch**: `main`
|
||||||
|
- **File Paths**: `docker-compose.yml`
|
||||||
|
|
||||||
|
### 3. Configure Environment Variables
|
||||||
|
|
||||||
|
In the stack configuration, add the following environment variables:
|
||||||
|
|
||||||
|
| Variable | Description | Example |
|
||||||
|
|----------|-------------|---------|
|
||||||
|
| `TS_AUTHKEY` | Tailscale auth key (reusable) | `tskey-auth-xxx` |
|
||||||
|
| `DB_HOST` | PostgreSQL host | `postgresql.taila5ad8.ts.net` |
|
||||||
|
| `DB_PORT` | PostgreSQL port | `5432` |
|
||||||
|
| `DB_USER` | Database user | `prefect` |
|
||||||
|
| `DB_PASSWORD` | Database password | *secret* |
|
||||||
|
|
||||||
|
> **Tip**: Use Komodo's secret variables (marked with 🔒) for sensitive values.
|
||||||
|
|
||||||
|
### 4. Deploy
|
||||||
|
|
||||||
|
Click **Deploy** in Komodo. The stack will clone the repository and start all services.
|
||||||
|
|
||||||
|
## GitOps Workflow
|
||||||
|
|
||||||
|
### Auto-deploy on Git Push
|
||||||
|
|
||||||
|
1. In Komodo, go to your stack settings
|
||||||
|
2. Enable **Auto Deploy** on push
|
||||||
|
3. Copy the webhook URL
|
||||||
|
4. Add it to your Gitea repository: Settings → Webhooks
|
||||||
|
|
||||||
## Access
|
## Access
|
||||||
|
|
||||||
@@ -61,14 +85,33 @@ Once deployed: https://prefect.taila5ad8.ts.net
|
|||||||
|
|
||||||
The `prefect-worker-pg-backup` service automatically creates and listens to the `pg-backup-pool` work pool (type: process).
|
The `prefect-worker-pg-backup` service automatically creates and listens to the `pg-backup-pool` work pool (type: process).
|
||||||
|
|
||||||
To deploy a flow to this pool:
|
## Secrets Management
|
||||||
|
|
||||||
|
Flow-specific secrets (S3 credentials, database passwords, API keys, etc.) should be managed via **Prefect Blocks**, not environment variables in the compose file.
|
||||||
|
|
||||||
|
### Creating a Block (example with S3/Garage)
|
||||||
|
|
||||||
|
```python
|
||||||
|
from prefect_aws import AwsCredentials
|
||||||
|
|
||||||
|
creds = AwsCredentials(
|
||||||
|
aws_access_key_id="xxx",
|
||||||
|
aws_secret_access_key="xxx",
|
||||||
|
aws_endpoint_url="https://s3.taila5ad8.ts.net"
|
||||||
|
)
|
||||||
|
creds.save("garage-credentials")
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using in a flow
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from prefect import flow
|
from prefect import flow
|
||||||
|
from prefect_aws import AwsCredentials
|
||||||
|
|
||||||
@flow
|
@flow
|
||||||
def my_backup_flow():
|
def my_backup_flow():
|
||||||
...
|
creds = AwsCredentials.load("garage-credentials")
|
||||||
|
# use creds...
|
||||||
|
|
||||||
my_backup_flow.deploy(
|
my_backup_flow.deploy(
|
||||||
name="my-backup",
|
name="my-backup",
|
||||||
@@ -76,10 +119,11 @@ my_backup_flow.deploy(
|
|||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
## Directory Structure
|
## Repository Structure
|
||||||
|
|
||||||
```
|
```
|
||||||
/opt/prefect/
|
prefect-deployment/
|
||||||
└── tailscale/
|
├── docker-compose.yml # Stack definition (relative paths)
|
||||||
└── serve-config.json # Tailscale HTTPS configuration
|
├── serve-config.json # Tailscale HTTPS serve config
|
||||||
|
└── README.md
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ services:
|
|||||||
- TS_SERVE_CONFIG=/config/serve-config.json
|
- TS_SERVE_CONFIG=/config/serve-config.json
|
||||||
volumes:
|
volumes:
|
||||||
- tailscale-state:/var/lib/tailscale
|
- tailscale-state:/var/lib/tailscale
|
||||||
- /opt/prefect/tailscale/serve-config.json:/config/serve-config.json:ro
|
- ./serve-config.json:/config/serve-config.json:ro
|
||||||
cap_add:
|
cap_add:
|
||||||
- NET_ADMIN
|
- NET_ADMIN
|
||||||
- SYS_MODULE
|
- SYS_MODULE
|
||||||
@@ -78,12 +78,7 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
- prefect-server
|
- prefect-server
|
||||||
environment:
|
environment:
|
||||||
# Prefect API connection (via Tailscale)
|
|
||||||
- PREFECT_API_URL=http://localhost:4200/api
|
- PREFECT_API_URL=http://localhost:4200/api
|
||||||
# S3 credentials for Garage
|
|
||||||
- AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY}
|
|
||||||
- AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY}
|
|
||||||
- AWS_ENDPOINT_URL=${S3_ENDPOINT_URL}
|
|
||||||
command: prefect worker start --pool pg-backup-pool --type process
|
command: prefect worker start --pool pg-backup-pool --type process
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
|
|||||||
20
stack.env
20
stack.env
@@ -1,20 +0,0 @@
|
|||||||
# =============================================================================
|
|
||||||
# Prefect Stack Environment Variables
|
|
||||||
# =============================================================================
|
|
||||||
# This file is used by Portainer when deploying from Git repository.
|
|
||||||
# Sensitive values (marked CHANGE_ME) must be set in Portainer UI.
|
|
||||||
# =============================================================================
|
|
||||||
|
|
||||||
# Tailscale
|
|
||||||
TS_AUTHKEY=CHANGE_ME
|
|
||||||
|
|
||||||
# PostgreSQL (external database)
|
|
||||||
DB_HOST=postgresql.taila5ad8.ts.net
|
|
||||||
DB_PORT=5432
|
|
||||||
DB_USER=prefect
|
|
||||||
DB_PASSWORD=CHANGE_ME
|
|
||||||
|
|
||||||
# S3 Storage (Garage) - for backup worker
|
|
||||||
S3_ACCESS_KEY=CHANGE_ME
|
|
||||||
S3_SECRET_KEY=CHANGE_ME
|
|
||||||
S3_ENDPOINT_URL=https://s3.taila5ad8.ts.net
|
|
||||||
Reference in New Issue
Block a user