diff --git a/proxy/README.md b/proxy/README.md index 946c749..9a1e46d 100644 --- a/proxy/README.md +++ b/proxy/README.md @@ -60,3 +60,60 @@ sudo ufw delete allow 22/tcp - Access NPM admin: `https://proxy..ts.net` - Default credentials: `admin@example.com` / `changeme` - Optionally approve exit-node in Tailscale admin console + +## Centralized log reception (rsyslog) + +A fail2ban jail running inside an exposed service's own LXC only ever sees +this proxy's tailnet IP as the connection source, so it would end up +banning the proxy itself instead of the actual client. Detection has to +stay where the signal is (the service's application log); banning has to +happen here, where public connections terminate. Services forward their +logs to this proxy over TCP so a jail here can act on them. + +| File | Purpose | +|------|---------| +| `/etc/rsyslog.d/10-remote-receiver.conf` | Generic `imtcp` listener only (`module`/`input`), port `RSYSLOG_PORT` (default `5514`). No rules here — see why below. | +| `/etc/rsyslog.d/50-.conf` | One per exposed service. Routes by tag/programname into that service's own logfile for its dedicated fail2ban jail, then `stop`s so the message doesn't also fall through to the catch-all. | +| `/etc/rsyslog.d/90-remote-fallback.conf` | Catch-all: anything a `50-.conf` didn't claim (or before one exists yet) lands in `/var/log/remote/.log`. | +| `/etc/logrotate.d/remote-logs` | Rotation for everything under `/var/log/remote/` (`copytruncate`, so fail2ban never loses its file descriptor across a rotation). | + +rsyslog loads `/etc/rsyslog.d/*.conf` in filename order, and rules within a +ruleset run in the order they were loaded — a catch-all in `10-` would fire +on *every* message before a `50-.conf` ever got a look, doubling +every claimed message into both files. Keeping the listener in `10-`, routing +in `50-`, and the catch-all in `90-` puts them in the right order without +depending on load-order accidents. + +Adding a new exposed service is a matter of dropping its `50-.conf` +here — nothing else in this list needs to change. A minimal example that +routes messages tagged `myservice` into their own file instead of the +generic catch-all: + +``` +$RuleSet remoteLogs +if $programname == 'myservice' then { + action(type="omfile" file="/var/log/myservice/myservice.log") + stop +} +$RuleSet RSYSLOG_DefaultRuleset +``` + +Deliberately on the legacy `$RuleSet ` directive rather than the +modern `ruleset(name="...") { ... }` object syntax: rsyslog rejects a named +ruleset declared with that object syntax more than once ("ruleset ... +specified more than once"), which breaks the moment a second +`50-.conf` (or `90-remote-fallback.conf`) tries to add its own +rules to the same `remoteLogs` ruleset. `$RuleSet ` is a context +selector, not a one-shot declaration — any number of files can reopen it to +append rules, which is the entire point of this pattern. + +`RSYSLOG_PORT` and `RSYSLOG_BIND_ADDR` (default `0.0.0.0`) are overridable +via environment. The default bind is safe as-is: UFW's default-deny only +opens `80/tcp` and `443/tcp` publicly, so port `5514` is reachable +exclusively over `tailscale0` regardless of the bind address. Binding to +the tailnet IP directly was considered and rejected — it would require +`tailscale up` to have already succeeded before rsyslog is configured, +which complicates the script's flow (a missing `TS_AUTHKEY` is tolerated +today). The residual risk is log injection from anything that reaches the +port (which can trigger a false fail2ban ban); narrow `RSYSLOG_BIND_ADDR` +to a specific tailnet IP if that risk becomes a concern. diff --git a/proxy/install.sh b/proxy/install.sh index cfca993..b5c73a3 100644 --- a/proxy/install.sh +++ b/proxy/install.sh @@ -44,6 +44,13 @@ ACME_EMAIL="${ACME_EMAIL:-}" # Generate at https://login.tailscale.com/admin/settings/keys TS_AUTHKEY="${TS_AUTHKEY:-}" +# rsyslog receiver for exposed services' logs (see "Configuring rsyslog" below). +RSYSLOG_PORT="${RSYSLOG_PORT:-5514}" +# Default 0.0.0.0 is fine: UFW's default-deny only opens 80/443 publicly, so +# this port is reachable exclusively over tailscale0 either way. Override to +# a specific tailnet IP to shrink the blast radius of log injection instead. +RSYSLOG_BIND_ADDR="${RSYSLOG_BIND_ADDR:-0.0.0.0}" + main() { log_info "=== Proxy Server Deployment (Traefik v3) ===" @@ -75,7 +82,7 @@ main() { log_info "Installing base packages..." sudo apt update -qq - sudo apt install -y -qq vim ca-certificates curl gnupg lsb-release fail2ban unattended-upgrades ufw ethtool networkd-dispatcher > /dev/null + sudo apt install -y -qq vim ca-certificates curl gnupg lsb-release fail2ban unattended-upgrades ufw ethtool networkd-dispatcher rsyslog > /dev/null log_info "Installing Tailscale..." curl -fsSL https://tailscale.com/install.sh | sh @@ -174,6 +181,60 @@ EOF sudo systemctl restart fail2ban + log_info "Configuring rsyslog receiver for exposed services..." + # A fail2ban jail running inside a service's own LXC only ever sees this + # proxy's tailnet IP as the source of connections, so it would end up + # banning the proxy itself. Detection has to stay where the signal is + # (the service's application log); banning has to happen here, at the + # edge where public connections actually terminate. Services forward + # their logs to this receiver over TCP; adding a new one is a matter of + # dropping a 50-.conf here (see proxy/README.md) — nothing else + # to touch. + # + # 10- carries only the listener (module/input): rsyslog loads + # /etc/rsyslog.d/*.conf in filename order, and rules within a ruleset + # execute in the order they were loaded. A catch-all defined here would + # run *before* any 50-.conf's rules ever get a chance — every + # message would double up into both the generic file and the + # service-specific one. The catch-all instead lives in + # 90-remote-fallback.conf (below, written after this block) so it loads + # last, and a service's `stop` actually prevents fallthrough into it. + sudo mkdir -p /var/log/remote + sudo tee /etc/rsyslog.d/10-remote-receiver.conf > /dev/null << EOF +module(load="imtcp") + +input(type="imtcp" port="${RSYSLOG_PORT}" address="${RSYSLOG_BIND_ADDR}" ruleset="remoteLogs") +EOF + + # Catch-all for anything a 50-.conf doesn't claim (or before one + # exists yet). Kept on the legacy $RuleSet/$template directives rather + # than the modern ruleset(name=...){...} object: rsyslog rejects a named + # ruleset declared via that object syntax more than once ("ruleset ... + # specified more than once"), which would break the moment a + # 50-.conf tries to add its own rules to the same "remoteLogs" + # ruleset — the entire point of this split. $RuleSet is a context + # selector, not a one-shot declaration, so any number of files can + # reopen it to append rules. + sudo tee /etc/rsyslog.d/90-remote-fallback.conf > /dev/null << 'EOF' +$RuleSet remoteLogs +$template RemoteLogPath,"/var/log/remote/%HOSTNAME%.log" +*.* ?RemoteLogPath +$RuleSet RSYSLOG_DefaultRuleset +EOF + + sudo tee /etc/logrotate.d/remote-logs > /dev/null << 'EOF' +/var/log/remote/*.log { + daily + rotate 7 + compress + missingok + notifempty + copytruncate +} +EOF + + sudo systemctl restart rsyslog + log_info "Creating Traefik stack under $TRAEFIK_DIR..." mkdir -p "$TRAEFIK_DIR/conf.d"