diff --git a/deploy/prod/README.md b/deploy/prod/README.md new file mode 100644 index 0000000..63f606b --- /dev/null +++ b/deploy/prod/README.md @@ -0,0 +1,74 @@ +# Production deployment (noema.dixon.cx) + +Noema runs as a Docker container supervised by systemd, talks to the host +PostgreSQL 16 cluster over the docker bridge, and sits behind nginx. Access +control is Noema's own sign-in page. Cloudflare proxies the hostname +(wildcard `*.dixon.cx`). + +``` +Cloudflare -> nginx :443 (TLS) -> 127.0.0.1:29630 -> container :8080 -> postgres 172.17.0.1:5432 +``` + +## Deploying a new version + +```sh +git pull +deploy/prod/build.sh +``` + +`build.sh` builds `noema:` and `noema:latest`, then restarts the +`noema` service. Migrations are applied automatically when the daemon starts. +Roll back with `docker tag noema: noema:latest && sudo systemctl restart noema` +(`docker images noema` lists the versions you have). + +## Where things live + +| What | Where | +|---|---| +| systemd unit | `/etc/systemd/system/noema.service` (source: `deploy/prod/noema.service`) | +| Environment and secrets | `/etc/noema/noema.env` (root, 0600): DB URL, master key, bootstrap admin | +| nginx vhost | `/etc/nginx/sites-enabled/noema.dixon.cx.conf` (source: `deploy/prod/noema.dixon.cx.conf`) | +| TLS certificate | `/etc/letsencrypt/live/noema.dixon.cx` (certbot, nginx authenticator, auto-renews) | +| Research folder | `/srv/noema/research` mounted at `/research` (owned by uid 10001) | +| Database | `noema` database, `noema` role on the host cluster; `vector` extension created by `postgres` | +| Logs | `journalctl -u noema -f` (JSON); nginx `/var/log/nginx/noema_*.log` | + +The daemon ignores proxy headers, so it sees every visitor as the docker +gateway address: the login rate limit (20/min) is shared by everyone, and the +per-account lockout is what actually stops password guessing. Use a strong +admin password and enable TOTP on the account page. + +**Back up `NOEMA_MASTER_KEY`.** It encrypts provider API keys stored in the +database; losing it means re-entering every key. + +## Common tasks + +```sh +# noemactl against the running instance +docker exec -it noema noemactl --help + +# psql as the app (from the host use the postgres superuser instead) +sudo -u postgres psql noema +``` + +## Rebuilding from scratch + +Applies to a fresh host; everything below was done once on this one. + +```sh +sudo -u postgres psql -c "CREATE ROLE noema LOGIN PASSWORD '...'" -c "CREATE DATABASE noema OWNER noema" +sudo -u postgres psql -d noema -c "CREATE EXTENSION vector" # needs superuser; migration 0005 expects it +# pg_hba.conf must allow 172.17.0.0/16 and postgresql.conf must listen on 172.17.0.1 + +sudo install -d -m 700 /etc/noema && sudo install -d -o 10001 -g 10001 /srv/noema/research +sudoedit /etc/noema/noema.env # keys as in .env.example, plus: +# NOEMA_ENV=production NOEMA_HTTP_ADDR=0.0.0.0:8080 NOEMA_RESEARCH_DIR=/research +# NOEMA_DATABASE_URL=postgres://noema:PASS@172.17.0.1:5432/noema?sslmode=disable +# NOEMA_MASTER_KEY=$(openssl rand -base64 32) NOEMA_BOOTSTRAP_ADMIN_PASSWORD=... + +sudo cp deploy/prod/noema.service /etc/systemd/system/ && sudo systemctl daemon-reload +deploy/prod/build.sh && sudo systemctl enable noema + +sudo certbot certonly --nginx -d noema.dixon.cx --key-type ecdsa +sudo cp deploy/prod/noema.dixon.cx.conf /etc/nginx/sites-enabled/ && sudo nginx -t && sudo systemctl reload nginx +``` diff --git a/deploy/prod/build.sh b/deploy/prod/build.sh new file mode 100755 index 0000000..9c52716 --- /dev/null +++ b/deploy/prod/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build the Noema image and restart the systemd service. Run from anywhere. +set -euo pipefail +cd "$(dirname "$0")/../.." +VERSION=$(git describe --tags --always --dirty 2>/dev/null || echo dev) +docker build -f deploy/Dockerfile --build-arg VERSION="$VERSION" -t noema:"$VERSION" -t noema:latest . +sudo systemctl restart noema +sleep 3 +systemctl --no-pager --lines=5 status noema diff --git a/deploy/prod/noema.dixon.cx.conf b/deploy/prod/noema.dixon.cx.conf new file mode 100644 index 0000000..5beca13 --- /dev/null +++ b/deploy/prod/noema.dixon.cx.conf @@ -0,0 +1,29 @@ +# noema.dixon.cx -> Noema in Docker on 127.0.0.1:29630 +server { + listen 80; + listen [::]:80; + server_name noema.dixon.cx; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl http2; + server_name noema.dixon.cx; + + ssl_certificate /etc/letsencrypt/live/noema.dixon.cx/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/noema.dixon.cx/privkey.pem; + include snippets/ssl-params.conf; + + access_log /var/log/nginx/noema_access.log; + error_log /var/log/nginx/noema_error.log; + + client_max_body_size 2m; + + location / { + proxy_pass http://127.0.0.1:29630; + include snippets/proxy.conf; + # SSE streams and model calls run long; don't buffer or cut them off. + proxy_buffering off; + proxy_read_timeout 300s; + } +} diff --git a/deploy/prod/noema.service b/deploy/prod/noema.service new file mode 100644 index 0000000..dd18e09 --- /dev/null +++ b/deploy/prod/noema.service @@ -0,0 +1,21 @@ +[Unit] +Description=Noema daemon (docker container) +After=docker.service network-online.target postgresql.service +Requires=docker.service +Wants=network-online.target + +[Service] +Restart=always +RestartSec=5 +TimeoutStartSec=0 +ExecStartPre=-/usr/bin/docker rm -f noema +ExecStart=/usr/bin/docker run --rm --name noema \ + --env-file /etc/noema/noema.env \ + -p 127.0.0.1:29630:8080 \ + -v /srv/noema/research:/research \ + --memory 2g --cpus 4 --pids-limit 512 \ + noema:latest +ExecStop=/usr/bin/docker stop -t 30 noema + +[Install] +WantedBy=multi-user.target