diff --git a/docs/site/build.py b/docs/site/build.py index 05d45d5..b8b9f43 100644 --- a/docs/site/build.py +++ b/docs/site/build.py @@ -28,8 +28,7 @@ CHAPTERS = [ ("index", "Noema", "A persistent, inspectable artificial cognitive runtime: minds that remember, reason with evidence, say how sure they are, and can always show their working."), ("concepts", "Concepts and vocabulary", "Minds, events, epistemic kinds, memory kinds, beliefs, drives and the interpretation boundary."), ("architecture", "Architecture", "The daemon, the cognitive cycle, the storage model and the boundaries between deterministic code and language models."), - ("getting-started", "Getting started", "From an empty machine to a running mind you can talk to, inspect and replay."), - ("configuration", "Configuration", "Environment, config file, secret references, runtime settings and the per-mind cognitive configuration."), + ("configuration", "Cognitive configuration", "The per-mind settings that shape a mind: workspace capacity, attention weights, budgets and background cognition."), ("personality", "Identity and personality", "Versioned personalities, templates, traits and how they modulate attention and voice."), ("cognition", "The cognitive cycle", "Processes, attention scoring, the global workspace, executive dispatch, modes and metacognition."), ("memory", "Memory", "Episodic, semantic, procedural and autobiographical memory; retrieval, decay, consolidation and provenance."), @@ -45,11 +44,10 @@ CHAPTERS = [ ("snapshots-replay", "Snapshots, forks, replay and experiments", "Capturing state, branching minds, re-running cognition and running controlled experiments."), ("ui-tour", "Interface tour", "Every screen of the web interface, what it shows and what an operator does there."), ("api", "HTTP API", "Authentication, roles, the error envelope, streaming and every route."), - ("cli", "Command line", "noemactl: every command with examples."), + ("cli", "Command line", "What an operator can do from a terminal: every command, with examples."), ("security", "Security and threat model", "Trust boundaries, secret handling, sessions, two-factor authentication and the threat-by-threat verification table."), - ("operations", "Operating Noema", "Deployment, backups, scheduled jobs, metrics, audit, performance and upgrades."), ("decisions", "Architecture decision records", "The recorded decisions and the alternatives they rejected."), - ("development", "Development and history", "Contributing, testing, the phased plan, changelog and release notes."), + ("development", "How it was built", "The phased plan, what each phase delivered, measured performance, release notes and changelog."), ("glossary", "Glossary", "Short definitions of every term used across the documentation."), ] @@ -209,7 +207,7 @@ LINK_MAP = { "docs/personality.md": "personality", "docs/cognition.md": "cognition", "docs/memory.md": "memory", "docs/beliefs.md": "beliefs", "docs/conversation.md": "conversation", "docs/motivation.md": "motivation", "docs/predictions.md": "predictions", "docs/graph.md": "graph", "docs/llm-providers.md": "llm-providers", "docs/capabilities.md": "capabilities", "docs/integrations.md": "integrations", - "docs/dreaming.md": "dreaming", "docs/snapshots-replay.md": "snapshots-replay", "docs/performance.md": "operations", + "docs/dreaming.md": "dreaming", "docs/snapshots-replay.md": "snapshots-replay", "docs/performance.md": "development", "docs/RELEASE_NOTES.md": "development", "docs/openapi.json": "api", "docs/adr": "decisions", } diff --git a/docs/site/pages/cli.md b/docs/site/pages/cli.md index 23b3e5a..7481d1f 100644 --- a/docs/site/pages/cli.md +++ b/docs/site/pages/cli.md @@ -1,12 +1,4 @@ -`noemactl` is a thin client for the HTTP API with a few offline commands that talk to the database directly. It prints tables by default and JSON with `-json` where a command supports it, exits non-zero on API errors with the error message, and never stores credentials. - -## Environment - -| Variable | Purpose | -|---|---| -| `NOEMA_URL` | base URL of the daemon, default `http://127.0.0.1:8080` | -| `NOEMA_API_TOKEN` | bearer token from the account page; required for API commands | -| `NOEMA_DATABASE_URL` | for the offline commands only (`migrate`, `user`) | +`noemactl` gives an operator the whole system from a terminal: talk to a mind, watch it think, inspect and correct what it knows, govern what it may do, connect sources, and snapshot, fork, replay and experiment. Everything it does goes through the same API and the same audit trail as the interface. ## Commands diff --git a/docs/site/pages/configuration.md b/docs/site/pages/configuration.md index de5ccd5..e8d17ae 100644 --- a/docs/site/pages/configuration.md +++ b/docs/site/pages/configuration.md @@ -1,30 +1,8 @@ -Configuration has two layers. **Daemon configuration** is fixed at start: where to listen, which database, the master key, resource limits. **Cognitive configuration** belongs to each mind, lives in the database, is edited from the interface or API, and is reloaded into the running mind without a restart. +Each mind carries its own cognitive configuration: how many thoughts it can hold, how it weighs what to attend to, how fast it cycles, how much it may spend on models, and which background processes run. These settings are data, edited from the interface or API, normalised on every write and reloaded into the running mind without a restart. They are the knobs an experiment turns. -## Daemon configuration +## The settings -{{include docs/configuration.md}} - -### Environment reference - -| Variable | Default | Notes | -|---|---|---| -| `NOEMA_HTTP_ADDR` | `127.0.0.1:8080` | bind to a loopback address and put a TLS proxy in front, or bind `0.0.0.0` inside a container | -| `NOEMA_ENV` | `development` | `production` forces `Secure` cookies and disables template reloading | -| `NOEMA_LOG_LEVEL` / `NOEMA_LOG_FORMAT` | `info` / `text` | `json` for log shippers; the log handler redacts values that look like secrets | -| `NOEMA_DATABASE_URL` | compose database | pgx connection string; `sslmode` as your database requires | -| `NOEMA_MASTER_KEY` | unset | 32 bytes base64; required for encrypted secret references and TOTP | -| `NOEMA_BOOTSTRAP_ADMIN_USER` / `_PASSWORD` | unset | first administrator when the users table is empty; remove after first start | -| `NOEMA_RESEARCH_DIR` | `./research` | default path offered when creating a research-folder integration | -| `NOEMA_MAX_WORKERS` | 8 | size of the cognitive worker pool shared by all minds | -| `NOEMA_MAX_CONCURRENT_LLM` | 4 | concurrent model calls across the daemon | -| `NOEMA_DAILY_TOKEN_BUDGET` | 2 000 000 | global hard stop on model tokens per day; per-mind budgets sit under it | -| `NOEMA_SESSION_IDLE_TTL` / `NOEMA_SESSION_ABSOLUTE_TTL` | 12h / 7d | browser session expiry; API tokens have their own optional expiry | - -A JSON file passed with `noemad -config path` can set the same keys; environment variables override the file. - -## Cognitive configuration (per mind) - -Each mind carries a `CognitiveConfig` document. It is shown on the mind overview, edited from the personality and dreaming pages and through `PATCH /api/v1/minds/{mind}`, and normalised on every write: out-of-range values fall back to defaults rather than being rejected silently or accepted dangerously. +Out-of-range values fall back to defaults rather than being rejected silently or accepted dangerously; every change is an operator action in the mind's history, and the mind encodes it as an autobiographical memory. | Field | Default | Effect | |---|---|---| @@ -52,6 +30,6 @@ Attention scoring uses a fixed set of ten features, each computed deterministica A small set of tunables that are safe to change while running (the global consolidation and dreaming pause switches, notification behaviour) are stored in the database and edited in the interface; they take effect on the next scheduler tick. Everything else requires a restart or, for per-mind settings, a `reload` command that the interface issues automatically after an edit. -## Development environment record +## Daemon-wide limits -{{include docs/ENVIRONMENT.md demote=1}} +Above the per-mind settings sit a few daemon-wide limits: a global daily token budget that no mind can exceed, a cap on concurrent model calls, and the size of the worker pool that runs cognitive processes for every mind. Per-mind budgets are enforced under the global one, so one talkative mind cannot starve the others. Secrets used by any part of the system (provider keys, integration secrets, second-factor secrets) exist only as references encrypted under a master key, never as values in configuration. diff --git a/docs/site/pages/development.md b/docs/site/pages/development.md index abea9e8..656ff4b 100644 --- a/docs/site/pages/development.md +++ b/docs/site/pages/development.md @@ -1,4 +1,4 @@ -Noema was built in sixteen explicit phases, each committed with tests and documentation before the next began. This page collects the contributing guide, the testing story, the phase plan, the changelog and the release notes. +Noema was built in sixteen explicit phases, each committed with tests and documentation before the next began, and each verifiable on its own. This page records what each phase delivered, how the result was measured, and the release notes and changelog. ## How it was built @@ -22,11 +22,13 @@ Noema was built in sixteen explicit phases, each committed with tests and docume | 15 | Snapshots, forks, comparison, export/import, replay, experiments | | 16 | Two-factor authentication, security suite, OpenAPI, scanners, performance pass, screenshots, release | -## Testing +## How it is verified -`go test -race ./...` runs unit and integration tests; anything that needs PostgreSQL uses a fresh schema per test and is enabled by `NOEMA_TEST_DATABASE_URL`. `tests/m1` is the end-to-end acceptance test for the first milestone with a fake model. `tests/security` is the cross-cutting security suite. `tests/perf` is the opt-in load test. `make lint` runs staticcheck; `make security` runs gosec and govulncheck. All were clean at release. +Every subsystem has unit tests that run against a fresh database schema; an end-to-end acceptance test drives the whole daemon with a fake model through the first milestone (start, log in, create a mind, converse, inspect memory and beliefs, restart and continue); a security suite exercises prompt injection, roles, cross-mind isolation, stored scripts, secret disclosure and hostile archives; and an opt-in load test seeds a realistic volume and rejects any hot query that drifts off its index. The race detector, staticcheck, gosec and govulncheck were clean at release. -{{include CONTRIBUTING.md}} +## Measured performance + +{{include docs/performance.md}} ## Release notes diff --git a/docs/site/pages/getting-started.md b/docs/site/pages/getting-started.md deleted file mode 100644 index a08c969..0000000 --- a/docs/site/pages/getting-started.md +++ /dev/null @@ -1,110 +0,0 @@ -This page takes you from nothing to a running mind that you can talk to, inspect, snapshot and replay. It assumes a Linux or macOS machine with Docker available; the daemon itself is a single static binary and needs only PostgreSQL 16 (pgvector recommended, optional). - -## Prerequisites - -- Go 1.26 (the module pins a patched toolchain and will fetch it automatically), or the prebuilt container image. -- Docker with Compose, used here for PostgreSQL. Any PostgreSQL 16 works; the `pgvector/pgvector:pg16` image adds semantic retrieval. -- A terminal and a browser. No Node, no build step for the interface; templates and assets are embedded. - -## 1. Clone and build - -``` -git clone git@github.com:andydixon/noema.git -cd noema -make build # produces ./bin/noemad and ./bin/noemactl -``` - -`make tools` installs staticcheck, gosec and govulncheck if you intend to develop; they are not needed to run. - -## 2. Start the database - -``` -make dev-db # docker compose: PostgreSQL 16 with pgvector on 127.0.0.1:5433 -``` - -The compose file lives in `deploy/docker-compose.yml`. It publishes the database on port 5433 to avoid colliding with a local PostgreSQL, and stores data in a named volume. - -## 3. Configure - -Copy the example environment file and set the two values that matter: - -``` -cp .env.example .env -openssl rand -base64 32 # paste as NOEMA_MASTER_KEY -``` - -| Variable | Purpose | -|---|---| -| `NOEMA_DATABASE_URL` | connection string; the example points at the compose database | -| `NOEMA_MASTER_KEY` | 32 bytes, base64. Encrypts provider API keys, integration secrets and TOTP secrets at rest. Without it those features are unavailable, and the daemon says so. | -| `NOEMA_BOOTSTRAP_ADMIN_USER` / `_PASSWORD` | optional first administrator, used only when the users table is empty | -| `NOEMA_HTTP_ADDR` | listen address, default `127.0.0.1:8080` | -| `NOEMA_RESEARCH_DIR` | default watched folder for the research integration | - -Secrets never go into the database in clear. Provider keys are entered once in the interface and stored as `enc:` references; you can also reference an environment variable (`env:OPENAI_API_KEY`) or a file (`file:/run/secrets/key`). The [configuration page](configuration) has the full list. - -## 4. Migrate and run - -``` -make migrate # applies the embedded, numbered migrations -make run # starts noemad with .env -``` - -The daemon applies migrations itself on start as well, under an advisory lock, so a fleet of instances cannot race. The log line `noemad ready url=http://127.0.0.1:8080` means the interface is up. - -## 5. First login - -Open the address in a browser. With no users, the interface shows a one-time **setup** page to create the first administrator (unless you set the bootstrap variables). Passwords are hashed with Argon2id; sessions are cookie-based with CSRF protection; you can add a second factor later from the account page. - -Create an API token from **Account → API tokens** for the command line: - -``` -export NOEMA_URL=http://127.0.0.1:8080 -export NOEMA_API_TOKEN=noema_… -./bin/noemactl status -``` - -## 6. Create a mind - -Either click **Minds → New mind** and pick a personality template, or use the sample mind shipped with the repository: - -``` -./bin/noemactl mind create --slug dani --name Dani --template investigator -``` - -The sample seed `docs/samples/dani-seed.json` gives Dani a personality, a few memories and some claims to reason about. Import it from the minds page or: - -``` -curl -s -H "Authorization: Bearer $NOEMA_API_TOKEN" -H 'Content-Type: application/json' \ - --data @docs/samples/dani-seed.json $NOEMA_URL/api/v1/minds/import -``` - -The mind starts immediately. Its runtime wakes on new events, runs an idle cycle on a timer, and persists its workspace after every cycle. - -## 7. Talk to it, and watch it think - -Open the mind, go to **Conversation** and say something. Or from the shell: - -``` -./bin/noemactl ask dani "The payments service failed right after the database migration finished." -``` - -Then open **Thoughts**. You will see the message arrive as an observation, the perception that analysed it, the memories it recalled with their retrieval scores, the thought candidates and which ones won the workspace, and, if the mind decided to answer, the response intent and the response with the evidence it rests on. Everything in that list is an event; click any of them for its full context: parents, children, payload, and the model call if one was made. - -Without a language model configured, all of this still works: perception is deterministic, retrieval is lexical and structural, and responses are rendered from templates over real state. Adding a model under **LLM** makes perception richer and answers more fluent; it does not change what the mind knows. - -## 8. Inspect beliefs and memories - -Under **Beliefs**, each proposition shows its confidence, status, evidence for and against, and the formula that produced the number. Mark a belief wrong and the correction becomes counter-evidence and, if you choose, a "do not infer this again" rule. Under **Memory**, search recalls with a visible score breakdown: lexical, semantic (if pgvector and an embedding model are configured), recency, importance, activation and relationship terms. - -## 9. Snapshot, fork, replay - -From **Snapshots**, take a snapshot. Fork it into `dani-b`, change something in the fork's personality, and use **Compare** to see the two diverge. Start a **Replay** from the snapshot: the recorded stimuli are fed to a disabled sandbox copy and every step is labelled exact, approximate or diverged. This is the loop the rest of the documentation keeps coming back to: change something, run it again, look at the difference. - -## 10. Where to go next - -- The [interface tour](ui-tour) walks every screen. -- [Configuration](configuration) explains the per-mind cognitive settings (workspace capacity, attention weights, budgets, background cognition). -- [Language models](llm-providers) covers providers, roles, budgets and prompt versions. -- [Capabilities](capabilities) explains how to let a mind do anything at all beyond thinking, and why the defaults say ASK. -- [Operations](operations) covers running it for real: backups, metrics, audit, upgrades. diff --git a/docs/site/pages/index.md b/docs/site/pages/index.md index 65da589..ea7cb56 100644 --- a/docs/site/pages/index.md +++ b/docs/site/pages/index.md @@ -1,7 +1,7 @@

An artificial cognitive runtime

Most AI systems answer. Noema thinks, remembers, doubts and can show its working. It is a persistent runtime that hosts minds: long-lived cognitive entities with their own identity, memories, beliefs with evidence, goals, drives and an unbroken record of every thought they have ever had. Language models are plug-in services it calls when it needs language. The mind is the thing that stays.

-

Run it in ten minutes See every screen Read the architecture

+

What it is How it thinks See every screen

## Why it is different @@ -66,10 +66,11 @@ Noema assumes every external byte is hostile. Prompt injection is a threat model Noema does not claim consciousness, sentience or personhood. Its components borrow names from cognitive science because those names describe what each part does; they are data structures and deterministic rules, sometimes assisted by a model. That honesty is a feature: it is what lets you trust the rest. The [interpretation boundary](concepts#the-interpretation-boundary) is part of the specification. -## Start here +## Read next -1. [Getting started](getting-started): database, binary, first mind, first conversation. -2. [Concepts](concepts): the vocabulary in one page. -3. [Interface tour](ui-tour): every screen with screenshots. -4. [Capabilities](capabilities): how to let a mind do anything, safely. +1. [Concepts](concepts): the vocabulary in one page, and the interpretation boundary. +2. [The cognitive cycle](cognition): how a thought is proposed, scored, admitted and acted on. +3. [Beliefs and evidence](beliefs): why confidence is a number you can check. +4. [Capabilities and policy](capabilities): how a mind acts, and why the defaults say ask. 5. [Snapshots, forks, replay and experiments](snapshots-replay): the laboratory. +6. [Interface tour](ui-tour): every screen with screenshots. diff --git a/docs/site/pages/operations.md b/docs/site/pages/operations.md deleted file mode 100644 index e75d412..0000000 --- a/docs/site/pages/operations.md +++ /dev/null @@ -1,56 +0,0 @@ -This page is for whoever runs Noema: how to deploy it, what it needs, what runs on a schedule, what to back up, what to watch, and how to upgrade. - -## Deployment - -The daemon is one static binary plus PostgreSQL. `deploy/Dockerfile` builds a minimal image; `deploy/docker-compose.yml` runs PostgreSQL with pgvector and, with the `app` profile, the daemon. For a systemd host: install the binary, create a `.env` from the example with a real master key and database URL, run `noemad` under a dedicated user with `NOEMA_ENV=production`, and put a TLS-terminating proxy in front of the loopback listener. The daemon applies migrations on start under an advisory lock, so rolling restarts and multiple instances are safe. - -Production settings to check: - -- `NOEMA_ENV=production` (Secure cookies, no template reloading); -- a strong `NOEMA_MASTER_KEY` kept out of the repository and backed up separately from the database (without it, encrypted provider keys, integration secrets and TOTP secrets cannot be read); -- remove the bootstrap administrator variables after first start; -- budgets: `NOEMA_DAILY_TOKEN_BUDGET`, per-mind `daily_token_budget`, `NOEMA_MAX_CONCURRENT_LLM`; -- `NOEMA_MAX_WORKERS` roughly equal to the cores you want cognition to use. - -## Scheduled jobs - -| Job | Interval | What it does | -|---|---|---| -| auth.sweep / auth.purge | 5 min / 1 h | rate-limit bucket cleanup, expired sessions and pending logins | -| memory.decay | 6 h | lowers activation of unretrieved memories, never deletes | -| memory.embed | 1 min | embeds memories that lack a vector when an embedding model exists | -| predictions.resolve | 1 min | evaluates open predictions against their criteria | -| cognition.background | 5 min | runs consolidation, reflection and dreaming for minds whose interval elapsed and switches are on | -| approvals.expire | 5 min | expires stale approval requests | -| integrations.tick | 1 min | runs due integrations | -| brain.reconcile | 30 s | starts runtimes for newly active minds and stops disabled ones | - -The system page shows each job's last run, duration and error. - -## Backups - -The database is the whole state, including history, snapshots and recorded model calls. Back it up with the PostgreSQL tools of your choice; the append-only tables compress well. The master key must be backed up separately and stored apart from the database dump. Mind archives (`noemactl snapshots export`) are portable, checksummed and secret-free; they are a convenient per-mind backup and a migration path between instances, but they do not include event history. - -## Metrics and logs - -`/metrics` exposes Prometheus metrics behind authentication: cycles and their duration per mind, thought candidates and admissions, events by type, model calls, tokens and cost by role and model, budget exhaustion, integration runs, HTTP request counts and latencies, and authentication failures. Logs are structured (`text` or `json`) and the handler redacts values that look like secrets. Every request has an id that appears in error envelopes and log lines. - -## Audit - -Every operator action (login, configuration change, policy edit, approval, correction, snapshot, fork, import, experiment run) is written to an append-only audit log with actor, target and details, and most also appear as `operator_action` events in the affected mind's own history. The audit page is searchable; the API exposes it to operators and above. - -## Sessions, tokens and two-factor authentication - -Sessions have idle and absolute expiry, are listed and revocable per user, and are all revoked on password change. API tokens are hashed at rest and optionally expire. Two-factor authentication is per user (TOTP with recovery codes); administrators can reset it. Repeated failed logins lock an account with a rising back-off; login attempts are rate-limited per address. - -## Performance - -{{include docs/performance.md}} - -## Upgrades - -Upgrades are forward-only migrations applied on start; each migration ships with a down script for emergencies. Read the changelog under [development](development) for behavioural changes, take a database backup, deploy the new binary, and watch the log for `applied migrations`. Snapshots taken by an older version remain importable as long as their schema version is supported; the archive format is versioned and validated. - -## Shutting a mind down - -Disable it (its runtime stops; state and history remain), archive it (hidden from lists), or delete it through the administrator API (which the interface confirms twice). None of these are decisions a mind can make, influence or resist; there is no self-preservation drive and no capability that touches its own lifecycle.