// Command noemad is the Noema daemon: API server, web UI, scheduler, // integration manager and cognitive workers in one process. package main import ( "context" "encoding/json" "errors" "flag" "fmt" "net/http" "os" "os/signal" "syscall" "time" "github.com/andydixon/noema/internal/api" "github.com/andydixon/noema/internal/app" "github.com/andydixon/noema/internal/audit" "github.com/andydixon/noema/internal/auth" "github.com/andydixon/noema/internal/beliefs" "github.com/andydixon/noema/internal/brain" "github.com/andydixon/noema/internal/capabilities" "github.com/andydixon/noema/internal/cognition" "github.com/andydixon/noema/internal/config" "github.com/andydixon/noema/internal/consolidation" "github.com/andydixon/noema/internal/conversation" "github.com/andydixon/noema/internal/dreaming" evstore "github.com/andydixon/noema/internal/events" "github.com/andydixon/noema/internal/experiments" "github.com/andydixon/noema/internal/goals" "github.com/andydixon/noema/internal/graph" "github.com/andydixon/noema/internal/integrations" "github.com/andydixon/noema/internal/llm/prompts" "github.com/andydixon/noema/internal/llm/router" "github.com/andydixon/noema/internal/memory" "github.com/andydixon/noema/internal/minds" "github.com/andydixon/noema/internal/motivation" "github.com/andydixon/noema/internal/narrative" "github.com/andydixon/noema/internal/notifications" "github.com/andydixon/noema/internal/observability" "github.com/andydixon/noema/internal/predictions" "github.com/andydixon/noema/internal/reflection" "github.com/andydixon/noema/internal/replay" "github.com/andydixon/noema/internal/scheduler" "github.com/andydixon/noema/internal/security/secrets" "github.com/andydixon/noema/internal/snapshots" "github.com/andydixon/noema/internal/storage" "github.com/andydixon/noema/internal/ui" "github.com/andydixon/noema/internal/version" "github.com/andydixon/noema/migrations" "github.com/andydixon/noema/pkg/events" "github.com/andydixon/noema/pkg/ids" ) func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, "noemad:", err) os.Exit(1) } } func run() error { var ( cfgPath = flag.String("config", "", "path to JSON config file (optional)") migrate = flag.Bool("migrate", true, "apply pending migrations at start") show = flag.Bool("version", false, "print version and exit") ) flag.Parse() if *show { fmt.Println(version.Version) return nil } cfg, err := config.Load(*cfgPath, os.LookupEnv) if err != nil { return err } log := observability.NewLogger(os.Stderr, cfg.LogLevel, cfg.LogFormat) log.Info("starting noemad", "version", version.Version, "env", cfg.Env, "addr", cfg.HTTPAddr) log.Debug("configuration", "config", fmt.Sprintf("%+v", cfg.Redacted())) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() db, err := storage.Open(ctx, cfg.DatabaseURL) if err != nil { return err } defer db.Close() if *migrate { applied, err := storage.MigrateUp(ctx, db, migrations.FS) if err != nil { return fmt.Errorf("migrate: %w", err) } if len(applied) > 0 { log.Info("applied migrations", "migrations", applied) } } metrics := observability.NewMetrics() authSvc := auth.NewService(db, cfg.SessionIdleTTL, cfg.SessionAbsoluteTTL) auditLog := audit.New(db, log) deps := app.Deps{Config: cfg, Log: log, DB: db, Auth: authSvc, Audit: auditLog, Metrics: metrics, Minds: minds.NewStore(db), Events: evstore.NewStore(db, evstore.NewBus(200), metrics), Migrations: migrations.FS, Started: time.Now()} if err := bootstrapAdmin(ctx, deps); err != nil { return err } group := scheduler.NewGroup(ctx, log) sec, err := secrets.New(cfg.MasterKey) if err != nil { return err } deps.Secrets = sec authSvc.SetSecrets(sec) if !sec.CanEncrypt() { log.Warn("NOEMA_MASTER_KEY not set: API keys must be given as env: or file: references") } deps.LLMStore = router.NewStore(db) deps.LLM = router.NewRouter(deps.LLMStore, sec, log, metrics, cfg.MaxConcurrentLLM) deps.LLM.DailyTokenBudget = cfg.DailyTokenBudget deps.LLM.MindBudget = func(id ids.ID) int64 { m, err := deps.Minds.Get(ctx, id) if err != nil { return 0 } return m.Config.DailyTokenBudget } deps.Prompts = prompts.NewStore(db) if n, err := deps.Prompts.SeedDefaults(ctx); err != nil { return fmt.Errorf("seed prompts: %w", err) } else if n > 0 { log.Info("installed default prompts", "count", n) } deps.Memory = memory.NewStore(ctx, db, deps.LLM.Embedder(ctx)) deps.Beliefs = beliefs.NewStore(db) deps.Conversations = conversation.NewStore(db) deps.Goals = goals.NewStore(db) deps.Motivation = motivation.NewStore(db) deps.Predictions = predictions.NewStore(db) deps.Resolver = &predictions.Resolver{Store: deps.Predictions, Events: deps.Events, Beliefs: deps.Beliefs, Log: log, Metrics: metrics} deps.Graph = graph.NewStore(db) capReg := capabilities.NewRegistry() for _, c := range capabilities.Builtins() { capReg.Add(c) } deps.Integrations = &integrations.Manager{Store: integrations.NewStore(db), Events: deps.Events, Graph: deps.Graph, Secrets: sec, Log: log, Pool: scheduler.NewPool(2), Pollers: map[integrations.Kind]integrations.Poller{integrations.ResearchFolder: integrations.ResearchPoller{}, integrations.Git: integrations.GitPoller{DataDir: "./data/git"}, integrations.Prometheus: integrations.PrometheusPoller{Secrets: sec}}} for _, c := range integrations.Capabilities(deps.Integrations, integrations.GitPoller{DataDir: "./data/git"}, integrations.PrometheusPoller{Secrets: sec}) { capReg.Add(c) } deps.Capabilities = &capabilities.Service{Registry: capReg, DB: db, Events: deps.Events, Log: log, Metrics: metrics, ApprovalTTL: 24 * time.Hour, SessionTTL: 8 * time.Hour} selfSync := cognition.SelfModelSync{Minds: deps.Minds, Caps: deps.Capabilities, Events: deps.Events, Log: log} deps.Capabilities.OnSelfModel = selfSync.Sync deps.Narrative = narrative.NewStore(db) deps.Notifications = notifications.NewStore(db) deps.Consolidation = &consolidation.Service{DB: db, Memory: deps.Memory, Beliefs: deps.Beliefs, Events: deps.Events, LLM: deps.LLM, Prompts: deps.Prompts, Minds: deps.Minds, Log: log} deps.Dreaming = &dreaming.Service{DB: db, Memory: deps.Memory, Beliefs: deps.Beliefs, Events: deps.Events, LLM: deps.LLM, Prompts: deps.Prompts, Log: log} deps.Reflection = &reflection.Service{DB: db, Events: deps.Events, Beliefs: deps.Beliefs, Goals: deps.Goals, Memory: deps.Memory, Predictions: deps.Predictions, LLM: deps.LLM, Prompts: deps.Prompts, Log: log} registry := brain.NewRegistry().AddProcess(cognition.Attend{}).AddProcess(cognition.Retrieve{}).AddProcess(cognition.Associate{}).AddProcess(cognition.Perceive{}). AddProcess(cognition.GoalMonitor{}).AddProcess(cognition.Curiosity{}).AddProcess(cognition.Metacognition{}).AddProcess(cognition.Act{}). AddHook(cognition.MotivationHook{Store: deps.Motivation}).AddHook(cognition.NarrativeHook{Store: deps.Narrative}).AddHook(cognition.NotifyHook{Store: deps.Notifications}) cognition.Handlers{Memory: deps.Memory, Beliefs: deps.Beliefs, Conversations: deps.Conversations, Graph: deps.Graph}.Register(registry) brainCfg := brain.Config{DB: db, Events: deps.Events, Minds: deps.Minds, Registry: registry, Pool: scheduler.NewPool(cfg.MaxWorkers), Log: log, Metrics: metrics, Services: brain.Services{Events: deps.Events, Memory: deps.Memory, Beliefs: deps.Beliefs, LLM: deps.LLM, Prompts: deps.Prompts, Goals: deps.Goals, Motivation: deps.Motivation, Predictions: deps.Predictions, Capabilities: deps.Capabilities}} deps.Brain = brain.NewManager(brainCfg, group) // Sandbox runtimes (replay, experiments) share the registry but use a // router that serves recorded model responses when it can. replayLLM := router.NewRouter(deps.LLMStore, sec, log, metrics, cfg.MaxConcurrentLLM) replayLLM.Replay = true replayLLM.DailyTokenBudget, replayLLM.MindBudget = deps.LLM.DailyTokenBudget, deps.LLM.MindBudget sandboxCfg := brainCfg sandboxCfg.Pool = scheduler.NewPool(2) sandboxCfg.Services.LLM = replayLLM deps.Snapshots = &snapshots.Store{DB: db, Events: deps.Events, Minds: deps.Minds} deps.Replay = &replay.Service{DB: db, Events: deps.Events, Minds: deps.Minds, Snapshots: deps.Snapshots, Brain: sandboxCfg, Log: log} deps.Experiments = &experiments.Service{DB: db, Minds: deps.Minds, Snapshots: deps.Snapshots, Brain: sandboxCfg, Log: log, Routing: func(ctx context.Context) []map[string]any { roles, _ := deps.LLMStore.ListRoles(ctx) b, _ := json.Marshal(roles) var out []map[string]any json.Unmarshal(b, &out) //nolint:errcheck return out }} selfSync.Sync(ctx, ids.Nil) group.Every("integrations.tick", time.Minute, deps.Integrations.Tick) group.Every("approvals.expire", 5*time.Minute, func(ctx context.Context) error { _, err := deps.Capabilities.ExpirePending(ctx); return err }) group.Every("predictions.resolve", time.Minute, deps.Resolver.Run) group.Every("cognition.background", 5*time.Minute, func(ctx context.Context) error { return backgroundCognition(ctx, deps) }) group.Every("memory.decay", 6*time.Hour, func(ctx context.Context) error { return decayMemories(ctx, deps) }) group.Every("memory.embed", time.Minute, func(ctx context.Context) error { _, err := deps.Memory.EmbedMissing(ctx, 50); return err }) if err := deps.Brain.Start(ctx); err != nil { return fmt.Errorf("start minds: %w", err) } var srv *api.Server uiHandler, err := ui.New(deps, ui.Actions{ Status: func(r *http.Request) any { return srv.Status(r.Context()) }, Context: func(r *http.Request, id ids.ID) (api.EventContext, error) { return srv.Context(r, id) }, Workspace: func(r *http.Request, m minds.Mind) (api.WorkspaceView, error) { return srv.Workspace(r.Context(), m) }, Memory: func(r *http.Request, mind, id ids.ID) (api.MemoryDetail, error) { return srv.MemoryDetail(r, mind, id) }, Belief: func(r *http.Request, mind, id ids.ID) (api.BeliefDetail, error) { return srv.BeliefDetail(r, mind, id) }, Emitter: func() beliefs.Emitter { return srv.Emitter() }, Post: func(r *http.Request, mind ids.ID, conv conversation.Conversation, text, who string) (conversation.Message, error) { return srv.PostMessage(r, mind, conv, text, who) }, Goal: func(r *http.Request, n goals.New) (goals.Goal, error) { return srv.CreateGoal(r, n) }, GoalUpdate: func(r *http.Request, mind, id ids.ID, u goals.Update) (goals.Goal, error) { return srv.UpdateGoal(r, mind, id, u) }, Predict: func(r *http.Request, n predictions.New) (predictions.Prediction, error) { return srv.CreatePrediction(r, n) }, Resolve: func(r *http.Request, mind, id ids.ID, outcome predictions.Status, reason string) (predictions.Result, error) { return srv.ResolvePrediction(r, mind, id, outcome, reason) }, ResetState: func(r *http.Request, mind ids.ID) (motivation.State, error) { return srv.ResetState(r, mind) }, Consolidate: func(r *http.Request, mind ids.ID) (consolidation.Job, error) { return srv.RunConsolidation(r, mind) }, Study: func() *api.Server { return srv }, Reflect: func(r *http.Request, mind ids.ID) (reflection.Reflection, error) { return srv.RunReflection(r, mind) }, Entity: func(r *http.Request, mind, id ids.ID) (api.EntityDetail, error) { return srv.EntityDetail(r, mind, id) }, Merge: func(r *http.Request, mind ids.ID, e graph.Entity, other string) error { return srv.MergeEntities(r, mind, e, other) }, Distinct: func(r *http.Request, mind ids.ID, e graph.Entity, other string) error { return srv.DistinctEntities(r, mind, e, other) }, }) if err != nil { return err } srv = api.New(deps, uiHandler) httpSrv := &http.Server{Addr: cfg.HTTPAddr, Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, IdleTimeout: 120 * time.Second, MaxHeaderBytes: 64 << 10} group.Go("http", func(ctx context.Context) error { errc := make(chan error, 1) go func() { errc <- httpSrv.ListenAndServe() }() // owned: joined via errc below select { case err := <-errc: if errors.Is(err, http.ErrServerClosed) { return nil } return err case <-ctx.Done(): shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return httpSrv.Shutdown(shutdownCtx) } }) group.Every("auth.sweep", 5*time.Minute, func(context.Context) error { authSvc.SweepLimiter(); return nil }) group.Every("auth.purge", time.Hour, authSvc.PurgeExpiredSessions) log.Info("noemad ready", "url", "http://"+cfg.HTTPAddr) <-ctx.Done() log.Info("shutting down") return group.Stop() } // bootstrapAdmin creates the first administrator from the environment when the // users table is empty, so headless deployments can start without the setup page. func bootstrapAdmin(ctx context.Context, d app.Deps) error { n, err := d.Auth.CountUsers(ctx) if err != nil { return err } if n > 0 || d.Config.BootstrapAdminPassword == "" { return nil } u, err := d.Auth.CreateUser(ctx, d.Config.BootstrapAdminUser, d.Config.BootstrapAdminPassword, auth.RoleAdministrator) if err != nil { return fmt.Errorf("bootstrap admin: %w", err) } d.Audit.Log(ctx, "user.bootstrap", "user", u.ID.String(), map[string]any{"username": u.Username, "via": "environment"}) d.Log.Info("created bootstrap administrator", "username", u.Username) return nil } // decayMemories lowers activation of unused memories for every mind and leaves // one summary event per mind so the decay is visible without spamming. func decayMemories(ctx context.Context, d app.Deps) error { list, err := d.Minds.List(ctx, false) if err != nil { return err } for _, m := range list { n, err := d.Memory.Decay(ctx, m.ID, 14*24*time.Hour) if err != nil { return err } if n > 0 { d.Events.Append(ctx, events.New{MindID: m.ID, Type: events.MemoryDecayed, Source: "memory.decay", Summary: fmt.Sprintf("activation decayed on %d memories", n), //nolint:errcheck Salience: 0.05, Confidence: 1, Metadata: map[string]any{"count": n}}) } } return nil } // backgroundCognition runs consolidation and reflection for each active mind // whose interval has elapsed, switching the runtime mode while it works. func backgroundCognition(ctx context.Context, d app.Deps) error { list, err := d.Minds.List(ctx, false) if err != nil { return err } for _, m := range list { if m.Status != minds.StatusActive { continue } if m.Config.ConsolidationEnabled && !consolidation.IsPaused() { if jobs, _ := d.Consolidation.Jobs(ctx, m.ID, 1); len(jobs) == 0 || time.Since(jobs[0].CreatedAt) >= m.Config.ConsolidationInterval.D() { d.Brain.Command(ctx, m.ID, "setmode", map[string]bool{"consolidating": true}) //nolint:errcheck if _, err := d.Consolidation.Run(ctx, m, "schedule", "noemad"); err != nil { d.Log.Warn("consolidation failed", "mind", m.Slug, "err", err) } d.Brain.Command(ctx, m.ID, "setmode", map[string]bool{"awake": true}) //nolint:errcheck } } if m.Config.DreamingEnabled && !dreaming.IsPaused() && dreaming.InWindow(m.Config.DreamHours, time.Now()) { if ds, _ := d.Dreaming.Dreams(ctx, m.ID, 1); len(ds) == 0 || time.Since(ds[0].CreatedAt) >= m.Config.DreamInterval.D() { d.Brain.Command(ctx, m.ID, "setmode", map[string]bool{"dreaming": true}) //nolint:errcheck if _, err := d.Dreaming.Run(ctx, m, "schedule", "noemad"); err != nil { d.Log.Warn("dream failed", "mind", m.Slug, "err", err) } d.Brain.Command(ctx, m.ID, "setmode", map[string]bool{"awake": true}) //nolint:errcheck } } if m.Config.ReflectionEnabled { if last, err := d.Reflection.Latest(ctx, m.ID); err != nil || time.Since(last.CreatedAt) >= m.Config.ReflectionInterval.D() { d.Brain.Command(ctx, m.ID, "setmode", map[string]bool{"reflecting": true}) //nolint:errcheck if _, err := d.Reflection.Run(ctx, m, "schedule", "noemad"); err != nil { d.Log.Warn("reflection failed", "mind", m.Slug, "err", err) } d.Brain.Command(ctx, m.ID, "setmode", map[string]bool{"awake": true}) //nolint:errcheck } } } return nil }