From 5816dea6b1c658b042fb5e469d6ceeccfd28d7f6 Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Sun, 25 Jan 2026 09:10:28 +0000 Subject: [PATCH] Add Go HTTP/SSE API server with embedded frontend Implements the API routes (jobs, progress events, snapshots, search, entities, source delivery with release) to net/http and serves the static frontend from the same binary on the same origin. --- main.go | 654 ++++++++++++++++++++++++++++++++++++++++ main_test.go | 257 ++++++++++++++++ web/api.mjs | 35 +++ web/app.js | 776 ++++++++++++++++++++++++++++++++++++++++++++++++ web/config.js | 4 + web/favicon.svg | 6 + web/index.html | 151 ++++++++++ web/renderer.js | 149 ++++++++++ web/styles.css | 319 ++++++++++++++++++++ 9 files changed, 2351 insertions(+) create mode 100644 main.go create mode 100644 main_test.go create mode 100644 web/api.mjs create mode 100644 web/app.js create mode 100644 web/config.js create mode 100644 web/favicon.svg create mode 100644 web/index.html create mode 100644 web/renderer.js create mode 100644 web/styles.css diff --git a/main.go b/main.go new file mode 100644 index 0000000..5bf53a6 --- /dev/null +++ b/main.go @@ -0,0 +1,654 @@ +package main + +import ( + "bufio" + "bytes" + "embed" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + "unicode/utf8" +) + +//go:embed web +var webFiles embed.FS + +type Job struct { + ID string `json:"id"` + SourceKind string `json:"sourceKind"` + Name string `json:"name"` + Phase string `json:"phase"` + Completed int `json:"completed"` + Total int `json:"total"` + Message string `json:"message"` + SnapshotID *string `json:"snapshotId"` + Error *string `json:"error"` + Warnings []string `json:"warnings"` + root string +} + +type Server struct { + jobsMu sync.Mutex + jobs map[string]*Job + snapshotsMu sync.Mutex + snapshots map[string]*Snapshot + workspace string // parent of every job directory; only paths under it are ever removed + cors string + static http.Handler +} + +func NewServer(workspace string) *Server { + cors := os.Getenv("CORS_ORIGIN") + if cors == "" || strings.ContainsAny(cors, "\r\n") { + cors = "*" + } + web, _ := fs.Sub(webFiles, "web") + return &Server{ + jobs: map[string]*Job{}, + snapshots: map[string]*Snapshot{}, + workspace: workspace, + cors: cors, + static: http.FileServerFS(web), + } +} + +func main() { + port := 4177 + if p, err := strconv.Atoi(os.Getenv("PORT")); err == nil { + port = p + } + flag.IntVar(&port, "port", port, "listening port (default $PORT or 4177)") + open := flag.Bool("open", false, "open the viewer in a browser") + flag.Parse() + + server := &http.Server{ + Addr: fmt.Sprintf("0.0.0.0:%d", port), + Handler: NewServer(filepath.Join(os.TempDir(), "codenavigator")), + ReadHeaderTimeout: 30 * time.Second, + IdleTimeout: 2 * time.Minute, + } + log.Printf("CodeNavigator listening on http://127.0.0.1:%d", port) + if *open { + _ = exec.Command("xdg-open", fmt.Sprintf("http://127.0.0.1:%d", port)).Start() + } + log.Fatal(server.ListenAndServe()) +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("Cache-Control", "no-store") + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Access-Control-Allow-Origin", s.cors) + h.Set("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS") + h.Set("Access-Control-Allow-Headers", "Content-Type, ngrok-skip-browser-warning") + h.Set("Access-Control-Max-Age", "86400") + + path := r.URL.Path + switch { + case r.Method == http.MethodOptions: + w.WriteHeader(http.StatusNoContent) + case path == "/api/health" && r.Method == http.MethodGet: + writeJSON(w, 200, map[string]any{"ok": true, "version": "0.1.0"}) + case path == "/api/jobs" && r.Method == http.MethodPost: + s.createJob(w, r) + case strings.HasPrefix(path, "/api/jobs/"): + s.jobRoute(w, r, strings.Split(strings.TrimPrefix(path, "/api/jobs/"), "/")) + case strings.HasPrefix(path, "/api/snapshots/"): + s.snapshotRoute(w, r, strings.Split(strings.TrimPrefix(path, "/api/snapshots/"), "/")) + case !strings.HasPrefix(path, "/api/") && (r.Method == http.MethodGet || r.Method == http.MethodHead): + s.static.ServeHTTP(w, r) + default: + writeError(w, 404, "Not found") + } +} + +func (s *Server) createJob(w http.ResponseWriter, r *http.Request) { + var request struct { + Kind string `json:"kind"` + Name *string `json:"name"` + URL *string `json:"url"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&request); err != nil || request.Kind == "" { + writeError(w, 400, "Invalid job request") + return + } + url := "" + if request.Kind == "github" { + if request.URL == nil { + writeError(w, 400, "Missing GitHub URL") + return + } + if url = cleanGithubURL(*request.URL); url == "" { + writeError(w, 400, "Use a public github.com owner/repository URL") + return + } + } + id := uniqueID("job") + root := filepath.Join(s.workspace, id) + if err := os.MkdirAll(root, 0o755); err != nil { + writeError(w, 500, err.Error()) + return + } + job := &Job{ID: id, SourceKind: request.Kind, Name: "Local codebase", Phase: "receiving", Message: "Waiting for files…", Warnings: []string{}, root: root} + if request.Name != nil { + job.Name = *request.Name + } + if request.Kind == "github" { + job.Phase, job.Message = "cloning", "Cloning repository…" + } + s.jobsMu.Lock() + s.jobs[id] = job + s.jobsMu.Unlock() + + if request.Kind == "github" { + go func() { + checkout := filepath.Join(root, "repo") + output, err := exec.Command("git", "clone", "--depth=1", "--single-branch", "--quiet", "--", url, checkout).CombinedOutput() + if err != nil { + s.failJob(id, fmt.Sprintf("git clone failed: %v %s", err, strings.TrimSpace(string(output)))) + s.removeWorkspace(root) + return + } + s.indexJob(id, checkout) + }() + } + writeJSON(w, 201, map[string]string{"jobId": id}) +} + +func (s *Server) jobRoute(w http.ResponseWriter, r *http.Request, parts []string) { + id, action := parts[0], "" + if len(parts) > 1 { + action = parts[1] + } + switch { + case r.Method == http.MethodGet && action == "": + if job, ok := s.jobSnapshot(id); ok { + writeJSON(w, 200, job) + } else { + writeError(w, 404, "Unknown job") + } + case r.Method == http.MethodGet && action == "events": + s.streamJobEvents(w, r, id) + case r.Method == http.MethodPost && action == "files": + relative, ok := safeRelativePath(r.URL.Query().Get("path")) + if !ok { + writeError(w, 400, "Unsafe file path") + return + } + root, ok := s.localJobRoot(id) + if !ok { + writeError(w, 404, "Unknown local job") + return + } + output := filepath.Join(root, "files", relative) + if err := writeUpload(output, http.MaxBytesReader(w, r.Body, 64<<20)); err != nil { + writeError(w, 400, "Could not store file") + return + } + s.jobsMu.Lock() + if job := s.jobs[id]; job != nil { + job.Completed++ + job.Total = job.Completed + job.Message = fmt.Sprintf("Received %d files", job.Completed) + } + s.jobsMu.Unlock() + writeJSON(w, 200, map[string]bool{"ok": true}) + case r.Method == http.MethodPost && action == "commit": + s.jobsMu.Lock() + job := s.jobs[id] + if job == nil || job.SourceKind != "local" { + s.jobsMu.Unlock() + writeError(w, 404, "Unknown local job") + return + } + job.Phase, job.Message = "scanning", "Scanning files…" + root := filepath.Join(job.root, "files") + s.jobsMu.Unlock() + go s.indexJob(id, root) + writeJSON(w, 202, map[string]bool{"ok": true}) + default: + writeError(w, 404, "Unknown job route") + } +} + +func writeUpload(output string, body io.Reader) error { + if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil { + return err + } + file, err := os.Create(output) + if err != nil { + return err + } + _, err = io.Copy(file, body) + return errors.Join(err, file.Close()) +} + +func (s *Server) snapshotRoute(w http.ResponseWriter, r *http.Request, parts []string) { + part := func(i int) string { + if i < len(parts) { + return parts[i] + } + return "" + } + id, action := part(0), part(1) + s.snapshotsMu.Lock() + snapshot := s.snapshots[id] + if snapshot == nil { + s.snapshotsMu.Unlock() + writeError(w, 404, "Unknown snapshot") + return + } + switch action { + case "": + body := map[string]any{ + "id": snapshot.ID, "name": snapshot.Name, "source": snapshot.Source, "fileCount": len(snapshot.Files), + "totalLines": snapshot.TotalLines, "definitions": snapshot.Definitions, "references": snapshot.References, + "edges": len(snapshot.Edges), "warnings": snapshot.Warnings, + } + s.snapshotsMu.Unlock() + writeJSON(w, 200, body) + case "scene": + body := sceneJSON(snapshot) + s.snapshotsMu.Unlock() + if sendJSON(w, 200, body) == nil { + // The response owned the bytes; release the duplicate previews. + s.snapshotsMu.Lock() + for i := range snapshot.Files { + snapshot.Files[i].Preview = "" + } + s.snapshotsMu.Unlock() + } + case "search": + hits := search(snapshot, r.URL.Query().Get("q")) + s.snapshotsMu.Unlock() + writeJSON(w, 200, hits) + case "entities": + entityID, err := strconv.ParseUint(part(2), 10, 32) + if err != nil || entityID == 0 || int(entityID) > len(snapshot.Files) { + s.snapshotsMu.Unlock() + writeError(w, 404, "Unknown entity") + return + } + file := &snapshot.Files[entityID-1] + switch part(3) { + case "": + body := marshal(file) + s.snapshotsMu.Unlock() + sendJSON(w, 200, body) + case "source": + s.serveSource(w, r, snapshot, file) // unlocks + default: + s.snapshotsMu.Unlock() + writeError(w, 404, "Unknown entity route") + } + default: + s.snapshotsMu.Unlock() + writeError(w, 404, "Unknown snapshot route") + } +} + +// serveSource is called with snapshotsMu held and releases it before writing. +func (s *Server) serveSource(w http.ResponseWriter, r *http.Request, snapshot *Snapshot, file *FileRecord) { + query := r.URL.Query() + start, err := strconv.Atoi(query.Get("start")) + if err != nil || start < 0 { + start = 0 + } + start = min(start, file.Lines) + limit, err := strconv.Atoi(query.Get("limit")) + if err != nil || limit < 0 { + limit = 600 + } + limit = min(max(limit, 1), 4096) + fileID, totalLines := file.ID, file.Lines + relative, ok := safeRelativePath(file.Path) + if !ok { + s.snapshotsMu.Unlock() + writeError(w, 400, "Unsafe file path") + return + } + sourcePath := filepath.Join(snapshot.Root, relative) + lines, err := readLines(sourcePath, start, limit) + s.snapshotsMu.Unlock() + if err != nil { + writeError(w, 404, "Source unavailable") + return + } + body := marshal(map[string]any{"start": start, "lines": lines, "totalLines": totalLines}) + if sendJSON(w, 200, body) != nil { + return + } + + s.snapshotsMu.Lock() + defer s.snapshotsMu.Unlock() + if snapshot.released[fileID] { + return + } + ranges, complete := recordDelivery(snapshot.delivered[fileID], start, start+len(lines), totalLines) + snapshot.delivered[fileID] = ranges + if complete { + snapshot.released[fileID] = true + os.Remove(sourcePath) + if len(snapshot.released) == len(snapshot.Files) { + s.removeWorkspace(snapshot.Root) + } + } +} + +// readLines returns up to limit lines after skipping start, stopping at the first +// invalid UTF-8 line like Rust's BufRead::lines. +func readLines(path string, start, limit int) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64*1024), 9*1024*1024) + lines := []string{} + for index := 0; len(lines) < limit && scanner.Scan(); index++ { + if index < start { + continue + } + if !utf8.Valid(scanner.Bytes()) { + break + } + lines = append(lines, scanner.Text()) + } + return lines, nil +} + +type sceneFile struct { + ID uint32 `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Directory string `json:"directory"` + Extension string `json:"extension"` + Language string `json:"language"` + Layer string `json:"layer"` + Lines int `json:"lines"` + Bytes int `json:"bytes"` + Complexity int `json:"complexity"` + Preview string `json:"preview"` + SymbolCount int `json:"symbolCount"` +} + +func sceneJSON(snapshot *Snapshot) []byte { + files := make([]sceneFile, len(snapshot.Files)) + for i, f := range snapshot.Files { + files[i] = sceneFile{f.ID, f.Path, f.Name, f.Directory, f.Extension, f.Language, f.Layer, f.Lines, f.Bytes, f.Complexity, f.Preview, len(f.Symbols)} + } + return marshal(struct { + ID string `json:"id"` + Name string `json:"name"` + Source string `json:"source"` + Files []sceneFile `json:"files"` + Edges []Edge `json:"edges"` + TotalLines int `json:"totalLines"` + Definitions int `json:"definitions"` + References int `json:"references"` + }{snapshot.ID, snapshot.Name, snapshot.Source, files, snapshot.Edges, snapshot.TotalLines, snapshot.Definitions, snapshot.References}) +} + +type searchHit struct { + EntityID uint32 `json:"entityId"` + Path string `json:"path"` + Name string `json:"name"` + Line int `json:"line"` + Kind string `json:"kind"` + Preview string `json:"preview"` +} + +func search(snapshot *Snapshot, query string) []searchHit { + const maxHits = 180 + needle := strings.ToLower(query) + hits := []searchHit{} + if len(needle) < 2 { + return hits + } + for _, file := range snapshot.Files { + if len(hits) >= maxHits { + break + } + if strings.Contains(strings.ToLower(file.Path), needle) { + firstLine, _, _ := strings.Cut(file.Preview, "\n") + hits = append(hits, searchHit{file.ID, file.Path, file.Name, 1, "file", strings.TrimSuffix(firstLine, "\r")}) + } + for _, symbol := range file.Symbols { + if len(hits) >= maxHits { + break + } + if strings.Contains(strings.ToLower(symbol.Name), needle) { + hits = append(hits, searchHit{file.ID, file.Path, symbol.Name, symbol.Line, "definition", symbol.Signature}) + } + } + } + return hits +} + +func (s *Server) streamJobEvents(w http.ResponseWriter, r *http.Request, id string) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(200) + if flusher != nil { + flusher.Flush() + } + last := "" + ticker := time.NewTicker(150 * time.Millisecond) + defer ticker.Stop() + for range 1200 { + job, ok := s.jobSnapshot(id) + if !ok { + return + } + encoded := marshal(job) + if string(encoded) != last { + if _, err := fmt.Fprintf(w, "event: progress\ndata: %s\n\n", encoded); err != nil { + return + } + if flusher != nil { + flusher.Flush() + } + last = string(encoded) + } + if job.SnapshotID != nil || job.Error != nil { + return + } + select { + case <-r.Context().Done(): + return + case <-ticker.C: + } + } +} + +func (s *Server) jobSnapshot(id string) (Job, bool) { + s.jobsMu.Lock() + defer s.jobsMu.Unlock() + job := s.jobs[id] + if job == nil { + return Job{}, false + } + return *job, true +} + +func (s *Server) localJobRoot(id string) (string, bool) { + s.jobsMu.Lock() + defer s.jobsMu.Unlock() + job := s.jobs[id] + if job == nil || job.SourceKind != "local" { + return "", false + } + return job.root, true +} + +func (s *Server) indexJob(id, root string) { + s.updateJob(id, "scanning", 0, 0, "Discovering source files…") + snapshot, err := indexDirectory(root, func(phase string, completed, total int, message string) { + s.updateJob(id, phase, completed, total, message) + }) + if err != nil { + s.removeWorkspace(root) + s.failJob(id, err.Error()) + return + } + s.jobsMu.Lock() + if job := s.jobs[id]; job != nil { + snapshot.Name, snapshot.Source = job.Name, job.SourceKind + } + s.jobsMu.Unlock() + snapshot.ID = uniqueID("snapshot") + + // Only the latest snapshot is kept; older workspaces are deleted. + s.snapshotsMu.Lock() + var oldRoots []string + for _, old := range s.snapshots { + oldRoots = append(oldRoots, old.Root) + } + clear(s.snapshots) + s.snapshots[snapshot.ID] = snapshot + s.snapshotsMu.Unlock() + for _, old := range oldRoots { + if old != root { + s.removeWorkspace(old) + } + } + + s.jobsMu.Lock() + if job := s.jobs[id]; job != nil { + job.Phase, job.Message = "ready", "Landscape ready" + job.SnapshotID = &snapshot.ID + job.Completed = max(job.Total, job.Completed) + } + s.jobsMu.Unlock() +} + +func (s *Server) updateJob(id, phase string, completed, total int, message string) { + s.jobsMu.Lock() + defer s.jobsMu.Unlock() + if job := s.jobs[id]; job != nil { + job.Phase, job.Completed, job.Total, job.Message = phase, completed, total, message + } +} + +func (s *Server) failJob(id, message string) { + s.jobsMu.Lock() + defer s.jobsMu.Unlock() + if job := s.jobs[id]; job != nil { + job.Phase, job.Message, job.Error = "error", "Indexing failed", &message + } +} + +// removeWorkspace deletes a checkout and its now-empty job directory, never leaving the workspace. +func (s *Server) removeWorkspace(root string) { + if !strings.HasPrefix(root, s.workspace+string(filepath.Separator)) { + return + } + os.RemoveAll(root) + if parent := filepath.Dir(root); parent != s.workspace && strings.HasPrefix(parent, s.workspace) { + os.Remove(parent) + } +} + +// recordDelivery merges [start,end) into ranges and reports whether [0,total) is fully covered. +func recordDelivery(ranges [][2]int, start, end, total int) ([][2]int, bool) { + if end > start { + ranges = append(ranges, [2]int{start, min(end, total)}) + slices.SortFunc(ranges, func(a, b [2]int) int { return a[0] - b[0] }) + merged := ranges[:1] + for _, r := range ranges[1:] { + if last := &merged[len(merged)-1]; r[0] <= last[1] { + last[1] = max(last[1], r[1]) + } else { + merged = append(merged, r) + } + } + ranges = merged + } + return ranges, total == 0 || (len(ranges) > 0 && ranges[0][0] == 0 && ranges[0][1] >= total) +} + +// cleanGithubURL returns the normalised https://github.com/owner/repo URL, or "" if invalid. +func cleanGithubURL(value string) string { + clean := strings.TrimSuffix(strings.TrimRight(strings.TrimSpace(value), "/"), ".git") + rest, ok := strings.CutPrefix(clean, "https://github.com/") + if !ok { + return "" + } + owner, repo, ok := strings.Cut(rest, "/") + if !ok || !slug(owner) || !slug(repo) { + return "" + } + return clean +} + +func slug(s string) bool { + return s != "" && strings.Trim(s, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.") == "" +} + +func safeRelativePath(value string) (string, bool) { + if value == "" || strings.HasPrefix(value, "/") || strings.HasPrefix(value, "./") || value == "." { + return "", false + } + var clean []string + for part := range strings.SplitSeq(value, "/") { + switch part { + case "", ".": + case "..": + return "", false + default: + clean = append(clean, part) + } + } + if len(clean) == 0 { + return "", false + } + return filepath.Join(clean...), true +} + +var idCounter atomic.Uint64 + +func uniqueID(prefix string) string { + return fmt.Sprintf("%s-%x%x", prefix, time.Now().UnixNano(), idCounter.Add(1)) +} + +func writeError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]string{"error": message}) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + sendJSON(w, status, marshal(value)) +} + +// marshal skips HTML escaping: source previews are full of < > & and the payloads are large. +func marshal(value any) []byte { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if encoder.Encode(value) != nil { + return []byte(`{"error":"serialization failed"}`) + } + return bytes.TrimSuffix(buf.Bytes(), []byte("\n")) +} + +func sendJSON(w http.ResponseWriter, status int, body []byte) error { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(status) + _, err := w.Write(body) + return err +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..9448d0f --- /dev/null +++ b/main_test.go @@ -0,0 +1,257 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestDeliveryRangesMergeBeforeSourceIsReleased(t *testing.T) { + var ranges [][2]int + var done bool + for _, step := range []struct { + start, end int + want bool + }{{600, 1000, false}, {0, 400, false}, {350, 550, false}, {500, 700, true}} { + if ranges, done = recordDelivery(ranges, step.start, step.end, 1000); done != step.want { + t.Fatalf("recordDelivery(%d,%d) = %v, want %v", step.start, step.end, done, step.want) + } + } + if !reflect.DeepEqual(ranges, [][2]int{{0, 1000}}) { + t.Fatalf("ranges = %v", ranges) + } +} + +func TestEmptyOrDuplicateDeliveryDoesNotFakeCoverage(t *testing.T) { + var ranges [][2]int + var done bool + for _, end := range []int{0, 5, 5} { + if ranges, done = recordDelivery(ranges, 0, end, 10); done { + t.Fatalf("coverage reported after [0,%d)", end) + } + } + if !reflect.DeepEqual(ranges, [][2]int{{0, 5}}) { + t.Fatalf("ranges = %v", ranges) + } +} + +func TestGithubURLValidation(t *testing.T) { + for in, want := range map[string]string{ + "https://github.com/makepad/makepad": "https://github.com/makepad/makepad", + " https://github.com/a/b.git/ ": "https://github.com/a/b", + "https://example.com/a/b": "", + "https://github.com/a/b/tree/main": "", + "https://github.com/a/b;rm -rf": "", + "https://github.com/-/--upload-pack=": "", + } { + if got := cleanGithubURL(in); got != want { + t.Errorf("cleanGithubURL(%q) = %q, want %q", in, got, want) + } + } +} + +func TestPathValidation(t *testing.T) { + for in, ok := range map[string]bool{"src/main.rs": true, "a/./b": true, "../secret": false, "a/../../b": false, "/etc/passwd": false, "./a": false, "": false, ".": false} { + if _, got := safeRelativePath(in); got != ok { + t.Errorf("safeRelativePath(%q) ok = %v, want %v", in, got, ok) + } + } +} + +type client struct { + t *testing.T + url string +} + +func (c client) do(method, path, body string) (int, []byte) { + c.t.Helper() + req, _ := http.NewRequest(method, c.url+path, strings.NewReader(body)) + res, err := http.DefaultClient.Do(req) + if err != nil { + c.t.Fatal(err) + } + defer res.Body.Close() + data, _ := io.ReadAll(res.Body) + return res.StatusCode, data +} + +func (c client) json(method, path, body string, wantStatus int, out any) { + c.t.Helper() + status, data := c.do(method, path, body) + if status != wantStatus { + c.t.Fatalf("%s %s = %d %s, want %d", method, path, status, data, wantStatus) + } + if out != nil { + if err := json.Unmarshal(data, out); err != nil { + c.t.Fatalf("%s %s: %v in %s", method, path, err, data) + } + } +} + +func newTestServer(t *testing.T) (*Server, client) { + server := NewServer(filepath.Join(t.TempDir(), "codenavigator")) + ts := httptest.NewServer(server) + t.Cleanup(ts.Close) + return server, client{t, ts.URL} +} + +func TestLocalJobLifecycle(t *testing.T) { + server, c := newTestServer(t) + var created struct{ JobID string } + c.json("POST", "/api/jobs", `{"kind":"local","name":"demo"}`, 201, &created) + + sources := map[string]string{ + "src/app.js": "import { helper } from './helper.js';\nfunction start() {\n helper();\n}\n", + "src/helper.js": "export function helper() {\n return '';\n}\n", + } + for path, content := range sources { + c.json("POST", "/api/jobs/"+created.JobID+"/files?path="+strings.ReplaceAll(path, "/", "%2F"), content, 200, nil) + } + c.json("POST", "/api/jobs/"+created.JobID+"/files?path=..%2Fescape.js", "x", 400, nil) + c.json("POST", "/api/jobs/"+created.JobID+"/commit", "", 202, nil) + + res, err := http.Get(c.url + "/api/jobs/" + created.JobID + "/events") + if err != nil { + t.Fatal(err) + } + if ct := res.Header.Get("Content-Type"); ct != "text/event-stream" { + t.Fatalf("events content type = %q", ct) + } + var job Job + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + if data, ok := strings.CutPrefix(scanner.Text(), "data: "); ok { + job = Job{} + json.Unmarshal([]byte(data), &job) + } + } + res.Body.Close() + if job.SnapshotID == nil || job.Phase != "ready" { + t.Fatalf("final job event = %+v", job) + } + snapshotURL := "/api/snapshots/" + *job.SnapshotID + + var summary map[string]any + c.json("GET", snapshotURL, "", 200, &summary) + if summary["fileCount"] != 2.0 || summary["references"] != 1.0 || summary["name"] != "demo" { + t.Fatalf("summary = %v", summary) + } + + status, raw := c.do("GET", snapshotURL+"/scene", "") + if status != 200 || !strings.Contains(string(raw), `''`) { + t.Fatalf("scene = %d %s (previews must not be HTML-escaped)", status, raw) + } + var scene struct { + Files []sceneFile + Edges []Edge + } + json.Unmarshal(raw, &scene) + if len(scene.Files) != 2 || scene.Files[0].Path != "src/app.js" || scene.Files[0].SymbolCount != 1 || len(scene.Edges) != 1 { + t.Fatalf("scene = %+v", scene) + } + if server.snapshots[*job.SnapshotID].Files[0].Preview != "" { + t.Fatal("previews should be released after the scene is delivered") + } + + var hits []searchHit + c.json("GET", snapshotURL+"/search?q=HELP", "", 200, &hits) + if len(hits) != 2 || hits[0].Kind != "file" || hits[1].Name != "helper" || hits[1].Line != 1 { + t.Fatalf("hits = %+v", hits) + } + c.json("GET", snapshotURL+"/search?q=h", "", 200, &hits) + if len(hits) != 0 { + t.Fatalf("single-character search returned %v", hits) + } + + var entity FileRecord + c.json("GET", snapshotURL+"/entities/1", "", 200, &entity) + if entity.Path != "src/app.js" || entity.Imports == nil || len(entity.Symbols) != 1 { + t.Fatalf("entity = %+v", entity) + } + c.json("GET", snapshotURL+"/entities/99", "", 404, nil) + c.json("GET", snapshotURL+"/entities/1/nope", "", 404, nil) + + root := server.snapshots[*job.SnapshotID].Root + var chunk struct { + Start, TotalLines int + Lines []string + } + c.json("GET", snapshotURL+"/entities/1/source?start=2&limit=5", "", 200, &chunk) + if chunk.Start != 2 || chunk.TotalLines != 4 || !reflect.DeepEqual(chunk.Lines, []string{" helper();", "}"}) { + t.Fatalf("chunk = %+v", chunk) + } + if _, err := os.Stat(filepath.Join(root, "src/app.js")); err != nil { + t.Fatal("partially delivered source was released early") + } + c.json("GET", snapshotURL+"/entities/1/source?start=0&limit=2", "", 200, &chunk) + if _, err := os.Stat(filepath.Join(root, "src/app.js")); !os.IsNotExist(err) { + t.Fatal("fully delivered source should be deleted") + } + c.json("GET", snapshotURL+"/entities/2/source", "", 200, &chunk) + if _, err := os.Stat(filepath.Dir(root)); !os.IsNotExist(err) { + t.Fatal("workspace should be removed once every source is delivered") + } + c.json("GET", snapshotURL+"/entities/2/source", "", 404, nil) +} + +func TestRoutesAndHeaders(t *testing.T) { + _, c := newTestServer(t) + var health map[string]any + c.json("GET", "/api/health", "", 200, &health) + if health["ok"] != true { + t.Fatalf("health = %v", health) + } + c.json("POST", "/api/jobs", `not json`, 400, nil) + c.json("POST", "/api/jobs", `{"kind":"github"}`, 400, nil) + c.json("POST", "/api/jobs", `{"kind":"github","url":"https://evil.example/a/b"}`, 400, nil) + c.json("GET", "/api/jobs/missing", "", 404, nil) + c.json("POST", "/api/jobs/missing/commit", "", 404, nil) + c.json("GET", "/api/snapshots/missing/scene", "", 404, nil) + c.json("GET", "/api/unknown", "", 404, nil) + + res, err := http.DefaultClient.Do(must(http.NewRequest("OPTIONS", c.url+"/api/jobs", nil))) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 204 || res.Header.Get("Access-Control-Allow-Origin") != "*" || !strings.Contains(res.Header.Get("Access-Control-Allow-Headers"), "ngrok-skip-browser-warning") { + t.Fatalf("preflight = %d %v", res.StatusCode, res.Header) + } + + for path, contentType := range map[string]string{"/": "text/html", "/api.mjs": "text/javascript", "/styles.css": "text/css", "/config.js": "text/javascript"} { + res, err := http.Get(c.url + path) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != 200 || !strings.HasPrefix(res.Header.Get("Content-Type"), contentType) { + t.Errorf("GET %s = %d %s", path, res.StatusCode, res.Header.Get("Content-Type")) + } + } +} + +func TestFailedIndexReportsErrorAndCleansUp(t *testing.T) { + server, c := newTestServer(t) + var created struct{ JobID string } + c.json("POST", "/api/jobs", `{"kind":"local"}`, 201, &created) + server.indexJob(created.JobID, filepath.Join(server.workspace, created.JobID, "files")) + var job Job + c.json("GET", "/api/jobs/"+created.JobID, "", 200, &job) + if job.Phase != "error" || job.Error == nil || job.Name != "Local codebase" { + t.Fatalf("job = %+v", job) + } +} + +func must[T any](v T, err error) T { + if err != nil { + panic(fmt.Sprint(err)) + } + return v +} diff --git a/web/api.mjs b/web/api.mjs new file mode 100644 index 0000000..ca40c40 --- /dev/null +++ b/web/api.mjs @@ -0,0 +1,35 @@ +export function apiFetch(url, options = {}) { + const headers = new Headers(options.headers); + headers.set('ngrok-skip-browser-warning', 'true'); + return fetch(url, { ...options, headers }); +} + +// Read SSE over fetch so progress requests can carry the same headers as JSON. +export async function* progressEvents(response) { + if (!response.ok || !response.body || !response.headers.get('content-type')?.includes('text/event-stream')) { + throw new Error('Could not connect to indexing progress.'); + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = '', event = '', data = []; + try { + while (true) { + const { value, done } = await reader.read(); + buffer += decoder.decode(value, { stream: !done }); + let end; + while ((end = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, end).replace(/\r$/, ''); + buffer = buffer.slice(end + 1); + if (!line) { + if (event === 'progress' && data.length) yield JSON.parse(data.join('\n')); + event = ''; data = []; + } else if (line.startsWith('event:')) event = line.slice(6).trim(); + else if (line.startsWith('data:')) data.push(line.slice(5).replace(/^ /, '')); + } + if (done) break; + } + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } +} diff --git a/web/app.js b/web/app.js new file mode 100644 index 0000000..6f3ec19 --- /dev/null +++ b/web/app.js @@ -0,0 +1,776 @@ +import { LandscapeRenderer } from './renderer.js'; +import { apiFetch, progressEvents } from './api.mjs'; + +const configuredBackend=window.CODENAVIGATOR_CONFIG?.backendUrl?.trim()||''; +const API_BASE=configuredBackend.replace(/\/+$/,''); +const apiUrl=path=>`${API_BASE}${path}`; +const $ = selector => document.querySelector(selector); +const $$ = selector => [...document.querySelectorAll(selector)]; +document.addEventListener('pointerdown',()=>{document.documentElement.dataset.inputMethod='pointer';},true); +document.addEventListener('keydown',()=>{document.documentElement.dataset.inputMethod='keyboard';},true); +const glCanvas = $('#glCanvas'); +const overlay = $('#overlayCanvas'); +const overlayCtx = overlay.getContext('2d'); +const viewport = $('#viewport'); +const MAX_2D_ZOOM=260; +const MIN_3D_DISTANCE=.52; +let renderer; +try { renderer = new LandscapeRenderer(glCanvas); } +catch (error) { $('#emptyState').hidden=false; $('#emptyState strong').textContent='WebGL2 is unavailable'; $('#emptyState span').textContent=error.message; throw error; } + +const palettes = [ + { application:'#1fbf8c', library:'#28aa8b', test:'#c7a62e', generated:'#cc7741', vendor:'#4e83bc', platform:'#d05492', docs:'#8d9561', unknown:'#65706a' }, + { application:'#48a9e6', library:'#7874db', test:'#e6a04b', generated:'#d6627c', vendor:'#45b39d', platform:'#b781d4', docs:'#9ca35c', unknown:'#697277' }, + { application:'#87b95d', library:'#39a5a0', test:'#e2bf52', generated:'#e37d52', vendor:'#527ac2', platform:'#c76699', docs:'#a4a4a4', unknown:'#6b706d' }, +]; +let paletteIndex=0; +let files=[]; +let edges=[]; +let layoutItems=[]; +let directoryRects=[]; +let fileById=new Map(); +let layoutById=new Map(); +let referenceDegree=new Map(); +let edgesByFile=new Map(); +let sceneStats={definitions:0,totalLines:0,known:0,inferred:0,layers:new Map()}; +let spatialGrid=new Map(); +let visibleScreenItems=[]; +let overviewCanvas=null,overviewContext=null,overviewIndex=0; +let codeTextureBytes=0; +const codeTextureLru=new Map(),codeTextureBudget=192*1024*1024,overviewScale=3; +let selected=null; +let history=[]; +let searchHits=[]; +let expandedCoverageLayer=null; +let coverageActiveIndex=-1; +let cameraAnimation=null; +let currentSnapshot=null; +let currentName=''; +let currentMetric='lines'; +let showCode=true; +let showLegend=false; +let dirty=true; +let pointer={down:false,x:0,y:0,startX:0,startY:0,action:'orbit',id:null}; +let lastFrame=performance.now(), frameSamples=[]; + +const layerLabels = { application:'Application',library:'Libraries',test:'Tests',generated:'Generated',vendor:'Vendor',platform:'Platform',docs:'Documentation',unknown:'Other' }; + +function toRgb(hex){const value=parseInt(hex.slice(1),16);return[((value>>16)&255)/255,((value>>8)&255)/255,(value&255)/255];} +function fileWeight(file){ + if(currentMetric==='references'){ + const degree=referenceDegree.get(file.id)||0; + return Math.max(1,degree*55+(file.symbolCount??file.symbols?.length??0)*7); + } + return Math.max(1,file.lines); +} + +function makeTree(list){ + // Repositories frequently keep source files at their root. Root participates + // in the same layout contract as every child directory. + const root={name:currentName,path:'',children:new Map(),files:[],weight:0,depth:0}; + for(const file of list){ + const parts=file.path.split('/'), filename=parts.pop(); let node=root; + for(const part of parts){ + if(!node.children.has(part)) node.children.set(part,{name:part,path:node.path?`${node.path}/${part}`:part,children:new Map(),files:[],weight:0,depth:node.depth+1}); + node=node.children.get(part); + } + node.files.push({name:filename,file,weight:fileWeight(file)}); + } + function weigh(node){ + let weight=0;const layers=new Map(); + for(const item of node.files||[]){weight+=item.weight;const layer=item.file.layer in layerLabels?item.file.layer:'unknown';layers.set(layer,(layers.get(layer)||0)+item.weight);} + for(const child of node.children.values()){const childWeight=weigh(child);weight+=childWeight;layers.set(child.layer,(layers.get(child.layer)||0)+childWeight);} + node.weight=weight;node.layer=[...layers.entries()].sort((a,b)=>b[1]-a[1])[0]?.[0]||'unknown';return weight; + } + weigh(root); return root; +} + +function squarifiedLayout(entries,x,y,w,h,depth,output){ + if(!entries.length||w<=0||h<=0)return; + const total=entries.reduce((sum,entry)=>sum+Math.max(.0001,entry.weight),0),scale=w*h/total; + const remaining=entries.map(entry=>({entry,area:Math.max(.0001,entry.weight)*scale})); + const worst=(row,side)=>{ + if(!row.length)return Infinity; + const sum=row.reduce((value,item)=>value+item.area,0),largest=Math.max(...row.map(item=>item.area)),smallest=Math.min(...row.map(item=>item.area)),sideSquared=side*side; + return Math.max(sideSquared*largest/(sum*sum),(sum*sum)/(sideSquared*smallest)); + }; + const place=row=>{ + const area=row.reduce((value,item)=>value+item.area,0); + if(w>=h){ + const rowWidth=Math.min(w,area/Math.max(.0001,h));let rowY=y; + for(const item of row){const itemHeight=item.area/Math.max(.0001,rowWidth);output(item.entry,x,rowY,rowWidth,itemHeight,depth);rowY+=itemHeight;} + x+=rowWidth;w=Math.max(0,w-rowWidth); + }else{ + const rowHeight=Math.min(h,area/Math.max(.0001,w));let rowX=x; + for(const item of row){const itemWidth=item.area/Math.max(.0001,rowHeight);output(item.entry,rowX,y,itemWidth,rowHeight,depth);rowX+=itemWidth;} + y+=rowHeight;h=Math.max(0,h-rowHeight); + } + }; + let row=[]; + while(remaining.length){ + const next=remaining[0],side=Math.max(.0001,Math.min(w,h)); + if(!row.length||worst([...row,next],side)<=worst(row,side)){row.push(remaining.shift());} + else{place(row);row=[];} + } + if(row.length)place(row); +} + +function computeLayout(){ + const root=makeTree(files); layoutItems=[];directoryRects=[]; + function visit(node,x,y,w,h,depth){ + directoryRects.push({node,x,y,w,h,depth}); + const pad=Math.min(3.2,1.2+depth*.35,w*.07,h*.07),header=Math.min(8,Math.max(0,h*.05)),ix=x+pad,iy=y+pad+header,iw=Math.max(.01,w-pad*2),ih=Math.max(.01,h-pad*2-header); + const values=[...node.children.values(),...(node.files||[])],total=values.reduce((sum,value)=>sum+value.weight,0); + // A file should be compared with its siblings, not visually erased by the + // aggregate weight of a huge neighboring subtree. The local floor keeps + // every entry readable while leaving at least 87.5% of each region fully + // proportional to the selected metric. + const floor=total/Math.max(1,values.length*8); + const entries=values.map(value=>({value,weight:Math.max(value.weight,floor)})).sort((a,b)=>b.weight-a.weight); + squarifiedLayout(entries,ix,iy,iw,ih,depth,(wrapped,rx,ry,rw,rh)=>{ + const entry=wrapped.value; + if(entry.file){ + const inset=Math.min(1,Math.min(rw,rh)*.08),layer=entry.file.layer in palettes[paletteIndex]?entry.file.layer:'unknown'; + layoutItems.push({...entry.file,x:rx+inset,y:ry+inset,w:Math.max(.1,rw-inset*2),h:Math.max(.1,rh-inset*2),color:toRgb(palettes[paletteIndex][layer]),height:4+Math.log2(entry.file.lines+entry.file.complexity*5)*5}); + }else visit(entry,rx,ry,rw,rh,depth+1); + }); + } + visit(root,0,0,1000,680,0); + layoutById=new Map(layoutItems.map(item=>[item.id,item]));buildSpatialGrid();resetCodeCaches(); + renderer.setData(layoutItems);dirty=true; +} + +function resetCodeCaches(){ + overviewCanvas=document.createElement('canvas');overviewCanvas.width=1000*overviewScale;overviewCanvas.height=680*overviewScale;overviewContext=overviewCanvas.getContext('2d',{alpha:true});overviewIndex=0;codeTextureBytes=0;codeTextureLru.clear(); +} + +const spatialCell=50; +function buildSpatialGrid(){ + spatialGrid=new Map(); + for(const item of layoutItems){ + const x0=Math.floor(item.x/spatialCell),x1=Math.floor((item.x+item.w)/spatialCell),y0=Math.floor(item.y/spatialCell),y1=Math.floor((item.y+item.h)/spatialCell); + for(let y=y0;y<=y1;y++)for(let x=x0;x<=x1;x++){const key=`${x}:${y}`,cell=spatialGrid.get(key);if(cell)cell.push(item);else spatialGrid.set(key,[item]);} + } +} +function visibleLayout(w,h){ + let candidates=layoutItems; + if(renderer.mode==='2d'&&renderer.camera.zoom>1){ + const halfW=w/(2*renderer.camera.zoom),halfH=h/(2*renderer.camera.zoom),x0=Math.floor((renderer.camera.x-halfW)/spatialCell),x1=Math.floor((renderer.camera.x+halfW)/spatialCell),y0=Math.floor((renderer.camera.y-halfH)/spatialCell),y1=Math.floor((renderer.camera.y+halfH)/spatialCell),seen=new Set(),near=[]; + for(let y=y0;y<=y1;y++)for(let x=x0;x<=x1;x++)for(const item of spatialGrid.get(`${x}:${y}`)||[])if(!seen.has(item.id)){seen.add(item.id);near.push(item);} + candidates=near; + } + const visible=[]; + for(const item of candidates){const rect=renderer.screenRect(item);if(rect.x>w||rect.y>h||rect.x+rect.w<0||rect.y+rect.h<0)continue;visible.push({item,rect});} + return visible; +} + +function applySnapshot(snapshot){ + sourceQueue.length=0;queuedSources.clear(); + files=snapshot.files||[];for(const file of files)file.symbols||=[];edges=snapshot.edges||[];currentName=snapshot.name||'codebase';currentSnapshot=snapshot.id||null;selected=null;history=[];searchHits=[];expandedCoverageLayer=null;coverageActiveIndex=-1;cameraAnimation=null; + fileById=new Map(files.map(file=>[file.id,file]));referenceDegree=new Map();edgesByFile=new Map();let known=0,inferred=0;for(const edge of edges){referenceDegree.set(edge.from,(referenceDegree.get(edge.from)||0)+1);referenceDegree.set(edge.to,(referenceDegree.get(edge.to)||0)+1);for(const id of [edge.from,edge.to]){const list=edgesByFile.get(id);if(list)list.push(edge);else edgesByFile.set(id,[edge]);}if(edge.confidence==='known')known++;else inferred++;} + const layers=new Map();for(const file of files){const layer=file.layer in layerLabels?file.layer:'unknown';layers.set(layer,(layers.get(layer)||0)+1);}sceneStats={definitions:snapshot.definitions??files.reduce((n,file)=>n+(file.symbolCount||0),0),totalLines:snapshot.totalLines??files.reduce((n,file)=>n+file.lines,0),known,inferred,layers}; + computeLayout();fitScene();updateStats(snapshot);renderInspector();renderResults([]);renderHistory(); + $('#breadcrumbText').textContent=currentName;$('#emptyState').hidden=files.length>0; +} + +function fitScene(){ + cameraAnimation=null; + renderer.camera.x=500;renderer.camera.y=340; + renderer.camera.zoom=Math.min((viewport.clientWidth-24)/1000,(viewport.clientHeight-24)/680); + renderer.camera.distance=3.5;renderer.camera.yaw=-.12;renderer.camera.pitch=.76;dirty=true; +} + +function pan3d(dx,dy){ + const {yaw,distance}=renderer.camera; + const scale=Math.max(.18,distance*.29); + const cy=Math.cos(yaw),sy=Math.sin(yaw); + // Keep the grabbed ground point beneath the pointer in screen space. + renderer.camera.x+=(-dx*cy-dy*sy)*scale; + renderer.camera.y+=( dx*sy-dy*cy)*scale; +} + +function dolly3d(amount){ + renderer.camera.distance=Math.max(MIN_3D_DISTANCE,Math.min(14,renderer.camera.distance*Math.exp(amount))); +} + +function updateStats(snapshot={}){ + const definitions=snapshot.definitions??files.reduce((n,file)=>n+file.symbols.length,0); + const references=snapshot.references??edges.length; + $('#headlineStats').textContent=`${format(definitions)} definitions · ${format(references)} references`; + $('#fileStats').textContent=`${format(files.length)} files · ${format(snapshot.totalLines??files.reduce((n,f)=>n+f.lines,0))} lines`; +} + +function format(value){return Number(value||0).toLocaleString();} +function escapeHtml(value=''){return String(value).replace(/[&<>"]/g,char=>({'&':'&','<':'<','>':'>','"':'"'}[char]));} + +const syntaxPalette={plain:'#aab5af',comment:'#64726b',string:'#89b88f',number:'#83a8d8',keyword:'#d8a75b',type:'#79b8d8',function:'#d6cf8b',property:'#b397d5',operator:'#8d9c95',constant:'#d98585',tag:'#65b9aa'}; +const syntaxKeywords=new Set(`abstract as async await break case catch class const continue crate def default defer delete do else enum export extends extern false final finally fn for from func function get go if impl import in instanceof interface is lambda let loop match mod module mut namespace new nil None null of override package private protected pub public raise readonly require return self set static struct super switch this throw trait true try type typeof undefined union unsafe use using var virtual void where while with yield`.split(' ')); +const syntaxConstants=new Set('true false null nil none undefined nan infinity self this super'.split(' ')); +const tokenPattern=/"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`|\b0x[\da-f]+\b|\b0b[01]+\b|#[\da-f]{3,8}\b|\b\d+(?:\.\d+)?(?:e[+-]?\d+)?\b|[A-Za-z_$][\w$-]*|\s+|./gi; + +function syntaxKind(file={}){ + const ext=(file.extension||file.name?.split('.').pop()||'').toLowerCase(),language=(file.language||'').toLowerCase(); + if(['html','htm','xml','svg','vue','svelte'].includes(ext))return'markup'; + if(['css','scss','sass','less'].includes(ext))return'css'; + if(['json','jsonc','toml','yaml','yml'].includes(ext))return'data'; + if(['md','mdx','rst','txt'].includes(ext)||language==='docs')return'docs'; + if(['py','pyi','rb','r','pl','sh','bash','zsh','fish'].includes(ext))return'hash-comment'; + if(['sql','lua','hs'].includes(ext))return'dash-comment'; + return language==='other'?'plain':'code'; +} + +function commentStart(line,kind){ + const markers=kind==='markup'?['