package main import ( "bytes" "fmt" "io/fs" "os" "path/filepath" "regexp" "runtime" "strings" "sync" "sync/atomic" "unicode/utf8" ) type Symbol struct { Name string `json:"name"` Kind string `json:"kind"` Line int `json:"line"` Signature string `json:"signature"` } type FileRecord struct { ID uint32 `json:"id"` Path string `json:"path"` Name string `json:"name"` Directory string `json:"directory"` Extension string `json:"extension"` Language string `json:"language"` Layer string `json:"layer"` Lines int `json:"lines"` Bytes int `json:"bytes"` Complexity int `json:"complexity"` Preview string `json:"preview"` Symbols []Symbol `json:"symbols"` Imports []string `json:"imports"` } type Edge struct { From uint32 `json:"from"` To uint32 `json:"to"` Kind string `json:"kind"` Confidence string `json:"confidence"` } type Snapshot struct { ID, Name, Source string Files []FileRecord Edges []Edge TotalLines int Definitions int References int Warnings []string Root string delivered map[uint32][][2]int released map[uint32]bool } type definitionPattern struct { kind string re *regexp.Regexp } var definitionPatterns = []definitionPattern{ {"class", regexp.MustCompile(`(?:class|struct|enum|trait|interface|record)\s+([A-Za-z_][A-Za-z0-9_]*)`)}, {"function", regexp.MustCompile(`(?:fn|func|def|function)\s+([A-Za-z_][A-Za-z0-9_]*)`)}, {"function", regexp.MustCompile(`(?:public|private|protected|static|async|export|const|let|var|unsafe|pub|virtual|override|inline|final|synchronized|abstract|extern|\s)+\s*([A-Za-z_][A-Za-z0-9_]*)\s*\([^;]*\)\s*(?:\{|=>|throws)`)}, } var importPatterns = []*regexp.Regexp{ regexp.MustCompile(`from\s+["']?([^"';\s]+)`), regexp.MustCompile(`import\s*["']([^"']+)`), regexp.MustCompile(`^\s*import\s+([A-Za-z_][A-Za-z0-9_.]*)\s*(?:;|$|\bas\b)`), regexp.MustCompile(`import\s*\(\s*["']([^"']+)`), regexp.MustCompile(`require\s*\(\s*["']([^"']+)`), regexp.MustCompile(`#include\s*[<"]([^>"]+)`), regexp.MustCompile(`(?:use|mod)\s+([A-Za-z_][A-Za-z0-9_:]*)`), regexp.MustCompile(`using\s+([A-Za-z_][A-Za-z0-9_.]*)`), } var complexityTokens = []string{" if ", " for ", " while ", " match ", " switch ", "&&", "||"} const previewBudget = 48 * 1024 * 1024 // indexDirectory walks root and parses files in parallel; IDs follow walk order. func indexDirectory(root string, progress func(phase string, completed, total int, message string)) (*Snapshot, error) { paths, err := walkSource(root) if err != nil { return nil, err } if len(paths) == 0 { return nil, fmt.Errorf("No readable files were found") } total := len(paths) records := make([]*FileRecord, total) warnings := make([]string, total) var next, done atomic.Int64 var wg sync.WaitGroup for range runtime.GOMAXPROCS(0) { wg.Go(func() { for { i := int(next.Add(1) - 1) if i >= total { return } records[i], warnings[i] = parseFile(root, paths[i]) if n := done.Add(1); n%20 == 1 { progress("parsing", int(n-1), total, fmt.Sprintf("Parsing %d of %d", n, total)) } } }) } wg.Wait() files := make([]FileRecord, 0, total) snapshot := &Snapshot{Root: root, Warnings: []string{}, delivered: map[uint32][][2]int{}, released: map[uint32]bool{}} for i, record := range records { if warnings[i] != "" { snapshot.Warnings = append(snapshot.Warnings, warnings[i]) } if record != nil { record.ID = uint32(len(files) + 1) files = append(files, *record) } } progress("linking", len(files), len(files), "Linking files and symbols…") previewBytes := 0 for i := range files { previewBytes += len(files[i].Preview) } if previewBytes > previewBudget { perFile := max(previewBudget/max(len(files), 1), 32) for i := range files { if p := files[i].Preview; len(p) > perFile { end := perFile for end > 0 && !utf8.RuneStart(p[end]) { end-- } files[i].Preview = p[:end] } } } snapshot.Edges = linkFiles(files) // Imports are only an indexing intermediate; edges carry what the frontend needs. for i := range files { files[i].Imports = []string{} snapshot.TotalLines += files[i].Lines snapshot.Definitions += len(files[i].Symbols) } snapshot.Files = files snapshot.References = len(snapshot.Edges) return snapshot, nil } // parseFile returns nil for skipped files (binary or >8MB) and a warning for unreadable ones. func parseFile(root, path string) (*FileRecord, string) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Sprintf("%s: %v", path, err) } if len(data) > 8*1024*1024 || bytes.IndexByte(data[:min(len(data), 4096)], 0) >= 0 { return nil, "" } text := string(data) if !utf8.ValidString(text) { text = strings.ToValidUTF8(text, "�") } relative, err := filepath.Rel(root, path) if err != nil { relative = path } relative = filepath.ToSlash(relative) name := filepath.Base(path) extension := strings.ToLower(extensionOf(name)) record := &FileRecord{ Path: relative, Name: name, Extension: extension, Language: languageFor(extension), Layer: layerFor(relative), Bytes: len(data), Complexity: 1, Symbols: []Symbol{}, } if i := strings.LastIndexByte(relative, '/'); i >= 0 { record.Directory = relative[:i] } preview := make([]string, 0, 60) for len(text) > 0 { line, rest, found := strings.Cut(text, "\n") if found { line = strings.TrimSuffix(line, "\r") } text = rest record.Lines++ if len(preview) < 60 { preview = append(preview, line) } for _, token := range complexityTokens { if strings.Contains(line, token) { record.Complexity++ } } if mayDefine(line) { for _, pattern := range definitionPatterns { if m := pattern.re.FindStringSubmatchIndex(line); m != nil && m[2] >= 0 { record.Symbols = append(record.Symbols, Symbol{strings.Clone(line[m[2]:m[3]]), pattern.kind, record.Lines, strings.Clone(truncateRunes(strings.TrimSpace(line), 180))}) break } } } if mayImport(line) { for _, re := range importPatterns { if m := re.FindStringSubmatchIndex(line); m != nil && m[2] >= 0 { record.Imports = append(record.Imports, strings.Clone(line[m[2]:m[3]])) } } } } record.Lines = max(record.Lines, 1) // Enough source for zoomed-in tiles without copying whole repositories into the scene. record.Preview = strings.Clone(strings.Join(preview, "\n")) // Join aliases a lone line return record, "" } func mayDefine(line string) bool { return strings.Contains(line, "(") || strings.Contains(line, "class ") || strings.Contains(line, "struct ") || strings.Contains(line, "enum ") || strings.Contains(line, "interface ") || strings.Contains(line, "fn ") || strings.Contains(line, "func ") || strings.Contains(line, "def ") || strings.Contains(line, "function ") } func mayImport(line string) bool { return strings.Contains(line, "import") || strings.Contains(line, "from ") || strings.Contains(line, "require") || strings.Contains(line, "#include") || strings.Contains(line, "use ") || strings.Contains(line, "mod ") || strings.Contains(line, "using ") } func truncateRunes(s string, n int) string { for i := range s { if n == 0 { return s[:i] } n-- } return s } // extensionOf mirrors Rust's Path::extension: dotfiles like ".gitignore" have none. func extensionOf(name string) string { if i := strings.LastIndexByte(name, '.'); i > 0 { return name[i+1:] } return "" } var skippedNames = map[string]bool{".git": true, "target": true, "node_modules": true, ".next": true, "dist": true, "build": true} // walkSource lists regular files in lexical order, honouring .ignore files everywhere // and .gitignore files inside git checkouts. func walkSource(root string) ([]string, error) { var paths []string var rules []ignoreRule inGit := false err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return nil // unreadable entries are skipped, like the ignore crate's walker } rel, _ := filepath.Rel(root, path) rel = filepath.ToSlash(rel) if path != root { if skippedNames[d.Name()] || ignored(rules, rel, d.IsDir()) { if d.IsDir() { return filepath.SkipDir } return nil } } if !d.IsDir() { if d.Type().IsRegular() { paths = append(paths, path) } return nil } if _, err := os.Stat(filepath.Join(path, ".git")); err == nil { inGit = true } base := "" if path != root { base = rel + "/" } if inGit { rules = append(rules, loadIgnoreFile(filepath.Join(path, ".gitignore"), base)...) } rules = append(rules, loadIgnoreFile(filepath.Join(path, ".ignore"), base)...) return nil }) return paths, err } type ignoreRule struct { re *regexp.Regexp negate bool dirOnly bool base string // slash-terminated directory the rule file lives in, "" for root anyDir bool // pattern without a slash matches the basename at any depth } // ponytail: covers standard gitignore globs (*, ?, **, [], !, trailing /); no global or .git/info/exclude files. func loadIgnoreFile(path, base string) []ignoreRule { data, err := os.ReadFile(path) if err != nil { return nil } var rules []ignoreRule for _, line := range strings.Split(string(data), "\n") { line = strings.TrimRight(strings.TrimSuffix(line, "\r"), " ") if line == "" || line[0] == '#' { continue } rule := ignoreRule{base: base} if line[0] == '!' { rule.negate, line = true, line[1:] } else if line[0] == '\\' { line = line[1:] } if strings.HasSuffix(line, "/") { rule.dirOnly, line = true, strings.TrimSuffix(line, "/") } rule.anyDir = !strings.Contains(line, "/") line = strings.TrimPrefix(line, "/") if line == "" { continue } if re, err := regexp.Compile("^" + globToRegexp(line) + "$"); err == nil { rule.re = re rules = append(rules, rule) } } return rules } func globToRegexp(glob string) string { var b strings.Builder for i := 0; i < len(glob); i++ { switch c := glob[i]; { case strings.HasPrefix(glob[i:], "**/"): b.WriteString("(?:.*/)?") i += 2 case strings.HasPrefix(glob[i:], "**"): b.WriteString(".*") i++ case c == '*': b.WriteString("[^/]*") case c == '?': b.WriteString("[^/]") case c == '[': if end := strings.IndexByte(glob[i+1:], ']'); end > 0 { class := glob[i+1 : i+1+end] if class[0] == '!' { class = "^" + class[1:] } b.WriteString("[" + strings.ReplaceAll(class, `\`, `\\`) + "]") i += end + 1 } else { b.WriteString(`\[`) } case c == '\\' && i+1 < len(glob): i++ b.WriteString(regexp.QuoteMeta(glob[i : i+1])) default: b.WriteString(regexp.QuoteMeta(glob[i : i+1])) } } return b.String() } // ignored applies rules in order; later (deeper) rules override earlier ones. func ignored(rules []ignoreRule, rel string, isDir bool) bool { result := false for _, rule := range rules { if (rule.dirOnly && !isDir) || !strings.HasPrefix(rel, rule.base) { continue } target := rel[len(rule.base):] if rule.anyDir { target = target[strings.LastIndexByte(target, '/')+1:] } if rule.re.MatchString(target) { result = !rule.negate } } return result } func linkFiles(files []FileRecord) []Edge { exact := make(map[string]uint32, len(files)) lookup := make(map[string]uint32, len(files)*4) // 0 marks an ambiguous key add := func(key string, id uint32) { if old, ok := lookup[key]; ok && old != id { lookup[key] = 0 } else if !ok { lookup[key] = id } } for i := range files { file := &files[i] path := strings.ReplaceAll(file.Path, `\`, "/") ext := extensionOf(file.Name) extensionless := path stem := file.Name if ext != "" { extensionless = path[:len(path)-len(ext)-1] stem = file.Name[:len(file.Name)-len(ext)-1] } exact[path] = file.ID add(path, file.ID) add(file.Name, file.ID) add(extensionless, file.ID) if dir, ok := strings.CutSuffix(extensionless, "/index"); ok { add(dir, file.ID) } for j := range len(path) { if path[j] == '/' { add(path[j+1:], file.ID) } } for j := range len(extensionless) { if extensionless[j] == '/' { add(extensionless[j+1:], file.ID) } } if stem != "" { add(stem, file.ID) } } resolve := func(key string) uint32 { if id, ok := exact[key]; ok { return id } return lookup[key] } type pair struct{ from, to uint32 } seen := map[pair]bool{} edges := []Edge{} for i := range files { file := &files[i] for _, imp := range file.Imports { imp = strings.ReplaceAll(imp, `\`, "/") relativeImport := strings.HasPrefix(imp, ".") direct := resolve(normalizePath(file.Directory + "/" + imp)) if direct == 0 && !relativeImport { direct = resolve(normalizePath(imp)) } target := direct if target == 0 && !relativeImport { module := strings.ReplaceAll(imp, "::", "/") if file.Language == "Python" || file.Language == "Java" || file.Language == "C#" { module = strings.ReplaceAll(module, ".", "/") } if target = resolve(module); target == 0 { target = lookup[module[strings.LastIndexByte(module, '/')+1:]] } } if target != 0 && target != file.ID && !seen[pair{file.ID, target}] { seen[pair{file.ID, target}] = true confidence := "inferred" if direct != 0 { confidence = "known" } edges = append(edges, Edge{file.ID, target, "import", confidence}) } } } return edges } func normalizePath(path string) string { parts := make([]string, 0, 8) for part := range strings.SplitSeq(path, "/") { switch part { case "", ".": case "..": if len(parts) > 0 { parts = parts[:len(parts)-1] } default: parts = append(parts, part) } } return strings.Join(parts, "/") } var languages = map[string]string{} func init() { for language, extensions := range map[string]string{ "JavaScript": "js jsx mjs cjs", "TypeScript": "ts tsx", "Python": "py pyi", "Rust": "rs", "Go": "go", "C": "c h", "C++": "cc cpp cxx hpp hh", "Java": "java", "C#": "cs", "F#": "fs fsx", "Kotlin": "kt kts", "Swift": "swift", "Scala": "scala sc", "Dart": "dart", "PHP": "php", "Ruby": "rb", "R": "r", "Perl": "pl pm", "Lua": "lua", "SQL": "sql", "Elixir": "ex exs", "Erlang": "erl hrl", "Haskell": "hs lhs", "OCaml": "ml mli", "Clojure": "clj cljs cljc edn", "Zig": "zig", "Nim": "nim", "Solidity": "sol", "Groovy": "groovy gradle", "HTML": "html htm", "Markup": "xml svg vue svelte", "CSS": "css scss sass less", "Data": "json jsonc toml yaml yml ini properties", "Docs": "md mdx txt rst", "Shell": "sh zsh bash fish", } { for ext := range strings.FieldsSeq(extensions) { languages[ext] = language } } } func languageFor(extension string) string { if language, ok := languages[extension]; ok { return language } return "Other" } func layerFor(path string) string { lower := strings.ToLower(path) has := func(subs ...string) bool { for _, s := range subs { if strings.Contains(lower, s) { return true } } return false } switch { case has("test", "spec", "fixture"): return "test" case has("vendor", "third_party", "external"): return "vendor" case has("generated", "gen/") || strings.HasSuffix(lower, ".min.js"): return "generated" case has("platform", "windows", "linux", "darwin"): return "platform" case has("lib/", "libs/", "packages/"): return "library" case has("docs/") || strings.HasSuffix(lower, ".md"): return "docs" } return "application" }