Add source indexer
Parallel file parsing, import linking, language/layer classification, and .gitignore/.ignore handling, with tests.
This commit is contained in:
Andy Dixon
@@ -0,0 +1 @@
|
|||||||
|
/codenavigator
|
||||||
+550
@@ -0,0 +1,550 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Symbol struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Line int `json:"line"`
|
||||||
|
Signature string `json:"signature"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type FileRecord struct {
|
||||||
|
ID uint32 `json:"id"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Directory string `json:"directory"`
|
||||||
|
Extension string `json:"extension"`
|
||||||
|
Language string `json:"language"`
|
||||||
|
Layer string `json:"layer"`
|
||||||
|
Lines int `json:"lines"`
|
||||||
|
Bytes int `json:"bytes"`
|
||||||
|
Complexity int `json:"complexity"`
|
||||||
|
Preview string `json:"preview"`
|
||||||
|
Symbols []Symbol `json:"symbols"`
|
||||||
|
Imports []string `json:"imports"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Edge struct {
|
||||||
|
From uint32 `json:"from"`
|
||||||
|
To uint32 `json:"to"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Confidence string `json:"confidence"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Snapshot struct {
|
||||||
|
ID, Name, Source string
|
||||||
|
Files []FileRecord
|
||||||
|
Edges []Edge
|
||||||
|
TotalLines int
|
||||||
|
Definitions int
|
||||||
|
References int
|
||||||
|
Warnings []string
|
||||||
|
Root string
|
||||||
|
delivered map[uint32][][2]int
|
||||||
|
released map[uint32]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type definitionPattern struct {
|
||||||
|
kind string
|
||||||
|
re *regexp.Regexp
|
||||||
|
}
|
||||||
|
|
||||||
|
var definitionPatterns = []definitionPattern{
|
||||||
|
{"class", regexp.MustCompile(`(?:class|struct|enum|trait|interface|record)\s+([A-Za-z_][A-Za-z0-9_]*)`)},
|
||||||
|
{"function", regexp.MustCompile(`(?:fn|func|def|function)\s+([A-Za-z_][A-Za-z0-9_]*)`)},
|
||||||
|
{"function", regexp.MustCompile(`(?:public|private|protected|static|async|export|const|let|var|unsafe|pub|virtual|override|inline|final|synchronized|abstract|extern|\s)+\s*([A-Za-z_][A-Za-z0-9_]*)\s*\([^;]*\)\s*(?:\{|=>|throws)`)},
|
||||||
|
}
|
||||||
|
|
||||||
|
var importPatterns = []*regexp.Regexp{
|
||||||
|
regexp.MustCompile(`from\s+["']?([^"';\s]+)`),
|
||||||
|
regexp.MustCompile(`import\s*["']([^"']+)`),
|
||||||
|
regexp.MustCompile(`^\s*import\s+([A-Za-z_][A-Za-z0-9_.]*)\s*(?:;|$|\bas\b)`),
|
||||||
|
regexp.MustCompile(`import\s*\(\s*["']([^"']+)`),
|
||||||
|
regexp.MustCompile(`require\s*\(\s*["']([^"']+)`),
|
||||||
|
regexp.MustCompile(`#include\s*[<"]([^>"]+)`),
|
||||||
|
regexp.MustCompile(`(?:use|mod)\s+([A-Za-z_][A-Za-z0-9_:]*)`),
|
||||||
|
regexp.MustCompile(`using\s+([A-Za-z_][A-Za-z0-9_.]*)`),
|
||||||
|
}
|
||||||
|
|
||||||
|
var complexityTokens = []string{" if ", " for ", " while ", " match ", " switch ", "&&", "||"}
|
||||||
|
|
||||||
|
const previewBudget = 48 * 1024 * 1024
|
||||||
|
|
||||||
|
// indexDirectory walks root and parses files in parallel; IDs follow walk order.
|
||||||
|
func indexDirectory(root string, progress func(phase string, completed, total int, message string)) (*Snapshot, error) {
|
||||||
|
paths, err := walkSource(root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(paths) == 0 {
|
||||||
|
return nil, fmt.Errorf("No readable files were found")
|
||||||
|
}
|
||||||
|
|
||||||
|
total := len(paths)
|
||||||
|
records := make([]*FileRecord, total)
|
||||||
|
warnings := make([]string, total)
|
||||||
|
var next, done atomic.Int64
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for range runtime.GOMAXPROCS(0) {
|
||||||
|
wg.Go(func() {
|
||||||
|
for {
|
||||||
|
i := int(next.Add(1) - 1)
|
||||||
|
if i >= total {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
records[i], warnings[i] = parseFile(root, paths[i])
|
||||||
|
if n := done.Add(1); n%20 == 1 {
|
||||||
|
progress("parsing", int(n-1), total, fmt.Sprintf("Parsing %d of %d", n, total))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
files := make([]FileRecord, 0, total)
|
||||||
|
snapshot := &Snapshot{Root: root, Warnings: []string{}, delivered: map[uint32][][2]int{}, released: map[uint32]bool{}}
|
||||||
|
for i, record := range records {
|
||||||
|
if warnings[i] != "" {
|
||||||
|
snapshot.Warnings = append(snapshot.Warnings, warnings[i])
|
||||||
|
}
|
||||||
|
if record != nil {
|
||||||
|
record.ID = uint32(len(files) + 1)
|
||||||
|
files = append(files, *record)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
progress("linking", len(files), len(files), "Linking files and symbols…")
|
||||||
|
|
||||||
|
previewBytes := 0
|
||||||
|
for i := range files {
|
||||||
|
previewBytes += len(files[i].Preview)
|
||||||
|
}
|
||||||
|
if previewBytes > previewBudget {
|
||||||
|
perFile := max(previewBudget/max(len(files), 1), 32)
|
||||||
|
for i := range files {
|
||||||
|
if p := files[i].Preview; len(p) > perFile {
|
||||||
|
end := perFile
|
||||||
|
for end > 0 && !utf8.RuneStart(p[end]) {
|
||||||
|
end--
|
||||||
|
}
|
||||||
|
files[i].Preview = p[:end]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.Edges = linkFiles(files)
|
||||||
|
// Imports are only an indexing intermediate; edges carry what the frontend needs.
|
||||||
|
for i := range files {
|
||||||
|
files[i].Imports = []string{}
|
||||||
|
snapshot.TotalLines += files[i].Lines
|
||||||
|
snapshot.Definitions += len(files[i].Symbols)
|
||||||
|
}
|
||||||
|
snapshot.Files = files
|
||||||
|
snapshot.References = len(snapshot.Edges)
|
||||||
|
return snapshot, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseFile returns nil for skipped files (binary or >8MB) and a warning for unreadable ones.
|
||||||
|
func parseFile(root, path string) (*FileRecord, string) {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Sprintf("%s: %v", path, err)
|
||||||
|
}
|
||||||
|
if len(data) > 8*1024*1024 || bytes.IndexByte(data[:min(len(data), 4096)], 0) >= 0 {
|
||||||
|
return nil, ""
|
||||||
|
}
|
||||||
|
text := string(data)
|
||||||
|
if !utf8.ValidString(text) {
|
||||||
|
text = strings.ToValidUTF8(text, "�")
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(root, path)
|
||||||
|
if err != nil {
|
||||||
|
relative = path
|
||||||
|
}
|
||||||
|
relative = filepath.ToSlash(relative)
|
||||||
|
name := filepath.Base(path)
|
||||||
|
extension := strings.ToLower(extensionOf(name))
|
||||||
|
|
||||||
|
record := &FileRecord{
|
||||||
|
Path: relative,
|
||||||
|
Name: name,
|
||||||
|
Extension: extension,
|
||||||
|
Language: languageFor(extension),
|
||||||
|
Layer: layerFor(relative),
|
||||||
|
Bytes: len(data),
|
||||||
|
Complexity: 1,
|
||||||
|
Symbols: []Symbol{},
|
||||||
|
}
|
||||||
|
if i := strings.LastIndexByte(relative, '/'); i >= 0 {
|
||||||
|
record.Directory = relative[:i]
|
||||||
|
}
|
||||||
|
|
||||||
|
preview := make([]string, 0, 60)
|
||||||
|
for len(text) > 0 {
|
||||||
|
line, rest, found := strings.Cut(text, "\n")
|
||||||
|
if found {
|
||||||
|
line = strings.TrimSuffix(line, "\r")
|
||||||
|
}
|
||||||
|
text = rest
|
||||||
|
record.Lines++
|
||||||
|
if len(preview) < 60 {
|
||||||
|
preview = append(preview, line)
|
||||||
|
}
|
||||||
|
for _, token := range complexityTokens {
|
||||||
|
if strings.Contains(line, token) {
|
||||||
|
record.Complexity++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mayDefine(line) {
|
||||||
|
for _, pattern := range definitionPatterns {
|
||||||
|
if m := pattern.re.FindStringSubmatchIndex(line); m != nil && m[2] >= 0 {
|
||||||
|
record.Symbols = append(record.Symbols, Symbol{strings.Clone(line[m[2]:m[3]]), pattern.kind, record.Lines, strings.Clone(truncateRunes(strings.TrimSpace(line), 180))})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mayImport(line) {
|
||||||
|
for _, re := range importPatterns {
|
||||||
|
if m := re.FindStringSubmatchIndex(line); m != nil && m[2] >= 0 {
|
||||||
|
record.Imports = append(record.Imports, strings.Clone(line[m[2]:m[3]]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record.Lines = max(record.Lines, 1)
|
||||||
|
// Enough source for zoomed-in tiles without copying whole repositories into the scene.
|
||||||
|
record.Preview = strings.Clone(strings.Join(preview, "\n")) // Join aliases a lone line
|
||||||
|
return record, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func mayDefine(line string) bool {
|
||||||
|
return strings.Contains(line, "(") || strings.Contains(line, "class ") || strings.Contains(line, "struct ") ||
|
||||||
|
strings.Contains(line, "enum ") || strings.Contains(line, "interface ") || strings.Contains(line, "fn ") ||
|
||||||
|
strings.Contains(line, "func ") || strings.Contains(line, "def ") || strings.Contains(line, "function ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func mayImport(line string) bool {
|
||||||
|
return strings.Contains(line, "import") || strings.Contains(line, "from ") || strings.Contains(line, "require") ||
|
||||||
|
strings.Contains(line, "#include") || strings.Contains(line, "use ") || strings.Contains(line, "mod ") ||
|
||||||
|
strings.Contains(line, "using ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncateRunes(s string, n int) string {
|
||||||
|
for i := range s {
|
||||||
|
if n == 0 {
|
||||||
|
return s[:i]
|
||||||
|
}
|
||||||
|
n--
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// extensionOf mirrors Rust's Path::extension: dotfiles like ".gitignore" have none.
|
||||||
|
func extensionOf(name string) string {
|
||||||
|
if i := strings.LastIndexByte(name, '.'); i > 0 {
|
||||||
|
return name[i+1:]
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var skippedNames = map[string]bool{".git": true, "target": true, "node_modules": true, ".next": true, "dist": true, "build": true}
|
||||||
|
|
||||||
|
// walkSource lists regular files in lexical order, honouring .ignore files everywhere
|
||||||
|
// and .gitignore files inside git checkouts.
|
||||||
|
func walkSource(root string) ([]string, error) {
|
||||||
|
var paths []string
|
||||||
|
var rules []ignoreRule
|
||||||
|
inGit := false
|
||||||
|
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return nil // unreadable entries are skipped, like the ignore crate's walker
|
||||||
|
}
|
||||||
|
rel, _ := filepath.Rel(root, path)
|
||||||
|
rel = filepath.ToSlash(rel)
|
||||||
|
if path != root {
|
||||||
|
if skippedNames[d.Name()] || ignored(rules, rel, d.IsDir()) {
|
||||||
|
if d.IsDir() {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !d.IsDir() {
|
||||||
|
if d.Type().IsRegular() {
|
||||||
|
paths = append(paths, path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(path, ".git")); err == nil {
|
||||||
|
inGit = true
|
||||||
|
}
|
||||||
|
base := ""
|
||||||
|
if path != root {
|
||||||
|
base = rel + "/"
|
||||||
|
}
|
||||||
|
if inGit {
|
||||||
|
rules = append(rules, loadIgnoreFile(filepath.Join(path, ".gitignore"), base)...)
|
||||||
|
}
|
||||||
|
rules = append(rules, loadIgnoreFile(filepath.Join(path, ".ignore"), base)...)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return paths, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type ignoreRule struct {
|
||||||
|
re *regexp.Regexp
|
||||||
|
negate bool
|
||||||
|
dirOnly bool
|
||||||
|
base string // slash-terminated directory the rule file lives in, "" for root
|
||||||
|
anyDir bool // pattern without a slash matches the basename at any depth
|
||||||
|
}
|
||||||
|
|
||||||
|
// ponytail: covers standard gitignore globs (*, ?, **, [], !, trailing /); no global or .git/info/exclude files.
|
||||||
|
func loadIgnoreFile(path, base string) []ignoreRule {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var rules []ignoreRule
|
||||||
|
for _, line := range strings.Split(string(data), "\n") {
|
||||||
|
line = strings.TrimRight(strings.TrimSuffix(line, "\r"), " ")
|
||||||
|
if line == "" || line[0] == '#' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rule := ignoreRule{base: base}
|
||||||
|
if line[0] == '!' {
|
||||||
|
rule.negate, line = true, line[1:]
|
||||||
|
} else if line[0] == '\\' {
|
||||||
|
line = line[1:]
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(line, "/") {
|
||||||
|
rule.dirOnly, line = true, strings.TrimSuffix(line, "/")
|
||||||
|
}
|
||||||
|
rule.anyDir = !strings.Contains(line, "/")
|
||||||
|
line = strings.TrimPrefix(line, "/")
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if re, err := regexp.Compile("^" + globToRegexp(line) + "$"); err == nil {
|
||||||
|
rule.re = re
|
||||||
|
rules = append(rules, rule)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rules
|
||||||
|
}
|
||||||
|
|
||||||
|
func globToRegexp(glob string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for i := 0; i < len(glob); i++ {
|
||||||
|
switch c := glob[i]; {
|
||||||
|
case strings.HasPrefix(glob[i:], "**/"):
|
||||||
|
b.WriteString("(?:.*/)?")
|
||||||
|
i += 2
|
||||||
|
case strings.HasPrefix(glob[i:], "**"):
|
||||||
|
b.WriteString(".*")
|
||||||
|
i++
|
||||||
|
case c == '*':
|
||||||
|
b.WriteString("[^/]*")
|
||||||
|
case c == '?':
|
||||||
|
b.WriteString("[^/]")
|
||||||
|
case c == '[':
|
||||||
|
if end := strings.IndexByte(glob[i+1:], ']'); end > 0 {
|
||||||
|
class := glob[i+1 : i+1+end]
|
||||||
|
if class[0] == '!' {
|
||||||
|
class = "^" + class[1:]
|
||||||
|
}
|
||||||
|
b.WriteString("[" + strings.ReplaceAll(class, `\`, `\\`) + "]")
|
||||||
|
i += end + 1
|
||||||
|
} else {
|
||||||
|
b.WriteString(`\[`)
|
||||||
|
}
|
||||||
|
case c == '\\' && i+1 < len(glob):
|
||||||
|
i++
|
||||||
|
b.WriteString(regexp.QuoteMeta(glob[i : i+1]))
|
||||||
|
default:
|
||||||
|
b.WriteString(regexp.QuoteMeta(glob[i : i+1]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ignored applies rules in order; later (deeper) rules override earlier ones.
|
||||||
|
func ignored(rules []ignoreRule, rel string, isDir bool) bool {
|
||||||
|
result := false
|
||||||
|
for _, rule := range rules {
|
||||||
|
if (rule.dirOnly && !isDir) || !strings.HasPrefix(rel, rule.base) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
target := rel[len(rule.base):]
|
||||||
|
if rule.anyDir {
|
||||||
|
target = target[strings.LastIndexByte(target, '/')+1:]
|
||||||
|
}
|
||||||
|
if rule.re.MatchString(target) {
|
||||||
|
result = !rule.negate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func linkFiles(files []FileRecord) []Edge {
|
||||||
|
exact := make(map[string]uint32, len(files))
|
||||||
|
lookup := make(map[string]uint32, len(files)*4) // 0 marks an ambiguous key
|
||||||
|
add := func(key string, id uint32) {
|
||||||
|
if old, ok := lookup[key]; ok && old != id {
|
||||||
|
lookup[key] = 0
|
||||||
|
} else if !ok {
|
||||||
|
lookup[key] = id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := range files {
|
||||||
|
file := &files[i]
|
||||||
|
path := strings.ReplaceAll(file.Path, `\`, "/")
|
||||||
|
ext := extensionOf(file.Name)
|
||||||
|
extensionless := path
|
||||||
|
stem := file.Name
|
||||||
|
if ext != "" {
|
||||||
|
extensionless = path[:len(path)-len(ext)-1]
|
||||||
|
stem = file.Name[:len(file.Name)-len(ext)-1]
|
||||||
|
}
|
||||||
|
exact[path] = file.ID
|
||||||
|
add(path, file.ID)
|
||||||
|
add(file.Name, file.ID)
|
||||||
|
add(extensionless, file.ID)
|
||||||
|
if dir, ok := strings.CutSuffix(extensionless, "/index"); ok {
|
||||||
|
add(dir, file.ID)
|
||||||
|
}
|
||||||
|
for j := range len(path) {
|
||||||
|
if path[j] == '/' {
|
||||||
|
add(path[j+1:], file.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for j := range len(extensionless) {
|
||||||
|
if extensionless[j] == '/' {
|
||||||
|
add(extensionless[j+1:], file.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if stem != "" {
|
||||||
|
add(stem, file.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolve := func(key string) uint32 {
|
||||||
|
if id, ok := exact[key]; ok {
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
return lookup[key]
|
||||||
|
}
|
||||||
|
|
||||||
|
type pair struct{ from, to uint32 }
|
||||||
|
seen := map[pair]bool{}
|
||||||
|
edges := []Edge{}
|
||||||
|
for i := range files {
|
||||||
|
file := &files[i]
|
||||||
|
for _, imp := range file.Imports {
|
||||||
|
imp = strings.ReplaceAll(imp, `\`, "/")
|
||||||
|
relativeImport := strings.HasPrefix(imp, ".")
|
||||||
|
direct := resolve(normalizePath(file.Directory + "/" + imp))
|
||||||
|
if direct == 0 && !relativeImport {
|
||||||
|
direct = resolve(normalizePath(imp))
|
||||||
|
}
|
||||||
|
target := direct
|
||||||
|
if target == 0 && !relativeImport {
|
||||||
|
module := strings.ReplaceAll(imp, "::", "/")
|
||||||
|
if file.Language == "Python" || file.Language == "Java" || file.Language == "C#" {
|
||||||
|
module = strings.ReplaceAll(module, ".", "/")
|
||||||
|
}
|
||||||
|
if target = resolve(module); target == 0 {
|
||||||
|
target = lookup[module[strings.LastIndexByte(module, '/')+1:]]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if target != 0 && target != file.ID && !seen[pair{file.ID, target}] {
|
||||||
|
seen[pair{file.ID, target}] = true
|
||||||
|
confidence := "inferred"
|
||||||
|
if direct != 0 {
|
||||||
|
confidence = "known"
|
||||||
|
}
|
||||||
|
edges = append(edges, Edge{file.ID, target, "import", confidence})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return edges
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizePath(path string) string {
|
||||||
|
parts := make([]string, 0, 8)
|
||||||
|
for part := range strings.SplitSeq(path, "/") {
|
||||||
|
switch part {
|
||||||
|
case "", ".":
|
||||||
|
case "..":
|
||||||
|
if len(parts) > 0 {
|
||||||
|
parts = parts[:len(parts)-1]
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
parts = append(parts, part)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
var languages = map[string]string{}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
for language, extensions := range map[string]string{
|
||||||
|
"JavaScript": "js jsx mjs cjs", "TypeScript": "ts tsx", "Python": "py pyi", "Rust": "rs", "Go": "go",
|
||||||
|
"C": "c h", "C++": "cc cpp cxx hpp hh", "Java": "java", "C#": "cs", "F#": "fs fsx", "Kotlin": "kt kts",
|
||||||
|
"Swift": "swift", "Scala": "scala sc", "Dart": "dart", "PHP": "php", "Ruby": "rb", "R": "r", "Perl": "pl pm",
|
||||||
|
"Lua": "lua", "SQL": "sql", "Elixir": "ex exs", "Erlang": "erl hrl", "Haskell": "hs lhs", "OCaml": "ml mli",
|
||||||
|
"Clojure": "clj cljs cljc edn", "Zig": "zig", "Nim": "nim", "Solidity": "sol", "Groovy": "groovy gradle",
|
||||||
|
"HTML": "html htm", "Markup": "xml svg vue svelte", "CSS": "css scss sass less",
|
||||||
|
"Data": "json jsonc toml yaml yml ini properties", "Docs": "md mdx txt rst", "Shell": "sh zsh bash fish",
|
||||||
|
} {
|
||||||
|
for ext := range strings.FieldsSeq(extensions) {
|
||||||
|
languages[ext] = language
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func languageFor(extension string) string {
|
||||||
|
if language, ok := languages[extension]; ok {
|
||||||
|
return language
|
||||||
|
}
|
||||||
|
return "Other"
|
||||||
|
}
|
||||||
|
|
||||||
|
func layerFor(path string) string {
|
||||||
|
lower := strings.ToLower(path)
|
||||||
|
has := func(subs ...string) bool {
|
||||||
|
for _, s := range subs {
|
||||||
|
if strings.Contains(lower, s) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case has("test", "spec", "fixture"):
|
||||||
|
return "test"
|
||||||
|
case has("vendor", "third_party", "external"):
|
||||||
|
return "vendor"
|
||||||
|
case has("generated", "gen/") || strings.HasSuffix(lower, ".min.js"):
|
||||||
|
return "generated"
|
||||||
|
case has("platform", "windows", "linux", "darwin"):
|
||||||
|
return "platform"
|
||||||
|
case has("lib/", "libs/", "packages/"):
|
||||||
|
return "library"
|
||||||
|
case has("docs/") || strings.HasSuffix(lower, ".md"):
|
||||||
|
return "docs"
|
||||||
|
}
|
||||||
|
return "application"
|
||||||
|
}
|
||||||
+209
@@ -0,0 +1,209 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func record(id uint32, path string, imports ...string) FileRecord {
|
||||||
|
dir, name := "", path
|
||||||
|
if i := strings.LastIndexByte(path, '/'); i >= 0 {
|
||||||
|
dir, name = path[:i], path[i+1:]
|
||||||
|
}
|
||||||
|
return FileRecord{ID: id, Path: path, Name: name, Directory: dir, Language: "JavaScript", Imports: imports}
|
||||||
|
}
|
||||||
|
|
||||||
|
func edgePairs(edges []Edge) [][2]uint32 {
|
||||||
|
pairs := [][2]uint32{}
|
||||||
|
for _, e := range edges {
|
||||||
|
pairs = append(pairs, [2]uint32{e.From, e.To})
|
||||||
|
}
|
||||||
|
return pairs
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvesRelativeImportsIncludesAndIndexFiles(t *testing.T) {
|
||||||
|
edges := linkFiles([]FileRecord{
|
||||||
|
record(1, "src/main.js", "./utils.js", "../include/config.h", "./widgets", "./utils.js"),
|
||||||
|
record(2, "src/utils.js"),
|
||||||
|
record(3, "include/config.h"),
|
||||||
|
record(4, "src/widgets/index.ts"),
|
||||||
|
record(5, "elsewhere/utils.js"),
|
||||||
|
})
|
||||||
|
if got, want := edgePairs(edges), [][2]uint32{{1, 2}, {1, 3}, {1, 4}}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("edges = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
if edges[0].Confidence != "known" {
|
||||||
|
t.Fatalf("relative import confidence = %q", edges[0].Confidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvesHeaderSuffixWithoutGuessingAmbiguousNames(t *testing.T) {
|
||||||
|
edges := linkFiles([]FileRecord{
|
||||||
|
record(1, "main.c", "config.h", "utils.js", "missing.js"),
|
||||||
|
record(2, "include/config.h"),
|
||||||
|
record(3, "a/utils.js"),
|
||||||
|
record(4, "b/utils.js"),
|
||||||
|
})
|
||||||
|
if got, want := edgePairs(edges), [][2]uint32{{1, 2}}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("edges = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvesDottedModulesAsInferred(t *testing.T) {
|
||||||
|
a := record(1, "app/main.py", "app.models.user")
|
||||||
|
a.Language = "Python"
|
||||||
|
edges := linkFiles([]FileRecord{a, record(2, "app/models/user.py")})
|
||||||
|
if len(edges) != 1 || edges[0].To != 2 || edges[0].Confidence != "inferred" {
|
||||||
|
t.Fatalf("edges = %+v", edges)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractsNamedJavaScriptImportTarget(t *testing.T) {
|
||||||
|
var imports []string
|
||||||
|
for _, re := range importPatterns {
|
||||||
|
if m := re.FindStringSubmatch("import { render } from './renderer.js';"); m != nil {
|
||||||
|
imports = append(imports, m[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(imports, []string{"./renderer.js"}) {
|
||||||
|
t.Fatalf("imports = %v", imports)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClassifiesLanguagesAndLayers(t *testing.T) {
|
||||||
|
for in, want := range map[string]string{"rs": "Rust", "tsx": "TypeScript", "cpp": "C++", "zzz": "Other"} {
|
||||||
|
if got := languageFor(in); got != want {
|
||||||
|
t.Errorf("languageFor(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for in, want := range map[string]string{"src/widget.test.ts": "test", "vendor/lib.c": "vendor", "a/gen/x.go": "generated", "README.md": "docs", "src/main.go": "application"} {
|
||||||
|
if got := layerFor(in); got != want {
|
||||||
|
t.Errorf("layerFor(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractsCommonSymbols(t *testing.T) {
|
||||||
|
for _, line := range []string{"pub fn render_scene() {", "class WindowManager {", "public void run() throws IOException"} {
|
||||||
|
matched := false
|
||||||
|
for _, p := range definitionPatterns {
|
||||||
|
matched = matched || p.re.MatchString(line)
|
||||||
|
}
|
||||||
|
if !matched {
|
||||||
|
t.Errorf("no definition pattern matched %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionOf(t *testing.T) {
|
||||||
|
for in, want := range map[string]string{"a.rs": "rs", ".gitignore": "", "a.tar.gz": "gz", "Makefile": ""} {
|
||||||
|
if got := extensionOf(in); got != want {
|
||||||
|
t.Errorf("extensionOf(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIgnoreRules(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("# c\n*.log\n/out/\n!keep.log\ndocs/**/*.tmp\n"), 0o644)
|
||||||
|
rules := loadIgnoreFile(filepath.Join(dir, ".gitignore"), "")
|
||||||
|
cases := []struct {
|
||||||
|
path string
|
||||||
|
isDir bool
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"a.log", false, true}, {"deep/b.log", false, true}, {"keep.log", false, false},
|
||||||
|
{"out", true, true}, {"src/out", true, false}, {"out", false, false},
|
||||||
|
{"docs/x/y/z.tmp", false, true}, {"docs/z.tmp", false, true}, {"z.tmp", false, false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := ignored(rules, c.path, c.isDir); got != c.want {
|
||||||
|
t.Errorf("ignored(%q, %v) = %v, want %v", c.path, c.isDir, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTree(t *testing.T, files map[string]string) string {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
for path, content := range files {
|
||||||
|
full := filepath.Join(root, path)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIndexDirectory(t *testing.T) {
|
||||||
|
root := writeTree(t, map[string]string{
|
||||||
|
"src/main.js": "import { util } from './util.js';\r\nfunction main() {\n if (a && b) { util(); }\n}\n",
|
||||||
|
"src/util.js": "export function util() {\n}\n",
|
||||||
|
"node_modules/x.js": "skipped",
|
||||||
|
"build": "skipped file named build",
|
||||||
|
"image.bin": "\x00\x01binary",
|
||||||
|
"ignored.log": "ignored by .ignore",
|
||||||
|
".ignore": "*.log\n",
|
||||||
|
"sub/.gitignore": "*.js\n", // not a git checkout, so this is not honoured
|
||||||
|
"sub/kept.js": "x\xffy",
|
||||||
|
".hidden/visible.txt": "hidden dirs are walked",
|
||||||
|
})
|
||||||
|
var phases []string
|
||||||
|
snapshot, err := indexDirectory(root, func(phase string, _, _ int, _ string) { phases = append(phases, phase) })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var paths []string
|
||||||
|
for i, f := range snapshot.Files {
|
||||||
|
if f.ID != uint32(i+1) {
|
||||||
|
t.Errorf("file %s has id %d, want %d", f.Path, f.ID, i+1)
|
||||||
|
}
|
||||||
|
paths = append(paths, f.Path)
|
||||||
|
}
|
||||||
|
want := []string{".hidden/visible.txt", ".ignore", "src/main.js", "src/util.js", "sub/.gitignore", "sub/kept.js"}
|
||||||
|
if !reflect.DeepEqual(paths, want) {
|
||||||
|
t.Fatalf("paths = %v, want %v", paths, want)
|
||||||
|
}
|
||||||
|
main := snapshot.Files[2]
|
||||||
|
if main.Lines != 4 || main.Complexity != 3 || main.Language != "JavaScript" || main.Directory != "src" || main.Name != "main.js" {
|
||||||
|
t.Errorf("main.js record = %+v", main)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(main.Preview, "import { util } from './util.js';\nfunction main() {") {
|
||||||
|
t.Errorf("preview = %q", main.Preview)
|
||||||
|
}
|
||||||
|
if len(main.Symbols) != 2 || main.Symbols[0].Name != "main" || main.Symbols[0].Line != 2 {
|
||||||
|
t.Errorf("symbols = %+v", main.Symbols)
|
||||||
|
}
|
||||||
|
if got := edgePairs(snapshot.Edges); !reflect.DeepEqual(got, [][2]uint32{{3, 4}}) {
|
||||||
|
t.Errorf("edges = %v", got)
|
||||||
|
}
|
||||||
|
if snapshot.Files[5].Preview != "x�y" {
|
||||||
|
t.Errorf("invalid UTF-8 preview = %q", snapshot.Files[5].Preview)
|
||||||
|
}
|
||||||
|
if snapshot.References != 1 || snapshot.Definitions != 3 || phases[len(phases)-1] != "linking" {
|
||||||
|
t.Errorf("snapshot totals refs=%d defs=%d phases=%v", snapshot.References, snapshot.Definitions, phases)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGitignoreHonouredInsideCheckout(t *testing.T) {
|
||||||
|
root := writeTree(t, map[string]string{".git/HEAD": "ref", ".gitignore": "gen/\n", "gen/a.go": "x", "main.go": "package main"})
|
||||||
|
paths, err := walkSource(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(paths) != 2 || !strings.HasSuffix(paths[0], ".gitignore") || !strings.HasSuffix(paths[1], "main.go") {
|
||||||
|
t.Fatalf("paths = %v", paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyDirectoryFails(t *testing.T) {
|
||||||
|
if _, err := indexDirectory(t.TempDir(), func(string, int, int, string) {}); err == nil {
|
||||||
|
t.Fatal("expected error for empty directory")
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user