Prefilter indexer regexps with literal checks

Go's regexp backtracker dominated indexing. Each pattern now runs only
when a cheap necessary-condition check passes, cutting indexing of the
Go standard library source from 4.7s to 1.4s.
Adds BenchmarkIndexDirectory.

indexer.go

@@ -57,26 +57,54 @@ type Snapshot struct {
 	released         map[uint32]bool
 }
 
-type definitionPattern struct {
+// Each pattern has a cheap literal prefilter that is a necessary condition for a match,
-	kind string
+// so the (much slower) regexp only runs on candidate lines.
-	re   *regexp.Regexp
+type pattern struct {
+	kind  string
+	maybe func(line string) bool
+	re    *regexp.Regexp
 }
 
-var definitionPatterns = []definitionPattern{
+func containsAny(literals ...string) func(string) bool {
-	{"class", regexp.MustCompile(`(?:class|struct|enum|trait|interface|record)\s+([A-Za-z_][A-Za-z0-9_]*)`)},
+	return func(line string) bool {
-	{"function", regexp.MustCompile(`(?:fn|func|def|function)\s+([A-Za-z_][A-Za-z0-9_]*)`)},
+		for _, literal := range literals {
-	{"function", regexp.MustCompile(`(?:public|private|protected|static|async|export|const|let|var|unsafe|pub|virtual|override|inline|final|synchronized|abstract|extern|\s)+\s*([A-Za-z_][A-Za-z0-9_]*)\s*\([^;]*\)\s*(?:\{|=>|throws)`)},
+			if strings.Contains(line, literal) {
+				return true
+			}
+		}
+		return false
+	}
 }
 
-var importPatterns = []*regexp.Regexp{
+// callWithBody needs "(", then ")", then "{", "=>" or "throws" after that ")".
-	regexp.MustCompile(`from\s+["']?([^"';\s]+)`),
+func callWithBody(line string) bool {
-	regexp.MustCompile(`import\s*["']([^"']+)`),
+	open := strings.IndexByte(line, '(')
-	regexp.MustCompile(`^\s*import\s+([A-Za-z_][A-Za-z0-9_.]*)\s*(?:;|$|\bas\b)`),
+	if open < 0 {
-	regexp.MustCompile(`import\s*\(\s*["']([^"']+)`),
+		return false
-	regexp.MustCompile(`require\s*\(\s*["']([^"']+)`),
+	}
-	regexp.MustCompile(`#include\s*[<"]([^>"]+)`),
+	close := strings.IndexByte(line[open:], ')')
-	regexp.MustCompile(`(?:use|mod)\s+([A-Za-z_][A-Za-z0-9_:]*)`),
+	if close < 0 {
-	regexp.MustCompile(`using\s+([A-Za-z_][A-Za-z0-9_.]*)`),
+		return false
+	}
+	tail := line[open+close:]
+	return strings.Contains(tail, "{") || strings.Contains(tail, "=>") || strings.Contains(tail, "throws")
+}
+
+var definitionPatterns = []pattern{
+	{"class", containsAny("class", "struct", "enum", "trait", "interface", "record"), regexp.MustCompile(`(?:class|struct|enum|trait|interface|record)\s+([A-Za-z_][A-Za-z0-9_]*)`)},
+	{"function", containsAny("fn", "func", "def"), regexp.MustCompile(`(?:fn|func|def|function)\s+([A-Za-z_][A-Za-z0-9_]*)`)},
+	{"function", callWithBody, regexp.MustCompile(`(?:public|private|protected|static|async|export|const|let|var|unsafe|pub|virtual|override|inline|final|synchronized|abstract|extern|\s)+\s*([A-Za-z_][A-Za-z0-9_]*)\s*\([^;]*\)\s*(?:\{|=>|throws)`)},
+}
+
+var importPatterns = []pattern{
+	{"", containsAny("from"), regexp.MustCompile(`from\s+["']?([^"';\s]+)`)},
+	{"", containsAny("import"), regexp.MustCompile(`import\s*["']([^"']+)`)},
+	{"", containsAny("import"), regexp.MustCompile(`^\s*import\s+([A-Za-z_][A-Za-z0-9_.]*)\s*(?:;|$|\bas\b)`)},
+	{"", containsAny("import"), regexp.MustCompile(`import\s*\(\s*["']([^"']+)`)},
+	{"", containsAny("require"), regexp.MustCompile(`require\s*\(\s*["']([^"']+)`)},
+	{"", containsAny("#include"), regexp.MustCompile(`#include\s*[<"]([^>"]+)`)},
+	{"", containsAny("use", "mod"), regexp.MustCompile(`(?:use|mod)\s+([A-Za-z_][A-Za-z0-9_:]*)`)},
+	{"", containsAny("using"), regexp.MustCompile(`using\s+([A-Za-z_][A-Za-z0-9_.]*)`)},
 }
 
 var complexityTokens = []string{" if ", " for ", " while ", " match ", " switch ", "&&", "||"}
@@ -209,6 +237,9 @@ func parseFile(root, path string) (*FileRecord, string) {
 		}
 		if mayDefine(line) {
 			for _, pattern := range definitionPatterns {
+				if !pattern.maybe(line) {
+					continue
+				}
 				if m := pattern.re.FindStringSubmatchIndex(line); m != nil && m[2] >= 0 {
 					record.Symbols = append(record.Symbols, Symbol{strings.Clone(line[m[2]:m[3]]), pattern.kind, record.Lines, strings.Clone(truncateRunes(strings.TrimSpace(line), 180))})
 					break
@@ -216,8 +247,11 @@ func parseFile(root, path string) (*FileRecord, string) {
 			}
 		}
 		if mayImport(line) {
-			for _, re := range importPatterns {
+			for _, pattern := range importPatterns {
-				if m := re.FindStringSubmatchIndex(line); m != nil && m[2] >= 0 {
+				if !pattern.maybe(line) {
+					continue
+				}
+				if m := pattern.re.FindStringSubmatchIndex(line); m != nil && m[2] >= 0 {
 					record.Imports = append(record.Imports, strings.Clone(line[m[2]:m[3]]))
 				}
 			}

indexer_test.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"strings"
 	"testing"
 )
@@ -63,8 +64,8 @@ func TestResolvesDottedModulesAsInferred(t *testing.T) {
 
 func TestExtractsNamedJavaScriptImportTarget(t *testing.T) {
 	var imports []string
-	for _, re := range importPatterns {
+	for _, p := range importPatterns {
-		if m := re.FindStringSubmatch("import { render } from './renderer.js';"); m != nil {
+		if m := p.re.FindStringSubmatch("import { render } from './renderer.js';"); m != nil {
 			imports = append(imports, m[1])
 		}
 	}
@@ -207,3 +208,19 @@ func TestEmptyDirectoryFails(t *testing.T) {
 		t.Fatal("expected error for empty directory")
 	}
 }
+
+// BenchmarkIndexDirectory indexes CODENAV_BENCH_DIR, defaulting to the Go standard library source.
+func BenchmarkIndexDirectory(b *testing.B) {
+	dir := os.Getenv("CODENAV_BENCH_DIR")
+	if dir == "" {
+		dir = filepath.Join(runtime.GOROOT(), "src")
+	}
+	if _, err := os.Stat(dir); err != nil {
+		b.Skip("no benchmark corpus:", err)
+	}
+	for b.Loop() {
+		if _, err := indexDirectory(dir, func(string, int, int, string) {}); err != nil {
+			b.Fatal(err)
+		}
+	}
+}