package parsing import ( "bytes" "encoding/json" "fmt" "io" "sort" "strconv" "strings" "github.com/andydixon/relata/internal/domain" ) // JSONParser flattens a JSON document into readable "path: value" lines so that // its content can be embedded and searched like prose, while object boundaries // remain visible to the chunker. // // Depth is bounded before decoding so that a deeply nested document cannot // exhaust the stack, and the flattened output is capped so that a wide array // cannot expand without limit. type JSONParser struct{} // Name identifies the parser. func (JSONParser) Name() string { return "json" } // Handles reports whether p can parse mime. func (JSONParser) Handles(mime string) bool { return isJSON(mime) } // defaultMaxJSONDepth is used when Input.MaxJSONDepth is unset. const defaultMaxJSONDepth = 20 // maxFlattenedLines bounds output from pathologically wide documents. const maxFlattenedLines = 20000 // Parse decodes the JSON and renders it as a flat, human-readable listing. func (JSONParser) Parse(in Input) (*Document, error) { maxDepth := in.MaxJSONDepth if maxDepth <= 0 { maxDepth = defaultMaxJSONDepth } var doc Document depth, err := scanDepth(in.Data, maxDepth) if err != nil { return nil, fmt.Errorf("json is not well-formed: %w", err) } truncatedDepth := depth > maxDepth var v any dec := json.NewDecoder(bytes.NewReader(in.Data)) dec.UseNumber() // keep numeric text exactly as written if err := dec.Decode(&v); err != nil { return nil, fmt.Errorf("decode json: %w", err) } // Reject trailing content: two concatenated documents are not one document. if _, err := dec.Token(); err != io.EOF { return nil, fmt.Errorf("json contains trailing data after the top-level value") } f := &flattener{maxDepth: maxDepth} f.walk("", v, 0) if f.truncated || truncatedDepth { doc.Warnings = append(doc.Warnings, domain.ParseWarning{ Code: domain.WarnJSONDepthExceeded, Message: "The JSON document was flattened only to the configured depth or size limit; deeper values are stored in the original but not indexed.", Detail: fmt.Sprintf("max_depth=%d observed_depth=%d lines=%d", maxDepth, depth, len(f.lines)), }) } var sb strings.Builder for i, l := range f.lines { if i > 0 { sb.WriteByte('\n') } start := sb.Len() sb.WriteString(l.text) doc.Blocks = append(doc.Blocks, Block{ Kind: BlockJSONField, Text: l.text, Start: start, End: start + len(l.text), Section: l.section, }) } doc.Text = sb.String() // Blocks were built against a builder whose offsets already account for the // newline separators, so they index correctly into doc.Text. return &doc, nil } type flatLine struct { text string section string } type flattener struct { lines []flatLine maxDepth int truncated bool } // walk renders v as one or more "path: value" lines. Objects contribute a // section name so the chunker can keep an object's fields together. func (f *flattener) walk(path string, v any, depth int) { if len(f.lines) >= maxFlattenedLines { f.truncated = true return } if depth > f.maxDepth { f.truncated = true f.emit(path, "[nested value omitted: depth limit reached]", path) return } switch t := v.(type) { case map[string]any: keys := make([]string, 0, len(t)) for k := range t { keys = append(keys, k) } sort.Strings(keys) // stable output, so hashes are reproducible if len(keys) == 0 { f.emit(path, "{}", path) return } for _, k := range keys { f.walk(join(path, sanitiseKey(k)), t[k], depth+1) } case []any: if len(t) == 0 { f.emit(path, "[]", path) return } for i, e := range t { f.walk(fmt.Sprintf("%s[%d]", path, i), e, depth+1) } case nil: f.emit(path, "null", sectionOf(path)) case bool: f.emit(path, strconv.FormatBool(t), sectionOf(path)) case json.Number: f.emit(path, t.String(), sectionOf(path)) case string: f.emit(path, t, sectionOf(path)) default: f.emit(path, fmt.Sprint(t), sectionOf(path)) } } func (f *flattener) emit(path, value, section string) { if len(f.lines) >= maxFlattenedLines { f.truncated = true return } if path == "" { path = "value" } f.lines = append(f.lines, flatLine{text: path + ": " + value, section: section}) } func join(base, key string) string { if base == "" { return key } return base + "." + key } // sectionOf returns the parent path, used to group an object's fields. func sectionOf(path string) string { if i := strings.LastIndexAny(path, "."); i > 0 { return path[:i] } return path } // sanitiseKey strips control characters from object keys so that a crafted key // cannot inject line breaks into the flattened rendering. func sanitiseKey(k string) string { return strings.Map(func(r rune) rune { if r == '\n' || r == '\r' || r == '\t' || r < 0x20 { return ' ' } return r }, k) } // scanDepth measures maximum nesting using a streaming token scan, so nesting // is bounded before any recursive structure is built. It returns the observed // depth, stopping early once limit is exceeded. func scanDepth(data []byte, limit int) (int, error) { dec := json.NewDecoder(bytes.NewReader(data)) depth, maxSeen := 0, 0 for { tok, err := dec.Token() if err == io.EOF { return maxSeen, nil } if err != nil { return maxSeen, err } switch d := tok.(type) { case json.Delim: switch d { case '{', '[': depth++ if depth > maxSeen { maxSeen = depth } // Bail out early rather than walking a deeply nested bomb. if depth > limit+50 { return maxSeen, nil } case '}', ']': depth-- } } } } // validJSON reports whether data is a single well-formed JSON value. func validJSON(data []byte) bool { return json.Valid(data) }