Code sync, security fixes

2026-08-11 10:40:10 +01:00
parent 4e00deedf8
commit 53dbc28383
141 changed files with 36194 additions and 0 deletions
@@ -0,0 +1,253 @@
+package parsing
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/andydixon/relata/internal/domain"
+)
+
+// DOCXParser extracts text from Office Open XML word documents.
+//
+// A DOCX is a zip. Only the single known part `word/document.xml` is read, by
+// exact name — the archive is never walked and no entry path is ever joined
+// with a filesystem path, so zip-slip does not apply. Decompressed size is
+// capped so a compression bomb cannot exhaust memory.
+type DOCXParser struct{}
+
+// Name identifies the parser.
+func (DOCXParser) Name() string { return "docx" }
+
+// Handles reports whether p can parse mime.
+func (DOCXParser) Handles(mime string) bool { return isDOCX(mime) }
+
+// maxDOCXPartBytes caps the decompressed size of word/document.xml.
+const maxDOCXPartBytes = 64 << 20
+
+// Parse reads paragraph runs from the main document part, mapping Word's
+// heading styles onto structural headings.
+func (DOCXParser) Parse(in Input) (*Document, error) {
+	zr, err := zip.NewReader(bytes.NewReader(in.Data), int64(len(in.Data)))
+	if err != nil {
+		return nil, fmt.Errorf("open docx container: %w", err)
+	}
+
+	var part *zip.File
+	for _, f := range zr.File {
+		if f.Name == "word/document.xml" {
+			part = f
+			break
+		}
+	}
+	if part == nil {
+		return nil, fmt.Errorf("docx is missing word/document.xml")
+	}
+	if part.UncompressedSize64 > maxDOCXPartBytes {
+		return nil, fmt.Errorf("docx main part is too large: %d bytes", part.UncompressedSize64)
+	}
+
+	rc, err := part.Open()
+	if err != nil {
+		return nil, fmt.Errorf("open docx main part: %w", err)
+	}
+	defer rc.Close()
+
+	// LimitReader guards against a lying uncompressed-size header.
+	raw, err := io.ReadAll(io.LimitReader(rc, maxDOCXPartBytes+1))
+	if err != nil {
+		return nil, fmt.Errorf("read docx main part: %w", err)
+	}
+	if int64(len(raw)) > maxDOCXPartBytes {
+		return nil, fmt.Errorf("docx main part exceeded the %d byte limit when decompressed", int64(maxDOCXPartBytes))
+	}
+
+	paras, warns, err := parseDocumentXML(raw)
+	if err != nil {
+		return nil, err
+	}
+
+	doc := &Document{Warnings: warns}
+	var sb strings.Builder
+	var sectionPath []string
+
+	for _, p := range paras {
+		text := strings.TrimRight(p.text, " \t")
+		if strings.TrimSpace(text) == "" {
+			continue
+		}
+		if sb.Len() > 0 {
+			sb.WriteString("\n\n")
+		}
+		start := sb.Len()
+		sb.WriteString(text)
+
+		kind := BlockParagraph
+		level := 0
+		switch {
+		case p.headingLevel > 0:
+			kind, level = BlockHeading, p.headingLevel
+			if level-1 < len(sectionPath) {
+				sectionPath = sectionPath[:level-1]
+			}
+			for len(sectionPath) < level-1 {
+				sectionPath = append(sectionPath, "")
+			}
+			sectionPath = append(sectionPath, text)
+		case p.isList:
+			kind = BlockList
+		}
+
+		doc.Blocks = append(doc.Blocks, Block{
+			Kind:    kind,
+			Text:    text,
+			Start:   start,
+			End:     start + len(text),
+			Level:   level,
+			Section: strings.Join(sectionPath, " > "),
+		})
+	}
+	doc.Text = sb.String()
+
+	if len(doc.Blocks) == 0 {
+		doc.Warnings = append(doc.Warnings, warn(domain.WarnLowExtractableText,
+			"No paragraph text was found in the Word document. If it contains only images, Relata does not perform OCR in this version.", ""))
+	}
+	return doc, nil
+}
+
+type docxPara struct {
+	text         string
+	headingLevel int
+	isList       bool
+}
+
+// parseDocumentXML streams the WordprocessingML, concatenating text runs into
+// paragraphs. Streaming (rather than unmarshalling the whole tree) keeps memory
+// bounded on large documents.
+func parseDocumentXML(raw []byte) ([]docxPara, []domain.ParseWarning, error) {
+	dec := xml.NewDecoder(bytes.NewReader(raw))
+	// Reject external entity references outright: this input is untrusted and
+	// there is no legitimate reason for a DOCX part to pull in outside content.
+	dec.Strict = false
+	dec.Entity = map[string]string{}
+	dec.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) {
+		// Only UTF-8 is expected; anything else is passed through as bytes
+		// rather than resolved via an external charset table.
+		return input, nil
+	}
+
+	var (
+		paras    []docxPara
+		warns    []domain.ParseWarning
+		cur      strings.Builder
+		inPara   bool
+		inText   bool
+		inStyle  bool
+		curStyle string
+		isList   bool
+		depth    int
+	)
+
+	flush := func() {
+		if !inPara {
+			return
+		}
+		paras = append(paras, docxPara{
+			text:         cur.String(),
+			headingLevel: headingLevelFromStyle(curStyle),
+			isList:       isList,
+		})
+		cur.Reset()
+		curStyle = ""
+		isList = false
+		inPara = false
+	}
+
+	for {
+		tok, err := dec.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			// Salvage what was parsed so far rather than losing the document.
+			warns = append(warns, warn(domain.WarnParseFailed,
+				"The Word document's XML ended unexpectedly; text extracted up to that point is indexed.",
+				truncate(err.Error(), 200)))
+			break
+		}
+
+		switch t := tok.(type) {
+		case xml.StartElement:
+			depth++
+			if depth > 1000 {
+				return paras, warns, fmt.Errorf("docx xml nesting is implausibly deep")
+			}
+			switch t.Name.Local {
+			case "p":
+				flush()
+				inPara = true
+			case "t":
+				inText = true
+			case "pStyle":
+				inStyle = true
+				for _, a := range t.Attr {
+					if a.Name.Local == "val" {
+						curStyle = a.Value
+					}
+				}
+			case "numPr":
+				isList = true
+			case "tab":
+				if inPara {
+					cur.WriteByte('\t')
+				}
+			case "br", "cr":
+				if inPara {
+					cur.WriteByte('\n')
+				}
+			}
+		case xml.CharData:
+			if inText && inPara {
+				cur.Write(t)
+			}
+		case xml.EndElement:
+			depth--
+			switch t.Name.Local {
+			case "p":
+				flush()
+			case "t":
+				inText = false
+			case "pStyle":
+				inStyle = false
+			}
+		}
+		_ = inStyle
+	}
+	flush()
+	return paras, warns, nil
+}
+
+// headingLevelFromStyle maps Word style identifiers ("Heading1", "heading 2",
+// "Title") onto a 1..6 structural level, returning 0 for body text.
+func headingLevelFromStyle(style string) int {
+	s := strings.ToLower(strings.ReplaceAll(style, " ", ""))
+	switch s {
+	case "title":
+		return 1
+	case "subtitle":
+		return 2
+	}
+	if !strings.HasPrefix(s, "heading") {
+		return 0
+	}
+	n, err := strconv.Atoi(strings.TrimPrefix(s, "heading"))
+	if err != nil || n < 1 || n > 6 {
+		return 0
+	}
+	return n
+}
@@ -0,0 +1,225 @@
+package parsing
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/andydixon/relata/internal/domain"
+)
+
+// JSONParser flattens a JSON document into readable "path: value" lines so that
+// its content can be embedded and searched like prose, while object boundaries
+// remain visible to the chunker.
+//
+// Depth is bounded before decoding so that a deeply nested document cannot
+// exhaust the stack, and the flattened output is capped so that a wide array
+// cannot expand without limit.
+type JSONParser struct{}
+
+// Name identifies the parser.
+func (JSONParser) Name() string { return "json" }
+
+// Handles reports whether p can parse mime.
+func (JSONParser) Handles(mime string) bool { return isJSON(mime) }
+
+// defaultMaxJSONDepth is used when Input.MaxJSONDepth is unset.
+const defaultMaxJSONDepth = 20
+
+// maxFlattenedLines bounds output from pathologically wide documents.
+const maxFlattenedLines = 20000
+
+// Parse decodes the JSON and renders it as a flat, human-readable listing.
+func (JSONParser) Parse(in Input) (*Document, error) {
+	maxDepth := in.MaxJSONDepth
+	if maxDepth <= 0 {
+		maxDepth = defaultMaxJSONDepth
+	}
+
+	var doc Document
+
+	depth, err := scanDepth(in.Data, maxDepth)
+	if err != nil {
+		return nil, fmt.Errorf("json is not well-formed: %w", err)
+	}
+	truncatedDepth := depth > maxDepth
+
+	var v any
+	dec := json.NewDecoder(bytes.NewReader(in.Data))
+	dec.UseNumber() // keep numeric text exactly as written
+	if err := dec.Decode(&v); err != nil {
+		return nil, fmt.Errorf("decode json: %w", err)
+	}
+	// Reject trailing content: two concatenated documents are not one document.
+	if _, err := dec.Token(); err != io.EOF {
+		return nil, fmt.Errorf("json contains trailing data after the top-level value")
+	}
+
+	f := &flattener{maxDepth: maxDepth}
+	f.walk("", v, 0)
+
+	if f.truncated || truncatedDepth {
+		doc.Warnings = append(doc.Warnings, domain.ParseWarning{
+			Code:    domain.WarnJSONDepthExceeded,
+			Message: "The JSON document was flattened only to the configured depth or size limit; deeper values are stored in the original but not indexed.",
+			Detail:  fmt.Sprintf("max_depth=%d observed_depth=%d lines=%d", maxDepth, depth, len(f.lines)),
+		})
+	}
+
+	var sb strings.Builder
+	for i, l := range f.lines {
+		if i > 0 {
+			sb.WriteByte('\n')
+		}
+		start := sb.Len()
+		sb.WriteString(l.text)
+		doc.Blocks = append(doc.Blocks, Block{
+			Kind:    BlockJSONField,
+			Text:    l.text,
+			Start:   start,
+			End:     start + len(l.text),
+			Section: l.section,
+		})
+	}
+	doc.Text = sb.String()
+
+	// Blocks were built against a builder whose offsets already account for the
+	// newline separators, so they index correctly into doc.Text.
+	return &doc, nil
+}
+
+type flatLine struct {
+	text    string
+	section string
+}
+
+type flattener struct {
+	lines     []flatLine
+	maxDepth  int
+	truncated bool
+}
+
+// walk renders v as one or more "path: value" lines. Objects contribute a
+// section name so the chunker can keep an object's fields together.
+func (f *flattener) walk(path string, v any, depth int) {
+	if len(f.lines) >= maxFlattenedLines {
+		f.truncated = true
+		return
+	}
+	if depth > f.maxDepth {
+		f.truncated = true
+		f.emit(path, "[nested value omitted: depth limit reached]", path)
+		return
+	}
+
+	switch t := v.(type) {
+	case map[string]any:
+		keys := make([]string, 0, len(t))
+		for k := range t {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys) // stable output, so hashes are reproducible
+		if len(keys) == 0 {
+			f.emit(path, "{}", path)
+			return
+		}
+		for _, k := range keys {
+			f.walk(join(path, sanitiseKey(k)), t[k], depth+1)
+		}
+	case []any:
+		if len(t) == 0 {
+			f.emit(path, "[]", path)
+			return
+		}
+		for i, e := range t {
+			f.walk(fmt.Sprintf("%s[%d]", path, i), e, depth+1)
+		}
+	case nil:
+		f.emit(path, "null", sectionOf(path))
+	case bool:
+		f.emit(path, strconv.FormatBool(t), sectionOf(path))
+	case json.Number:
+		f.emit(path, t.String(), sectionOf(path))
+	case string:
+		f.emit(path, t, sectionOf(path))
+	default:
+		f.emit(path, fmt.Sprint(t), sectionOf(path))
+	}
+}
+
+func (f *flattener) emit(path, value, section string) {
+	if len(f.lines) >= maxFlattenedLines {
+		f.truncated = true
+		return
+	}
+	if path == "" {
+		path = "value"
+	}
+	f.lines = append(f.lines, flatLine{text: path + ": " + value, section: section})
+}
+
+func join(base, key string) string {
+	if base == "" {
+		return key
+	}
+	return base + "." + key
+}
+
+// sectionOf returns the parent path, used to group an object's fields.
+func sectionOf(path string) string {
+	if i := strings.LastIndexAny(path, "."); i > 0 {
+		return path[:i]
+	}
+	return path
+}
+
+// sanitiseKey strips control characters from object keys so that a crafted key
+// cannot inject line breaks into the flattened rendering.
+func sanitiseKey(k string) string {
+	return strings.Map(func(r rune) rune {
+		if r == '\n' || r == '\r' || r == '\t' || r < 0x20 {
+			return ' '
+		}
+		return r
+	}, k)
+}
+
+// scanDepth measures maximum nesting using a streaming token scan, so nesting
+// is bounded before any recursive structure is built. It returns the observed
+// depth, stopping early once limit is exceeded.
+func scanDepth(data []byte, limit int) (int, error) {
+	dec := json.NewDecoder(bytes.NewReader(data))
+	depth, maxSeen := 0, 0
+	for {
+		tok, err := dec.Token()
+		if err == io.EOF {
+			return maxSeen, nil
+		}
+		if err != nil {
+			return maxSeen, err
+		}
+		switch d := tok.(type) {
+		case json.Delim:
+			switch d {
+			case '{', '[':
+				depth++
+				if depth > maxSeen {
+					maxSeen = depth
+				}
+				// Bail out early rather than walking a deeply nested bomb.
+				if depth > limit+50 {
+					return maxSeen, nil
+				}
+			case '}', ']':
+				depth--
+			}
+		}
+	}
+}
+
+// validJSON reports whether data is a single well-formed JSON value.
+func validJSON(data []byte) bool { return json.Valid(data) }
@@ -0,0 +1,236 @@
+package parsing
+
+import (
+	"strings"
+)
+
+// MarkdownParser extracts text from Markdown while preserving heading structure.
+//
+// It is deliberately not a full CommonMark implementation: Relata needs
+// paragraph and heading boundaries for chunking, not rendering. Markdown is
+// never converted to HTML anywhere in Relata — stored content is escaped on
+// output, so there is no path from a document to executable markup.
+type MarkdownParser struct{}
+
+// Name identifies the parser.
+func (MarkdownParser) Name() string { return "markdown" }
+
+// Handles reports whether p can parse mime.
+func (MarkdownParser) Handles(mime string) bool { return isMarkdown(mime, "") }
+
+// Parse walks the document line by line, emitting a block per heading,
+// paragraph, list group, fenced code block and block quote.
+func (MarkdownParser) Parse(in Input) (*Document, error) {
+	text := normaliseText(string(in.Data))
+	doc := &Document{Text: text}
+
+	// sectionPath tracks the enclosing heading trail, e.g. "Policy > Delivery".
+	var sectionPath []string
+
+	lines := lineSpans(text)
+	i := 0
+	for i < len(lines) {
+		ln := lines[i]
+		raw := text[ln.start:ln.end]
+		trimmed := strings.TrimSpace(raw)
+
+		switch {
+		case trimmed == "":
+			i++
+
+		case strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~"):
+			fence := trimmed[:3]
+			start := ln.start
+			i++
+			for i < len(lines) {
+				t := strings.TrimSpace(text[lines[i].start:lines[i].end])
+				i++
+				if strings.HasPrefix(t, fence) {
+					break
+				}
+			}
+			end := ln.end
+			if i-1 < len(lines) {
+				end = lines[min(i-1, len(lines)-1)].end
+			}
+			doc.Blocks = append(doc.Blocks, Block{
+				Kind: BlockCode, Text: text[start:end], Start: start, End: end,
+				Section: strings.Join(sectionPath, " > "),
+			})
+
+		case headingLevel(trimmed) > 0:
+			lvl := headingLevel(trimmed)
+			title := strings.TrimSpace(strings.TrimLeft(trimmed, "# "))
+			// Truncate the path to the parent level, then push this heading.
+			if lvl-1 < len(sectionPath) {
+				sectionPath = sectionPath[:lvl-1]
+			}
+			for len(sectionPath) < lvl-1 {
+				sectionPath = append(sectionPath, "")
+			}
+			sectionPath = append(sectionPath, title)
+			hs, he := trimSpan(text, ln.start, ln.end)
+			doc.Blocks = append(doc.Blocks, Block{
+				Kind: BlockHeading, Text: text[hs:he], Start: hs, End: he,
+				Level: lvl, Section: strings.Join(sectionPath, " > "),
+			})
+			i++
+
+		case isSetextUnderline(text, lines, i):
+			// "Title" followed by "=====" or "-----".
+			lvl := 1
+			if strings.HasPrefix(strings.TrimSpace(text[lines[i+1].start:lines[i+1].end]), "-") {
+				lvl = 2
+			}
+			if lvl-1 < len(sectionPath) {
+				sectionPath = sectionPath[:lvl-1]
+			}
+			sectionPath = append(sectionPath, trimmed)
+			hs, he := trimSpan(text, ln.start, ln.end)
+			doc.Blocks = append(doc.Blocks, Block{
+				Kind: BlockHeading, Text: text[hs:he], Start: hs, End: he,
+				Level: lvl, Section: strings.Join(sectionPath, " > "),
+			})
+			i += 2
+
+		case isListLine(trimmed):
+			start := ln.start
+			end := ln.end
+			for i < len(lines) {
+				t := strings.TrimSpace(text[lines[i].start:lines[i].end])
+				if t == "" || (!isListLine(t) && !strings.HasPrefix(text[lines[i].start:lines[i].end], "  ")) {
+					break
+				}
+				end = lines[i].end
+				i++
+			}
+			doc.Blocks = append(doc.Blocks, Block{
+				Kind: BlockList, Text: text[start:end], Start: start, End: end,
+				Section: strings.Join(sectionPath, " > "),
+			})
+
+		case strings.HasPrefix(trimmed, ">"):
+			start := ln.start
+			end := ln.end
+			for i < len(lines) && strings.HasPrefix(strings.TrimSpace(text[lines[i].start:lines[i].end]), ">") {
+				end = lines[i].end
+				i++
+			}
+			doc.Blocks = append(doc.Blocks, Block{
+				Kind: BlockQuote, Text: text[start:end], Start: start, End: end,
+				Section: strings.Join(sectionPath, " > "),
+			})
+
+		case strings.HasPrefix(trimmed, "|") && strings.Count(trimmed, "|") >= 2:
+			start := ln.start
+			end := ln.end
+			for i < len(lines) && strings.HasPrefix(strings.TrimSpace(text[lines[i].start:lines[i].end]), "|") {
+				end = lines[i].end
+				i++
+			}
+			doc.Blocks = append(doc.Blocks, Block{
+				Kind: BlockTable, Text: text[start:end], Start: start, End: end,
+				Section: strings.Join(sectionPath, " > "),
+			})
+
+		default:
+			start := ln.start
+			end := ln.end
+			for i < len(lines) {
+				t := strings.TrimSpace(text[lines[i].start:lines[i].end])
+				if t == "" || headingLevel(t) > 0 || isListLine(t) || strings.HasPrefix(t, "```") {
+					break
+				}
+				end = lines[i].end
+				i++
+			}
+			doc.Blocks = append(doc.Blocks, Block{
+				Kind: BlockParagraph, Text: text[start:end], Start: start, End: end,
+				Section: strings.Join(sectionPath, " > "),
+			})
+		}
+	}
+
+	if len(doc.Blocks) == 0 && strings.TrimSpace(text) != "" {
+		doc.Blocks = append(doc.Blocks, Block{Kind: BlockParagraph, Text: text, Start: 0, End: len(text)})
+	}
+	return doc, nil
+}
+
+// trimSpan narrows [start,end) past surrounding whitespace, so that a block's
+// Text always equals text[Start:End]. Every derived record's source reference
+// depends on that identity holding.
+func trimSpan(text string, start, end int) (int, int) {
+	for start < end && isSpaceByte(text[start]) {
+		start++
+	}
+	for end > start && isSpaceByte(text[end-1]) {
+		end--
+	}
+	return start, end
+}
+
+func isSpaceByte(b byte) bool {
+	return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\v' || b == '\f'
+}
+
+func lineSpans(text string) []span {
+	var out []span
+	start := 0
+	for i := 0; i < len(text); i++ {
+		if text[i] == '\n' {
+			out = append(out, span{start, i})
+			start = i + 1
+		}
+	}
+	if start <= len(text) {
+		out = append(out, span{start, len(text)})
+	}
+	return out
+}
+
+func headingLevel(trimmed string) int {
+	n := 0
+	for n < len(trimmed) && trimmed[n] == '#' {
+		n++
+	}
+	if n == 0 || n > 6 {
+		return 0
+	}
+	if n >= len(trimmed) || (trimmed[n] != ' ' && trimmed[n] != '\t') {
+		return 0
+	}
+	return n
+}
+
+func isListLine(t string) bool {
+	if strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* ") || strings.HasPrefix(t, "+ ") {
+		return true
+	}
+	for i := 0; i < len(t) && i < 4; i++ {
+		if t[i] >= '0' && t[i] <= '9' {
+			continue
+		}
+		return i > 0 && (t[i] == '.' || t[i] == ')') && i+1 < len(t) && t[i+1] == ' '
+	}
+	return false
+}
+
+func isSetextUnderline(text string, lines []span, i int) bool {
+	if i+1 >= len(lines) {
+		return false
+	}
+	cur := strings.TrimSpace(text[lines[i].start:lines[i].end])
+	next := strings.TrimSpace(text[lines[i+1].start:lines[i+1].end])
+	if cur == "" || len(next) < 2 {
+		return false
+	}
+	return strings.Trim(next, "=") == "" || strings.Trim(next, "-") == ""
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
@@ -0,0 +1,225 @@
+// Package parsing extracts text from supported source formats.
+//
+// Every parser treats its input as attacker-controlled. Parsers never execute
+// content, never follow URLs found inside documents, and never fetch remote
+// resources. When extraction fails or degrades, the failure is recorded as a
+// warning and the original bytes are preserved by the caller — content is never
+// silently discarded.
+package parsing
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/andydixon/relata/internal/domain"
+)
+
+// Version identifies the parser generation. It is stamped onto every item so
+// that a parser change can be detected and the item reprocessed.
+const Version = "parser-v1"
+
+// BlockKind describes the structural role of a text block. The chunker uses it
+// to prefer semantically meaningful boundaries.
+type BlockKind string
+
+const (
+	BlockHeading   BlockKind = "heading"
+	BlockParagraph BlockKind = "paragraph"
+	BlockList      BlockKind = "list"
+	BlockCode      BlockKind = "code"
+	BlockTable     BlockKind = "table"
+	BlockQuote     BlockKind = "quote"
+	BlockJSONField BlockKind = "json_field"
+)
+
+// Block is a contiguous run of extracted text with its structural context.
+// Start and End are byte offsets into Document.Text.
+type Block struct {
+	Kind    BlockKind
+	Text    string
+	Start   int
+	End     int
+	Level   int    // heading depth, 1..6
+	Page    int    // 1-based; 0 when unknown
+	Section string // nearest enclosing heading path
+}
+
+// Document is a parser's output.
+type Document struct {
+	// Text is the extracted plain text. Block offsets index into this string.
+	Text     string
+	Blocks   []Block
+	Warnings []domain.ParseWarning
+	// PageCount is set for paginated formats.
+	PageCount int
+	// Detected is the MIME type the parser actually handled, which may differ
+	// from what the client claimed.
+	Detected string
+	Version  string
+}
+
+// Input is a document to parse.
+type Input struct {
+	Data []byte
+	// MIMEType is the resolved type. Callers should sniff before parsing.
+	MIMEType string
+	Filename string
+	// MaxTextBytes truncates extracted output; 0 means unlimited.
+	MaxTextBytes int
+	// MaxJSONDepth bounds JSON nesting; 0 uses the default.
+	MaxJSONDepth int
+}
+
+// ErrUnsupportedType is returned for a MIME type no parser handles.
+var ErrUnsupportedType = errors.New("unsupported content type")
+
+// Parser extracts a Document from raw bytes.
+type Parser interface {
+	Parse(in Input) (*Document, error)
+	Handles(mime string) bool
+	Name() string
+}
+
+// Parse dispatches to the parser matching in.MIMEType.
+//
+// A parser failure is not fatal: the caller is expected to preserve the original
+// bytes and surface the returned warning. Parse returns a Document with a
+// parse_failed warning and empty text rather than nothing at all, so that the
+// item still exists and can be reprocessed later.
+func Parse(in Input) (*Document, error) {
+	var p Parser
+	switch {
+	case isTextLike(in.MIMEType):
+		p = TextParser{}
+	case isMarkdown(in.MIMEType, in.Filename):
+		p = MarkdownParser{}
+	case isJSON(in.MIMEType):
+		p = JSONParser{}
+	case isPDF(in.MIMEType):
+		p = PDFParser{}
+	case isDOCX(in.MIMEType):
+		p = DOCXParser{}
+	default:
+		return nil, fmt.Errorf("%w: %s", ErrUnsupportedType, in.MIMEType)
+	}
+
+	doc, err := p.Parse(in)
+	if err != nil {
+		return &Document{
+			Detected: in.MIMEType,
+			Version:  Version,
+			Warnings: []domain.ParseWarning{{
+				Code:    domain.WarnParseFailed,
+				Message: "Text could not be extracted from this file. The original is preserved unchanged.",
+				Detail:  truncate(err.Error(), 400),
+			}},
+		}, err
+	}
+	doc.Version = Version
+	if doc.Detected == "" {
+		doc.Detected = in.MIMEType
+	}
+	applyLimits(doc, in.MaxTextBytes)
+	checkExtractionQuality(doc, len(in.Data))
+	return doc, nil
+}
+
+func isMarkdown(mime, filename string) bool {
+	if mime == "text/markdown" || mime == "text/x-markdown" {
+		return true
+	}
+	lf := strings.ToLower(filename)
+	return strings.HasSuffix(lf, ".md") || strings.HasSuffix(lf, ".markdown")
+}
+
+func isTextLike(mime string) bool {
+	switch mime {
+	case "text/plain", "text/csv", "message/rfc822", "text/x-log":
+		return true
+	}
+	return false
+}
+
+func isJSON(mime string) bool {
+	return mime == "application/json" || mime == "text/json" || strings.HasSuffix(mime, "+json")
+}
+
+func isPDF(mime string) bool { return mime == "application/pdf" }
+
+func isDOCX(mime string) bool {
+	return mime == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+}
+
+// applyLimits truncates extracted text to max bytes, dropping any blocks that
+// fall entirely beyond the cut and clamping the one that straddles it.
+func applyLimits(doc *Document, max int) {
+	if max <= 0 || len(doc.Text) <= max {
+		return
+	}
+	cut := max
+	// Do not split a UTF-8 rune.
+	for cut > 0 && !utf8.RuneStart(doc.Text[cut]) {
+		cut--
+	}
+	doc.Text = doc.Text[:cut]
+
+	kept := doc.Blocks[:0]
+	for _, b := range doc.Blocks {
+		if b.Start >= cut {
+			continue
+		}
+		if b.End > cut {
+			b.End = cut
+			b.Text = doc.Text[b.Start:cut]
+		}
+		kept = append(kept, b)
+	}
+	doc.Blocks = kept
+	doc.Warnings = append(doc.Warnings, domain.ParseWarning{
+		Code:    domain.WarnTruncatedText,
+		Message: "Extracted text exceeded the configured limit and was truncated for indexing.",
+		Detail:  fmt.Sprintf("truncated to %d bytes; the original file is stored unchanged", max),
+	})
+}
+
+// checkExtractionQuality flags documents that yielded implausibly little text
+// for their size — most often a scanned PDF. Relata does not OCR in this
+// version, so the condition is reported rather than worked around.
+func checkExtractionQuality(doc *Document, rawSize int) {
+	if rawSize < 4096 {
+		return
+	}
+	printable := 0
+	for _, r := range doc.Text {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) {
+			printable++
+		}
+	}
+	// A text-bearing document of this size should yield far more than this.
+	minExpected := rawSize / 100
+	if minExpected < 80 {
+		minExpected = 80
+	}
+	if printable < minExpected {
+		doc.Warnings = append(doc.Warnings, domain.ParseWarning{
+			Code: domain.WarnLowExtractableText,
+			Message: "Very little machine-readable text was extracted. " +
+				"If this is a scanned document, Relata does not perform OCR in this version, " +
+				"so its contents will not be searchable.",
+			Detail: fmt.Sprintf("%d alphanumeric characters extracted from %d bytes", printable, rawSize),
+		})
+	}
+}
+
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	for n > 0 && !utf8.RuneStart(s[n]) {
+		n--
+	}
+	return s[:n] + "…"
+}
@@ -0,0 +1,519 @@
+package parsing
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/andydixon/relata/internal/domain"
+)
+
+func hasWarning(warns []domain.ParseWarning, code string) bool {
+	for _, w := range warns {
+		if w.Code == code {
+			return true
+		}
+	}
+	return false
+}
+
+// --- MIME sniffing ---------------------------------------------------------
+
+func TestSniffPrefersContentOverDeclaredType(t *testing.T) {
+	// A real PDF declared as plain text must still be parsed as a PDF.
+	res, err := Sniff([]byte("%PDF-1.7\n%âãÏÓ\n"), "text/plain", "notes.txt")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if res.MIME != "application/pdf" {
+		t.Fatalf("content-based sniffing failed: got %q", res.MIME)
+	}
+	if !res.FromContent {
+		t.Fatal("result should be marked as content-derived")
+	}
+}
+
+func TestSniffRejectsArchivesDisguisedAsDocuments(t *testing.T) {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, _ := zw.Create("payload.sh")
+	_, _ = w.Write([]byte("rm -rf /"))
+	_ = zw.Close()
+
+	// Claimed as a DOCX, with a .docx name, but it has no OOXML part.
+	_, err := Sniff(buf.Bytes(), docxMIME, "report.docx")
+	if !errors.Is(err, ErrArchiveRejected) {
+		t.Fatalf("archive masquerading as docx was accepted: %v", err)
+	}
+}
+
+func TestSniffAcceptsGenuineDOCX(t *testing.T) {
+	data := buildDOCX(t, `<w:p><w:r><w:t>Hello</w:t></w:r></w:p>`)
+	res, err := Sniff(data, "application/octet-stream", "anything.bin")
+	if err != nil {
+		t.Fatalf("genuine docx rejected: %v", err)
+	}
+	if res.MIME != docxMIME {
+		t.Fatalf("got %q, want docx", res.MIME)
+	}
+}
+
+func TestSniffRejectsExecutables(t *testing.T) {
+	for name, payload := range map[string][]byte{
+		"elf":    {0x7f, 'E', 'L', 'F', 2, 1, 1, 0},
+		"pe":     []byte("MZ\x90\x00\x03"),
+		"script": []byte("#!/bin/sh\necho hi\n"),
+		"macho":  {0xCA, 0xFE, 0xBA, 0xBE, 0, 0, 0, 2},
+	} {
+		if _, err := Sniff(payload, "text/plain", name+".txt"); !errors.Is(err, ErrExecutableRejected) {
+			t.Fatalf("%s payload was not rejected: %v", name, err)
+		}
+	}
+}
+
+func TestSniffRejectsCompressedFormats(t *testing.T) {
+	cases := map[string][]byte{
+		"gzip":  {0x1f, 0x8b, 0x08},
+		"bzip2": []byte("BZh9"),
+		"xz":    {0xfd, '7', 'z', 'X', 'Z', 0x00},
+		"rar":   []byte("Rar!\x1a\x07"),
+		"7z":    {0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C},
+	}
+	for name, payload := range cases {
+		if _, err := Sniff(payload, "application/octet-stream", name); !errors.Is(err, ErrArchiveRejected) {
+			t.Fatalf("%s was not rejected: %v", name, err)
+		}
+	}
+}
+
+func TestSniffRejectsBinaryPosingAsText(t *testing.T) {
+	binary := append([]byte("looks like text at first"), 0x00, 0x01, 0x02, 0xff, 0xfe)
+	binary = append(binary, bytes.Repeat([]byte{0x00}, 200)...)
+	if _, err := Sniff(binary, "text/plain", "notes.txt"); !errors.Is(err, ErrUnsupportedType) {
+		t.Fatalf("binary content accepted as text: %v", err)
+	}
+}
+
+func TestSniffDetectsJSONByContent(t *testing.T) {
+	res, err := Sniff([]byte(`{"cost": 42000, "vendor": "Example Ltd"}`), "text/plain", "data.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.MIME != "application/json" {
+		t.Fatalf("got %q, want application/json", res.MIME)
+	}
+}
+
+func TestSniffTreatsInvalidJSONAsText(t *testing.T) {
+	// Declared JSON that does not parse must still be stored, as text.
+	res, err := Sniff([]byte(`{"broken": `), "application/json", "data.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.MIME != "text/plain" {
+		t.Fatalf("got %q, want text/plain fallback", res.MIME)
+	}
+}
+
+// --- Text ------------------------------------------------------------------
+
+func TestTextParagraphOffsetsAddressTheirOwnText(t *testing.T) {
+	body := "First paragraph here.\n\nSecond paragraph here.\n\nThird one."
+	doc, err := Parse(Input{Data: []byte(body), MIMEType: "text/plain"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(doc.Blocks) != 3 {
+		t.Fatalf("expected 3 paragraphs, got %d", len(doc.Blocks))
+	}
+	for i, b := range doc.Blocks {
+		if doc.Text[b.Start:b.End] != b.Text {
+			t.Fatalf("block %d offsets do not match its text", i)
+		}
+	}
+}
+
+func TestTextPreservesContentExactly(t *testing.T) {
+	body := "Line one\nLine two\n\nAfter blank"
+	doc, _ := Parse(Input{Data: []byte(body), MIMEType: "text/plain"})
+	if doc.Text != body {
+		t.Fatalf("text was rewritten:\n got %q\nwant %q", doc.Text, body)
+	}
+}
+
+func TestEmailHeadersBecomeTheirOwnBlock(t *testing.T) {
+	body := "From: alice@example.com\nTo: bob@example.com\nSubject: Delivery date\nDate: Mon, 3 Mar 2025 09:00:00 +0000\n\nThe date has moved to 14 March."
+	doc, err := Parse(Input{Data: []byte(body), MIMEType: "text/plain"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(doc.Blocks) == 0 || doc.Blocks[0].Section != "headers" {
+		t.Fatal("email headers were not recognised as a header block")
+	}
+	if !strings.Contains(doc.Blocks[0].Text, "Subject: Delivery date") {
+		t.Fatal("header block lost the subject line")
+	}
+}
+
+// --- Markdown --------------------------------------------------------------
+
+func TestMarkdownBuildsSectionPaths(t *testing.T) {
+	body := "# Policy\n\nIntro text.\n\n## Delivery\n\nDelivery text.\n\n### Exceptions\n\nException text.\n"
+	doc, err := Parse(Input{Data: []byte(body), MIMEType: "text/markdown"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var deepest string
+	for _, b := range doc.Blocks {
+		if strings.Contains(b.Text, "Exception text") {
+			deepest = b.Section
+		}
+	}
+	if deepest != "Policy > Delivery > Exceptions" {
+		t.Fatalf("section path wrong: %q", deepest)
+	}
+}
+
+func TestMarkdownIsNeverConvertedToHTML(t *testing.T) {
+	// Markdown containing raw HTML must come out as literal text; Relata never
+	// renders uploaded markup.
+	body := "# Title\n\n<script>alert(1)</script>\n\n[link](javascript:alert(2))\n"
+	doc, err := Parse(Input{Data: []byte(body), MIMEType: "text/markdown"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(doc.Text, "<script>alert(1)</script>") {
+		t.Fatal("content was altered by the parser")
+	}
+	for _, b := range doc.Blocks {
+		if b.Kind == BlockCode {
+			continue
+		}
+		if strings.Contains(b.Text, "<b>") {
+			t.Fatal("parser produced markup")
+		}
+	}
+}
+
+func TestMarkdownListGroupsStayTogether(t *testing.T) {
+	body := "Intro.\n\n- first item\n- second item\n- third item\n\nOutro.\n"
+	doc, err := Parse(Input{Data: []byte(body), MIMEType: "text/markdown"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var list *Block
+	for i := range doc.Blocks {
+		if doc.Blocks[i].Kind == BlockList {
+			list = &doc.Blocks[i]
+		}
+	}
+	if list == nil {
+		t.Fatal("no list block produced")
+	}
+	for _, want := range []string{"first item", "second item", "third item"} {
+		if !strings.Contains(list.Text, want) {
+			t.Fatalf("list block missing %q", want)
+		}
+	}
+}
+
+// --- JSON ------------------------------------------------------------------
+
+func TestJSONFlattensToReadablePaths(t *testing.T) {
+	body := `{"project":{"name":"Atlas","cost":42000},"vendor":"Example Ltd"}`
+	doc, err := Parse(Input{Data: []byte(body), MIMEType: "application/json"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"project.name: Atlas", "project.cost: 42000", "vendor: Example Ltd"} {
+		if !strings.Contains(doc.Text, want) {
+			t.Fatalf("flattened output missing %q:\n%s", want, doc.Text)
+		}
+	}
+	for i, b := range doc.Blocks {
+		if doc.Text[b.Start:b.End] != b.Text {
+			t.Fatalf("json block %d offsets do not match its text", i)
+		}
+	}
+}
+
+func TestJSONDepthIsBounded(t *testing.T) {
+	// Deeply nested input must not recurse without limit.
+	deep := strings.Repeat(`{"a":`, 500) + `1` + strings.Repeat(`}`, 500)
+	doc, err := Parse(Input{Data: []byte(deep), MIMEType: "application/json", MaxJSONDepth: 10})
+	if err != nil {
+		t.Fatalf("deep json should parse with a warning, not fail: %v", err)
+	}
+	if !hasWarning(doc.Warnings, domain.WarnJSONDepthExceeded) {
+		t.Fatal("depth limit was hit but not reported to the user")
+	}
+}
+
+func TestJSONRejectsTrailingData(t *testing.T) {
+	_, err := Parse(Input{Data: []byte(`{"a":1} {"b":2}`), MIMEType: "application/json"})
+	if err == nil {
+		t.Fatal("two concatenated documents were accepted as one")
+	}
+}
+
+func TestJSONOutputIsDeterministic(t *testing.T) {
+	body := `{"z":1,"a":2,"m":3}`
+	first, _ := Parse(Input{Data: []byte(body), MIMEType: "application/json"})
+	second, _ := Parse(Input{Data: []byte(body), MIMEType: "application/json"})
+	if first.Text != second.Text {
+		t.Fatal("flattening is not deterministic, so content hashes would be unstable")
+	}
+}
+
+func TestJSONKeysCannotInjectLineBreaks(t *testing.T) {
+	body := "{\"evil\\nkey: injected\": \"x\"}"
+	doc, err := Parse(Input{Data: []byte(body), MIMEType: "application/json"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The crafted key must not create a second logical line.
+	if strings.Count(doc.Text, "\n") != 0 {
+		t.Fatalf("newline injected via object key:\n%q", doc.Text)
+	}
+}
+
+// --- PDF -------------------------------------------------------------------
+
+func TestMalformedPDFFailsCleanlyWithoutPanic(t *testing.T) {
+	cases := map[string][]byte{
+		"header only":     []byte("%PDF-1.4\n"),
+		"truncated":       []byte("%PDF-1.4\n1 0 obj\n<< /Type /Catalog"),
+		"garbage body":    append([]byte("%PDF-1.4\n"), bytes.Repeat([]byte{0xff, 0x00, 0xAB}, 500)...),
+		"fake xref":       []byte("%PDF-1.4\ntrailer\n<< /Size 99999999 >>\nstartxref\n999999\n%%EOF"),
+		"negative length": []byte("%PDF-1.5\n1 0 obj\n<< /Length -5 >>\nstream\nxx\nendstream\nendobj\n"),
+	}
+	for name, data := range cases {
+		t.Run(name, func(t *testing.T) {
+			doc, err := Parse(Input{Data: data, MIMEType: "application/pdf"})
+			// Either outcome is acceptable; a panic or hang is not.
+			if err != nil {
+				if doc == nil || !hasWarning(doc.Warnings, domain.WarnParseFailed) {
+					t.Fatal("parse failure was not recorded as a warning")
+				}
+				return
+			}
+			if doc == nil {
+				t.Fatal("nil document with nil error")
+			}
+		})
+	}
+}
+
+// --- DOCX ------------------------------------------------------------------
+
+func buildDOCX(t *testing.T, bodyXML string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, err := zw.Create("word/document.xml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	xml := `<?xml version="1.0" encoding="UTF-8"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>` +
+		bodyXML + `</w:body></w:document>`
+	if _, err := w.Write([]byte(xml)); err != nil {
+		t.Fatal(err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return buf.Bytes()
+}
+
+func TestDOCXExtractsParagraphsAndHeadings(t *testing.T) {
+	body := `<w:p><w:pPr><w:pStyle w:val="Heading1"/></w:pPr><w:r><w:t>Delivery policy</w:t></w:r></w:p>` +
+		`<w:p><w:r><w:t>The revised date is 14 March 2025.</w:t></w:r></w:p>` +
+		`<w:p><w:pPr><w:pStyle w:val="Heading2"/></w:pPr><w:r><w:t>Exceptions</w:t></w:r></w:p>` +
+		`<w:p><w:r><w:t>Contact operations.</w:t></w:r></w:p>`
+
+	doc, err := Parse(Input{Data: buildDOCX(t, body), MIMEType: docxMIME})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(doc.Text, "14 March 2025") {
+		t.Fatalf("body text missing:\n%s", doc.Text)
+	}
+
+	var headings int
+	var deepSection string
+	for _, b := range doc.Blocks {
+		if b.Kind == BlockHeading {
+			headings++
+		}
+		if strings.Contains(b.Text, "Contact operations") {
+			deepSection = b.Section
+		}
+	}
+	if headings != 2 {
+		t.Fatalf("expected 2 headings, got %d", headings)
+	}
+	if deepSection != "Delivery policy > Exceptions" {
+		t.Fatalf("section path wrong: %q", deepSection)
+	}
+	for i, b := range doc.Blocks {
+		if doc.Text[b.Start:b.End] != b.Text {
+			t.Fatalf("docx block %d offsets do not match its text", i)
+		}
+	}
+}
+
+func TestDOCXSplitRunsAreJoined(t *testing.T) {
+	// Word routinely splits a sentence across runs; they must rejoin.
+	body := `<w:p><w:r><w:t>The cost is </w:t></w:r><w:r><w:t>£42,000</w:t></w:r><w:r><w:t> excluding VAT.</w:t></w:r></w:p>`
+	doc, err := Parse(Input{Data: buildDOCX(t, body), MIMEType: docxMIME})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(doc.Text, "The cost is £42,000 excluding VAT.") {
+		t.Fatalf("runs were not joined: %q", doc.Text)
+	}
+}
+
+func TestDOCXWithoutMainPartIsRejected(t *testing.T) {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, _ := zw.Create("word/other.xml")
+	_, _ = w.Write([]byte("<x/>"))
+	_ = zw.Close()
+
+	if _, err := Parse(Input{Data: buf.Bytes(), MIMEType: docxMIME}); err == nil {
+		t.Fatal("a zip without word/document.xml was accepted as a docx")
+	}
+}
+
+func TestDOCXDoesNotResolveExternalEntities(t *testing.T) {
+	// An XXE attempt must not read a local file or make a network call.
+	body := `<w:p><w:r><w:t>&xxe;</w:t></w:r></w:p>`
+	xml := `<?xml version="1.0"?><!DOCTYPE d [<!ENTITY xxe SYSTEM "file:///etc/passwd">]>` +
+		`<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>` + body + `</w:body></w:document>`
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, _ := zw.Create("word/document.xml")
+	_, _ = w.Write([]byte(xml))
+	_ = zw.Close()
+
+	doc, err := Parse(Input{Data: buf.Bytes(), MIMEType: docxMIME})
+	if err == nil && doc != nil {
+		if strings.Contains(doc.Text, "root:") || strings.Contains(doc.Text, "/bin/") {
+			t.Fatal("external entity was resolved: local file contents leaked into extracted text")
+		}
+	}
+}
+
+// --- Limits and quality ----------------------------------------------------
+
+func TestTextTruncationIsReportedAndOffsetsStayValid(t *testing.T) {
+	body := strings.Repeat("Long content that goes on and on. ", 500)
+	doc, err := Parse(Input{Data: []byte(body), MIMEType: "text/plain", MaxTextBytes: 500})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(doc.Text) > 500 {
+		t.Fatalf("text not truncated: %d bytes", len(doc.Text))
+	}
+	if !hasWarning(doc.Warnings, domain.WarnTruncatedText) {
+		t.Fatal("truncation was not reported")
+	}
+	for i, b := range doc.Blocks {
+		if b.End > len(doc.Text) || doc.Text[b.Start:b.End] != b.Text {
+			t.Fatalf("block %d survived truncation with bad offsets", i)
+		}
+	}
+}
+
+func TestLowExtractableTextIsReported(t *testing.T) {
+	// A large file yielding almost no letters looks like a scan.
+	doc := &Document{Text: "  \n  x  \n "}
+	checkExtractionQuality(doc, 500_000)
+	if !hasWarning(doc.Warnings, domain.WarnLowExtractableText) {
+		t.Fatal("a scan-like document was not flagged")
+	}
+}
+
+func TestUnsupportedTypeIsRejected(t *testing.T) {
+	_, err := Parse(Input{Data: []byte("data"), MIMEType: "image/png"})
+	if !errors.Is(err, ErrUnsupportedType) {
+		t.Fatalf("expected ErrUnsupportedType, got %v", err)
+	}
+}
+
+// --- Fuzzing ---------------------------------------------------------------
+
+// FuzzParseText exercises the text and markdown paths for panics and for
+// offsets that do not address their own text.
+func FuzzParseText(f *testing.F) {
+	f.Add("hello world")
+	f.Add("# heading\n\n- a\n- b\n\n```\ncode\n```\n")
+	f.Add("From: a@b\nTo: c@d\n\nbody")
+	f.Add("\x00\x01binary-ish")
+	f.Add(strings.Repeat("a\n\n", 100))
+
+	f.Fuzz(func(t *testing.T, s string) {
+		for _, mime := range []string{"text/plain", "text/markdown"} {
+			doc, err := Parse(Input{Data: []byte(s), MIMEType: mime, MaxTextBytes: 4096})
+			if err != nil {
+				continue
+			}
+			for _, b := range doc.Blocks {
+				if b.Start < 0 || b.End > len(doc.Text) || b.Start > b.End {
+					t.Fatalf("block offsets [%d,%d) out of range for %d bytes (mime %s)",
+						b.Start, b.End, len(doc.Text), mime)
+				}
+				if doc.Text[b.Start:b.End] != b.Text {
+					t.Fatalf("block text/offset mismatch (mime %s)", mime)
+				}
+			}
+		}
+	})
+}
+
+// FuzzParseJSON exercises the JSON flattener, which handles the most
+// structurally adversarial input Relata accepts as a first-class format.
+func FuzzParseJSON(f *testing.F) {
+	f.Add(`{"a":1}`)
+	f.Add(`[1,[2,[3,[4]]]]`)
+	f.Add(`{"":{"":{"":null}}}`)
+	f.Add(`{"a":"\u0000\n\r"}`)
+
+	f.Fuzz(func(t *testing.T, s string) {
+		doc, err := Parse(Input{Data: []byte(s), MIMEType: "application/json", MaxJSONDepth: 8, MaxTextBytes: 8192})
+		if err != nil {
+			return
+		}
+		for _, b := range doc.Blocks {
+			if b.Start < 0 || b.End > len(doc.Text) || b.Start > b.End {
+				t.Fatalf("json block offsets [%d,%d) out of range for %d bytes", b.Start, b.End, len(doc.Text))
+			}
+			if doc.Text[b.Start:b.End] != b.Text {
+				t.Fatal("json block text/offset mismatch")
+			}
+		}
+	})
+}
+
+// FuzzSniff checks that type detection never panics and never accepts an
+// archive or executable, whatever the input.
+func FuzzSniff(f *testing.F) {
+	f.Add([]byte("%PDF-1.4"), "application/pdf", "a.pdf")
+	f.Add([]byte("PK\x03\x04"), "application/zip", "a.zip")
+	f.Add([]byte("{}"), "application/json", "a.json")
+
+	f.Fuzz(func(t *testing.T, data []byte, declared, name string) {
+		res, err := Sniff(data, declared, name)
+		if err != nil {
+			return
+		}
+		if res.MIME != "" && !Supported(res.MIME) {
+			t.Fatalf("sniff accepted an unsupported type: %q", res.MIME)
+		}
+	})
+}
@@ -0,0 +1,198 @@
+package parsing
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"net/http"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+)
+
+// Errors returned by MIME resolution.
+var (
+	// ErrArchiveRejected is returned for archive formats, which are out of scope
+	// for the MVP. Accepting them would mean walking untrusted archive entries,
+	// with the zip-slip and decompression-bomb handling that implies.
+	ErrArchiveRejected = errors.New("archive formats are not accepted")
+	// ErrExecutableRejected is returned for executable content.
+	ErrExecutableRejected = errors.New("executable content is not accepted")
+)
+
+// SupportedMIMETypes are the types Relata will ingest.
+var SupportedMIMETypes = []string{
+	"text/plain",
+	"text/markdown",
+	"application/json",
+	"application/pdf",
+	"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+}
+
+const docxMIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+
+// SniffResult is the outcome of content-based type detection.
+type SniffResult struct {
+	// MIME is the resolved type, decided primarily from file content.
+	MIME string
+	// Declared is what the client claimed, retained for auditing.
+	Declared string
+	// FromContent reports whether MIME came from magic bytes rather than a hint.
+	FromContent bool
+}
+
+// Sniff resolves the MIME type of data, preferring content inspection over the
+// client-supplied type and filename. A browser-supplied Content-Type is a hint,
+// never an authority: a .pdf named file containing a zip must be seen as a zip.
+func Sniff(data []byte, declared, filename string) (SniffResult, error) {
+	res := SniffResult{Declared: declared}
+
+	// 1. Unambiguous magic numbers win outright.
+	switch {
+	case bytes.HasPrefix(data, []byte("%PDF-")):
+		res.MIME, res.FromContent = "application/pdf", true
+		return res, nil
+	case isELFOrPE(data):
+		return res, ErrExecutableRejected
+	case bytes.HasPrefix(data, []byte{0x1f, 0x8b}): // gzip
+		return res, ErrArchiveRejected
+	case bytes.HasPrefix(data, []byte("BZh")): // bzip2
+		return res, ErrArchiveRejected
+	case bytes.HasPrefix(data, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}): // xz
+		return res, ErrArchiveRejected
+	case bytes.HasPrefix(data, []byte("Rar!")): // rar
+		return res, ErrArchiveRejected
+	case bytes.HasPrefix(data, []byte{0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C}): // 7z
+		return res, ErrArchiveRejected
+	}
+
+	// 2. ZIP containers: DOCX is a zip, but so is every archive we reject.
+	// Distinguish by looking for the OOXML word part, not by extension.
+	if bytes.HasPrefix(data, []byte("PK\x03\x04")) || bytes.HasPrefix(data, []byte("PK\x05\x06")) {
+		if isDOCXZip(data) {
+			res.MIME, res.FromContent = docxMIME, true
+			return res, nil
+		}
+		return res, ErrArchiveRejected
+	}
+
+	// 3. Textual formats. Require valid UTF-8 without NUL bytes before treating
+	// anything as text, so binary content cannot masquerade as a note.
+	if !looksTextual(data) {
+		detected := http.DetectContentType(data)
+		res.MIME = detected
+		res.FromContent = true
+		return res, ErrUnsupportedType
+	}
+
+	trimmed := bytes.TrimSpace(data)
+	if looksJSON(trimmed) {
+		res.MIME, res.FromContent = "application/json", true
+		return res, nil
+	}
+
+	// Between plain text and markdown the distinction is stylistic, so the
+	// filename and declared type are allowed to decide.
+	ext := strings.ToLower(filepath.Ext(filename))
+	switch {
+	case ext == ".md" || ext == ".markdown":
+		res.MIME = "text/markdown"
+	case declared == "text/markdown" || declared == "text/x-markdown":
+		res.MIME = "text/markdown"
+	case ext == ".json" || declared == "application/json":
+		// Declared JSON that does not parse is still stored, but as text, with
+		// the mismatch visible to the user.
+		res.MIME = "text/plain"
+	case looksMarkdown(trimmed):
+		res.MIME, res.FromContent = "text/markdown", true
+	default:
+		res.MIME = "text/plain"
+	}
+	return res, nil
+}
+
+// Supported reports whether Relata has a parser for mime.
+func Supported(mime string) bool {
+	for _, m := range SupportedMIMETypes {
+		if m == mime {
+			return true
+		}
+	}
+	return mime == "text/x-markdown" || mime == "text/json"
+}
+
+func isELFOrPE(data []byte) bool {
+	return bytes.HasPrefix(data, []byte{0x7f, 'E', 'L', 'F'}) ||
+		bytes.HasPrefix(data, []byte("MZ")) ||
+		bytes.HasPrefix(data, []byte("#!")) ||
+		bytes.HasPrefix(data, []byte{0xCA, 0xFE, 0xBA, 0xBE})
+}
+
+// isDOCXZip reports whether the zip contains the OOXML main document part.
+// It reads only the central directory, so a malformed archive costs little.
+func isDOCXZip(data []byte) bool {
+	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		return false
+	}
+	for _, f := range zr.File {
+		if f.Name == "word/document.xml" {
+			return true
+		}
+	}
+	return false
+}
+
+// looksTextual reports whether data is plausibly text: valid UTF-8, no NULs,
+// and few control characters.
+func looksTextual(data []byte) bool {
+	sample := data
+	if len(sample) > 8192 {
+		sample = sample[:8192]
+		// Trim to a rune boundary so a split multi-byte rune is not misread.
+		for len(sample) > 0 && !utf8.RuneStart(sample[len(sample)-1]) {
+			sample = sample[:len(sample)-1]
+		}
+	}
+	if bytes.IndexByte(sample, 0) >= 0 {
+		return false
+	}
+	if !utf8.Valid(sample) {
+		return false
+	}
+	ctrl := 0
+	for _, b := range sample {
+		if b < 0x09 || (b > 0x0d && b < 0x20) {
+			ctrl++
+		}
+	}
+	return len(sample) == 0 || ctrl*100/max(len(sample), 1) < 2
+}
+
+func looksJSON(t []byte) bool {
+	if len(t) == 0 {
+		return false
+	}
+	if t[0] != '{' && t[0] != '[' {
+		return false
+	}
+	return validJSON(t)
+}
+
+var mdSignals = regexp.MustCompile(`(?m)^(#{1,6}\s+\S|[-*+]\s+\S|\d+\.\s+\S|>\s+\S|` + "```" + `)`)
+
+func looksMarkdown(t []byte) bool {
+	sample := t
+	if len(sample) > 4096 {
+		sample = sample[:4096]
+	}
+	return len(mdSignals.FindAll(sample, 3)) >= 2
+}
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
@@ -0,0 +1,2 @@
+go test fuzz v1
+string("# 0 ")
@@ -0,0 +1,170 @@
+package parsing
+
+import (
+	"strings"
+
+	"github.com/andydixon/relata/internal/domain"
+)
+
+// TextParser handles plain text, CSV-ish content, transcripts and pasted emails.
+// It preserves paragraph boundaries and, for email-shaped input, records the
+// header block as a section so it stays attached to the body.
+type TextParser struct{}
+
+// Name identifies the parser.
+func (TextParser) Name() string { return "text" }
+
+// Handles reports whether p can parse mime.
+func (TextParser) Handles(mime string) bool { return isTextLike(mime) }
+
+// Parse splits text into paragraph blocks, keeping byte offsets into the
+// normalised text so every derived record can point back at its exact source.
+func (TextParser) Parse(in Input) (*Document, error) {
+	text := normaliseText(string(in.Data))
+	doc := &Document{Text: text}
+
+	section := ""
+	if hdrEnd, ok := emailHeaderEnd(text); ok {
+		section = "headers"
+		doc.Blocks = append(doc.Blocks, Block{
+			Kind:    BlockParagraph,
+			Text:    text[:hdrEnd],
+			Start:   0,
+			End:     hdrEnd,
+			Section: "headers",
+		})
+	}
+
+	for _, seg := range splitParagraphs(text) {
+		if seg.start < len(doc.Text) && len(doc.Blocks) > 0 && doc.Blocks[0].Section == "headers" && seg.end <= doc.Blocks[0].End {
+			continue // already captured in the header block
+		}
+		body := text[seg.start:seg.end]
+		if strings.TrimSpace(body) == "" {
+			continue
+		}
+		kind := BlockParagraph
+		if isListBlock(body) {
+			kind = BlockList
+		}
+		sec := section
+		if sec == "headers" {
+			sec = "body"
+		}
+		doc.Blocks = append(doc.Blocks, Block{
+			Kind:    kind,
+			Text:    body,
+			Start:   seg.start,
+			End:     seg.end,
+			Section: sec,
+		})
+	}
+
+	if len(doc.Blocks) == 0 && strings.TrimSpace(text) != "" {
+		doc.Blocks = append(doc.Blocks, Block{
+			Kind: BlockParagraph, Text: text, Start: 0, End: len(text),
+		})
+	}
+	return doc, nil
+}
+
+type span struct{ start, end int }
+
+// splitParagraphs returns byte spans separated by blank lines.
+func splitParagraphs(text string) []span {
+	var out []span
+	start := 0
+	i := 0
+	for i < len(text) {
+		// A blank-line separator is "\n" followed by optional spaces then "\n".
+		if text[i] == '\n' {
+			j := i + 1
+			for j < len(text) && (text[j] == ' ' || text[j] == '\t' || text[j] == '\r') {
+				j++
+			}
+			if j < len(text) && text[j] == '\n' {
+				if i > start {
+					out = append(out, span{start, i})
+				}
+				for j < len(text) && (text[j] == '\n' || text[j] == ' ' || text[j] == '\t' || text[j] == '\r') {
+					j++
+				}
+				start = j
+				i = j
+				continue
+			}
+		}
+		i++
+	}
+	if start < len(text) {
+		out = append(out, span{start, len(text)})
+	}
+	return out
+}
+
+func isListBlock(s string) bool {
+	lines := strings.Split(strings.TrimSpace(s), "\n")
+	if len(lines) < 2 {
+		return false
+	}
+	marked := 0
+	for _, l := range lines {
+		t := strings.TrimLeft(l, " \t")
+		if t == "" {
+			continue
+		}
+		if strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* ") || strings.HasPrefix(t, "+ ") ||
+			(len(t) > 2 && t[0] >= '0' && t[0] <= '9' && (t[1] == '.' || t[1] == ')')) {
+			marked++
+		}
+	}
+	return marked >= 2 && marked*2 >= len(lines)
+}
+
+// emailHeaderEnd finds the end of an RFC-822-style header block, if the text
+// begins with one. Recognising it keeps "From:"/"Date:" lines attached to the
+// message rather than scattered across chunks.
+func emailHeaderEnd(text string) (int, bool) {
+	if len(text) == 0 {
+		return 0, false
+	}
+	lines := strings.SplitAfter(text, "\n")
+	headerish := 0
+	pos := 0
+	for _, l := range lines {
+		trimmed := strings.TrimRight(l, "\r\n")
+		if trimmed == "" {
+			if headerish >= 2 {
+				return pos, true
+			}
+			return 0, false
+		}
+		if k, _, ok := strings.Cut(trimmed, ":"); ok && k != "" && !strings.ContainsAny(k, " \t") {
+			headerish++
+		} else if !strings.HasPrefix(l, " ") && !strings.HasPrefix(l, "\t") {
+			return 0, false
+		}
+		pos += len(l)
+		if headerish > 30 {
+			return 0, false
+		}
+	}
+	return 0, false
+}
+
+// normaliseText makes line endings consistent and strips a UTF-8 BOM. It does
+// not otherwise alter content: the item's original text is stored separately and
+// is never rewritten.
+func normaliseText(s string) string {
+	s = strings.TrimPrefix(s, "\uFEFF")
+	if strings.ContainsRune(s, '\r') {
+		s = strings.ReplaceAll(s, "\r\n", "\n")
+		s = strings.ReplaceAll(s, "\r", "\n")
+	}
+	return s
+}
+
+// warn builds a parse warning.
+func warn(code, msg, detail string) domain.ParseWarning {
+	return domain.ParseWarning{Code: code, Message: msg, Detail: detail}
+}