Code sync, security fixes

internal/parsing/text.go

@@ -0,0 +1,170 @@
+package parsing
+
+import (
+	"strings"
+
+	"github.com/andydixon/relata/internal/domain"
+)
+
+// TextParser handles plain text, CSV-ish content, transcripts and pasted emails.
+// It preserves paragraph boundaries and, for email-shaped input, records the
+// header block as a section so it stays attached to the body.
+type TextParser struct{}
+
+// Name identifies the parser.
+func (TextParser) Name() string { return "text" }
+
+// Handles reports whether p can parse mime.
+func (TextParser) Handles(mime string) bool { return isTextLike(mime) }
+
+// Parse splits text into paragraph blocks, keeping byte offsets into the
+// normalised text so every derived record can point back at its exact source.
+func (TextParser) Parse(in Input) (*Document, error) {
+	text := normaliseText(string(in.Data))
+	doc := &Document{Text: text}
+
+	section := ""
+	if hdrEnd, ok := emailHeaderEnd(text); ok {
+		section = "headers"
+		doc.Blocks = append(doc.Blocks, Block{
+			Kind:    BlockParagraph,
+			Text:    text[:hdrEnd],
+			Start:   0,
+			End:     hdrEnd,
+			Section: "headers",
+		})
+	}
+
+	for _, seg := range splitParagraphs(text) {
+		if seg.start < len(doc.Text) && len(doc.Blocks) > 0 && doc.Blocks[0].Section == "headers" && seg.end <= doc.Blocks[0].End {
+			continue // already captured in the header block
+		}
+		body := text[seg.start:seg.end]
+		if strings.TrimSpace(body) == "" {
+			continue
+		}
+		kind := BlockParagraph
+		if isListBlock(body) {
+			kind = BlockList
+		}
+		sec := section
+		if sec == "headers" {
+			sec = "body"
+		}
+		doc.Blocks = append(doc.Blocks, Block{
+			Kind:    kind,
+			Text:    body,
+			Start:   seg.start,
+			End:     seg.end,
+			Section: sec,
+		})
+	}
+
+	if len(doc.Blocks) == 0 && strings.TrimSpace(text) != "" {
+		doc.Blocks = append(doc.Blocks, Block{
+			Kind: BlockParagraph, Text: text, Start: 0, End: len(text),
+		})
+	}
+	return doc, nil
+}
+
+type span struct{ start, end int }
+
+// splitParagraphs returns byte spans separated by blank lines.
+func splitParagraphs(text string) []span {
+	var out []span
+	start := 0
+	i := 0
+	for i < len(text) {
+		// A blank-line separator is "\n" followed by optional spaces then "\n".
+		if text[i] == '\n' {
+			j := i + 1
+			for j < len(text) && (text[j] == ' ' || text[j] == '\t' || text[j] == '\r') {
+				j++
+			}
+			if j < len(text) && text[j] == '\n' {
+				if i > start {
+					out = append(out, span{start, i})
+				}
+				for j < len(text) && (text[j] == '\n' || text[j] == ' ' || text[j] == '\t' || text[j] == '\r') {
+					j++
+				}
+				start = j
+				i = j
+				continue
+			}
+		}
+		i++
+	}
+	if start < len(text) {
+		out = append(out, span{start, len(text)})
+	}
+	return out
+}
+
+func isListBlock(s string) bool {
+	lines := strings.Split(strings.TrimSpace(s), "\n")
+	if len(lines) < 2 {
+		return false
+	}
+	marked := 0
+	for _, l := range lines {
+		t := strings.TrimLeft(l, " \t")
+		if t == "" {
+			continue
+		}
+		if strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* ") || strings.HasPrefix(t, "+ ") ||
+			(len(t) > 2 && t[0] >= '0' && t[0] <= '9' && (t[1] == '.' || t[1] == ')')) {
+			marked++
+		}
+	}
+	return marked >= 2 && marked*2 >= len(lines)
+}
+
+// emailHeaderEnd finds the end of an RFC-822-style header block, if the text
+// begins with one. Recognising it keeps "From:"/"Date:" lines attached to the
+// message rather than scattered across chunks.
+func emailHeaderEnd(text string) (int, bool) {
+	if len(text) == 0 {
+		return 0, false
+	}
+	lines := strings.SplitAfter(text, "\n")
+	headerish := 0
+	pos := 0
+	for _, l := range lines {
+		trimmed := strings.TrimRight(l, "\r\n")
+		if trimmed == "" {
+			if headerish >= 2 {
+				return pos, true
+			}
+			return 0, false
+		}
+		if k, _, ok := strings.Cut(trimmed, ":"); ok && k != "" && !strings.ContainsAny(k, " \t") {
+			headerish++
+		} else if !strings.HasPrefix(l, " ") && !strings.HasPrefix(l, "\t") {
+			return 0, false
+		}
+		pos += len(l)
+		if headerish > 30 {
+			return 0, false
+		}
+	}
+	return 0, false
+}
+
+// normaliseText makes line endings consistent and strips a UTF-8 BOM. It does
+// not otherwise alter content: the item's original text is stored separately and
+// is never rewritten.
+func normaliseText(s string) string {
+	s = strings.TrimPrefix(s, "\uFEFF")
+	if strings.ContainsRune(s, '\r') {
+		s = strings.ReplaceAll(s, "\r\n", "\n")
+		s = strings.ReplaceAll(s, "\r", "\n")
+	}
+	return s
+}
+
+// warn builds a parse warning.
+func warn(code, msg, detail string) domain.ParseWarning {
+	return domain.ParseWarning{Code: code, Message: msg, Detail: detail}
+}