relata/internal/parsing/text.go

package parsing

import (
	"strings"

	"github.com/andydixon/relata/internal/domain"
)

// TextParser handles plain text, CSV-ish content, transcripts and pasted emails.
// It preserves paragraph boundaries and, for email-shaped input, records the
// header block as a section so it stays attached to the body.
type TextParser struct{}

// Name identifies the parser.
func (TextParser) Name() string { return "text" }

// Handles reports whether p can parse mime.
func (TextParser) Handles(mime string) bool { return isTextLike(mime) }

// Parse splits text into paragraph blocks, keeping byte offsets into the
// normalised text so every derived record can point back at its exact source.
func (TextParser) Parse(in Input) (*Document, error) {
	text := normaliseText(string(in.Data))
	doc := &Document{Text: text}

	section := ""
	if hdrEnd, ok := emailHeaderEnd(text); ok {
		section = "headers"
		doc.Blocks = append(doc.Blocks, Block{
			Kind:    BlockParagraph,
			Text:    text[:hdrEnd],
			Start:   0,
			End:     hdrEnd,
			Section: "headers",
		})
	}

	for _, seg := range splitParagraphs(text) {
		if seg.start < len(doc.Text) && len(doc.Blocks) > 0 && doc.Blocks[0].Section == "headers" && seg.end <= doc.Blocks[0].End {
			continue // already captured in the header block
		}
		body := text[seg.start:seg.end]
		if strings.TrimSpace(body) == "" {
			continue
		}
		kind := BlockParagraph
		if isListBlock(body) {
			kind = BlockList
		}
		sec := section
		if sec == "headers" {
			sec = "body"
		}
		doc.Blocks = append(doc.Blocks, Block{
			Kind:    kind,
			Text:    body,
			Start:   seg.start,
			End:     seg.end,
			Section: sec,
		})
	}

	if len(doc.Blocks) == 0 && strings.TrimSpace(text) != "" {
		doc.Blocks = append(doc.Blocks, Block{
			Kind: BlockParagraph, Text: text, Start: 0, End: len(text),
		})
	}
	return doc, nil
}

type span struct{ start, end int }

// splitParagraphs returns byte spans separated by blank lines.
func splitParagraphs(text string) []span {
	var out []span
	start := 0
	i := 0
	for i < len(text) {
		// A blank-line separator is "\n" followed by optional spaces then "\n".
		if text[i] == '\n' {
			j := i + 1
			for j < len(text) && (text[j] == ' ' || text[j] == '\t' || text[j] == '\r') {
				j++
			}
			if j < len(text) && text[j] == '\n' {
				if i > start {
					out = append(out, span{start, i})
				}
				for j < len(text) && (text[j] == '\n' || text[j] == ' ' || text[j] == '\t' || text[j] == '\r') {
					j++
				}
				start = j
				i = j
				continue
			}
		}
		i++
	}
	if start < len(text) {
		out = append(out, span{start, len(text)})
	}
	return out
}

func isListBlock(s string) bool {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	if len(lines) < 2 {
		return false
	}
	marked := 0
	for _, l := range lines {
		t := strings.TrimLeft(l, " \t")
		if t == "" {
			continue
		}
		if strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* ") || strings.HasPrefix(t, "+ ") ||
			(len(t) > 2 && t[0] >= '0' && t[0] <= '9' && (t[1] == '.' || t[1] == ')')) {
			marked++
		}
	}
	return marked >= 2 && marked*2 >= len(lines)
}

// emailHeaderEnd finds the end of an RFC-822-style header block, if the text
// begins with one. Recognising it keeps "From:"/"Date:" lines attached to the
// message rather than scattered across chunks.
func emailHeaderEnd(text string) (int, bool) {
	if len(text) == 0 {
		return 0, false
	}
	lines := strings.SplitAfter(text, "\n")
	headerish := 0
	pos := 0
	for _, l := range lines {
		trimmed := strings.TrimRight(l, "\r\n")
		if trimmed == "" {
			if headerish >= 2 {
				return pos, true
			}
			return 0, false
		}
		if k, _, ok := strings.Cut(trimmed, ":"); ok && k != "" && !strings.ContainsAny(k, " \t") {
			headerish++
		} else if !strings.HasPrefix(l, " ") && !strings.HasPrefix(l, "\t") {
			return 0, false
		}
		pos += len(l)
		if headerish > 30 {
			return 0, false
		}
	}
	return 0, false
}

// normaliseText makes line endings consistent and strips a UTF-8 BOM. It does
// not otherwise alter content: the item's original text is stored separately and
// is never rewritten.
func normaliseText(s string) string {
	s = strings.TrimPrefix(s, "\uFEFF")
	if strings.ContainsRune(s, '\r') {
		s = strings.ReplaceAll(s, "\r\n", "\n")
		s = strings.ReplaceAll(s, "\r", "\n")
	}
	return s
}

// warn builds a parse warning.
func warn(code, msg, detail string) domain.ParseWarning {
	return domain.ParseWarning{Code: code, Message: msg, Detail: detail}
}