package parsing import ( "strings" ) // MarkdownParser extracts text from Markdown while preserving heading structure. // // It is deliberately not a full CommonMark implementation: Relata needs // paragraph and heading boundaries for chunking, not rendering. Markdown is // never converted to HTML anywhere in Relata — stored content is escaped on // output, so there is no path from a document to executable markup. type MarkdownParser struct{} // Name identifies the parser. func (MarkdownParser) Name() string { return "markdown" } // Handles reports whether p can parse mime. func (MarkdownParser) Handles(mime string) bool { return isMarkdown(mime, "") } // Parse walks the document line by line, emitting a block per heading, // paragraph, list group, fenced code block and block quote. func (MarkdownParser) Parse(in Input) (*Document, error) { text := normaliseText(string(in.Data)) doc := &Document{Text: text} // sectionPath tracks the enclosing heading trail, e.g. "Policy > Delivery". var sectionPath []string lines := lineSpans(text) i := 0 for i < len(lines) { ln := lines[i] raw := text[ln.start:ln.end] trimmed := strings.TrimSpace(raw) switch { case trimmed == "": i++ case strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~"): fence := trimmed[:3] start := ln.start i++ for i < len(lines) { t := strings.TrimSpace(text[lines[i].start:lines[i].end]) i++ if strings.HasPrefix(t, fence) { break } } end := ln.end if i-1 < len(lines) { end = lines[min(i-1, len(lines)-1)].end } doc.Blocks = append(doc.Blocks, Block{ Kind: BlockCode, Text: text[start:end], Start: start, End: end, Section: strings.Join(sectionPath, " > "), }) case headingLevel(trimmed) > 0: lvl := headingLevel(trimmed) title := strings.TrimSpace(strings.TrimLeft(trimmed, "# ")) // Truncate the path to the parent level, then push this heading. if lvl-1 < len(sectionPath) { sectionPath = sectionPath[:lvl-1] } for len(sectionPath) < lvl-1 { sectionPath = append(sectionPath, "") } sectionPath = append(sectionPath, title) hs, he := trimSpan(text, ln.start, ln.end) doc.Blocks = append(doc.Blocks, Block{ Kind: BlockHeading, Text: text[hs:he], Start: hs, End: he, Level: lvl, Section: strings.Join(sectionPath, " > "), }) i++ case isSetextUnderline(text, lines, i): // "Title" followed by "=====" or "-----". lvl := 1 if strings.HasPrefix(strings.TrimSpace(text[lines[i+1].start:lines[i+1].end]), "-") { lvl = 2 } if lvl-1 < len(sectionPath) { sectionPath = sectionPath[:lvl-1] } sectionPath = append(sectionPath, trimmed) hs, he := trimSpan(text, ln.start, ln.end) doc.Blocks = append(doc.Blocks, Block{ Kind: BlockHeading, Text: text[hs:he], Start: hs, End: he, Level: lvl, Section: strings.Join(sectionPath, " > "), }) i += 2 case isListLine(trimmed): start := ln.start end := ln.end for i < len(lines) { t := strings.TrimSpace(text[lines[i].start:lines[i].end]) if t == "" || (!isListLine(t) && !strings.HasPrefix(text[lines[i].start:lines[i].end], " ")) { break } end = lines[i].end i++ } doc.Blocks = append(doc.Blocks, Block{ Kind: BlockList, Text: text[start:end], Start: start, End: end, Section: strings.Join(sectionPath, " > "), }) case strings.HasPrefix(trimmed, ">"): start := ln.start end := ln.end for i < len(lines) && strings.HasPrefix(strings.TrimSpace(text[lines[i].start:lines[i].end]), ">") { end = lines[i].end i++ } doc.Blocks = append(doc.Blocks, Block{ Kind: BlockQuote, Text: text[start:end], Start: start, End: end, Section: strings.Join(sectionPath, " > "), }) case strings.HasPrefix(trimmed, "|") && strings.Count(trimmed, "|") >= 2: start := ln.start end := ln.end for i < len(lines) && strings.HasPrefix(strings.TrimSpace(text[lines[i].start:lines[i].end]), "|") { end = lines[i].end i++ } doc.Blocks = append(doc.Blocks, Block{ Kind: BlockTable, Text: text[start:end], Start: start, End: end, Section: strings.Join(sectionPath, " > "), }) default: start := ln.start end := ln.end for i < len(lines) { t := strings.TrimSpace(text[lines[i].start:lines[i].end]) if t == "" || headingLevel(t) > 0 || isListLine(t) || strings.HasPrefix(t, "```") { break } end = lines[i].end i++ } doc.Blocks = append(doc.Blocks, Block{ Kind: BlockParagraph, Text: text[start:end], Start: start, End: end, Section: strings.Join(sectionPath, " > "), }) } } if len(doc.Blocks) == 0 && strings.TrimSpace(text) != "" { doc.Blocks = append(doc.Blocks, Block{Kind: BlockParagraph, Text: text, Start: 0, End: len(text)}) } return doc, nil } // trimSpan narrows [start,end) past surrounding whitespace, so that a block's // Text always equals text[Start:End]. Every derived record's source reference // depends on that identity holding. func trimSpan(text string, start, end int) (int, int) { for start < end && isSpaceByte(text[start]) { start++ } for end > start && isSpaceByte(text[end-1]) { end-- } return start, end } func isSpaceByte(b byte) bool { return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\v' || b == '\f' } func lineSpans(text string) []span { var out []span start := 0 for i := 0; i < len(text); i++ { if text[i] == '\n' { out = append(out, span{start, i}) start = i + 1 } } if start <= len(text) { out = append(out, span{start, len(text)}) } return out } func headingLevel(trimmed string) int { n := 0 for n < len(trimmed) && trimmed[n] == '#' { n++ } if n == 0 || n > 6 { return 0 } if n >= len(trimmed) || (trimmed[n] != ' ' && trimmed[n] != '\t') { return 0 } return n } func isListLine(t string) bool { if strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* ") || strings.HasPrefix(t, "+ ") { return true } for i := 0; i < len(t) && i < 4; i++ { if t[i] >= '0' && t[i] <= '9' { continue } return i > 0 && (t[i] == '.' || t[i] == ')') && i+1 < len(t) && t[i+1] == ' ' } return false } func isSetextUnderline(text string, lines []span, i int) bool { if i+1 >= len(lines) { return false } cur := strings.TrimSpace(text[lines[i].start:lines[i].end]) next := strings.TrimSpace(text[lines[i+1].start:lines[i+1].end]) if cur == "" || len(next) < 2 { return false } return strings.Trim(next, "=") == "" || strings.Trim(next, "-") == "" } func min(a, b int) int { if a < b { return a } return b }