127 lines
3.6 KiB
Go
127 lines
3.6 KiB
Go
package parsing
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/andydixon/relata/internal/domain"
|
|
"github.com/ledongthuc/pdf"
|
|
)
|
|
|
|
// PDFParser extracts text from PDFs, recording the page each block came from.
|
|
//
|
|
// No OCR is performed in this version. A scanned PDF yields little or no text;
|
|
// that condition is detected and reported rather than hidden, and the original
|
|
// file is retained so it can be reprocessed if OCR is added later.
|
|
type PDFParser struct{}
|
|
|
|
// Name identifies the parser.
|
|
func (PDFParser) Name() string { return "pdf" }
|
|
|
|
// Handles reports whether p can parse mime.
|
|
func (PDFParser) Handles(mime string) bool { return isPDF(mime) }
|
|
|
|
// maxPDFPages bounds work on a document with an absurd page count.
|
|
const maxPDFPages = 2000
|
|
|
|
// Parse reads the PDF page by page. A page that fails to decode is skipped with
|
|
// a warning; one bad page does not discard the rest of the document.
|
|
func (PDFParser) Parse(in Input) (doc *Document, err error) {
|
|
// The PDF library panics on some malformed files. Recover so that a hostile
|
|
// or corrupt document becomes a recorded parse failure, not a crashed worker.
|
|
defer func() {
|
|
if recover() != nil {
|
|
// Third-party panic values may contain fragments of attacker-supplied
|
|
// input. Return a stable diagnostic rather than storing or logging them.
|
|
doc, err = nil, fmt.Errorf("pdf structure is malformed")
|
|
}
|
|
}()
|
|
|
|
r, perr := pdf.NewReader(bytes.NewReader(in.Data), int64(len(in.Data)))
|
|
if perr != nil {
|
|
return nil, fmt.Errorf("open pdf: %w", perr)
|
|
}
|
|
|
|
pages := r.NumPage()
|
|
if pages < 0 {
|
|
pages = 0
|
|
}
|
|
d := &Document{PageCount: pages}
|
|
if pages > maxPDFPages {
|
|
d.Warnings = append(d.Warnings, warn(domain.WarnTruncatedText,
|
|
"The PDF has more pages than Relata will index; later pages were skipped.",
|
|
fmt.Sprintf("page_count=%d indexed=%d", pages, maxPDFPages)))
|
|
pages = maxPDFPages
|
|
}
|
|
|
|
var sb strings.Builder
|
|
failedPages := 0
|
|
|
|
for pageNo := 1; pageNo <= pages; pageNo++ {
|
|
text, perr := extractPage(r, pageNo)
|
|
if perr != nil {
|
|
failedPages++
|
|
continue
|
|
}
|
|
text = normaliseText(text)
|
|
if strings.TrimSpace(text) == "" {
|
|
continue
|
|
}
|
|
// Emit one block per paragraph so that page and paragraph structure both
|
|
// survive into chunking.
|
|
base := sb.Len()
|
|
if base > 0 {
|
|
sb.WriteString("\n\n")
|
|
base = sb.Len()
|
|
}
|
|
sb.WriteString(text)
|
|
section := fmt.Sprintf("page %d", pageNo)
|
|
for _, seg := range splitParagraphs(text) {
|
|
body := text[seg.start:seg.end]
|
|
if strings.TrimSpace(body) == "" {
|
|
continue
|
|
}
|
|
d.Blocks = append(d.Blocks, Block{
|
|
Kind: BlockParagraph,
|
|
Text: body,
|
|
Start: base + seg.start,
|
|
End: base + seg.end,
|
|
Page: pageNo,
|
|
Section: section,
|
|
})
|
|
}
|
|
}
|
|
|
|
d.Text = sb.String()
|
|
if failedPages > 0 {
|
|
d.Warnings = append(d.Warnings, warn(domain.WarnParseFailed,
|
|
"Some pages could not be decoded and were skipped. The original file is preserved.",
|
|
fmt.Sprintf("failed_pages=%d of %d", failedPages, pages)))
|
|
}
|
|
if pages == 0 {
|
|
d.Warnings = append(d.Warnings, warn(domain.WarnLowExtractableText,
|
|
"The PDF reported no pages, so no text could be extracted.", ""))
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// extractPage pulls the text of one page, isolating panics to that page.
|
|
func extractPage(r *pdf.Reader, pageNo int) (text string, err error) {
|
|
defer func() {
|
|
if rec := recover(); rec != nil {
|
|
text, err = "", fmt.Errorf("page %d: %v", pageNo, rec)
|
|
}
|
|
}()
|
|
p := r.Page(pageNo)
|
|
if p.V.IsNull() {
|
|
return "", fmt.Errorf("page %d is null", pageNo)
|
|
}
|
|
// GetPlainText with a nil font map uses the page's own fonts.
|
|
s, err := p.GetPlainText(nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return s, nil
|
|
}
|