Code sync, security fixes
This commit is contained in:
Andy Dixon
@@ -0,0 +1,185 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSanitiseFilenameStripsPathTraversal(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"../../../etc/passwd": "passwd",
|
||||
"..\\..\\..\\windows\\system32\\cmd.exe": "cmd.exe",
|
||||
"/etc/shadow": "shadow",
|
||||
"foo/../../bar.txt": "bar.txt",
|
||||
"normal.pdf": "normal.pdf",
|
||||
"with spaces.docx": "with spaces.docx",
|
||||
"unicode-ünïcödé.txt": "unicode-ünïcödé.txt",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := SanitiseFilename(in); got != want {
|
||||
t.Errorf("SanitiseFilename(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitiseFilenameRemovesDangerousCharacters(t *testing.T) {
|
||||
hostile := []string{
|
||||
"file\x00.txt", "file\nname.txt", "file\rname.txt",
|
||||
"a:b.txt", "a*b.txt", "a?b.txt", `a"b.txt`, "a<b>.txt", "a|b.txt",
|
||||
}
|
||||
for _, in := range hostile {
|
||||
got := SanitiseFilename(in)
|
||||
for _, bad := range []string{"\x00", "\n", "\r", "/", "\\", ":", "*", "?", `"`, "<", ">", "|"} {
|
||||
if strings.Contains(got, bad) {
|
||||
t.Errorf("SanitiseFilename(%q) = %q, which still contains %q", in, got, bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitiseFilenameRefusesHiddenAndReservedNames(t *testing.T) {
|
||||
// A leading dot creates a hidden file and can conceal the real extension.
|
||||
for _, in := range []string{".hidden", "...", ". leading space.txt"} {
|
||||
if got := SanitiseFilename(in); strings.HasPrefix(got, ".") {
|
||||
t.Errorf("SanitiseFilename(%q) = %q, still hidden", in, got)
|
||||
}
|
||||
}
|
||||
// Windows device names cause surprising behaviour if the file is ever
|
||||
// copied to such a system.
|
||||
for _, in := range []string{"CON", "PRN", "AUX", "NUL", "COM1", "LPT1", "con.txt", "nul.pdf"} {
|
||||
got := SanitiseFilename(in)
|
||||
if strings.EqualFold(got, in) {
|
||||
t.Errorf("reserved device name %q was passed through unchanged", in)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitiseFilenameHandlesDegenerateInput(t *testing.T) {
|
||||
for _, in := range []string{"", ".", "..", "/", "//", " ", "...."} {
|
||||
got := SanitiseFilename(in)
|
||||
if strings.ContainsAny(got, "/\\") || got == ".." || got == "." {
|
||||
t.Errorf("SanitiseFilename(%q) = %q, which is not a safe base name", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitiseFilenameIsBounded(t *testing.T) {
|
||||
long := strings.Repeat("a", 5000) + ".txt"
|
||||
if got := SanitiseFilename(long); len(got) > 200 {
|
||||
t.Fatalf("a %d-character name produced %d characters", len(long), len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitiseFilenameIsIdempotent(t *testing.T) {
|
||||
for _, in := range []string{"../../etc/passwd", "CON", ".hidden", "normal.pdf", "a:b*c.txt"} {
|
||||
once := SanitiseFilename(in)
|
||||
if twice := SanitiseFilename(once); once != twice {
|
||||
t.Errorf("not idempotent for %q: %q then %q", in, once, twice)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzSanitiseFilename(f *testing.F) {
|
||||
f.Add("../../etc/passwd")
|
||||
f.Add("normal.pdf")
|
||||
f.Add("CON")
|
||||
f.Add("\x00\xff")
|
||||
f.Add(strings.Repeat("../", 100) + "x")
|
||||
|
||||
f.Fuzz(func(t *testing.T, name string) {
|
||||
got := SanitiseFilename(name)
|
||||
|
||||
// Whatever the input, the result must be usable as a bare base name.
|
||||
for _, forbidden := range []string{"/", "\\", "\x00"} {
|
||||
if strings.Contains(got, forbidden) {
|
||||
t.Fatalf("SanitiseFilename(%q) = %q, containing %q", name, got, forbidden)
|
||||
}
|
||||
}
|
||||
if got == "." || got == ".." || strings.HasPrefix(got, ".") {
|
||||
t.Fatalf("SanitiseFilename(%q) = %q, which is a relative or hidden path", name, got)
|
||||
}
|
||||
if len(got) > 200 {
|
||||
t.Fatalf("SanitiseFilename(%q) produced %d characters", name, len(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStripControlRemovesInvisibleCharacters(t *testing.T) {
|
||||
got := stripControl(" hello\x00\x07world\x1b ")
|
||||
if strings.ContainsAny(got, "\x00\x07\x1b") {
|
||||
t.Fatalf("control characters survived: %q", got)
|
||||
}
|
||||
if got != "helloworld" {
|
||||
t.Fatalf("stripControl produced %q", got)
|
||||
}
|
||||
// Newlines and tabs are legitimate in a pasted note and must survive.
|
||||
if stripControl("line one\nline two\tindented") != "line one\nline two\tindented" {
|
||||
t.Fatal("stripControl removed a legitimate newline or tab")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataKeyPatternRejectsOperatorShapedKeys(t *testing.T) {
|
||||
valid := []string{"project", "department", "a", "a1", "a.b", "a-b", "a_b", strings.Repeat("k", 64)}
|
||||
for _, k := range valid {
|
||||
if !metadataKeyPattern.MatchString(k) {
|
||||
t.Errorf("valid metadata key %q was rejected", k)
|
||||
}
|
||||
}
|
||||
|
||||
// A key MongoDB would treat specially, or that carries markup, must not
|
||||
// reach storage.
|
||||
invalid := []string{
|
||||
"", "$ne", "$where", "$gt", ".leading", "-leading", "_leading",
|
||||
"has space", "has\nnewline", "<script>", "a/b", "a\\b",
|
||||
strings.Repeat("k", 65), "键", "a$b",
|
||||
}
|
||||
for _, k := range invalid {
|
||||
if metadataKeyPattern.MatchString(k) {
|
||||
t.Errorf("invalid metadata key %q was accepted", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceTypeInference(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"application/json": "json",
|
||||
"text/markdown": "markdown",
|
||||
"application/pdf": "document",
|
||||
"text/plain": "note",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "document",
|
||||
}
|
||||
for mime, want := range cases {
|
||||
if got := string(inferSourceType(mime)); got != want {
|
||||
t.Errorf("inferSourceType(%q) = %q, want %q", mime, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextualTypesAreStoredInline(t *testing.T) {
|
||||
// Text is kept on the item; binary formats go to disk. Getting this wrong
|
||||
// would either lose content or put arbitrary bytes in a database field.
|
||||
for _, mime := range []string{"text/plain", "text/markdown", "application/json"} {
|
||||
if !isTextual(mime) {
|
||||
t.Errorf("%q should be stored inline", mime)
|
||||
}
|
||||
}
|
||||
for _, mime := range []string{"application/pdf",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document"} {
|
||||
if isTextual(mime) {
|
||||
t.Errorf("%q should be stored on disk, not inline", mime)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateRespectsRuneBoundaries(t *testing.T) {
|
||||
s := strings.Repeat("é", 100) // two bytes each
|
||||
got := truncate(s, 15)
|
||||
for _, r := range got {
|
||||
if r == '�' {
|
||||
t.Fatalf("truncation split a rune: %q", got)
|
||||
}
|
||||
}
|
||||
if len(got) > 20 {
|
||||
t.Fatalf("truncate produced %d bytes for a limit of 15", len(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/ids"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
"github.com/andydixon/relata/internal/parsing"
|
||||
"github.com/andydixon/relata/internal/relationships"
|
||||
)
|
||||
|
||||
// DeletionPlan describes what deleting an item would remove.
|
||||
//
|
||||
// It is produced by the dry-run API so a user can see the blast radius before
|
||||
// committing — particularly the entities that exist only because of this item
|
||||
// and would therefore disappear with it.
|
||||
type DeletionPlan struct {
|
||||
ItemID string `json:"item_id"`
|
||||
Title string `json:"title"`
|
||||
Chunks int `json:"chunks"`
|
||||
Vectors int64 `json:"vectors"`
|
||||
Claims int `json:"claims"`
|
||||
Relationships int `json:"relationships"`
|
||||
OrphanedEntities []string `json:"orphaned_entities"`
|
||||
SharedEntities []string `json:"shared_entities_retained"`
|
||||
OriginalFile string `json:"original_file,omitempty"`
|
||||
Jobs int `json:"queued_jobs"`
|
||||
}
|
||||
|
||||
// PlanDeletion reports what would be removed, without changing anything.
|
||||
func (p *Pipeline) PlanDeletion(ctx context.Context, workspaceID, itemID string) (*DeletionPlan, error) {
|
||||
item, err := p.repos.Items.Get(ctx, workspaceID, itemID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
plan := &DeletionPlan{ItemID: item.ID, Title: item.Title}
|
||||
|
||||
chunks, err := p.repos.Chunks.ListByItem(ctx, workspaceID, itemID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plan.Chunks = len(chunks)
|
||||
plan.Vectors = int64(len(chunks))
|
||||
|
||||
claims, err := p.repos.Claims.ListByItem(ctx, workspaceID, itemID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plan.Claims = len(claims)
|
||||
|
||||
out, in, err := p.repos.Rels.ListForNode(ctx, workspaceID, domain.NodeItem, itemID, domain.RelationshipFilter{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plan.Relationships = len(out) + len(in)
|
||||
|
||||
ents, err := p.repos.Entities.ListByItem(ctx, workspaceID, itemID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range ents {
|
||||
// An entity referenced only by this item has no evidence left once the
|
||||
// item is gone; one referenced elsewhere survives.
|
||||
if len(e.ItemIDs) <= 1 {
|
||||
plan.OrphanedEntities = append(plan.OrphanedEntities, e.CanonicalName)
|
||||
} else {
|
||||
plan.SharedEntities = append(plan.SharedEntities, e.CanonicalName)
|
||||
}
|
||||
}
|
||||
|
||||
if item.OriginalStoredPath != "" {
|
||||
plan.OriginalFile = item.OriginalFilename
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// RequestDeletion queues an item for deletion and tombstones it immediately.
|
||||
//
|
||||
// Tombstoning first means the item stops appearing in search and on the
|
||||
// dashboard straight away, even though the derived records are cleaned up
|
||||
// asynchronously. The user sees the deletion take effect at once.
|
||||
func (p *Pipeline) RequestDeletion(ctx context.Context, pr *domain.Principal, itemID string) (string, error) {
|
||||
if pr == nil || !pr.Role.CanWrite() {
|
||||
return "", domain.ErrForbidden
|
||||
}
|
||||
item, err := p.repos.Items.Get(ctx, pr.WorkspaceID, itemID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err := p.repos.Items.Tombstone(ctx, pr.WorkspaceID, item.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
job := &domain.Job{
|
||||
ID: ids.New(),
|
||||
WorkspaceID: pr.WorkspaceID,
|
||||
Type: domain.JobDeleteItemData,
|
||||
Status: domain.JobPending,
|
||||
ItemID: item.ID,
|
||||
Priority: 20, // deletion outranks analysis
|
||||
MaxAttempts: p.cfg.Workers.MaxAttempts,
|
||||
Payload: map[string]string{"actor": pr.UserID},
|
||||
IdempotencyKey: ids.Deterministic(pr.WorkspaceID, item.ID, string(domain.JobDeleteItemData)),
|
||||
}
|
||||
if err := p.repos.Jobs.Enqueue(ctx, job); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
p.audit.Record(ctx, pr, domain.AuditItemDeleted, "item", item.ID, "requested", map[string]string{
|
||||
"title": item.Title,
|
||||
})
|
||||
return job.ID, nil
|
||||
}
|
||||
|
||||
// HandleDelete removes an item's derived records and vectors.
|
||||
//
|
||||
// The order matters: vectors go first, because an orphaned vector is the one
|
||||
// artefact that nothing else would ever clean up. Everything after it is
|
||||
// reachable from MongoDB and could be swept later if a step failed.
|
||||
func (p *Pipeline) HandleDelete(ctx context.Context, j *domain.Job) error {
|
||||
log := observability.LoggerFrom(ctx).With("item_id", j.ItemID)
|
||||
|
||||
// Deletion is by filter, so it removes vectors even for chunks that are
|
||||
// already gone from MongoDB.
|
||||
if err := p.vectors.DeleteByItem(ctx, j.WorkspaceID, j.ItemID); err != nil {
|
||||
return fmt.Errorf("delete vectors: %w", err)
|
||||
}
|
||||
|
||||
if err := p.repos.Chunks.DeleteByItem(ctx, j.WorkspaceID, j.ItemID); err != nil {
|
||||
return fmt.Errorf("delete chunks: %w", err)
|
||||
}
|
||||
if err := p.repos.Claims.DeleteByItem(ctx, j.WorkspaceID, j.ItemID); err != nil {
|
||||
return fmt.Errorf("delete claims: %w", err)
|
||||
}
|
||||
|
||||
// Strip this item from every entity, then remove entities left with no
|
||||
// evidence at all. An entity still referenced elsewhere is retained.
|
||||
orphans, err := p.repos.Entities.RemoveItemRefs(ctx, j.WorkspaceID, j.ItemID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update entity references: %w", err)
|
||||
}
|
||||
for _, id := range orphans {
|
||||
if err := p.repos.Aliases.DeleteByEntity(ctx, j.WorkspaceID, id); err != nil {
|
||||
log.Warn("could not delete aliases for an orphaned entity", "entity_id", id, "error", err.Error())
|
||||
}
|
||||
}
|
||||
if len(orphans) > 0 {
|
||||
if err := p.repos.Entities.DeleteMany(ctx, j.WorkspaceID, orphans); err != nil {
|
||||
return fmt.Errorf("delete orphaned entities: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
relCount, err := p.repos.Rels.DeleteForItem(ctx, j.WorkspaceID, j.ItemID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete relationships: %w", err)
|
||||
}
|
||||
|
||||
if _, err := p.repos.Jobs.DeleteForItem(ctx, j.WorkspaceID, j.ItemID); err != nil {
|
||||
log.Warn("could not remove queued jobs for the deleted item", "error", err.Error())
|
||||
}
|
||||
|
||||
// Remove the stored original, then the item record itself.
|
||||
item, err := p.repos.Items.Get(ctx, j.WorkspaceID, j.ItemID)
|
||||
if err != nil && !errors.Is(err, domain.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
if item != nil {
|
||||
if err := p.ingest.DeleteOriginal(item); err != nil {
|
||||
log.Warn("could not delete the stored original file", "error", err.Error())
|
||||
}
|
||||
if err := p.repos.Items.HardDelete(ctx, j.WorkspaceID, j.ItemID); err != nil && !errors.Is(err, domain.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("deleted item and its derived records",
|
||||
"orphaned_entities", len(orphans), "relationships", relCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RequestReprocess queues an item to be processed again.
|
||||
//
|
||||
// Reprocessing replaces derived records rather than adding to them: chunks and
|
||||
// claims are swapped wholesale, entity observations merge by exact key, and
|
||||
// relationships upsert on a dedupe key that leaves human review decisions
|
||||
// untouched. That is what keeps a reprocess from multiplying data.
|
||||
func (p *Pipeline) RequestReprocess(ctx context.Context, pr *domain.Principal, itemID, reason string) (string, error) {
|
||||
if pr == nil || !pr.Role.CanWrite() {
|
||||
return "", domain.ErrForbidden
|
||||
}
|
||||
item, err := p.repos.Items.Get(ctx, pr.WorkspaceID, itemID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if item.Deleted {
|
||||
return "", domain.ErrNotFound
|
||||
}
|
||||
|
||||
runID := ids.New()
|
||||
run := &domain.ProcessingRun{
|
||||
ID: runID,
|
||||
WorkspaceID: item.WorkspaceID,
|
||||
ItemID: item.ID,
|
||||
Reason: reasonOr(reason, "manual reprocess"),
|
||||
StartedAt: time.Now().UTC(),
|
||||
ParserVersion: parsing.Version,
|
||||
EmbeddingModel: p.embedder.ModelID(),
|
||||
AnalysisModel: p.analysis.ModelID(),
|
||||
ScoreVersion: relationships.ScoreVersion,
|
||||
}
|
||||
if err := p.repos.Runs.Create(ctx, run); err != nil {
|
||||
observability.LoggerFrom(ctx).Warn("could not open a processing run", "error", err.Error())
|
||||
}
|
||||
|
||||
job := &domain.Job{
|
||||
ID: ids.New(),
|
||||
WorkspaceID: pr.WorkspaceID,
|
||||
Type: domain.JobParseItem,
|
||||
Status: domain.JobPending,
|
||||
ItemID: item.ID,
|
||||
RunID: runID,
|
||||
Priority: 12,
|
||||
MaxAttempts: p.cfg.Workers.MaxAttempts,
|
||||
Payload: map[string]string{"reason": run.Reason},
|
||||
// The run ID is part of the key, so a reprocess is never suppressed as
|
||||
// a duplicate of the original ingest.
|
||||
IdempotencyKey: ids.Deterministic(pr.WorkspaceID, item.ID, string(domain.JobParseItem), runID),
|
||||
}
|
||||
if err := p.repos.Jobs.Enqueue(ctx, job); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
p.audit.Record(ctx, pr, domain.AuditItemReprocessed, "item", item.ID, "requested", map[string]string{
|
||||
"reason": run.Reason,
|
||||
"parser_version": parsing.Version,
|
||||
"embedding_model": p.embedder.ModelID(),
|
||||
"score_version": relationships.ScoreVersion,
|
||||
})
|
||||
return job.ID, nil
|
||||
}
|
||||
|
||||
// HandleReprocess is the entry point for a queued reprocess request; it simply
|
||||
// restarts the chain at parsing.
|
||||
func (p *Pipeline) HandleReprocess(ctx context.Context, j *domain.Job) error {
|
||||
return p.HandleParse(ctx, j)
|
||||
}
|
||||
|
||||
// HandleReembedWorkspace re-embeds chunks whose vectors were produced by a
|
||||
// different model.
|
||||
//
|
||||
// The job re-enqueues itself while work remains, so a large workspace is
|
||||
// processed in bounded batches instead of one enormous job that would hold a
|
||||
// lease for hours.
|
||||
func (p *Pipeline) HandleReembedWorkspace(ctx context.Context, j *domain.Job) error {
|
||||
log := observability.LoggerFrom(ctx)
|
||||
model := p.embedder.ModelID()
|
||||
|
||||
if err := p.vectors.EnsureCollection(ctx, p.embedder.Dimensions()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
const batchSize = 64
|
||||
chunks, err := p.repos.Chunks.ListUnembedded(ctx, j.WorkspaceID, model, batchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
log.Info("workspace re-embedding complete", "model", model)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Item metadata is needed for the vector payload.
|
||||
itemIDs := make([]string, 0, len(chunks))
|
||||
seen := map[string]bool{}
|
||||
for _, c := range chunks {
|
||||
if !seen[c.ItemID] {
|
||||
seen[c.ItemID] = true
|
||||
itemIDs = append(itemIDs, c.ItemID)
|
||||
}
|
||||
}
|
||||
items, err := p.repos.Items.GetMany(ctx, j.WorkspaceID, itemIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byID := make(map[string]*domain.Item, len(items))
|
||||
for _, it := range items {
|
||||
byID[it.ID] = it
|
||||
}
|
||||
|
||||
texts := make([]string, len(chunks))
|
||||
for i, c := range chunks {
|
||||
texts[i] = c.Text
|
||||
}
|
||||
|
||||
release, err := p.pool.AcquireAI(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vecs, err := p.embedder.Embed(ctx, texts)
|
||||
release()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(vecs) != len(chunks) {
|
||||
return fmt.Errorf("embedding provider returned %d vectors for %d chunks", len(vecs), len(chunks))
|
||||
}
|
||||
|
||||
points := make([]domain.VectorPoint, 0, len(chunks))
|
||||
ok := make([]string, 0, len(chunks))
|
||||
for i, c := range chunks {
|
||||
it := byID[c.ItemID]
|
||||
if it == nil {
|
||||
// The chunk's item is gone; drop the stale chunk rather than
|
||||
// creating a vector that points at nothing.
|
||||
continue
|
||||
}
|
||||
points = append(points, domain.VectorPoint{
|
||||
ChunkID: c.ID,
|
||||
WorkspaceID: c.WorkspaceID,
|
||||
ItemID: c.ItemID,
|
||||
SourceType: it.SourceType,
|
||||
CreatedAt: it.CreatedAt,
|
||||
EmbeddingModel: model,
|
||||
Vector: vecs[i],
|
||||
})
|
||||
ok = append(ok, c.ID)
|
||||
}
|
||||
|
||||
if err := p.vectors.Upsert(ctx, points); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.repos.Chunks.MarkEmbedded(ctx, j.WorkspaceID, ok, model); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = p.repos.Models.Observe(ctx, j.WorkspaceID, "embedding", model, p.embedder.Dimensions())
|
||||
|
||||
log.Info("re-embedded a batch", "count", len(ok), "model", model)
|
||||
|
||||
// Queue the next batch. A fresh idempotency key is required or the unique
|
||||
// pending-job index would suppress it.
|
||||
next := &domain.Job{
|
||||
ID: ids.New(),
|
||||
WorkspaceID: j.WorkspaceID,
|
||||
Type: domain.JobReembedWS,
|
||||
Status: domain.JobPending,
|
||||
Priority: 3,
|
||||
MaxAttempts: p.cfg.Workers.MaxAttempts,
|
||||
IdempotencyKey: ids.Deterministic(j.WorkspaceID, string(domain.JobReembedWS), ids.New()),
|
||||
}
|
||||
return p.repos.Jobs.Enqueue(ctx, next)
|
||||
}
|
||||
|
||||
// RequestReembed queues a workspace-wide re-embedding.
|
||||
func (p *Pipeline) RequestReembed(ctx context.Context, pr *domain.Principal) (string, error) {
|
||||
if pr == nil || pr.Role != domain.RoleAdmin {
|
||||
return "", domain.ErrForbidden
|
||||
}
|
||||
job := &domain.Job{
|
||||
ID: ids.New(),
|
||||
WorkspaceID: pr.WorkspaceID,
|
||||
Type: domain.JobReembedWS,
|
||||
Status: domain.JobPending,
|
||||
Priority: 3,
|
||||
MaxAttempts: p.cfg.Workers.MaxAttempts,
|
||||
IdempotencyKey: ids.Deterministic(pr.WorkspaceID, string(domain.JobReembedWS), "manual"),
|
||||
}
|
||||
if err := p.repos.Jobs.Enqueue(ctx, job); err != nil {
|
||||
return "", err
|
||||
}
|
||||
p.audit.Record(ctx, pr, domain.AuditWorkspaceReembed, "workspace", pr.WorkspaceID, "requested",
|
||||
map[string]string{"embedding_model": p.embedder.ModelID()})
|
||||
return job.ID, nil
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/relata/internal/analysis"
|
||||
"github.com/andydixon/relata/internal/audit"
|
||||
"github.com/andydixon/relata/internal/chunking"
|
||||
"github.com/andydixon/relata/internal/config"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/entities"
|
||||
"github.com/andydixon/relata/internal/ids"
|
||||
"github.com/andydixon/relata/internal/jobs"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
"github.com/andydixon/relata/internal/parsing"
|
||||
"github.com/andydixon/relata/internal/relationships"
|
||||
)
|
||||
|
||||
// Repos bundles the repositories the pipeline writes to.
|
||||
type Repos struct {
|
||||
Items domain.ItemRepository
|
||||
Chunks domain.ChunkRepository
|
||||
Entities domain.EntityRepository
|
||||
Aliases domain.AliasRepository
|
||||
Claims domain.ClaimRepository
|
||||
Rels domain.RelationshipRepository
|
||||
Jobs domain.JobRepository
|
||||
Runs domain.RunRepository
|
||||
Models domain.ModelVersionRepository
|
||||
}
|
||||
|
||||
// Pipeline executes the processing stages as background jobs.
|
||||
//
|
||||
// Each stage is a separate job type so that progress is visible in the queue
|
||||
// and a failure is attributable to a specific stage. Stages hand off by
|
||||
// enqueuing the next one, and every stage is idempotent: re-running it replaces
|
||||
// its own output rather than adding to it.
|
||||
type Pipeline struct {
|
||||
repos Repos
|
||||
vectors domain.VectorStore
|
||||
embedder domain.Embedder
|
||||
analysis *analysis.Service
|
||||
entities *entities.Service
|
||||
discover *relationships.Discoverer
|
||||
ingest *Service
|
||||
cfg *config.Config
|
||||
pool *jobs.Pool
|
||||
audit *audit.Recorder
|
||||
}
|
||||
|
||||
// NewPipeline builds the pipeline.
|
||||
func NewPipeline(repos Repos, vectors domain.VectorStore, embedder domain.Embedder,
|
||||
an *analysis.Service, ent *entities.Service, disc *relationships.Discoverer,
|
||||
ing *Service, cfg *config.Config, pool *jobs.Pool, rec *audit.Recorder) *Pipeline {
|
||||
return &Pipeline{
|
||||
repos: repos, vectors: vectors, embedder: embedder, analysis: an,
|
||||
entities: ent, discover: disc, ingest: ing, cfg: cfg, pool: pool, audit: rec,
|
||||
}
|
||||
}
|
||||
|
||||
// Register wires the handlers into a worker pool.
|
||||
func (p *Pipeline) Register(pool *jobs.Pool) {
|
||||
pool.Register(domain.JobParseItem, p.HandleParse)
|
||||
pool.Register(domain.JobChunkItem, p.HandleChunk)
|
||||
pool.Register(domain.JobEmbedItem, p.HandleEmbed)
|
||||
pool.Register(domain.JobExtractEntity, p.HandleExtractEntities)
|
||||
pool.Register(domain.JobExtractClaims, p.HandleExtractClaims)
|
||||
pool.Register(domain.JobDiscoverRels, p.HandleDiscoverRelationships)
|
||||
pool.Register(domain.JobReprocessItem, p.HandleReprocess)
|
||||
pool.Register(domain.JobDeleteItemData, p.HandleDelete)
|
||||
pool.Register(domain.JobReembedWS, p.HandleReembedWorkspace)
|
||||
}
|
||||
|
||||
// enqueueNext queues the following stage.
|
||||
//
|
||||
// The idempotency key includes the run ID, so a reprocess enqueues a fresh
|
||||
// chain while a duplicate enqueue within the same run is a no-op.
|
||||
func (p *Pipeline) enqueueNext(ctx context.Context, j *domain.Job, next domain.JobType, priority int) error {
|
||||
nextJob := &domain.Job{
|
||||
ID: ids.New(),
|
||||
WorkspaceID: j.WorkspaceID,
|
||||
Type: next,
|
||||
Status: domain.JobPending,
|
||||
ItemID: j.ItemID,
|
||||
Payload: j.Payload,
|
||||
RunID: j.RunID,
|
||||
Priority: priority,
|
||||
MaxAttempts: p.cfg.Workers.MaxAttempts,
|
||||
IdempotencyKey: ids.Deterministic(j.WorkspaceID, j.ItemID, string(next), j.RunID),
|
||||
}
|
||||
return p.repos.Jobs.Enqueue(ctx, nextJob)
|
||||
}
|
||||
|
||||
// loadItem fetches the job's item, treating a missing item as a permanent
|
||||
// failure — it was deleted while queued, and retrying will not bring it back.
|
||||
func (p *Pipeline) loadItem(ctx context.Context, j *domain.Job) (*domain.Item, error) {
|
||||
if j.ItemID == "" {
|
||||
return nil, fmt.Errorf("%w: job has no item", domain.ErrInvalidInput)
|
||||
}
|
||||
item, err := p.repos.Items.Get(ctx, j.WorkspaceID, j.ItemID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if item.Deleted {
|
||||
return nil, fmt.Errorf("%w: item has been deleted", domain.ErrNotFound)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage 1: parse
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// HandleParse extracts text from the item's original content.
|
||||
//
|
||||
// A parse failure does not fail the job in the sense of losing the item: the
|
||||
// original is retained, the failure is recorded as a warning, and the item is
|
||||
// marked partial so a user can see what happened and reprocess later.
|
||||
func (p *Pipeline) HandleParse(ctx context.Context, j *domain.Job) error {
|
||||
item, err := p.loadItem(ctx, j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log := observability.LoggerFrom(ctx).With("item_id", item.ID)
|
||||
|
||||
if err := p.repos.Items.SetStatus(ctx, item.WorkspaceID, item.ID, domain.StatusParsing, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open a processing run so that what each generation of parser and model
|
||||
// produced stays on the record.
|
||||
runID := j.RunID
|
||||
if runID == "" {
|
||||
runID = ids.New()
|
||||
j.RunID = runID
|
||||
run := &domain.ProcessingRun{
|
||||
ID: runID,
|
||||
WorkspaceID: item.WorkspaceID,
|
||||
ItemID: item.ID,
|
||||
Reason: reasonOr(j.Payload["reason"], "ingest"),
|
||||
StartedAt: time.Now().UTC(),
|
||||
ParserVersion: parsing.Version,
|
||||
EmbeddingModel: p.embedder.ModelID(),
|
||||
AnalysisModel: p.analysis.ModelID(),
|
||||
ScoreVersion: relationships.ScoreVersion,
|
||||
}
|
||||
if err := p.repos.Runs.Create(ctx, run); err != nil {
|
||||
log.Warn("could not open a processing run", "error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
content, err := p.ingest.ReadOriginal(item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read original content: %w", err)
|
||||
}
|
||||
|
||||
doc, perr := parsing.Parse(parsing.Input{
|
||||
Data: content,
|
||||
MIMEType: item.MIMEType,
|
||||
Filename: item.OriginalFilename,
|
||||
MaxTextBytes: p.cfg.Uploads.MaxExtractedTextBytes,
|
||||
MaxJSONDepth: p.cfg.Uploads.MaxJSONDepth,
|
||||
})
|
||||
|
||||
if perr != nil {
|
||||
// Record the failure and stop the chain, keeping the original intact.
|
||||
for _, w := range doc.Warnings {
|
||||
_ = p.repos.Items.AddWarning(ctx, item.WorkspaceID, item.ID, w)
|
||||
}
|
||||
item.ParserVersion = parsing.Version
|
||||
item.Status = domain.StatusFailed
|
||||
item.Error = "Text could not be extracted from this item. The original content is preserved and can be reprocessed."
|
||||
if err := p.repos.Items.Update(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
p.finishRun(ctx, runID, "failed", perr.Error(), nil)
|
||||
log.Warn("parsing failed; the original is preserved", "error", perr.Error())
|
||||
// Returning nil: the job did its work correctly. The item's failure is
|
||||
// recorded as item state, not as an endlessly retried job.
|
||||
return nil
|
||||
}
|
||||
|
||||
item.Warnings = append(item.Warnings, doc.Warnings...)
|
||||
item.ExtractedText = doc.Text
|
||||
item.ParserVersion = doc.Version
|
||||
item.Status = domain.StatusChunking
|
||||
item.Error = ""
|
||||
if err := p.repos.Items.Update(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_ = p.repos.Models.Observe(ctx, item.WorkspaceID, "parser", parsing.Version, 0)
|
||||
log.Info("parsed item", "text_bytes", len(doc.Text), "blocks", len(doc.Blocks),
|
||||
"warnings", len(doc.Warnings))
|
||||
|
||||
// The parsed document is not carried in the job payload — payloads never
|
||||
// hold content — so the next stage re-parses from the preserved original.
|
||||
return p.enqueueNext(ctx, j, domain.JobChunkItem, 9)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage 2: chunk
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// HandleChunk divides the extracted text into retrievable chunks.
|
||||
func (p *Pipeline) HandleChunk(ctx context.Context, j *domain.Job) error {
|
||||
item, err := p.loadItem(ctx, j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log := observability.LoggerFrom(ctx).With("item_id", item.ID)
|
||||
|
||||
if err := p.repos.Items.SetStatus(ctx, item.WorkspaceID, item.ID, domain.StatusChunking, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
doc, err := p.reparse(item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
res := chunking.Split(doc, chunking.Options{
|
||||
MaxChars: p.cfg.Chunking.MaxChars,
|
||||
TargetChars: p.cfg.Chunking.TargetChars,
|
||||
MinChars: p.cfg.Chunking.MinChars,
|
||||
OverlapChars: p.cfg.Chunking.OverlapChars,
|
||||
MaxChunks: p.cfg.Chunking.MaxPerItem,
|
||||
MinSplitChars: p.cfg.Chunking.MinSplitChars,
|
||||
})
|
||||
|
||||
if res.Truncated {
|
||||
_ = p.repos.Items.AddWarning(ctx, item.WorkspaceID, item.ID, domain.ParseWarning{
|
||||
Code: domain.WarnTruncatedChunks,
|
||||
Message: "This item produced more sections than the configured limit, so its later content is not searchable.",
|
||||
Detail: fmt.Sprintf("limit=%d", p.cfg.Chunking.MaxPerItem),
|
||||
})
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
chunks := make([]*domain.Chunk, 0, len(res.Chunks))
|
||||
for _, c := range res.Chunks {
|
||||
chunks = append(chunks, &domain.Chunk{
|
||||
// A deterministic ID keyed on item and sequence means re-chunking
|
||||
// reuses the same Qdrant point IDs, so vectors are overwritten
|
||||
// rather than orphaned.
|
||||
ID: ids.DeterministicUUID(item.WorkspaceID, item.ID, fmt.Sprint(c.Sequence), c.Hash),
|
||||
WorkspaceID: item.WorkspaceID,
|
||||
ItemID: item.ID,
|
||||
Sequence: c.Sequence,
|
||||
Text: c.Text,
|
||||
ContentHash: c.Hash,
|
||||
StartOffset: c.Start,
|
||||
EndOffset: c.End,
|
||||
Page: c.Page,
|
||||
Section: c.Section,
|
||||
Heading: c.Heading,
|
||||
CreatedAt: now,
|
||||
})
|
||||
}
|
||||
|
||||
// Chunk IDs change when content changes, so vectors for the previous
|
||||
// generation must go before the new chunks are written.
|
||||
if err := p.vectors.DeleteByItem(ctx, item.WorkspaceID, item.ID); err != nil {
|
||||
log.Warn("could not clear previous vectors before re-chunking", "error", err.Error())
|
||||
}
|
||||
if err := p.repos.Chunks.ReplaceForItem(ctx, item.WorkspaceID, item.ID, chunks); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
item.ChunkCount = len(chunks)
|
||||
item.Status = domain.StatusEmbedding
|
||||
if err := p.repos.Items.Update(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("chunked item", "chunks", len(chunks), "truncated", res.Truncated)
|
||||
if len(chunks) == 0 {
|
||||
// Nothing to embed or analyse. The item is complete, if empty.
|
||||
return p.markReady(ctx, item, j)
|
||||
}
|
||||
return p.enqueueNext(ctx, j, domain.JobEmbedItem, 8)
|
||||
}
|
||||
|
||||
// reparse re-derives the parsed document from the preserved original.
|
||||
func (p *Pipeline) reparse(item *domain.Item) (*parsing.Document, error) {
|
||||
content, err := p.ingest.ReadOriginal(item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read original content: %w", err)
|
||||
}
|
||||
doc, err := parsing.Parse(parsing.Input{
|
||||
Data: content,
|
||||
MIMEType: item.MIMEType,
|
||||
Filename: item.OriginalFilename,
|
||||
MaxTextBytes: p.cfg.Uploads.MaxExtractedTextBytes,
|
||||
MaxJSONDepth: p.cfg.Uploads.MaxJSONDepth,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", domain.ErrUnsupported, err)
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage 3: embed
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// HandleEmbed generates and stores vectors for the item's chunks.
|
||||
func (p *Pipeline) HandleEmbed(ctx context.Context, j *domain.Job) error {
|
||||
item, err := p.loadItem(ctx, j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log := observability.LoggerFrom(ctx).With("item_id", item.ID)
|
||||
|
||||
if err := p.repos.Items.SetStatus(ctx, item.WorkspaceID, item.ID, domain.StatusEmbedding, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunks, err := p.repos.Chunks.ListByItem(ctx, item.WorkspaceID, item.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return p.markReady(ctx, item, j)
|
||||
}
|
||||
|
||||
texts := make([]string, len(chunks))
|
||||
for i, c := range chunks {
|
||||
texts[i] = c.Text
|
||||
}
|
||||
|
||||
// Hold an AI slot for the duration of the provider call so that concurrent
|
||||
// model usage stays within the configured limit.
|
||||
release, err := p.pool.AcquireAI(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vecs, err := p.embedder.Embed(ctx, texts)
|
||||
release()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate embeddings: %w", err)
|
||||
}
|
||||
if len(vecs) != len(chunks) {
|
||||
return fmt.Errorf("embedding provider returned %d vectors for %d chunks", len(vecs), len(chunks))
|
||||
}
|
||||
|
||||
model := p.embedder.ModelID()
|
||||
points := make([]domain.VectorPoint, 0, len(chunks))
|
||||
for i, c := range chunks {
|
||||
points = append(points, domain.VectorPoint{
|
||||
ChunkID: c.ID,
|
||||
WorkspaceID: c.WorkspaceID,
|
||||
ItemID: c.ItemID,
|
||||
SourceType: item.SourceType,
|
||||
CreatedAt: item.CreatedAt,
|
||||
EmbeddingModel: model,
|
||||
Vector: vecs[i],
|
||||
})
|
||||
}
|
||||
if err := p.vectors.Upsert(ctx, points); err != nil {
|
||||
return fmt.Errorf("store vectors: %w", err)
|
||||
}
|
||||
|
||||
chunkIDs := make([]string, len(chunks))
|
||||
for i, c := range chunks {
|
||||
chunkIDs[i] = c.ID
|
||||
}
|
||||
if err := p.repos.Chunks.MarkEmbedded(ctx, item.WorkspaceID, chunkIDs, model); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
item.EmbeddingModel = model
|
||||
item.Status = domain.StatusAnalysing
|
||||
if err := p.repos.Items.Update(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = p.repos.Models.Observe(ctx, item.WorkspaceID, "embedding", model, p.embedder.Dimensions())
|
||||
|
||||
log.Info("embedded item", "chunks", len(chunks), "model", model)
|
||||
return p.enqueueNext(ctx, j, domain.JobExtractEntity, 7)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage 4: entities
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// HandleExtractEntities runs deterministic and model-assisted entity extraction.
|
||||
func (p *Pipeline) HandleExtractEntities(ctx context.Context, j *domain.Job) error {
|
||||
item, err := p.loadItem(ctx, j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log := observability.LoggerFrom(ctx).With("item_id", item.ID)
|
||||
|
||||
if err := p.repos.Items.SetStatus(ctx, item.WorkspaceID, item.ID, domain.StatusAnalysing, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunks, err := p.repos.Chunks.ListByItem(ctx, item.WorkspaceID, item.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := p.entities.ExtractForItem(ctx, item, chunks, p.pool)
|
||||
if err != nil {
|
||||
// Model-assisted extraction is best-effort: the deterministic results
|
||||
// are already stored, so a model failure degrades quality rather than
|
||||
// stopping the pipeline.
|
||||
if errors.Is(err, analysis.ErrDisabled) || errors.Is(err, analysis.ErrInvalidOutput) {
|
||||
log.Warn("model-assisted entity extraction unavailable; deterministic results kept",
|
||||
"error", err.Error())
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
item.EntityCount = n
|
||||
item.AnalysisModel = p.analysis.ModelID()
|
||||
if err := p.repos.Items.Update(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
if m := p.analysis.ModelID(); m != "" {
|
||||
_ = p.repos.Models.Observe(ctx, item.WorkspaceID, "analysis", m, 0)
|
||||
}
|
||||
|
||||
log.Info("extracted entities", "count", n)
|
||||
return p.enqueueNext(ctx, j, domain.JobExtractClaims, 6)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage 5: claims
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// HandleExtractClaims extracts the statements the item makes.
|
||||
func (p *Pipeline) HandleExtractClaims(ctx context.Context, j *domain.Job) error {
|
||||
item, err := p.loadItem(ctx, j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log := observability.LoggerFrom(ctx).With("item_id", item.ID)
|
||||
|
||||
chunks, err := p.repos.Chunks.ListByItem(ctx, item.WorkspaceID, item.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := p.entities.ExtractClaimsForItem(ctx, item, chunks, p.pool)
|
||||
if err != nil {
|
||||
if errors.Is(err, analysis.ErrDisabled) || errors.Is(err, analysis.ErrInvalidOutput) {
|
||||
log.Warn("claim extraction unavailable", "error", err.Error())
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
item.ClaimCount = n
|
||||
item.Status = domain.StatusRelating
|
||||
if err := p.repos.Items.Update(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("extracted claims", "count", n)
|
||||
return p.enqueueNext(ctx, j, domain.JobDiscoverRels, 5)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage 6: relationships
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// HandleDiscoverRelationships finds and scores candidate relationships.
|
||||
func (p *Pipeline) HandleDiscoverRelationships(ctx context.Context, j *domain.Job) error {
|
||||
item, err := p.loadItem(ctx, j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log := observability.LoggerFrom(ctx).With("item_id", item.ID)
|
||||
|
||||
if err := p.repos.Items.SetStatus(ctx, item.WorkspaceID, item.ID, domain.StatusRelating, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stats, err := p.discover.DiscoverForItem(ctx, item, p.pool)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
item.RelationshipSeen = stats.Stored
|
||||
item.ScoreVersion = relationships.ScoreVersion
|
||||
if err := p.repos.Items.Update(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("discovered relationships",
|
||||
"candidates", stats.Candidates, "stored", stats.Stored, "model_calls", stats.ModelCalls)
|
||||
|
||||
p.finishRun(ctx, j.RunID, "succeeded", "", map[string]int{
|
||||
"chunks": item.ChunkCount,
|
||||
"entities": item.EntityCount,
|
||||
"claims": item.ClaimCount,
|
||||
"relationships": stats.Stored,
|
||||
"candidates": stats.Candidates,
|
||||
})
|
||||
return p.markReady(ctx, item, j)
|
||||
}
|
||||
|
||||
func (p *Pipeline) markReady(ctx context.Context, item *domain.Item, j *domain.Job) error {
|
||||
status := domain.StatusReady
|
||||
// An item that produced no searchable text is "partial": it exists and is
|
||||
// preserved, but it will not be found by content.
|
||||
if item.ChunkCount == 0 || item.HasWarning(domain.WarnLowExtractableText) {
|
||||
status = domain.StatusPartial
|
||||
}
|
||||
if err := p.repos.Items.SetStatus(ctx, item.WorkspaceID, item.ID, status, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
if j != nil && j.RunID != "" {
|
||||
p.finishRun(ctx, j.RunID, "succeeded", "", nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Pipeline) finishRun(ctx context.Context, runID, outcome, errMsg string, stats map[string]int) {
|
||||
if runID == "" {
|
||||
return
|
||||
}
|
||||
if err := p.repos.Runs.Finish(ctx, runID, outcome, errMsg, stats); err != nil {
|
||||
observability.LoggerFrom(ctx).Debug("could not close processing run", "error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func reasonOr(v, def string) string {
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,605 @@
|
||||
// Package ingest accepts information into Relata and drives it through the
|
||||
// processing pipeline.
|
||||
//
|
||||
// The governing rule is that the original is preserved. Whatever a user
|
||||
// supplies is stored verbatim — bytes for uploads, text for pastes — before any
|
||||
// parsing is attempted. If parsing later fails, the failure is recorded as a
|
||||
// warning on the item and the original remains retrievable and reprocessable.
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/andydixon/relata/internal/audit"
|
||||
"github.com/andydixon/relata/internal/config"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/ids"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
"github.com/andydixon/relata/internal/parsing"
|
||||
)
|
||||
|
||||
// Request is a single item to ingest.
|
||||
type Request struct {
|
||||
Title string
|
||||
Content []byte
|
||||
Filename string
|
||||
// DeclaredMIME is what the client claimed. It is a hint only; the real type
|
||||
// is determined from the content.
|
||||
DeclaredMIME string
|
||||
SourceType domain.SourceType
|
||||
Metadata map[string]string
|
||||
// CreatedAt is the user-asserted creation time of the information itself.
|
||||
CreatedAt *time.Time
|
||||
// AllowDuplicate must be set explicitly to store content that already
|
||||
// exists byte-for-byte.
|
||||
AllowDuplicate bool
|
||||
}
|
||||
|
||||
// Result describes what ingestion produced.
|
||||
type Result struct {
|
||||
Item *domain.Item
|
||||
// Duplicate is the existing item when identical content was found.
|
||||
Duplicate *domain.Item
|
||||
// Stored is false when an exact duplicate was refused.
|
||||
Stored bool
|
||||
JobID string
|
||||
}
|
||||
|
||||
// Service ingests items.
|
||||
type Service struct {
|
||||
items domain.ItemRepository
|
||||
jobs domain.JobRepository
|
||||
cfg *config.Config
|
||||
audit *audit.Recorder
|
||||
metrics *observability.Metrics
|
||||
// admit bounds concurrent ingestion so a burst of uploads cannot exhaust
|
||||
// memory: each in-flight request holds its content in memory.
|
||||
admit chan struct{}
|
||||
}
|
||||
|
||||
// NewService builds the ingestion service.
|
||||
func NewService(items domain.ItemRepository, jobRepo domain.JobRepository, cfg *config.Config,
|
||||
rec *audit.Recorder, m *observability.Metrics) *Service {
|
||||
n := cfg.Uploads.MaxConcurrentIngest
|
||||
if n <= 0 {
|
||||
n = 4
|
||||
}
|
||||
return &Service{
|
||||
items: items,
|
||||
jobs: jobRepo,
|
||||
cfg: cfg,
|
||||
audit: rec,
|
||||
metrics: m,
|
||||
admit: make(chan struct{}, n),
|
||||
}
|
||||
}
|
||||
|
||||
// ValidationError describes why an ingestion request was rejected.
|
||||
type ValidationError struct {
|
||||
Field string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *ValidationError) Error() string { return e.Field + ": " + e.Message }
|
||||
func (e *ValidationError) Unwrap() error { return domain.ErrInvalidInput }
|
||||
|
||||
// DuplicateError reports that identical content already exists.
|
||||
type DuplicateError struct {
|
||||
ExistingID string
|
||||
ExistingTitle string
|
||||
}
|
||||
|
||||
func (e *DuplicateError) Error() string {
|
||||
if e.ExistingID == "" {
|
||||
// A race we lost, where the winning item was no longer readable.
|
||||
return "identical content already exists"
|
||||
}
|
||||
return fmt.Sprintf("identical content already exists as item %s (%q)", e.ExistingID, e.ExistingTitle)
|
||||
}
|
||||
func (e *DuplicateError) Unwrap() error { return domain.ErrDuplicate }
|
||||
|
||||
// Ingest validates, stores and queues an item for processing.
|
||||
func (s *Service) Ingest(ctx context.Context, p *domain.Principal, req Request) (*Result, error) {
|
||||
select {
|
||||
case s.admit <- struct{}{}:
|
||||
defer func() { <-s.admit }()
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
if p == nil || p.WorkspaceID == "" {
|
||||
return nil, domain.ErrWorkspaceMiss
|
||||
}
|
||||
if !p.Role.CanWrite() {
|
||||
return nil, domain.ErrForbidden
|
||||
}
|
||||
|
||||
item, err := s.build(ctx, p, req)
|
||||
if err != nil {
|
||||
s.countIngest(string(req.SourceType), "rejected")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Exact duplicate detection, before anything is written.
|
||||
existing, err := s.items.FindByContentHash(ctx, p.WorkspaceID, item.ContentHash)
|
||||
if err != nil && !errors.Is(err, domain.ErrNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
if existing != nil {
|
||||
if !req.AllowDuplicate {
|
||||
s.countIngest(string(req.SourceType), "duplicate")
|
||||
return &Result{Duplicate: existing, Stored: false},
|
||||
&DuplicateError{ExistingID: existing.ID, ExistingTitle: existing.Title}
|
||||
}
|
||||
// The user asked for it explicitly; record the link rather than hide it.
|
||||
item.DuplicateOf = existing.ID
|
||||
}
|
||||
|
||||
// Persist the original bytes for binary formats before the item row, so a
|
||||
// row never claims a file that does not exist.
|
||||
if item.MIMEType == "application/pdf" || isDOCX(item.MIMEType) {
|
||||
path, err := s.storeOriginal(p.WorkspaceID, item.ID, req.Filename, req.Content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store original file: %w", err)
|
||||
}
|
||||
item.OriginalStoredPath = path
|
||||
}
|
||||
|
||||
// discard removes the stored original when the item will not be created.
|
||||
discard := func() {
|
||||
if item.OriginalStoredPath != "" {
|
||||
_ = os.Remove(item.OriginalStoredPath)
|
||||
item.OriginalStoredPath = ""
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.items.Create(ctx, item); err != nil {
|
||||
// A concurrent request stored identical content between the check above
|
||||
// and this insert: the unique index on content_hash caught what the
|
||||
// check could not.
|
||||
if !errors.Is(err, domain.ErrDuplicate) {
|
||||
discard()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
winner, lookupErr := s.items.FindByContentHash(ctx, p.WorkspaceID, item.ContentHash)
|
||||
if lookupErr != nil || winner == nil {
|
||||
// The winner is not readable — deleted in the interim, or the
|
||||
// lookup failed. Still a duplicate, just without a name to give.
|
||||
discard()
|
||||
s.countIngest(string(req.SourceType), "duplicate")
|
||||
return &Result{Stored: false}, &DuplicateError{}
|
||||
}
|
||||
|
||||
if req.AllowDuplicate {
|
||||
// The caller asked for a second copy to be kept. Losing the race
|
||||
// does not change that answer — it only reveals which item this one
|
||||
// is a copy of. Recording that link puts the item outside the
|
||||
// uniqueness constraint, so the retry succeeds.
|
||||
item.DuplicateOf = winner.ID
|
||||
if err := s.items.Create(ctx, item); err != nil {
|
||||
discard()
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
discard()
|
||||
s.countIngest(string(req.SourceType), "duplicate")
|
||||
return &Result{Duplicate: winner, Stored: false},
|
||||
&DuplicateError{ExistingID: winner.ID, ExistingTitle: winner.Title}
|
||||
}
|
||||
}
|
||||
|
||||
job := &domain.Job{
|
||||
ID: ids.New(),
|
||||
WorkspaceID: p.WorkspaceID,
|
||||
Type: domain.JobParseItem,
|
||||
Status: domain.JobPending,
|
||||
ItemID: item.ID,
|
||||
Priority: 10,
|
||||
MaxAttempts: s.cfg.Workers.MaxAttempts,
|
||||
IdempotencyKey: ids.Deterministic(p.WorkspaceID, item.ID, string(domain.JobParseItem), "initial"),
|
||||
}
|
||||
if err := s.jobs.Enqueue(ctx, job); err != nil {
|
||||
// The item exists and is recoverable; surface the queueing failure
|
||||
// rather than pretending processing has started.
|
||||
return &Result{Item: item, Stored: true}, fmt.Errorf("queue item for processing: %w", err)
|
||||
}
|
||||
|
||||
s.audit.Record(ctx, p, domain.AuditItemCreated, "item", item.ID, "success", map[string]string{
|
||||
"source_type": string(item.SourceType),
|
||||
"mime_type": item.MIMEType,
|
||||
"size_bytes": fmt.Sprint(item.SizeBytes),
|
||||
"duplicate": boolStr(item.DuplicateOf != ""),
|
||||
})
|
||||
s.countIngest(string(item.SourceType), "accepted")
|
||||
|
||||
return &Result{Item: item, Stored: true, JobID: job.ID, Duplicate: existing}, nil
|
||||
}
|
||||
|
||||
func (s *Service) countIngest(sourceType, outcome string) {
|
||||
if s.metrics != nil {
|
||||
if sourceType == "" {
|
||||
sourceType = "unknown"
|
||||
}
|
||||
s.metrics.IngestItems.WithLabelValues(sourceType, outcome).Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// build validates a request and constructs the item record.
|
||||
func (s *Service) build(ctx context.Context, p *domain.Principal, req Request) (*domain.Item, error) {
|
||||
lim := s.cfg.Uploads
|
||||
|
||||
if len(req.Content) == 0 {
|
||||
return nil, &ValidationError{Field: "content", Message: "no content was supplied"}
|
||||
}
|
||||
if int64(len(req.Content)) > lim.MaxUploadBytes {
|
||||
return nil, &ValidationError{
|
||||
Field: "content",
|
||||
Message: fmt.Sprintf("content is %d bytes, larger than the %d byte limit", len(req.Content), lim.MaxUploadBytes),
|
||||
}
|
||||
}
|
||||
|
||||
// Determine the real type from the bytes. A client's declared type and the
|
||||
// filename are hints; neither is trusted.
|
||||
sniff, err := parsing.Sniff(req.Content, req.DeclaredMIME, req.Filename)
|
||||
switch {
|
||||
case errors.Is(err, parsing.ErrArchiveRejected):
|
||||
return nil, &ValidationError{
|
||||
Field: "content",
|
||||
Message: "archive files are not accepted. Please upload the documents individually.",
|
||||
}
|
||||
case errors.Is(err, parsing.ErrExecutableRejected):
|
||||
return nil, &ValidationError{
|
||||
Field: "content",
|
||||
Message: "executable content is not accepted.",
|
||||
}
|
||||
case err != nil:
|
||||
return nil, &ValidationError{
|
||||
Field: "content",
|
||||
Message: fmt.Sprintf("this file type is not supported. Relata accepts plain text, Markdown, JSON, PDF and DOCX (detected: %s)",
|
||||
sniff.MIME),
|
||||
}
|
||||
}
|
||||
if !parsing.Supported(sniff.MIME) {
|
||||
return nil, &ValidationError{
|
||||
Field: "content",
|
||||
Message: "this file type is not supported. Relata accepts plain text, Markdown, JSON, PDF and DOCX.",
|
||||
}
|
||||
}
|
||||
|
||||
title, err := s.cleanTitle(req.Title, req.Filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta, err := s.cleanMetadata(req.Metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sourceType := req.SourceType
|
||||
if sourceType == "" {
|
||||
sourceType = inferSourceType(sniff.MIME)
|
||||
}
|
||||
if !sourceType.Valid() {
|
||||
return nil, &ValidationError{Field: "source_type", Message: "unrecognised source type"}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
created := now
|
||||
if req.CreatedAt != nil {
|
||||
created = req.CreatedAt.UTC()
|
||||
// A creation date far in the future is almost certainly a mistake and
|
||||
// would distort every temporal signal computed from it.
|
||||
if created.After(now.Add(24 * time.Hour)) {
|
||||
return nil, &ValidationError{Field: "created_at", Message: "the creation date cannot be in the future"}
|
||||
}
|
||||
}
|
||||
|
||||
sum := sha256.Sum256(req.Content)
|
||||
item := &domain.Item{
|
||||
ID: ids.New(),
|
||||
WorkspaceID: p.WorkspaceID,
|
||||
Title: title,
|
||||
SourceType: sourceType,
|
||||
MIMEType: sniff.MIME,
|
||||
SizeBytes: int64(len(req.Content)),
|
||||
ContentHash: hex.EncodeToString(sum[:]),
|
||||
Metadata: meta,
|
||||
CreatedAt: created,
|
||||
IngestedAt: now,
|
||||
UpdatedAt: now,
|
||||
Status: domain.StatusPending,
|
||||
CreatedBy: p.UserID,
|
||||
}
|
||||
|
||||
if fn := SanitiseFilename(req.Filename); fn != "" {
|
||||
item.OriginalFilename = fn
|
||||
}
|
||||
|
||||
// Text-shaped content is kept inline; binary content is kept on disk.
|
||||
if isTextual(sniff.MIME) {
|
||||
if !utf8.Valid(req.Content) {
|
||||
return nil, &ValidationError{Field: "content", Message: "text content is not valid UTF-8"}
|
||||
}
|
||||
item.OriginalText = string(req.Content)
|
||||
}
|
||||
|
||||
if sniff.Declared != "" && sniff.FromContent && sniff.Declared != sniff.MIME {
|
||||
// Worth recording: it may be innocent, or it may be an attempt to
|
||||
// smuggle one format past a check for another.
|
||||
item.Warnings = append(item.Warnings, domain.ParseWarning{
|
||||
Code: "mime_mismatch",
|
||||
Message: "The file's actual type differs from the type the browser reported. Relata used the type detected from the file's contents.",
|
||||
Detail: fmt.Sprintf("declared=%s detected=%s", sniff.Declared, sniff.MIME),
|
||||
})
|
||||
}
|
||||
|
||||
observability.LoggerFrom(ctx).Info("item accepted for ingestion",
|
||||
"item_id", item.ID, "mime_type", item.MIMEType, "size_bytes", item.SizeBytes,
|
||||
"source_type", string(item.SourceType))
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *Service) cleanTitle(title, filename string) (string, error) {
|
||||
t := strings.TrimSpace(title)
|
||||
if t == "" {
|
||||
t = strings.TrimSpace(SanitiseFilename(filename))
|
||||
}
|
||||
if t == "" {
|
||||
t = "Untitled item"
|
||||
}
|
||||
t = stripControl(t)
|
||||
if utf8.RuneCountInString(t) > s.cfg.Uploads.MaxTitleLen {
|
||||
return "", &ValidationError{
|
||||
Field: "title",
|
||||
Message: fmt.Sprintf("title must be at most %d characters", s.cfg.Uploads.MaxTitleLen),
|
||||
}
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// metadataKeyPattern restricts metadata keys to a safe, predictable shape.
|
||||
//
|
||||
// This is not cosmetic: keys become part of stored documents and are displayed
|
||||
// and compared. Restricting them keeps a key from resembling a MongoDB operator
|
||||
// or carrying markup.
|
||||
var metadataKeyPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.\-]{0,63}$`)
|
||||
|
||||
func (s *Service) cleanMetadata(in map[string]string) (map[string]string, error) {
|
||||
if len(in) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
lim := s.cfg.Uploads
|
||||
if len(in) > lim.MaxMetadataKeys {
|
||||
return nil, &ValidationError{
|
||||
Field: "metadata",
|
||||
Message: fmt.Sprintf("at most %d metadata entries are allowed", lim.MaxMetadataKeys),
|
||||
}
|
||||
}
|
||||
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
key := strings.TrimSpace(k)
|
||||
if !metadataKeyPattern.MatchString(key) {
|
||||
return nil, &ValidationError{
|
||||
Field: "metadata",
|
||||
Message: fmt.Sprintf("metadata key %q is invalid; use letters, digits, dot, dash or underscore", truncate(key, 40)),
|
||||
}
|
||||
}
|
||||
// A key beginning with $ or containing a dot has special meaning to
|
||||
// MongoDB. The pattern above already excludes a leading $; dots are
|
||||
// permitted in the key text but the value is always stored as a plain
|
||||
// string inside a map, never interpreted as a path.
|
||||
val := stripControl(v)
|
||||
if len(val) > lim.MaxMetadataValueLen {
|
||||
return nil, &ValidationError{
|
||||
Field: "metadata",
|
||||
Message: fmt.Sprintf("metadata value for %q exceeds %d characters", key, lim.MaxMetadataValueLen),
|
||||
}
|
||||
}
|
||||
out[key] = val
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// storeOriginal writes uploaded bytes to disk.
|
||||
//
|
||||
// The stored name is derived from the item's own UUID, never from the
|
||||
// user-supplied filename, so a crafted name cannot influence the path at all.
|
||||
// Files are written without any execute permission and the directory layout is
|
||||
// per-workspace.
|
||||
func (s *Service) storeOriginal(workspaceID, itemID, filename string, content []byte) (string, error) {
|
||||
if !ids.Valid(itemID) || !ids.Valid(workspaceID) {
|
||||
return "", fmt.Errorf("refusing to store a file under a non-UUID identifier")
|
||||
}
|
||||
|
||||
dir := filepath.Join(s.cfg.Uploads.Dir, workspaceID)
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// The extension is taken from a fixed allow-list keyed on the *detected*
|
||||
// type, not from the filename, so ".php" or ".sh" can never appear.
|
||||
ext := ""
|
||||
switch {
|
||||
case strings.HasSuffix(strings.ToLower(SanitiseFilename(filename)), ".pdf"):
|
||||
ext = ".pdf"
|
||||
case strings.HasSuffix(strings.ToLower(SanitiseFilename(filename)), ".docx"):
|
||||
ext = ".docx"
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, itemID+ext)
|
||||
|
||||
// Belt and braces: confirm the final path is inside the upload directory.
|
||||
absDir, err := filepath.Abs(s.cfg.Uploads.Dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !strings.HasPrefix(absPath, absDir+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("refusing to write outside the upload directory")
|
||||
}
|
||||
|
||||
// 0600: readable and writable by the service account only, never executable.
|
||||
if err := os.WriteFile(absPath, content, 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return absPath, nil
|
||||
}
|
||||
|
||||
// ReadOriginal returns the stored bytes of an uploaded file.
|
||||
func (s *Service) ReadOriginal(item *domain.Item) ([]byte, error) {
|
||||
if item.OriginalStoredPath == "" {
|
||||
if item.OriginalText != "" {
|
||||
return []byte(item.OriginalText), nil
|
||||
}
|
||||
return nil, domain.ErrNotFound
|
||||
}
|
||||
absDir, err := filepath.Abs(s.cfg.Uploads.Dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Re-check containment on read: the stored path came from the database,
|
||||
// which is trusted, but a cheap check costs nothing and bounds the damage
|
||||
// from a corrupted record.
|
||||
if !strings.HasPrefix(item.OriginalStoredPath, absDir+string(os.PathSeparator)) {
|
||||
return nil, fmt.Errorf("stored file path is outside the upload directory")
|
||||
}
|
||||
return os.ReadFile(item.OriginalStoredPath)
|
||||
}
|
||||
|
||||
// DeleteOriginal removes an item's stored file.
|
||||
func (s *Service) DeleteOriginal(item *domain.Item) error {
|
||||
if item.OriginalStoredPath == "" {
|
||||
return nil
|
||||
}
|
||||
absDir, err := filepath.Abs(s.cfg.Uploads.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.HasPrefix(item.OriginalStoredPath, absDir+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("refusing to delete outside the upload directory")
|
||||
}
|
||||
if err := os.Remove(item.OriginalStoredPath); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SanitiseFilename reduces a user-supplied filename to a safe display name.
|
||||
//
|
||||
// Only the base name survives, so "../../etc/passwd" becomes "passwd". Path
|
||||
// separators, control characters, leading dots and Windows reserved names are
|
||||
// all removed. The result is used for display and for choosing an extension —
|
||||
// never as a path component on its own.
|
||||
func SanitiseFilename(name string) string {
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
// Handle both separators: a Windows client may send backslashes.
|
||||
name = strings.ReplaceAll(name, "\\", "/")
|
||||
name = filepath.Base(name)
|
||||
|
||||
// filepath.Base returns "." or "/" for degenerate input.
|
||||
if name == "." || name == ".." || name == "/" {
|
||||
return ""
|
||||
}
|
||||
|
||||
name = strings.Map(func(r rune) rune {
|
||||
switch {
|
||||
case r < 0x20 || r == 0x7f:
|
||||
return -1
|
||||
case r == '/' || r == '\\' || r == ':' || r == '*' || r == '?' ||
|
||||
r == '"' || r == '<' || r == '>' || r == '|' || r == 0:
|
||||
return '_'
|
||||
}
|
||||
return r
|
||||
}, name)
|
||||
|
||||
// A leading dot would create a hidden file and can hide the real extension.
|
||||
name = strings.TrimLeft(name, ". ")
|
||||
name = strings.TrimRight(name, ". ")
|
||||
|
||||
// Windows reserved device names, which can cause surprising behaviour if
|
||||
// the file is ever copied to such a system.
|
||||
base := name
|
||||
if i := strings.IndexByte(name, '.'); i > 0 {
|
||||
base = name[:i]
|
||||
}
|
||||
switch strings.ToUpper(base) {
|
||||
case "CON", "PRN", "AUX", "NUL",
|
||||
"COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9",
|
||||
"LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9":
|
||||
name = "file_" + name
|
||||
}
|
||||
|
||||
if len(name) > 200 {
|
||||
name = name[:200]
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func inferSourceType(mime string) domain.SourceType {
|
||||
switch mime {
|
||||
case "application/json":
|
||||
return domain.SourceTypeJSON
|
||||
case "text/markdown", "text/x-markdown":
|
||||
return domain.SourceTypeMarkdown
|
||||
case "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
|
||||
return domain.SourceTypeDocument
|
||||
default:
|
||||
return domain.SourceTypeNote
|
||||
}
|
||||
}
|
||||
|
||||
func isTextual(mime string) bool {
|
||||
switch mime {
|
||||
case "text/plain", "text/markdown", "text/x-markdown", "application/json", "text/json", "text/csv":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isDOCX(mime string) bool {
|
||||
return mime == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||
}
|
||||
|
||||
func stripControl(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r < 0x20 && r != '\n' && r != '\t' || r == 0x7f {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, strings.TrimSpace(s))
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
for n > 0 && !utf8.RuneStart(s[n]) {
|
||||
n--
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
|
||||
func boolStr(b bool) string {
|
||||
if b {
|
||||
return "true"
|
||||
}
|
||||
return "false"
|
||||
}
|
||||
Reference in New Issue
Block a user