fix: harden credential and input handling
This commit is contained in:
Andy Dixon
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,680 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/andydixon/relata/internal/config"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
)
|
||||
|
||||
// --- JSON decoding ---------------------------------------------------------
|
||||
|
||||
type testBody struct {
|
||||
Title string `json:"title"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
func postJSON(body string) (*httptest.ResponseRecorder, *http.Request) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/api/v1/items", strings.NewReader(body))
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
return httptest.NewRecorder(), r
|
||||
}
|
||||
|
||||
func TestDecodeJSONRejectsUnknownFields(t *testing.T) {
|
||||
w, r := postJSON(`{"title":"a","count":1,"is_admin":true}`)
|
||||
|
||||
var dst testBody
|
||||
err := DecodeJSON(w, r, 1<<20, &dst)
|
||||
if err == nil {
|
||||
t.Fatal("an unknown field was silently ignored")
|
||||
}
|
||||
var apiErr *APIError
|
||||
if !asAPIError(err, &apiErr) {
|
||||
t.Fatalf("wrong error type: %T", err)
|
||||
}
|
||||
if apiErr.Status != http.StatusBadRequest || apiErr.Code != CodeValidationFailed {
|
||||
t.Fatalf("got %d/%s, want 400/%s", apiErr.Status, apiErr.Code, CodeValidationFailed)
|
||||
}
|
||||
if len(apiErr.Fields) == 0 || apiErr.Fields[0].Field != "is_admin" {
|
||||
t.Fatalf("the offending field was not identified: %+v", apiErr.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeJSONRejectsMalformedInput(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"truncated": `{"title":"a"`,
|
||||
"not json": `title=a`,
|
||||
"array": `[1,2,3]`,
|
||||
"empty": ``,
|
||||
"two documents": `{"title":"a"}{"title":"b"}`,
|
||||
}
|
||||
for name, body := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
w, r := postJSON(body)
|
||||
var dst testBody
|
||||
if err := DecodeJSON(w, r, 1<<20, &dst); err == nil {
|
||||
t.Fatalf("malformed body %q was accepted", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeJSONEnforcesSizeLimit(t *testing.T) {
|
||||
big := `{"title":"` + strings.Repeat("x", 5000) + `"}`
|
||||
w, r := postJSON(big)
|
||||
|
||||
var dst testBody
|
||||
err := DecodeJSON(w, r, 1024, &dst)
|
||||
var apiErr *APIError
|
||||
if !asAPIError(err, &apiErr) || apiErr.Status != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("oversized body not rejected with 413: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeJSONRequiresJSONContentType(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/api/v1/items", strings.NewReader(`{"title":"a"}`))
|
||||
r.Header.Set("Content-Type", "text/html")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
var dst testBody
|
||||
err := DecodeJSON(w, r, 1<<20, &dst)
|
||||
var apiErr *APIError
|
||||
if !asAPIError(err, &apiErr) || apiErr.Status != http.StatusUnsupportedMediaType {
|
||||
t.Fatalf("non-JSON content type accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func asAPIError(err error, target **APIError) bool {
|
||||
for err != nil {
|
||||
if e, ok := err.(*APIError); ok {
|
||||
*target = e
|
||||
return true
|
||||
}
|
||||
u, ok := err.(interface{ Unwrap() error })
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
err = u.Unwrap()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- Error responses -------------------------------------------------------
|
||||
|
||||
func TestErrorResponsesNeverLeakInternalDetail(t *testing.T) {
|
||||
// An internal error must not surface the underlying message, which could
|
||||
// contain a connection string, a file path or schema detail.
|
||||
internal := Errorf(http.StatusInternalServerError, CodeInternal, "An unexpected error occurred.")
|
||||
internal.Internal = errNamed("mongodb://user:hunter2@db:27017 connection refused: /var/lib/secret")
|
||||
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/items", nil)
|
||||
w := httptest.NewRecorder()
|
||||
WriteError(w, r, internal)
|
||||
|
||||
body := w.Body.String()
|
||||
for _, forbidden := range []string{"hunter2", "mongodb://", "/var/lib/secret", "goroutine"} {
|
||||
if strings.Contains(body, forbidden) {
|
||||
t.Fatalf("the response leaked internal detail %q:\n%s", forbidden, body)
|
||||
}
|
||||
}
|
||||
if w.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("status = %d, want 500", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorResponsesUseStableCodes(t *testing.T) {
|
||||
cases := []struct {
|
||||
err error
|
||||
wantStatus int
|
||||
wantCode string
|
||||
}{
|
||||
{domain.ErrNotFound, http.StatusNotFound, CodeNotFound},
|
||||
{domain.ErrForbidden, http.StatusForbidden, CodeForbidden},
|
||||
{domain.ErrDuplicate, http.StatusConflict, CodeDuplicate},
|
||||
{domain.ErrTooLarge, http.StatusRequestEntityTooLarge, CodePayloadTooLarge},
|
||||
{domain.ErrUnsupported, http.StatusUnsupportedMediaType, CodeUnsupportedMedia},
|
||||
// A cross-workspace reference must look identical to a missing record,
|
||||
// so that probing cannot confirm an ID exists elsewhere.
|
||||
{domain.ErrWorkspaceMiss, http.StatusNotFound, CodeNotFound},
|
||||
}
|
||||
for _, c := range cases {
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/items/x", nil)
|
||||
w := httptest.NewRecorder()
|
||||
WriteError(w, r, c.err)
|
||||
|
||||
if w.Code != c.wantStatus {
|
||||
t.Errorf("%v → status %d, want %d", c.err, w.Code, c.wantStatus)
|
||||
}
|
||||
var resp ErrorResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("response is not valid JSON: %v", err)
|
||||
}
|
||||
if resp.Error.Code != c.wantCode {
|
||||
t.Errorf("%v → code %q, want %q", c.err, resp.Error.Code, c.wantCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceMismatchIsIndistinguishableFromNotFound(t *testing.T) {
|
||||
render := func(err error) string {
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/items/x", nil)
|
||||
w := httptest.NewRecorder()
|
||||
WriteError(w, r, err)
|
||||
return w.Body.String()
|
||||
}
|
||||
if render(domain.ErrNotFound) != render(domain.ErrWorkspaceMiss) {
|
||||
t.Fatal("a cross-workspace access is distinguishable from a missing record, " +
|
||||
"which lets an attacker confirm that an ID exists in another workspace")
|
||||
}
|
||||
}
|
||||
|
||||
type namedErr string
|
||||
|
||||
func (e namedErr) Error() string { return string(e) }
|
||||
func errNamed(s string) error { return namedErr(s) }
|
||||
|
||||
// --- Security headers ------------------------------------------------------
|
||||
|
||||
func TestSecurityHeadersAreRestrictive(t *testing.T) {
|
||||
h := SecurityHeaders(false)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
csp := w.Header().Get("Content-Security-Policy")
|
||||
for _, want := range []string{
|
||||
"default-src 'none'",
|
||||
"script-src 'self'",
|
||||
"frame-ancestors 'none'",
|
||||
"object-src 'none'",
|
||||
"base-uri 'none'",
|
||||
"form-action 'self'",
|
||||
} {
|
||||
if !strings.Contains(csp, want) {
|
||||
t.Errorf("CSP is missing %q: %s", want, csp)
|
||||
}
|
||||
}
|
||||
// Inline script and remote origins must not be permitted: they are how a
|
||||
// stored payload would execute or exfiltrate.
|
||||
for _, forbidden := range []string{"unsafe-inline", "unsafe-eval", "*", "http:"} {
|
||||
if strings.Contains(csp, forbidden) {
|
||||
t.Errorf("CSP permits %q: %s", forbidden, csp)
|
||||
}
|
||||
}
|
||||
|
||||
checks := map[string]string{
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"X-Frame-Options": "DENY",
|
||||
"Referrer-Policy": "same-origin",
|
||||
"Cache-Control": "no-store",
|
||||
}
|
||||
for header, want := range checks {
|
||||
if got := w.Header().Get(header); got != want {
|
||||
t.Errorf("%s = %q, want %q", header, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Request IDs -----------------------------------------------------------
|
||||
|
||||
func TestRequestIDRejectsClientSuppliedGarbage(t *testing.T) {
|
||||
h := RequestID()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
|
||||
// A crafted value must not end up in log fields or the response header.
|
||||
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
r.Header.Set("X-Request-ID", "not-a-uuid\ninjected=evil")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
got := w.Header().Get("X-Request-ID")
|
||||
if strings.Contains(got, "injected") || strings.Contains(got, "\n") {
|
||||
t.Fatalf("a crafted request ID was echoed back: %q", got)
|
||||
}
|
||||
if len(got) != 36 {
|
||||
t.Fatalf("no valid request ID was generated: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestIDAcceptsAValidUUID(t *testing.T) {
|
||||
const valid = "3f2504e0-4f89-41d3-9a0c-0305e82c3301"
|
||||
h := RequestID()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
|
||||
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
r.Header.Set("X-Request-ID", valid)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if got := w.Header().Get("X-Request-ID"); got != valid {
|
||||
t.Fatalf("a valid client request ID was discarded: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Body limits -----------------------------------------------------------
|
||||
|
||||
func TestMaxBodyRejectsOversizedRequests(t *testing.T) {
|
||||
h := MaxBody(100)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(strings.Repeat("x", 500)))
|
||||
r.ContentLength = 500
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("status = %d, want 413", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// --- CSRF ------------------------------------------------------------------
|
||||
|
||||
func csrfHandler(session *domain.Session) http.Handler {
|
||||
c := CSRF{TrustedOrigins: []string{"http://127.0.0.1:8080"}}
|
||||
inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if session != nil {
|
||||
r = r.WithContext(withSession(r.Context(), session))
|
||||
}
|
||||
c.Middleware()(inner).ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCSRFAllowsSafeMethods(t *testing.T) {
|
||||
sess := &domain.Session{CSRFToken: "the-token"}
|
||||
h := csrfHandler(sess)
|
||||
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead, http.MethodOptions} {
|
||||
r := httptest.NewRequest(method, "/items", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Errorf("%s was refused: %d", method, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSRFBlocksStateChangeWithoutToken(t *testing.T) {
|
||||
sess := &domain.Session{CSRFToken: "the-token"}
|
||||
h := csrfHandler(sess)
|
||||
|
||||
r := httptest.NewRequest(http.MethodPost, "/items", nil)
|
||||
r.Header.Set("Origin", "http://127.0.0.1:8080")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("a POST without a CSRF token returned %d, want 403", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSRFBlocksWrongToken(t *testing.T) {
|
||||
sess := &domain.Session{CSRFToken: "the-token"}
|
||||
h := csrfHandler(sess)
|
||||
|
||||
r := httptest.NewRequest(http.MethodPost, "/items", nil)
|
||||
r.Header.Set("Origin", "http://127.0.0.1:8080")
|
||||
r.Header.Set("X-CSRF-Token", "the-wrong-token")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("a POST with the wrong CSRF token returned %d, want 403", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSRFBlocksForeignOrigin(t *testing.T) {
|
||||
sess := &domain.Session{CSRFToken: "the-token"}
|
||||
h := csrfHandler(sess)
|
||||
|
||||
// Even with the correct token, a request claiming another origin is refused.
|
||||
r := httptest.NewRequest(http.MethodPost, "/items", nil)
|
||||
r.Header.Set("Origin", "https://evil.example.com")
|
||||
r.Header.Set("X-CSRF-Token", "the-token")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("a cross-origin POST returned %d, want 403", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSRFAcceptsCorrectTokenAndOrigin(t *testing.T) {
|
||||
sess := &domain.Session{CSRFToken: "the-token"}
|
||||
h := csrfHandler(sess)
|
||||
|
||||
r := httptest.NewRequest(http.MethodPost, "/items", nil)
|
||||
r.Header.Set("Origin", "http://127.0.0.1:8080")
|
||||
r.Header.Set("X-CSRF-Token", "the-token")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("a valid request returned %d, want 200", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSRFRequiresASession(t *testing.T) {
|
||||
h := csrfHandler(nil)
|
||||
|
||||
r := httptest.NewRequest(http.MethodPost, "/items", nil)
|
||||
r.Header.Set("Origin", "http://127.0.0.1:8080")
|
||||
r.Header.Set("X-CSRF-Token", "anything")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("an unauthenticated POST returned %d, want 401", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSRFDerivesOriginFromReferer(t *testing.T) {
|
||||
sess := &domain.Session{CSRFToken: "the-token"}
|
||||
h := csrfHandler(sess)
|
||||
|
||||
r := httptest.NewRequest(http.MethodPost, "/items", nil)
|
||||
r.Header.Set("Referer", "https://evil.example.com/some/page?x=1")
|
||||
r.Header.Set("X-CSRF-Token", "the-token")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("a foreign Referer returned %d, want 403", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Authorisation ---------------------------------------------------------
|
||||
|
||||
func TestRequireWriteRejectsViewers(t *testing.T) {
|
||||
h := RequireWrite()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
cases := map[domain.Role]int{
|
||||
domain.RoleViewer: http.StatusForbidden,
|
||||
domain.RoleEditor: http.StatusOK,
|
||||
domain.RoleAdmin: http.StatusOK,
|
||||
}
|
||||
for role, want := range cases {
|
||||
r := httptest.NewRequest(http.MethodPost, "/items", nil)
|
||||
r = r.WithContext(WithPrincipal(r.Context(), &domain.Principal{
|
||||
UserID: "u", WorkspaceID: "ws", Role: role,
|
||||
}))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != want {
|
||||
t.Errorf("role %s got %d, want %d", role, w.Code, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireWriteRejectsAnonymous(t *testing.T) {
|
||||
h := RequireWrite()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
r := httptest.NewRequest(http.MethodPost, "/items", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("an anonymous write returned %d, want 403", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Panic recovery --------------------------------------------------------
|
||||
|
||||
func TestRecoverConvertsPanicToCleanError(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
logger := slog.New(slog.NewTextHandler(&logs, nil))
|
||||
h := Chain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
panic("a secret value hunter2 appeared in the panic")
|
||||
}), RequestID(), Recover())
|
||||
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/items", nil)
|
||||
r = r.WithContext(observability.WithLogger(r.Context(), logger))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("status = %d, want 500", w.Code)
|
||||
}
|
||||
if strings.Contains(w.Body.String(), "hunter2") {
|
||||
t.Fatalf("the panic value reached the client:\n%s", w.Body.String())
|
||||
}
|
||||
if strings.Contains(logs.String(), "hunter2") {
|
||||
t.Fatalf("the panic value reached the logs:\n%s", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchQueryRejectsNegativeOffsetAsZero(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/search?q=test&offset=-7", nil)
|
||||
q, err := searchQueryFromRequest(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q.Offset != 0 {
|
||||
t.Fatalf("offset = %d, want 0; a negative value can panic slice paging", q.Offset)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Pagination ------------------------------------------------------------
|
||||
|
||||
func TestPaginationIsBounded(t *testing.T) {
|
||||
cases := []struct {
|
||||
query string
|
||||
wantLimit int
|
||||
wantOffsetAtLst int
|
||||
}{
|
||||
{"", 20, 0},
|
||||
{"?limit=5&offset=10", 5, 10},
|
||||
{"?limit=99999", 200, 0}, // clamped to the maximum
|
||||
{"?limit=-1", 20, 0}, // falls back to the default
|
||||
{"?offset=-5", 20, 0}, // negative offsets are clamped
|
||||
{"?limit=abc", 20, 0}, // unparseable values fall back
|
||||
}
|
||||
for _, c := range cases {
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/items"+c.query, nil)
|
||||
p := pageFromQuery(r)
|
||||
if p.Limit != c.wantLimit {
|
||||
t.Errorf("%q → limit %d, want %d", c.query, p.Limit, c.wantLimit)
|
||||
}
|
||||
if p.Offset != c.wantOffsetAtLst {
|
||||
t.Errorf("%q → offset %d, want %d", c.query, p.Offset, c.wantOffsetAtLst)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Query filter validation ----------------------------------------------
|
||||
|
||||
// A filter value that MongoDB would treat as an operator must be rejected as
|
||||
// an invalid enum long before it reaches a query.
|
||||
func TestFiltersRejectOperatorShapedValues(t *testing.T) {
|
||||
hostile := []string{
|
||||
`$ne`, `{"$ne":null}`, `$where`, `$gt`, `[$ne]`, `{"$regex":".*"}`,
|
||||
}
|
||||
for _, v := range hostile {
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/items?source_type="+v, nil)
|
||||
if _, err := itemFilterFromQuery(r); err == nil {
|
||||
t.Errorf("source_type=%q was accepted", v)
|
||||
}
|
||||
|
||||
r = httptest.NewRequest(http.MethodGet, "/api/v1/items?type="+v, nil)
|
||||
if _, err := relFilterFromQuery(r); err == nil {
|
||||
t.Errorf("relationship type=%q was accepted", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFiltersAcceptValidValues(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodGet,
|
||||
"/api/v1/items?source_type=note,document&status=ready&created_from=2025-01-01", nil)
|
||||
f, err := itemFilterFromQuery(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.SourceTypes) != 2 {
|
||||
t.Fatalf("expected 2 source types, got %d", len(f.SourceTypes))
|
||||
}
|
||||
if f.CreatedFrom == nil || f.CreatedFrom.Year() != 2025 {
|
||||
t.Fatal("created_from was not parsed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentHashFilterRejectsNonHex(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/items?content_hash=../../etc/passwd", nil)
|
||||
if _, err := itemFilterFromQuery(r); err == nil {
|
||||
t.Fatal("a non-hexadecimal content hash was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEntityIDFiltersMustBeUUIDs(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodGet,
|
||||
"/api/v1/search?entity_id=not-a-uuid,3f2504e0-4f89-41d3-9a0c-0305e82c3301", nil)
|
||||
q, err := searchQueryFromRequest(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.EntityIDs) != 1 {
|
||||
t.Fatalf("expected only the valid UUID to survive, got %v", q.EntityIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchQueryLengthIsBounded(t *testing.T) {
|
||||
long := strings.Repeat("a", 3000)
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/search?q="+long, nil)
|
||||
if _, err := searchQueryFromRequest(r); err == nil {
|
||||
t.Fatal("an unbounded query string was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Trusted origins -------------------------------------------------------
|
||||
|
||||
func TestTrustedOriginsIncludeBaseURLAndListenAddress(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
cfg.HTTP.BaseURL = "https://relata.example.org"
|
||||
cfg.HTTP.Addr = "127.0.0.1:8080"
|
||||
|
||||
origins := trustedOrigins(cfg)
|
||||
want := []string{"https://relata.example.org", "http://127.0.0.1:8080", "http://localhost:8080"}
|
||||
for _, w := range want {
|
||||
if !containsStr(origins, w) {
|
||||
t.Errorf("origin %q is missing from %v", w, origins)
|
||||
}
|
||||
}
|
||||
// Nothing else may be trusted.
|
||||
for _, o := range origins {
|
||||
if strings.Contains(o, "evil") || o == "*" {
|
||||
t.Errorf("unexpected trusted origin %q", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A provider that ran out of time must be reported as a dependency timeout.
|
||||
// It previously surfaced as an opaque internal error, leaving a user with no
|
||||
// indication that the fix is a longer timeout or a smaller model.
|
||||
func TestProviderTimeoutIsReportedAsAGatewayTimeout(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/api/v1/search/answer", nil)
|
||||
w := httptest.NewRecorder()
|
||||
WriteError(w, r, fmt.Errorf("call the model: %w", context.DeadlineExceeded))
|
||||
|
||||
if w.Code != http.StatusGatewayTimeout {
|
||||
t.Fatalf("status = %d, want 504", w.Code)
|
||||
}
|
||||
var resp ErrorResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.Error.Code != CodeDependencyTimeout {
|
||||
t.Fatalf("code = %q, want %q", resp.Error.Code, CodeDependencyTimeout)
|
||||
}
|
||||
// The message must tell the user what to do about it.
|
||||
if !strings.Contains(resp.Error.Message, "RELATA_ANALYSIS_TIMEOUT") {
|
||||
t.Fatalf("the message gives no remedy: %q", resp.Error.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelledRequestIsNotAnInternalError(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/search", nil)
|
||||
w := httptest.NewRecorder()
|
||||
WriteError(w, r, fmt.Errorf("search: %w", context.Canceled))
|
||||
|
||||
if w.Code == http.StatusInternalServerError {
|
||||
t.Fatal("a client disconnect was recorded as an internal error")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Fuzzing ---------------------------------------------------------------
|
||||
|
||||
// Request decoding is the outermost trust boundary for the JSON API.
|
||||
func FuzzDecodeJSON(f *testing.F) {
|
||||
f.Add(`{"title":"a","count":1}`)
|
||||
f.Add(`{"title":"a","unknown":true}`)
|
||||
f.Add(`{"a":1}{"b":2}`)
|
||||
f.Add("\x00\xff")
|
||||
f.Add(`{"title":` + strings.Repeat("[", 200))
|
||||
|
||||
f.Fuzz(func(t *testing.T, body string) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/api/v1/items", strings.NewReader(body))
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
var dst testBody
|
||||
err := DecodeJSON(w, r, 1<<16, &dst)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
// Every rejection must be a typed API error with a client-safe status,
|
||||
// never a bare error that would become an opaque 500.
|
||||
var apiErr *APIError
|
||||
if !asAPIError(err, &apiErr) {
|
||||
t.Fatalf("decoding %q produced an untyped error: %v", body, err)
|
||||
}
|
||||
if apiErr.Status < 400 || apiErr.Status >= 500 {
|
||||
t.Fatalf("decoding %q produced status %d; malformed input is a client error",
|
||||
body, apiErr.Status)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Query parameters are attacker-controlled and become database filter values.
|
||||
func FuzzQueryFilters(f *testing.F) {
|
||||
f.Add("note", "ready", "2025-01-01", "abc123")
|
||||
f.Add(`{"$ne":null}`, "$where", "not-a-date", "../etc")
|
||||
|
||||
f.Fuzz(func(t *testing.T, sourceType, status, from, hash string) {
|
||||
q := url.Values{}
|
||||
q.Set("source_type", sourceType)
|
||||
q.Set("status", status)
|
||||
q.Set("created_from", from)
|
||||
q.Set("content_hash", hash)
|
||||
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/items?"+q.Encode(), nil)
|
||||
filter, err := itemFilterFromQuery(r)
|
||||
if err != nil {
|
||||
return // rejected, which is the safe outcome
|
||||
}
|
||||
// Anything accepted must be a value from a closed set, never arbitrary
|
||||
// text that could reach a query as an operator.
|
||||
for _, s := range filter.SourceTypes {
|
||||
if !s.Valid() {
|
||||
t.Fatalf("accepted an invalid source type %q", s)
|
||||
}
|
||||
}
|
||||
if filter.ContentHash != "" && !isHex(filter.ContentHash) {
|
||||
t.Fatalf("accepted a non-hexadecimal content hash %q", filter.ContentHash)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,550 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/relata/internal/auth"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/ids"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
)
|
||||
|
||||
type ctxKey int
|
||||
|
||||
const (
|
||||
ctxKeyPrincipal ctxKey = iota
|
||||
ctxKeySession
|
||||
)
|
||||
|
||||
// PrincipalFrom returns the authenticated caller, or nil.
|
||||
func PrincipalFrom(ctx context.Context) *domain.Principal {
|
||||
p, _ := ctx.Value(ctxKeyPrincipal).(*domain.Principal)
|
||||
return p
|
||||
}
|
||||
|
||||
// SessionFrom returns the active session, or nil.
|
||||
func SessionFrom(ctx context.Context) *domain.Session {
|
||||
s, _ := ctx.Value(ctxKeySession).(*domain.Session)
|
||||
return s
|
||||
}
|
||||
|
||||
// WithPrincipal attaches a principal to a context. Exported for tests.
|
||||
func WithPrincipal(ctx context.Context, p *domain.Principal) context.Context {
|
||||
return context.WithValue(ctx, ctxKeyPrincipal, p)
|
||||
}
|
||||
|
||||
// Middleware is the standard decorator shape.
|
||||
type Middleware func(http.Handler) http.Handler
|
||||
|
||||
// Chain applies middleware so that the first listed runs outermost.
|
||||
func Chain(h http.Handler, mw ...Middleware) http.Handler {
|
||||
for i := len(mw) - 1; i >= 0; i-- {
|
||||
h = mw[i](h)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// statusWriter captures the response status for logging and metrics.
|
||||
type statusWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
bytes int
|
||||
}
|
||||
|
||||
func (w *statusWriter) WriteHeader(code int) {
|
||||
if w.status == 0 {
|
||||
w.status = code
|
||||
w.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *statusWriter) Write(b []byte) (int, error) {
|
||||
if w.status == 0 {
|
||||
w.status = http.StatusOK
|
||||
}
|
||||
n, err := w.ResponseWriter.Write(b)
|
||||
w.bytes += n
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Flush supports streaming handlers behind the wrapper.
|
||||
func (w *statusWriter) Flush() {
|
||||
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// RequestID assigns a correlation ID to every request and echoes it back.
|
||||
//
|
||||
// A client-supplied ID is accepted only when it looks like one Relata would
|
||||
// generate; anything else is replaced, so a caller cannot inject arbitrary text
|
||||
// into log fields.
|
||||
func RequestID() Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.Header.Get("X-Request-ID")
|
||||
if !ids.Valid(id) {
|
||||
id = ids.New()
|
||||
}
|
||||
w.Header().Set("X-Request-ID", id)
|
||||
next.ServeHTTP(w, r.WithContext(observability.WithRequestID(r.Context(), id)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Logging records each request's outcome and duration.
|
||||
type Logging struct {
|
||||
Metrics *observability.Metrics
|
||||
}
|
||||
|
||||
// Middleware returns the logging middleware. Route is the pattern rather than
|
||||
// the literal path, so metric label cardinality stays bounded.
|
||||
func (l Logging) Middleware(route string) Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
started := time.Now()
|
||||
sw := &statusWriter{ResponseWriter: w}
|
||||
|
||||
if l.Metrics != nil {
|
||||
l.Metrics.HTTPInFlight.Inc()
|
||||
defer l.Metrics.HTTPInFlight.Dec()
|
||||
}
|
||||
|
||||
next.ServeHTTP(sw, r)
|
||||
|
||||
if sw.status == 0 {
|
||||
sw.status = http.StatusOK
|
||||
}
|
||||
dur := time.Since(started)
|
||||
|
||||
observability.LoggerFrom(r.Context()).Info("request",
|
||||
"method", r.Method,
|
||||
"path", r.URL.Path,
|
||||
"status", sw.status,
|
||||
"duration_ms", dur.Milliseconds(),
|
||||
"bytes", sw.bytes,
|
||||
)
|
||||
|
||||
if l.Metrics != nil {
|
||||
l.Metrics.HTTPRequests.WithLabelValues(route, r.Method, statusClass(sw.status)).Inc()
|
||||
l.Metrics.HTTPDuration.WithLabelValues(route, r.Method).Observe(dur.Seconds())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func statusClass(code int) string {
|
||||
switch {
|
||||
case code < 200:
|
||||
return "1xx"
|
||||
case code < 300:
|
||||
return "2xx"
|
||||
case code < 400:
|
||||
return "3xx"
|
||||
case code < 500:
|
||||
return "4xx"
|
||||
default:
|
||||
return "5xx"
|
||||
}
|
||||
}
|
||||
|
||||
// Recover turns a panic into a 500 rather than a dropped connection.
|
||||
func Recover() Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
// Panic values can contain credentials or request content. The
|
||||
// request ID and route identify the failure without persisting it.
|
||||
observability.LoggerFrom(r.Context()).Error("panic while handling request",
|
||||
"path", r.URL.Path, "method", r.Method)
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusInternalServerError, Code: CodeInternal,
|
||||
Message: "An unexpected error occurred.",
|
||||
})
|
||||
}
|
||||
}()
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// SecurityHeaders sets the response headers that constrain what a browser will
|
||||
// do with Relata's pages.
|
||||
//
|
||||
// The Content-Security-Policy is deliberately restrictive. Relata renders
|
||||
// stored content as escaped text and ships its own CSS and JavaScript as files,
|
||||
// so no inline script, no remote origin and no framing are needed — and
|
||||
// forbidding them means a stored-XSS payload that somehow reached the page
|
||||
// still could not execute or exfiltrate.
|
||||
func SecurityHeaders(allowInlineStyle bool) Middleware {
|
||||
directives := []string{
|
||||
"default-src 'none'",
|
||||
"script-src 'self'",
|
||||
"style-src 'self'",
|
||||
"img-src 'self' data:",
|
||||
"font-src 'self'",
|
||||
"connect-src 'self'",
|
||||
"form-action 'self'",
|
||||
"base-uri 'none'",
|
||||
"frame-ancestors 'none'",
|
||||
"object-src 'none'",
|
||||
"manifest-src 'self'",
|
||||
}
|
||||
csp := strings.Join(directives, "; ")
|
||||
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h := w.Header()
|
||||
h.Set("Content-Security-Policy", csp)
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("X-Frame-Options", "DENY")
|
||||
h.Set("Referrer-Policy", "same-origin")
|
||||
h.Set("Cross-Origin-Opener-Policy", "same-origin")
|
||||
h.Set("Cross-Origin-Resource-Policy", "same-origin")
|
||||
h.Set("Permissions-Policy", "geolocation=(), microphone=(), camera=(), payment=(), usb=()")
|
||||
// Caches must not retain authenticated pages.
|
||||
h.Set("Cache-Control", "no-store")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// MaxBody rejects request bodies over a limit before a handler reads them.
|
||||
func MaxBody(limit int64) Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.ContentLength > limit {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusRequestEntityTooLarge, Code: CodePayloadTooLarge,
|
||||
Message: "The request body is larger than the configured limit.",
|
||||
})
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, limit)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Authenticator resolves sessions and, on the API, bearer tokens.
|
||||
type Authenticator struct {
|
||||
Auth *auth.Service
|
||||
// Tokens enables bearer-token authentication. It is set only on the API
|
||||
// chain: a token is never a way into the web interface.
|
||||
Tokens *auth.TokenService
|
||||
// APIMode returns a JSON 401 rather than redirecting to the login page.
|
||||
APIMode bool
|
||||
// LoginPath is where browser requests are sent when unauthenticated.
|
||||
LoginPath string
|
||||
}
|
||||
|
||||
// resolve authenticates a request, preferring the session cookie.
|
||||
//
|
||||
// The ordering is load-bearing. A request carrying a valid session cookie
|
||||
// authenticates as that session even if it also sends an Authorization header,
|
||||
// so the CSRF exemption below — which keys off how the request authenticated —
|
||||
// cannot be switched off by adding a header to a cookie-bearing request.
|
||||
func (a Authenticator) resolve(r *http.Request) (*domain.Principal, *domain.Session, error) {
|
||||
if p, sess, err := a.Auth.Resolve(r.Context(), a.Auth.TokenFromRequest(r)); err == nil && p != nil {
|
||||
return p, sess, nil
|
||||
}
|
||||
if a.Tokens == nil {
|
||||
return nil, nil, auth.ErrNoSession
|
||||
}
|
||||
secret := auth.BearerFromRequest(r)
|
||||
if secret == "" {
|
||||
return nil, nil, auth.ErrNoSession
|
||||
}
|
||||
p, err := a.Tokens.Authenticate(r.Context(), secret)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return p, nil, nil
|
||||
}
|
||||
|
||||
// RequireAuth rejects unauthenticated requests.
|
||||
func (a Authenticator) RequireAuth() Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p, sess, err := a.resolve(r)
|
||||
if err != nil || p == nil {
|
||||
a.rejectWith(w, r, err)
|
||||
return
|
||||
}
|
||||
ctx := context.WithValue(r.Context(), ctxKeyPrincipal, p)
|
||||
ctx = context.WithValue(ctx, ctxKeySession, sess)
|
||||
// Correlate logs with the user without recording anything sensitive.
|
||||
ctx = observability.WithLogger(ctx,
|
||||
observability.LoggerFrom(ctx).With("user_id", p.UserID, "workspace_id", p.WorkspaceID))
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// OptionalAuth resolves a session when present but permits anonymous requests.
|
||||
func (a Authenticator) OptionalAuth() Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p, sess, err := a.resolve(r)
|
||||
if err == nil && p != nil {
|
||||
ctx := context.WithValue(r.Context(), ctxKeyPrincipal, p)
|
||||
ctx = context.WithValue(ctx, ctxKeySession, sess)
|
||||
r = r.WithContext(ctx)
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// rejectWith refuses a request, distinguishing the one failure a caller can act
|
||||
// on — being rate limited — from every other, which are deliberately identical.
|
||||
func (a Authenticator) rejectWith(w http.ResponseWriter, r *http.Request, err error) {
|
||||
if a.APIMode && errors.Is(err, auth.ErrTokenRateLimit) {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusTooManyRequests, Code: CodeRateLimited,
|
||||
Message: "This token is making too many requests. Slow down and retry.",
|
||||
})
|
||||
return
|
||||
}
|
||||
a.reject(w, r)
|
||||
}
|
||||
|
||||
func (a Authenticator) reject(w http.ResponseWriter, r *http.Request) {
|
||||
if a.APIMode {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusUnauthorized, Code: CodeUnauthorised,
|
||||
Message: "Authentication is required. Sign in to obtain a session.",
|
||||
})
|
||||
return
|
||||
}
|
||||
path := a.LoginPath
|
||||
if path == "" {
|
||||
path = "/login"
|
||||
}
|
||||
http.Redirect(w, r, path, http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// RequireScope rejects a caller that may not perform an action.
|
||||
//
|
||||
// The check is the intersection of the person's role and, when the request came
|
||||
// from an API token, that token's scopes — so a token can never do more than
|
||||
// the person who created it, and demoting someone narrows every token they hold
|
||||
// without any of those tokens being touched.
|
||||
func RequireScope(scope domain.TokenScope) Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p := PrincipalFrom(r.Context())
|
||||
if p == nil {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusUnauthorized, Code: CodeUnauthorised,
|
||||
Message: "Authentication is required.",
|
||||
})
|
||||
return
|
||||
}
|
||||
if p.Permits(scope) {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Say which scope was needed. A caller holding the wrong token
|
||||
// should be able to fix it without guessing, and the scope names
|
||||
// are public API surface rather than a secret.
|
||||
if p.ViaToken() {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusForbidden, Code: CodeInsufficientScope,
|
||||
Message: "This API token does not carry the scope this request needs.",
|
||||
Detail: map[string]string{"required_scope": string(scope)},
|
||||
})
|
||||
return
|
||||
}
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusForbidden, Code: CodeForbidden,
|
||||
Message: "Your role does not permit this.",
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// RequireSession rejects anything not authenticated by an interactive session.
|
||||
//
|
||||
// Token management lives behind this: a leaked token must not be able to mint
|
||||
// more tokens, extend its own life, or revoke the tokens that would replace it.
|
||||
func RequireSession() Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p := PrincipalFrom(r.Context())
|
||||
if p == nil || p.ViaToken() {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusForbidden, Code: CodeForbidden,
|
||||
Message: "This action requires a signed-in session. API tokens cannot manage API tokens.",
|
||||
})
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// RequireWrite rejects a caller whose role cannot modify content.
|
||||
func RequireWrite() Middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p := PrincipalFrom(r.Context())
|
||||
if p == nil || !p.Role.CanWrite() {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusForbidden, Code: CodeForbidden,
|
||||
Message: "Your role does not permit changes.",
|
||||
})
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// CSRF protects state-changing browser requests.
|
||||
//
|
||||
// The check has two independent parts:
|
||||
//
|
||||
// 1. Origin/Referer must match the expected host. A cross-site form post
|
||||
// carries the attacker's origin, and browsers will not let script forge it.
|
||||
// 2. A per-session token, submitted in a form field or header, must match the
|
||||
// session's own token in a constant-time comparison.
|
||||
//
|
||||
// Safe methods are exempt because they must not change state in the first place.
|
||||
type CSRF struct {
|
||||
// TrustedOrigins are the origins permitted to submit state-changing
|
||||
// requests, derived from the configured base URL.
|
||||
TrustedOrigins []string
|
||||
// HeaderName is where a fetch/HTMX request supplies the token.
|
||||
HeaderName string
|
||||
// FieldName is the form field carrying the token.
|
||||
FieldName string
|
||||
}
|
||||
|
||||
// Middleware returns the CSRF middleware.
|
||||
func (c CSRF) Middleware() Middleware {
|
||||
header := c.HeaderName
|
||||
if header == "" {
|
||||
header = "X-CSRF-Token"
|
||||
}
|
||||
field := c.FieldName
|
||||
if field == "" {
|
||||
field = "csrf_token"
|
||||
}
|
||||
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodTrace:
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// A bearer-authenticated request needs no CSRF token. CSRF exists
|
||||
// to stop a third-party page spending *ambient* credentials, and an
|
||||
// Authorization header is not ambient: a browser never attaches one
|
||||
// on its own.
|
||||
//
|
||||
// The exemption keys off how the request authenticated, not off a
|
||||
// header being present. Because resolve() prefers the session
|
||||
// cookie, a request holding a cookie is never ViaToken, so adding
|
||||
// an Authorization header cannot disable this check.
|
||||
if p := PrincipalFrom(r.Context()); p != nil && p.ViaToken() {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
sess := SessionFrom(r.Context())
|
||||
if sess == nil {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusUnauthorized, Code: CodeUnauthorised,
|
||||
Message: "Authentication is required.",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if !c.originAllowed(r) {
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusForbidden, Code: CodeCSRFFailed,
|
||||
Message: "This request appears to originate from another site and was refused.",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
token := r.Header.Get(header)
|
||||
if token == "" {
|
||||
// Reading the form consumes the body; only do so for form
|
||||
// content types, so a JSON body is left intact for the handler.
|
||||
ct := r.Header.Get("Content-Type")
|
||||
base, _, _ := strings.Cut(ct, ";")
|
||||
switch strings.TrimSpace(base) {
|
||||
case "application/x-www-form-urlencoded", "multipart/form-data":
|
||||
_ = r.ParseMultipartForm(8 << 20)
|
||||
token = r.FormValue(field)
|
||||
}
|
||||
}
|
||||
|
||||
if !auth.CheckCSRF(sess.CSRFToken, token) {
|
||||
observability.LoggerFrom(r.Context()).Warn("CSRF check failed",
|
||||
"path", r.URL.Path, "method", r.Method, "has_token", token != "")
|
||||
WriteError(w, r, &APIError{
|
||||
Status: http.StatusForbidden, Code: CodeCSRFFailed,
|
||||
Message: "Your session's security token was missing or invalid. Reload the page and try again.",
|
||||
})
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// originAllowed checks the Origin header, falling back to Referer.
|
||||
//
|
||||
// A request with neither header is refused rather than allowed: every browser
|
||||
// that can perform a cross-site state-changing request sends at least one.
|
||||
func (c CSRF) originAllowed(r *http.Request) bool {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
if ref := r.Header.Get("Referer"); ref != "" {
|
||||
origin = originOf(ref)
|
||||
}
|
||||
}
|
||||
if origin == "" {
|
||||
// A non-browser API client with a valid session token and no Origin
|
||||
// header is not subject to CSRF (there is no ambient credential to
|
||||
// abuse from a browser), but it must still present the token, which is
|
||||
// checked next.
|
||||
return true
|
||||
}
|
||||
for _, allowed := range c.TrustedOrigins {
|
||||
if strings.EqualFold(origin, allowed) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// originOf extracts scheme://host from a URL string without full parsing.
|
||||
func originOf(u string) string {
|
||||
i := strings.Index(u, "://")
|
||||
if i < 0 {
|
||||
return ""
|
||||
}
|
||||
rest := u[i+3:]
|
||||
if j := strings.IndexAny(rest, "/?#"); j >= 0 {
|
||||
rest = rest[:j]
|
||||
}
|
||||
return u[:i+3] + rest
|
||||
}
|
||||
|
||||
// withSession attaches a session to a context. Exported for tests in this
|
||||
// package; the middleware sets it during normal operation.
|
||||
func withSession(ctx context.Context, s *domain.Session) context.Context {
|
||||
return context.WithValue(ctx, ctxKeySession, s)
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHashProducesDistinctHashesForTheSamePassword(t *testing.T) {
|
||||
const pw = "correct horse battery staple"
|
||||
a, err := HashPassword(pw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := HashPassword(pw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a == b {
|
||||
t.Fatal("identical hashes for the same password: the salt is not random")
|
||||
}
|
||||
for _, h := range []string{a, b} {
|
||||
if strings.Contains(h, pw) {
|
||||
t.Fatal("the hash contains the plain-text password")
|
||||
}
|
||||
if !strings.HasPrefix(h, "$argon2id$") {
|
||||
t.Fatalf("hash is not Argon2id: %q", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyAcceptsCorrectAndRejectsWrongPasswords(t *testing.T) {
|
||||
const pw = "correct horse battery staple"
|
||||
h, err := HashPassword(pw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ok, err := VerifyPassword(pw, h)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("correct password rejected: ok=%v err=%v", ok, err)
|
||||
}
|
||||
for _, wrong := range []string{
|
||||
"correct horse battery stapl",
|
||||
"correct horse battery staple ",
|
||||
"Correct horse battery staple",
|
||||
"",
|
||||
strings.Repeat("x", 200),
|
||||
} {
|
||||
ok, err := VerifyPassword(wrong, h)
|
||||
if err != nil && !errors.Is(err, ErrPasswordTooLong) {
|
||||
t.Fatalf("unexpected error for %q: %v", wrong, err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatalf("wrong password %q was accepted", wrong)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashEncodesItsOwnParameters(t *testing.T) {
|
||||
h, err := HashPassword("a sufficiently long password")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Parameters travel with the hash, so they can be raised later without
|
||||
// invalidating existing passwords.
|
||||
for _, want := range []string{"v=19", "m=", "t=", "p="} {
|
||||
if !strings.Contains(h, want) {
|
||||
t.Fatalf("hash %q does not record %q", h, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyRejectsMalformedHashes(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"empty": "",
|
||||
"not phc": "plaintext",
|
||||
"wrong algorithm": "$argon2i$v=19$m=65536,t=3,p=2$c2FsdA$aGFzaA",
|
||||
"bad version": "$argon2id$v=99$m=65536,t=3,p=2$c2FsdA$aGFzaA",
|
||||
"missing sections": "$argon2id$v=19$m=65536,t=3,p=2",
|
||||
"bad base64": "$argon2id$v=19$m=65536,t=3,p=2$!!!$aGFzaA",
|
||||
"weak params": "$argon2id$v=19$m=1,t=1,p=1$c2FsdA$aGFzaA",
|
||||
"zero parallelism": "$argon2id$v=19$m=65536,t=3,p=0$c2FsdA$aGFzaA",
|
||||
"unknown param": "$argon2id$v=19$m=65536,t=3,x=2$c2FsdA$aGFzaA",
|
||||
"excessive memory": "$argon2id$v=19$m=4294967295,t=3,p=2$YWJjZGVmZ2hpamtsbW5vcA$YWJjZGVmZ2hpamtsbW5vcA",
|
||||
"excessive time": "$argon2id$v=19$m=65536,t=4294967295,p=2$YWJjZGVmZ2hpamtsbW5vcA$YWJjZGVmZ2hpamtsbW5vcA",
|
||||
"oversized hash": strings.Repeat("x", maxEncodedHashLen+1),
|
||||
}
|
||||
for name, h := range cases {
|
||||
ok, err := VerifyPassword("anything", h)
|
||||
if ok {
|
||||
t.Errorf("%s: a malformed hash verified successfully", name)
|
||||
}
|
||||
if err == nil {
|
||||
t.Errorf("%s: malformed hash produced no error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPasswordPolicy(t *testing.T) {
|
||||
if err := ValidatePassword(strings.Repeat("a", MinPasswordLength-1)); !errors.Is(err, ErrPasswordTooShort) {
|
||||
t.Fatalf("short password accepted: %v", err)
|
||||
}
|
||||
if err := ValidatePassword(strings.Repeat("a", MaxPasswordLength+1)); !errors.Is(err, ErrPasswordTooLong) {
|
||||
t.Fatalf("overlong password accepted: %v", err)
|
||||
}
|
||||
if err := ValidatePassword(strings.Repeat("a", MinPasswordLength)); err != nil {
|
||||
t.Fatalf("valid password rejected: %v", err)
|
||||
}
|
||||
// A very long password must be rejected rather than hashed: Argon2id would
|
||||
// happily consume it, but the cost is attacker-controlled.
|
||||
if _, err := HashPassword(strings.Repeat("a", MaxPasswordLength+1)); err == nil {
|
||||
t.Fatal("an unbounded password was hashed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsernameValidation(t *testing.T) {
|
||||
valid := map[string]string{
|
||||
"Admin": "admin",
|
||||
" ops.team ": "ops.team",
|
||||
"a_b-c.123": "a_b-c.123",
|
||||
}
|
||||
for in, want := range valid {
|
||||
got, err := ValidateUsername(in)
|
||||
if err != nil {
|
||||
t.Errorf("ValidateUsername(%q) errored: %v", in, err)
|
||||
continue
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("ValidateUsername(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "ab", strings.Repeat("a", 65), "has space", "quote'", "semi;colon", "$ne", "../../etc"} {
|
||||
if _, err := ValidateUsername(bad); err == nil {
|
||||
t.Errorf("ValidateUsername(%q) was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A username that would be meaningful to MongoDB as an operator must be
|
||||
// rejected before it ever reaches a query.
|
||||
func TestUsernameRejectsOperatorShapedInput(t *testing.T) {
|
||||
for _, bad := range []string{"$ne", "$gt", "{\"$ne\":null}", "a$b"} {
|
||||
if _, err := ValidateUsername(bad); err == nil {
|
||||
t.Errorf("operator-shaped username %q accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDummyVerificationCostsRealTime(t *testing.T) {
|
||||
// The unknown-user path must not be obviously faster than a real
|
||||
// verification, or response timing would enumerate accounts.
|
||||
h, err := HashPassword("a sufficiently long password")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
_, _ = VerifyPassword("wrong password entirely", h)
|
||||
real := time.Since(start)
|
||||
|
||||
start = time.Now()
|
||||
SpendDummyVerification("wrong password entirely")
|
||||
dummy := time.Since(start)
|
||||
|
||||
if dummy < real/4 {
|
||||
t.Fatalf("dummy verification took %v against a real %v: the timing gap would leak account existence", dummy, real)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSRFComparison(t *testing.T) {
|
||||
if !CheckCSRF("token-value", "token-value") {
|
||||
t.Fatal("matching CSRF tokens rejected")
|
||||
}
|
||||
for _, bad := range []string{"", "token-valu", "token-value ", "TOKEN-VALUE", "x"} {
|
||||
if CheckCSRF("token-value", bad) {
|
||||
t.Fatalf("CSRF token %q accepted", bad)
|
||||
}
|
||||
}
|
||||
if CheckCSRF("", "") {
|
||||
t.Fatal("two empty tokens compared as a match")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Rate limiting ---------------------------------------------------------
|
||||
|
||||
func TestLoginLimiterEnforcesBurst(t *testing.T) {
|
||||
l := newLoginLimiter(3, time.Minute)
|
||||
for i := 0; i < 3; i++ {
|
||||
if !l.allow("client-a") {
|
||||
t.Fatalf("attempt %d refused within the burst", i+1)
|
||||
}
|
||||
}
|
||||
if l.allow("client-a") {
|
||||
t.Fatal("attempt beyond the burst was permitted")
|
||||
}
|
||||
// A different client is unaffected.
|
||||
if !l.allow("client-b") {
|
||||
t.Fatal("one client's limit blocked another")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginLimiterResetsOnSuccess(t *testing.T) {
|
||||
l := newLoginLimiter(2, time.Minute)
|
||||
l.allow("c")
|
||||
l.allow("c")
|
||||
if l.allow("c") {
|
||||
t.Fatal("limit not reached")
|
||||
}
|
||||
l.reset("c")
|
||||
if !l.allow("c") {
|
||||
t.Fatal("a successful login did not clear the attempt budget")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginLimiterWindowExpires(t *testing.T) {
|
||||
l := newLoginLimiter(1, 20*time.Millisecond)
|
||||
if !l.allow("c") {
|
||||
t.Fatal("first attempt refused")
|
||||
}
|
||||
if l.allow("c") {
|
||||
t.Fatal("second attempt allowed within the window")
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
if !l.allow("c") {
|
||||
t.Fatal("the window did not expire")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginLimiterDoesNotGrowWithoutBound(t *testing.T) {
|
||||
l := newLoginLimiter(1, 10*time.Millisecond)
|
||||
for i := 0; i < 500; i++ {
|
||||
l.allow(string(rune('a'+i%26)) + string(rune('0'+i%10)) + "-key")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
// A sweep runs on the next call, discarding expired buckets.
|
||||
l.allow("trigger-sweep")
|
||||
|
||||
l.mu.Lock()
|
||||
n := len(l.buckets)
|
||||
l.mu.Unlock()
|
||||
if n > 50 {
|
||||
t.Fatalf("limiter retained %d buckets after expiry; memory would grow without bound", n)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Client key ------------------------------------------------------------
|
||||
|
||||
func TestClientKeyIgnoresProxyHeadersByDefault(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/login", nil)
|
||||
r.RemoteAddr = "192.0.2.10:5555"
|
||||
r.Header.Set("X-Forwarded-For", "203.0.113.1")
|
||||
|
||||
// Trusting the header by default would let anyone bypass rate limiting by
|
||||
// varying it.
|
||||
if got := ClientKey(r, false); got != "192.0.2.10" {
|
||||
t.Fatalf("ClientKey = %q; a spoofable header was trusted", got)
|
||||
}
|
||||
if got := ClientKey(r, true); got != "203.0.113.1" {
|
||||
t.Fatalf("ClientKey with trusted proxy = %q, want the forwarded address", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientKeyHandlesIPv6AndMalformedAddresses(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"[2001:db8::1]:443": "2001:db8::1",
|
||||
"192.0.2.1:80": "192.0.2.1",
|
||||
"garbage": "garbage",
|
||||
}
|
||||
for addr, want := range cases {
|
||||
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
r.RemoteAddr = addr
|
||||
if got := ClientKey(r, false); got != want {
|
||||
t.Errorf("ClientKey(%q) = %q, want %q", addr, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserAgentIsHashedNotStored(t *testing.T) {
|
||||
ua := "Mozilla/5.0 (very identifying string)"
|
||||
h := hashUserAgent(ua)
|
||||
if h == "" {
|
||||
t.Fatal("no hash produced")
|
||||
}
|
||||
if strings.Contains(h, "Mozilla") {
|
||||
t.Fatal("the user agent was stored verbatim")
|
||||
}
|
||||
if h != hashUserAgent(ua) {
|
||||
t.Fatal("hashing is not deterministic")
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzDecodeHash(f *testing.F) {
|
||||
h, _ := HashPassword("a sufficiently long password")
|
||||
f.Add(h)
|
||||
f.Add("$argon2id$v=19$m=65536,t=3,p=2$c2FsdA$aGFzaA")
|
||||
f.Add("")
|
||||
|
||||
f.Fuzz(func(t *testing.T, encoded string) {
|
||||
// Must never panic, and must never verify an arbitrary string.
|
||||
ok, err := VerifyPassword("some candidate password", encoded)
|
||||
if ok && err != nil {
|
||||
t.Fatal("verification both succeeded and errored")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
// Package auth implements local authentication: Argon2id password hashing,
|
||||
// server-side sessions, CSRF protection and login rate limiting.
|
||||
//
|
||||
// Plain-text passwords are never stored, never logged and never compared with
|
||||
// anything other than a constant-time function.
|
||||
package auth
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
|
||||
// Argon2 parameters.
|
||||
//
|
||||
// These target roughly 64 MiB and a few hundred milliseconds on a modern
|
||||
// machine — deliberately expensive, because the whole point is to make an
|
||||
// offline guessing attack against a stolen hash uneconomic. The parameters are
|
||||
// stored in each hash, so raising them later does not invalidate existing
|
||||
// passwords: an old hash still verifies with its own parameters.
|
||||
const (
|
||||
argonTime uint32 = 3
|
||||
argonMemory uint32 = 64 * 1024 // KiB
|
||||
argonKeyLen uint32 = 32
|
||||
argonSaltLen = 16
|
||||
|
||||
// Stored hashes are data, not trusted instructions. These generous ceilings
|
||||
// prevent a corrupt or attacker-modified database value from turning login
|
||||
// into an extreme allocation or CPU request while leaving room for upgrades.
|
||||
maxEncodedHashLen = 512
|
||||
maxArgonMemory uint32 = 256 * 1024 // KiB
|
||||
maxArgonTime uint32 = 10
|
||||
maxArgonParallelism = 16
|
||||
maxArgonSaltLen = 64
|
||||
maxArgonKeyLen = 64
|
||||
)
|
||||
|
||||
// Password policy.
|
||||
const (
|
||||
// MinPasswordLength is deliberately generous rather than demanding
|
||||
// character classes: length beats composition rules for real strength.
|
||||
MinPasswordLength = 12
|
||||
MaxPasswordLength = 1024
|
||||
MinUsernameLength = 3
|
||||
MaxUsernameLength = 64
|
||||
)
|
||||
|
||||
// Errors returned by password handling.
|
||||
var (
|
||||
ErrPasswordTooShort = fmt.Errorf("password must be at least %d characters", MinPasswordLength)
|
||||
ErrPasswordTooLong = fmt.Errorf("password must be at most %d characters", MaxPasswordLength)
|
||||
ErrInvalidHashFormat = errors.New("stored password hash is not in the expected format")
|
||||
ErrIncompatibleHash = errors.New("stored password hash uses an unsupported algorithm version")
|
||||
ErrInvalidUsername = errors.New("username must be 3-64 characters using letters, digits, dot, dash or underscore")
|
||||
)
|
||||
|
||||
// argonParallelism picks a lane count from the machine's CPUs, bounded so that
|
||||
// a hash produced on a large host still verifies affordably on a small one.
|
||||
func argonParallelism() uint8 {
|
||||
n := runtime.NumCPU()
|
||||
if n < 1 {
|
||||
n = 1
|
||||
}
|
||||
if n > 4 {
|
||||
n = 4
|
||||
}
|
||||
return uint8(n)
|
||||
}
|
||||
|
||||
// HashPassword returns an encoded Argon2id hash in the standard PHC string
|
||||
// format, which embeds the algorithm, its parameters and the salt.
|
||||
func HashPassword(password string) (string, error) {
|
||||
if err := ValidatePassword(password); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
salt := make([]byte, argonSaltLen)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return "", fmt.Errorf("generate salt: %w", err)
|
||||
}
|
||||
|
||||
par := argonParallelism()
|
||||
key := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, par, argonKeyLen)
|
||||
|
||||
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
|
||||
argon2.Version, argonMemory, argonTime, par,
|
||||
base64.RawStdEncoding.EncodeToString(salt),
|
||||
base64.RawStdEncoding.EncodeToString(key),
|
||||
), nil
|
||||
}
|
||||
|
||||
// VerifyPassword checks a candidate password against an encoded hash.
|
||||
//
|
||||
// The comparison is constant-time so that verification duration cannot reveal
|
||||
// how much of the hash matched.
|
||||
func VerifyPassword(password, encoded string) (bool, error) {
|
||||
params, salt, want, err := decodeHash(encoded)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(password) > MaxPasswordLength {
|
||||
return false, ErrPasswordTooLong
|
||||
}
|
||||
|
||||
got := argon2.IDKey([]byte(password), salt, params.time, params.memory, params.parallelism, uint32(len(want)))
|
||||
return subtle.ConstantTimeCompare(got, want) == 1, nil
|
||||
}
|
||||
|
||||
type argonParams struct {
|
||||
memory uint32
|
||||
time uint32
|
||||
parallelism uint8
|
||||
}
|
||||
|
||||
func decodeHash(encoded string) (argonParams, []byte, []byte, error) {
|
||||
var p argonParams
|
||||
if len(encoded) == 0 || len(encoded) > maxEncodedHashLen {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
|
||||
parts := strings.Split(encoded, "$")
|
||||
// "", "argon2id", "v=19", "m=...,t=...,p=...", salt, key
|
||||
if len(parts) != 6 || parts[0] != "" {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
if parts[1] != "argon2id" {
|
||||
return p, nil, nil, ErrIncompatibleHash
|
||||
}
|
||||
|
||||
var version int
|
||||
if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
if version != argon2.Version {
|
||||
return p, nil, nil, ErrIncompatibleHash
|
||||
}
|
||||
|
||||
kv := strings.Split(parts[3], ",")
|
||||
if len(kv) != 3 {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
for _, item := range kv {
|
||||
k, v, ok := strings.Cut(item, "=")
|
||||
if !ok {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
if err != nil {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
switch k {
|
||||
case "m":
|
||||
p.memory = uint32(n)
|
||||
case "t":
|
||||
p.time = uint32(n)
|
||||
case "p":
|
||||
if n == 0 || n > 255 {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
p.parallelism = uint8(n)
|
||||
default:
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
}
|
||||
// Reject nonsensical parameters that would make verification trivial or
|
||||
// impossibly expensive.
|
||||
if p.memory < 1024 || p.memory > maxArgonMemory ||
|
||||
p.time == 0 || p.time > maxArgonTime ||
|
||||
p.parallelism == 0 || p.parallelism > maxArgonParallelism {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
|
||||
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
|
||||
if err != nil || len(salt) < argonSaltLen || len(salt) > maxArgonSaltLen {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
key, err := base64.RawStdEncoding.DecodeString(parts[5])
|
||||
if err != nil || len(key) < 16 || len(key) > maxArgonKeyLen {
|
||||
return p, nil, nil, ErrInvalidHashFormat
|
||||
}
|
||||
return p, salt, key, nil
|
||||
}
|
||||
|
||||
// ValidatePassword enforces the password policy.
|
||||
func ValidatePassword(password string) error {
|
||||
n := utf8.RuneCountInString(password)
|
||||
if n < MinPasswordLength {
|
||||
return ErrPasswordTooShort
|
||||
}
|
||||
if len(password) > MaxPasswordLength {
|
||||
return ErrPasswordTooLong
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateUsername enforces the username policy and returns the canonical
|
||||
// lowercase form.
|
||||
func ValidateUsername(username string) (string, error) {
|
||||
u := strings.ToLower(strings.TrimSpace(username))
|
||||
if len(u) < MinUsernameLength || len(u) > MaxUsernameLength {
|
||||
return "", ErrInvalidUsername
|
||||
}
|
||||
for _, r := range u {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z',
|
||||
r >= '0' && r <= '9',
|
||||
r == '.', r == '-', r == '_':
|
||||
default:
|
||||
return "", ErrInvalidUsername
|
||||
}
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// dummyHash is a real Argon2id hash of a random value, used to spend
|
||||
// comparable time when a username does not exist.
|
||||
//
|
||||
// Without it, a failed lookup would return noticeably faster than a wrong
|
||||
// password, and the response time would disclose which usernames are
|
||||
// registered.
|
||||
var dummyHash = func() string {
|
||||
buf := make([]byte, 32)
|
||||
_, _ = rand.Read(buf)
|
||||
h, err := HashPassword(base64.RawStdEncoding.EncodeToString(buf))
|
||||
if err != nil {
|
||||
// Only reachable if the policy rejects a 43-character random string.
|
||||
return "$argon2id$v=19$m=65536,t=3,p=2$YWJjZGVmZ2hpamtsbW5vcA$" +
|
||||
"c29tZWZpeGVkdmFsdWV0aGF0d2lsbG5ldmVybWF0Y2g"
|
||||
}
|
||||
return h
|
||||
}()
|
||||
|
||||
// SpendDummyVerification performs a password verification against a throwaway
|
||||
// hash. Callers use it on the "no such user" path so that both outcomes cost
|
||||
// roughly the same.
|
||||
func SpendDummyVerification(password string) {
|
||||
_, _ = VerifyPassword(password, dummyHash)
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
// Package jobs runs Relata's background work: a bounded pool of workers
|
||||
// draining a persistent, leased queue stored in MongoDB.
|
||||
//
|
||||
// The design goals are recoverability and boundedness. A crashed worker's job
|
||||
// returns to the queue when its lease expires; no work is ever started without
|
||||
// a worker slot; AI calls are additionally limited by their own semaphore; and
|
||||
// shutdown drains in-flight jobs rather than abandoning them.
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/relata/internal/analysis"
|
||||
"github.com/andydixon/relata/internal/config"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/embeddings"
|
||||
"github.com/andydixon/relata/internal/ids"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
)
|
||||
|
||||
// Handler executes one job. It must honour ctx cancellation and should be
|
||||
// idempotent: a job may run more than once after a crash.
|
||||
type Handler func(ctx context.Context, j *domain.Job) error
|
||||
|
||||
// Pool is a set of workers draining the queue.
|
||||
type Pool struct {
|
||||
repo domain.JobRepository
|
||||
cfg config.WorkerConfig
|
||||
metrics *observability.Metrics
|
||||
handlers map[domain.JobType]Handler
|
||||
|
||||
// aiSem bounds concurrent AI provider calls independently of worker count,
|
||||
// so a small local model is not swamped by every worker at once.
|
||||
aiSem chan struct{}
|
||||
|
||||
workerID string
|
||||
wg sync.WaitGroup
|
||||
stopOnce sync.Once
|
||||
stopped chan struct{}
|
||||
}
|
||||
|
||||
// NewPool builds a worker pool. Handlers are registered before Start.
|
||||
func NewPool(repo domain.JobRepository, cfg config.WorkerConfig, m *observability.Metrics) *Pool {
|
||||
host, _ := os.Hostname()
|
||||
if host == "" {
|
||||
host = "relata"
|
||||
}
|
||||
aiConc := cfg.AIConcurrency
|
||||
if aiConc <= 0 {
|
||||
aiConc = 1
|
||||
}
|
||||
return &Pool{
|
||||
repo: repo,
|
||||
cfg: cfg,
|
||||
metrics: m,
|
||||
handlers: make(map[domain.JobType]Handler),
|
||||
aiSem: make(chan struct{}, aiConc),
|
||||
workerID: fmt.Sprintf("%s-%s", host, ids.New()[:8]),
|
||||
stopped: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Register attaches a handler to a job type. It panics on an unknown type,
|
||||
// which can only be a programming error and is better caught at startup.
|
||||
func (p *Pool) Register(t domain.JobType, h Handler) {
|
||||
if !t.Valid() {
|
||||
panic("jobs: cannot register handler for unknown job type " + string(t))
|
||||
}
|
||||
p.handlers[t] = h
|
||||
}
|
||||
|
||||
// AcquireAI blocks until an AI slot is free, returning a release function.
|
||||
//
|
||||
// Handlers call this around provider requests so that the number of concurrent
|
||||
// model calls stays bounded regardless of how many workers are running.
|
||||
func (p *Pool) AcquireAI(ctx context.Context) (release func(), err error) {
|
||||
select {
|
||||
case p.aiSem <- struct{}{}:
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { <-p.aiSem }) }, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Start launches the workers and the lease reaper.
|
||||
//
|
||||
// The returned context governs the pool's lifetime: cancelling it begins a
|
||||
// graceful shutdown in which no new jobs are claimed and in-flight jobs are
|
||||
// given until Stop's deadline to finish.
|
||||
func (p *Pool) Start(ctx context.Context) {
|
||||
if !p.cfg.Enabled {
|
||||
observability.LoggerFrom(ctx).Info("background workers are disabled by configuration")
|
||||
return
|
||||
}
|
||||
n := p.cfg.Count
|
||||
if n <= 0 {
|
||||
n = 1
|
||||
}
|
||||
|
||||
types := make([]domain.JobType, 0, len(p.handlers))
|
||||
for t := range p.handlers {
|
||||
types = append(types, t)
|
||||
}
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
p.wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer p.wg.Done()
|
||||
p.run(ctx, fmt.Sprintf("%s-w%d", p.workerID, idx), types)
|
||||
}(i)
|
||||
}
|
||||
|
||||
p.wg.Add(1)
|
||||
go func() {
|
||||
defer p.wg.Done()
|
||||
p.reapLeases(ctx)
|
||||
}()
|
||||
|
||||
observability.LoggerFrom(ctx).Info("background workers started",
|
||||
"workers", n, "ai_concurrency", cap(p.aiSem), "job_types", len(types))
|
||||
}
|
||||
|
||||
// Stop waits for in-flight jobs to finish, up to the given timeout.
|
||||
func (p *Pool) Stop(timeout time.Duration) {
|
||||
p.stopOnce.Do(func() { close(p.stopped) })
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
p.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(timeout):
|
||||
// In-flight jobs keep their leases, which expire and return them to
|
||||
// the queue for another worker to pick up.
|
||||
}
|
||||
}
|
||||
|
||||
// run is one worker's claim-execute-report loop.
|
||||
func (p *Pool) run(ctx context.Context, workerID string, types []domain.JobType) {
|
||||
log := observability.LoggerFrom(ctx).With("worker", workerID)
|
||||
|
||||
// A small random offset stops every worker polling on the same tick.
|
||||
jitter := time.Duration(rand.Int63n(int64(p.cfg.PollInterval) + 1))
|
||||
timer := time.NewTimer(jitter)
|
||||
defer timer.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-p.stopped:
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
|
||||
job, err := p.repo.Claim(ctx, workerID, types, p.cfg.LeaseDuration)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
log.Error("could not claim a job", "error", err.Error())
|
||||
}
|
||||
timer.Reset(p.cfg.PollInterval)
|
||||
continue
|
||||
}
|
||||
if job == nil {
|
||||
timer.Reset(p.cfg.PollInterval)
|
||||
continue
|
||||
}
|
||||
|
||||
p.execute(ctx, workerID, job)
|
||||
// Work was found, so look again immediately rather than idling.
|
||||
timer.Reset(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// execute runs one job with its own timeout, heartbeat and outcome reporting.
|
||||
func (p *Pool) execute(ctx context.Context, workerID string, job *domain.Job) {
|
||||
log := observability.LoggerFrom(ctx).With(
|
||||
"worker", workerID, "job_id", job.ID, "job_type", string(job.Type),
|
||||
"attempt", job.Attempts, "item_id", job.ItemID)
|
||||
|
||||
handler, ok := p.handlers[job.Type]
|
||||
if !ok {
|
||||
// Nothing can run this job, and retrying will not change that.
|
||||
p.finish(ctx, workerID, job, fmt.Errorf("no handler registered for job type %q", job.Type),
|
||||
domain.ErrClassPermanent, time.Now())
|
||||
return
|
||||
}
|
||||
|
||||
jobCtx, cancel := context.WithTimeout(observability.WithJobID(ctx, job.ID), p.cfg.JobTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Renew the lease while the job runs, so a long job is not reclaimed from
|
||||
// under this worker. Losing the lease cancels the work: another worker has
|
||||
// taken over and this attempt's result is no longer wanted.
|
||||
hbDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(hbDone)
|
||||
t := time.NewTicker(p.cfg.HeartbeatPeriod)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-jobCtx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if err := p.repo.Heartbeat(jobCtx, job.ID, workerID, p.cfg.LeaseDuration); err != nil {
|
||||
if errors.Is(err, domain.ErrConflict) {
|
||||
log.Warn("lease lost; abandoning this attempt")
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
log.Debug("heartbeat failed", "error", err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
started := time.Now()
|
||||
err := runGuarded(jobCtx, handler, job)
|
||||
|
||||
// Classify before cancelling: the check distinguishes a job that ran out of
|
||||
// time from one cancelled here, and cancelling first would conflate them.
|
||||
class := Classify(err, jobCtx)
|
||||
|
||||
// Stop the heartbeat and wait for it to exit. The cancel must come before
|
||||
// the wait — the goroutine's only exit is jobCtx being done, so waiting
|
||||
// first would block until the deferred cancel that never runs.
|
||||
cancel()
|
||||
<-hbDone
|
||||
p.finish(ctx, workerID, job, err, class, started)
|
||||
|
||||
if p.metrics != nil {
|
||||
outcome := "success"
|
||||
if err != nil {
|
||||
outcome = string(class)
|
||||
}
|
||||
p.metrics.JobsProcessed.WithLabelValues(string(job.Type), outcome).Inc()
|
||||
p.metrics.JobDuration.WithLabelValues(string(job.Type)).Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Error("job failed", "error", err.Error(), "class", string(class),
|
||||
"duration_ms", time.Since(started).Milliseconds())
|
||||
} else {
|
||||
log.Info("job completed", "duration_ms", time.Since(started).Milliseconds())
|
||||
}
|
||||
}
|
||||
|
||||
// runGuarded executes a handler, converting a panic into an error so that one
|
||||
// malformed document cannot take down a worker.
|
||||
func runGuarded(ctx context.Context, h Handler, job *domain.Job) (err error) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
// Never persist the panic value: handler panics can include document
|
||||
// content or credentials. The job and request IDs provide correlation.
|
||||
err = errors.New("job handler panicked")
|
||||
}
|
||||
}()
|
||||
return h(ctx, job)
|
||||
}
|
||||
|
||||
// finish reports the attempt's outcome and decides whether to retry.
|
||||
func (p *Pool) finish(ctx context.Context, workerID string, job *domain.Job, err error, class domain.ErrorClass, started time.Time) {
|
||||
// Report using a fresh context: the job's context may already be cancelled,
|
||||
// and the outcome must still be recorded.
|
||||
repCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
att := domain.JobAttempt{
|
||||
Number: job.Attempts,
|
||||
StartedAt: started,
|
||||
EndedAt: time.Now().UTC(),
|
||||
Worker: workerID,
|
||||
DurationMS: time.Since(started).Milliseconds(),
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
if cerr := p.repo.Complete(repCtx, job.ID, workerID, att); cerr != nil {
|
||||
observability.LoggerFrom(ctx).Error("could not mark job complete",
|
||||
"job_id", job.ID, "error", cerr.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
att.Error = truncateError(err.Error())
|
||||
att.ErrorClass = string(class)
|
||||
|
||||
maxAttempts := job.MaxAttempts
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = p.cfg.MaxAttempts
|
||||
}
|
||||
|
||||
// Only transient failures are retried, and only within the attempt budget.
|
||||
// A permanent failure — invalid model output, an unparseable document, a
|
||||
// missing handler — is terminal: repeating it would waste the queue.
|
||||
var retryAfter *time.Time
|
||||
if class == domain.ErrClassTransient && job.Attempts < maxAttempts {
|
||||
delay := backoff(job.Attempts)
|
||||
t := time.Now().UTC().Add(delay)
|
||||
retryAfter = &t
|
||||
}
|
||||
|
||||
if ferr := p.repo.Fail(repCtx, job.ID, workerID, att, retryAfter); ferr != nil {
|
||||
observability.LoggerFrom(ctx).Error("could not record job failure",
|
||||
"job_id", job.ID, "error", ferr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// backoff returns the delay before attempt n+1, with jitter.
|
||||
func backoff(attempt int) time.Duration {
|
||||
base := 2 * time.Second
|
||||
d := base << minInt(attempt, 6) // capped so the shift cannot overflow
|
||||
if d > 5*time.Minute {
|
||||
d = 5 * time.Minute
|
||||
}
|
||||
return d/2 + time.Duration(rand.Int63n(int64(d/2)+1))
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Classify decides whether a failure is worth retrying.
|
||||
func Classify(err error, ctx context.Context) domain.ErrorClass {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
return domain.ErrClassCancelled
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
// A job that ran out of time may succeed with a fresh budget.
|
||||
return domain.ErrClassTimeout
|
||||
case errors.Is(err, analysis.ErrInvalidOutput):
|
||||
// Retrying a model that produced structurally invalid output tends to
|
||||
// produce more of the same; the failure is recorded instead.
|
||||
return domain.ErrClassInvalid
|
||||
case errors.Is(err, analysis.ErrDisabled):
|
||||
return domain.ErrClassPermanent
|
||||
case embeddings.IsPermanent(err):
|
||||
return domain.ErrClassPermanent
|
||||
case errors.Is(err, domain.ErrNotFound),
|
||||
errors.Is(err, domain.ErrInvalidInput),
|
||||
errors.Is(err, domain.ErrUnsupported),
|
||||
errors.Is(err, domain.ErrWorkspaceMiss):
|
||||
return domain.ErrClassPermanent
|
||||
case embeddings.Retryable(err):
|
||||
return domain.ErrClassTransient
|
||||
}
|
||||
// An unrecognised error is treated as transient: the attempt budget bounds
|
||||
// the cost, and a genuinely permanent failure will exhaust it quickly.
|
||||
return domain.ErrClassTransient
|
||||
}
|
||||
|
||||
// reapLeases periodically returns jobs from crashed workers to the queue and
|
||||
// refreshes the queue-depth gauges.
|
||||
func (p *Pool) reapLeases(ctx context.Context) {
|
||||
interval := p.cfg.LeaseDuration / 2
|
||||
if interval < 10*time.Second {
|
||||
interval = 10 * time.Second
|
||||
}
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-p.stopped:
|
||||
return
|
||||
case <-t.C:
|
||||
released, err := p.repo.ReleaseExpired(ctx, time.Now().UTC())
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
observability.LoggerFrom(ctx).Error("could not release expired leases", "error", err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
if released > 0 {
|
||||
observability.LoggerFrom(ctx).Warn(
|
||||
"returned jobs from expired leases to the queue", "count", released)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// truncateError bounds an error string before it is stored. Error text can
|
||||
// include untrusted content, so its length is capped.
|
||||
func truncateError(s string) string {
|
||||
const max = 2000
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + "…"
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/relata/internal/analysis"
|
||||
"github.com/andydixon/relata/internal/config"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/embeddings"
|
||||
)
|
||||
|
||||
// fakeQueue is an in-memory job repository for exercising the worker loop.
|
||||
type fakeQueue struct {
|
||||
mu sync.Mutex
|
||||
jobs map[string]*domain.Job
|
||||
order []string
|
||||
completed []string
|
||||
failed []string
|
||||
retried []string
|
||||
heartbeats int
|
||||
}
|
||||
|
||||
func newFakeQueue() *fakeQueue {
|
||||
return &fakeQueue{jobs: map[string]*domain.Job{}}
|
||||
}
|
||||
|
||||
func (q *fakeQueue) Enqueue(_ context.Context, j *domain.Job) error {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
q.jobs[j.ID] = j
|
||||
q.order = append(q.order, j.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *fakeQueue) Claim(_ context.Context, owner string, _ []domain.JobType, lease time.Duration) (*domain.Job, error) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
for _, id := range q.order {
|
||||
j := q.jobs[id]
|
||||
if j.Status != domain.JobPending {
|
||||
continue
|
||||
}
|
||||
j.Status = domain.JobRunning
|
||||
j.LeaseOwner = owner
|
||||
j.Attempts++
|
||||
exp := time.Now().Add(lease)
|
||||
j.LeaseExpires = &exp
|
||||
cp := *j
|
||||
return &cp, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (q *fakeQueue) Heartbeat(_ context.Context, id, owner string, _ time.Duration) error {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
q.heartbeats++
|
||||
j := q.jobs[id]
|
||||
if j == nil || j.LeaseOwner != owner {
|
||||
return domain.ErrConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *fakeQueue) Complete(_ context.Context, id, owner string, _ domain.JobAttempt) error {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
j := q.jobs[id]
|
||||
if j == nil || j.LeaseOwner != owner {
|
||||
return domain.ErrConflict
|
||||
}
|
||||
j.Status = domain.JobSucceeded
|
||||
q.completed = append(q.completed, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *fakeQueue) Fail(_ context.Context, id, owner string, att domain.JobAttempt, retryAfter *time.Time) error {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
j := q.jobs[id]
|
||||
if j == nil || j.LeaseOwner != owner {
|
||||
return domain.ErrConflict
|
||||
}
|
||||
if retryAfter != nil {
|
||||
j.Status = domain.JobPending
|
||||
q.retried = append(q.retried, id)
|
||||
} else {
|
||||
j.Status = domain.JobFailed
|
||||
q.failed = append(q.failed, id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *fakeQueue) Get(_ context.Context, id string) (*domain.Job, error) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if j := q.jobs[id]; j != nil {
|
||||
cp := *j
|
||||
return &cp, nil
|
||||
}
|
||||
return nil, domain.ErrNotFound
|
||||
}
|
||||
|
||||
func (q *fakeQueue) List(context.Context, string, []domain.JobStatus, []domain.JobType, domain.Page) ([]*domain.Job, int64, error) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
func (q *fakeQueue) Retry(context.Context, string, string) (*domain.Job, error) { return nil, nil }
|
||||
func (q *fakeQueue) CountByStatus(context.Context, string) (map[domain.JobStatus]int64, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (q *fakeQueue) ReleaseExpired(context.Context, time.Time) (int64, error) { return 0, nil }
|
||||
func (q *fakeQueue) DeleteForItem(context.Context, string, string) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (q *fakeQueue) counts() (done, failed, retried int) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return len(q.completed), len(q.failed), len(q.retried)
|
||||
}
|
||||
|
||||
func testConfig() config.WorkerConfig {
|
||||
return config.WorkerConfig{
|
||||
Count: 2,
|
||||
AIConcurrency: 1,
|
||||
PollInterval: 10 * time.Millisecond,
|
||||
LeaseDuration: 2 * time.Second,
|
||||
HeartbeatPeriod: 200 * time.Millisecond,
|
||||
MaxAttempts: 3,
|
||||
JobTimeout: 5 * time.Second,
|
||||
Enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
func queueJob(q *fakeQueue, id string, t domain.JobType) {
|
||||
_ = q.Enqueue(context.Background(), &domain.Job{
|
||||
ID: id, WorkspaceID: "ws", Type: t, Status: domain.JobPending,
|
||||
MaxAttempts: 3, RunAfter: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
// waitFor polls cond until it holds or the deadline passes.
|
||||
func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out after %s waiting for %s", timeout, what)
|
||||
}
|
||||
|
||||
// This is the regression test for a deadlock in which the worker waited for its
|
||||
// heartbeat goroutine to exit, while that goroutine waited for a context that
|
||||
// was only cancelled once the worker returned. Every worker wedged after one
|
||||
// job and the queue stopped draining.
|
||||
func TestWorkerCompletesJobsAndDoesNotDeadlock(t *testing.T) {
|
||||
q := newFakeQueue()
|
||||
pool := NewPool(q, testConfig(), nil)
|
||||
|
||||
var ran int
|
||||
var mu sync.Mutex
|
||||
pool.Register(domain.JobParseItem, func(ctx context.Context, j *domain.Job) error {
|
||||
mu.Lock()
|
||||
ran++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
|
||||
// More jobs than workers: if a worker wedges after its first job, the
|
||||
// remainder never complete.
|
||||
for i := 0; i < 8; i++ {
|
||||
queueJob(q, string(rune('a'+i)), domain.JobParseItem)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
pool.Start(ctx)
|
||||
defer pool.Stop(2 * time.Second)
|
||||
|
||||
waitFor(t, 5*time.Second, "all 8 jobs to complete", func() bool {
|
||||
done, _, _ := q.counts()
|
||||
return done == 8
|
||||
})
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if ran != 8 {
|
||||
t.Fatalf("handler ran %d times, want 8", ran)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerRetriesTransientFailures(t *testing.T) {
|
||||
q := newFakeQueue()
|
||||
pool := NewPool(q, testConfig(), nil)
|
||||
|
||||
pool.Register(domain.JobParseItem, func(context.Context, *domain.Job) error {
|
||||
// A network-shaped error is transient and should be retried.
|
||||
return &embeddings.StatusError{Status: 503, Op: "test", Body: "unavailable"}
|
||||
})
|
||||
queueJob(q, "j1", domain.JobParseItem)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
pool.Start(ctx)
|
||||
defer pool.Stop(2 * time.Second)
|
||||
|
||||
waitFor(t, 5*time.Second, "the job to be retried", func() bool {
|
||||
_, _, retried := q.counts()
|
||||
return retried >= 1
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkerDoesNotRetryPermanentFailures(t *testing.T) {
|
||||
q := newFakeQueue()
|
||||
pool := NewPool(q, testConfig(), nil)
|
||||
|
||||
pool.Register(domain.JobParseItem, func(context.Context, *domain.Job) error {
|
||||
// Invalid model output will never become valid by repeating the call.
|
||||
return analysis.Reject("task", analysis.ReasonBadEnum, "nope", "raw")
|
||||
})
|
||||
queueJob(q, "j1", domain.JobParseItem)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
pool.Start(ctx)
|
||||
defer pool.Stop(2 * time.Second)
|
||||
|
||||
waitFor(t, 5*time.Second, "the job to fail terminally", func() bool {
|
||||
_, failed, _ := q.counts()
|
||||
return failed == 1
|
||||
})
|
||||
if _, _, retried := q.counts(); retried != 0 {
|
||||
t.Fatalf("a permanently invalid job was retried %d times", retried)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerSurvivesAPanickingHandler(t *testing.T) {
|
||||
q := newFakeQueue()
|
||||
pool := NewPool(q, testConfig(), nil)
|
||||
|
||||
pool.Register(domain.JobParseItem, func(context.Context, *domain.Job) error {
|
||||
panic("a malformed document exploded")
|
||||
})
|
||||
pool.Register(domain.JobChunkItem, func(context.Context, *domain.Job) error { return nil })
|
||||
|
||||
queueJob(q, "boom", domain.JobParseItem)
|
||||
queueJob(q, "fine", domain.JobChunkItem)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
pool.Start(ctx)
|
||||
defer pool.Stop(2 * time.Second)
|
||||
|
||||
// The panicking job must not take the worker down: the other job still runs.
|
||||
waitFor(t, 5*time.Second, "the healthy job to complete despite a panic", func() bool {
|
||||
done, _, _ := q.counts()
|
||||
return done == 1
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunGuardedDoesNotPersistPanicValue(t *testing.T) {
|
||||
err := runGuarded(context.Background(), func(context.Context, *domain.Job) error {
|
||||
panic("bearer credential rlt_v1_secret")
|
||||
}, &domain.Job{ID: "job-1"})
|
||||
if err == nil {
|
||||
t.Fatal("panic was not converted to an error")
|
||||
}
|
||||
if strings.Contains(err.Error(), "rlt_v1_secret") {
|
||||
t.Fatalf("panic value reached the persistent job error: %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnregisteredJobTypeFailsPermanently(t *testing.T) {
|
||||
q := newFakeQueue()
|
||||
pool := NewPool(q, testConfig(), nil)
|
||||
pool.Register(domain.JobParseItem, func(context.Context, *domain.Job) error { return nil })
|
||||
|
||||
// A job whose type nothing handles must not spin in the queue.
|
||||
queueJob(q, "orphan", domain.JobExtractClaims)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
pool.Start(ctx)
|
||||
defer pool.Stop(2 * time.Second)
|
||||
|
||||
waitFor(t, 5*time.Second, "the unhandled job to fail", func() bool {
|
||||
_, failed, _ := q.counts()
|
||||
return failed == 1
|
||||
})
|
||||
}
|
||||
|
||||
func TestAIConcurrencyIsBounded(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
cfg.Count = 4
|
||||
cfg.AIConcurrency = 2
|
||||
|
||||
q := newFakeQueue()
|
||||
pool := NewPool(q, cfg, nil)
|
||||
|
||||
var mu sync.Mutex
|
||||
var inFlight, peak int
|
||||
|
||||
pool.Register(domain.JobEmbedItem, func(ctx context.Context, j *domain.Job) error {
|
||||
release, err := pool.AcquireAI(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
|
||||
mu.Lock()
|
||||
inFlight++
|
||||
if inFlight > peak {
|
||||
peak = inFlight
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
inFlight--
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
|
||||
for i := 0; i < 8; i++ {
|
||||
queueJob(q, string(rune('a'+i)), domain.JobEmbedItem)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
pool.Start(ctx)
|
||||
defer pool.Stop(3 * time.Second)
|
||||
|
||||
waitFor(t, 8*time.Second, "all embedding jobs to finish", func() bool {
|
||||
done, _, _ := q.counts()
|
||||
return done == 8
|
||||
})
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if peak > cfg.AIConcurrency {
|
||||
t.Fatalf("%d concurrent AI calls observed, limit is %d", peak, cfg.AIConcurrency)
|
||||
}
|
||||
if peak < 2 {
|
||||
t.Fatalf("AI calls never overlapped (peak %d); the semaphore may be serialising everything", peak)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopDrainsInFlightWork(t *testing.T) {
|
||||
q := newFakeQueue()
|
||||
pool := NewPool(q, testConfig(), nil)
|
||||
|
||||
started := make(chan struct{})
|
||||
var once sync.Once
|
||||
pool.Register(domain.JobParseItem, func(ctx context.Context, j *domain.Job) error {
|
||||
once.Do(func() { close(started) })
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
return nil
|
||||
})
|
||||
queueJob(q, "j1", domain.JobParseItem)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
pool.Start(ctx)
|
||||
|
||||
<-started
|
||||
pool.Stop(3 * time.Second)
|
||||
|
||||
// A job that was running when shutdown began should have been finished,
|
||||
// not abandoned.
|
||||
done, _, _ := q.counts()
|
||||
if done != 1 {
|
||||
t.Fatalf("in-flight job was not completed during shutdown (completed=%d)", done)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyDistinguishesFailureKinds(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want domain.ErrorClass
|
||||
}{
|
||||
{"nil", nil, ""},
|
||||
{"cancelled", context.Canceled, domain.ErrClassCancelled},
|
||||
{"timeout", context.DeadlineExceeded, domain.ErrClassTimeout},
|
||||
{"invalid model output", analysis.Reject("t", "r", "d", ""), domain.ErrClassInvalid},
|
||||
{"analysis disabled", analysis.ErrDisabled, domain.ErrClassPermanent},
|
||||
{"not found", domain.ErrNotFound, domain.ErrClassPermanent},
|
||||
{"invalid input", domain.ErrInvalidInput, domain.ErrClassPermanent},
|
||||
{"permanent provider error", embeddings.Permanent(errors.New("bad model")), domain.ErrClassPermanent},
|
||||
{"503 from provider", &embeddings.StatusError{Status: 503}, domain.ErrClassTransient},
|
||||
{"unknown", errors.New("something odd"), domain.ErrClassTransient},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := Classify(c.err, ctx); got != c.want {
|
||||
t.Errorf("Classify(%s) = %q, want %q", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackoffIsBoundedAndJittered(t *testing.T) {
|
||||
seen := map[time.Duration]bool{}
|
||||
for attempt := 1; attempt <= 12; attempt++ {
|
||||
for i := 0; i < 20; i++ {
|
||||
d := backoff(attempt)
|
||||
if d < 0 || d > 5*time.Minute {
|
||||
t.Fatalf("backoff(%d) = %v, outside the expected bounds", attempt, d)
|
||||
}
|
||||
seen[d] = true
|
||||
}
|
||||
}
|
||||
// Jitter means repeated calls must not all return the same value, or a
|
||||
// provider restart would be met by every worker retrying in lockstep.
|
||||
if len(seen) < 10 {
|
||||
t.Fatalf("backoff produced only %d distinct delays; jitter is not being applied", len(seen))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package parsing
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/ledongthuc/pdf"
|
||||
)
|
||||
|
||||
// PDFParser extracts text from PDFs, recording the page each block came from.
|
||||
//
|
||||
// No OCR is performed in this version. A scanned PDF yields little or no text;
|
||||
// that condition is detected and reported rather than hidden, and the original
|
||||
// file is retained so it can be reprocessed if OCR is added later.
|
||||
type PDFParser struct{}
|
||||
|
||||
// Name identifies the parser.
|
||||
func (PDFParser) Name() string { return "pdf" }
|
||||
|
||||
// Handles reports whether p can parse mime.
|
||||
func (PDFParser) Handles(mime string) bool { return isPDF(mime) }
|
||||
|
||||
// maxPDFPages bounds work on a document with an absurd page count.
|
||||
const maxPDFPages = 2000
|
||||
|
||||
// Parse reads the PDF page by page. A page that fails to decode is skipped with
|
||||
// a warning; one bad page does not discard the rest of the document.
|
||||
func (PDFParser) Parse(in Input) (doc *Document, err error) {
|
||||
// The PDF library panics on some malformed files. Recover so that a hostile
|
||||
// or corrupt document becomes a recorded parse failure, not a crashed worker.
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
// Third-party panic values may contain fragments of attacker-supplied
|
||||
// input. Return a stable diagnostic rather than storing or logging them.
|
||||
doc, err = nil, fmt.Errorf("pdf structure is malformed")
|
||||
}
|
||||
}()
|
||||
|
||||
r, perr := pdf.NewReader(bytes.NewReader(in.Data), int64(len(in.Data)))
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("open pdf: %w", perr)
|
||||
}
|
||||
|
||||
pages := r.NumPage()
|
||||
if pages < 0 {
|
||||
pages = 0
|
||||
}
|
||||
d := &Document{PageCount: pages}
|
||||
if pages > maxPDFPages {
|
||||
d.Warnings = append(d.Warnings, warn(domain.WarnTruncatedText,
|
||||
"The PDF has more pages than Relata will index; later pages were skipped.",
|
||||
fmt.Sprintf("page_count=%d indexed=%d", pages, maxPDFPages)))
|
||||
pages = maxPDFPages
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
failedPages := 0
|
||||
|
||||
for pageNo := 1; pageNo <= pages; pageNo++ {
|
||||
text, perr := extractPage(r, pageNo)
|
||||
if perr != nil {
|
||||
failedPages++
|
||||
continue
|
||||
}
|
||||
text = normaliseText(text)
|
||||
if strings.TrimSpace(text) == "" {
|
||||
continue
|
||||
}
|
||||
// Emit one block per paragraph so that page and paragraph structure both
|
||||
// survive into chunking.
|
||||
base := sb.Len()
|
||||
if base > 0 {
|
||||
sb.WriteString("\n\n")
|
||||
base = sb.Len()
|
||||
}
|
||||
sb.WriteString(text)
|
||||
section := fmt.Sprintf("page %d", pageNo)
|
||||
for _, seg := range splitParagraphs(text) {
|
||||
body := text[seg.start:seg.end]
|
||||
if strings.TrimSpace(body) == "" {
|
||||
continue
|
||||
}
|
||||
d.Blocks = append(d.Blocks, Block{
|
||||
Kind: BlockParagraph,
|
||||
Text: body,
|
||||
Start: base + seg.start,
|
||||
End: base + seg.end,
|
||||
Page: pageNo,
|
||||
Section: section,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
d.Text = sb.String()
|
||||
if failedPages > 0 {
|
||||
d.Warnings = append(d.Warnings, warn(domain.WarnParseFailed,
|
||||
"Some pages could not be decoded and were skipped. The original file is preserved.",
|
||||
fmt.Sprintf("failed_pages=%d of %d", failedPages, pages)))
|
||||
}
|
||||
if pages == 0 {
|
||||
d.Warnings = append(d.Warnings, warn(domain.WarnLowExtractableText,
|
||||
"The PDF reported no pages, so no text could be extracted.", ""))
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// extractPage pulls the text of one page, isolating panics to that page.
|
||||
func extractPage(r *pdf.Reader, pageNo int) (text string, err error) {
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
text, err = "", fmt.Errorf("page %d: %v", pageNo, rec)
|
||||
}
|
||||
}()
|
||||
p := r.Page(pageNo)
|
||||
if p.V.IsNull() {
|
||||
return "", fmt.Errorf("page %d is null", pageNo)
|
||||
}
|
||||
// GetPlainText with a nil font map uses the page's own fonts.
|
||||
s, err := p.GetPlainText(nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
@@ -0,0 +1,839 @@
|
||||
// Package search implements hybrid retrieval over stored items.
|
||||
//
|
||||
// A query runs through several independent paths — exact and lexical matching,
|
||||
// vector similarity, entity matching, and expansion along confirmed
|
||||
// relationships — whose results are fused by rank. Every result carries the
|
||||
// reasons it was returned, and the three distinct numbers involved (query
|
||||
// relevance, semantic similarity, relationship strength) are reported
|
||||
// separately rather than merged into one figure.
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/andydixon/relata/internal/config"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/entities"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
"github.com/andydixon/relata/internal/ranking"
|
||||
"github.com/andydixon/relata/internal/relationships"
|
||||
)
|
||||
|
||||
// Query is a search request. Every field is a typed value; no part of it ever
|
||||
// becomes a database operator.
|
||||
type Query struct {
|
||||
Text string
|
||||
|
||||
SourceTypes []domain.SourceType
|
||||
MIMETypes []string
|
||||
EntityIDs []string
|
||||
RelTypes []domain.RelationshipType
|
||||
CreatedFrom *time.Time
|
||||
CreatedTo *time.Time
|
||||
// ConfirmedOnly restricts relationship expansion to human-confirmed links.
|
||||
ConfirmedOnly bool
|
||||
// MinRelStrength filters expanded results by relationship strength.
|
||||
MinRelStrength float64
|
||||
// Expand enables relationship expansion. On by default for the UI.
|
||||
Expand bool
|
||||
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
// Result is one item returned for a query.
|
||||
type Result struct {
|
||||
Item *domain.Item `json:"item"`
|
||||
|
||||
// QueryRelevance is how well this item answers THIS query, 0..1 relative to
|
||||
// the best result. It is not a similarity and not a probability.
|
||||
QueryRelevance float64 `json:"query_relevance"`
|
||||
// SemanticSimilarity is the best cosine similarity of any matching chunk,
|
||||
// reported separately because similarity is not relevance.
|
||||
SemanticSimilarity float64 `json:"semantic_similarity,omitempty"`
|
||||
|
||||
Excerpt string `json:"excerpt"`
|
||||
MatchedChunks []MatchedChunk `json:"matched_chunks,omitempty"`
|
||||
MatchedEntities []MatchedEntity `json:"matched_entities,omitempty"`
|
||||
Related []RelatedItem `json:"related_items,omitempty"`
|
||||
WhyMatched []string `json:"why_matched"`
|
||||
Contributions []ranking.Contribution `json:"contributions,omitempty"`
|
||||
|
||||
// ViaRelationship is set when this item was reached by expansion rather
|
||||
// than by matching the query itself.
|
||||
ViaRelationship *RelatedItem `json:"via_relationship,omitempty"`
|
||||
|
||||
RankVersion string `json:"rank_version"`
|
||||
}
|
||||
|
||||
// MatchedChunk is a chunk that matched, with its own similarity.
|
||||
type MatchedChunk struct {
|
||||
ChunkID string `json:"chunk_id"`
|
||||
Sequence int `json:"sequence"`
|
||||
Text string `json:"text"`
|
||||
Similarity float64 `json:"semantic_similarity,omitempty"`
|
||||
Page int `json:"page,omitempty"`
|
||||
Section string `json:"section,omitempty"`
|
||||
}
|
||||
|
||||
// MatchedEntity is an entity the query matched.
|
||||
type MatchedEntity struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type domain.EntityType `json:"type"`
|
||||
}
|
||||
|
||||
// RelatedItem describes a relationship from a result to another item.
|
||||
type RelatedItem struct {
|
||||
RelationshipID string `json:"relationship_id"`
|
||||
ItemID string `json:"item_id"`
|
||||
Title string `json:"title"`
|
||||
Type domain.RelationshipType `json:"type"`
|
||||
Provenance domain.Provenance `json:"provenance"`
|
||||
// Strength is the relationship's heuristic strength, NOT query relevance.
|
||||
Strength float64 `json:"relationship_strength"`
|
||||
ScoreVersion string `json:"score_version"`
|
||||
ReviewStatus domain.ReviewStatus `json:"review_status"`
|
||||
Explanation []string `json:"explanation,omitempty"`
|
||||
Direction string `json:"direction"`
|
||||
}
|
||||
|
||||
// Response is a complete search result set.
|
||||
type Response struct {
|
||||
Query string `json:"query"`
|
||||
Results []Result `json:"results"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
TookMS int64 `json:"took_ms"`
|
||||
// Notes carries advisory messages, such as vector search being unavailable.
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// Service performs searches.
|
||||
type Service struct {
|
||||
items domain.ItemRepository
|
||||
chunks domain.ChunkRepository
|
||||
entities domain.EntityRepository
|
||||
rels domain.RelationshipRepository
|
||||
vectors domain.VectorStore
|
||||
embedder domain.Embedder
|
||||
cfg *config.Config
|
||||
metrics *observability.Metrics
|
||||
}
|
||||
|
||||
// NewService builds the search service.
|
||||
func NewService(items domain.ItemRepository, chunks domain.ChunkRepository,
|
||||
ents domain.EntityRepository, rels domain.RelationshipRepository,
|
||||
vectors domain.VectorStore, embedder domain.Embedder,
|
||||
cfg *config.Config, m *observability.Metrics) *Service {
|
||||
return &Service{
|
||||
items: items, chunks: chunks, entities: ents, rels: rels,
|
||||
vectors: vectors, embedder: embedder, cfg: cfg, metrics: m,
|
||||
}
|
||||
}
|
||||
|
||||
// recencyHalfLife controls how gently recency nudges the ordering.
|
||||
const recencyHalfLife = 180 * 24 * time.Hour
|
||||
|
||||
// Search runs the hybrid retrieval pipeline.
|
||||
func (s *Service) Search(ctx context.Context, p *domain.Principal, q Query) (*Response, error) {
|
||||
if p == nil || p.WorkspaceID == "" {
|
||||
return nil, domain.ErrWorkspaceMiss
|
||||
}
|
||||
started := time.Now()
|
||||
|
||||
limit := q.Limit
|
||||
if limit <= 0 {
|
||||
limit = s.cfg.Search.DefaultLimit
|
||||
}
|
||||
if limit > s.cfg.Search.MaxResults {
|
||||
limit = s.cfg.Search.MaxResults
|
||||
}
|
||||
offset := q.Offset
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
resp := &Response{Query: q.Text, Limit: limit, Offset: offset}
|
||||
cands := map[string]*ranking.Candidate{}
|
||||
get := func(itemID string) *ranking.Candidate {
|
||||
c := cands[itemID]
|
||||
if c == nil {
|
||||
c = &ranking.Candidate{ItemID: itemID}
|
||||
cands[itemID] = c
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
itemFilter := domain.ItemFilter{
|
||||
SourceTypes: q.SourceTypes,
|
||||
MIMETypes: q.MIMETypes,
|
||||
CreatedFrom: q.CreatedFrom,
|
||||
CreatedTo: q.CreatedTo,
|
||||
Statuses: []domain.ProcessingStatus{
|
||||
domain.StatusReady, domain.StatusPartial,
|
||||
domain.StatusAnalysing, domain.StatusRelating, domain.StatusEmbedding,
|
||||
},
|
||||
}
|
||||
|
||||
trimmed := strings.TrimSpace(q.Text)
|
||||
|
||||
// --- Path 1: lexical item search ---------------------------------------
|
||||
if trimmed != "" {
|
||||
hits, err := s.items.TextSearch(ctx, p.WorkspaceID, trimmed, itemFilter, s.cfg.Search.LexicalCandidates)
|
||||
if err != nil {
|
||||
resp.Notes = append(resp.Notes, "Keyword search was unavailable for this query.")
|
||||
observability.LoggerFrom(ctx).Warn("lexical item search failed", "error", err.Error())
|
||||
}
|
||||
for i, h := range hits {
|
||||
get(h.ItemID).Add(ranking.SourceLexical, i+1,
|
||||
"The item's text contains words from your query.")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path 2: lexical chunk search, for excerpts -------------------------
|
||||
chunkHits := map[string][]domain.ChunkHit{}
|
||||
if trimmed != "" {
|
||||
hits, err := s.chunks.TextSearch(ctx, p.WorkspaceID, trimmed, s.cfg.Search.LexicalCandidates)
|
||||
if err == nil {
|
||||
for i, h := range hits {
|
||||
chunkHits[h.ItemID] = append(chunkHits[h.ItemID], h)
|
||||
c := get(h.ItemID)
|
||||
c.MatchedChunks = appendUnique(c.MatchedChunks, h.ChunkID, 5)
|
||||
if i < 20 {
|
||||
c.Add(ranking.SourceLexical, i+1,
|
||||
"A section of this item contains words from your query.")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path 3: vector similarity -----------------------------------------
|
||||
vectorSims := map[string]float64{}
|
||||
chunkSims := map[string]float64{}
|
||||
if trimmed != "" {
|
||||
vecs, err := s.embedder.Embed(ctx, []string{trimmed})
|
||||
if err != nil || len(vecs) == 0 {
|
||||
resp.Notes = append(resp.Notes,
|
||||
"Meaning-based search was unavailable, so these results are from keyword matching only.")
|
||||
observability.LoggerFrom(ctx).Warn("could not embed the query", "error", errString(err))
|
||||
} else {
|
||||
matches, err := s.vectors.Search(ctx, vecs[0], s.cfg.Search.VectorCandidates, domain.VectorFilter{
|
||||
WorkspaceID: p.WorkspaceID,
|
||||
SourceTypes: q.SourceTypes,
|
||||
EmbeddingModel: s.embedder.ModelID(),
|
||||
CreatedFrom: q.CreatedFrom,
|
||||
CreatedTo: q.CreatedTo,
|
||||
})
|
||||
if err != nil {
|
||||
resp.Notes = append(resp.Notes, "Meaning-based search was unavailable for this query.")
|
||||
observability.LoggerFrom(ctx).Warn("vector search failed", "error", err.Error())
|
||||
}
|
||||
for i, m := range matches {
|
||||
c := get(m.ItemID)
|
||||
c.Add(ranking.SourceVector, i+1,
|
||||
fmt.Sprintf("A section of this item is close in meaning to your query (similarity %.2f).", m.Score))
|
||||
if m.Score > c.BestSimilarity {
|
||||
c.BestSimilarity = m.Score
|
||||
}
|
||||
if m.Score > vectorSims[m.ItemID] {
|
||||
vectorSims[m.ItemID] = m.Score
|
||||
}
|
||||
chunkSims[m.ChunkID] = m.Score
|
||||
c.MatchedChunks = appendUnique(c.MatchedChunks, m.ChunkID, 5)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path 4: entity matching -------------------------------------------
|
||||
matchedEntities := map[string][]*domain.Entity{}
|
||||
if err := s.matchEntities(ctx, p.WorkspaceID, q, trimmed, get, matchedEntities); err != nil {
|
||||
observability.LoggerFrom(ctx).Warn("entity matching failed", "error", err.Error())
|
||||
}
|
||||
|
||||
// An empty query with filters is a legitimate browse request.
|
||||
if trimmed == "" && len(cands) == 0 {
|
||||
items, total, err := s.items.List(ctx, p.WorkspaceID, itemFilter,
|
||||
domain.Page{Limit: limit, Offset: offset})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, it := range items {
|
||||
c := get(it.ID)
|
||||
c.CreatedAt = it.CreatedAt
|
||||
c.Add(ranking.SourceLexical, i+1, "Matched your filters.")
|
||||
}
|
||||
resp.Total = int(total)
|
||||
}
|
||||
|
||||
if len(cands) == 0 {
|
||||
resp.TookMS = time.Since(started).Milliseconds()
|
||||
s.count("hybrid", "empty")
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Load the candidate items, applying the workspace and filter constraints
|
||||
// that the vector store cannot express.
|
||||
ids := make([]string, 0, len(cands))
|
||||
for id := range cands {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
items, err := s.items.GetMany(ctx, p.WorkspaceID, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byID := make(map[string]*domain.Item, len(items))
|
||||
for _, it := range items {
|
||||
if it.Deleted || !matchesFilter(it, itemFilter, q) {
|
||||
delete(cands, it.ID)
|
||||
continue
|
||||
}
|
||||
byID[it.ID] = it
|
||||
cands[it.ID].CreatedAt = it.CreatedAt
|
||||
}
|
||||
// Drop candidates whose items were not returned at all: they are deleted,
|
||||
// in another workspace, or otherwise not visible to this principal.
|
||||
for id := range cands {
|
||||
if byID[id] == nil {
|
||||
delete(cands, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Title matching is cheap and a strong signal a user expects to work.
|
||||
if trimmed != "" {
|
||||
lower := strings.ToLower(trimmed)
|
||||
for id, it := range byID {
|
||||
if strings.Contains(strings.ToLower(it.Title), lower) {
|
||||
cands[id].AddScaled(ranking.SourceTitle, 1,
|
||||
fmt.Sprintf("The title %q contains your query.", it.Title))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path 5: relationship expansion ------------------------------------
|
||||
expansions := map[string]*RelatedItem{}
|
||||
relsByItem := map[string][]*domain.Relationship{}
|
||||
if q.Expand {
|
||||
s.expand(ctx, p.WorkspaceID, q, cands, byID, get, expansions, relsByItem)
|
||||
|
||||
// Newly-expanded items still need their records.
|
||||
var missing []string
|
||||
for id := range cands {
|
||||
if byID[id] == nil {
|
||||
missing = append(missing, id)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
more, err := s.items.GetMany(ctx, p.WorkspaceID, missing)
|
||||
if err == nil {
|
||||
for _, it := range more {
|
||||
if it.Deleted || !matchesFilter(it, itemFilter, q) {
|
||||
delete(cands, it.ID)
|
||||
continue
|
||||
}
|
||||
byID[it.ID] = it
|
||||
cands[it.ID].CreatedAt = it.CreatedAt
|
||||
}
|
||||
}
|
||||
for _, id := range missing {
|
||||
if byID[id] == nil {
|
||||
delete(cands, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Fuse and order -----------------------------------------------------
|
||||
list := make([]*ranking.Candidate, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
list = append(list, c)
|
||||
}
|
||||
ranked := ranking.Rank(list, time.Now(), recencyHalfLife)
|
||||
|
||||
if resp.Total == 0 {
|
||||
resp.Total = len(ranked)
|
||||
}
|
||||
|
||||
// Apply paging after ranking, so a page boundary does not change ordering.
|
||||
start := offset
|
||||
if start > len(ranked) {
|
||||
start = len(ranked)
|
||||
}
|
||||
end := start + limit
|
||||
if end > len(ranked) {
|
||||
end = len(ranked)
|
||||
}
|
||||
page := ranked[start:end]
|
||||
|
||||
// Fetch the chunks needed for excerpts, in one query.
|
||||
var wanted []string
|
||||
for _, c := range page {
|
||||
wanted = append(wanted, c.MatchedChunks...)
|
||||
}
|
||||
chunkByID := map[string]*domain.Chunk{}
|
||||
if len(wanted) > 0 {
|
||||
got, err := s.chunks.GetMany(ctx, p.WorkspaceID, wanted)
|
||||
if err == nil {
|
||||
for _, ch := range got {
|
||||
chunkByID[ch.ID] = ch
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range page {
|
||||
it := byID[c.ItemID]
|
||||
if it == nil {
|
||||
continue
|
||||
}
|
||||
r := Result{
|
||||
Item: it,
|
||||
QueryRelevance: c.Relevance,
|
||||
SemanticSimilarity: c.BestSimilarity,
|
||||
WhyMatched: ranking.Explain(c),
|
||||
Contributions: c.Contributions,
|
||||
RankVersion: ranking.RankVersion,
|
||||
}
|
||||
|
||||
for _, cid := range c.MatchedChunks {
|
||||
ch := chunkByID[cid]
|
||||
if ch == nil {
|
||||
continue
|
||||
}
|
||||
r.MatchedChunks = append(r.MatchedChunks, MatchedChunk{
|
||||
ChunkID: ch.ID,
|
||||
Sequence: ch.Sequence,
|
||||
Text: ch.Text,
|
||||
Similarity: chunkSims[ch.ID],
|
||||
Page: ch.Page,
|
||||
Section: ch.Section,
|
||||
})
|
||||
}
|
||||
sort.SliceStable(r.MatchedChunks, func(i, j int) bool {
|
||||
return r.MatchedChunks[i].Similarity > r.MatchedChunks[j].Similarity
|
||||
})
|
||||
|
||||
r.Excerpt = buildExcerpt(it, r.MatchedChunks, chunkHits[it.ID], trimmed)
|
||||
|
||||
for _, e := range matchedEntities[it.ID] {
|
||||
r.MatchedEntities = append(r.MatchedEntities, MatchedEntity{
|
||||
ID: e.ID, Name: e.CanonicalName, Type: e.Type,
|
||||
})
|
||||
}
|
||||
|
||||
if via, ok := expansions[it.ID]; ok {
|
||||
r.ViaRelationship = via
|
||||
}
|
||||
r.Related = s.relatedFor(it.ID, relsByItem[it.ID], byID)
|
||||
|
||||
resp.Results = append(resp.Results, r)
|
||||
}
|
||||
|
||||
resp.TookMS = time.Since(started).Milliseconds()
|
||||
s.count("hybrid", "success")
|
||||
if s.metrics != nil {
|
||||
s.metrics.SearchLatency.Observe(time.Since(started).Seconds())
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *Service) count(mode, outcome string) {
|
||||
if s.metrics != nil {
|
||||
s.metrics.SearchOps.WithLabelValues(mode, outcome).Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// matchEntities finds entities the query names and adds the items citing them.
|
||||
func (s *Service) matchEntities(ctx context.Context, workspaceID string, q Query, text string,
|
||||
get func(string) *ranking.Candidate, out map[string][]*domain.Entity) error {
|
||||
|
||||
var found []*domain.Entity
|
||||
|
||||
// Explicit entity filters are authoritative.
|
||||
if len(q.EntityIDs) > 0 {
|
||||
ents, err := s.entities.GetMany(ctx, workspaceID, q.EntityIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found = append(found, ents...)
|
||||
}
|
||||
|
||||
// Otherwise look for entity names appearing in the query text.
|
||||
if text != "" {
|
||||
keys := candidateEntityKeys(text)
|
||||
if len(keys) > 0 {
|
||||
ents, err := s.entities.FindByNormalised(ctx, workspaceID, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found = append(found, ents...)
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, e := range found {
|
||||
if seen[e.ID] {
|
||||
continue
|
||||
}
|
||||
seen[e.ID] = true
|
||||
for _, itemID := range e.ItemIDs {
|
||||
c := get(itemID)
|
||||
c.AddScaled(ranking.SourceEntity, 1,
|
||||
fmt.Sprintf("This item refers to %s (%s), which your query names.", e.CanonicalName, e.Type))
|
||||
c.MatchedEntities = appendUnique(c.MatchedEntities, e.ID, 8)
|
||||
out[itemID] = append(out[itemID], e)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// candidateEntityKeys generates normalised keys for word sequences in the
|
||||
// query, so "Example Ltd" matches the stored organisation.
|
||||
//
|
||||
// Only short n-grams are tried, and each becomes a value in an indexed $in
|
||||
// lookup — never a pattern or an operator.
|
||||
func candidateEntityKeys(text string) []string {
|
||||
words := strings.FieldsFunc(text, func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '\'' && r != '&'
|
||||
})
|
||||
if len(words) == 0 || len(words) > 40 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var keys []string
|
||||
seen := map[string]bool{}
|
||||
addKey := func(phrase string) {
|
||||
norm := entities.Normalise(phrase)
|
||||
if norm == "" || len(norm) < 3 {
|
||||
return
|
||||
}
|
||||
for _, t := range domain.ValidEntityTypes {
|
||||
k := string(t) + ":" + norm
|
||||
if !seen[k] {
|
||||
seen[k] = true
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unigrams through trigrams.
|
||||
for n := 1; n <= 3; n++ {
|
||||
for i := 0; i+n <= len(words); i++ {
|
||||
if len(keys) > 600 {
|
||||
return keys
|
||||
}
|
||||
addKey(strings.Join(words[i:i+n], " "))
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// expand pulls in items connected to the current matches by a relationship.
|
||||
//
|
||||
// Expanded items are labelled with the relationship that brought them in, and
|
||||
// their contribution is weighted well below a direct match: being related to a
|
||||
// hit is weaker evidence than being one. Rejected relationships never expand.
|
||||
func (s *Service) expand(ctx context.Context, workspaceID string, q Query,
|
||||
cands map[string]*ranking.Candidate, byID map[string]*domain.Item,
|
||||
get func(string) *ranking.Candidate, expansions map[string]*RelatedItem,
|
||||
relsByItem map[string][]*domain.Relationship) {
|
||||
|
||||
// Expand only from the strongest current matches.
|
||||
seeds := make([]*ranking.Candidate, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
seeds = append(seeds, c)
|
||||
}
|
||||
sort.SliceStable(seeds, func(i, j int) bool {
|
||||
return len(seeds[i].Contributions) > len(seeds[j].Contributions)
|
||||
})
|
||||
max := s.cfg.Search.MaxExpandedItems
|
||||
if max <= 0 {
|
||||
max = 20
|
||||
}
|
||||
if len(seeds) > max {
|
||||
seeds = seeds[:max]
|
||||
}
|
||||
|
||||
seedIDs := make([]string, 0, len(seeds))
|
||||
for _, c := range seeds {
|
||||
seedIDs = append(seedIDs, c.ItemID)
|
||||
}
|
||||
|
||||
minScore := q.MinRelStrength
|
||||
if minScore <= 0 {
|
||||
minScore = s.cfg.Search.ExpansionMinScore
|
||||
}
|
||||
|
||||
rels, err := s.rels.ListForItems(ctx, workspaceID, seedIDs, minScore)
|
||||
if err != nil {
|
||||
observability.LoggerFrom(ctx).Warn("relationship expansion failed", "error", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
isSeed := map[string]bool{}
|
||||
for _, id := range seedIDs {
|
||||
isSeed[id] = true
|
||||
}
|
||||
|
||||
added := 0
|
||||
for _, rel := range rels {
|
||||
if q.ConfirmedOnly && rel.ReviewStatus != domain.ReviewConfirmed {
|
||||
continue
|
||||
}
|
||||
if len(q.RelTypes) > 0 && !containsRelType(q.RelTypes, rel.Type) {
|
||||
continue
|
||||
}
|
||||
|
||||
relsByItem[rel.SourceID] = append(relsByItem[rel.SourceID], rel)
|
||||
relsByItem[rel.TargetID] = append(relsByItem[rel.TargetID], rel)
|
||||
|
||||
// Identify the far end of the relationship.
|
||||
var otherID, direction string
|
||||
switch {
|
||||
case isSeed[rel.SourceID] && !isSeed[rel.TargetID]:
|
||||
otherID, direction = rel.TargetID, "outgoing"
|
||||
case isSeed[rel.TargetID] && !isSeed[rel.SourceID]:
|
||||
otherID, direction = rel.SourceID, "incoming"
|
||||
default:
|
||||
continue // both ends already matched, or neither did
|
||||
}
|
||||
if added >= max {
|
||||
break
|
||||
}
|
||||
added++
|
||||
|
||||
c := get(otherID)
|
||||
c.ExpandedFrom = rel.SourceID
|
||||
label := "related to"
|
||||
if rel.ReviewStatus == domain.ReviewConfirmed {
|
||||
label = "confirmed as related to"
|
||||
c.AddScaled(ranking.SourceConfirmedRel, 1,
|
||||
"A person has confirmed this item's relationship to a matching item.")
|
||||
}
|
||||
c.AddScaled(ranking.SourceExpansion, rel.Score, fmt.Sprintf(
|
||||
"Not a direct match: this item is %s a matching item by a %s relationship (strength %.2f, %s).",
|
||||
label, rel.Type, rel.Score, provenanceLabel(rel.Provenance)))
|
||||
|
||||
expansions[otherID] = &RelatedItem{
|
||||
RelationshipID: rel.ID,
|
||||
ItemID: otherID,
|
||||
Type: rel.Type,
|
||||
Provenance: rel.Provenance,
|
||||
Strength: rel.Score,
|
||||
ScoreVersion: rel.ScoreVersion,
|
||||
ReviewStatus: rel.ReviewStatus,
|
||||
Direction: direction,
|
||||
Explanation: relationships.Explain(rel),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// relatedFor renders an item's relationships for display.
|
||||
func (s *Service) relatedFor(itemID string, rels []*domain.Relationship, byID map[string]*domain.Item) []RelatedItem {
|
||||
if len(rels) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
out := make([]RelatedItem, 0, len(rels))
|
||||
|
||||
for _, rel := range rels {
|
||||
if seen[rel.ID] {
|
||||
continue
|
||||
}
|
||||
seen[rel.ID] = true
|
||||
|
||||
otherID, direction := rel.TargetID, "outgoing"
|
||||
if rel.TargetID == itemID {
|
||||
otherID, direction = rel.SourceID, "incoming"
|
||||
}
|
||||
title := ""
|
||||
if it := byID[otherID]; it != nil {
|
||||
title = it.Title
|
||||
}
|
||||
out = append(out, RelatedItem{
|
||||
RelationshipID: rel.ID,
|
||||
ItemID: otherID,
|
||||
Title: title,
|
||||
Type: rel.Type,
|
||||
Provenance: rel.Provenance,
|
||||
Strength: rel.Score,
|
||||
ScoreVersion: rel.ScoreVersion,
|
||||
ReviewStatus: rel.ReviewStatus,
|
||||
Direction: direction,
|
||||
Explanation: relationships.Explain(rel),
|
||||
})
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Strength > out[j].Strength })
|
||||
if len(out) > 8 {
|
||||
out = out[:8]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildExcerpt picks the most useful snippet to show for a result.
|
||||
func buildExcerpt(item *domain.Item, matched []MatchedChunk, lexical []domain.ChunkHit, query string) string {
|
||||
const width = 320
|
||||
|
||||
// Prefer a chunk that lexically contains a query term, so the excerpt shows
|
||||
// the user their own words where possible.
|
||||
if query != "" {
|
||||
terms := strings.Fields(strings.ToLower(query))
|
||||
for _, mc := range matched {
|
||||
if snippet := findTermWindow(mc.Text, terms, width); snippet != "" {
|
||||
return snippet
|
||||
}
|
||||
}
|
||||
for _, lh := range lexical {
|
||||
if snippet := findTermWindow(lh.Text, terms, width); snippet != "" {
|
||||
return snippet
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(matched) > 0 {
|
||||
return truncateWords(matched[0].Text, width)
|
||||
}
|
||||
if len(lexical) > 0 {
|
||||
return truncateWords(lexical[0].Text, width)
|
||||
}
|
||||
return truncateWords(item.ExtractedText, width)
|
||||
}
|
||||
|
||||
// findTermWindow returns a window of text around the first matching term.
|
||||
func findTermWindow(text string, terms []string, width int) string {
|
||||
lower := strings.ToLower(text)
|
||||
best := -1
|
||||
for _, t := range terms {
|
||||
if len(t) < 3 {
|
||||
continue
|
||||
}
|
||||
if i := strings.Index(lower, t); i >= 0 && (best < 0 || i < best) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return ""
|
||||
}
|
||||
start := best - width/3
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
for start > 0 && text[start]&0xC0 == 0x80 {
|
||||
start--
|
||||
}
|
||||
end := start + width
|
||||
if end > len(text) {
|
||||
end = len(text)
|
||||
}
|
||||
for end < len(text) && text[end]&0xC0 == 0x80 {
|
||||
end++
|
||||
}
|
||||
out := strings.TrimSpace(collapse(text[start:end]))
|
||||
if start > 0 {
|
||||
out = "…" + out
|
||||
}
|
||||
if end < len(text) {
|
||||
out += "…"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func truncateWords(s string, n int) string {
|
||||
s = collapse(strings.TrimSpace(s))
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := n
|
||||
for cut > 0 && s[cut]&0xC0 == 0x80 {
|
||||
cut--
|
||||
}
|
||||
if i := strings.LastIndexByte(s[:cut], ' '); i > n/2 {
|
||||
cut = i
|
||||
}
|
||||
return s[:cut] + "…"
|
||||
}
|
||||
|
||||
func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
// matchesFilter applies the filters the storage query could not express.
|
||||
func matchesFilter(it *domain.Item, f domain.ItemFilter, q Query) bool {
|
||||
if len(f.SourceTypes) > 0 && !containsSourceType(f.SourceTypes, it.SourceType) {
|
||||
return false
|
||||
}
|
||||
if len(f.MIMETypes) > 0 && !containsString(f.MIMETypes, it.MIMEType) {
|
||||
return false
|
||||
}
|
||||
if f.CreatedFrom != nil && it.CreatedAt.Before(*f.CreatedFrom) {
|
||||
return false
|
||||
}
|
||||
if f.CreatedTo != nil && it.CreatedAt.After(*f.CreatedTo) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func containsSourceType(list []domain.SourceType, v domain.SourceType) bool {
|
||||
for _, e := range list {
|
||||
if e == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsString(list []string, v string) bool {
|
||||
for _, e := range list {
|
||||
if e == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsRelType(list []domain.RelationshipType, v domain.RelationshipType) bool {
|
||||
for _, e := range list {
|
||||
if e == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func appendUnique(list []string, v string, max int) []string {
|
||||
if v == "" || len(list) >= max {
|
||||
return list
|
||||
}
|
||||
for _, e := range list {
|
||||
if e == v {
|
||||
return list
|
||||
}
|
||||
}
|
||||
return append(list, v)
|
||||
}
|
||||
|
||||
func provenanceLabel(p domain.Provenance) string {
|
||||
switch p {
|
||||
case domain.ProvExplicit:
|
||||
return "stated explicitly"
|
||||
case domain.ProvDeterministic:
|
||||
return "derived from matching evidence"
|
||||
case domain.ProvModelInferred:
|
||||
return "suggested by analysis, not yet reviewed"
|
||||
case domain.ProvUserConfirmed:
|
||||
return "confirmed by a person"
|
||||
case domain.ProvUserRejected:
|
||||
return "rejected by a person"
|
||||
}
|
||||
return string(p)
|
||||
}
|
||||
|
||||
func errString(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Relata's only client-side script.
|
||||
//
|
||||
// It does two small things that plain HTML cannot: confirm destructive actions,
|
||||
// and let HTMX requests carry the CSRF token. Everything else is server
|
||||
// rendered. There is no inline script anywhere, which is what allows the
|
||||
// Content-Security-Policy to forbid it outright.
|
||||
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
// Confirm destructive submissions. The prompt text lives in the markup so
|
||||
// that it is translatable and reviewable alongside the form it guards.
|
||||
document.addEventListener("submit", function (event) {
|
||||
var form = event.target;
|
||||
if (!(form instanceof HTMLFormElement)) return;
|
||||
var message = form.getAttribute("data-confirm");
|
||||
if (message && !window.confirm(message)) {
|
||||
event.preventDefault();
|
||||
}
|
||||
});
|
||||
|
||||
// HTMX issues requests outside the normal form flow, so the CSRF token has to
|
||||
// be attached explicitly. It is read from the form field the server rendered,
|
||||
// never from a cookie.
|
||||
// Copy-to-clipboard for a freshly issued API token. Delegated, so it works
|
||||
// for content rendered after load. The value is read from the DOM only; it is
|
||||
// never sent anywhere.
|
||||
document.addEventListener("click", function (event) {
|
||||
var trigger = event.target.closest("[data-copy]");
|
||||
if (!trigger) {
|
||||
return;
|
||||
}
|
||||
var source = document.getElementById(trigger.getAttribute("data-copy"));
|
||||
if (!source) {
|
||||
console.warn("Copy control has no matching source element");
|
||||
return;
|
||||
}
|
||||
var original = trigger.textContent;
|
||||
function showStatus(message) {
|
||||
trigger.textContent = message;
|
||||
window.setTimeout(function () {
|
||||
trigger.textContent = original;
|
||||
}, 2000);
|
||||
}
|
||||
if (!navigator.clipboard) {
|
||||
console.warn("Clipboard API is unavailable in this browser context");
|
||||
showStatus("Copy unavailable");
|
||||
return;
|
||||
}
|
||||
navigator.clipboard.writeText(source.textContent.trim()).then(function () {
|
||||
showStatus("Copied");
|
||||
}).catch(function () {
|
||||
// Do not log the clipboard value or the full error object: the source is
|
||||
// a bearer credential and browser errors add no actionable context here.
|
||||
console.warn("Clipboard write was refused");
|
||||
showStatus("Copy failed");
|
||||
});
|
||||
});
|
||||
|
||||
document.body.addEventListener("htmx:configRequest", function (event) {
|
||||
var field = document.querySelector('input[name="csrf_token"]');
|
||||
if (field && field.value) {
|
||||
event.detail.headers["X-CSRF-Token"] = field.value;
|
||||
}
|
||||
});
|
||||
})();
|
||||
@@ -0,0 +1,114 @@
|
||||
{{define "content"}}
|
||||
<h1>API tokens</h1>
|
||||
<p class="lede">
|
||||
Non-interactive credentials for <code>/api/v1</code>. They authenticate programs, not people.
|
||||
</p>
|
||||
|
||||
{{if .NewSecret}}
|
||||
<section class="panel panel-highlight">
|
||||
<h2>Your new token{{if .NewName}}: {{.NewName}}{{end}}</h2>
|
||||
<p class="callout">
|
||||
<strong>This is the only time it will be shown.</strong> Relata stores only a hash of it and
|
||||
cannot show it again. Copy it now.
|
||||
</p>
|
||||
<pre class="secret"><code id="new-token">{{.NewSecret}}</code></pre>
|
||||
<p>
|
||||
<button type="button" class="btn" data-copy="new-token">Copy to clipboard</button>
|
||||
</p>
|
||||
<p class="muted">
|
||||
The value exists only in this response and is not included in the page URL. Copy it before
|
||||
navigating away; refreshing the page will not recover the same secret.
|
||||
</p>
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
<section class="panel">
|
||||
<h2>Create a token</h2>
|
||||
<form method="post" action="/settings/tokens" class="stack">
|
||||
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
|
||||
|
||||
<div class="field">
|
||||
<label for="name">Name</label>
|
||||
<input type="text" id="name" name="name" maxlength="80" required
|
||||
placeholder="e.g. nightly ingestion script">
|
||||
<p class="hint">So you can tell your tokens apart later.</p>
|
||||
</div>
|
||||
|
||||
<fieldset class="field">
|
||||
<legend>Scopes</legend>
|
||||
{{range .Scopes}}
|
||||
<label class="check">
|
||||
<input type="checkbox" name="scopes" value="{{.Scope}}" {{if not .Available}}disabled{{end}}>
|
||||
<span><strong>{{.Label}}</strong> — {{.Description}}
|
||||
{{if not .Available}}<em class="muted">(not available to your role)</em>{{end}}</span>
|
||||
</label>
|
||||
{{end}}
|
||||
<p class="hint">Grant only what the program needs. Scopes narrow permission; they never widen it.</p>
|
||||
</fieldset>
|
||||
|
||||
<div class="field">
|
||||
<label for="ttl_days">Expires after (days)</label>
|
||||
<input type="number" id="ttl_days" name="ttl_days" min="1" max="{{.MaxTTL}}" value="{{.DefaultTTL}}">
|
||||
<p class="hint">Maximum {{.MaxTTL}} days. A token stops working at its expiry with no further action.</p>
|
||||
</div>
|
||||
|
||||
<div class="actions">
|
||||
<button type="submit" class="btn btn-primary">Create token</button>
|
||||
<a class="btn btn-quiet" href="/settings">Back to settings</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Your tokens</h2>
|
||||
{{if .Tokens}}
|
||||
<table class="data">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th><th>Prefix</th><th>Scopes</th>
|
||||
<th>Created</th><th>Expires</th><th>Last used</th><th>Status</th><th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{range .Tokens}}
|
||||
<tr {{if or .Revoked .Expired}}class="row-inactive"{{end}}>
|
||||
<td>{{.Token.Name}}</td>
|
||||
<td><code>{{.Token.Prefix}}…</code></td>
|
||||
<td>{{.Scopes}}</td>
|
||||
<td>{{.Token.CreatedAt.Format "2 Jan 2006"}}</td>
|
||||
<td>{{if .Token.ExpiresAt.IsZero}}never{{else}}{{.Token.ExpiresAt.Format "2 Jan 2006"}}{{end}}</td>
|
||||
<td>{{if .Token.LastUsedAt}}{{.Token.LastUsedAt.Format "2 Jan 2006 15:04"}}{{else}}never{{end}}</td>
|
||||
<td><span class="status status-{{if .Revoked}}rejected{{else if .Expired}}muted{{else}}confirmed{{end}}">{{.Status}}</span></td>
|
||||
<td>
|
||||
{{if not .Revoked}}
|
||||
<form method="post" action="/settings/tokens/{{.Token.ID}}/revoke"
|
||||
data-confirm="Revoke this token? Any program using it will stop working immediately.">
|
||||
<input type="hidden" name="csrf_token" value="{{$.CSRFToken}}">
|
||||
<button type="submit" class="btn btn-quiet btn-danger">Revoke</button>
|
||||
</form>
|
||||
{{end}}
|
||||
</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
<p class="muted">
|
||||
Revoked and expired tokens stay listed. Hiding them would make this page a worse record of
|
||||
what existed than the audit log.
|
||||
</p>
|
||||
{{else}}
|
||||
<p class="empty">You have no API tokens. Create one above to use the API.</p>
|
||||
{{end}}
|
||||
</section>
|
||||
|
||||
<section class="panel panel-quiet">
|
||||
<h2>Using a token</h2>
|
||||
<pre><code>curl -H "Authorization: Bearer $RELATA_TOKEN" \
|
||||
'{{"https://your-host/api/v1/search?q=delivery"}}'</code></pre>
|
||||
<p class="muted">
|
||||
Send it in the <code>Authorization</code> header only — never in a query parameter, where it
|
||||
would end up in access logs, browser history and referrer headers.
|
||||
<a href="/api">Full API description</a>.
|
||||
</p>
|
||||
</section>
|
||||
{{end}}
|
||||
@@ -0,0 +1,188 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/relata/internal/api"
|
||||
"github.com/andydixon/relata/internal/auth"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
)
|
||||
|
||||
// tokenRow is one token as shown on the page.
|
||||
type tokenRow struct {
|
||||
Token *domain.APIToken
|
||||
Status string
|
||||
Expired bool
|
||||
Revoked bool
|
||||
Scopes string
|
||||
}
|
||||
|
||||
// scopeChoice is one selectable scope on the creation form.
|
||||
type scopeChoice struct {
|
||||
Scope domain.TokenScope
|
||||
Label string
|
||||
Description string
|
||||
Available bool
|
||||
}
|
||||
|
||||
// TokensPage lists the caller's API tokens and offers the creation form.
|
||||
func (h *Handlers) TokensPage(w http.ResponseWriter, r *http.Request) {
|
||||
p := api.PrincipalFrom(r.Context())
|
||||
if p == nil {
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
h.renderTokensPage(w, r, p, "", "", http.StatusOK)
|
||||
}
|
||||
|
||||
// renderTokensPage renders the token list and, after creation, the one-time
|
||||
// secret directly in the POST response. A bearer credential must never travel
|
||||
// through a redirect URL, where it would enter browser history and proxy logs.
|
||||
func (h *Handlers) renderTokensPage(w http.ResponseWriter, r *http.Request, p *domain.Principal,
|
||||
newSecret, newName string, status int) {
|
||||
toks, err := h.d.Tokens.List(r.Context(), p)
|
||||
if err != nil {
|
||||
if newSecret == "" {
|
||||
h.renderError(w, r, http.StatusInternalServerError, "Could not list tokens",
|
||||
"The token list could not be read. Please try again.")
|
||||
return
|
||||
}
|
||||
// Creation has already succeeded and the secret is unrecoverable. Show it
|
||||
// even if the follow-up list refresh failed rather than losing it forever.
|
||||
toks = nil
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
rows := make([]tokenRow, 0, len(toks))
|
||||
for _, t := range toks {
|
||||
row := tokenRow{Token: t, Status: "Active", Scopes: joinScopes(t.Scopes)}
|
||||
switch {
|
||||
case t.RevokedAt != nil:
|
||||
row.Status, row.Revoked = "Revoked", true
|
||||
case !t.ExpiresAt.IsZero() && !now.Before(t.ExpiresAt):
|
||||
row.Status, row.Expired = "Expired", true
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
|
||||
data := struct {
|
||||
base
|
||||
Tokens []tokenRow
|
||||
Scopes []scopeChoice
|
||||
// NewSecret is set only immediately after creation.
|
||||
NewSecret string
|
||||
NewName string
|
||||
DefaultTTL int
|
||||
MaxTTL int
|
||||
}{
|
||||
base: h.newBase(r, "API tokens", "settings"),
|
||||
Tokens: rows,
|
||||
Scopes: scopeChoicesFor(p.Role),
|
||||
NewSecret: newSecret,
|
||||
NewName: newName,
|
||||
DefaultTTL: int(h.d.Cfg.Auth.OAuth.TokenDefaultTTL.Hours() / 24),
|
||||
MaxTTL: int(h.d.Cfg.Auth.OAuth.TokenMaxTTL.Hours() / 24),
|
||||
}
|
||||
if err != nil {
|
||||
data.Error = "The token was created, but the token list could not be refreshed. Copy the new token now."
|
||||
}
|
||||
h.d.Renderer.Render(w, r, "tokens.html", status, data)
|
||||
}
|
||||
|
||||
// CreateToken issues a token from the interface.
|
||||
func (h *Handlers) CreateToken(w http.ResponseWriter, r *http.Request) {
|
||||
p := api.PrincipalFrom(r.Context())
|
||||
if p == nil {
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Redirect(w, r, "/settings/tokens?warn=token_invalid", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
scopes := make([]domain.TokenScope, 0, 3)
|
||||
for _, raw := range r.PostForm["scopes"] {
|
||||
sc := domain.TokenScope(strings.TrimSpace(raw))
|
||||
if sc.Valid() {
|
||||
scopes = append(scopes, sc)
|
||||
}
|
||||
}
|
||||
|
||||
days, _ := strconv.Atoi(r.PostFormValue("ttl_days"))
|
||||
ttl := time.Duration(days) * 24 * time.Hour
|
||||
|
||||
issued, err := h.d.Tokens.Create(r.Context(), p, auth.CreateRequest{
|
||||
Name: r.PostFormValue("name"),
|
||||
Scopes: scopes,
|
||||
TTL: ttl,
|
||||
})
|
||||
if err != nil {
|
||||
observability.LoggerFrom(r.Context()).Warn("token creation refused", "error", err.Error())
|
||||
http.Redirect(w, r, "/settings/tokens?warn=token_invalid", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
h.d.Audit.Record(r.Context(), p, domain.AuditTokenCreated, "api_token", issued.Token.ID, "success",
|
||||
map[string]string{"name": issued.Token.Name, "prefix": issued.Token.Prefix})
|
||||
|
||||
// Render the one-time secret in this response. Redirect URLs must never carry
|
||||
// bearer credentials: URLs are retained by browsers, proxies and access logs.
|
||||
h.renderTokensPage(w, r, p, issued.Secret, issued.Token.Name, http.StatusCreated)
|
||||
}
|
||||
|
||||
// RevokeToken makes a token unusable from the interface.
|
||||
func (h *Handlers) RevokeToken(w http.ResponseWriter, r *http.Request) {
|
||||
p := api.PrincipalFrom(r.Context())
|
||||
if p == nil {
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
id := r.PathValue("id")
|
||||
if err := h.d.Tokens.Revoke(r.Context(), p, id); err != nil {
|
||||
http.Redirect(w, r, "/settings/tokens?warn=token_invalid", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
h.d.Audit.Record(r.Context(), p, domain.AuditTokenRevoked, "api_token", id, "success", nil)
|
||||
http.Redirect(w, r, "/settings/tokens?ok=token_revoked", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// scopeChoicesFor describes the scopes, marking those the role cannot grant.
|
||||
//
|
||||
// An unavailable scope is shown and disabled rather than hidden, so a viewer
|
||||
// can see that ingestion exists and why it is not offered to them.
|
||||
func scopeChoicesFor(role domain.Role) []scopeChoice {
|
||||
return []scopeChoice{
|
||||
{
|
||||
Scope: domain.ScopeQuery, Label: "Query",
|
||||
Description: "Search, generate answers, and read stored items, entities and relationships.",
|
||||
Available: domain.RoleGrants(role, domain.ScopeQuery),
|
||||
},
|
||||
{
|
||||
Scope: domain.ScopeIngest, Label: "Ingest",
|
||||
Description: "Add items, upload files, reprocess and delete.",
|
||||
Available: domain.RoleGrants(role, domain.ScopeIngest),
|
||||
},
|
||||
{
|
||||
Scope: domain.ScopeReview, Label: "Review",
|
||||
Description: "Confirm, reject and retype relationships; confirm entity aliases.",
|
||||
Available: domain.RoleGrants(role, domain.ScopeReview),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func joinScopes(scopes []domain.TokenScope) string {
|
||||
if len(scopes) == 0 {
|
||||
return "none"
|
||||
}
|
||||
parts := make([]string, 0, len(scopes))
|
||||
for _, s := range scopes {
|
||||
parts = append(parts, string(s))
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/relata/internal/api"
|
||||
"github.com/andydixon/relata/internal/auth"
|
||||
"github.com/andydixon/relata/internal/config"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
)
|
||||
|
||||
type tokenPageRepo struct {
|
||||
domain.APITokenRepository
|
||||
tokens []*domain.APIToken
|
||||
}
|
||||
|
||||
func (r *tokenPageRepo) Create(_ context.Context, tok *domain.APIToken) error {
|
||||
r.tokens = append(r.tokens, tok)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *tokenPageRepo) ListForUser(_ context.Context, workspaceID, userID string) ([]*domain.APIToken, error) {
|
||||
var out []*domain.APIToken
|
||||
for _, tok := range r.tokens {
|
||||
if tok.WorkspaceID == workspaceID && tok.UserID == userID {
|
||||
out = append(out, tok)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type tokenPageRels struct{ domain.RelationshipRepository }
|
||||
|
||||
func (tokenPageRels) CountPendingReview(context.Context, string) (int64, error) { return 0, nil }
|
||||
|
||||
func newTokenPageHandlers(t *testing.T) (*Handlers, *domain.Principal) {
|
||||
t.Helper()
|
||||
renderer, err := NewRenderer()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := &tokenPageRepo{}
|
||||
tokens := auth.NewTokenService(repo, nil, auth.TokenConfig{
|
||||
DefaultTTL: 24 * time.Hour,
|
||||
MaxTTL: 30 * 24 * time.Hour,
|
||||
})
|
||||
cfg := &config.Config{}
|
||||
cfg.Auth.OAuth.TokenDefaultTTL = 24 * time.Hour
|
||||
cfg.Auth.OAuth.TokenMaxTTL = 30 * 24 * time.Hour
|
||||
h := NewHandlers(Deps{Cfg: cfg, Renderer: renderer, Tokens: tokens, Rels: tokenPageRels{}})
|
||||
p := &domain.Principal{
|
||||
UserID: "user-1", Username: "admin", WorkspaceID: "workspace-1", Role: domain.RoleAdmin,
|
||||
}
|
||||
return h, p
|
||||
}
|
||||
|
||||
func TestCreateTokenNeverPlacesSecretInRedirectURL(t *testing.T) {
|
||||
h, p := newTokenPageHandlers(t)
|
||||
form := url.Values{
|
||||
"name": {"automation"},
|
||||
"scopes": {string(domain.ScopeQuery)},
|
||||
"ttl_days": {"7"},
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPost, "/settings/tokens", strings.NewReader(form.Encode()))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
r = r.WithContext(api.WithPrincipal(r.Context(), p))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.CreateToken(w, r)
|
||||
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d, want 201", w.Code)
|
||||
}
|
||||
if location := w.Header().Get("Location"); location != "" {
|
||||
t.Fatalf("token creation redirected to %q; the secret could enter URL logs", location)
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "rlt_v1_") {
|
||||
t.Fatal("the one-time token was not rendered in the creation response")
|
||||
}
|
||||
if strings.Contains(w.Body.String(), "?secret=") {
|
||||
t.Fatal("the rendered page still describes a secret-bearing URL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokensPageIgnoresSecretQueryParameter(t *testing.T) {
|
||||
h, p := newTokenPageHandlers(t)
|
||||
r := httptest.NewRequest(http.MethodGet, "/settings/tokens?secret=rlt_v1_attacker_controlled", nil)
|
||||
r = r.WithContext(api.WithPrincipal(r.Context(), p))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.TokensPage(w, r)
|
||||
|
||||
if strings.Contains(w.Body.String(), "rlt_v1_attacker_controlled") {
|
||||
t.Fatal("a secret query parameter was reflected into the token page")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user