// Package browser provides read-only web capabilities that behave like the // reading half of a browser: open a page, read its text, list its links. // Pages are fetched through the SSRF-safe client, scripts never run, and // everything a page says arrives as low-trust observation data. The policy // target is the page's host, so operators can allow or deny by domain. // // Writing operations (click, fill, submit, download) are designed in // ADR-0011 but deliberately not registered: they need a sandboxed rendering // engine and per-domain write policy that this release does not ship. package browser import ( "context" "errors" "fmt" "html" "net/http" "net/url" "regexp" "strings" "time" "github.com/andydixon/noema/internal/capabilities" "github.com/andydixon/noema/internal/security/httpclient" "github.com/andydixon/noema/pkg/events" ) // Options configure the browser capabilities. type Options struct { // Client overrides the HTTP client (tests). Nil uses the SSRF-safe client. Client *http.Client MaxBytes int64 // default 512 KiB } // Host returns the lower-cased host of an http(s) URL parameter. func Host(params map[string]any) (string, error) { raw, _ := params["url"].(string) u, err := url.Parse(raw) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" { return "", errors.New("url must be an absolute http(s) URL") } if u.User != nil { return "", errors.New("credentials in URLs are not accepted") } return strings.ToLower(u.Hostname()), nil } // Capabilities returns browser.navigate, browser.read and browser.links. func Capabilities(o Options) []capabilities.Capability { if o.MaxBytes <= 0 { o.MaxBytes = 512 << 10 } param := map[string]capabilities.Param{"url": {Type: "url", Required: true, Description: "page to open", MaxLen: 2000}} common := func(name, desc string, exec capabilities.Executor) capabilities.Capability { return capabilities.Capability{Name: name, Description: desc, Risk: capabilities.RiskMedium, TargetKind: "domain", Access: capabilities.AccessRead, Idempotent: true, Timeout: 25 * time.Second, SideEffects: "one GET request to the page's server, which may log it; no scripts run and nothing is submitted", DefaultEffect: capabilities.Ask, Params: param, TargetFrom: Host, OutputTrust: events.TrustLow, Estimate: capabilities.Cost{WallSeconds: 2, Network: 1}, Executor: exec} } return []capabilities.Capability{ common("browser.navigate", "Open a page and report its final address, status, title and link count.", func(ctx context.Context, inv capabilities.Invocation) (capabilities.Result, error) { p, err := o.load(ctx, inv) if err != nil { return capabilities.Result{}, err } return capabilities.Result{Summary: fmt.Sprintf("HTTP %d %s: %q (%d links)", p.status, p.final, p.title, len(p.links)), Data: map[string]any{"status": p.status, "final_url": p.final, "title": p.title, "content_type": p.contentType, "links": len(p.links), "bytes": p.bytes}}, nil }), common("browser.read", "Open a page and return its readable text with scripts, styles and markup removed.", func(ctx context.Context, inv capabilities.Invocation) (capabilities.Result, error) { p, err := o.load(ctx, inv) if err != nil { return capabilities.Result{}, err } return capabilities.Result{Summary: fmt.Sprintf("read %q from %s (%d characters)", p.title, p.final, len(p.text)), Output: p.text, Data: map[string]any{"status": p.status, "final_url": p.final, "title": p.title}}, nil }), common("browser.links", "Open a page and list its links as absolute URLs with their text.", func(ctx context.Context, inv capabilities.Invocation) (capabilities.Result, error) { p, err := o.load(ctx, inv) if err != nil { return capabilities.Result{}, err } var b strings.Builder for _, l := range p.links { fmt.Fprintf(&b, "%s\t%s\n", l.URL, l.Text) } return capabilities.Result{Summary: fmt.Sprintf("%d links on %s", len(p.links), p.final), Output: b.String(), Data: map[string]any{"final_url": p.final, "links": p.links}}, nil }), } } // Link is one anchor on a page. type Link struct { URL string `json:"url"` Text string `json:"text"` } type page struct { status int final, contentType string title, text string links []Link bytes int } func (o Options) load(ctx context.Context, inv capabilities.Invocation) (page, error) { raw, _ := inv.Params["url"].(string) client := o.Client if client == nil { client = httpclient.New(httpclient.Options{Timeout: 20 * time.Second, MaxBody: o.MaxBytes}) } req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil) if err != nil { return page{}, err } req.Header.Set("User-Agent", "noema/1 (+capability browser; read-only)") req.Header.Set("Accept", "text/html, text/plain;q=0.8") res, err := client.Do(req) if err != nil { return page{}, err } defer res.Body.Close() body, err := httpclient.ReadAll(res.Body, o.MaxBytes) if err != nil { return page{}, err } p := page{status: res.StatusCode, final: res.Request.URL.String(), contentType: res.Header.Get("Content-Type"), bytes: len(body)} // A redirect must not leave the policy target: the host that was // authorised is the host that is read. if want, _ := Host(inv.Params); !strings.EqualFold(res.Request.URL.Hostname(), want) { return page{}, fmt.Errorf("redirected from %s to %s; open the new address as its own request", want, res.Request.URL.Hostname()) } doc := string(body) if !strings.Contains(p.contentType, "html") { p.text = doc return p, nil } p.title, p.text, p.links = Extract(doc, res.Request.URL) return p, nil } var ( dropRe = regexp.MustCompile(`(?is)<(script|style|noscript|template|iframe|svg)\b.*?(script|style|noscript|template|iframe|svg)\s*>`) titleRe = regexp.MustCompile(`(?is)