Phase 4: authorised HTTPS interception (Mode B)

Toggleable transparent interception via mitmdump in a venv. Own RSA CA generated
in-app, downloadable (public cert only, key never leaves the box), with per-OS
install guide and fingerprint. nftables redirect accepts only dnat'd connections
(never an open proxy). HTTP Explorer with headers and opt-in body viewers (JSON/
text/hex/image, sandboxed). Honest status: pinned/untrusting = reported errors,
excluded hosts pass through, bodies off by default. Helper op + systemd unit +
installer venv build + boot restore.

web/src/App.tsx

@@ -15,6 +15,8 @@ import DnsExplorer from "./pages/DnsExplorer";
 import TlsExplorer from "./pages/TlsExplorer";
 import Statistics from "./pages/Statistics";
 import Findings from "./pages/Findings";
+import HttpExplorer from "./pages/HttpExplorer";
+import Certificates from "./pages/Certificates";
 import Settings from "./pages/Settings";
 
 export interface Prefs { theme: "dark" | "light" | "system"; density: "compact" | "comfortable"; scale: number; motion: "system" | "reduce" }
@@ -29,12 +31,14 @@ const NAV: [string, string, string][] = [
   ["/conversations", "Conversations", "⇌"],
   ["/dns", "DNS Explorer", "◇"],
   ["/tls", "TLS Explorer", "⛨"],
+  ["/http", "HTTP Explorer", "⇅"],
   ["/statistics", "Statistics", "▦"],
   ["/findings", "Findings", "⚑"],
   ["/library", "Capture Library", "▤"],
   ["/devices", "Devices", "⌬"],
   ["", "", ""],
   ["/network", "Network Settings", "⇄"],
+  ["/certificates", "Certificates", "⛉"],
   ["/health", "System Health", "♥"],
   ["/settings", "Settings", "⚙"],
 ];
@@ -101,6 +105,8 @@ export default function App() {
     if (path.startsWith("/tls")) return <TlsExplorer sessionId={path.split("/")[2] ?? ""} />;
     if (path.startsWith("/statistics")) return <Statistics sessionId={path.split("/")[2] ?? ""} />;
     if (path.startsWith("/findings")) return <Findings sessionId={path.split("/")[2] ?? ""} />;
+    if (path.startsWith("/http")) return <HttpExplorer />;
+    if (path === "/certificates") return <Certificates />;
     if (path === "/library") return <Library />;
     if (path === "/devices") return <Devices />;
     if (path === "/network") return <NetworkPage />;

web/src/api.ts

@@ -56,7 +56,7 @@ let es: EventSource | null = null;
 export function connectEvents() {
   if (es) return;
   es = new EventSource("/api/events");
-  for (const t of ["hello", "packets", "capture", "resync", "ping"]) {
+  for (const t of ["hello", "packets", "capture", "resync", "ping", "http"]) {
     es.addEventListener(t, (e) => {
       let d: any = {};
       try { d = JSON.parse((e as MessageEvent).data); } catch { /* ignore */ }
@@ -98,3 +98,14 @@ export interface TLSRecord { num: number; seg: number; time: number; client: str
 export interface CertRecord { num: number; seg: number; time: number; server: string; subject: string; issuer: string; sans: string; not_before: string; not_after: string }
 
 export interface Finding { rule: string; title: string; severity: "info" | "low" | "medium"; count: number; flows: number; examples: number[]; filter: string; note: string }
+
+// ---- Phase 4 interception types ----
+export interface CAInfo { present: boolean; subject: string; sha256: string; not_before: string; not_after: string; serial: string; days_left: number; key_type: string }
+export interface InterceptionConfig { enabled: boolean; capture_bodies: boolean; max_body_kb: number; exclude_hosts: string[] }
+export interface InterceptionStatus { config: InterceptionConfig; ca: CAInfo; running: boolean; port: number; note: string }
+export interface HttpFlow {
+  id: string; ts: number; client: string; method: string; scheme: string; host: string; port: number; path: string;
+  http_version: string; status: number; reason: string; req_ctype: string; resp_ctype: string; req_size: number; resp_size: number;
+  duration_ms: number; tls_version: string; sni: string; alpn: string; intercepted: boolean; error: string; websocket: boolean;
+  has_req_body: boolean; has_resp_body: boolean; req_headers: Record<string,string>; resp_headers: Record<string,string>;
+}

web/src/pages/Certificates.tsx

@@ -0,0 +1,146 @@
+import { useState } from "react";
+import { api, type ApplyResult, type InterceptionStatus } from "../api";
+import { Badge, Modal, Steps, toast, usePoll } from "../ui";
+
+const INSTALL: Record<string, string[]> = {
+  "iOS / iPadOS": [
+    "Download the CA to the device (or AirDrop it).",
+    "Settings opens a profile prompt → Install → enter your passcode.",
+    "Then Settings → General → About → Certificate Trust Settings → turn ON full trust for ‘PiLens Interception CA’.",
+    "Without that second step iOS will NOT trust it. This is deliberate on Apple's part.",
+  ],
+  "Android": [
+    "Settings → Security → Encryption & credentials → Install a certificate → CA certificate.",
+    "Pick the downloaded .crt.",
+    "Note: Android apps only trust user-added CAs if they opt in. Many apps (and all of Chrome's pinned sites) will not be decryptable. That is expected.",
+  ],
+  "Windows": [
+    "Double-click the .crt → Install Certificate → Local Machine → ‘Trusted Root Certification Authorities’.",
+    "Browsers using the system store (Edge, Chrome) will then trust it; Firefox uses its own store (below).",
+  ],
+  "macOS": [
+    "Double-click the .crt → add to the ‘login’ or ‘System’ keychain.",
+    "Open Keychain Access → find ‘PiLens Interception CA’ → Get Info → Trust → ‘Always Trust’.",
+  ],
+  "Firefox": [
+    "Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import.",
+    "Tick ‘Trust this CA to identify websites’.",
+  ],
+};
+
+export default function Certificates() {
+  const [st, err, reload] = usePoll<InterceptionStatus>("/api/interception", 8000);
+  const [busy, setBusy] = useState(false);
+  const [result, setResult] = useState<ApplyResult | null>(null);
+  const [applyErr, setApplyErr] = useState("");
+  const [os, setOs] = useState("iOS / iPadOS");
+  const [confirmBodies, setConfirmBodies] = useState(false);
+  const [draft, setDraft] = useState<InterceptionStatus["config"] | null>(null);
+
+  const cfg = draft ?? st?.config;
+  const dirty = draft !== null && JSON.stringify(draft) !== JSON.stringify(st?.config);
+
+  const set = (p: Partial<InterceptionStatus["config"]>) => setDraft({ ...(cfg as any), ...p });
+
+  const apply = async (override?: Partial<InterceptionStatus["config"]>) => {
+    const body = { ...(cfg as any), ...override };
+    setBusy(true); setApplyErr(""); setResult(null);
+    try {
+      const r = await api<{ result: ApplyResult }>("/api/interception", { method: "PUT", body });
+      setResult(r.result); setDraft(null); reload();
+      toast(body.enabled ? "Interception enabled" : "Interception disabled");
+    } catch (e: any) {
+      setApplyErr(e.message); setResult(e.body?.result ?? null); reload();
+    }
+    setBusy(false);
+  };
+
+  const ca = st?.ca;
+
+  return (
+    <div className="page">
+      <div className="page-head">
+        <div><h1>Certificates &amp; HTTPS Inspection</h1>
+          <p>Authorised HTTPS interception (Mode B). Decrypts only for devices whose owner has installed and trusted the CA below.</p></div>
+      </div>
+
+      <div className="banner warn" style={{ marginBottom: "1rem" }}><div>
+        <b>Consent is required, by design.</b> PiLens can only decrypt a device after <em>you</em> install this CA on it and turn on trust.
+        A trusted certificate can never be pushed over the network or via DHCP. Pinned apps, private trust stores and QUIC are reported as
+        not intercepted — never silently bypassed. The CA private key never leaves the appliance and is not downloadable.
+      </div></div>
+
+      {err && <div className="banner bad"><div>{err}</div></div>}
+
+      <div className="grid g2">
+        <div className="card stack">
+          <h3>Interception</h3>
+          <label className="inline" style={{ fontSize: "1rem" }}>
+            <input type="checkbox" checked={!!cfg?.enabled} disabled={busy || !ca?.present}
+              onChange={(e) => apply({ enabled: e.target.checked })} />
+            <b>{cfg?.enabled ? "Enabled" : "Disabled"}</b> — redirect AP clients' HTTP/HTTPS through the proxy
+          </label>
+          {st && <div className="row">
+            <Badge kind={st.running ? "ok" : "plain"}>{st.running ? "proxy active" : "proxy stopped"}</Badge>
+            <span className="hint">transparent, port {st.port}</span>
+          </div>}
+          {!ca?.present && <div className="banner bad"><div>The interception CA is not available. Is the mitmproxy venv installed? Run <code>sudo /opt/pilens/repair.sh</code>.</div></div>}
+          {applyErr && <div className="err">{applyErr}</div>}
+          {result && <Steps steps={result.steps} />}
+          <p className="hint">Enabling adds an nftables redirect for ports 80/443 from the Wi-Fi only. The proxy never becomes a direct open proxy for clients (it accepts only redirected connections).</p>
+        </div>
+
+        <div className="card stack">
+          <h3>The appliance CA</h3>
+          {ca?.present ? <>
+            <dl className="kv">
+              <dt>Subject</dt><dd className="mono">{ca.subject}</dd>
+              <dt>Key</dt><dd>{ca.key_type}</dd>
+              <dt>Fingerprint</dt><dd className="mono" style={{ fontSize: ".75rem", wordBreak: "break-all" }}>SHA-256 {ca.sha256}</dd>
+              <dt>Valid</dt><dd>{ca.not_before} → {ca.not_after} <span className="hint">({ca.days_left} days left)</span></dd>
+            </dl>
+            <div className="row">
+              <a className="btn primary" href="/api/interception/ca.crt" download>Download CA certificate</a>
+              <button disabled={cfg?.enabled || busy} title={cfg?.enabled ? "Disable interception first" : ""}
+                onClick={async () => { if (confirm("Rotate the CA? Every device must install the new one; current interception breaks until they do.")) { await api("/api/interception/ca/rotate", { method: "POST" }); reload(); toast("CA rotated"); } }}>Rotate CA</button>
+            </div>
+            <p className="hint">Install this on each authorised test device, then verify the fingerprint above matches before trusting it.</p>
+          </> : <p className="hint">No CA yet.</p>}
+        </div>
+      </div>
+
+      <div className="card stack" style={{ marginTop: ".9rem" }}>
+        <h3>Install instructions</h3>
+        <div className="row" role="tablist">
+          {Object.keys(INSTALL).map((k) => <button key={k} className={os === k ? "on" : ""} onClick={() => setOs(k)}>{k}</button>)}
+        </div>
+        <ol style={{ margin: 0, paddingLeft: "1.2rem" }}>{INSTALL[os].map((s, i) => <li key={i} style={{ marginBottom: ".3rem" }}>{s}</li>)}</ol>
+      </div>
+
+      {cfg && <div className="card stack" style={{ marginTop: ".9rem" }}>
+        <h3>Options</h3>
+        <label>Excluded hosts (not decrypted — passed straight through)
+          <textarea rows={3} className="mono" value={cfg.exclude_hosts.join("\n")} placeholder={"one host/regex per line\n(?i)\\.apple\\.com$\nexample\\.com"}
+            onChange={(e) => set({ exclude_hosts: e.target.value.split("\n").map((x) => x.trim()).filter(Boolean) })} />
+          <span className="hint">Use for pinned apps or privacy-sensitive hosts. Matched hosts keep working but stay encrypted.</span>
+        </label>
+        <label className="inline">
+          <input type="checkbox" checked={cfg.capture_bodies} onChange={(e) => { if (e.target.checked) setConfirmBodies(true); else set({ capture_bodies: false }); }} />
+          Retain request/response bodies on disk
+        </label>
+        <div className="row">
+          <label>Max body size (KiB)<input type="number" min={0} max={10240} value={cfg.max_body_kb} onChange={(e) => set({ max_body_kb: +e.target.value })} /></label>
+        </div>
+        {dirty && <div className="row"><button className="primary" disabled={busy} onClick={() => apply()}>Apply options</button>
+          <button disabled={busy} onClick={() => setDraft(null)}>Discard</button></div>}
+      </div>}
+
+      <Modal open={confirmBodies} title="Retain decrypted bodies?" onClose={() => setConfirmBodies(false)}
+        footer={<><button onClick={() => setConfirmBodies(false)}>Cancel</button>
+          <button className="danger" onClick={() => { set({ capture_bodies: true }); setConfirmBodies(false); }}>I understand, retain bodies</button></>}>
+        <p>Response and request bodies can contain passwords, tokens, messages and personal data in clear text. They will be written to the appliance's disk until you clear them.</p>
+        <p className="hint">Off by default. Only enable for a specific investigation, and clear the data (HTTP Explorer → Clear) when done. Exports and body views are audited.</p>
+      </Modal>
+    </div>
+  );
+}

web/src/pages/HttpExplorer.tsx

@@ -0,0 +1,162 @@
+import { useEffect, useMemo, useRef, useState } from "react";
+import { api, onEvent, fmtBytes, fmtTime, type HttpFlow } from "../api";
+import { Badge, Modal, copyText, toast, usePersist } from "../ui";
+
+function statusKind(f: HttpFlow): "ok" | "warn" | "bad" | "plain" {
+  if (f.error) return "bad";
+  if (f.status >= 500) return "bad";
+  if (f.status >= 400) return "warn";
+  if (f.status >= 200 && f.status < 400) return "ok";
+  return "plain";
+}
+
+export default function HttpExplorer() {
+  const [flows, setFlows] = useState<HttpFlow[]>([]);
+  const [q, setQ] = useState("");
+  const [onlyErr, setOnlyErr] = useState(false);
+  const [live, setLive] = usePersist("http.live", true);
+  const [sel, setSel] = useState<HttpFlow | null>(null);
+  const [err, setErr] = useState("");
+  const liveRef = useRef(live); liveRef.current = live;
+
+  const load = () => api<HttpFlow[]>("/api/http?limit=2000").then(setFlows).catch((e) => setErr(e.message));
+  useEffect(() => { load(); }, []);
+  useEffect(() => onEvent((type, d: HttpFlow) => {
+    if (type !== "http" || !liveRef.current || !d.id) return;
+    setFlows((f) => [d, ...f.filter((x) => x.id !== d.id)].slice(0, 3000));
+  }), []);
+
+  const rows = useMemo(() => flows.filter((f) =>
+    (!q || `${f.host}${f.path} ${f.client} ${f.method}`.toLowerCase().includes(q.toLowerCase())) &&
+    (!onlyErr || f.error || f.status >= 400)), [flows, q, onlyErr]);
+
+  return (
+    <div className="page" style={{ display: "flex", flexDirection: "column" }}>
+      <div className="page-head">
+        <div><h1>HTTP Explorer</h1><p>Decrypted HTTP(S) transactions from trusting devices. Requires interception (Certificates page).</p></div>
+        <div className="row">
+          <button className={live ? "on" : ""} onClick={() => setLive(!live)} title="Append new transactions live">⚡ Live</button>
+          <button onClick={load}>Refresh</button>
+          <button className="danger" onClick={async () => { if (confirm("Delete all intercepted HTTP data (flows and any retained bodies)? Captures are untouched.")) { await api("/api/interception/clear", { method: "POST" }); setFlows([]); setSel(null); toast("Cleared"); } }}>Clear</button>
+        </div>
+      </div>
+      <div className="banner info" style={{ marginBottom: ".8rem" }}><div>
+        Shown here are connections PiLens could decrypt because the device trusts the CA. Pinned/untrusting connections appear as <b>errors</b>, not content. Nothing here is captured from devices that did not opt in.
+      </div></div>
+      <div className="row" style={{ marginBottom: ".6rem" }}>
+        <input placeholder="Filter host / path / client / method" value={q} onChange={(e) => setQ(e.target.value)} style={{ minWidth: 280 }} />
+        <label className="inline"><input type="checkbox" checked={onlyErr} onChange={(e) => setOnlyErr(e.target.checked)} />Errors only</label>
+        <span className="hint">{rows.length.toLocaleString()} transactions</span>
+      </div>
+      {err && <div className="banner bad"><div>{err}</div></div>}
+      <div className="card" style={{ padding: 0, flex: 1, overflow: "auto" }}>
+        <table className="data">
+          <thead><tr><th>Time</th><th>Client</th><th>Method</th><th>Host</th><th>Path</th><th>Status</th><th>Type</th><th>Size</th><th>Time</th><th></th></tr></thead>
+          <tbody>
+            {rows.map((f) => (
+              <tr key={f.id} style={{ cursor: "pointer" }} onClick={() => setSel(f)}>
+                <td className="hint" style={{ whiteSpace: "nowrap" }}>{fmtTime(f.ts * 1000).split(", ")[1] ?? fmtTime(f.ts * 1000)}</td>
+                <td className="mono">{f.client}</td>
+                <td className="mono">{f.method}{f.websocket && " ⇄"}</td>
+                <td className="mono">{f.scheme === "https" ? "🔒" : ""}{f.host}</td>
+                <td className="mono" style={{ maxWidth: 360, overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{f.path}</td>
+                <td>{f.error ? <Badge kind="bad">error</Badge> : <Badge kind={statusKind(f)}>{f.status}</Badge>}</td>
+                <td className="mono hint">{(f.resp_ctype || f.req_ctype || "").split(";")[0]}</td>
+                <td className="mono">{fmtBytes(f.resp_size)}</td>
+                <td className="mono hint">{f.duration_ms} ms</td>
+                <td>›</td>
+              </tr>
+            ))}
+            {rows.length === 0 && <tr><td colSpan={10} className="empty">No transactions. Enable interception and install the CA on a test device.</td></tr>}
+          </tbody>
+        </table>
+      </div>
+      {sel && <FlowDetail flow={sel} onClose={() => setSel(null)} />}
+    </div>
+  );
+}
+
+function Headers({ h }: { h: Record<string, string> }) {
+  const entries = Object.entries(h || {});
+  if (!entries.length) return <div className="hint">none</div>;
+  return <table className="data"><tbody>{entries.map(([k, v]) => <tr key={k}><td className="mono" style={{ color: "var(--muted)", whiteSpace: "nowrap", verticalAlign: "top" }}>{k}</td><td className="mono" style={{ wordBreak: "break-all" }}>{v}</td></tr>)}</tbody></table>;
+}
+
+function FlowDetail({ flow, onClose }: { flow: HttpFlow; onClose: () => void }) {
+  const [tab, setTab] = useState<"req" | "resp">("resp");
+  return (
+    <Modal open title={<span className="mono" style={{ fontSize: ".95rem" }}>{flow.method} {flow.scheme}://{flow.host}{flow.path}</span>} onClose={onClose}
+      footer={<><button onClick={() => copyText(`${flow.method} ${flow.scheme}://${flow.host}${flow.path}`)}>Copy URL</button><button className="primary" onClick={onClose}>Close</button></>}>
+      <div className="row" style={{ gap: ".5rem", flexWrap: "wrap" }}>
+        <Badge kind={flow.error ? "bad" : statusKind(flow)}>{flow.error ? "error" : `${flow.status} ${flow.reason}`}</Badge>
+        {flow.intercepted ? <Badge kind="ok">decrypted</Badge> : <Badge kind="warn">not decrypted</Badge>}
+        <span className="hint mono">{flow.client} · {flow.http_version} · {flow.duration_ms} ms</span>
+      </div>
+      {flow.tls_version && <div className="hint mono">{flow.tls_version}{flow.sni ? ` · SNI ${flow.sni}` : ""}{flow.alpn ? ` · ALPN ${flow.alpn}` : ""}</div>}
+      {flow.error && <div className="banner bad" style={{ margin: ".5rem 0" }}><div>{flow.error}</div></div>}
+
+      <div className="row" style={{ gap: ".25rem", marginTop: ".5rem" }}>
+        <button className={tab === "req" ? "on" : ""} onClick={() => setTab("req")}>Request</button>
+        <button className={tab === "resp" ? "on" : ""} onClick={() => setTab("resp")}>Response</button>
+      </div>
+      {tab === "req" ? <div className="stack">
+        <h3>Request headers</h3><Headers h={flow.req_headers} />
+        <Body flow={flow} which="req" has={flow.has_req_body} size={flow.req_size} ctype={flow.req_ctype} />
+      </div> : <div className="stack">
+        <h3>Response headers</h3><Headers h={flow.resp_headers} />
+        <Body flow={flow} which="resp" has={flow.has_resp_body} size={flow.resp_size} ctype={flow.resp_ctype} />
+      </div>}
+    </Modal>
+  );
+}
+
+function Body({ flow, which, has, size, ctype }: { flow: HttpFlow; which: "req" | "resp"; has: boolean; size: number; ctype: string }) {
+  const [data, setData] = useState<Uint8Array | null>(null);
+  const [mode, setMode] = useState<"auto" | "text" | "hex" | "image">("auto");
+  const [loading, setLoading] = useState(false);
+  const ct = (ctype || "").split(";")[0];
+
+  const load = async () => {
+    setLoading(true);
+    try {
+      const r = await fetch(`/api/http/${flow.id}/body/${which}`, { headers: { "X-Requested-With": "pilens" }, credentials: "same-origin" });
+      if (!r.ok) throw new Error(await r.text());
+      setData(new Uint8Array(await r.arrayBuffer()));
+    } catch (e) { toast((e as Error).message, true); }
+    setLoading(false);
+  };
+
+  if (!has) return <><h3>Body</h3><div className="hint">{size > 0 ? `${fmtBytes(size)} not retained (body capture is off for privacy). Enable it on the Certificates page for a specific investigation.` : "empty"}</div></>;
+
+  const view = mode === "auto" ? (ct.startsWith("image/") ? "image" : /json|text|xml|javascript|html|urlencoded/.test(ct) ? "text" : "hex") : mode;
+
+  return <><div className="row spread"><h3>Body ({fmtBytes(size)}, {ct || "unknown"})</h3>
+    <div className="row" style={{ gap: ".25rem" }}>
+      {["auto", "text", "hex", "image"].map((m) => <button key={m} className={"small" + (mode === m ? " on" : "")} onClick={() => setMode(m as any)}>{m}</button>)}
+    </div></div>
+    {!data ? <button disabled={loading} onClick={load}>{loading ? "Loading…" : "Load body"}</button> : <BodyView data={data} view={view} id={flow.id} which={which} ct={ct} />}
+  </>;
+}
+
+function BodyView({ data, view, id, which, ct }: { data: Uint8Array; view: string; id: string; which: string; ct: string }) {
+  const text = useMemo(() => { try { return new TextDecoder().decode(data); } catch { return ""; } }, [data]);
+  if (view === "image") {
+    // Sandboxed: served as application/octet-stream with nosniff; render via blob URL.
+    const url = useMemo(() => URL.createObjectURL(new Blob([data as BlobPart], { type: ct || "application/octet-stream" })), [data, ct]);
+    return <img src={url} alt="response body" style={{ maxWidth: "100%", maxHeight: "50vh", background: "var(--panel-2)" }} />;
+  }
+  if (view === "hex") {
+    const lines = [];
+    for (let o = 0; o < Math.min(data.length, 16384); o += 16) {
+      const slice = Array.from(data.slice(o, o + 16));
+      lines.push(o.toString(16).padStart(6, "0") + "  " + slice.map((b) => b.toString(16).padStart(2, "0")).join(" ").padEnd(47) + "  " + slice.map((b) => b >= 32 && b < 127 ? String.fromCharCode(b) : ".").join(""));
+    }
+    return <pre style={preStyle}>{lines.join("\n")}{data.length > 16384 ? "\n… truncated" : ""}</pre>;
+  }
+  let pretty = text;
+  if (/json/.test(ct)) { try { pretty = JSON.stringify(JSON.parse(text), null, 2); } catch { /* keep raw */ } }
+  return <><pre style={preStyle}>{pretty.slice(0, 200000)}{pretty.length > 200000 ? "\n… truncated" : ""}</pre>
+    <a className="btn small" href={`/api/http/${id}/body/${which}`} download>Download raw</a></>;
+}
+
+const preStyle: React.CSSProperties = { maxHeight: "48vh", overflow: "auto", background: "var(--panel-2)", padding: ".6rem", borderRadius: 6, fontSize: ".82rem", whiteSpace: "pre-wrap", wordBreak: "break-all", margin: 0 };