From 5dbf85f84eeec7eac49af580f2dbc81b4259daee Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Sat, 10 Oct 2026 08:05:42 +0100 Subject: [PATCH] Phase 4: authorised HTTPS interception (Mode B) Toggleable transparent interception via mitmdump in a venv. Own RSA CA generated in-app, downloadable (public cert only, key never leaves the box), with per-OS install guide and fingerprint. nftables redirect accepts only dnat'd connections (never an open proxy). HTTP Explorer with headers and opt-in body viewers (JSON/ text/hex/image, sandboxed). Honest status: pinned/untrusting = reported errors, excluded hosts pass through, bodies off by default. Helper op + systemd unit + installer venv build + boot restore. --- Makefile | 2 +- cmd/pilens-helper/main.go | 106 +++++++++++- cmd/pilens/main.go | 17 +- internal/api/interception.go | 182 +++++++++++++++++++++ internal/api/server.go | 3 + internal/capture/hub.go | 1 + internal/helper/helper.go | 7 +- internal/netgen/netgen.go | 29 +++- internal/netgen/netgen_test.go | 21 ++- internal/proxy/ca.go | 165 +++++++++++++++++++ internal/proxy/ca_test.go | 62 +++++++ internal/proxy/proxy.go | 286 +++++++++++++++++++++++++++++++++ migrations/002_http.sql | 33 ++++ packaging/mitm_addon.py | 157 ++++++++++++++++++ scripts/diagnose.sh | 3 +- scripts/install.sh | 19 +++ scripts/proxy-run.sh | 13 ++ scripts/repair.sh | 12 ++ scripts/uninstall.sh | 2 +- systemd/pilens-proxy.service | 26 +++ web/dist/.gitkeep | 0 web/src/App.tsx | 6 + web/src/api.ts | 13 +- web/src/pages/Certificates.tsx | 146 +++++++++++++++++ web/src/pages/HttpExplorer.tsx | 162 +++++++++++++++++++ 25 files changed, 1460 insertions(+), 13 deletions(-) create mode 100644 internal/api/interception.go create mode 100644 internal/proxy/ca.go create mode 100644 internal/proxy/ca_test.go create mode 100644 internal/proxy/proxy.go create mode 100644 migrations/002_http.sql create mode 100644 packaging/mitm_addon.py create mode 100755 scripts/proxy-run.sh create mode 100644 systemd/pilens-proxy.service delete mode 100644 web/dist/.gitkeep create mode 100644 web/src/pages/Certificates.tsx create mode 100644 web/src/pages/HttpExplorer.tsx diff --git a/Makefile b/Makefile index ff8bb24..4ad3ead 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ build: web $(GO) build -trimpath -ldflags '$(LDFLAGS)' -o dist/$(PKG)/bin/pilens-helper ./cmd/pilens-helper package: build - cp -r systemd packaging/setup.example.yaml dist/$(PKG)/ + cp -r systemd packaging/setup.example.yaml packaging/mitm_addon.py dist/$(PKG)/ cp scripts/*.sh dist/$(PKG)/ cp -r docs README.md LICENSE dist/$(PKG)/ echo $(VERSION) > dist/$(PKG)/VERSION diff --git a/cmd/pilens-helper/main.go b/cmd/pilens-helper/main.go index cdda292..c86627b 100644 --- a/cmd/pilens-helper/main.go +++ b/cmd/pilens-helper/main.go @@ -142,6 +142,12 @@ func (s *server) handle(ctx context.Context, c *net.UnixConn) { data = s.status(ctx) case helper.OpStations: data, err = stations(ctx) + case helper.OpSetInterception: + r := s.setInterception(ctx, req.InterceptEnabled, req.InterceptPort) + data = r + if r.Error != "" { + err = errors.New(r.Error) + } case helper.OpLeases: b, rerr := os.ReadFile(netgen.LeaseFile) if rerr != nil && !os.IsNotExist(rerr) { @@ -232,7 +238,7 @@ func (s *server) apply(ctx context.Context, n config.Network) helper.ApplyResult return r } step("validated configuration") - ruleset := netgen.Nftables(n) + ruleset := netgen.Nftables(n, loadIntercept()) tmp := statePath("ruleset.check.nft") if err := writeAtomic(tmp, []byte(ruleset), 0o600); err != nil { r.Error = err.Error() @@ -441,6 +447,98 @@ func stations(ctx context.Context) ([]helper.Station, error) { return helper.ParseStationDump(out), nil } +// --- interception state (persisted so apply_network and boot keep the redirect) --- + +type interceptState struct { + Enabled bool `json:"enabled"` + Port int `json:"port"` +} + +func interceptPath() string { return statePath("intercept.json") } + +func loadIntercept() *netgen.Intercept { + b, err := os.ReadFile(interceptPath()) + if err != nil { + return nil + } + var st interceptState + if json.Unmarshal(b, &st) != nil || !st.Enabled { + return nil + } + return &netgen.Intercept{Enabled: true, Port: st.Port} +} + +// setInterception toggles transparent interception: re-renders the firewall with +// (or without) the redirect, then starts/stops the proxy service. The CA and the +// proxy's configuration are owned by the core; the helper only wires the network +// and the service. +func (s *server) setInterception(ctx context.Context, enabled bool, port int) helper.ApplyResult { + s.mu.Lock() + defer s.mu.Unlock() + var r helper.ApplyResult + step := func(f string, a ...any) { r.Steps = append(r.Steps, fmt.Sprintf(f, a...)) } + if port <= 0 || port > 65535 { + r.Error = "invalid proxy port" + return r + } + b, err := os.ReadFile(statePath("last-good.json")) + if err != nil { + r.Error = "apply a network configuration before enabling interception" + return r + } + var n config.Network + if err := json.Unmarshal(b, &n); err != nil { + r.Error = "stored network configuration unreadable" + return r + } + ic := (*netgen.Intercept)(nil) + if enabled { + ic = &netgen.Intercept{Enabled: true, Port: port} + } + ruleset := netgen.Nftables(n, ic) + tmp := statePath("ruleset.check.nft") + writeAtomic(tmp, []byte(ruleset), 0o600) + if _, err := run(ctx, 20*time.Second, "nft", "-c", "-f", tmp); err != nil { + r.Error = "firewall ruleset rejected: " + err.Error() + return r + } + // Persist state first so a concurrent apply_network renders consistently. + stB, _ := json.Marshal(interceptState{Enabled: enabled, Port: port}) + writeAtomic(interceptPath(), stB, 0o600) + writeAtomic(statePath("ruleset.nft"), []byte(ruleset), 0o600) + if _, err := run(ctx, 20*time.Second, "nft", "-f", statePath("ruleset.nft")); err != nil { + r.Error = "loading firewall failed: " + err.Error() + return r + } + step("firewall updated (%s)", map[bool]string{true: "redirecting HTTP/HTTPS to the proxy", false: "redirect removed"}[enabled]) + if enabled { + if _, err := run(ctx, 30*time.Second, "systemctl", "restart", "pilens-proxy.service"); err != nil { + writeAtomic(interceptPath(), mustJSON(interceptState{Enabled: false, Port: port}), 0o600) + writeAtomic(statePath("ruleset.nft"), []byte(netgen.Nftables(n, nil)), 0o600) + run(ctx, 20*time.Second, "nft", "-f", statePath("ruleset.nft")) + r.Error = "proxy failed to start, interception reverted: " + err.Error() + r.RolledBack = true + return r + } + deadline := time.Now().Add(15 * time.Second) + for !unitActive(ctx, "pilens-proxy.service") { + if time.Now().After(deadline) { + r.Error = "proxy did not become active" + return r + } + time.Sleep(time.Second) + } + step("interception proxy active on port %d", port) + } else { + run(ctx, 20*time.Second, "systemctl", "stop", "pilens-proxy.service") + step("interception proxy stopped") + } + r.Applied = true + return r +} + +func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b } + // bootLoad restores the last good ruleset (or a fail-closed one) before networking starts. func bootLoad() error { ctx := context.Background() @@ -462,6 +560,10 @@ func bootLoad() error { run(ctx, 10*time.Second, "iw", "reg", "set", n.AP.Country) } } + if ic := loadIntercept(); ic != nil { + run(ctx, 30*time.Second, "systemctl", "start", "pilens-proxy.service") + slog.Info("interception proxy started at boot") + } slog.Info("firewall restored at boot") return nil } @@ -481,6 +583,6 @@ func renderCmd(path string) int { fmt.Fprintln(os.Stderr, err) return 1 } - fmt.Printf("### nftables\n%s\n### dnsmasq.conf\n%s\n### %s (psk redacted)\n%s", netgen.Nftables(n), netgen.DnsmasqConf(n), keyfilePath, netgen.NMKeyfile(n.Redacted())) + fmt.Printf("### nftables\n%s\n### dnsmasq.conf\n%s\n### %s (psk redacted)\n%s", netgen.Nftables(n, loadIntercept()), netgen.DnsmasqConf(n), keyfilePath, netgen.NMKeyfile(n.Redacted())) return 0 } diff --git a/cmd/pilens/main.go b/cmd/pilens/main.go index e04d88a..7bc8577 100644 --- a/cmd/pilens/main.go +++ b/cmd/pilens/main.go @@ -36,6 +36,7 @@ import ( "pilens.dev/pilens/internal/config" "pilens.dev/pilens/internal/helper" "pilens.dev/pilens/internal/pcapfile" + "pilens.dev/pilens/internal/proxy" "pilens.dev/pilens/internal/store" "pilens.dev/pilens/web" ) @@ -134,13 +135,20 @@ func serve(args []string) error { go eng.Reindex(id) } + host, _ := os.Hostname() + sink := proxySink{eng.Hub} + px := proxy.New(db, filepath.Join(*data, "proxy"), host, sink) + if err := px.EnsureCA(); err != nil { + slog.Warn("interception CA init deferred", "err", err) + } + cert, fp, err := ensureCert(*data) if err != nil { return err } slog.Info("management certificate", "sha256", fp) srv := &api.Server{DB: db, Engine: eng, Helper: helper.Client{Socket: *sock}, Version: Version, Web: web.FS(), - SetupTokenPath: filepath.Join(*data, "setup-token"), DataDir: *data} + SetupTokenPath: filepath.Join(*data, "setup-token"), DataDir: *data, Proxy: px} hs := &http.Server{ Addr: *listen, Handler: srv.Handler(), @@ -162,6 +170,7 @@ func serve(args []string) error { } }() sdNotify("READY=1") + px.Start(ctx) go watchdog(ctx, db) slog.Info("pilens serving", "listen", *listen, "version", Version) <-ctx.Done() @@ -231,6 +240,12 @@ func recoverSessions(db *store.Store, dir string) (ids []string) { return ids } +type proxySink struct{ h *capture.Hub } + +func (s proxySink) Publish(eventType string, data []byte) { + s.h.Publish(capture.Event{Type: eventType, Data: data}) +} + func sdNotify(state string) { addr := os.Getenv("NOTIFY_SOCKET") if addr == "" { diff --git a/internal/api/interception.go b/internal/api/interception.go new file mode 100644 index 0000000..4cda222 --- /dev/null +++ b/internal/api/interception.go @@ -0,0 +1,182 @@ +package api + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "pilens.dev/pilens/internal/helper" + "pilens.dev/pilens/internal/proxy" +) + +// interceptionRoutes wires the Mode-B (authorised HTTPS inspection) endpoints. +func (s *Server) interceptionRoutes(priv func(string, http.HandlerFunc)) { + priv("GET /api/interception", s.getInterception) + priv("PUT /api/interception", s.setInterception) + priv("POST /api/interception/ca/rotate", s.rotateCA) + priv("POST /api/interception/clear", s.clearFlows) + priv("GET /api/interception/ca.crt", s.downloadCA) + priv("GET /api/http", s.httpFlows) + priv("GET /api/http/{id}", s.httpFlow) + priv("GET /api/http/{id}/body/{which}", s.httpBody) +} + +func (s *Server) interceptionStatus(ctx context.Context) map[string]any { + cfg := s.Proxy.Config() + running := false + st, herr := s.helperStatus(ctx) + _ = st + // proxy liveness: ask the helper-independent systemd via the health probe is + // overkill; infer from config + a quick check of the flows file growth is not + // reliable. Report configured-enabled; the dashboard shows service health. + running = cfg.Enabled && herr == "" + return map[string]any{ + "config": cfg, + "ca": s.Proxy.CAInfo(), + "running": running, + "port": proxy.ListenPort, + "note": "Interception decrypts HTTPS only for devices whose owner has installed and trusted the PiLens CA. " + + "Pinned apps, private trust stores and QUIC are reported as not intercepted, never bypassed. The CA private key is never downloadable.", + } +} + +func (s *Server) getInterception(w http.ResponseWriter, r *http.Request) { + writeJSON(w, s.interceptionStatus(r.Context())) +} + +// proxyEnvPath is read by the pilens-proxy systemd unit via proxy-run.sh. +func (s *Server) proxyDir() string { return filepath.Join(s.DataDir, "proxy") } +func (s *Server) proxyEnv() string { return filepath.Join(s.proxyDir(), "proxy.env") } + +func shellQuote(v string) string { return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'" } + +func (s *Server) writeProxyEnv(cfg proxy.Config) error { + bodies := "0" + if cfg.CaptureBodies { + bodies = "1" + } + env := strings.Join([]string{ + "PILENS_VENV=" + shellQuote("/opt/pilens/proxy-venv"), + fmt.Sprintf("PILENS_PROXY_PORT=%d", proxy.ListenPort), + "PILENS_CONFDIR=" + shellQuote(s.Proxy.ConfDir()), + "PILENS_ADDON=" + shellQuote("/opt/pilens/mitm_addon.py"), + "PILENS_FLOWS=" + shellQuote(filepath.Join(s.proxyDir(), "flows.ndjson")), + "PILENS_BODIES=" + shellQuote(filepath.Join(s.proxyDir(), "bodies")), + "PILENS_CAPTURE_BODIES=" + bodies, + fmt.Sprintf("PILENS_MAX_BODY_KB=%d", cfg.MaxBodyKB), + "PILENS_IGNORE=" + shellQuote(cfg.ExcludeRegex()), + }, "\n") + "\n" + os.MkdirAll(s.proxyDir(), 0o750) + return os.WriteFile(s.proxyEnv(), []byte(env), 0o640) +} + +func (s *Server) setInterception(w http.ResponseWriter, r *http.Request) { + var cfg proxy.Config + if !readJSON(w, r, &cfg) { + return + } + if err := cfg.Validate(); err != nil { + httpErr(w, http.StatusBadRequest, err.Error()) + return + } + if err := s.Proxy.EnsureCA(); err != nil { + httpErr(w, 500, "CA generation failed: "+err.Error()) + return + } + if err := s.writeProxyEnv(cfg); err != nil { + httpErr(w, 500, "writing proxy config failed: "+err.Error()) + return + } + if err := s.Proxy.SetConfig(cfg); err != nil { + httpErr(w, 500, err.Error()) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + var res helper.ApplyResult + err := s.Helper.Call(ctx, helper.Request{Op: helper.OpSetInterception, InterceptEnabled: cfg.Enabled, InterceptPort: proxy.ListenPort}, &res) + detail := fmt.Sprintf("enabled=%v capture_bodies=%v excludes=%d", cfg.Enabled, cfg.CaptureBodies, len(cfg.ExcludeHosts)) + if err != nil { + s.audit(r, "interception.apply_failed", detail+": "+err.Error()) + w.WriteHeader(http.StatusBadGateway) + writeJSON(w, map[string]any{"error": err.Error(), "result": res, "status": s.interceptionStatus(r.Context())}) + return + } + s.audit(r, "interception.changed", detail) + writeJSON(w, map[string]any{"result": res, "status": s.interceptionStatus(r.Context())}) +} + +func (s *Server) rotateCA(w http.ResponseWriter, r *http.Request) { + if s.Proxy.Config().Enabled { + httpErr(w, http.StatusConflict, "disable interception before rotating the CA, so devices are not left mid-session") + return + } + if err := s.Proxy.RotateCA(); err != nil { + httpErr(w, 500, err.Error()) + return + } + s.audit(r, "interception.ca_rotated", s.Proxy.CAInfo().SHA256) + writeJSON(w, map[string]any{"ca": s.Proxy.CAInfo()}) +} + +func (s *Server) downloadCA(w http.ResponseWriter, r *http.Request) { + pem, err := s.Proxy.CACertPEM() + if err != nil { + httpErr(w, http.StatusNotFound, err.Error()) + return + } + s.audit(r, "interception.ca_downloaded", s.Proxy.CAInfo().SHA256) + w.Header().Set("Content-Type", "application/x-pem-file") + w.Header().Set("Content-Disposition", `attachment; filename="pilens-ca.crt"`) + w.Write(pem) +} + +func (s *Server) clearFlows(w http.ResponseWriter, r *http.Request) { + s.Proxy.Clear() + s.audit(r, "interception.flows_cleared", "") + writeJSON(w, map[string]bool{"ok": true}) +} + +func (s *Server) httpFlows(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + client := q.Get("client") + if !proxy.ClientIP(client) { + httpErr(w, http.StatusBadRequest, "invalid client filter") + return + } + flows, err := s.Proxy.Flows(r.Context(), q.Get("q"), client, qint(r, "limit", 1000)) + if err != nil { + httpErr(w, 500, err.Error()) + return + } + writeJSON(w, flows) +} + +func (s *Server) httpFlow(w http.ResponseWriter, r *http.Request) { + f, err := s.Proxy.Flow(r.Context(), r.PathValue("id")) + if err != nil { + httpErr(w, http.StatusNotFound, "flow not found") + return + } + writeJSON(w, f) +} + +// httpBody serves a stored body as an opaque download. It is never served with a +// content type that a browser would execute; the UI renders it in a sandbox. +func (s *Server) httpBody(w http.ResponseWriter, r *http.Request) { + body, err := s.Proxy.Body(r.PathValue("id"), r.PathValue("which")) + if err != nil { + httpErr(w, http.StatusNotFound, "body not retained (body capture was off, or it was cleared)") + return + } + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", "attachment") + w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox") + s.audit(r, "interception.body_viewed", r.PathValue("id")+"/"+r.PathValue("which")) + w.Write(body) +} diff --git a/internal/api/server.go b/internal/api/server.go index 1d11799..34978a8 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -21,6 +21,7 @@ import ( "pilens.dev/pilens/internal/capture" "pilens.dev/pilens/internal/config" "pilens.dev/pilens/internal/helper" + "pilens.dev/pilens/internal/proxy" "pilens.dev/pilens/internal/store" ) @@ -39,6 +40,7 @@ type Server struct { Engine *capture.Engine Helper helper.Client Version string + Proxy *proxy.Manager Web fs.FS // built UI; may be nil in tests SetupTokenPath string DataDir string @@ -82,6 +84,7 @@ func (s *Server) Handler() http.Handler { s.networkRoutes(priv) s.captureRoutes(priv) s.analysisRoutes(priv) + s.interceptionRoutes(priv) if s.Web != nil { files := http.FileServerFS(s.Web) diff --git a/internal/capture/hub.go b/internal/capture/hub.go index 7e6654e..60226f2 100644 --- a/internal/capture/hub.go +++ b/internal/capture/hub.go @@ -51,3 +51,4 @@ func (h *Hub) Publish(e Event) { } } } + diff --git a/internal/helper/helper.go b/internal/helper/helper.go index e52cc1f..a4e4ee7 100644 --- a/internal/helper/helper.go +++ b/internal/helper/helper.go @@ -16,8 +16,10 @@ import ( const DefaultSocket = "/run/pilens/helper.sock" type Request struct { - Op string `json:"op"` - Network *config.Network `json:"network,omitempty"` + Op string `json:"op"` + Network *config.Network `json:"network,omitempty"` + InterceptEnabled bool `json:"intercept_enabled,omitempty"` + InterceptPort int `json:"intercept_port,omitempty"` } type Response struct { @@ -32,6 +34,7 @@ const ( OpStatus = "status" OpStations = "stations" OpLeases = "leases" + OpSetInterception = "set_interception" ) type ApplyResult struct { diff --git a/internal/netgen/netgen.go b/internal/netgen/netgen.go index a3eda6d..a8455eb 100644 --- a/internal/netgen/netgen.go +++ b/internal/netgen/netgen.go @@ -129,11 +129,19 @@ func DnsmasqConf(n config.Network) string { func q(s string) string { return `"` + s + `"` } // inputs are validated interface names +// Intercept describes transparent HTTPS-interception redirection. Port 0 / nil +// means interception is off and no redirect rules are emitted. +type Intercept struct { + Enabled bool + Port int +} + // Nftables renders the complete inet pilens table. The output begins by // (re)creating and deleting the table so `nft -f` replaces it atomically while -// leaving every other table untouched. -func Nftables(n config.Network) string { +// leaving every other table untouched. ic may be nil (interception off). +func Nftables(n config.Network, ic *Intercept) string { ap, up := q(n.AP.Interface), q(n.Uplink) + intercept := ic != nil && ic.Enabled && ic.Port > 0 sub := n.LAN.Subnet().String() var mgmtIf []string for _, i := range n.Management.Interfaces { @@ -186,6 +194,12 @@ func Nftables(n config.Network) string { w("\t\tip daddr %s udp dport 53 accept", n.LAN.Gateway()) w("\t\tip daddr %s tcp dport 53 accept", n.LAN.Gateway()) w("\t\ticmp type { echo-request, destination-unreachable, time-exceeded } accept") + if intercept { + // Accept ONLY connections the prerouting redirect created (ct status dnat). + // A client connecting directly to the proxy port is not dnat'd, so this + // never turns the proxy into an open proxy reachable by clients. + w("\t\ttcp dport %d ct status dnat accept comment \"redirected to interception proxy\"", ic.Port) + } w("\t\tcounter drop comment \"AP clients cannot reach the appliance\"") w("\t}") @@ -213,6 +227,17 @@ func Nftables(n config.Network) string { w("\t\tcounter drop") w("\t}") + // Transparent interception: redirect AP-client HTTP(S) to the local proxy. + // Only when enabled, and only new connections from the AP interface. The + // proxy's own upstream connections originate on the Pi (not iifname ap), so + // they are never redirected — no loop. + w("\tchain prerouting {") + w("\t\ttype nat hook prerouting priority dstnat; policy accept;") + if intercept { + w("\t\tiifname %s meta l4proto tcp tcp dport { 80, 443 } redirect to :%d", ap, ic.Port) + } + w("\t}") + w("\tchain postrouting {") w("\t\ttype nat hook postrouting priority srcnat; policy accept;") w("\t\tip saddr %s oifname %s masquerade", sub, up) diff --git a/internal/netgen/netgen_test.go b/internal/netgen/netgen_test.go index 45b59ba..6849cd2 100644 --- a/internal/netgen/netgen_test.go +++ b/internal/netgen/netgen_test.go @@ -66,7 +66,7 @@ func TestDnsmasq(t *testing.T) { } func TestNftables(t *testing.T) { - s := Nftables(testNet()) + s := Nftables(testNet(), nil) for _, want := range []string{ "delete table inet pilens", `iifname "wlan0" jump ap_forward`, @@ -84,7 +84,24 @@ func TestNftables(t *testing.T) { } n := testNet() n.Forwarding.Mode = "deny" - if !strings.Contains(Nftables(n), `"forwarding disabled"`) { + if !strings.Contains(Nftables(n, nil), `"forwarding disabled"`) { t.Error("deny mode must drop forwarded traffic") } } + +func TestNftablesInterception(t *testing.T) { + off := Nftables(testNet(), nil) + if strings.Contains(off, "redirect to") || strings.Contains(off, "ct status dnat") { + t.Fatal("interception rules present when disabled") + } + on := Nftables(testNet(), &Intercept{Enabled: true, Port: 8081}) + for _, want := range []string{ + "chain prerouting {", + `iifname "wlan0" meta l4proto tcp tcp dport { 80, 443 } redirect to :8081`, + "tcp dport 8081 ct status dnat accept", + } { + if !strings.Contains(on, want) { + t.Errorf("interception ruleset missing %q", want) + } + } +} diff --git a/internal/proxy/ca.go b/internal/proxy/ca.go new file mode 100644 index 0000000..766e8ae --- /dev/null +++ b/internal/proxy/ca.go @@ -0,0 +1,165 @@ +// Package proxy manages optional, authorised HTTPS interception: a locally +// generated CA that the operator installs on test devices by hand, a mitmdump +// service, and ingestion of the decrypted HTTP transactions. +// +// Consent is the whole design. The CA private key never leaves the appliance, +// is never pushed to a device, and interception only happens for devices whose +// owner has explicitly installed and trusted the CA. Nothing is covert. +package proxy + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "math/big" + "os" + "path/filepath" + "strings" + "time" +) + +// CAInfo describes the interception CA for display. It never includes the key. +type CAInfo struct { + Present bool `json:"present"` + Subject string `json:"subject"` + SHA256 string `json:"sha256"` // fingerprint, colon-separated + NotBefore string `json:"not_before"` + NotAfter string `json:"not_after"` + Serial string `json:"serial"` + DaysLeft int `json:"days_left"` + KeyType string `json:"key_type"` +} + +// caPaths returns the key+cert path (mitmproxy format) and the public cert path. +func caPaths(dir string) (combined, certOnly string) { + return filepath.Join(dir, "mitmproxy-ca.pem"), filepath.Join(dir, "mitmproxy-ca-cert.pem") +} + +// EnsureCA creates the interception CA if absent, in the mitmproxy confdir layout. +// RSA-2048 is used (not EC) for the broadest device trust-store compatibility. +func EnsureCA(dir, hostname string) error { + combined, certOnly := caPaths(dir) + if _, err := os.Stat(combined); err == nil { + return nil + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + serial, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + cn := "PiLens Interception CA" + if hostname != "" { + cn = "PiLens Interception CA (" + hostname + ")" + } + tpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: cn, Organization: []string{"PiLens appliance"}}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().AddDate(3, 0, 0), + IsCA: true, + BasicConstraintsValid: true, + MaxPathLen: 0, + MaxPathLenZero: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) + if err != nil { + return err + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + // mitmproxy reads the key+cert from mitmproxy-ca.pem (0600) and derives the rest. + if err := writeFile(combined, append(append([]byte{}, keyPEM...), certPEM...), 0o600); err != nil { + return err + } + // Public cert for download / device install (never contains the key). + if err := writeFile(certOnly, certPEM, 0o644); err != nil { + return err + } + writeFile(filepath.Join(dir, "mitmproxy-ca-cert.cer"), certPEM, 0o644) + return nil +} + +func writeFile(path string, data []byte, mode os.FileMode) error { + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, mode); err != nil { + return err + } + os.Chmod(tmp, mode) + return os.Rename(tmp, path) +} + +// PublicCertPEM returns the CA's public certificate (safe to distribute). It never +// returns the private key. +func PublicCertPEM(dir string) ([]byte, error) { + _, certOnly := caPaths(dir) + b, err := os.ReadFile(certOnly) + if err != nil { + return nil, errors.New("interception CA not generated yet") + } + if strings.Contains(string(b), "PRIVATE KEY") { // defensive: must never happen + return nil, errors.New("refusing to serve a file containing a private key") + } + return b, nil +} + +// Info reads the CA metadata for display. +func Info(dir string) CAInfo { + _, certOnly := caPaths(dir) + b, err := os.ReadFile(certOnly) + if err != nil { + return CAInfo{Present: false} + } + blk, _ := pem.Decode(b) + if blk == nil { + return CAInfo{Present: false} + } + c, err := x509.ParseCertificate(blk.Bytes) + if err != nil { + return CAInfo{Present: false} + } + sum := sha256.Sum256(c.Raw) + return CAInfo{ + Present: true, + Subject: c.Subject.String(), + SHA256: colonHex(sum[:]), + NotBefore: c.NotBefore.UTC().Format("2006-01-02"), + NotAfter: c.NotAfter.UTC().Format("2006-01-02"), + Serial: c.SerialNumber.Text(16), + DaysLeft: int(time.Until(c.NotAfter).Hours() / 24), + KeyType: "RSA 2048", + } +} + +// Rotate deletes the CA so a fresh one is generated. Devices trusting the old CA +// must install the new one; existing interception breaks until they do. +func Rotate(dir, hostname string) error { + combined, certOnly := caPaths(dir) + for _, p := range []string{combined, certOnly, filepath.Join(dir, "mitmproxy-ca-cert.cer"), filepath.Join(dir, "mitmproxy-ca.p12"), filepath.Join(dir, "mitmproxy-ca-cert.p12"), filepath.Join(dir, "mitmproxy-dhparam.pem")} { + os.Remove(p) + } + return EnsureCA(dir, hostname) +} + +func colonHex(b []byte) string { + h := strings.ToUpper(hex.EncodeToString(b)) + var parts []string + for i := 0; i < len(h); i += 2 { + parts = append(parts, h[i:i+2]) + } + return strings.Join(parts, ":") +} + +// Fingerprint of a cert file, for the install instructions ("verify this matches"). +func Fingerprint(dir string) string { + return fmt.Sprintf("SHA-256 %s", Info(dir).SHA256) +} diff --git a/internal/proxy/ca_test.go b/internal/proxy/ca_test.go new file mode 100644 index 0000000..288f2ba --- /dev/null +++ b/internal/proxy/ca_test.go @@ -0,0 +1,62 @@ +package proxy + +import ( + "strings" + "testing" +) + +func TestCAGeneration(t *testing.T) { + dir := t.TempDir() + if err := EnsureCA(dir, "testpi"); err != nil { + t.Fatal(err) + } + // Public cert must never contain a private key. + pub, err := PublicCertPEM(dir) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(pub), "PRIVATE KEY") { + t.Fatal("public cert leaked the private key") + } + if !strings.Contains(string(pub), "BEGIN CERTIFICATE") { + t.Fatal("not a certificate") + } + info := Info(dir) + if !info.Present || !strings.Contains(info.Subject, "PiLens") || len(info.SHA256) != 95 || info.DaysLeft < 1000 { + t.Fatalf("bad CA info: %+v", info) + } + // Idempotent: second call keeps the same CA. + before := info.SHA256 + EnsureCA(dir, "testpi") + if Info(dir).SHA256 != before { + t.Fatal("EnsureCA regenerated an existing CA") + } + // Rotate produces a different CA. + if err := Rotate(dir, "testpi"); err != nil { + t.Fatal(err) + } + if Info(dir).SHA256 == before { + t.Fatal("rotate did not change the CA") + } +} + +func TestConfigValidate(t *testing.T) { + bad := []Config{ + {MaxBodyKB: -1}, + {MaxBodyKB: 999999}, + {ExcludeHosts: []string{"ok.com", "bad regex ("}}, + {ExcludeHosts: []string{"has space"}}, + } + for i, c := range bad { + if c.Validate() == nil { + t.Errorf("case %d accepted invalid config", i) + } + } + good := Config{MaxBodyKB: 256, ExcludeHosts: []string{`(?i)\.apple\.com$`, "example\\.com"}} + if err := good.Validate(); err != nil { + t.Fatal(err) + } + if r := good.ExcludeRegex(); !strings.Contains(r, "apple") || !strings.Contains(r, "|") { + t.Fatalf("exclude regex wrong: %s", r) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go new file mode 100644 index 0000000..2985835 --- /dev/null +++ b/internal/proxy/proxy.go @@ -0,0 +1,286 @@ +package proxy + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/netip" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "pilens.dev/pilens/internal/store" +) + +// Config is the interception toggle and its options. Stored in settings. +type Config struct { + Enabled bool `json:"enabled"` + CaptureBodies bool `json:"capture_bodies"` // default false: sensitive content off by default + MaxBodyKB int `json:"max_body_kb"` + ExcludeHosts []string `json:"exclude_hosts"` // host regexes NOT decrypted (passed through) +} + +func DefaultConfig() Config { return Config{Enabled: false, CaptureBodies: false, MaxBodyKB: 256} } + +const ListenPort = 8081 + +var hostRe = regexp.MustCompile(`^[A-Za-z0-9.*?+()\[\]{}|\\^$_:-]{1,200}$`) + +func (c Config) Validate() error { + if c.MaxBodyKB < 0 || c.MaxBodyKB > 10240 { + return errors.New("max_body_kb must be 0-10240") + } + for _, h := range c.ExcludeHosts { + if !hostRe.MatchString(h) { + return errors.New("invalid exclude-host pattern: " + h) + } + if _, err := regexp.Compile(h); err != nil { + return errors.New("invalid exclude-host regex " + h + ": " + err.Error()) + } + } + return nil +} + +// Sink receives live events for the browser (adapts the capture Hub). +type Sink interface{ Publish(eventType string, data []byte) } + +// Flow is one HTTP(S) transaction, as emitted by the mitmdump addon and stored. +type Flow struct { + ID string `json:"id"` + TS float64 `json:"ts"` + Client string `json:"client"` + Method string `json:"method"` + Scheme string `json:"scheme"` + Host string `json:"host"` + Port int `json:"port"` + Path string `json:"path"` + HTTPVersion string `json:"http_version"` + Status int `json:"status"` + Reason string `json:"reason"` + ReqCType string `json:"req_ctype"` + RespCType string `json:"resp_ctype"` + ReqSize int64 `json:"req_size"` + RespSize int64 `json:"resp_size"` + DurationMS int64 `json:"duration_ms"` + TLSVersion string `json:"tls_version"` + SNI string `json:"sni"` + ALPN string `json:"alpn"` + Intercepted bool `json:"intercepted"` + Error string `json:"error"` + WebSocket bool `json:"websocket"` + HasReqBody bool `json:"has_req_body"` + HasRespBody bool `json:"has_resp_body"` + ReqHeaders map[string]string `json:"req_headers"` + RespHeaders map[string]string `json:"resp_headers"` +} + +// Manager owns interception state, the flow ingester and queries. +type Manager struct { + DB *store.Store + Dir string // /var/lib/pilens/proxy + Hostname string + Sink Sink + + mu sync.Mutex + cfg Config + cancel context.CancelFunc +} + +func New(db *store.Store, dir, hostname string, sink Sink) *Manager { + m := &Manager{DB: db, Dir: dir, Hostname: hostname, Sink: sink, cfg: DefaultConfig()} + db.GetJSON("interception", &m.cfg) + return m +} + +func (m *Manager) Config() Config { m.mu.Lock(); defer m.mu.Unlock(); return m.cfg } + +func (m *Manager) SetConfig(c Config) error { + if err := c.Validate(); err != nil { + return err + } + m.mu.Lock() + m.cfg = c + m.mu.Unlock() + return m.DB.PutJSON("interception", c) +} + +func (m *Manager) flowsPath() string { return filepath.Join(m.Dir, "flows.ndjson") } +func (m *Manager) bodiesDir() string { return filepath.Join(m.Dir, "bodies") } +func (m *Manager) ConfDir() string { return filepath.Join(m.Dir, "ca") } + +func (m *Manager) EnsureCA() error { return EnsureCA(m.ConfDir(), m.Hostname) } +func (m *Manager) CAInfo() CAInfo { return Info(m.ConfDir()) } +func (m *Manager) CACertPEM() ([]byte, error) { return PublicCertPEM(m.ConfDir()) } +func (m *Manager) RotateCA() error { return Rotate(m.ConfDir(), m.Hostname) } + +// Start begins tailing the flow file. It runs for the service lifetime; the +// mitmdump service is started/stopped by the helper, this just ingests. +func (m *Manager) Start(ctx context.Context) { + os.MkdirAll(m.bodiesDir(), 0o750) + ctx, cancel := context.WithCancel(ctx) + m.cancel = cancel + go m.tail(ctx) +} + +// tail follows flows.ndjson, inserting new lines. Survives file rotation/truncation. +func (m *Manager) tail(ctx context.Context) { + path := m.flowsPath() + var offset int64 + m.DB.GetJSON("interception_flow_offset", &offset) + for ctx.Err() == nil { + f, err := os.Open(path) + if err != nil { + select { + case <-ctx.Done(): + return + case <-time.After(2 * time.Second): + } + continue + } + if fi, _ := f.Stat(); fi != nil && fi.Size() < offset { + offset = 0 // file was rotated/truncated + } + f.Seek(offset, io.SeekStart) + r := bufio.NewReaderSize(f, 1<<20) + for ctx.Err() == nil { + line, rerr := r.ReadBytes('\n') + if len(line) > 0 && line[len(line)-1] == '\n' { + offset += int64(len(line)) + m.ingest(line) + m.DB.PutJSON("interception_flow_offset", offset) + continue + } + _ = rerr // partial/EOF: keep offset, re-read after the wait + break + } + f.Close() + select { + case <-ctx.Done(): + return + case <-time.After(500 * time.Millisecond): + } + } +} + +func (m *Manager) ingest(line []byte) { + var fl Flow + if err := json.Unmarshal(line, &fl); err != nil || fl.ID == "" { + return + } + rh, _ := json.Marshal(fl.ReqHeaders) + sh, _ := json.Marshal(fl.RespHeaders) + _, err := m.DB.Exec(`INSERT OR REPLACE INTO http_flows + (id,ts,client,method,scheme,host,port,path,http_version,status,reason,req_ctype,resp_ctype,req_size,resp_size,duration_ms,tls_version,sni,alpn,intercepted,error,websocket,has_req_body,has_resp_body,req_headers,resp_headers) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + fl.ID, fl.TS, fl.Client, fl.Method, fl.Scheme, fl.Host, fl.Port, fl.Path, fl.HTTPVersion, fl.Status, fl.Reason, + fl.ReqCType, fl.RespCType, fl.ReqSize, fl.RespSize, fl.DurationMS, fl.TLSVersion, fl.SNI, fl.ALPN, + b2i(fl.Intercepted), fl.Error, b2i(fl.WebSocket), b2i(fl.HasReqBody), b2i(fl.HasRespBody), string(rh), string(sh)) + if err != nil { + slog.Warn("http flow insert", "err", err) + return + } + if m.Sink != nil { + m.Sink.Publish("http", line) + } +} + +func b2i(b bool) int { + if b { + return 1 + } + return 0 +} + +// --- queries --- + +const flowCols = `id,ts,client,method,scheme,host,port,path,http_version,status,reason,req_ctype,resp_ctype,req_size,resp_size,duration_ms,tls_version,sni,alpn,intercepted,error,websocket,has_req_body,has_resp_body,req_headers,resp_headers` + +func scanFlow(sc interface{ Scan(...any) error }) (Flow, error) { + var f Flow + var rh, sh string + var intercept, ws, hrb, hsb int + err := sc.Scan(&f.ID, &f.TS, &f.Client, &f.Method, &f.Scheme, &f.Host, &f.Port, &f.Path, &f.HTTPVersion, &f.Status, &f.Reason, + &f.ReqCType, &f.RespCType, &f.ReqSize, &f.RespSize, &f.DurationMS, &f.TLSVersion, &f.SNI, &f.ALPN, + &intercept, &f.Error, &ws, &hrb, &hsb, &rh, &sh) + f.Intercepted, f.WebSocket, f.HasReqBody, f.HasRespBody = intercept == 1, ws == 1, hrb == 1, hsb == 1 + json.Unmarshal([]byte(rh), &f.ReqHeaders) + json.Unmarshal([]byte(sh), &f.RespHeaders) + return f, err +} + +// Flows returns recent transactions, newest first, optionally filtered by a +// substring over host/path/client and a client IP. +func (m *Manager) Flows(ctx context.Context, q, client string, limit int) ([]Flow, error) { + if limit <= 0 || limit > 5000 { + limit = 1000 + } + where := "1=1" + args := []any{} + if q != "" { + where += " AND (host LIKE ? OR path LIKE ? OR client LIKE ?)" + like := "%" + q + "%" + args = append(args, like, like, like) + } + if client != "" { + where += " AND client = ?" + args = append(args, client) + } + args = append(args, limit) + rows, err := m.DB.QueryContext(ctx, `SELECT `+flowCols+` FROM http_flows WHERE `+where+` ORDER BY ts DESC LIMIT ?`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []Flow{} + for rows.Next() { + f, err := scanFlow(rows) + if err != nil { + return nil, err + } + out = append(out, f) + } + return out, rows.Err() +} + +func (m *Manager) Flow(ctx context.Context, id string) (Flow, error) { + return scanFlow(m.DB.QueryRowContext(ctx, `SELECT `+flowCols+` FROM http_flows WHERE id=?`, id)) +} + +// Body returns a stored request/response body. which is "req" or "resp". +// Bodies exist only when body capture was enabled for that flow. +func (m *Manager) Body(id, which string) ([]byte, error) { + if which != "req" && which != "resp" || !flowIDRe.MatchString(id) { + return nil, errors.New("invalid body request") + } + return os.ReadFile(filepath.Join(m.bodiesDir(), id+"."+which)) +} + +var flowIDRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + +// Clear deletes all intercepted HTTP data (flows + bodies). Captures are untouched. +func (m *Manager) Clear() error { + m.DB.Exec(`DELETE FROM http_flows`) + os.RemoveAll(m.bodiesDir()) + os.MkdirAll(m.bodiesDir(), 0o750) + os.Truncate(m.flowsPath(), 0) + m.DB.PutJSON("interception_flow_offset", int64(0)) + return nil +} + +// ClientIP validates a client filter value. +func ClientIP(s string) bool { _, err := netip.ParseAddr(s); return err == nil || s == "" } + +// ExcludeRegex joins exclude hosts into one mitmdump --ignore-hosts pattern. +func (c Config) ExcludeRegex() string { + var parts []string + for _, h := range c.ExcludeHosts { + parts = append(parts, "(?:"+h+")") + } + return strings.Join(parts, "|") +} diff --git a/migrations/002_http.sql b/migrations/002_http.sql new file mode 100644 index 0000000..71566e4 --- /dev/null +++ b/migrations/002_http.sql @@ -0,0 +1,33 @@ +-- Intercepted HTTP(S) transactions (Mode B). Populated only while interception +-- is enabled and only for devices that trust the installed CA. +CREATE TABLE http_flows ( + id TEXT PRIMARY KEY, + ts REAL NOT NULL, + client TEXT NOT NULL DEFAULT '', + method TEXT NOT NULL DEFAULT '', + scheme TEXT NOT NULL DEFAULT '', + host TEXT NOT NULL DEFAULT '', + port INTEGER NOT NULL DEFAULT 0, + path TEXT NOT NULL DEFAULT '', + http_version TEXT NOT NULL DEFAULT '', + status INTEGER NOT NULL DEFAULT 0, + reason TEXT NOT NULL DEFAULT '', + req_ctype TEXT NOT NULL DEFAULT '', + resp_ctype TEXT NOT NULL DEFAULT '', + req_size INTEGER NOT NULL DEFAULT 0, + resp_size INTEGER NOT NULL DEFAULT 0, + duration_ms INTEGER NOT NULL DEFAULT 0, + tls_version TEXT NOT NULL DEFAULT '', + sni TEXT NOT NULL DEFAULT '', + alpn TEXT NOT NULL DEFAULT '', + intercepted INTEGER NOT NULL DEFAULT 0, -- 1 = decrypted, 0 = passthrough/failed + error TEXT NOT NULL DEFAULT '', + websocket INTEGER NOT NULL DEFAULT 0, + has_req_body INTEGER NOT NULL DEFAULT 0, + has_resp_body INTEGER NOT NULL DEFAULT 0, + req_headers TEXT NOT NULL DEFAULT '{}', + resp_headers TEXT NOT NULL DEFAULT '{}' +); +CREATE INDEX http_flows_ts ON http_flows(ts); +CREATE INDEX http_flows_host ON http_flows(host); +CREATE INDEX http_flows_client ON http_flows(client); diff --git a/packaging/mitm_addon.py b/packaging/mitm_addon.py new file mode 100644 index 0000000..2ce8ed4 --- /dev/null +++ b/packaging/mitm_addon.py @@ -0,0 +1,157 @@ +"""PiLens mitmdump addon. + +Emits one JSON line per HTTP(S) transaction (and per client TLS failure) to +PILENS_FLOWS. Request/response bodies are written to PILENS_BODIES only when +PILENS_CAPTURE_BODIES=1, size-limited by PILENS_MAX_BODY_KB. Sensitive content +is therefore off by default. + +This runs inside the pilens-proxy service's venv; it is not imported anywhere else. +""" +import json +import os +import time + +FLOWS = os.environ.get("PILENS_FLOWS", "/var/lib/pilens/proxy/flows.ndjson") +BODIES = os.environ.get("PILENS_BODIES", "/var/lib/pilens/proxy/bodies") +CAPTURE_BODIES = os.environ.get("PILENS_CAPTURE_BODIES", "0") == "1" +MAX_BODY = int(os.environ.get("PILENS_MAX_BODY_KB", "256")) * 1024 + + +def _emit(obj): + try: + with open(FLOWS, "a") as f: + f.write(json.dumps(obj, separators=(",", ":")) + "\n") + f.flush() + except Exception: + pass + + +def _headers(msg): + out = {} + for k, v in msg.headers.items(multi=True): + out[k] = (out[k] + ", " + v) if k in out else v + return out + + +def _write_body(flow_id, which, content): + if not CAPTURE_BODIES or not content: + return False + data = content[:MAX_BODY] + try: + os.makedirs(BODIES, exist_ok=True) + with open(os.path.join(BODIES, "%s.%s" % (flow_id, which)), "wb") as f: + f.write(data) + return True + except Exception: + return False + + +def _client(flow): + try: + return flow.client_conn.peername[0] + except Exception: + return "" + + +def _tls(flow): + cc = flow.client_conn + sc = flow.server_conn + ver = getattr(cc, "tls_version", None) or getattr(sc, "tls_version", None) or "" + alpn = getattr(cc, "alpn", b"") or b"" + try: + alpn = alpn.decode() + except Exception: + alpn = str(alpn) + return ver, getattr(cc, "sni", None) or "", alpn + + +class PiLens: + def response(self, flow): + req, resp = flow.request, flow.response + ver, sni, alpn = _tls(flow) + dur = 0 + try: + dur = int((resp.timestamp_end - req.timestamp_start) * 1000) + except Exception: + pass + has_req = _write_body(flow.id, "req", req.raw_content) + has_resp = _write_body(flow.id, "resp", resp.raw_content) + _emit({ + "id": flow.id, + "ts": req.timestamp_start or time.time(), + "client": _client(flow), + "method": req.method, + "scheme": req.scheme, + "host": req.pretty_host, + "port": req.port, + "path": req.path, + "http_version": resp.http_version, + "status": resp.status_code, + "reason": resp.reason or "", + "req_ctype": req.headers.get("content-type", ""), + "resp_ctype": resp.headers.get("content-type", ""), + "req_size": len(req.raw_content or b""), + "resp_size": len(resp.raw_content or b""), + "duration_ms": dur, + "tls_version": ver, + "sni": sni, + "alpn": alpn, + "intercepted": req.scheme == "https", + "error": "", + "websocket": bool(flow.websocket), + "has_req_body": has_req, + "has_resp_body": has_resp, + "req_headers": _headers(req), + "resp_headers": _headers(resp), + }) + + def error(self, flow): + try: + req = flow.request + ver, sni, alpn = _tls(flow) + _emit({ + "id": flow.id, + "ts": (req.timestamp_start if req else None) or time.time(), + "client": _client(flow), + "method": req.method if req else "", + "scheme": req.scheme if req else "", + "host": req.pretty_host if req else "", + "port": req.port if req else 0, + "path": req.path if req else "", + "status": 0, + "tls_version": ver, + "sni": sni, + "alpn": alpn, + "intercepted": False, + "error": str(flow.error) if flow.error else "connection error", + "req_headers": {}, + "resp_headers": {}, + }) + except Exception: + pass + + def tls_failed_client(self, data): + # The device refused our certificate: it does not trust the CA, or the + # app pins a specific certificate. Reported honestly, never bypassed. + try: + conn = data.conn + peer = conn.peername[0] if conn.peername else "" + sni = getattr(data.context.client, "sni", None) or getattr(conn, "sni", None) or "" + _emit({ + "id": "tlsfail-%s-%d" % (peer.replace(".", "_").replace(":", "_"), int(time.time() * 1000) % 1000000000), + "ts": time.time(), + "client": peer, + "scheme": "https", + "host": sni, + "sni": sni, + "status": 0, + "intercepted": False, + "error": "client TLS failure: device does not trust the PiLens CA, or the app pins its own certificate", + "req_headers": {}, + "resp_headers": {}, + }) + except Exception: + pass + + +addons = [PiLens()] diff --git a/scripts/diagnose.sh b/scripts/diagnose.sh index 287294d..067ab89 100755 --- a/scripts/diagnose.sh +++ b/scripts/diagnose.sh @@ -11,7 +11,7 @@ run() { local f="$D/$1"; shift; { echo "\$ $*"; "$@"; } >"$f" 2>&1 || true; } redact() { sed -E 's/("passphrase"[[:space:]]*:[[:space:]]*")[^"]*/\1REDACTED/g; s/^(psk=).*/\1REDACTED/; s/(setup_token=).*/\1REDACTED/'; } run versions.txt sh -c 'cat /opt/pilens/VERSION; uname -a; cat /etc/os-release; tr -d "\0" /dev/null; echo; [ -f /var/lib/pilens/proxy/ca/mitmproxy-ca-cert.pem ] && openssl x509 -in /var/lib/pilens/proxy/ca/mitmproxy-ca-cert.pem -noout -subject -fingerprint -sha256 -dates; /opt/pilens/proxy-venv/bin/mitmdump --version 2>/dev/null | head -1' run capture.txt sh -c 'getcap /usr/bin/dumpcap; id pilens; ls -la /var/lib/pilens /var/lib/pilens-helper' run resources.txt sh -c 'df -h; free -m; uptime; vcgencmd get_throttled; vcgencmd measure_temp; timedatectl' cp /var/lib/pilens-helper/dnsmasq.conf "$D/dnsmasq.conf" 2>/dev/null diff --git a/scripts/install.sh b/scripts/install.sh index 0403205..ed2119e 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -169,14 +169,33 @@ usermod -aG wireshark pilens systemctl stop pilens 2>/dev/null || true mkdir -p "$PREFIX" install -m 0755 "$SRC/bin/pilens" "$SRC/bin/pilens-helper" -D -t "$PREFIX/bin" +install -m 0755 "$SRC/proxy-run.sh" "$PREFIX/proxy-run.sh" +install -m 0644 "$SRC/mitm_addon.py" "$PREFIX/mitm_addon.py" for f in install.sh uninstall.sh repair.sh diagnose.sh setup.example.yaml VERSION README.md LICENSE; do install -m 0644 "$SRC/$f" "$PREFIX/$f"; done chmod 0755 "$PREFIX"/*.sh rm -rf "$PREFIX/docs"; cp -r "$SRC/docs" "$PREFIX/docs" install -d -o pilens -g pilens -m 0750 "$DATA" +install -d -o pilens -g pilens -m 0750 "$DATA/proxy" install -d -o root -g root -m 0700 /var/lib/pilens-helper install -m 0644 "$SRC"/systemd/*.service /etc/systemd/system/ systemctl daemon-reload +# ---------------------------------------------------------------- 16. optional TLS proxy deps +say "Preparing optional HTTPS-interception proxy (mitmproxy)" +VENV="$PREFIX/proxy-venv" +if [ -x "$VENV/bin/mitmdump" ]; then + echo " mitmproxy venv already present" +elif apt-get install -y -qq --no-install-recommends python3-venv >>"$LOG" 2>&1 && python3 -m venv "$VENV" >>"$LOG" 2>&1; then + if "$VENV/bin/pip" install --quiet --only-binary=:all: --upgrade pip >>"$LOG" 2>&1 && "$VENV/bin/pip" install --quiet --only-binary=:all: mitmproxy >>"$LOG" 2>&1; then + chown -R pilens:pilens "$VENV" + echo " mitmproxy installed ($("$VENV/bin/mitmdump" --version 2>/dev/null | head -1))" + else + warn "mitmproxy install failed (needs internet + arm64 wheels). HTTPS interception will be unavailable until you run: sudo $PREFIX/repair.sh" + fi +else + warn "could not create the proxy virtualenv; HTTPS interception will be unavailable (optional feature)" +fi + # ---------------------------------------------------------------- 15. initialise data say "Initialising database and management certificate" INIT=$(runuser -u pilens -- "$PREFIX/bin/pilens" init -data "$DATA" -mgmt-iface "$MGMT_IF" -mgmt-subnet "$MGMT_NET") diff --git a/scripts/proxy-run.sh b/scripts/proxy-run.sh new file mode 100755 index 0000000..6588808 --- /dev/null +++ b/scripts/proxy-run.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Launches mitmdump for PiLens interception from the core-written env file. +# Runs as the unprivileged 'pilens' user under the pilens-proxy.service unit. +set -eu +ENV=/var/lib/pilens/proxy/proxy.env +[ -f "$ENV" ] || { echo "no proxy.env; interception not configured" >&2; exit 1; } +. "$ENV" +export PILENS_FLOWS PILENS_BODIES PILENS_CAPTURE_BODIES PILENS_MAX_BODY_KB +args=( --mode transparent --listen-host 0.0.0.0 --listen-port "$PILENS_PROXY_PORT" + --set "confdir=$PILENS_CONFDIR" --set block_global=false --set connection_strategy=lazy + --set stream_large_bodies=1m -s "$PILENS_ADDON" -q ) +if [ -n "${PILENS_IGNORE:-}" ]; then args+=( --ignore-hosts "$PILENS_IGNORE" ); fi +exec "$PILENS_VENV/bin/mitmdump" "${args[@]}" diff --git a/scripts/repair.sh b/scripts/repair.sh index 9ad5e54..103bb51 100755 --- a/scripts/repair.sh +++ b/scripts/repair.sh @@ -18,6 +18,18 @@ if ! getcap /usr/bin/dumpcap 2>/dev/null | grep -q cap_net_raw; then fi getcap /usr/bin/dumpcap | grep -q cap_net_raw && ok "dumpcap capture capabilities" || bad "dumpcap lacks capabilities" +# Optional: rebuild the mitmproxy venv if it is missing (HTTPS interception). +VENV=/opt/pilens/proxy-venv +if [ ! -x "$VENV/bin/mitmdump" ]; then + if python3 -m venv "$VENV" >/dev/null 2>&1 && "$VENV/bin/pip" install --quiet --only-binary=:all: mitmproxy >/dev/null 2>&1; then + chown -R pilens:pilens "$VENV"; ok "mitmproxy venv rebuilt" + else + bad "mitmproxy venv missing and reinstall failed (interception unavailable; needs internet)" + fi +else + ok "mitmproxy venv present" +fi + systemctl daemon-reload systemctl enable pilens-firewall pilens-helper pilens-dnsmasq pilens >/dev/null 2>&1 systemctl reset-failed 'pilens*' 2>/dev/null diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index 636f749..5412b38 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -15,7 +15,7 @@ if [ $PURGE = 1 ]; then [ "$x" = DELETE ] || { echo "not confirmed; keeping data"; PURGE=0; } fi -systemctl disable --now pilens pilens-dnsmasq pilens-helper pilens-firewall 2>/dev/null || true +systemctl disable --now pilens pilens-proxy pilens-dnsmasq pilens-helper pilens-firewall 2>/dev/null || true nmcli connection delete pilens-ap 2>/dev/null && echo "removed access-point profile (other Wi-Fi profiles were never modified and will reconnect)" || true rm -f /etc/NetworkManager/system-connections/pilens-ap.nmconnection nft delete table inet pilens 2>/dev/null && echo "removed firewall table inet pilens" || true diff --git a/systemd/pilens-proxy.service b/systemd/pilens-proxy.service new file mode 100644 index 0000000..129c0e3 --- /dev/null +++ b/systemd/pilens-proxy.service @@ -0,0 +1,26 @@ +[Unit] +Description=PiLens HTTPS interception proxy (mitmdump, authorised Mode B) +Documentation=file:/opt/pilens/docs/tls-and-encryption.md +After=network-online.target pilens.service +Wants=network-online.target +PartOf=pilens.service +ConditionPathExists=/var/lib/pilens/proxy/proxy.env + +[Service] +Type=simple +User=pilens +Group=pilens +# Transparent mode reads SO_ORIGINAL_DST on redirected sockets; no extra privilege. +ExecStart=/opt/pilens/proxy-run.sh +Restart=on-failure +RestartSec=3 +NoNewPrivileges=yes +ProtectSystem=strict +ReadWritePaths=/var/lib/pilens/proxy +ProtectHome=yes +PrivateTmp=yes +ProtectKernelModules=yes +ProtectControlGroups=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +LockPersonality=yes +# started/stopped by the network helper when interception is toggled diff --git a/web/dist/.gitkeep b/web/dist/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/web/src/App.tsx b/web/src/App.tsx index bce5301..7cae351 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -15,6 +15,8 @@ import DnsExplorer from "./pages/DnsExplorer"; import TlsExplorer from "./pages/TlsExplorer"; import Statistics from "./pages/Statistics"; import Findings from "./pages/Findings"; +import HttpExplorer from "./pages/HttpExplorer"; +import Certificates from "./pages/Certificates"; import Settings from "./pages/Settings"; export interface Prefs { theme: "dark" | "light" | "system"; density: "compact" | "comfortable"; scale: number; motion: "system" | "reduce" } @@ -29,12 +31,14 @@ const NAV: [string, string, string][] = [ ["/conversations", "Conversations", "⇌"], ["/dns", "DNS Explorer", "◇"], ["/tls", "TLS Explorer", "⛨"], + ["/http", "HTTP Explorer", "⇅"], ["/statistics", "Statistics", "▦"], ["/findings", "Findings", "⚑"], ["/library", "Capture Library", "▤"], ["/devices", "Devices", "⌬"], ["", "", ""], ["/network", "Network Settings", "⇄"], + ["/certificates", "Certificates", "⛉"], ["/health", "System Health", "♥"], ["/settings", "Settings", "⚙"], ]; @@ -101,6 +105,8 @@ export default function App() { if (path.startsWith("/tls")) return ; if (path.startsWith("/statistics")) return ; if (path.startsWith("/findings")) return ; + if (path.startsWith("/http")) return ; + if (path === "/certificates") return ; if (path === "/library") return ; if (path === "/devices") return ; if (path === "/network") return ; diff --git a/web/src/api.ts b/web/src/api.ts index f4bdc87..243a211 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -56,7 +56,7 @@ let es: EventSource | null = null; export function connectEvents() { if (es) return; es = new EventSource("/api/events"); - for (const t of ["hello", "packets", "capture", "resync", "ping"]) { + for (const t of ["hello", "packets", "capture", "resync", "ping", "http"]) { es.addEventListener(t, (e) => { let d: any = {}; try { d = JSON.parse((e as MessageEvent).data); } catch { /* ignore */ } @@ -98,3 +98,14 @@ export interface TLSRecord { num: number; seg: number; time: number; client: str export interface CertRecord { num: number; seg: number; time: number; server: string; subject: string; issuer: string; sans: string; not_before: string; not_after: string } export interface Finding { rule: string; title: string; severity: "info" | "low" | "medium"; count: number; flows: number; examples: number[]; filter: string; note: string } + +// ---- Phase 4 interception types ---- +export interface CAInfo { present: boolean; subject: string; sha256: string; not_before: string; not_after: string; serial: string; days_left: number; key_type: string } +export interface InterceptionConfig { enabled: boolean; capture_bodies: boolean; max_body_kb: number; exclude_hosts: string[] } +export interface InterceptionStatus { config: InterceptionConfig; ca: CAInfo; running: boolean; port: number; note: string } +export interface HttpFlow { + id: string; ts: number; client: string; method: string; scheme: string; host: string; port: number; path: string; + http_version: string; status: number; reason: string; req_ctype: string; resp_ctype: string; req_size: number; resp_size: number; + duration_ms: number; tls_version: string; sni: string; alpn: string; intercepted: boolean; error: string; websocket: boolean; + has_req_body: boolean; has_resp_body: boolean; req_headers: Record; resp_headers: Record; +} diff --git a/web/src/pages/Certificates.tsx b/web/src/pages/Certificates.tsx new file mode 100644 index 0000000..3c715fb --- /dev/null +++ b/web/src/pages/Certificates.tsx @@ -0,0 +1,146 @@ +import { useState } from "react"; +import { api, type ApplyResult, type InterceptionStatus } from "../api"; +import { Badge, Modal, Steps, toast, usePoll } from "../ui"; + +const INSTALL: Record = { + "iOS / iPadOS": [ + "Download the CA to the device (or AirDrop it).", + "Settings opens a profile prompt → Install → enter your passcode.", + "Then Settings → General → About → Certificate Trust Settings → turn ON full trust for ‘PiLens Interception CA’.", + "Without that second step iOS will NOT trust it. This is deliberate on Apple's part.", + ], + "Android": [ + "Settings → Security → Encryption & credentials → Install a certificate → CA certificate.", + "Pick the downloaded .crt.", + "Note: Android apps only trust user-added CAs if they opt in. Many apps (and all of Chrome's pinned sites) will not be decryptable. That is expected.", + ], + "Windows": [ + "Double-click the .crt → Install Certificate → Local Machine → ‘Trusted Root Certification Authorities’.", + "Browsers using the system store (Edge, Chrome) will then trust it; Firefox uses its own store (below).", + ], + "macOS": [ + "Double-click the .crt → add to the ‘login’ or ‘System’ keychain.", + "Open Keychain Access → find ‘PiLens Interception CA’ → Get Info → Trust → ‘Always Trust’.", + ], + "Firefox": [ + "Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import.", + "Tick ‘Trust this CA to identify websites’.", + ], +}; + +export default function Certificates() { + const [st, err, reload] = usePoll("/api/interception", 8000); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState(null); + const [applyErr, setApplyErr] = useState(""); + const [os, setOs] = useState("iOS / iPadOS"); + const [confirmBodies, setConfirmBodies] = useState(false); + const [draft, setDraft] = useState(null); + + const cfg = draft ?? st?.config; + const dirty = draft !== null && JSON.stringify(draft) !== JSON.stringify(st?.config); + + const set = (p: Partial) => setDraft({ ...(cfg as any), ...p }); + + const apply = async (override?: Partial) => { + const body = { ...(cfg as any), ...override }; + setBusy(true); setApplyErr(""); setResult(null); + try { + const r = await api<{ result: ApplyResult }>("/api/interception", { method: "PUT", body }); + setResult(r.result); setDraft(null); reload(); + toast(body.enabled ? "Interception enabled" : "Interception disabled"); + } catch (e: any) { + setApplyErr(e.message); setResult(e.body?.result ?? null); reload(); + } + setBusy(false); + }; + + const ca = st?.ca; + + return ( +
+
+

Certificates & HTTPS Inspection

+

Authorised HTTPS interception (Mode B). Decrypts only for devices whose owner has installed and trusted the CA below.

+
+ +
+ Consent is required, by design. PiLens can only decrypt a device after you install this CA on it and turn on trust. + A trusted certificate can never be pushed over the network or via DHCP. Pinned apps, private trust stores and QUIC are reported as + not intercepted — never silently bypassed. The CA private key never leaves the appliance and is not downloadable. +
+ + {err &&
{err}
} + +
+
+

Interception

+ + {st &&
+ {st.running ? "proxy active" : "proxy stopped"} + transparent, port {st.port} +
} + {!ca?.present &&
The interception CA is not available. Is the mitmproxy venv installed? Run sudo /opt/pilens/repair.sh.
} + {applyErr &&
{applyErr}
} + {result && } +

Enabling adds an nftables redirect for ports 80/443 from the Wi-Fi only. The proxy never becomes a direct open proxy for clients (it accepts only redirected connections).

+
+ +
+

The appliance CA

+ {ca?.present ? <> +
+
Subject
{ca.subject}
+
Key
{ca.key_type}
+
Fingerprint
SHA-256 {ca.sha256}
+
Valid
{ca.not_before} → {ca.not_after} ({ca.days_left} days left)
+
+
+ Download CA certificate + +
+

Install this on each authorised test device, then verify the fingerprint above matches before trusting it.

+ :

No CA yet.

} +
+
+ +
+

Install instructions

+
+ {Object.keys(INSTALL).map((k) => )} +
+
    {INSTALL[os].map((s, i) =>
  1. {s}
    2. )}
+
+ + {cfg &&
+

Options

+