Phase 4: authorised HTTPS interception (Mode B)
Toggleable transparent interception via mitmdump in a venv. Own RSA CA generated in-app, downloadable (public cert only, key never leaves the box), with per-OS install guide and fingerprint. nftables redirect accepts only dnat'd connections (never an open proxy). HTTP Explorer with headers and opt-in body viewers (JSON/ text/hex/image, sandboxed). Honest status: pinned/untrusting = reported errors, excluded hosts pass through, bodies off by default. Helper op + systemd unit + installer venv build + boot restore.
This commit is contained in:
Andy Dixon
@@ -19,7 +19,7 @@ build: web
|
||||
$(GO) build -trimpath -ldflags '$(LDFLAGS)' -o dist/$(PKG)/bin/pilens-helper ./cmd/pilens-helper
|
||||
|
||||
package: build
|
||||
cp -r systemd packaging/setup.example.yaml dist/$(PKG)/
|
||||
cp -r systemd packaging/setup.example.yaml packaging/mitm_addon.py dist/$(PKG)/
|
||||
cp scripts/*.sh dist/$(PKG)/
|
||||
cp -r docs README.md LICENSE dist/$(PKG)/
|
||||
echo $(VERSION) > dist/$(PKG)/VERSION
|
||||
|
||||
+104
-2
@@ -142,6 +142,12 @@ func (s *server) handle(ctx context.Context, c *net.UnixConn) {
|
||||
data = s.status(ctx)
|
||||
case helper.OpStations:
|
||||
data, err = stations(ctx)
|
||||
case helper.OpSetInterception:
|
||||
r := s.setInterception(ctx, req.InterceptEnabled, req.InterceptPort)
|
||||
data = r
|
||||
if r.Error != "" {
|
||||
err = errors.New(r.Error)
|
||||
}
|
||||
case helper.OpLeases:
|
||||
b, rerr := os.ReadFile(netgen.LeaseFile)
|
||||
if rerr != nil && !os.IsNotExist(rerr) {
|
||||
@@ -232,7 +238,7 @@ func (s *server) apply(ctx context.Context, n config.Network) helper.ApplyResult
|
||||
return r
|
||||
}
|
||||
step("validated configuration")
|
||||
ruleset := netgen.Nftables(n)
|
||||
ruleset := netgen.Nftables(n, loadIntercept())
|
||||
tmp := statePath("ruleset.check.nft")
|
||||
if err := writeAtomic(tmp, []byte(ruleset), 0o600); err != nil {
|
||||
r.Error = err.Error()
|
||||
@@ -441,6 +447,98 @@ func stations(ctx context.Context) ([]helper.Station, error) {
|
||||
return helper.ParseStationDump(out), nil
|
||||
}
|
||||
|
||||
// --- interception state (persisted so apply_network and boot keep the redirect) ---
|
||||
|
||||
type interceptState struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Port int `json:"port"`
|
||||
}
|
||||
|
||||
func interceptPath() string { return statePath("intercept.json") }
|
||||
|
||||
func loadIntercept() *netgen.Intercept {
|
||||
b, err := os.ReadFile(interceptPath())
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var st interceptState
|
||||
if json.Unmarshal(b, &st) != nil || !st.Enabled {
|
||||
return nil
|
||||
}
|
||||
return &netgen.Intercept{Enabled: true, Port: st.Port}
|
||||
}
|
||||
|
||||
// setInterception toggles transparent interception: re-renders the firewall with
|
||||
// (or without) the redirect, then starts/stops the proxy service. The CA and the
|
||||
// proxy's configuration are owned by the core; the helper only wires the network
|
||||
// and the service.
|
||||
func (s *server) setInterception(ctx context.Context, enabled bool, port int) helper.ApplyResult {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var r helper.ApplyResult
|
||||
step := func(f string, a ...any) { r.Steps = append(r.Steps, fmt.Sprintf(f, a...)) }
|
||||
if port <= 0 || port > 65535 {
|
||||
r.Error = "invalid proxy port"
|
||||
return r
|
||||
}
|
||||
b, err := os.ReadFile(statePath("last-good.json"))
|
||||
if err != nil {
|
||||
r.Error = "apply a network configuration before enabling interception"
|
||||
return r
|
||||
}
|
||||
var n config.Network
|
||||
if err := json.Unmarshal(b, &n); err != nil {
|
||||
r.Error = "stored network configuration unreadable"
|
||||
return r
|
||||
}
|
||||
ic := (*netgen.Intercept)(nil)
|
||||
if enabled {
|
||||
ic = &netgen.Intercept{Enabled: true, Port: port}
|
||||
}
|
||||
ruleset := netgen.Nftables(n, ic)
|
||||
tmp := statePath("ruleset.check.nft")
|
||||
writeAtomic(tmp, []byte(ruleset), 0o600)
|
||||
if _, err := run(ctx, 20*time.Second, "nft", "-c", "-f", tmp); err != nil {
|
||||
r.Error = "firewall ruleset rejected: " + err.Error()
|
||||
return r
|
||||
}
|
||||
// Persist state first so a concurrent apply_network renders consistently.
|
||||
stB, _ := json.Marshal(interceptState{Enabled: enabled, Port: port})
|
||||
writeAtomic(interceptPath(), stB, 0o600)
|
||||
writeAtomic(statePath("ruleset.nft"), []byte(ruleset), 0o600)
|
||||
if _, err := run(ctx, 20*time.Second, "nft", "-f", statePath("ruleset.nft")); err != nil {
|
||||
r.Error = "loading firewall failed: " + err.Error()
|
||||
return r
|
||||
}
|
||||
step("firewall updated (%s)", map[bool]string{true: "redirecting HTTP/HTTPS to the proxy", false: "redirect removed"}[enabled])
|
||||
if enabled {
|
||||
if _, err := run(ctx, 30*time.Second, "systemctl", "restart", "pilens-proxy.service"); err != nil {
|
||||
writeAtomic(interceptPath(), mustJSON(interceptState{Enabled: false, Port: port}), 0o600)
|
||||
writeAtomic(statePath("ruleset.nft"), []byte(netgen.Nftables(n, nil)), 0o600)
|
||||
run(ctx, 20*time.Second, "nft", "-f", statePath("ruleset.nft"))
|
||||
r.Error = "proxy failed to start, interception reverted: " + err.Error()
|
||||
r.RolledBack = true
|
||||
return r
|
||||
}
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for !unitActive(ctx, "pilens-proxy.service") {
|
||||
if time.Now().After(deadline) {
|
||||
r.Error = "proxy did not become active"
|
||||
return r
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
step("interception proxy active on port %d", port)
|
||||
} else {
|
||||
run(ctx, 20*time.Second, "systemctl", "stop", "pilens-proxy.service")
|
||||
step("interception proxy stopped")
|
||||
}
|
||||
r.Applied = true
|
||||
return r
|
||||
}
|
||||
|
||||
func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }
|
||||
|
||||
// bootLoad restores the last good ruleset (or a fail-closed one) before networking starts.
|
||||
func bootLoad() error {
|
||||
ctx := context.Background()
|
||||
@@ -462,6 +560,10 @@ func bootLoad() error {
|
||||
run(ctx, 10*time.Second, "iw", "reg", "set", n.AP.Country)
|
||||
}
|
||||
}
|
||||
if ic := loadIntercept(); ic != nil {
|
||||
run(ctx, 30*time.Second, "systemctl", "start", "pilens-proxy.service")
|
||||
slog.Info("interception proxy started at boot")
|
||||
}
|
||||
slog.Info("firewall restored at boot")
|
||||
return nil
|
||||
}
|
||||
@@ -481,6 +583,6 @@ func renderCmd(path string) int {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("### nftables\n%s\n### dnsmasq.conf\n%s\n### %s (psk redacted)\n%s", netgen.Nftables(n), netgen.DnsmasqConf(n), keyfilePath, netgen.NMKeyfile(n.Redacted()))
|
||||
fmt.Printf("### nftables\n%s\n### dnsmasq.conf\n%s\n### %s (psk redacted)\n%s", netgen.Nftables(n, loadIntercept()), netgen.DnsmasqConf(n), keyfilePath, netgen.NMKeyfile(n.Redacted()))
|
||||
return 0
|
||||
}
|
||||
|
||||
+16
-1
@@ -36,6 +36,7 @@ import (
|
||||
"pilens.dev/pilens/internal/config"
|
||||
"pilens.dev/pilens/internal/helper"
|
||||
"pilens.dev/pilens/internal/pcapfile"
|
||||
"pilens.dev/pilens/internal/proxy"
|
||||
"pilens.dev/pilens/internal/store"
|
||||
"pilens.dev/pilens/web"
|
||||
)
|
||||
@@ -134,13 +135,20 @@ func serve(args []string) error {
|
||||
go eng.Reindex(id)
|
||||
}
|
||||
|
||||
host, _ := os.Hostname()
|
||||
sink := proxySink{eng.Hub}
|
||||
px := proxy.New(db, filepath.Join(*data, "proxy"), host, sink)
|
||||
if err := px.EnsureCA(); err != nil {
|
||||
slog.Warn("interception CA init deferred", "err", err)
|
||||
}
|
||||
|
||||
cert, fp, err := ensureCert(*data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Info("management certificate", "sha256", fp)
|
||||
srv := &api.Server{DB: db, Engine: eng, Helper: helper.Client{Socket: *sock}, Version: Version, Web: web.FS(),
|
||||
SetupTokenPath: filepath.Join(*data, "setup-token"), DataDir: *data}
|
||||
SetupTokenPath: filepath.Join(*data, "setup-token"), DataDir: *data, Proxy: px}
|
||||
hs := &http.Server{
|
||||
Addr: *listen,
|
||||
Handler: srv.Handler(),
|
||||
@@ -162,6 +170,7 @@ func serve(args []string) error {
|
||||
}
|
||||
}()
|
||||
sdNotify("READY=1")
|
||||
px.Start(ctx)
|
||||
go watchdog(ctx, db)
|
||||
slog.Info("pilens serving", "listen", *listen, "version", Version)
|
||||
<-ctx.Done()
|
||||
@@ -231,6 +240,12 @@ func recoverSessions(db *store.Store, dir string) (ids []string) {
|
||||
return ids
|
||||
}
|
||||
|
||||
type proxySink struct{ h *capture.Hub }
|
||||
|
||||
func (s proxySink) Publish(eventType string, data []byte) {
|
||||
s.h.Publish(capture.Event{Type: eventType, Data: data})
|
||||
}
|
||||
|
||||
func sdNotify(state string) {
|
||||
addr := os.Getenv("NOTIFY_SOCKET")
|
||||
if addr == "" {
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pilens.dev/pilens/internal/helper"
|
||||
"pilens.dev/pilens/internal/proxy"
|
||||
)
|
||||
|
||||
// interceptionRoutes wires the Mode-B (authorised HTTPS inspection) endpoints.
|
||||
func (s *Server) interceptionRoutes(priv func(string, http.HandlerFunc)) {
|
||||
priv("GET /api/interception", s.getInterception)
|
||||
priv("PUT /api/interception", s.setInterception)
|
||||
priv("POST /api/interception/ca/rotate", s.rotateCA)
|
||||
priv("POST /api/interception/clear", s.clearFlows)
|
||||
priv("GET /api/interception/ca.crt", s.downloadCA)
|
||||
priv("GET /api/http", s.httpFlows)
|
||||
priv("GET /api/http/{id}", s.httpFlow)
|
||||
priv("GET /api/http/{id}/body/{which}", s.httpBody)
|
||||
}
|
||||
|
||||
func (s *Server) interceptionStatus(ctx context.Context) map[string]any {
|
||||
cfg := s.Proxy.Config()
|
||||
running := false
|
||||
st, herr := s.helperStatus(ctx)
|
||||
_ = st
|
||||
// proxy liveness: ask the helper-independent systemd via the health probe is
|
||||
// overkill; infer from config + a quick check of the flows file growth is not
|
||||
// reliable. Report configured-enabled; the dashboard shows service health.
|
||||
running = cfg.Enabled && herr == ""
|
||||
return map[string]any{
|
||||
"config": cfg,
|
||||
"ca": s.Proxy.CAInfo(),
|
||||
"running": running,
|
||||
"port": proxy.ListenPort,
|
||||
"note": "Interception decrypts HTTPS only for devices whose owner has installed and trusted the PiLens CA. " +
|
||||
"Pinned apps, private trust stores and QUIC are reported as not intercepted, never bypassed. The CA private key is never downloadable.",
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) getInterception(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, s.interceptionStatus(r.Context()))
|
||||
}
|
||||
|
||||
// proxyEnvPath is read by the pilens-proxy systemd unit via proxy-run.sh.
|
||||
func (s *Server) proxyDir() string { return filepath.Join(s.DataDir, "proxy") }
|
||||
func (s *Server) proxyEnv() string { return filepath.Join(s.proxyDir(), "proxy.env") }
|
||||
|
||||
func shellQuote(v string) string { return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'" }
|
||||
|
||||
func (s *Server) writeProxyEnv(cfg proxy.Config) error {
|
||||
bodies := "0"
|
||||
if cfg.CaptureBodies {
|
||||
bodies = "1"
|
||||
}
|
||||
env := strings.Join([]string{
|
||||
"PILENS_VENV=" + shellQuote("/opt/pilens/proxy-venv"),
|
||||
fmt.Sprintf("PILENS_PROXY_PORT=%d", proxy.ListenPort),
|
||||
"PILENS_CONFDIR=" + shellQuote(s.Proxy.ConfDir()),
|
||||
"PILENS_ADDON=" + shellQuote("/opt/pilens/mitm_addon.py"),
|
||||
"PILENS_FLOWS=" + shellQuote(filepath.Join(s.proxyDir(), "flows.ndjson")),
|
||||
"PILENS_BODIES=" + shellQuote(filepath.Join(s.proxyDir(), "bodies")),
|
||||
"PILENS_CAPTURE_BODIES=" + bodies,
|
||||
fmt.Sprintf("PILENS_MAX_BODY_KB=%d", cfg.MaxBodyKB),
|
||||
"PILENS_IGNORE=" + shellQuote(cfg.ExcludeRegex()),
|
||||
}, "\n") + "\n"
|
||||
os.MkdirAll(s.proxyDir(), 0o750)
|
||||
return os.WriteFile(s.proxyEnv(), []byte(env), 0o640)
|
||||
}
|
||||
|
||||
func (s *Server) setInterception(w http.ResponseWriter, r *http.Request) {
|
||||
var cfg proxy.Config
|
||||
if !readJSON(w, r, &cfg) {
|
||||
return
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
httpErr(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.Proxy.EnsureCA(); err != nil {
|
||||
httpErr(w, 500, "CA generation failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.writeProxyEnv(cfg); err != nil {
|
||||
httpErr(w, 500, "writing proxy config failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.Proxy.SetConfig(cfg); err != nil {
|
||||
httpErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
var res helper.ApplyResult
|
||||
err := s.Helper.Call(ctx, helper.Request{Op: helper.OpSetInterception, InterceptEnabled: cfg.Enabled, InterceptPort: proxy.ListenPort}, &res)
|
||||
detail := fmt.Sprintf("enabled=%v capture_bodies=%v excludes=%d", cfg.Enabled, cfg.CaptureBodies, len(cfg.ExcludeHosts))
|
||||
if err != nil {
|
||||
s.audit(r, "interception.apply_failed", detail+": "+err.Error())
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
writeJSON(w, map[string]any{"error": err.Error(), "result": res, "status": s.interceptionStatus(r.Context())})
|
||||
return
|
||||
}
|
||||
s.audit(r, "interception.changed", detail)
|
||||
writeJSON(w, map[string]any{"result": res, "status": s.interceptionStatus(r.Context())})
|
||||
}
|
||||
|
||||
func (s *Server) rotateCA(w http.ResponseWriter, r *http.Request) {
|
||||
if s.Proxy.Config().Enabled {
|
||||
httpErr(w, http.StatusConflict, "disable interception before rotating the CA, so devices are not left mid-session")
|
||||
return
|
||||
}
|
||||
if err := s.Proxy.RotateCA(); err != nil {
|
||||
httpErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
s.audit(r, "interception.ca_rotated", s.Proxy.CAInfo().SHA256)
|
||||
writeJSON(w, map[string]any{"ca": s.Proxy.CAInfo()})
|
||||
}
|
||||
|
||||
func (s *Server) downloadCA(w http.ResponseWriter, r *http.Request) {
|
||||
pem, err := s.Proxy.CACertPEM()
|
||||
if err != nil {
|
||||
httpErr(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
s.audit(r, "interception.ca_downloaded", s.Proxy.CAInfo().SHA256)
|
||||
w.Header().Set("Content-Type", "application/x-pem-file")
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="pilens-ca.crt"`)
|
||||
w.Write(pem)
|
||||
}
|
||||
|
||||
func (s *Server) clearFlows(w http.ResponseWriter, r *http.Request) {
|
||||
s.Proxy.Clear()
|
||||
s.audit(r, "interception.flows_cleared", "")
|
||||
writeJSON(w, map[string]bool{"ok": true})
|
||||
}
|
||||
|
||||
func (s *Server) httpFlows(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
client := q.Get("client")
|
||||
if !proxy.ClientIP(client) {
|
||||
httpErr(w, http.StatusBadRequest, "invalid client filter")
|
||||
return
|
||||
}
|
||||
flows, err := s.Proxy.Flows(r.Context(), q.Get("q"), client, qint(r, "limit", 1000))
|
||||
if err != nil {
|
||||
httpErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, flows)
|
||||
}
|
||||
|
||||
func (s *Server) httpFlow(w http.ResponseWriter, r *http.Request) {
|
||||
f, err := s.Proxy.Flow(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
httpErr(w, http.StatusNotFound, "flow not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, f)
|
||||
}
|
||||
|
||||
// httpBody serves a stored body as an opaque download. It is never served with a
|
||||
// content type that a browser would execute; the UI renders it in a sandbox.
|
||||
func (s *Server) httpBody(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := s.Proxy.Body(r.PathValue("id"), r.PathValue("which"))
|
||||
if err != nil {
|
||||
httpErr(w, http.StatusNotFound, "body not retained (body capture was off, or it was cleared)")
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("Content-Disposition", "attachment")
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox")
|
||||
s.audit(r, "interception.body_viewed", r.PathValue("id")+"/"+r.PathValue("which"))
|
||||
w.Write(body)
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"pilens.dev/pilens/internal/capture"
|
||||
"pilens.dev/pilens/internal/config"
|
||||
"pilens.dev/pilens/internal/helper"
|
||||
"pilens.dev/pilens/internal/proxy"
|
||||
"pilens.dev/pilens/internal/store"
|
||||
)
|
||||
|
||||
@@ -39,6 +40,7 @@ type Server struct {
|
||||
Engine *capture.Engine
|
||||
Helper helper.Client
|
||||
Version string
|
||||
Proxy *proxy.Manager
|
||||
Web fs.FS // built UI; may be nil in tests
|
||||
SetupTokenPath string
|
||||
DataDir string
|
||||
@@ -82,6 +84,7 @@ func (s *Server) Handler() http.Handler {
|
||||
s.networkRoutes(priv)
|
||||
s.captureRoutes(priv)
|
||||
s.analysisRoutes(priv)
|
||||
s.interceptionRoutes(priv)
|
||||
|
||||
if s.Web != nil {
|
||||
files := http.FileServerFS(s.Web)
|
||||
|
||||
@@ -51,3 +51,4 @@ func (h *Hub) Publish(e Event) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,8 @@ const DefaultSocket = "/run/pilens/helper.sock"
|
||||
type Request struct {
|
||||
Op string `json:"op"`
|
||||
Network *config.Network `json:"network,omitempty"`
|
||||
InterceptEnabled bool `json:"intercept_enabled,omitempty"`
|
||||
InterceptPort int `json:"intercept_port,omitempty"`
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
@@ -32,6 +34,7 @@ const (
|
||||
OpStatus = "status"
|
||||
OpStations = "stations"
|
||||
OpLeases = "leases"
|
||||
OpSetInterception = "set_interception"
|
||||
)
|
||||
|
||||
type ApplyResult struct {
|
||||
|
||||
@@ -129,11 +129,19 @@ func DnsmasqConf(n config.Network) string {
|
||||
|
||||
func q(s string) string { return `"` + s + `"` } // inputs are validated interface names
|
||||
|
||||
// Intercept describes transparent HTTPS-interception redirection. Port 0 / nil
|
||||
// means interception is off and no redirect rules are emitted.
|
||||
type Intercept struct {
|
||||
Enabled bool
|
||||
Port int
|
||||
}
|
||||
|
||||
// Nftables renders the complete inet pilens table. The output begins by
|
||||
// (re)creating and deleting the table so `nft -f` replaces it atomically while
|
||||
// leaving every other table untouched.
|
||||
func Nftables(n config.Network) string {
|
||||
// leaving every other table untouched. ic may be nil (interception off).
|
||||
func Nftables(n config.Network, ic *Intercept) string {
|
||||
ap, up := q(n.AP.Interface), q(n.Uplink)
|
||||
intercept := ic != nil && ic.Enabled && ic.Port > 0
|
||||
sub := n.LAN.Subnet().String()
|
||||
var mgmtIf []string
|
||||
for _, i := range n.Management.Interfaces {
|
||||
@@ -186,6 +194,12 @@ func Nftables(n config.Network) string {
|
||||
w("\t\tip daddr %s udp dport 53 accept", n.LAN.Gateway())
|
||||
w("\t\tip daddr %s tcp dport 53 accept", n.LAN.Gateway())
|
||||
w("\t\ticmp type { echo-request, destination-unreachable, time-exceeded } accept")
|
||||
if intercept {
|
||||
// Accept ONLY connections the prerouting redirect created (ct status dnat).
|
||||
// A client connecting directly to the proxy port is not dnat'd, so this
|
||||
// never turns the proxy into an open proxy reachable by clients.
|
||||
w("\t\ttcp dport %d ct status dnat accept comment \"redirected to interception proxy\"", ic.Port)
|
||||
}
|
||||
w("\t\tcounter drop comment \"AP clients cannot reach the appliance\"")
|
||||
w("\t}")
|
||||
|
||||
@@ -213,6 +227,17 @@ func Nftables(n config.Network) string {
|
||||
w("\t\tcounter drop")
|
||||
w("\t}")
|
||||
|
||||
// Transparent interception: redirect AP-client HTTP(S) to the local proxy.
|
||||
// Only when enabled, and only new connections from the AP interface. The
|
||||
// proxy's own upstream connections originate on the Pi (not iifname ap), so
|
||||
// they are never redirected — no loop.
|
||||
w("\tchain prerouting {")
|
||||
w("\t\ttype nat hook prerouting priority dstnat; policy accept;")
|
||||
if intercept {
|
||||
w("\t\tiifname %s meta l4proto tcp tcp dport { 80, 443 } redirect to :%d", ap, ic.Port)
|
||||
}
|
||||
w("\t}")
|
||||
|
||||
w("\tchain postrouting {")
|
||||
w("\t\ttype nat hook postrouting priority srcnat; policy accept;")
|
||||
w("\t\tip saddr %s oifname %s masquerade", sub, up)
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestDnsmasq(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNftables(t *testing.T) {
|
||||
s := Nftables(testNet())
|
||||
s := Nftables(testNet(), nil)
|
||||
for _, want := range []string{
|
||||
"delete table inet pilens",
|
||||
`iifname "wlan0" jump ap_forward`,
|
||||
@@ -84,7 +84,24 @@ func TestNftables(t *testing.T) {
|
||||
}
|
||||
n := testNet()
|
||||
n.Forwarding.Mode = "deny"
|
||||
if !strings.Contains(Nftables(n), `"forwarding disabled"`) {
|
||||
if !strings.Contains(Nftables(n, nil), `"forwarding disabled"`) {
|
||||
t.Error("deny mode must drop forwarded traffic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNftablesInterception(t *testing.T) {
|
||||
off := Nftables(testNet(), nil)
|
||||
if strings.Contains(off, "redirect to") || strings.Contains(off, "ct status dnat") {
|
||||
t.Fatal("interception rules present when disabled")
|
||||
}
|
||||
on := Nftables(testNet(), &Intercept{Enabled: true, Port: 8081})
|
||||
for _, want := range []string{
|
||||
"chain prerouting {",
|
||||
`iifname "wlan0" meta l4proto tcp tcp dport { 80, 443 } redirect to :8081`,
|
||||
"tcp dport 8081 ct status dnat accept",
|
||||
} {
|
||||
if !strings.Contains(on, want) {
|
||||
t.Errorf("interception ruleset missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
// Package proxy manages optional, authorised HTTPS interception: a locally
|
||||
// generated CA that the operator installs on test devices by hand, a mitmdump
|
||||
// service, and ingestion of the decrypted HTTP transactions.
|
||||
//
|
||||
// Consent is the whole design. The CA private key never leaves the appliance,
|
||||
// is never pushed to a device, and interception only happens for devices whose
|
||||
// owner has explicitly installed and trusted the CA. Nothing is covert.
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CAInfo describes the interception CA for display. It never includes the key.
|
||||
type CAInfo struct {
|
||||
Present bool `json:"present"`
|
||||
Subject string `json:"subject"`
|
||||
SHA256 string `json:"sha256"` // fingerprint, colon-separated
|
||||
NotBefore string `json:"not_before"`
|
||||
NotAfter string `json:"not_after"`
|
||||
Serial string `json:"serial"`
|
||||
DaysLeft int `json:"days_left"`
|
||||
KeyType string `json:"key_type"`
|
||||
}
|
||||
|
||||
// caPaths returns the key+cert path (mitmproxy format) and the public cert path.
|
||||
func caPaths(dir string) (combined, certOnly string) {
|
||||
return filepath.Join(dir, "mitmproxy-ca.pem"), filepath.Join(dir, "mitmproxy-ca-cert.pem")
|
||||
}
|
||||
|
||||
// EnsureCA creates the interception CA if absent, in the mitmproxy confdir layout.
|
||||
// RSA-2048 is used (not EC) for the broadest device trust-store compatibility.
|
||||
func EnsureCA(dir, hostname string) error {
|
||||
combined, certOnly := caPaths(dir)
|
||||
if _, err := os.Stat(combined); err == nil {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serial, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
cn := "PiLens Interception CA"
|
||||
if hostname != "" {
|
||||
cn = "PiLens Interception CA (" + hostname + ")"
|
||||
}
|
||||
tpl := &x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: cn, Organization: []string{"PiLens appliance"}},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().AddDate(3, 0, 0),
|
||||
IsCA: true,
|
||||
BasicConstraintsValid: true,
|
||||
MaxPathLen: 0,
|
||||
MaxPathLenZero: true,
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
|
||||
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
// mitmproxy reads the key+cert from mitmproxy-ca.pem (0600) and derives the rest.
|
||||
if err := writeFile(combined, append(append([]byte{}, keyPEM...), certPEM...), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
// Public cert for download / device install (never contains the key).
|
||||
if err := writeFile(certOnly, certPEM, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
writeFile(filepath.Join(dir, "mitmproxy-ca-cert.cer"), certPEM, 0o644)
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeFile(path string, data []byte, mode os.FileMode) error {
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, mode); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Chmod(tmp, mode)
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// PublicCertPEM returns the CA's public certificate (safe to distribute). It never
|
||||
// returns the private key.
|
||||
func PublicCertPEM(dir string) ([]byte, error) {
|
||||
_, certOnly := caPaths(dir)
|
||||
b, err := os.ReadFile(certOnly)
|
||||
if err != nil {
|
||||
return nil, errors.New("interception CA not generated yet")
|
||||
}
|
||||
if strings.Contains(string(b), "PRIVATE KEY") { // defensive: must never happen
|
||||
return nil, errors.New("refusing to serve a file containing a private key")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Info reads the CA metadata for display.
|
||||
func Info(dir string) CAInfo {
|
||||
_, certOnly := caPaths(dir)
|
||||
b, err := os.ReadFile(certOnly)
|
||||
if err != nil {
|
||||
return CAInfo{Present: false}
|
||||
}
|
||||
blk, _ := pem.Decode(b)
|
||||
if blk == nil {
|
||||
return CAInfo{Present: false}
|
||||
}
|
||||
c, err := x509.ParseCertificate(blk.Bytes)
|
||||
if err != nil {
|
||||
return CAInfo{Present: false}
|
||||
}
|
||||
sum := sha256.Sum256(c.Raw)
|
||||
return CAInfo{
|
||||
Present: true,
|
||||
Subject: c.Subject.String(),
|
||||
SHA256: colonHex(sum[:]),
|
||||
NotBefore: c.NotBefore.UTC().Format("2006-01-02"),
|
||||
NotAfter: c.NotAfter.UTC().Format("2006-01-02"),
|
||||
Serial: c.SerialNumber.Text(16),
|
||||
DaysLeft: int(time.Until(c.NotAfter).Hours() / 24),
|
||||
KeyType: "RSA 2048",
|
||||
}
|
||||
}
|
||||
|
||||
// Rotate deletes the CA so a fresh one is generated. Devices trusting the old CA
|
||||
// must install the new one; existing interception breaks until they do.
|
||||
func Rotate(dir, hostname string) error {
|
||||
combined, certOnly := caPaths(dir)
|
||||
for _, p := range []string{combined, certOnly, filepath.Join(dir, "mitmproxy-ca-cert.cer"), filepath.Join(dir, "mitmproxy-ca.p12"), filepath.Join(dir, "mitmproxy-ca-cert.p12"), filepath.Join(dir, "mitmproxy-dhparam.pem")} {
|
||||
os.Remove(p)
|
||||
}
|
||||
return EnsureCA(dir, hostname)
|
||||
}
|
||||
|
||||
func colonHex(b []byte) string {
|
||||
h := strings.ToUpper(hex.EncodeToString(b))
|
||||
var parts []string
|
||||
for i := 0; i < len(h); i += 2 {
|
||||
parts = append(parts, h[i:i+2])
|
||||
}
|
||||
return strings.Join(parts, ":")
|
||||
}
|
||||
|
||||
// Fingerprint of a cert file, for the install instructions ("verify this matches").
|
||||
func Fingerprint(dir string) string {
|
||||
return fmt.Sprintf("SHA-256 %s", Info(dir).SHA256)
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCAGeneration(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := EnsureCA(dir, "testpi"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Public cert must never contain a private key.
|
||||
pub, err := PublicCertPEM(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(pub), "PRIVATE KEY") {
|
||||
t.Fatal("public cert leaked the private key")
|
||||
}
|
||||
if !strings.Contains(string(pub), "BEGIN CERTIFICATE") {
|
||||
t.Fatal("not a certificate")
|
||||
}
|
||||
info := Info(dir)
|
||||
if !info.Present || !strings.Contains(info.Subject, "PiLens") || len(info.SHA256) != 95 || info.DaysLeft < 1000 {
|
||||
t.Fatalf("bad CA info: %+v", info)
|
||||
}
|
||||
// Idempotent: second call keeps the same CA.
|
||||
before := info.SHA256
|
||||
EnsureCA(dir, "testpi")
|
||||
if Info(dir).SHA256 != before {
|
||||
t.Fatal("EnsureCA regenerated an existing CA")
|
||||
}
|
||||
// Rotate produces a different CA.
|
||||
if err := Rotate(dir, "testpi"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if Info(dir).SHA256 == before {
|
||||
t.Fatal("rotate did not change the CA")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidate(t *testing.T) {
|
||||
bad := []Config{
|
||||
{MaxBodyKB: -1},
|
||||
{MaxBodyKB: 999999},
|
||||
{ExcludeHosts: []string{"ok.com", "bad regex ("}},
|
||||
{ExcludeHosts: []string{"has space"}},
|
||||
}
|
||||
for i, c := range bad {
|
||||
if c.Validate() == nil {
|
||||
t.Errorf("case %d accepted invalid config", i)
|
||||
}
|
||||
}
|
||||
good := Config{MaxBodyKB: 256, ExcludeHosts: []string{`(?i)\.apple\.com$`, "example\\.com"}}
|
||||
if err := good.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := good.ExcludeRegex(); !strings.Contains(r, "apple") || !strings.Contains(r, "|") {
|
||||
t.Fatalf("exclude regex wrong: %s", r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pilens.dev/pilens/internal/store"
|
||||
)
|
||||
|
||||
// Config is the interception toggle and its options. Stored in settings.
|
||||
type Config struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
CaptureBodies bool `json:"capture_bodies"` // default false: sensitive content off by default
|
||||
MaxBodyKB int `json:"max_body_kb"`
|
||||
ExcludeHosts []string `json:"exclude_hosts"` // host regexes NOT decrypted (passed through)
|
||||
}
|
||||
|
||||
func DefaultConfig() Config { return Config{Enabled: false, CaptureBodies: false, MaxBodyKB: 256} }
|
||||
|
||||
const ListenPort = 8081
|
||||
|
||||
var hostRe = regexp.MustCompile(`^[A-Za-z0-9.*?+()\[\]{}|\\^$_:-]{1,200}$`)
|
||||
|
||||
func (c Config) Validate() error {
|
||||
if c.MaxBodyKB < 0 || c.MaxBodyKB > 10240 {
|
||||
return errors.New("max_body_kb must be 0-10240")
|
||||
}
|
||||
for _, h := range c.ExcludeHosts {
|
||||
if !hostRe.MatchString(h) {
|
||||
return errors.New("invalid exclude-host pattern: " + h)
|
||||
}
|
||||
if _, err := regexp.Compile(h); err != nil {
|
||||
return errors.New("invalid exclude-host regex " + h + ": " + err.Error())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sink receives live events for the browser (adapts the capture Hub).
|
||||
type Sink interface{ Publish(eventType string, data []byte) }
|
||||
|
||||
// Flow is one HTTP(S) transaction, as emitted by the mitmdump addon and stored.
|
||||
type Flow struct {
|
||||
ID string `json:"id"`
|
||||
TS float64 `json:"ts"`
|
||||
Client string `json:"client"`
|
||||
Method string `json:"method"`
|
||||
Scheme string `json:"scheme"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
Path string `json:"path"`
|
||||
HTTPVersion string `json:"http_version"`
|
||||
Status int `json:"status"`
|
||||
Reason string `json:"reason"`
|
||||
ReqCType string `json:"req_ctype"`
|
||||
RespCType string `json:"resp_ctype"`
|
||||
ReqSize int64 `json:"req_size"`
|
||||
RespSize int64 `json:"resp_size"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
TLSVersion string `json:"tls_version"`
|
||||
SNI string `json:"sni"`
|
||||
ALPN string `json:"alpn"`
|
||||
Intercepted bool `json:"intercepted"`
|
||||
Error string `json:"error"`
|
||||
WebSocket bool `json:"websocket"`
|
||||
HasReqBody bool `json:"has_req_body"`
|
||||
HasRespBody bool `json:"has_resp_body"`
|
||||
ReqHeaders map[string]string `json:"req_headers"`
|
||||
RespHeaders map[string]string `json:"resp_headers"`
|
||||
}
|
||||
|
||||
// Manager owns interception state, the flow ingester and queries.
|
||||
type Manager struct {
|
||||
DB *store.Store
|
||||
Dir string // /var/lib/pilens/proxy
|
||||
Hostname string
|
||||
Sink Sink
|
||||
|
||||
mu sync.Mutex
|
||||
cfg Config
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func New(db *store.Store, dir, hostname string, sink Sink) *Manager {
|
||||
m := &Manager{DB: db, Dir: dir, Hostname: hostname, Sink: sink, cfg: DefaultConfig()}
|
||||
db.GetJSON("interception", &m.cfg)
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *Manager) Config() Config { m.mu.Lock(); defer m.mu.Unlock(); return m.cfg }
|
||||
|
||||
func (m *Manager) SetConfig(c Config) error {
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.cfg = c
|
||||
m.mu.Unlock()
|
||||
return m.DB.PutJSON("interception", c)
|
||||
}
|
||||
|
||||
func (m *Manager) flowsPath() string { return filepath.Join(m.Dir, "flows.ndjson") }
|
||||
func (m *Manager) bodiesDir() string { return filepath.Join(m.Dir, "bodies") }
|
||||
func (m *Manager) ConfDir() string { return filepath.Join(m.Dir, "ca") }
|
||||
|
||||
func (m *Manager) EnsureCA() error { return EnsureCA(m.ConfDir(), m.Hostname) }
|
||||
func (m *Manager) CAInfo() CAInfo { return Info(m.ConfDir()) }
|
||||
func (m *Manager) CACertPEM() ([]byte, error) { return PublicCertPEM(m.ConfDir()) }
|
||||
func (m *Manager) RotateCA() error { return Rotate(m.ConfDir(), m.Hostname) }
|
||||
|
||||
// Start begins tailing the flow file. It runs for the service lifetime; the
|
||||
// mitmdump service is started/stopped by the helper, this just ingests.
|
||||
func (m *Manager) Start(ctx context.Context) {
|
||||
os.MkdirAll(m.bodiesDir(), 0o750)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
m.cancel = cancel
|
||||
go m.tail(ctx)
|
||||
}
|
||||
|
||||
// tail follows flows.ndjson, inserting new lines. Survives file rotation/truncation.
|
||||
func (m *Manager) tail(ctx context.Context) {
|
||||
path := m.flowsPath()
|
||||
var offset int64
|
||||
m.DB.GetJSON("interception_flow_offset", &offset)
|
||||
for ctx.Err() == nil {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
continue
|
||||
}
|
||||
if fi, _ := f.Stat(); fi != nil && fi.Size() < offset {
|
||||
offset = 0 // file was rotated/truncated
|
||||
}
|
||||
f.Seek(offset, io.SeekStart)
|
||||
r := bufio.NewReaderSize(f, 1<<20)
|
||||
for ctx.Err() == nil {
|
||||
line, rerr := r.ReadBytes('\n')
|
||||
if len(line) > 0 && line[len(line)-1] == '\n' {
|
||||
offset += int64(len(line))
|
||||
m.ingest(line)
|
||||
m.DB.PutJSON("interception_flow_offset", offset)
|
||||
continue
|
||||
}
|
||||
_ = rerr // partial/EOF: keep offset, re-read after the wait
|
||||
break
|
||||
}
|
||||
f.Close()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) ingest(line []byte) {
|
||||
var fl Flow
|
||||
if err := json.Unmarshal(line, &fl); err != nil || fl.ID == "" {
|
||||
return
|
||||
}
|
||||
rh, _ := json.Marshal(fl.ReqHeaders)
|
||||
sh, _ := json.Marshal(fl.RespHeaders)
|
||||
_, err := m.DB.Exec(`INSERT OR REPLACE INTO http_flows
|
||||
(id,ts,client,method,scheme,host,port,path,http_version,status,reason,req_ctype,resp_ctype,req_size,resp_size,duration_ms,tls_version,sni,alpn,intercepted,error,websocket,has_req_body,has_resp_body,req_headers,resp_headers)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
fl.ID, fl.TS, fl.Client, fl.Method, fl.Scheme, fl.Host, fl.Port, fl.Path, fl.HTTPVersion, fl.Status, fl.Reason,
|
||||
fl.ReqCType, fl.RespCType, fl.ReqSize, fl.RespSize, fl.DurationMS, fl.TLSVersion, fl.SNI, fl.ALPN,
|
||||
b2i(fl.Intercepted), fl.Error, b2i(fl.WebSocket), b2i(fl.HasReqBody), b2i(fl.HasRespBody), string(rh), string(sh))
|
||||
if err != nil {
|
||||
slog.Warn("http flow insert", "err", err)
|
||||
return
|
||||
}
|
||||
if m.Sink != nil {
|
||||
m.Sink.Publish("http", line)
|
||||
}
|
||||
}
|
||||
|
||||
func b2i(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// --- queries ---
|
||||
|
||||
const flowCols = `id,ts,client,method,scheme,host,port,path,http_version,status,reason,req_ctype,resp_ctype,req_size,resp_size,duration_ms,tls_version,sni,alpn,intercepted,error,websocket,has_req_body,has_resp_body,req_headers,resp_headers`
|
||||
|
||||
func scanFlow(sc interface{ Scan(...any) error }) (Flow, error) {
|
||||
var f Flow
|
||||
var rh, sh string
|
||||
var intercept, ws, hrb, hsb int
|
||||
err := sc.Scan(&f.ID, &f.TS, &f.Client, &f.Method, &f.Scheme, &f.Host, &f.Port, &f.Path, &f.HTTPVersion, &f.Status, &f.Reason,
|
||||
&f.ReqCType, &f.RespCType, &f.ReqSize, &f.RespSize, &f.DurationMS, &f.TLSVersion, &f.SNI, &f.ALPN,
|
||||
&intercept, &f.Error, &ws, &hrb, &hsb, &rh, &sh)
|
||||
f.Intercepted, f.WebSocket, f.HasReqBody, f.HasRespBody = intercept == 1, ws == 1, hrb == 1, hsb == 1
|
||||
json.Unmarshal([]byte(rh), &f.ReqHeaders)
|
||||
json.Unmarshal([]byte(sh), &f.RespHeaders)
|
||||
return f, err
|
||||
}
|
||||
|
||||
// Flows returns recent transactions, newest first, optionally filtered by a
|
||||
// substring over host/path/client and a client IP.
|
||||
func (m *Manager) Flows(ctx context.Context, q, client string, limit int) ([]Flow, error) {
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 1000
|
||||
}
|
||||
where := "1=1"
|
||||
args := []any{}
|
||||
if q != "" {
|
||||
where += " AND (host LIKE ? OR path LIKE ? OR client LIKE ?)"
|
||||
like := "%" + q + "%"
|
||||
args = append(args, like, like, like)
|
||||
}
|
||||
if client != "" {
|
||||
where += " AND client = ?"
|
||||
args = append(args, client)
|
||||
}
|
||||
args = append(args, limit)
|
||||
rows, err := m.DB.QueryContext(ctx, `SELECT `+flowCols+` FROM http_flows WHERE `+where+` ORDER BY ts DESC LIMIT ?`, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []Flow{}
|
||||
for rows.Next() {
|
||||
f, err := scanFlow(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (m *Manager) Flow(ctx context.Context, id string) (Flow, error) {
|
||||
return scanFlow(m.DB.QueryRowContext(ctx, `SELECT `+flowCols+` FROM http_flows WHERE id=?`, id))
|
||||
}
|
||||
|
||||
// Body returns a stored request/response body. which is "req" or "resp".
|
||||
// Bodies exist only when body capture was enabled for that flow.
|
||||
func (m *Manager) Body(id, which string) ([]byte, error) {
|
||||
if which != "req" && which != "resp" || !flowIDRe.MatchString(id) {
|
||||
return nil, errors.New("invalid body request")
|
||||
}
|
||||
return os.ReadFile(filepath.Join(m.bodiesDir(), id+"."+which))
|
||||
}
|
||||
|
||||
var flowIDRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
||||
|
||||
// Clear deletes all intercepted HTTP data (flows + bodies). Captures are untouched.
|
||||
func (m *Manager) Clear() error {
|
||||
m.DB.Exec(`DELETE FROM http_flows`)
|
||||
os.RemoveAll(m.bodiesDir())
|
||||
os.MkdirAll(m.bodiesDir(), 0o750)
|
||||
os.Truncate(m.flowsPath(), 0)
|
||||
m.DB.PutJSON("interception_flow_offset", int64(0))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClientIP validates a client filter value.
|
||||
func ClientIP(s string) bool { _, err := netip.ParseAddr(s); return err == nil || s == "" }
|
||||
|
||||
// ExcludeRegex joins exclude hosts into one mitmdump --ignore-hosts pattern.
|
||||
func (c Config) ExcludeRegex() string {
|
||||
var parts []string
|
||||
for _, h := range c.ExcludeHosts {
|
||||
parts = append(parts, "(?:"+h+")")
|
||||
}
|
||||
return strings.Join(parts, "|")
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
-- Intercepted HTTP(S) transactions (Mode B). Populated only while interception
|
||||
-- is enabled and only for devices that trust the installed CA.
|
||||
CREATE TABLE http_flows (
|
||||
id TEXT PRIMARY KEY,
|
||||
ts REAL NOT NULL,
|
||||
client TEXT NOT NULL DEFAULT '',
|
||||
method TEXT NOT NULL DEFAULT '',
|
||||
scheme TEXT NOT NULL DEFAULT '',
|
||||
host TEXT NOT NULL DEFAULT '',
|
||||
port INTEGER NOT NULL DEFAULT 0,
|
||||
path TEXT NOT NULL DEFAULT '',
|
||||
http_version TEXT NOT NULL DEFAULT '',
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
reason TEXT NOT NULL DEFAULT '',
|
||||
req_ctype TEXT NOT NULL DEFAULT '',
|
||||
resp_ctype TEXT NOT NULL DEFAULT '',
|
||||
req_size INTEGER NOT NULL DEFAULT 0,
|
||||
resp_size INTEGER NOT NULL DEFAULT 0,
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0,
|
||||
tls_version TEXT NOT NULL DEFAULT '',
|
||||
sni TEXT NOT NULL DEFAULT '',
|
||||
alpn TEXT NOT NULL DEFAULT '',
|
||||
intercepted INTEGER NOT NULL DEFAULT 0, -- 1 = decrypted, 0 = passthrough/failed
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
websocket INTEGER NOT NULL DEFAULT 0,
|
||||
has_req_body INTEGER NOT NULL DEFAULT 0,
|
||||
has_resp_body INTEGER NOT NULL DEFAULT 0,
|
||||
req_headers TEXT NOT NULL DEFAULT '{}',
|
||||
resp_headers TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE INDEX http_flows_ts ON http_flows(ts);
|
||||
CREATE INDEX http_flows_host ON http_flows(host);
|
||||
CREATE INDEX http_flows_client ON http_flows(client);
|
||||
@@ -0,0 +1,157 @@
|
||||
"""PiLens mitmdump addon.
|
||||
|
||||
Emits one JSON line per HTTP(S) transaction (and per client TLS failure) to
|
||||
PILENS_FLOWS. Request/response bodies are written to PILENS_BODIES only when
|
||||
PILENS_CAPTURE_BODIES=1, size-limited by PILENS_MAX_BODY_KB. Sensitive content
|
||||
is therefore off by default.
|
||||
|
||||
This runs inside the pilens-proxy service's venv; it is not imported anywhere else.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
FLOWS = os.environ.get("PILENS_FLOWS", "/var/lib/pilens/proxy/flows.ndjson")
|
||||
BODIES = os.environ.get("PILENS_BODIES", "/var/lib/pilens/proxy/bodies")
|
||||
CAPTURE_BODIES = os.environ.get("PILENS_CAPTURE_BODIES", "0") == "1"
|
||||
MAX_BODY = int(os.environ.get("PILENS_MAX_BODY_KB", "256")) * 1024
|
||||
|
||||
|
||||
def _emit(obj):
|
||||
try:
|
||||
with open(FLOWS, "a") as f:
|
||||
f.write(json.dumps(obj, separators=(",", ":")) + "\n")
|
||||
f.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _headers(msg):
|
||||
out = {}
|
||||
for k, v in msg.headers.items(multi=True):
|
||||
out[k] = (out[k] + ", " + v) if k in out else v
|
||||
return out
|
||||
|
||||
|
||||
def _write_body(flow_id, which, content):
|
||||
if not CAPTURE_BODIES or not content:
|
||||
return False
|
||||
data = content[:MAX_BODY]
|
||||
try:
|
||||
os.makedirs(BODIES, exist_ok=True)
|
||||
with open(os.path.join(BODIES, "%s.%s" % (flow_id, which)), "wb") as f:
|
||||
f.write(data)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _client(flow):
|
||||
try:
|
||||
return flow.client_conn.peername[0]
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _tls(flow):
|
||||
cc = flow.client_conn
|
||||
sc = flow.server_conn
|
||||
ver = getattr(cc, "tls_version", None) or getattr(sc, "tls_version", None) or ""
|
||||
alpn = getattr(cc, "alpn", b"") or b""
|
||||
try:
|
||||
alpn = alpn.decode()
|
||||
except Exception:
|
||||
alpn = str(alpn)
|
||||
return ver, getattr(cc, "sni", None) or "", alpn
|
||||
|
||||
|
||||
class PiLens:
|
||||
def response(self, flow):
|
||||
req, resp = flow.request, flow.response
|
||||
ver, sni, alpn = _tls(flow)
|
||||
dur = 0
|
||||
try:
|
||||
dur = int((resp.timestamp_end - req.timestamp_start) * 1000)
|
||||
except Exception:
|
||||
pass
|
||||
has_req = _write_body(flow.id, "req", req.raw_content)
|
||||
has_resp = _write_body(flow.id, "resp", resp.raw_content)
|
||||
_emit({
|
||||
"id": flow.id,
|
||||
"ts": req.timestamp_start or time.time(),
|
||||
"client": _client(flow),
|
||||
"method": req.method,
|
||||
"scheme": req.scheme,
|
||||
"host": req.pretty_host,
|
||||
"port": req.port,
|
||||
"path": req.path,
|
||||
"http_version": resp.http_version,
|
||||
"status": resp.status_code,
|
||||
"reason": resp.reason or "",
|
||||
"req_ctype": req.headers.get("content-type", ""),
|
||||
"resp_ctype": resp.headers.get("content-type", ""),
|
||||
"req_size": len(req.raw_content or b""),
|
||||
"resp_size": len(resp.raw_content or b""),
|
||||
"duration_ms": dur,
|
||||
"tls_version": ver,
|
||||
"sni": sni,
|
||||
"alpn": alpn,
|
||||
"intercepted": req.scheme == "https",
|
||||
"error": "",
|
||||
"websocket": bool(flow.websocket),
|
||||
"has_req_body": has_req,
|
||||
"has_resp_body": has_resp,
|
||||
"req_headers": _headers(req),
|
||||
"resp_headers": _headers(resp),
|
||||
})
|
||||
|
||||
def error(self, flow):
|
||||
try:
|
||||
req = flow.request
|
||||
ver, sni, alpn = _tls(flow)
|
||||
_emit({
|
||||
"id": flow.id,
|
||||
"ts": (req.timestamp_start if req else None) or time.time(),
|
||||
"client": _client(flow),
|
||||
"method": req.method if req else "",
|
||||
"scheme": req.scheme if req else "",
|
||||
"host": req.pretty_host if req else "",
|
||||
"port": req.port if req else 0,
|
||||
"path": req.path if req else "",
|
||||
"status": 0,
|
||||
"tls_version": ver,
|
||||
"sni": sni,
|
||||
"alpn": alpn,
|
||||
"intercepted": False,
|
||||
"error": str(flow.error) if flow.error else "connection error",
|
||||
"req_headers": {},
|
||||
"resp_headers": {},
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def tls_failed_client(self, data):
|
||||
# The device refused our certificate: it does not trust the CA, or the
|
||||
# app pins a specific certificate. Reported honestly, never bypassed.
|
||||
try:
|
||||
conn = data.conn
|
||||
peer = conn.peername[0] if conn.peername else ""
|
||||
sni = getattr(data.context.client, "sni", None) or getattr(conn, "sni", None) or ""
|
||||
_emit({
|
||||
"id": "tlsfail-%s-%d" % (peer.replace(".", "_").replace(":", "_"), int(time.time() * 1000) % 1000000000),
|
||||
"ts": time.time(),
|
||||
"client": peer,
|
||||
"scheme": "https",
|
||||
"host": sni,
|
||||
"sni": sni,
|
||||
"status": 0,
|
||||
"intercepted": False,
|
||||
"error": "client TLS failure: device does not trust the PiLens CA, or the app pins its own certificate",
|
||||
"req_headers": {},
|
||||
"resp_headers": {},
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
addons = [PiLens()]
|
||||
+2
-1
@@ -11,7 +11,7 @@ run() { local f="$D/$1"; shift; { echo "\$ $*"; "$@"; } >"$f" 2>&1 || true; }
|
||||
redact() { sed -E 's/("passphrase"[[:space:]]*:[[:space:]]*")[^"]*/\1REDACTED/g; s/^(psk=).*/\1REDACTED/; s/(setup_token=).*/\1REDACTED/'; }
|
||||
|
||||
run versions.txt sh -c 'cat /opt/pilens/VERSION; uname -a; cat /etc/os-release; tr -d "\0" </proc/device-tree/model; echo; tshark --version | head -1; nmcli --version; dnsmasq --version | head -1; nft --version'
|
||||
run services.txt systemctl status --no-pager pilens pilens-helper pilens-firewall pilens-dnsmasq NetworkManager
|
||||
run services.txt systemctl status --no-pager pilens pilens-helper pilens-firewall pilens-dnsmasq pilens-proxy NetworkManager
|
||||
run journal.txt journalctl --no-pager -n 3000 -u pilens -u pilens-helper -u pilens-firewall -u pilens-dnsmasq
|
||||
run journal-nm.txt journalctl --no-pager -n 800 -u NetworkManager
|
||||
run nft-pilens.txt nft list table inet pilens
|
||||
@@ -20,6 +20,7 @@ run ip.txt sh -c 'ip -br link; ip addr; ip route; ip -6 route; sysctl net.ipv4.i
|
||||
run nm.txt sh -c 'nmcli general; nmcli device; nmcli connection show'
|
||||
run nm-pilens-ap.txt nmcli connection show pilens-ap
|
||||
run wifi.txt sh -c 'iw dev; iw reg get; iw list'
|
||||
run interception.txt sh -c 'ls -la /var/lib/pilens/proxy 2>/dev/null; echo; [ -f /var/lib/pilens/proxy/ca/mitmproxy-ca-cert.pem ] && openssl x509 -in /var/lib/pilens/proxy/ca/mitmproxy-ca-cert.pem -noout -subject -fingerprint -sha256 -dates; /opt/pilens/proxy-venv/bin/mitmdump --version 2>/dev/null | head -1'
|
||||
run capture.txt sh -c 'getcap /usr/bin/dumpcap; id pilens; ls -la /var/lib/pilens /var/lib/pilens-helper'
|
||||
run resources.txt sh -c 'df -h; free -m; uptime; vcgencmd get_throttled; vcgencmd measure_temp; timedatectl'
|
||||
cp /var/lib/pilens-helper/dnsmasq.conf "$D/dnsmasq.conf" 2>/dev/null
|
||||
|
||||
@@ -169,14 +169,33 @@ usermod -aG wireshark pilens
|
||||
systemctl stop pilens 2>/dev/null || true
|
||||
mkdir -p "$PREFIX"
|
||||
install -m 0755 "$SRC/bin/pilens" "$SRC/bin/pilens-helper" -D -t "$PREFIX/bin"
|
||||
install -m 0755 "$SRC/proxy-run.sh" "$PREFIX/proxy-run.sh"
|
||||
install -m 0644 "$SRC/mitm_addon.py" "$PREFIX/mitm_addon.py"
|
||||
for f in install.sh uninstall.sh repair.sh diagnose.sh setup.example.yaml VERSION README.md LICENSE; do install -m 0644 "$SRC/$f" "$PREFIX/$f"; done
|
||||
chmod 0755 "$PREFIX"/*.sh
|
||||
rm -rf "$PREFIX/docs"; cp -r "$SRC/docs" "$PREFIX/docs"
|
||||
install -d -o pilens -g pilens -m 0750 "$DATA"
|
||||
install -d -o pilens -g pilens -m 0750 "$DATA/proxy"
|
||||
install -d -o root -g root -m 0700 /var/lib/pilens-helper
|
||||
install -m 0644 "$SRC"/systemd/*.service /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
|
||||
# ---------------------------------------------------------------- 16. optional TLS proxy deps
|
||||
say "Preparing optional HTTPS-interception proxy (mitmproxy)"
|
||||
VENV="$PREFIX/proxy-venv"
|
||||
if [ -x "$VENV/bin/mitmdump" ]; then
|
||||
echo " mitmproxy venv already present"
|
||||
elif apt-get install -y -qq --no-install-recommends python3-venv >>"$LOG" 2>&1 && python3 -m venv "$VENV" >>"$LOG" 2>&1; then
|
||||
if "$VENV/bin/pip" install --quiet --only-binary=:all: --upgrade pip >>"$LOG" 2>&1 && "$VENV/bin/pip" install --quiet --only-binary=:all: mitmproxy >>"$LOG" 2>&1; then
|
||||
chown -R pilens:pilens "$VENV"
|
||||
echo " mitmproxy installed ($("$VENV/bin/mitmdump" --version 2>/dev/null | head -1))"
|
||||
else
|
||||
warn "mitmproxy install failed (needs internet + arm64 wheels). HTTPS interception will be unavailable until you run: sudo $PREFIX/repair.sh"
|
||||
fi
|
||||
else
|
||||
warn "could not create the proxy virtualenv; HTTPS interception will be unavailable (optional feature)"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------- 15. initialise data
|
||||
say "Initialising database and management certificate"
|
||||
INIT=$(runuser -u pilens -- "$PREFIX/bin/pilens" init -data "$DATA" -mgmt-iface "$MGMT_IF" -mgmt-subnet "$MGMT_NET")
|
||||
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
# Launches mitmdump for PiLens interception from the core-written env file.
|
||||
# Runs as the unprivileged 'pilens' user under the pilens-proxy.service unit.
|
||||
set -eu
|
||||
ENV=/var/lib/pilens/proxy/proxy.env
|
||||
[ -f "$ENV" ] || { echo "no proxy.env; interception not configured" >&2; exit 1; }
|
||||
. "$ENV"
|
||||
export PILENS_FLOWS PILENS_BODIES PILENS_CAPTURE_BODIES PILENS_MAX_BODY_KB
|
||||
args=( --mode transparent --listen-host 0.0.0.0 --listen-port "$PILENS_PROXY_PORT"
|
||||
--set "confdir=$PILENS_CONFDIR" --set block_global=false --set connection_strategy=lazy
|
||||
--set stream_large_bodies=1m -s "$PILENS_ADDON" -q )
|
||||
if [ -n "${PILENS_IGNORE:-}" ]; then args+=( --ignore-hosts "$PILENS_IGNORE" ); fi
|
||||
exec "$PILENS_VENV/bin/mitmdump" "${args[@]}"
|
||||
@@ -18,6 +18,18 @@ if ! getcap /usr/bin/dumpcap 2>/dev/null | grep -q cap_net_raw; then
|
||||
fi
|
||||
getcap /usr/bin/dumpcap | grep -q cap_net_raw && ok "dumpcap capture capabilities" || bad "dumpcap lacks capabilities"
|
||||
|
||||
# Optional: rebuild the mitmproxy venv if it is missing (HTTPS interception).
|
||||
VENV=/opt/pilens/proxy-venv
|
||||
if [ ! -x "$VENV/bin/mitmdump" ]; then
|
||||
if python3 -m venv "$VENV" >/dev/null 2>&1 && "$VENV/bin/pip" install --quiet --only-binary=:all: mitmproxy >/dev/null 2>&1; then
|
||||
chown -R pilens:pilens "$VENV"; ok "mitmproxy venv rebuilt"
|
||||
else
|
||||
bad "mitmproxy venv missing and reinstall failed (interception unavailable; needs internet)"
|
||||
fi
|
||||
else
|
||||
ok "mitmproxy venv present"
|
||||
fi
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable pilens-firewall pilens-helper pilens-dnsmasq pilens >/dev/null 2>&1
|
||||
systemctl reset-failed 'pilens*' 2>/dev/null
|
||||
|
||||
@@ -15,7 +15,7 @@ if [ $PURGE = 1 ]; then
|
||||
[ "$x" = DELETE ] || { echo "not confirmed; keeping data"; PURGE=0; }
|
||||
fi
|
||||
|
||||
systemctl disable --now pilens pilens-dnsmasq pilens-helper pilens-firewall 2>/dev/null || true
|
||||
systemctl disable --now pilens pilens-proxy pilens-dnsmasq pilens-helper pilens-firewall 2>/dev/null || true
|
||||
nmcli connection delete pilens-ap 2>/dev/null && echo "removed access-point profile (other Wi-Fi profiles were never modified and will reconnect)" || true
|
||||
rm -f /etc/NetworkManager/system-connections/pilens-ap.nmconnection
|
||||
nft delete table inet pilens 2>/dev/null && echo "removed firewall table inet pilens" || true
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
[Unit]
|
||||
Description=PiLens HTTPS interception proxy (mitmdump, authorised Mode B)
|
||||
Documentation=file:/opt/pilens/docs/tls-and-encryption.md
|
||||
After=network-online.target pilens.service
|
||||
Wants=network-online.target
|
||||
PartOf=pilens.service
|
||||
ConditionPathExists=/var/lib/pilens/proxy/proxy.env
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pilens
|
||||
Group=pilens
|
||||
# Transparent mode reads SO_ORIGINAL_DST on redirected sockets; no extra privilege.
|
||||
ExecStart=/opt/pilens/proxy-run.sh
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/pilens/proxy
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||
LockPersonality=yes
|
||||
# started/stopped by the network helper when interception is toggled
|
||||
Vendored
@@ -15,6 +15,8 @@ import DnsExplorer from "./pages/DnsExplorer";
|
||||
import TlsExplorer from "./pages/TlsExplorer";
|
||||
import Statistics from "./pages/Statistics";
|
||||
import Findings from "./pages/Findings";
|
||||
import HttpExplorer from "./pages/HttpExplorer";
|
||||
import Certificates from "./pages/Certificates";
|
||||
import Settings from "./pages/Settings";
|
||||
|
||||
export interface Prefs { theme: "dark" | "light" | "system"; density: "compact" | "comfortable"; scale: number; motion: "system" | "reduce" }
|
||||
@@ -29,12 +31,14 @@ const NAV: [string, string, string][] = [
|
||||
["/conversations", "Conversations", "⇌"],
|
||||
["/dns", "DNS Explorer", "◇"],
|
||||
["/tls", "TLS Explorer", "⛨"],
|
||||
["/http", "HTTP Explorer", "⇅"],
|
||||
["/statistics", "Statistics", "▦"],
|
||||
["/findings", "Findings", "⚑"],
|
||||
["/library", "Capture Library", "▤"],
|
||||
["/devices", "Devices", "⌬"],
|
||||
["", "", ""],
|
||||
["/network", "Network Settings", "⇄"],
|
||||
["/certificates", "Certificates", "⛉"],
|
||||
["/health", "System Health", "♥"],
|
||||
["/settings", "Settings", "⚙"],
|
||||
];
|
||||
@@ -101,6 +105,8 @@ export default function App() {
|
||||
if (path.startsWith("/tls")) return <TlsExplorer sessionId={path.split("/")[2] ?? ""} />;
|
||||
if (path.startsWith("/statistics")) return <Statistics sessionId={path.split("/")[2] ?? ""} />;
|
||||
if (path.startsWith("/findings")) return <Findings sessionId={path.split("/")[2] ?? ""} />;
|
||||
if (path.startsWith("/http")) return <HttpExplorer />;
|
||||
if (path === "/certificates") return <Certificates />;
|
||||
if (path === "/library") return <Library />;
|
||||
if (path === "/devices") return <Devices />;
|
||||
if (path === "/network") return <NetworkPage />;
|
||||
|
||||
+12
-1
@@ -56,7 +56,7 @@ let es: EventSource | null = null;
|
||||
export function connectEvents() {
|
||||
if (es) return;
|
||||
es = new EventSource("/api/events");
|
||||
for (const t of ["hello", "packets", "capture", "resync", "ping"]) {
|
||||
for (const t of ["hello", "packets", "capture", "resync", "ping", "http"]) {
|
||||
es.addEventListener(t, (e) => {
|
||||
let d: any = {};
|
||||
try { d = JSON.parse((e as MessageEvent).data); } catch { /* ignore */ }
|
||||
@@ -98,3 +98,14 @@ export interface TLSRecord { num: number; seg: number; time: number; client: str
|
||||
export interface CertRecord { num: number; seg: number; time: number; server: string; subject: string; issuer: string; sans: string; not_before: string; not_after: string }
|
||||
|
||||
export interface Finding { rule: string; title: string; severity: "info" | "low" | "medium"; count: number; flows: number; examples: number[]; filter: string; note: string }
|
||||
|
||||
// ---- Phase 4 interception types ----
|
||||
export interface CAInfo { present: boolean; subject: string; sha256: string; not_before: string; not_after: string; serial: string; days_left: number; key_type: string }
|
||||
export interface InterceptionConfig { enabled: boolean; capture_bodies: boolean; max_body_kb: number; exclude_hosts: string[] }
|
||||
export interface InterceptionStatus { config: InterceptionConfig; ca: CAInfo; running: boolean; port: number; note: string }
|
||||
export interface HttpFlow {
|
||||
id: string; ts: number; client: string; method: string; scheme: string; host: string; port: number; path: string;
|
||||
http_version: string; status: number; reason: string; req_ctype: string; resp_ctype: string; req_size: number; resp_size: number;
|
||||
duration_ms: number; tls_version: string; sni: string; alpn: string; intercepted: boolean; error: string; websocket: boolean;
|
||||
has_req_body: boolean; has_resp_body: boolean; req_headers: Record<string,string>; resp_headers: Record<string,string>;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
import { useState } from "react";
|
||||
import { api, type ApplyResult, type InterceptionStatus } from "../api";
|
||||
import { Badge, Modal, Steps, toast, usePoll } from "../ui";
|
||||
|
||||
const INSTALL: Record<string, string[]> = {
|
||||
"iOS / iPadOS": [
|
||||
"Download the CA to the device (or AirDrop it).",
|
||||
"Settings opens a profile prompt → Install → enter your passcode.",
|
||||
"Then Settings → General → About → Certificate Trust Settings → turn ON full trust for ‘PiLens Interception CA’.",
|
||||
"Without that second step iOS will NOT trust it. This is deliberate on Apple's part.",
|
||||
],
|
||||
"Android": [
|
||||
"Settings → Security → Encryption & credentials → Install a certificate → CA certificate.",
|
||||
"Pick the downloaded .crt.",
|
||||
"Note: Android apps only trust user-added CAs if they opt in. Many apps (and all of Chrome's pinned sites) will not be decryptable. That is expected.",
|
||||
],
|
||||
"Windows": [
|
||||
"Double-click the .crt → Install Certificate → Local Machine → ‘Trusted Root Certification Authorities’.",
|
||||
"Browsers using the system store (Edge, Chrome) will then trust it; Firefox uses its own store (below).",
|
||||
],
|
||||
"macOS": [
|
||||
"Double-click the .crt → add to the ‘login’ or ‘System’ keychain.",
|
||||
"Open Keychain Access → find ‘PiLens Interception CA’ → Get Info → Trust → ‘Always Trust’.",
|
||||
],
|
||||
"Firefox": [
|
||||
"Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import.",
|
||||
"Tick ‘Trust this CA to identify websites’.",
|
||||
],
|
||||
};
|
||||
|
||||
export default function Certificates() {
|
||||
const [st, err, reload] = usePoll<InterceptionStatus>("/api/interception", 8000);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [result, setResult] = useState<ApplyResult | null>(null);
|
||||
const [applyErr, setApplyErr] = useState("");
|
||||
const [os, setOs] = useState("iOS / iPadOS");
|
||||
const [confirmBodies, setConfirmBodies] = useState(false);
|
||||
const [draft, setDraft] = useState<InterceptionStatus["config"] | null>(null);
|
||||
|
||||
const cfg = draft ?? st?.config;
|
||||
const dirty = draft !== null && JSON.stringify(draft) !== JSON.stringify(st?.config);
|
||||
|
||||
const set = (p: Partial<InterceptionStatus["config"]>) => setDraft({ ...(cfg as any), ...p });
|
||||
|
||||
const apply = async (override?: Partial<InterceptionStatus["config"]>) => {
|
||||
const body = { ...(cfg as any), ...override };
|
||||
setBusy(true); setApplyErr(""); setResult(null);
|
||||
try {
|
||||
const r = await api<{ result: ApplyResult }>("/api/interception", { method: "PUT", body });
|
||||
setResult(r.result); setDraft(null); reload();
|
||||
toast(body.enabled ? "Interception enabled" : "Interception disabled");
|
||||
} catch (e: any) {
|
||||
setApplyErr(e.message); setResult(e.body?.result ?? null); reload();
|
||||
}
|
||||
setBusy(false);
|
||||
};
|
||||
|
||||
const ca = st?.ca;
|
||||
|
||||
return (
|
||||
<div className="page">
|
||||
<div className="page-head">
|
||||
<div><h1>Certificates & HTTPS Inspection</h1>
|
||||
<p>Authorised HTTPS interception (Mode B). Decrypts only for devices whose owner has installed and trusted the CA below.</p></div>
|
||||
</div>
|
||||
|
||||
<div className="banner warn" style={{ marginBottom: "1rem" }}><div>
|
||||
<b>Consent is required, by design.</b> PiLens can only decrypt a device after <em>you</em> install this CA on it and turn on trust.
|
||||
A trusted certificate can never be pushed over the network or via DHCP. Pinned apps, private trust stores and QUIC are reported as
|
||||
not intercepted — never silently bypassed. The CA private key never leaves the appliance and is not downloadable.
|
||||
</div></div>
|
||||
|
||||
{err && <div className="banner bad"><div>{err}</div></div>}
|
||||
|
||||
<div className="grid g2">
|
||||
<div className="card stack">
|
||||
<h3>Interception</h3>
|
||||
<label className="inline" style={{ fontSize: "1rem" }}>
|
||||
<input type="checkbox" checked={!!cfg?.enabled} disabled={busy || !ca?.present}
|
||||
onChange={(e) => apply({ enabled: e.target.checked })} />
|
||||
<b>{cfg?.enabled ? "Enabled" : "Disabled"}</b> — redirect AP clients' HTTP/HTTPS through the proxy
|
||||
</label>
|
||||
{st && <div className="row">
|
||||
<Badge kind={st.running ? "ok" : "plain"}>{st.running ? "proxy active" : "proxy stopped"}</Badge>
|
||||
<span className="hint">transparent, port {st.port}</span>
|
||||
</div>}
|
||||
{!ca?.present && <div className="banner bad"><div>The interception CA is not available. Is the mitmproxy venv installed? Run <code>sudo /opt/pilens/repair.sh</code>.</div></div>}
|
||||
{applyErr && <div className="err">{applyErr}</div>}
|
||||
{result && <Steps steps={result.steps} />}
|
||||
<p className="hint">Enabling adds an nftables redirect for ports 80/443 from the Wi-Fi only. The proxy never becomes a direct open proxy for clients (it accepts only redirected connections).</p>
|
||||
</div>
|
||||
|
||||
<div className="card stack">
|
||||
<h3>The appliance CA</h3>
|
||||
{ca?.present ? <>
|
||||
<dl className="kv">
|
||||
<dt>Subject</dt><dd className="mono">{ca.subject}</dd>
|
||||
<dt>Key</dt><dd>{ca.key_type}</dd>
|
||||
<dt>Fingerprint</dt><dd className="mono" style={{ fontSize: ".75rem", wordBreak: "break-all" }}>SHA-256 {ca.sha256}</dd>
|
||||
<dt>Valid</dt><dd>{ca.not_before} → {ca.not_after} <span className="hint">({ca.days_left} days left)</span></dd>
|
||||
</dl>
|
||||
<div className="row">
|
||||
<a className="btn primary" href="/api/interception/ca.crt" download>Download CA certificate</a>
|
||||
<button disabled={cfg?.enabled || busy} title={cfg?.enabled ? "Disable interception first" : ""}
|
||||
onClick={async () => { if (confirm("Rotate the CA? Every device must install the new one; current interception breaks until they do.")) { await api("/api/interception/ca/rotate", { method: "POST" }); reload(); toast("CA rotated"); } }}>Rotate CA</button>
|
||||
</div>
|
||||
<p className="hint">Install this on each authorised test device, then verify the fingerprint above matches before trusting it.</p>
|
||||
</> : <p className="hint">No CA yet.</p>}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="card stack" style={{ marginTop: ".9rem" }}>
|
||||
<h3>Install instructions</h3>
|
||||
<div className="row" role="tablist">
|
||||
{Object.keys(INSTALL).map((k) => <button key={k} className={os === k ? "on" : ""} onClick={() => setOs(k)}>{k}</button>)}
|
||||
</div>
|
||||
<ol style={{ margin: 0, paddingLeft: "1.2rem" }}>{INSTALL[os].map((s, i) => <li key={i} style={{ marginBottom: ".3rem" }}>{s}</li>)}</ol>
|
||||
</div>
|
||||
|
||||
{cfg && <div className="card stack" style={{ marginTop: ".9rem" }}>
|
||||
<h3>Options</h3>
|
||||
<label>Excluded hosts (not decrypted — passed straight through)
|
||||
<textarea rows={3} className="mono" value={cfg.exclude_hosts.join("\n")} placeholder={"one host/regex per line\n(?i)\\.apple\\.com$\nexample\\.com"}
|
||||
onChange={(e) => set({ exclude_hosts: e.target.value.split("\n").map((x) => x.trim()).filter(Boolean) })} />
|
||||
<span className="hint">Use for pinned apps or privacy-sensitive hosts. Matched hosts keep working but stay encrypted.</span>
|
||||
</label>
|
||||
<label className="inline">
|
||||
<input type="checkbox" checked={cfg.capture_bodies} onChange={(e) => { if (e.target.checked) setConfirmBodies(true); else set({ capture_bodies: false }); }} />
|
||||
Retain request/response bodies on disk
|
||||
</label>
|
||||
<div className="row">
|
||||
<label>Max body size (KiB)<input type="number" min={0} max={10240} value={cfg.max_body_kb} onChange={(e) => set({ max_body_kb: +e.target.value })} /></label>
|
||||
</div>
|
||||
{dirty && <div className="row"><button className="primary" disabled={busy} onClick={() => apply()}>Apply options</button>
|
||||
<button disabled={busy} onClick={() => setDraft(null)}>Discard</button></div>}
|
||||
</div>}
|
||||
|
||||
<Modal open={confirmBodies} title="Retain decrypted bodies?" onClose={() => setConfirmBodies(false)}
|
||||
footer={<><button onClick={() => setConfirmBodies(false)}>Cancel</button>
|
||||
<button className="danger" onClick={() => { set({ capture_bodies: true }); setConfirmBodies(false); }}>I understand, retain bodies</button></>}>
|
||||
<p>Response and request bodies can contain passwords, tokens, messages and personal data in clear text. They will be written to the appliance's disk until you clear them.</p>
|
||||
<p className="hint">Off by default. Only enable for a specific investigation, and clear the data (HTTP Explorer → Clear) when done. Exports and body views are audited.</p>
|
||||
</Modal>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { api, onEvent, fmtBytes, fmtTime, type HttpFlow } from "../api";
|
||||
import { Badge, Modal, copyText, toast, usePersist } from "../ui";
|
||||
|
||||
function statusKind(f: HttpFlow): "ok" | "warn" | "bad" | "plain" {
|
||||
if (f.error) return "bad";
|
||||
if (f.status >= 500) return "bad";
|
||||
if (f.status >= 400) return "warn";
|
||||
if (f.status >= 200 && f.status < 400) return "ok";
|
||||
return "plain";
|
||||
}
|
||||
|
||||
export default function HttpExplorer() {
|
||||
const [flows, setFlows] = useState<HttpFlow[]>([]);
|
||||
const [q, setQ] = useState("");
|
||||
const [onlyErr, setOnlyErr] = useState(false);
|
||||
const [live, setLive] = usePersist("http.live", true);
|
||||
const [sel, setSel] = useState<HttpFlow | null>(null);
|
||||
const [err, setErr] = useState("");
|
||||
const liveRef = useRef(live); liveRef.current = live;
|
||||
|
||||
const load = () => api<HttpFlow[]>("/api/http?limit=2000").then(setFlows).catch((e) => setErr(e.message));
|
||||
useEffect(() => { load(); }, []);
|
||||
useEffect(() => onEvent((type, d: HttpFlow) => {
|
||||
if (type !== "http" || !liveRef.current || !d.id) return;
|
||||
setFlows((f) => [d, ...f.filter((x) => x.id !== d.id)].slice(0, 3000));
|
||||
}), []);
|
||||
|
||||
const rows = useMemo(() => flows.filter((f) =>
|
||||
(!q || `${f.host}${f.path} ${f.client} ${f.method}`.toLowerCase().includes(q.toLowerCase())) &&
|
||||
(!onlyErr || f.error || f.status >= 400)), [flows, q, onlyErr]);
|
||||
|
||||
return (
|
||||
<div className="page" style={{ display: "flex", flexDirection: "column" }}>
|
||||
<div className="page-head">
|
||||
<div><h1>HTTP Explorer</h1><p>Decrypted HTTP(S) transactions from trusting devices. Requires interception (Certificates page).</p></div>
|
||||
<div className="row">
|
||||
<button className={live ? "on" : ""} onClick={() => setLive(!live)} title="Append new transactions live">⚡ Live</button>
|
||||
<button onClick={load}>Refresh</button>
|
||||
<button className="danger" onClick={async () => { if (confirm("Delete all intercepted HTTP data (flows and any retained bodies)? Captures are untouched.")) { await api("/api/interception/clear", { method: "POST" }); setFlows([]); setSel(null); toast("Cleared"); } }}>Clear</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="banner info" style={{ marginBottom: ".8rem" }}><div>
|
||||
Shown here are connections PiLens could decrypt because the device trusts the CA. Pinned/untrusting connections appear as <b>errors</b>, not content. Nothing here is captured from devices that did not opt in.
|
||||
</div></div>
|
||||
<div className="row" style={{ marginBottom: ".6rem" }}>
|
||||
<input placeholder="Filter host / path / client / method" value={q} onChange={(e) => setQ(e.target.value)} style={{ minWidth: 280 }} />
|
||||
<label className="inline"><input type="checkbox" checked={onlyErr} onChange={(e) => setOnlyErr(e.target.checked)} />Errors only</label>
|
||||
<span className="hint">{rows.length.toLocaleString()} transactions</span>
|
||||
</div>
|
||||
{err && <div className="banner bad"><div>{err}</div></div>}
|
||||
<div className="card" style={{ padding: 0, flex: 1, overflow: "auto" }}>
|
||||
<table className="data">
|
||||
<thead><tr><th>Time</th><th>Client</th><th>Method</th><th>Host</th><th>Path</th><th>Status</th><th>Type</th><th>Size</th><th>Time</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
{rows.map((f) => (
|
||||
<tr key={f.id} style={{ cursor: "pointer" }} onClick={() => setSel(f)}>
|
||||
<td className="hint" style={{ whiteSpace: "nowrap" }}>{fmtTime(f.ts * 1000).split(", ")[1] ?? fmtTime(f.ts * 1000)}</td>
|
||||
<td className="mono">{f.client}</td>
|
||||
<td className="mono">{f.method}{f.websocket && " ⇄"}</td>
|
||||
<td className="mono">{f.scheme === "https" ? "🔒" : ""}{f.host}</td>
|
||||
<td className="mono" style={{ maxWidth: 360, overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{f.path}</td>
|
||||
<td>{f.error ? <Badge kind="bad">error</Badge> : <Badge kind={statusKind(f)}>{f.status}</Badge>}</td>
|
||||
<td className="mono hint">{(f.resp_ctype || f.req_ctype || "").split(";")[0]}</td>
|
||||
<td className="mono">{fmtBytes(f.resp_size)}</td>
|
||||
<td className="mono hint">{f.duration_ms} ms</td>
|
||||
<td>›</td>
|
||||
</tr>
|
||||
))}
|
||||
{rows.length === 0 && <tr><td colSpan={10} className="empty">No transactions. Enable interception and install the CA on a test device.</td></tr>}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{sel && <FlowDetail flow={sel} onClose={() => setSel(null)} />}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Headers({ h }: { h: Record<string, string> }) {
|
||||
const entries = Object.entries(h || {});
|
||||
if (!entries.length) return <div className="hint">none</div>;
|
||||
return <table className="data"><tbody>{entries.map(([k, v]) => <tr key={k}><td className="mono" style={{ color: "var(--muted)", whiteSpace: "nowrap", verticalAlign: "top" }}>{k}</td><td className="mono" style={{ wordBreak: "break-all" }}>{v}</td></tr>)}</tbody></table>;
|
||||
}
|
||||
|
||||
function FlowDetail({ flow, onClose }: { flow: HttpFlow; onClose: () => void }) {
|
||||
const [tab, setTab] = useState<"req" | "resp">("resp");
|
||||
return (
|
||||
<Modal open title={<span className="mono" style={{ fontSize: ".95rem" }}>{flow.method} {flow.scheme}://{flow.host}{flow.path}</span>} onClose={onClose}
|
||||
footer={<><button onClick={() => copyText(`${flow.method} ${flow.scheme}://${flow.host}${flow.path}`)}>Copy URL</button><button className="primary" onClick={onClose}>Close</button></>}>
|
||||
<div className="row" style={{ gap: ".5rem", flexWrap: "wrap" }}>
|
||||
<Badge kind={flow.error ? "bad" : statusKind(flow)}>{flow.error ? "error" : `${flow.status} ${flow.reason}`}</Badge>
|
||||
{flow.intercepted ? <Badge kind="ok">decrypted</Badge> : <Badge kind="warn">not decrypted</Badge>}
|
||||
<span className="hint mono">{flow.client} · {flow.http_version} · {flow.duration_ms} ms</span>
|
||||
</div>
|
||||
{flow.tls_version && <div className="hint mono">{flow.tls_version}{flow.sni ? ` · SNI ${flow.sni}` : ""}{flow.alpn ? ` · ALPN ${flow.alpn}` : ""}</div>}
|
||||
{flow.error && <div className="banner bad" style={{ margin: ".5rem 0" }}><div>{flow.error}</div></div>}
|
||||
|
||||
<div className="row" style={{ gap: ".25rem", marginTop: ".5rem" }}>
|
||||
<button className={tab === "req" ? "on" : ""} onClick={() => setTab("req")}>Request</button>
|
||||
<button className={tab === "resp" ? "on" : ""} onClick={() => setTab("resp")}>Response</button>
|
||||
</div>
|
||||
{tab === "req" ? <div className="stack">
|
||||
<h3>Request headers</h3><Headers h={flow.req_headers} />
|
||||
<Body flow={flow} which="req" has={flow.has_req_body} size={flow.req_size} ctype={flow.req_ctype} />
|
||||
</div> : <div className="stack">
|
||||
<h3>Response headers</h3><Headers h={flow.resp_headers} />
|
||||
<Body flow={flow} which="resp" has={flow.has_resp_body} size={flow.resp_size} ctype={flow.resp_ctype} />
|
||||
</div>}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
function Body({ flow, which, has, size, ctype }: { flow: HttpFlow; which: "req" | "resp"; has: boolean; size: number; ctype: string }) {
|
||||
const [data, setData] = useState<Uint8Array | null>(null);
|
||||
const [mode, setMode] = useState<"auto" | "text" | "hex" | "image">("auto");
|
||||
const [loading, setLoading] = useState(false);
|
||||
const ct = (ctype || "").split(";")[0];
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const r = await fetch(`/api/http/${flow.id}/body/${which}`, { headers: { "X-Requested-With": "pilens" }, credentials: "same-origin" });
|
||||
if (!r.ok) throw new Error(await r.text());
|
||||
setData(new Uint8Array(await r.arrayBuffer()));
|
||||
} catch (e) { toast((e as Error).message, true); }
|
||||
setLoading(false);
|
||||
};
|
||||
|
||||
if (!has) return <><h3>Body</h3><div className="hint">{size > 0 ? `${fmtBytes(size)} not retained (body capture is off for privacy). Enable it on the Certificates page for a specific investigation.` : "empty"}</div></>;
|
||||
|
||||
const view = mode === "auto" ? (ct.startsWith("image/") ? "image" : /json|text|xml|javascript|html|urlencoded/.test(ct) ? "text" : "hex") : mode;
|
||||
|
||||
return <><div className="row spread"><h3>Body ({fmtBytes(size)}, {ct || "unknown"})</h3>
|
||||
<div className="row" style={{ gap: ".25rem" }}>
|
||||
{["auto", "text", "hex", "image"].map((m) => <button key={m} className={"small" + (mode === m ? " on" : "")} onClick={() => setMode(m as any)}>{m}</button>)}
|
||||
</div></div>
|
||||
{!data ? <button disabled={loading} onClick={load}>{loading ? "Loading…" : "Load body"}</button> : <BodyView data={data} view={view} id={flow.id} which={which} ct={ct} />}
|
||||
</>;
|
||||
}
|
||||
|
||||
function BodyView({ data, view, id, which, ct }: { data: Uint8Array; view: string; id: string; which: string; ct: string }) {
|
||||
const text = useMemo(() => { try { return new TextDecoder().decode(data); } catch { return ""; } }, [data]);
|
||||
if (view === "image") {
|
||||
// Sandboxed: served as application/octet-stream with nosniff; render via blob URL.
|
||||
const url = useMemo(() => URL.createObjectURL(new Blob([data as BlobPart], { type: ct || "application/octet-stream" })), [data, ct]);
|
||||
return <img src={url} alt="response body" style={{ maxWidth: "100%", maxHeight: "50vh", background: "var(--panel-2)" }} />;
|
||||
}
|
||||
if (view === "hex") {
|
||||
const lines = [];
|
||||
for (let o = 0; o < Math.min(data.length, 16384); o += 16) {
|
||||
const slice = Array.from(data.slice(o, o + 16));
|
||||
lines.push(o.toString(16).padStart(6, "0") + " " + slice.map((b) => b.toString(16).padStart(2, "0")).join(" ").padEnd(47) + " " + slice.map((b) => b >= 32 && b < 127 ? String.fromCharCode(b) : ".").join(""));
|
||||
}
|
||||
return <pre style={preStyle}>{lines.join("\n")}{data.length > 16384 ? "\n… truncated" : ""}</pre>;
|
||||
}
|
||||
let pretty = text;
|
||||
if (/json/.test(ct)) { try { pretty = JSON.stringify(JSON.parse(text), null, 2); } catch { /* keep raw */ } }
|
||||
return <><pre style={preStyle}>{pretty.slice(0, 200000)}{pretty.length > 200000 ? "\n… truncated" : ""}</pre>
|
||||
<a className="btn small" href={`/api/http/${id}/body/${which}`} download>Download raw</a></>;
|
||||
}
|
||||
|
||||
const preStyle: React.CSSProperties = { maxHeight: "48vh", overflow: "auto", background: "var(--panel-2)", padding: ".6rem", borderRadius: 6, fontSize: ".82rem", whiteSpace: "pre-wrap", wordBreak: "break-all", margin: 0 };
|
||||
Reference in New Issue
Block a user