Phase 4: authorised HTTPS interception (Mode B)
Toggleable transparent interception via mitmdump in a venv. Own RSA CA generated in-app, downloadable (public cert only, key never leaves the box), with per-OS install guide and fingerprint. nftables redirect accepts only dnat'd connections (never an open proxy). HTTP Explorer with headers and opt-in body viewers (JSON/ text/hex/image, sandboxed). Honest status: pinned/untrusting = reported errors, excluded hosts pass through, bodies off by default. Helper op + systemd unit + installer venv build + boot restore.
This commit is contained in:
Andy Dixon
@@ -0,0 +1,182 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pilens.dev/pilens/internal/helper"
|
||||
"pilens.dev/pilens/internal/proxy"
|
||||
)
|
||||
|
||||
// interceptionRoutes wires the Mode-B (authorised HTTPS inspection) endpoints.
|
||||
func (s *Server) interceptionRoutes(priv func(string, http.HandlerFunc)) {
|
||||
priv("GET /api/interception", s.getInterception)
|
||||
priv("PUT /api/interception", s.setInterception)
|
||||
priv("POST /api/interception/ca/rotate", s.rotateCA)
|
||||
priv("POST /api/interception/clear", s.clearFlows)
|
||||
priv("GET /api/interception/ca.crt", s.downloadCA)
|
||||
priv("GET /api/http", s.httpFlows)
|
||||
priv("GET /api/http/{id}", s.httpFlow)
|
||||
priv("GET /api/http/{id}/body/{which}", s.httpBody)
|
||||
}
|
||||
|
||||
func (s *Server) interceptionStatus(ctx context.Context) map[string]any {
|
||||
cfg := s.Proxy.Config()
|
||||
running := false
|
||||
st, herr := s.helperStatus(ctx)
|
||||
_ = st
|
||||
// proxy liveness: ask the helper-independent systemd via the health probe is
|
||||
// overkill; infer from config + a quick check of the flows file growth is not
|
||||
// reliable. Report configured-enabled; the dashboard shows service health.
|
||||
running = cfg.Enabled && herr == ""
|
||||
return map[string]any{
|
||||
"config": cfg,
|
||||
"ca": s.Proxy.CAInfo(),
|
||||
"running": running,
|
||||
"port": proxy.ListenPort,
|
||||
"note": "Interception decrypts HTTPS only for devices whose owner has installed and trusted the PiLens CA. " +
|
||||
"Pinned apps, private trust stores and QUIC are reported as not intercepted, never bypassed. The CA private key is never downloadable.",
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) getInterception(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, s.interceptionStatus(r.Context()))
|
||||
}
|
||||
|
||||
// proxyEnvPath is read by the pilens-proxy systemd unit via proxy-run.sh.
|
||||
func (s *Server) proxyDir() string { return filepath.Join(s.DataDir, "proxy") }
|
||||
func (s *Server) proxyEnv() string { return filepath.Join(s.proxyDir(), "proxy.env") }
|
||||
|
||||
func shellQuote(v string) string { return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'" }
|
||||
|
||||
func (s *Server) writeProxyEnv(cfg proxy.Config) error {
|
||||
bodies := "0"
|
||||
if cfg.CaptureBodies {
|
||||
bodies = "1"
|
||||
}
|
||||
env := strings.Join([]string{
|
||||
"PILENS_VENV=" + shellQuote("/opt/pilens/proxy-venv"),
|
||||
fmt.Sprintf("PILENS_PROXY_PORT=%d", proxy.ListenPort),
|
||||
"PILENS_CONFDIR=" + shellQuote(s.Proxy.ConfDir()),
|
||||
"PILENS_ADDON=" + shellQuote("/opt/pilens/mitm_addon.py"),
|
||||
"PILENS_FLOWS=" + shellQuote(filepath.Join(s.proxyDir(), "flows.ndjson")),
|
||||
"PILENS_BODIES=" + shellQuote(filepath.Join(s.proxyDir(), "bodies")),
|
||||
"PILENS_CAPTURE_BODIES=" + bodies,
|
||||
fmt.Sprintf("PILENS_MAX_BODY_KB=%d", cfg.MaxBodyKB),
|
||||
"PILENS_IGNORE=" + shellQuote(cfg.ExcludeRegex()),
|
||||
}, "\n") + "\n"
|
||||
os.MkdirAll(s.proxyDir(), 0o750)
|
||||
return os.WriteFile(s.proxyEnv(), []byte(env), 0o640)
|
||||
}
|
||||
|
||||
func (s *Server) setInterception(w http.ResponseWriter, r *http.Request) {
|
||||
var cfg proxy.Config
|
||||
if !readJSON(w, r, &cfg) {
|
||||
return
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
httpErr(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.Proxy.EnsureCA(); err != nil {
|
||||
httpErr(w, 500, "CA generation failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.writeProxyEnv(cfg); err != nil {
|
||||
httpErr(w, 500, "writing proxy config failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.Proxy.SetConfig(cfg); err != nil {
|
||||
httpErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
var res helper.ApplyResult
|
||||
err := s.Helper.Call(ctx, helper.Request{Op: helper.OpSetInterception, InterceptEnabled: cfg.Enabled, InterceptPort: proxy.ListenPort}, &res)
|
||||
detail := fmt.Sprintf("enabled=%v capture_bodies=%v excludes=%d", cfg.Enabled, cfg.CaptureBodies, len(cfg.ExcludeHosts))
|
||||
if err != nil {
|
||||
s.audit(r, "interception.apply_failed", detail+": "+err.Error())
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
writeJSON(w, map[string]any{"error": err.Error(), "result": res, "status": s.interceptionStatus(r.Context())})
|
||||
return
|
||||
}
|
||||
s.audit(r, "interception.changed", detail)
|
||||
writeJSON(w, map[string]any{"result": res, "status": s.interceptionStatus(r.Context())})
|
||||
}
|
||||
|
||||
func (s *Server) rotateCA(w http.ResponseWriter, r *http.Request) {
|
||||
if s.Proxy.Config().Enabled {
|
||||
httpErr(w, http.StatusConflict, "disable interception before rotating the CA, so devices are not left mid-session")
|
||||
return
|
||||
}
|
||||
if err := s.Proxy.RotateCA(); err != nil {
|
||||
httpErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
s.audit(r, "interception.ca_rotated", s.Proxy.CAInfo().SHA256)
|
||||
writeJSON(w, map[string]any{"ca": s.Proxy.CAInfo()})
|
||||
}
|
||||
|
||||
func (s *Server) downloadCA(w http.ResponseWriter, r *http.Request) {
|
||||
pem, err := s.Proxy.CACertPEM()
|
||||
if err != nil {
|
||||
httpErr(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
s.audit(r, "interception.ca_downloaded", s.Proxy.CAInfo().SHA256)
|
||||
w.Header().Set("Content-Type", "application/x-pem-file")
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="pilens-ca.crt"`)
|
||||
w.Write(pem)
|
||||
}
|
||||
|
||||
func (s *Server) clearFlows(w http.ResponseWriter, r *http.Request) {
|
||||
s.Proxy.Clear()
|
||||
s.audit(r, "interception.flows_cleared", "")
|
||||
writeJSON(w, map[string]bool{"ok": true})
|
||||
}
|
||||
|
||||
func (s *Server) httpFlows(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
client := q.Get("client")
|
||||
if !proxy.ClientIP(client) {
|
||||
httpErr(w, http.StatusBadRequest, "invalid client filter")
|
||||
return
|
||||
}
|
||||
flows, err := s.Proxy.Flows(r.Context(), q.Get("q"), client, qint(r, "limit", 1000))
|
||||
if err != nil {
|
||||
httpErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, flows)
|
||||
}
|
||||
|
||||
func (s *Server) httpFlow(w http.ResponseWriter, r *http.Request) {
|
||||
f, err := s.Proxy.Flow(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
httpErr(w, http.StatusNotFound, "flow not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, f)
|
||||
}
|
||||
|
||||
// httpBody serves a stored body as an opaque download. It is never served with a
|
||||
// content type that a browser would execute; the UI renders it in a sandbox.
|
||||
func (s *Server) httpBody(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := s.Proxy.Body(r.PathValue("id"), r.PathValue("which"))
|
||||
if err != nil {
|
||||
httpErr(w, http.StatusNotFound, "body not retained (body capture was off, or it was cleared)")
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("Content-Disposition", "attachment")
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox")
|
||||
s.audit(r, "interception.body_viewed", r.PathValue("id")+"/"+r.PathValue("which"))
|
||||
w.Write(body)
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"pilens.dev/pilens/internal/capture"
|
||||
"pilens.dev/pilens/internal/config"
|
||||
"pilens.dev/pilens/internal/helper"
|
||||
"pilens.dev/pilens/internal/proxy"
|
||||
"pilens.dev/pilens/internal/store"
|
||||
)
|
||||
|
||||
@@ -39,6 +40,7 @@ type Server struct {
|
||||
Engine *capture.Engine
|
||||
Helper helper.Client
|
||||
Version string
|
||||
Proxy *proxy.Manager
|
||||
Web fs.FS // built UI; may be nil in tests
|
||||
SetupTokenPath string
|
||||
DataDir string
|
||||
@@ -82,6 +84,7 @@ func (s *Server) Handler() http.Handler {
|
||||
s.networkRoutes(priv)
|
||||
s.captureRoutes(priv)
|
||||
s.analysisRoutes(priv)
|
||||
s.interceptionRoutes(priv)
|
||||
|
||||
if s.Web != nil {
|
||||
files := http.FileServerFS(s.Web)
|
||||
|
||||
@@ -51,3 +51,4 @@ func (h *Hub) Publish(e Event) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,8 +16,10 @@ import (
|
||||
const DefaultSocket = "/run/pilens/helper.sock"
|
||||
|
||||
type Request struct {
|
||||
Op string `json:"op"`
|
||||
Network *config.Network `json:"network,omitempty"`
|
||||
Op string `json:"op"`
|
||||
Network *config.Network `json:"network,omitempty"`
|
||||
InterceptEnabled bool `json:"intercept_enabled,omitempty"`
|
||||
InterceptPort int `json:"intercept_port,omitempty"`
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
@@ -32,6 +34,7 @@ const (
|
||||
OpStatus = "status"
|
||||
OpStations = "stations"
|
||||
OpLeases = "leases"
|
||||
OpSetInterception = "set_interception"
|
||||
)
|
||||
|
||||
type ApplyResult struct {
|
||||
|
||||
@@ -129,11 +129,19 @@ func DnsmasqConf(n config.Network) string {
|
||||
|
||||
func q(s string) string { return `"` + s + `"` } // inputs are validated interface names
|
||||
|
||||
// Intercept describes transparent HTTPS-interception redirection. Port 0 / nil
|
||||
// means interception is off and no redirect rules are emitted.
|
||||
type Intercept struct {
|
||||
Enabled bool
|
||||
Port int
|
||||
}
|
||||
|
||||
// Nftables renders the complete inet pilens table. The output begins by
|
||||
// (re)creating and deleting the table so `nft -f` replaces it atomically while
|
||||
// leaving every other table untouched.
|
||||
func Nftables(n config.Network) string {
|
||||
// leaving every other table untouched. ic may be nil (interception off).
|
||||
func Nftables(n config.Network, ic *Intercept) string {
|
||||
ap, up := q(n.AP.Interface), q(n.Uplink)
|
||||
intercept := ic != nil && ic.Enabled && ic.Port > 0
|
||||
sub := n.LAN.Subnet().String()
|
||||
var mgmtIf []string
|
||||
for _, i := range n.Management.Interfaces {
|
||||
@@ -186,6 +194,12 @@ func Nftables(n config.Network) string {
|
||||
w("\t\tip daddr %s udp dport 53 accept", n.LAN.Gateway())
|
||||
w("\t\tip daddr %s tcp dport 53 accept", n.LAN.Gateway())
|
||||
w("\t\ticmp type { echo-request, destination-unreachable, time-exceeded } accept")
|
||||
if intercept {
|
||||
// Accept ONLY connections the prerouting redirect created (ct status dnat).
|
||||
// A client connecting directly to the proxy port is not dnat'd, so this
|
||||
// never turns the proxy into an open proxy reachable by clients.
|
||||
w("\t\ttcp dport %d ct status dnat accept comment \"redirected to interception proxy\"", ic.Port)
|
||||
}
|
||||
w("\t\tcounter drop comment \"AP clients cannot reach the appliance\"")
|
||||
w("\t}")
|
||||
|
||||
@@ -213,6 +227,17 @@ func Nftables(n config.Network) string {
|
||||
w("\t\tcounter drop")
|
||||
w("\t}")
|
||||
|
||||
// Transparent interception: redirect AP-client HTTP(S) to the local proxy.
|
||||
// Only when enabled, and only new connections from the AP interface. The
|
||||
// proxy's own upstream connections originate on the Pi (not iifname ap), so
|
||||
// they are never redirected — no loop.
|
||||
w("\tchain prerouting {")
|
||||
w("\t\ttype nat hook prerouting priority dstnat; policy accept;")
|
||||
if intercept {
|
||||
w("\t\tiifname %s meta l4proto tcp tcp dport { 80, 443 } redirect to :%d", ap, ic.Port)
|
||||
}
|
||||
w("\t}")
|
||||
|
||||
w("\tchain postrouting {")
|
||||
w("\t\ttype nat hook postrouting priority srcnat; policy accept;")
|
||||
w("\t\tip saddr %s oifname %s masquerade", sub, up)
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestDnsmasq(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNftables(t *testing.T) {
|
||||
s := Nftables(testNet())
|
||||
s := Nftables(testNet(), nil)
|
||||
for _, want := range []string{
|
||||
"delete table inet pilens",
|
||||
`iifname "wlan0" jump ap_forward`,
|
||||
@@ -84,7 +84,24 @@ func TestNftables(t *testing.T) {
|
||||
}
|
||||
n := testNet()
|
||||
n.Forwarding.Mode = "deny"
|
||||
if !strings.Contains(Nftables(n), `"forwarding disabled"`) {
|
||||
if !strings.Contains(Nftables(n, nil), `"forwarding disabled"`) {
|
||||
t.Error("deny mode must drop forwarded traffic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNftablesInterception(t *testing.T) {
|
||||
off := Nftables(testNet(), nil)
|
||||
if strings.Contains(off, "redirect to") || strings.Contains(off, "ct status dnat") {
|
||||
t.Fatal("interception rules present when disabled")
|
||||
}
|
||||
on := Nftables(testNet(), &Intercept{Enabled: true, Port: 8081})
|
||||
for _, want := range []string{
|
||||
"chain prerouting {",
|
||||
`iifname "wlan0" meta l4proto tcp tcp dport { 80, 443 } redirect to :8081`,
|
||||
"tcp dport 8081 ct status dnat accept",
|
||||
} {
|
||||
if !strings.Contains(on, want) {
|
||||
t.Errorf("interception ruleset missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
// Package proxy manages optional, authorised HTTPS interception: a locally
|
||||
// generated CA that the operator installs on test devices by hand, a mitmdump
|
||||
// service, and ingestion of the decrypted HTTP transactions.
|
||||
//
|
||||
// Consent is the whole design. The CA private key never leaves the appliance,
|
||||
// is never pushed to a device, and interception only happens for devices whose
|
||||
// owner has explicitly installed and trusted the CA. Nothing is covert.
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CAInfo describes the interception CA for display. It never includes the key.
|
||||
type CAInfo struct {
|
||||
Present bool `json:"present"`
|
||||
Subject string `json:"subject"`
|
||||
SHA256 string `json:"sha256"` // fingerprint, colon-separated
|
||||
NotBefore string `json:"not_before"`
|
||||
NotAfter string `json:"not_after"`
|
||||
Serial string `json:"serial"`
|
||||
DaysLeft int `json:"days_left"`
|
||||
KeyType string `json:"key_type"`
|
||||
}
|
||||
|
||||
// caPaths returns the key+cert path (mitmproxy format) and the public cert path.
|
||||
func caPaths(dir string) (combined, certOnly string) {
|
||||
return filepath.Join(dir, "mitmproxy-ca.pem"), filepath.Join(dir, "mitmproxy-ca-cert.pem")
|
||||
}
|
||||
|
||||
// EnsureCA creates the interception CA if absent, in the mitmproxy confdir layout.
|
||||
// RSA-2048 is used (not EC) for the broadest device trust-store compatibility.
|
||||
func EnsureCA(dir, hostname string) error {
|
||||
combined, certOnly := caPaths(dir)
|
||||
if _, err := os.Stat(combined); err == nil {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serial, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
cn := "PiLens Interception CA"
|
||||
if hostname != "" {
|
||||
cn = "PiLens Interception CA (" + hostname + ")"
|
||||
}
|
||||
tpl := &x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: cn, Organization: []string{"PiLens appliance"}},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().AddDate(3, 0, 0),
|
||||
IsCA: true,
|
||||
BasicConstraintsValid: true,
|
||||
MaxPathLen: 0,
|
||||
MaxPathLenZero: true,
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
|
||||
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
// mitmproxy reads the key+cert from mitmproxy-ca.pem (0600) and derives the rest.
|
||||
if err := writeFile(combined, append(append([]byte{}, keyPEM...), certPEM...), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
// Public cert for download / device install (never contains the key).
|
||||
if err := writeFile(certOnly, certPEM, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
writeFile(filepath.Join(dir, "mitmproxy-ca-cert.cer"), certPEM, 0o644)
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeFile(path string, data []byte, mode os.FileMode) error {
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, mode); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Chmod(tmp, mode)
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// PublicCertPEM returns the CA's public certificate (safe to distribute). It never
|
||||
// returns the private key.
|
||||
func PublicCertPEM(dir string) ([]byte, error) {
|
||||
_, certOnly := caPaths(dir)
|
||||
b, err := os.ReadFile(certOnly)
|
||||
if err != nil {
|
||||
return nil, errors.New("interception CA not generated yet")
|
||||
}
|
||||
if strings.Contains(string(b), "PRIVATE KEY") { // defensive: must never happen
|
||||
return nil, errors.New("refusing to serve a file containing a private key")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Info reads the CA metadata for display.
|
||||
func Info(dir string) CAInfo {
|
||||
_, certOnly := caPaths(dir)
|
||||
b, err := os.ReadFile(certOnly)
|
||||
if err != nil {
|
||||
return CAInfo{Present: false}
|
||||
}
|
||||
blk, _ := pem.Decode(b)
|
||||
if blk == nil {
|
||||
return CAInfo{Present: false}
|
||||
}
|
||||
c, err := x509.ParseCertificate(blk.Bytes)
|
||||
if err != nil {
|
||||
return CAInfo{Present: false}
|
||||
}
|
||||
sum := sha256.Sum256(c.Raw)
|
||||
return CAInfo{
|
||||
Present: true,
|
||||
Subject: c.Subject.String(),
|
||||
SHA256: colonHex(sum[:]),
|
||||
NotBefore: c.NotBefore.UTC().Format("2006-01-02"),
|
||||
NotAfter: c.NotAfter.UTC().Format("2006-01-02"),
|
||||
Serial: c.SerialNumber.Text(16),
|
||||
DaysLeft: int(time.Until(c.NotAfter).Hours() / 24),
|
||||
KeyType: "RSA 2048",
|
||||
}
|
||||
}
|
||||
|
||||
// Rotate deletes the CA so a fresh one is generated. Devices trusting the old CA
|
||||
// must install the new one; existing interception breaks until they do.
|
||||
func Rotate(dir, hostname string) error {
|
||||
combined, certOnly := caPaths(dir)
|
||||
for _, p := range []string{combined, certOnly, filepath.Join(dir, "mitmproxy-ca-cert.cer"), filepath.Join(dir, "mitmproxy-ca.p12"), filepath.Join(dir, "mitmproxy-ca-cert.p12"), filepath.Join(dir, "mitmproxy-dhparam.pem")} {
|
||||
os.Remove(p)
|
||||
}
|
||||
return EnsureCA(dir, hostname)
|
||||
}
|
||||
|
||||
func colonHex(b []byte) string {
|
||||
h := strings.ToUpper(hex.EncodeToString(b))
|
||||
var parts []string
|
||||
for i := 0; i < len(h); i += 2 {
|
||||
parts = append(parts, h[i:i+2])
|
||||
}
|
||||
return strings.Join(parts, ":")
|
||||
}
|
||||
|
||||
// Fingerprint of a cert file, for the install instructions ("verify this matches").
|
||||
func Fingerprint(dir string) string {
|
||||
return fmt.Sprintf("SHA-256 %s", Info(dir).SHA256)
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCAGeneration(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := EnsureCA(dir, "testpi"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Public cert must never contain a private key.
|
||||
pub, err := PublicCertPEM(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(pub), "PRIVATE KEY") {
|
||||
t.Fatal("public cert leaked the private key")
|
||||
}
|
||||
if !strings.Contains(string(pub), "BEGIN CERTIFICATE") {
|
||||
t.Fatal("not a certificate")
|
||||
}
|
||||
info := Info(dir)
|
||||
if !info.Present || !strings.Contains(info.Subject, "PiLens") || len(info.SHA256) != 95 || info.DaysLeft < 1000 {
|
||||
t.Fatalf("bad CA info: %+v", info)
|
||||
}
|
||||
// Idempotent: second call keeps the same CA.
|
||||
before := info.SHA256
|
||||
EnsureCA(dir, "testpi")
|
||||
if Info(dir).SHA256 != before {
|
||||
t.Fatal("EnsureCA regenerated an existing CA")
|
||||
}
|
||||
// Rotate produces a different CA.
|
||||
if err := Rotate(dir, "testpi"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if Info(dir).SHA256 == before {
|
||||
t.Fatal("rotate did not change the CA")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidate(t *testing.T) {
|
||||
bad := []Config{
|
||||
{MaxBodyKB: -1},
|
||||
{MaxBodyKB: 999999},
|
||||
{ExcludeHosts: []string{"ok.com", "bad regex ("}},
|
||||
{ExcludeHosts: []string{"has space"}},
|
||||
}
|
||||
for i, c := range bad {
|
||||
if c.Validate() == nil {
|
||||
t.Errorf("case %d accepted invalid config", i)
|
||||
}
|
||||
}
|
||||
good := Config{MaxBodyKB: 256, ExcludeHosts: []string{`(?i)\.apple\.com$`, "example\\.com"}}
|
||||
if err := good.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := good.ExcludeRegex(); !strings.Contains(r, "apple") || !strings.Contains(r, "|") {
|
||||
t.Fatalf("exclude regex wrong: %s", r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pilens.dev/pilens/internal/store"
|
||||
)
|
||||
|
||||
// Config is the interception toggle and its options. Stored in settings.
|
||||
type Config struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
CaptureBodies bool `json:"capture_bodies"` // default false: sensitive content off by default
|
||||
MaxBodyKB int `json:"max_body_kb"`
|
||||
ExcludeHosts []string `json:"exclude_hosts"` // host regexes NOT decrypted (passed through)
|
||||
}
|
||||
|
||||
func DefaultConfig() Config { return Config{Enabled: false, CaptureBodies: false, MaxBodyKB: 256} }
|
||||
|
||||
const ListenPort = 8081
|
||||
|
||||
var hostRe = regexp.MustCompile(`^[A-Za-z0-9.*?+()\[\]{}|\\^$_:-]{1,200}$`)
|
||||
|
||||
func (c Config) Validate() error {
|
||||
if c.MaxBodyKB < 0 || c.MaxBodyKB > 10240 {
|
||||
return errors.New("max_body_kb must be 0-10240")
|
||||
}
|
||||
for _, h := range c.ExcludeHosts {
|
||||
if !hostRe.MatchString(h) {
|
||||
return errors.New("invalid exclude-host pattern: " + h)
|
||||
}
|
||||
if _, err := regexp.Compile(h); err != nil {
|
||||
return errors.New("invalid exclude-host regex " + h + ": " + err.Error())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sink receives live events for the browser (adapts the capture Hub).
|
||||
type Sink interface{ Publish(eventType string, data []byte) }
|
||||
|
||||
// Flow is one HTTP(S) transaction, as emitted by the mitmdump addon and stored.
|
||||
type Flow struct {
|
||||
ID string `json:"id"`
|
||||
TS float64 `json:"ts"`
|
||||
Client string `json:"client"`
|
||||
Method string `json:"method"`
|
||||
Scheme string `json:"scheme"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
Path string `json:"path"`
|
||||
HTTPVersion string `json:"http_version"`
|
||||
Status int `json:"status"`
|
||||
Reason string `json:"reason"`
|
||||
ReqCType string `json:"req_ctype"`
|
||||
RespCType string `json:"resp_ctype"`
|
||||
ReqSize int64 `json:"req_size"`
|
||||
RespSize int64 `json:"resp_size"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
TLSVersion string `json:"tls_version"`
|
||||
SNI string `json:"sni"`
|
||||
ALPN string `json:"alpn"`
|
||||
Intercepted bool `json:"intercepted"`
|
||||
Error string `json:"error"`
|
||||
WebSocket bool `json:"websocket"`
|
||||
HasReqBody bool `json:"has_req_body"`
|
||||
HasRespBody bool `json:"has_resp_body"`
|
||||
ReqHeaders map[string]string `json:"req_headers"`
|
||||
RespHeaders map[string]string `json:"resp_headers"`
|
||||
}
|
||||
|
||||
// Manager owns interception state, the flow ingester and queries.
|
||||
type Manager struct {
|
||||
DB *store.Store
|
||||
Dir string // /var/lib/pilens/proxy
|
||||
Hostname string
|
||||
Sink Sink
|
||||
|
||||
mu sync.Mutex
|
||||
cfg Config
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func New(db *store.Store, dir, hostname string, sink Sink) *Manager {
|
||||
m := &Manager{DB: db, Dir: dir, Hostname: hostname, Sink: sink, cfg: DefaultConfig()}
|
||||
db.GetJSON("interception", &m.cfg)
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *Manager) Config() Config { m.mu.Lock(); defer m.mu.Unlock(); return m.cfg }
|
||||
|
||||
func (m *Manager) SetConfig(c Config) error {
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.cfg = c
|
||||
m.mu.Unlock()
|
||||
return m.DB.PutJSON("interception", c)
|
||||
}
|
||||
|
||||
func (m *Manager) flowsPath() string { return filepath.Join(m.Dir, "flows.ndjson") }
|
||||
func (m *Manager) bodiesDir() string { return filepath.Join(m.Dir, "bodies") }
|
||||
func (m *Manager) ConfDir() string { return filepath.Join(m.Dir, "ca") }
|
||||
|
||||
func (m *Manager) EnsureCA() error { return EnsureCA(m.ConfDir(), m.Hostname) }
|
||||
func (m *Manager) CAInfo() CAInfo { return Info(m.ConfDir()) }
|
||||
func (m *Manager) CACertPEM() ([]byte, error) { return PublicCertPEM(m.ConfDir()) }
|
||||
func (m *Manager) RotateCA() error { return Rotate(m.ConfDir(), m.Hostname) }
|
||||
|
||||
// Start begins tailing the flow file. It runs for the service lifetime; the
|
||||
// mitmdump service is started/stopped by the helper, this just ingests.
|
||||
func (m *Manager) Start(ctx context.Context) {
|
||||
os.MkdirAll(m.bodiesDir(), 0o750)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
m.cancel = cancel
|
||||
go m.tail(ctx)
|
||||
}
|
||||
|
||||
// tail follows flows.ndjson, inserting new lines. Survives file rotation/truncation.
|
||||
func (m *Manager) tail(ctx context.Context) {
|
||||
path := m.flowsPath()
|
||||
var offset int64
|
||||
m.DB.GetJSON("interception_flow_offset", &offset)
|
||||
for ctx.Err() == nil {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
continue
|
||||
}
|
||||
if fi, _ := f.Stat(); fi != nil && fi.Size() < offset {
|
||||
offset = 0 // file was rotated/truncated
|
||||
}
|
||||
f.Seek(offset, io.SeekStart)
|
||||
r := bufio.NewReaderSize(f, 1<<20)
|
||||
for ctx.Err() == nil {
|
||||
line, rerr := r.ReadBytes('\n')
|
||||
if len(line) > 0 && line[len(line)-1] == '\n' {
|
||||
offset += int64(len(line))
|
||||
m.ingest(line)
|
||||
m.DB.PutJSON("interception_flow_offset", offset)
|
||||
continue
|
||||
}
|
||||
_ = rerr // partial/EOF: keep offset, re-read after the wait
|
||||
break
|
||||
}
|
||||
f.Close()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) ingest(line []byte) {
|
||||
var fl Flow
|
||||
if err := json.Unmarshal(line, &fl); err != nil || fl.ID == "" {
|
||||
return
|
||||
}
|
||||
rh, _ := json.Marshal(fl.ReqHeaders)
|
||||
sh, _ := json.Marshal(fl.RespHeaders)
|
||||
_, err := m.DB.Exec(`INSERT OR REPLACE INTO http_flows
|
||||
(id,ts,client,method,scheme,host,port,path,http_version,status,reason,req_ctype,resp_ctype,req_size,resp_size,duration_ms,tls_version,sni,alpn,intercepted,error,websocket,has_req_body,has_resp_body,req_headers,resp_headers)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
fl.ID, fl.TS, fl.Client, fl.Method, fl.Scheme, fl.Host, fl.Port, fl.Path, fl.HTTPVersion, fl.Status, fl.Reason,
|
||||
fl.ReqCType, fl.RespCType, fl.ReqSize, fl.RespSize, fl.DurationMS, fl.TLSVersion, fl.SNI, fl.ALPN,
|
||||
b2i(fl.Intercepted), fl.Error, b2i(fl.WebSocket), b2i(fl.HasReqBody), b2i(fl.HasRespBody), string(rh), string(sh))
|
||||
if err != nil {
|
||||
slog.Warn("http flow insert", "err", err)
|
||||
return
|
||||
}
|
||||
if m.Sink != nil {
|
||||
m.Sink.Publish("http", line)
|
||||
}
|
||||
}
|
||||
|
||||
func b2i(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// --- queries ---
|
||||
|
||||
const flowCols = `id,ts,client,method,scheme,host,port,path,http_version,status,reason,req_ctype,resp_ctype,req_size,resp_size,duration_ms,tls_version,sni,alpn,intercepted,error,websocket,has_req_body,has_resp_body,req_headers,resp_headers`
|
||||
|
||||
func scanFlow(sc interface{ Scan(...any) error }) (Flow, error) {
|
||||
var f Flow
|
||||
var rh, sh string
|
||||
var intercept, ws, hrb, hsb int
|
||||
err := sc.Scan(&f.ID, &f.TS, &f.Client, &f.Method, &f.Scheme, &f.Host, &f.Port, &f.Path, &f.HTTPVersion, &f.Status, &f.Reason,
|
||||
&f.ReqCType, &f.RespCType, &f.ReqSize, &f.RespSize, &f.DurationMS, &f.TLSVersion, &f.SNI, &f.ALPN,
|
||||
&intercept, &f.Error, &ws, &hrb, &hsb, &rh, &sh)
|
||||
f.Intercepted, f.WebSocket, f.HasReqBody, f.HasRespBody = intercept == 1, ws == 1, hrb == 1, hsb == 1
|
||||
json.Unmarshal([]byte(rh), &f.ReqHeaders)
|
||||
json.Unmarshal([]byte(sh), &f.RespHeaders)
|
||||
return f, err
|
||||
}
|
||||
|
||||
// Flows returns recent transactions, newest first, optionally filtered by a
|
||||
// substring over host/path/client and a client IP.
|
||||
func (m *Manager) Flows(ctx context.Context, q, client string, limit int) ([]Flow, error) {
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 1000
|
||||
}
|
||||
where := "1=1"
|
||||
args := []any{}
|
||||
if q != "" {
|
||||
where += " AND (host LIKE ? OR path LIKE ? OR client LIKE ?)"
|
||||
like := "%" + q + "%"
|
||||
args = append(args, like, like, like)
|
||||
}
|
||||
if client != "" {
|
||||
where += " AND client = ?"
|
||||
args = append(args, client)
|
||||
}
|
||||
args = append(args, limit)
|
||||
rows, err := m.DB.QueryContext(ctx, `SELECT `+flowCols+` FROM http_flows WHERE `+where+` ORDER BY ts DESC LIMIT ?`, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []Flow{}
|
||||
for rows.Next() {
|
||||
f, err := scanFlow(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (m *Manager) Flow(ctx context.Context, id string) (Flow, error) {
|
||||
return scanFlow(m.DB.QueryRowContext(ctx, `SELECT `+flowCols+` FROM http_flows WHERE id=?`, id))
|
||||
}
|
||||
|
||||
// Body returns a stored request/response body. which is "req" or "resp".
|
||||
// Bodies exist only when body capture was enabled for that flow.
|
||||
func (m *Manager) Body(id, which string) ([]byte, error) {
|
||||
if which != "req" && which != "resp" || !flowIDRe.MatchString(id) {
|
||||
return nil, errors.New("invalid body request")
|
||||
}
|
||||
return os.ReadFile(filepath.Join(m.bodiesDir(), id+"."+which))
|
||||
}
|
||||
|
||||
var flowIDRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
||||
|
||||
// Clear deletes all intercepted HTTP data (flows + bodies). Captures are untouched.
|
||||
func (m *Manager) Clear() error {
|
||||
m.DB.Exec(`DELETE FROM http_flows`)
|
||||
os.RemoveAll(m.bodiesDir())
|
||||
os.MkdirAll(m.bodiesDir(), 0o750)
|
||||
os.Truncate(m.flowsPath(), 0)
|
||||
m.DB.PutJSON("interception_flow_offset", int64(0))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClientIP validates a client filter value.
|
||||
func ClientIP(s string) bool { _, err := netip.ParseAddr(s); return err == nil || s == "" }
|
||||
|
||||
// ExcludeRegex joins exclude hosts into one mitmdump --ignore-hosts pattern.
|
||||
func (c Config) ExcludeRegex() string {
|
||||
var parts []string
|
||||
for _, h := range c.ExcludeHosts {
|
||||
parts = append(parts, "(?:"+h+")")
|
||||
}
|
||||
return strings.Join(parts, "|")
|
||||
}
|
||||
Reference in New Issue
Block a user