Phase 4: authorised HTTPS interception (Mode B)

Toggleable transparent interception via mitmdump in a venv. Own RSA CA generated
in-app, downloadable (public cert only, key never leaves the box), with per-OS
install guide and fingerprint. nftables redirect accepts only dnat'd connections
(never an open proxy). HTTP Explorer with headers and opt-in body viewers (JSON/
text/hex/image, sandboxed). Honest status: pinned/untrusting = reported errors,
excluded hosts pass through, bodies off by default. Helper op + systemd unit +
installer venv build + boot restore.
This commit is contained in:
Andy Dixon
2026-10-10 08:05:42 +01:00
parent 4320428f41
commit 5dbf85f84e
25 changed files with 1460 additions and 13 deletions
+165
View File
@@ -0,0 +1,165 @@
// Package proxy manages optional, authorised HTTPS interception: a locally
// generated CA that the operator installs on test devices by hand, a mitmdump
// service, and ingestion of the decrypted HTTP transactions.
//
// Consent is the whole design. The CA private key never leaves the appliance,
// is never pushed to a device, and interception only happens for devices whose
// owner has explicitly installed and trusted the CA. Nothing is covert.
package proxy
import (
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"crypto/x509"
"crypto/x509/pkix"
"encoding/hex"
"encoding/pem"
"errors"
"fmt"
"math/big"
"os"
"path/filepath"
"strings"
"time"
)
// CAInfo describes the interception CA for display. It never includes the key.
type CAInfo struct {
Present bool `json:"present"`
Subject string `json:"subject"`
SHA256 string `json:"sha256"` // fingerprint, colon-separated
NotBefore string `json:"not_before"`
NotAfter string `json:"not_after"`
Serial string `json:"serial"`
DaysLeft int `json:"days_left"`
KeyType string `json:"key_type"`
}
// caPaths returns the key+cert path (mitmproxy format) and the public cert path.
func caPaths(dir string) (combined, certOnly string) {
return filepath.Join(dir, "mitmproxy-ca.pem"), filepath.Join(dir, "mitmproxy-ca-cert.pem")
}
// EnsureCA creates the interception CA if absent, in the mitmproxy confdir layout.
// RSA-2048 is used (not EC) for the broadest device trust-store compatibility.
func EnsureCA(dir, hostname string) error {
combined, certOnly := caPaths(dir)
if _, err := os.Stat(combined); err == nil {
return nil
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return err
}
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return err
}
serial, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
cn := "PiLens Interception CA"
if hostname != "" {
cn = "PiLens Interception CA (" + hostname + ")"
}
tpl := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: cn, Organization: []string{"PiLens appliance"}},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().AddDate(3, 0, 0),
IsCA: true,
BasicConstraintsValid: true,
MaxPathLen: 0,
MaxPathLenZero: true,
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
}
der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key)
if err != nil {
return err
}
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
// mitmproxy reads the key+cert from mitmproxy-ca.pem (0600) and derives the rest.
if err := writeFile(combined, append(append([]byte{}, keyPEM...), certPEM...), 0o600); err != nil {
return err
}
// Public cert for download / device install (never contains the key).
if err := writeFile(certOnly, certPEM, 0o644); err != nil {
return err
}
writeFile(filepath.Join(dir, "mitmproxy-ca-cert.cer"), certPEM, 0o644)
return nil
}
func writeFile(path string, data []byte, mode os.FileMode) error {
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, mode); err != nil {
return err
}
os.Chmod(tmp, mode)
return os.Rename(tmp, path)
}
// PublicCertPEM returns the CA's public certificate (safe to distribute). It never
// returns the private key.
func PublicCertPEM(dir string) ([]byte, error) {
_, certOnly := caPaths(dir)
b, err := os.ReadFile(certOnly)
if err != nil {
return nil, errors.New("interception CA not generated yet")
}
if strings.Contains(string(b), "PRIVATE KEY") { // defensive: must never happen
return nil, errors.New("refusing to serve a file containing a private key")
}
return b, nil
}
// Info reads the CA metadata for display.
func Info(dir string) CAInfo {
_, certOnly := caPaths(dir)
b, err := os.ReadFile(certOnly)
if err != nil {
return CAInfo{Present: false}
}
blk, _ := pem.Decode(b)
if blk == nil {
return CAInfo{Present: false}
}
c, err := x509.ParseCertificate(blk.Bytes)
if err != nil {
return CAInfo{Present: false}
}
sum := sha256.Sum256(c.Raw)
return CAInfo{
Present: true,
Subject: c.Subject.String(),
SHA256: colonHex(sum[:]),
NotBefore: c.NotBefore.UTC().Format("2006-01-02"),
NotAfter: c.NotAfter.UTC().Format("2006-01-02"),
Serial: c.SerialNumber.Text(16),
DaysLeft: int(time.Until(c.NotAfter).Hours() / 24),
KeyType: "RSA 2048",
}
}
// Rotate deletes the CA so a fresh one is generated. Devices trusting the old CA
// must install the new one; existing interception breaks until they do.
func Rotate(dir, hostname string) error {
combined, certOnly := caPaths(dir)
for _, p := range []string{combined, certOnly, filepath.Join(dir, "mitmproxy-ca-cert.cer"), filepath.Join(dir, "mitmproxy-ca.p12"), filepath.Join(dir, "mitmproxy-ca-cert.p12"), filepath.Join(dir, "mitmproxy-dhparam.pem")} {
os.Remove(p)
}
return EnsureCA(dir, hostname)
}
func colonHex(b []byte) string {
h := strings.ToUpper(hex.EncodeToString(b))
var parts []string
for i := 0; i < len(h); i += 2 {
parts = append(parts, h[i:i+2])
}
return strings.Join(parts, ":")
}
// Fingerprint of a cert file, for the install instructions ("verify this matches").
func Fingerprint(dir string) string {
return fmt.Sprintf("SHA-256 %s", Info(dir).SHA256)
}
+62
View File
@@ -0,0 +1,62 @@
package proxy
import (
"strings"
"testing"
)
func TestCAGeneration(t *testing.T) {
dir := t.TempDir()
if err := EnsureCA(dir, "testpi"); err != nil {
t.Fatal(err)
}
// Public cert must never contain a private key.
pub, err := PublicCertPEM(dir)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(pub), "PRIVATE KEY") {
t.Fatal("public cert leaked the private key")
}
if !strings.Contains(string(pub), "BEGIN CERTIFICATE") {
t.Fatal("not a certificate")
}
info := Info(dir)
if !info.Present || !strings.Contains(info.Subject, "PiLens") || len(info.SHA256) != 95 || info.DaysLeft < 1000 {
t.Fatalf("bad CA info: %+v", info)
}
// Idempotent: second call keeps the same CA.
before := info.SHA256
EnsureCA(dir, "testpi")
if Info(dir).SHA256 != before {
t.Fatal("EnsureCA regenerated an existing CA")
}
// Rotate produces a different CA.
if err := Rotate(dir, "testpi"); err != nil {
t.Fatal(err)
}
if Info(dir).SHA256 == before {
t.Fatal("rotate did not change the CA")
}
}
func TestConfigValidate(t *testing.T) {
bad := []Config{
{MaxBodyKB: -1},
{MaxBodyKB: 999999},
{ExcludeHosts: []string{"ok.com", "bad regex ("}},
{ExcludeHosts: []string{"has space"}},
}
for i, c := range bad {
if c.Validate() == nil {
t.Errorf("case %d accepted invalid config", i)
}
}
good := Config{MaxBodyKB: 256, ExcludeHosts: []string{`(?i)\.apple\.com$`, "example\\.com"}}
if err := good.Validate(); err != nil {
t.Fatal(err)
}
if r := good.ExcludeRegex(); !strings.Contains(r, "apple") || !strings.Contains(r, "|") {
t.Fatalf("exclude regex wrong: %s", r)
}
}
+286
View File
@@ -0,0 +1,286 @@
package proxy
import (
"bufio"
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"net/netip"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"pilens.dev/pilens/internal/store"
)
// Config is the interception toggle and its options. Stored in settings.
type Config struct {
Enabled bool `json:"enabled"`
CaptureBodies bool `json:"capture_bodies"` // default false: sensitive content off by default
MaxBodyKB int `json:"max_body_kb"`
ExcludeHosts []string `json:"exclude_hosts"` // host regexes NOT decrypted (passed through)
}
func DefaultConfig() Config { return Config{Enabled: false, CaptureBodies: false, MaxBodyKB: 256} }
const ListenPort = 8081
var hostRe = regexp.MustCompile(`^[A-Za-z0-9.*?+()\[\]{}|\\^$_:-]{1,200}$`)
func (c Config) Validate() error {
if c.MaxBodyKB < 0 || c.MaxBodyKB > 10240 {
return errors.New("max_body_kb must be 0-10240")
}
for _, h := range c.ExcludeHosts {
if !hostRe.MatchString(h) {
return errors.New("invalid exclude-host pattern: " + h)
}
if _, err := regexp.Compile(h); err != nil {
return errors.New("invalid exclude-host regex " + h + ": " + err.Error())
}
}
return nil
}
// Sink receives live events for the browser (adapts the capture Hub).
type Sink interface{ Publish(eventType string, data []byte) }
// Flow is one HTTP(S) transaction, as emitted by the mitmdump addon and stored.
type Flow struct {
ID string `json:"id"`
TS float64 `json:"ts"`
Client string `json:"client"`
Method string `json:"method"`
Scheme string `json:"scheme"`
Host string `json:"host"`
Port int `json:"port"`
Path string `json:"path"`
HTTPVersion string `json:"http_version"`
Status int `json:"status"`
Reason string `json:"reason"`
ReqCType string `json:"req_ctype"`
RespCType string `json:"resp_ctype"`
ReqSize int64 `json:"req_size"`
RespSize int64 `json:"resp_size"`
DurationMS int64 `json:"duration_ms"`
TLSVersion string `json:"tls_version"`
SNI string `json:"sni"`
ALPN string `json:"alpn"`
Intercepted bool `json:"intercepted"`
Error string `json:"error"`
WebSocket bool `json:"websocket"`
HasReqBody bool `json:"has_req_body"`
HasRespBody bool `json:"has_resp_body"`
ReqHeaders map[string]string `json:"req_headers"`
RespHeaders map[string]string `json:"resp_headers"`
}
// Manager owns interception state, the flow ingester and queries.
type Manager struct {
DB *store.Store
Dir string // /var/lib/pilens/proxy
Hostname string
Sink Sink
mu sync.Mutex
cfg Config
cancel context.CancelFunc
}
func New(db *store.Store, dir, hostname string, sink Sink) *Manager {
m := &Manager{DB: db, Dir: dir, Hostname: hostname, Sink: sink, cfg: DefaultConfig()}
db.GetJSON("interception", &m.cfg)
return m
}
func (m *Manager) Config() Config { m.mu.Lock(); defer m.mu.Unlock(); return m.cfg }
func (m *Manager) SetConfig(c Config) error {
if err := c.Validate(); err != nil {
return err
}
m.mu.Lock()
m.cfg = c
m.mu.Unlock()
return m.DB.PutJSON("interception", c)
}
func (m *Manager) flowsPath() string { return filepath.Join(m.Dir, "flows.ndjson") }
func (m *Manager) bodiesDir() string { return filepath.Join(m.Dir, "bodies") }
func (m *Manager) ConfDir() string { return filepath.Join(m.Dir, "ca") }
func (m *Manager) EnsureCA() error { return EnsureCA(m.ConfDir(), m.Hostname) }
func (m *Manager) CAInfo() CAInfo { return Info(m.ConfDir()) }
func (m *Manager) CACertPEM() ([]byte, error) { return PublicCertPEM(m.ConfDir()) }
func (m *Manager) RotateCA() error { return Rotate(m.ConfDir(), m.Hostname) }
// Start begins tailing the flow file. It runs for the service lifetime; the
// mitmdump service is started/stopped by the helper, this just ingests.
func (m *Manager) Start(ctx context.Context) {
os.MkdirAll(m.bodiesDir(), 0o750)
ctx, cancel := context.WithCancel(ctx)
m.cancel = cancel
go m.tail(ctx)
}
// tail follows flows.ndjson, inserting new lines. Survives file rotation/truncation.
func (m *Manager) tail(ctx context.Context) {
path := m.flowsPath()
var offset int64
m.DB.GetJSON("interception_flow_offset", &offset)
for ctx.Err() == nil {
f, err := os.Open(path)
if err != nil {
select {
case <-ctx.Done():
return
case <-time.After(2 * time.Second):
}
continue
}
if fi, _ := f.Stat(); fi != nil && fi.Size() < offset {
offset = 0 // file was rotated/truncated
}
f.Seek(offset, io.SeekStart)
r := bufio.NewReaderSize(f, 1<<20)
for ctx.Err() == nil {
line, rerr := r.ReadBytes('\n')
if len(line) > 0 && line[len(line)-1] == '\n' {
offset += int64(len(line))
m.ingest(line)
m.DB.PutJSON("interception_flow_offset", offset)
continue
}
_ = rerr // partial/EOF: keep offset, re-read after the wait
break
}
f.Close()
select {
case <-ctx.Done():
return
case <-time.After(500 * time.Millisecond):
}
}
}
func (m *Manager) ingest(line []byte) {
var fl Flow
if err := json.Unmarshal(line, &fl); err != nil || fl.ID == "" {
return
}
rh, _ := json.Marshal(fl.ReqHeaders)
sh, _ := json.Marshal(fl.RespHeaders)
_, err := m.DB.Exec(`INSERT OR REPLACE INTO http_flows
(id,ts,client,method,scheme,host,port,path,http_version,status,reason,req_ctype,resp_ctype,req_size,resp_size,duration_ms,tls_version,sni,alpn,intercepted,error,websocket,has_req_body,has_resp_body,req_headers,resp_headers)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
fl.ID, fl.TS, fl.Client, fl.Method, fl.Scheme, fl.Host, fl.Port, fl.Path, fl.HTTPVersion, fl.Status, fl.Reason,
fl.ReqCType, fl.RespCType, fl.ReqSize, fl.RespSize, fl.DurationMS, fl.TLSVersion, fl.SNI, fl.ALPN,
b2i(fl.Intercepted), fl.Error, b2i(fl.WebSocket), b2i(fl.HasReqBody), b2i(fl.HasRespBody), string(rh), string(sh))
if err != nil {
slog.Warn("http flow insert", "err", err)
return
}
if m.Sink != nil {
m.Sink.Publish("http", line)
}
}
func b2i(b bool) int {
if b {
return 1
}
return 0
}
// --- queries ---
const flowCols = `id,ts,client,method,scheme,host,port,path,http_version,status,reason,req_ctype,resp_ctype,req_size,resp_size,duration_ms,tls_version,sni,alpn,intercepted,error,websocket,has_req_body,has_resp_body,req_headers,resp_headers`
func scanFlow(sc interface{ Scan(...any) error }) (Flow, error) {
var f Flow
var rh, sh string
var intercept, ws, hrb, hsb int
err := sc.Scan(&f.ID, &f.TS, &f.Client, &f.Method, &f.Scheme, &f.Host, &f.Port, &f.Path, &f.HTTPVersion, &f.Status, &f.Reason,
&f.ReqCType, &f.RespCType, &f.ReqSize, &f.RespSize, &f.DurationMS, &f.TLSVersion, &f.SNI, &f.ALPN,
&intercept, &f.Error, &ws, &hrb, &hsb, &rh, &sh)
f.Intercepted, f.WebSocket, f.HasReqBody, f.HasRespBody = intercept == 1, ws == 1, hrb == 1, hsb == 1
json.Unmarshal([]byte(rh), &f.ReqHeaders)
json.Unmarshal([]byte(sh), &f.RespHeaders)
return f, err
}
// Flows returns recent transactions, newest first, optionally filtered by a
// substring over host/path/client and a client IP.
func (m *Manager) Flows(ctx context.Context, q, client string, limit int) ([]Flow, error) {
if limit <= 0 || limit > 5000 {
limit = 1000
}
where := "1=1"
args := []any{}
if q != "" {
where += " AND (host LIKE ? OR path LIKE ? OR client LIKE ?)"
like := "%" + q + "%"
args = append(args, like, like, like)
}
if client != "" {
where += " AND client = ?"
args = append(args, client)
}
args = append(args, limit)
rows, err := m.DB.QueryContext(ctx, `SELECT `+flowCols+` FROM http_flows WHERE `+where+` ORDER BY ts DESC LIMIT ?`, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := []Flow{}
for rows.Next() {
f, err := scanFlow(rows)
if err != nil {
return nil, err
}
out = append(out, f)
}
return out, rows.Err()
}
func (m *Manager) Flow(ctx context.Context, id string) (Flow, error) {
return scanFlow(m.DB.QueryRowContext(ctx, `SELECT `+flowCols+` FROM http_flows WHERE id=?`, id))
}
// Body returns a stored request/response body. which is "req" or "resp".
// Bodies exist only when body capture was enabled for that flow.
func (m *Manager) Body(id, which string) ([]byte, error) {
if which != "req" && which != "resp" || !flowIDRe.MatchString(id) {
return nil, errors.New("invalid body request")
}
return os.ReadFile(filepath.Join(m.bodiesDir(), id+"."+which))
}
var flowIDRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
// Clear deletes all intercepted HTTP data (flows + bodies). Captures are untouched.
func (m *Manager) Clear() error {
m.DB.Exec(`DELETE FROM http_flows`)
os.RemoveAll(m.bodiesDir())
os.MkdirAll(m.bodiesDir(), 0o750)
os.Truncate(m.flowsPath(), 0)
m.DB.PutJSON("interception_flow_offset", int64(0))
return nil
}
// ClientIP validates a client filter value.
func ClientIP(s string) bool { _, err := netip.ParseAddr(s); return err == nil || s == "" }
// ExcludeRegex joins exclude hosts into one mitmdump --ignore-hosts pattern.
func (c Config) ExcludeRegex() string {
var parts []string
for _, h := range c.ExcludeHosts {
parts = append(parts, "(?:"+h+")")
}
return strings.Join(parts, "|")
}