Core packages: config validation, network renderers, root helper, auth, SQLite store, capture engine
Capture engine streams dumpcap pcapng into verbatim, hashed segments with per-segment tshark summaries; display filters, PDML dissection with verified byte ranges, export and import. Tests run in a Debian trixie container.
This commit is contained in:
Andy Dixon
@@ -0,0 +1,824 @@
|
||||
// Package capture runs dumpcap, stores original packets as pcapng segments,
|
||||
// indexes per-packet summaries in SQLite and serves on-demand dissection.
|
||||
package capture
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"pilens.dev/pilens/internal/auth"
|
||||
"pilens.dev/pilens/internal/config"
|
||||
"pilens.dev/pilens/internal/pcapfile"
|
||||
"pilens.dev/pilens/internal/store"
|
||||
)
|
||||
|
||||
type Limits struct {
|
||||
MaxBytes int64 `json:"max_bytes"` // total bytes of capture files
|
||||
ReserveBytes int64 `json:"reserve_bytes"` // free space that must remain on the filesystem
|
||||
SegmentBytes int64 `json:"segment_bytes"` // rotate segment files at this size
|
||||
ImportBytes int64 `json:"import_bytes"` // largest accepted upload
|
||||
}
|
||||
|
||||
func DefaultLimits() Limits {
|
||||
return Limits{MaxBytes: 8 << 30, ReserveBytes: 1 << 30, SegmentBytes: 64 << 20, ImportBytes: 1 << 30}
|
||||
}
|
||||
|
||||
type Engine struct {
|
||||
DB *store.Store
|
||||
Dir string // root of capture storage
|
||||
Tools *Tools
|
||||
Hub *Hub
|
||||
Version string
|
||||
|
||||
limMu sync.Mutex
|
||||
limits Limits
|
||||
|
||||
mu sync.Mutex
|
||||
live *live
|
||||
|
||||
fcMu sync.Mutex
|
||||
fcache map[string]*filterEntry
|
||||
}
|
||||
|
||||
func (e *Engine) Limits() Limits { e.limMu.Lock(); defer e.limMu.Unlock(); return e.limits }
|
||||
func (e *Engine) SetLimits(l Limits) { e.limMu.Lock(); e.limits = l; e.limMu.Unlock() }
|
||||
|
||||
// Session is the API view of a capture session.
|
||||
type Session struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Kind string `json:"kind"`
|
||||
State string `json:"state"`
|
||||
Interfaces []string `json:"interfaces"`
|
||||
CaptureFilter string `json:"capture_filter"`
|
||||
StartedAt int64 `json:"started_at"`
|
||||
EndedAt int64 `json:"ended_at"`
|
||||
Packets int64 `json:"packets"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Drops string `json:"drops"`
|
||||
Error string `json:"error"`
|
||||
Preserved bool `json:"preserved"`
|
||||
Tags string `json:"tags"`
|
||||
Notes string `json:"notes"`
|
||||
RingSegments int `json:"ring_segments"`
|
||||
Meta json.RawMessage `json:"meta"`
|
||||
Paused bool `json:"paused"`
|
||||
Lag int64 `json:"ingest_lag"` // packets written but not yet summarised
|
||||
}
|
||||
|
||||
type Segment struct {
|
||||
Seq int `json:"seq"`
|
||||
File string `json:"file"`
|
||||
Format string `json:"format"`
|
||||
Base int64 `json:"base"`
|
||||
Count int64 `json:"count"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Complete bool `json:"complete"`
|
||||
Deleted bool `json:"deleted"`
|
||||
}
|
||||
|
||||
func newID() string {
|
||||
return time.Now().UTC().Format("20060102T150405Z") + "-" + auth.RandomToken(3)
|
||||
}
|
||||
|
||||
const sessCols = `id,name,description,kind,state,interfaces,capture_filter,started_at,ended_at,packets,bytes,drops,error,preserved,tags,notes,ring_segments,meta`
|
||||
|
||||
func scanSession(sc interface{ Scan(...any) error }) (Session, error) {
|
||||
var s Session
|
||||
var ifs, meta string
|
||||
err := sc.Scan(&s.ID, &s.Name, &s.Description, &s.Kind, &s.State, &ifs, &s.CaptureFilter, &s.StartedAt, &s.EndedAt, &s.Packets, &s.Bytes, &s.Drops, &s.Error, &s.Preserved, &s.Tags, &s.Notes, &s.RingSegments, &meta)
|
||||
json.Unmarshal([]byte(ifs), &s.Interfaces)
|
||||
s.Meta = json.RawMessage(meta)
|
||||
return s, err
|
||||
}
|
||||
|
||||
func (e *Engine) withLive(s Session) Session {
|
||||
e.mu.Lock()
|
||||
l := e.live
|
||||
e.mu.Unlock()
|
||||
if l != nil && l.id == s.ID {
|
||||
s.Packets = l.packets.Load()
|
||||
s.Bytes = l.bytes.Load()
|
||||
s.Paused = l.paused.Load()
|
||||
s.Lag = s.Packets - l.summarised.Load()
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (e *Engine) Sessions() ([]Session, error) {
|
||||
rows, err := e.DB.Query(`SELECT ` + sessCols + ` FROM capture_sessions ORDER BY started_at DESC`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []Session{}
|
||||
for rows.Next() {
|
||||
s, err := scanSession(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, e.withLive(s))
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (e *Engine) Get(id string) (Session, error) {
|
||||
s, err := scanSession(e.DB.QueryRow(`SELECT `+sessCols+` FROM capture_sessions WHERE id=?`, id))
|
||||
if err == sql.ErrNoRows {
|
||||
return s, ErrNotFound
|
||||
}
|
||||
return e.withLive(s), err
|
||||
}
|
||||
|
||||
var ErrNotFound = errors.New("not found")
|
||||
|
||||
func (e *Engine) Segments(id string) ([]Segment, error) {
|
||||
rows, err := e.DB.Query(`SELECT seq,file,format,base,count,bytes,sha256,complete,deleted FROM segments WHERE session_id=? ORDER BY seq`, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Segment
|
||||
for rows.Next() {
|
||||
var s Segment
|
||||
if err := rows.Scan(&s.Seq, &s.File, &s.Format, &s.Base, &s.Count, &s.Bytes, &s.SHA256, &s.Complete, &s.Deleted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (e *Engine) sessionDir(id string) string { return filepath.Join(e.Dir, id) }
|
||||
|
||||
// Update edits analyst metadata.
|
||||
func (e *Engine) Update(id, name, desc, tags, notes string, preserved bool) error {
|
||||
r, err := e.DB.Exec(`UPDATE capture_sessions SET name=?,description=?,tags=?,notes=?,preserved=? WHERE id=?`, name, desc, tags, notes, preserved, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, _ := r.RowsAffected(); n == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) Delete(id string) error {
|
||||
e.mu.Lock()
|
||||
running := e.live != nil && e.live.id == id
|
||||
e.mu.Unlock()
|
||||
if running {
|
||||
return errors.New("stop the capture before deleting it")
|
||||
}
|
||||
if _, err := e.Get(id); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(e.sessionDir(id)); err != nil {
|
||||
return err
|
||||
}
|
||||
e.DB.Exec(`DELETE FROM packets WHERE session_id=?`, id)
|
||||
_, err := e.DB.Exec(`DELETE FROM capture_sessions WHERE id=?`, id)
|
||||
e.dropFilterCache(id)
|
||||
return err
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- live capture
|
||||
|
||||
type StartOptions struct {
|
||||
Name string `json:"name"`
|
||||
Interfaces []string `json:"interfaces"`
|
||||
CaptureFilter string `json:"capture_filter"`
|
||||
RingSegments int `json:"ring_segments"`
|
||||
}
|
||||
|
||||
type live struct {
|
||||
id string
|
||||
cmd *exec.Cmd
|
||||
done chan struct{}
|
||||
stopping atomic.Bool
|
||||
paused atomic.Bool
|
||||
discarded atomic.Int64 // packets discarded while paused
|
||||
packets atomic.Int64
|
||||
bytes atomic.Int64
|
||||
summarised atomic.Int64
|
||||
stderr limitedBuffer
|
||||
|
||||
mu sync.Mutex // guards seg
|
||||
seg *segWriter
|
||||
wg sync.WaitGroup // summarisers
|
||||
}
|
||||
|
||||
type segWriter struct {
|
||||
seq int
|
||||
base int64
|
||||
path string
|
||||
f *os.File
|
||||
bw *bufio.Writer
|
||||
h hash.Hash
|
||||
size int64
|
||||
count int64
|
||||
offs []int64
|
||||
|
||||
feed chan []byte
|
||||
feedLost bool
|
||||
}
|
||||
|
||||
type limitedBuffer struct {
|
||||
mu sync.Mutex
|
||||
b bytes.Buffer
|
||||
}
|
||||
|
||||
func (l *limitedBuffer) Write(p []byte) (int, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.b.Len() < 64<<10 {
|
||||
l.b.Write(p)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (l *limitedBuffer) String() string { l.mu.Lock(); defer l.mu.Unlock(); return l.b.String() }
|
||||
|
||||
func clockSynced() string {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, "timedatectl", "show", "-p", "NTPSynchronized", "--value").Output()
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func (e *Engine) meta(extra map[string]any) string {
|
||||
host, _ := os.Hostname()
|
||||
m := map[string]any{
|
||||
"pilens_version": e.Version,
|
||||
"tshark_version": e.Tools.Version(),
|
||||
"clock_synchronised": clockSynced(),
|
||||
"hostname": host,
|
||||
"recorded_utc": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
}
|
||||
for k, v := range extra {
|
||||
m[k] = v
|
||||
}
|
||||
b, _ := json.Marshal(m)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func (e *Engine) Live() (Session, bool) {
|
||||
e.mu.Lock()
|
||||
l := e.live
|
||||
e.mu.Unlock()
|
||||
if l == nil {
|
||||
return Session{}, false
|
||||
}
|
||||
s, err := e.Get(l.id)
|
||||
return s, err == nil
|
||||
}
|
||||
|
||||
func (e *Engine) Start(ctx context.Context, o StartOptions) (Session, error) {
|
||||
if len(o.Interfaces) == 0 || len(o.Interfaces) > 8 {
|
||||
return Session{}, errors.New("choose 1-8 interfaces")
|
||||
}
|
||||
for _, i := range o.Interfaces {
|
||||
if i == "any" {
|
||||
continue
|
||||
}
|
||||
if !config.ValidInterface(i) {
|
||||
return Session{}, fmt.Errorf("invalid interface %q", i)
|
||||
}
|
||||
if _, err := net.InterfaceByName(i); err != nil {
|
||||
return Session{}, fmt.Errorf("interface %q not found", i)
|
||||
}
|
||||
}
|
||||
if o.CaptureFilter != "" {
|
||||
if err := e.Tools.ValidateCaptureFilter(ctx, o.Interfaces[0], o.CaptureFilter); err != nil {
|
||||
return Session{}, fmt.Errorf("capture filter: %v", err)
|
||||
}
|
||||
}
|
||||
if o.RingSegments < 0 || o.RingSegments > 100000 {
|
||||
return Session{}, errors.New("invalid ring size")
|
||||
}
|
||||
if strings.TrimSpace(o.Name) == "" {
|
||||
o.Name = "Capture " + time.Now().Format("2006-01-02 15:04")
|
||||
}
|
||||
if len(o.Name) > 200 {
|
||||
return Session{}, errors.New("name too long")
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if e.live != nil {
|
||||
return Session{}, errors.New("a capture is already running")
|
||||
}
|
||||
if err := e.enforceStorage(""); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
id := newID()
|
||||
dir := e.sessionDir(id)
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
ifs, _ := json.Marshal(o.Interfaces)
|
||||
lim := e.Limits()
|
||||
meta := e.meta(map[string]any{"segment_bytes": lim.SegmentBytes, "snaplen": "dumpcap default (full)"})
|
||||
_, err := e.DB.Exec(`INSERT INTO capture_sessions(id,name,kind,state,interfaces,capture_filter,started_at,dir,meta,ring_segments) VALUES(?,?,?,?,?,?,?,?,?,?)`,
|
||||
id, o.Name, "live", "running", string(ifs), o.CaptureFilter, time.Now().UnixMilli(), dir, meta, o.RingSegments)
|
||||
if err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
|
||||
args := []string{"-q", "-n", "-w", "-"}
|
||||
for _, i := range o.Interfaces {
|
||||
args = append(args, "-i", i)
|
||||
if o.CaptureFilter != "" {
|
||||
args = append(args, "-f", o.CaptureFilter) // applies to the preceding -i
|
||||
}
|
||||
}
|
||||
cmd := exec.Command(e.Tools.Dumpcap, args...)
|
||||
l := &live{id: id, cmd: cmd, done: make(chan struct{})}
|
||||
cmd.Stderr = &l.stderr
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
e.finishSession(id, "failed", "dumpcap failed to start: "+err.Error(), "")
|
||||
return Session{}, err
|
||||
}
|
||||
e.live = l
|
||||
go e.runLive(l, stdout)
|
||||
slog.Info("capture started", "session", id, "interfaces", o.Interfaces)
|
||||
s, err := scanSession(e.DB.QueryRow(`SELECT `+sessCols+` FROM capture_sessions WHERE id=?`, id))
|
||||
return s, err // e.mu is held: must not call e.Get (withLive locks it)
|
||||
}
|
||||
|
||||
func (e *Engine) SetPaused(p bool) error {
|
||||
e.mu.Lock()
|
||||
l := e.live
|
||||
e.mu.Unlock()
|
||||
if l == nil {
|
||||
return errors.New("no capture running")
|
||||
}
|
||||
if l.paused.Swap(p) != p {
|
||||
ev := "resumed"
|
||||
if p {
|
||||
ev = "paused"
|
||||
}
|
||||
e.DB.Exec(`UPDATE capture_sessions SET meta=json_insert(meta,'$.pause_log[#]',json_object('event',?,'utc',?)) WHERE id=?`, ev, time.Now().UTC().Format(time.RFC3339Nano), l.id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) Stop() error {
|
||||
e.mu.Lock()
|
||||
l := e.live
|
||||
e.mu.Unlock()
|
||||
if l == nil {
|
||||
return errors.New("no capture running")
|
||||
}
|
||||
l.stopping.Store(true)
|
||||
l.cmd.Process.Signal(syscall.SIGINT) // dumpcap flushes and reports statistics
|
||||
select {
|
||||
case <-l.done:
|
||||
case <-time.After(30 * time.Second):
|
||||
l.cmd.Process.Kill()
|
||||
<-l.done
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) runLive(l *live, stdout io.Reader) {
|
||||
defer close(l.done)
|
||||
rowsC := make(chan []Row, 64)
|
||||
insDone := make(chan struct{})
|
||||
go func() { e.inserter(l, rowsC); close(insDone) }()
|
||||
|
||||
// Periodic flush so on-demand reads of the active segment see recent packets,
|
||||
// and status events for the UI.
|
||||
stopTick := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stopTick:
|
||||
return
|
||||
case <-t.C:
|
||||
l.mu.Lock()
|
||||
if l.seg != nil {
|
||||
l.seg.bw.Flush()
|
||||
}
|
||||
l.mu.Unlock()
|
||||
e.publishStatus(l)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var failure string
|
||||
br := pcapfile.NewBlockReader(bufio.NewReaderSize(stdout, 1<<20))
|
||||
var hdrs [][]byte
|
||||
for {
|
||||
b, err := br.Next()
|
||||
if err != nil {
|
||||
if err != io.EOF && !l.stopping.Load() {
|
||||
failure = "capture stream error: " + err.Error()
|
||||
}
|
||||
break
|
||||
}
|
||||
switch b.Type {
|
||||
case pcapfile.BlockSHB:
|
||||
hdrs = [][]byte{b.Raw}
|
||||
case pcapfile.BlockIDB:
|
||||
hdrs = append(hdrs, b.Raw)
|
||||
}
|
||||
if b.IsPacket() && l.paused.Load() {
|
||||
l.discarded.Add(1)
|
||||
continue
|
||||
}
|
||||
if err := e.writeBlock(l, b, hdrs, rowsC); err != nil {
|
||||
failure = err.Error()
|
||||
l.stopping.Store(true)
|
||||
l.cmd.Process.Signal(syscall.SIGINT)
|
||||
io.Copy(io.Discard, stdout)
|
||||
break
|
||||
}
|
||||
}
|
||||
waitErr := l.cmd.Wait()
|
||||
l.mu.Lock()
|
||||
e.closeSegment(l, rowsC)
|
||||
l.mu.Unlock()
|
||||
l.wg.Wait()
|
||||
close(rowsC)
|
||||
<-insDone
|
||||
close(stopTick)
|
||||
|
||||
state := "stopped"
|
||||
if failure == "" && !l.stopping.Load() {
|
||||
failure = "dumpcap exited unexpectedly"
|
||||
if waitErr != nil {
|
||||
failure += ": " + waitErr.Error()
|
||||
}
|
||||
failure += ": " + strings.TrimSpace(l.stderr.String())
|
||||
}
|
||||
if failure != "" {
|
||||
state = "failed"
|
||||
}
|
||||
e.DB.Exec(`UPDATE capture_sessions SET packets=?,bytes=?,meta=json_set(meta,'$.discarded_while_paused',?) WHERE id=?`, l.packets.Load(), l.bytes.Load(), l.discarded.Load(), l.id)
|
||||
e.finishSession(l.id, state, failure, dropLines(l.stderr.String()))
|
||||
e.mu.Lock()
|
||||
e.live = nil
|
||||
e.mu.Unlock()
|
||||
e.publishStatus(l)
|
||||
slog.Info("capture finished", "session", l.id, "state", state, "packets", l.packets.Load(), "error", failure)
|
||||
}
|
||||
|
||||
// dropLines keeps dumpcap's per-interface "Packets received/dropped" report verbatim.
|
||||
func dropLines(s string) string {
|
||||
var out []string
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if strings.Contains(l, "received/dropped") {
|
||||
out = append(out, strings.TrimSpace(l))
|
||||
}
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
func (e *Engine) finishSession(id, state, errMsg, drops string) {
|
||||
e.DB.Exec(`UPDATE capture_sessions SET state=?,error=?,drops=?,ended_at=? WHERE id=?`, state, errMsg, drops, time.Now().UnixMilli(), id)
|
||||
e.dropFilterCache(id)
|
||||
}
|
||||
|
||||
// writeBlock is called only from the splitter goroutine.
|
||||
func (e *Engine) writeBlock(l *live, b pcapfile.Block, hdrs [][]byte, rowsC chan []Row) error {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
lim := e.Limits()
|
||||
if b.IsPacket() && l.seg != nil && l.seg.size >= lim.SegmentBytes {
|
||||
e.closeSegment(l, rowsC)
|
||||
if err := e.enforceStorage(l.id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if l.seg == nil {
|
||||
if !b.IsPacket() {
|
||||
return nil // headers are written when the first packet opens a segment
|
||||
}
|
||||
if err := e.openSegment(l, hdrs, rowsC); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s := l.seg
|
||||
if b.IsPacket() {
|
||||
s.offs = append(s.offs, s.size)
|
||||
s.count++
|
||||
l.packets.Add(1)
|
||||
if d, err := pcapfile.PacketData(b); err == nil {
|
||||
l.bytes.Add(int64(len(d)))
|
||||
}
|
||||
}
|
||||
if _, err := s.bw.Write(b.Raw); err != nil {
|
||||
return fmt.Errorf("writing capture file: %w", err)
|
||||
}
|
||||
s.size += int64(len(b.Raw))
|
||||
s.sendFeed(b.Raw)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *segWriter) sendFeed(raw []byte) {
|
||||
if s.feedLost {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.feed <- raw:
|
||||
default:
|
||||
// tshark cannot keep up. Never block capture: stop live summaries for this
|
||||
// segment; they are regenerated from the file when the segment closes.
|
||||
s.feedLost = true
|
||||
close(s.feed)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) openSegment(l *live, hdrs [][]byte, rowsC chan []Row) error {
|
||||
var seq int
|
||||
var base int64
|
||||
e.DB.QueryRow(`SELECT COALESCE(MAX(seq),0)+1 FROM segments WHERE session_id=?`, l.id).Scan(&seq)
|
||||
base = l.packets.Load() + 1
|
||||
name := fmt.Sprintf("seg-%06d.pcapng", seq)
|
||||
path := filepath.Join(e.sessionDir(l.id), name)
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h := sha256.New()
|
||||
s := &segWriter{seq: seq, base: base, path: path, f: f, h: h, bw: bufio.NewWriterSize(io.MultiWriter(f, h), 1<<20), feed: make(chan []byte, 2048)}
|
||||
if _, err := e.DB.Exec(`INSERT INTO segments(session_id,seq,file,format,base) VALUES(?,?,?,?,?)`, l.id, seq, name, "pcapng", base); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
for _, hb := range hdrs {
|
||||
s.bw.Write(hb)
|
||||
s.size += int64(len(hb))
|
||||
s.sendFeed(hb)
|
||||
}
|
||||
l.seg = s
|
||||
l.wg.Add(1)
|
||||
go e.summarise(l, s, rowsC)
|
||||
return nil
|
||||
}
|
||||
|
||||
// closeSegment finalises the current segment. Caller holds l.mu.
|
||||
func (e *Engine) closeSegment(l *live, rowsC chan []Row) {
|
||||
s := l.seg
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
l.seg = nil
|
||||
s.bw.Flush()
|
||||
s.f.Sync()
|
||||
s.f.Close()
|
||||
if !s.feedLost {
|
||||
close(s.feed)
|
||||
}
|
||||
sum := hex.EncodeToString(s.h.Sum(nil))
|
||||
e.DB.Exec(`UPDATE segments SET count=?,bytes=?,sha256=?,complete=1 WHERE session_id=? AND seq=?`, s.count, s.size, sum, l.id, s.seq)
|
||||
e.DB.Exec(`UPDATE capture_sessions SET packets=?,bytes=? WHERE id=?`, l.packets.Load(), l.bytes.Load(), l.id)
|
||||
e.applyRing(l.id)
|
||||
}
|
||||
|
||||
// summarise runs one tshark per segment fed from the splitter, then backfills
|
||||
// from the completed file if the live feed was cut short.
|
||||
func (e *Engine) summarise(l *live, s *segWriter, rowsC chan []Row) {
|
||||
defer l.wg.Done()
|
||||
offAt := func(local int64) int64 {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if local < 1 || local > int64(len(s.offs)) {
|
||||
return -1
|
||||
}
|
||||
return s.offs[local-1]
|
||||
}
|
||||
var got int64
|
||||
emit := func(batch *[]Row, r Row) {
|
||||
r.Off = offAt(r.Num)
|
||||
if r.Off < 0 {
|
||||
return
|
||||
}
|
||||
r.Num = s.base + r.Num - 1
|
||||
r.Seg = s.seq
|
||||
got++
|
||||
*batch = append(*batch, r)
|
||||
if len(*batch) >= 500 {
|
||||
rowsC <- *batch
|
||||
*batch = nil
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cmd := e.Tools.Command(ctx, 0, e.Tools.SummaryArgs("-")...)
|
||||
stdin, _ := cmd.StdinPipe()
|
||||
stdout, _ := cmd.StdoutPipe()
|
||||
if err := cmd.Start(); err != nil {
|
||||
slog.Error("tshark start", "err", err)
|
||||
for range s.feed {
|
||||
}
|
||||
} else {
|
||||
go func() {
|
||||
for b := range s.feed {
|
||||
if _, err := stdin.Write(b); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
stdin.Close()
|
||||
for range s.feed { // drain if tshark died
|
||||
}
|
||||
}()
|
||||
var batch []Row
|
||||
lines := make(chan Row, 1024)
|
||||
go func() { ReadSummaries(stdout, func(r Row) { lines <- r }); close(lines) }()
|
||||
tick := time.NewTicker(300 * time.Millisecond)
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
case r, ok := <-lines:
|
||||
if !ok {
|
||||
break loop
|
||||
}
|
||||
emit(&batch, r)
|
||||
case <-tick.C:
|
||||
if len(batch) > 0 {
|
||||
rowsC <- batch
|
||||
batch = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
tick.Stop()
|
||||
if len(batch) > 0 {
|
||||
rowsC <- batch
|
||||
}
|
||||
cmd.Wait()
|
||||
}
|
||||
|
||||
// Backfill from the finished file if live summaries are incomplete.
|
||||
l.mu.Lock()
|
||||
closed := l.seg != s
|
||||
count := s.count
|
||||
l.mu.Unlock()
|
||||
if closed && got < count {
|
||||
slog.Info("backfilling summaries", "session", l.id, "segment", s.seq, "have", got, "want", count)
|
||||
cmd := e.Tools.Command(ctx, 3600, e.Tools.SummaryArgs(s.path)...)
|
||||
out, _ := cmd.StdoutPipe()
|
||||
if cmd.Start() == nil {
|
||||
var batch []Row
|
||||
ReadSummaries(out, func(r Row) { emit(&batch, r) })
|
||||
if len(batch) > 0 {
|
||||
rowsC <- batch
|
||||
}
|
||||
cmd.Wait()
|
||||
}
|
||||
e.Hub.Publish(Event{Type: "resync", Data: []byte(`{"session":"` + l.id + `"}`)})
|
||||
}
|
||||
}
|
||||
|
||||
func insertRows(tx *sql.Tx, session string, rows []Row) error {
|
||||
st, err := tx.Prepare(`INSERT OR IGNORE INTO packets(session_id,num,seg,off,ts,src,dst,proto,len,info,sport,dport,stream,tcpflags,ttl,vlan,color) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer st.Close()
|
||||
for _, r := range rows {
|
||||
if _, err := st.Exec(session, r.Num, r.Seg, r.Off, r.TS, r.Src, r.Dst, r.Proto, r.Len, r.Info, r.SPort, r.DPort, r.Stream, r.TCPFlags, r.TTL, r.VLAN, r.Color); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) inserter(l *live, rowsC chan []Row) {
|
||||
for batch := range rowsC {
|
||||
tx, err := e.DB.Begin()
|
||||
if err != nil {
|
||||
slog.Error("index insert", "err", err)
|
||||
continue
|
||||
}
|
||||
if err := insertRows(tx, l.id, batch); err != nil {
|
||||
tx.Rollback()
|
||||
slog.Error("index insert", "err", err)
|
||||
continue
|
||||
}
|
||||
tx.Commit()
|
||||
l.summarised.Add(int64(len(batch)))
|
||||
data, _ := json.Marshal(map[string]any{"session": l.id, "rows": batch})
|
||||
e.Hub.Publish(Event{Type: "packets", Data: data})
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) publishStatus(l *live) {
|
||||
s, err := e.Get(l.id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
data, _ := json.Marshal(s)
|
||||
e.Hub.Publish(Event{Type: "capture", Data: data})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- storage
|
||||
|
||||
func (e *Engine) usedBytes() int64 {
|
||||
var n int64
|
||||
e.DB.QueryRow(`SELECT COALESCE(SUM(bytes),0) FROM segments WHERE deleted=0`).Scan(&n)
|
||||
return n
|
||||
}
|
||||
|
||||
func freeBytes(dir string) int64 {
|
||||
var st syscall.Statfs_t
|
||||
if syscall.Statfs(dir, &st) != nil {
|
||||
return 0
|
||||
}
|
||||
return int64(st.Bavail) * int64(st.Bsize)
|
||||
}
|
||||
|
||||
type StorageInfo struct {
|
||||
Used int64 `json:"used"`
|
||||
Free int64 `json:"free"`
|
||||
Limits Limits `json:"limits"`
|
||||
Dir string `json:"dir"`
|
||||
LowWarning bool `json:"low_warning"`
|
||||
}
|
||||
|
||||
func (e *Engine) Storage() StorageInfo {
|
||||
lim := e.Limits()
|
||||
i := StorageInfo{Used: e.usedBytes(), Free: freeBytes(e.Dir), Limits: lim, Dir: e.Dir}
|
||||
i.LowWarning = i.Free < 2*lim.ReserveBytes || i.Used > lim.MaxBytes*9/10
|
||||
return i
|
||||
}
|
||||
|
||||
// enforceStorage deletes the oldest unpreserved, finished sessions until within
|
||||
// limits. If that is not possible it returns an error and nothing is overwritten.
|
||||
func (e *Engine) enforceStorage(activeID string) error {
|
||||
lim := e.Limits()
|
||||
for {
|
||||
used, free := e.usedBytes(), freeBytes(e.Dir)
|
||||
if used+lim.SegmentBytes <= lim.MaxBytes && free-lim.SegmentBytes >= lim.ReserveBytes {
|
||||
return nil
|
||||
}
|
||||
var victim string
|
||||
e.DB.QueryRow(`SELECT id FROM capture_sessions WHERE preserved=0 AND id!=? AND state NOT IN ('running','importing') ORDER BY started_at LIMIT 1`, activeID).Scan(&victim)
|
||||
if victim == "" {
|
||||
return fmt.Errorf("storage limit reached (used %d of %d bytes, %d free, reserve %d) and no unpreserved sessions remain to delete", used, lim.MaxBytes, free, lim.ReserveBytes)
|
||||
}
|
||||
slog.Warn("storage limit: deleting oldest unpreserved session", "session", victim)
|
||||
e.DB.Audit("system", "capture.retention_delete", victim, "")
|
||||
if err := os.RemoveAll(e.sessionDir(victim)); err != nil {
|
||||
return err
|
||||
}
|
||||
e.DB.Exec(`DELETE FROM packets WHERE session_id=?`, victim)
|
||||
e.DB.Exec(`DELETE FROM capture_sessions WHERE id=?`, victim)
|
||||
e.dropFilterCache(victim)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) applyRing(id string) {
|
||||
var ring int
|
||||
var preserved bool
|
||||
e.DB.QueryRow(`SELECT ring_segments,preserved FROM capture_sessions WHERE id=?`, id).Scan(&ring, &preserved)
|
||||
if ring <= 0 || preserved {
|
||||
return
|
||||
}
|
||||
segs, _ := e.Segments(id)
|
||||
var live []Segment
|
||||
for _, s := range segs {
|
||||
if s.Complete && !s.Deleted {
|
||||
live = append(live, s)
|
||||
}
|
||||
}
|
||||
for len(live) > ring {
|
||||
s := live[0]
|
||||
live = live[1:]
|
||||
os.Remove(filepath.Join(e.sessionDir(id), s.File))
|
||||
e.DB.Exec(`UPDATE segments SET deleted=1 WHERE session_id=? AND seq=?`, id, s.Seq)
|
||||
e.DB.Exec(`DELETE FROM packets WHERE session_id=? AND num>=? AND num<?`, id, s.Base, s.Base+s.Count)
|
||||
}
|
||||
e.dropFilterCache(id)
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pilens.dev/pilens/internal/pcapfile"
|
||||
"pilens.dev/pilens/internal/store"
|
||||
)
|
||||
|
||||
const fixture = "../../tests/fixtures/sample-loopback-http-dns.pcapng"
|
||||
|
||||
func newEngine(t *testing.T) *Engine {
|
||||
ts, err := exec.LookPath("tshark")
|
||||
if err != nil {
|
||||
t.Skip("tshark not installed (run `make test-docker`)")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
db, err := store.Open(filepath.Join(dir, "t.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tools := &Tools{TShark: ts, Dumpcap: "dumpcap"}
|
||||
if err := tools.Init(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := &Engine{DB: db, Dir: filepath.Join(dir, "cap"), Tools: tools, Hub: NewHub(), Version: "test"}
|
||||
e.SetLimits(Limits{MaxBytes: 1 << 30, ReserveBytes: 0, SegmentBytes: 1 << 20, ImportBytes: 10 << 20})
|
||||
os.MkdirAll(e.Dir, 0o750)
|
||||
return e
|
||||
}
|
||||
|
||||
func TestImportQueryExport(t *testing.T) {
|
||||
e := newEngine(t)
|
||||
ctx := context.Background()
|
||||
f, err := os.Open(fixture)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
s, err := e.Import(ctx, "fixture", f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
s, _ = e.Get(s.ID)
|
||||
if s.State != "importing" {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if s.State != "ready" || s.Error != "" || s.Packets < 20 {
|
||||
t.Fatalf("import: %+v", s)
|
||||
}
|
||||
p, err := e.Packets(ctx, s.ID, "", 0, 1000)
|
||||
if err != nil || int64(len(p.Rows)) != s.Packets || p.Rows[0].Num != 1 {
|
||||
t.Fatalf("page: %v %+v", err, p)
|
||||
}
|
||||
fp, err := e.Packets(ctx, s.ID, "http.request", 0, 100)
|
||||
if err != nil || fp.Total != 2 || len(fp.Rows) != 2 || !strings.HasPrefix(fp.Rows[0].Info, "GET") {
|
||||
t.Fatalf("filter: %v %+v", err, fp)
|
||||
}
|
||||
if _, err := e.Packets(ctx, s.ID, "http.request ==", 0, 10); err == nil {
|
||||
t.Fatal("bad filter accepted")
|
||||
}
|
||||
if err := e.Tools.ValidateDisplayFilter(ctx, "tcp.port == 443 && dns"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
d, err := e.Packet(ctx, s.ID, fp.Rows[0].Num)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sawIP, sawHTTP bool
|
||||
var walk func([]Node)
|
||||
walk = func(ns []Node) {
|
||||
for _, n := range ns {
|
||||
if n.Field == "ip.src" && n.Verified && n.Size == 4 {
|
||||
sawIP = true
|
||||
}
|
||||
if n.Field == "http" {
|
||||
sawHTTP = true
|
||||
}
|
||||
walk(n.Children)
|
||||
}
|
||||
}
|
||||
walk(d.Tree)
|
||||
if !sawIP || !sawHTTP || len(d.Bytes) == 0 {
|
||||
t.Fatalf("detail tree incomplete: ip=%v http=%v", sawIP, sawHTTP)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
n, err := e.Export(ctx, &buf, s.ID, "udp")
|
||||
if err != nil || n != 4 {
|
||||
t.Fatalf("export: %d %v", n, err)
|
||||
}
|
||||
out := filepath.Join(t.TempDir(), "x.pcapng")
|
||||
os.WriteFile(out, buf.Bytes(), 0o600)
|
||||
count := 0
|
||||
if err := pcapfile.Scan(out, "pcapng", func(int, int64, []byte) error { count++; return nil }); err != nil || count != 4 {
|
||||
t.Fatalf("re-read export: %d %v", count, err)
|
||||
}
|
||||
if b, err := exec.Command(e.Tools.TShark, "-r", out).CombinedOutput(); err != nil {
|
||||
t.Fatalf("tshark rejects export: %s", b)
|
||||
}
|
||||
v, _ := e.Verify(s.ID)
|
||||
if len(v) != 1 || v[0].Status != "ok" {
|
||||
t.Fatalf("verify: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRejectsGarbage(t *testing.T) {
|
||||
e := newEngine(t)
|
||||
if _, err := e.Import(context.Background(), "x", strings.NewReader("PK\x03\x04 not a capture")); err == nil {
|
||||
t.Fatal("garbage accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveCaptureRotation(t *testing.T) {
|
||||
e := newEngine(t)
|
||||
if os.Getenv("PILENS_LIVE_TEST") == "" {
|
||||
t.Skip("set PILENS_LIVE_TEST=1 where dumpcap may capture on lo")
|
||||
}
|
||||
ctx := context.Background()
|
||||
sub := e.Hub.Subscribe()
|
||||
defer e.Hub.Unsubscribe(sub)
|
||||
s, err := e.Start(ctx, StartOptions{Name: "live", Interfaces: []string{"lo"}, CaptureFilter: "udp port 9999"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := e.Start(ctx, StartOptions{Interfaces: []string{"lo"}}); err == nil {
|
||||
t.Fatal("second concurrent capture allowed")
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
c, err := net.Dial("udp", "127.0.0.1:9999")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := bytes.Repeat([]byte("x"), 1000)
|
||||
for i := 0; i < 3000; i++ {
|
||||
c.Write(payload)
|
||||
if i%200 == 0 {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
c.Close()
|
||||
time.Sleep(2 * time.Second)
|
||||
if err := e.Stop(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, _ = e.Get(s.ID)
|
||||
if s.State != "stopped" || s.Packets < 1000 || !strings.Contains(s.Drops, fmt.Sprintf("%d/0", s.Packets)) {
|
||||
t.Fatalf("session: %+v", s)
|
||||
}
|
||||
segs, _ := e.Segments(s.ID)
|
||||
if len(segs) < 2 {
|
||||
t.Fatalf("expected rotation, got %d segments", len(segs))
|
||||
}
|
||||
var total int64
|
||||
for _, sg := range segs {
|
||||
if !sg.Complete || sg.SHA256 == "" {
|
||||
t.Fatalf("segment not finalised: %+v", sg)
|
||||
}
|
||||
total += sg.Count
|
||||
// every segment must be a standalone valid capture
|
||||
if b, err := exec.Command(e.Tools.TShark, "-r", filepath.Join(e.Dir, s.ID, sg.File), "-c", "1").CombinedOutput(); err != nil {
|
||||
t.Fatalf("segment %d unreadable: %s", sg.Seq, b)
|
||||
}
|
||||
}
|
||||
p, _ := e.Packets(ctx, s.ID, "", 0, 2000)
|
||||
if total != s.Packets || p.Total != s.Packets {
|
||||
t.Fatalf("counts disagree: segments %d session %d index %d", total, s.Packets, p.Total)
|
||||
}
|
||||
last, err := e.Packet(ctx, s.ID, s.Packets)
|
||||
if err != nil || len(last.Bytes) == 0 {
|
||||
t.Fatalf("last packet detail: %v", err)
|
||||
}
|
||||
if v, _ := e.Verify(s.ID); v[0].Status != "ok" {
|
||||
t.Fatalf("verify %+v", v)
|
||||
}
|
||||
got := 0
|
||||
for len(sub.C) > 0 {
|
||||
if ev := <-sub.C; ev.Type == "packets" {
|
||||
got++
|
||||
}
|
||||
}
|
||||
if got == 0 && sub.Dropped.Load() == 0 {
|
||||
t.Fatal("no live packet events")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Hub fans out live events to browser subscribers. A subscriber that cannot
|
||||
// keep up loses events (counted) and is told to resync; capture never waits.
|
||||
type Hub struct {
|
||||
mu sync.Mutex
|
||||
subs map[*Sub]struct{}
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
Type string // SSE event name
|
||||
Data []byte // JSON
|
||||
}
|
||||
|
||||
type Sub struct {
|
||||
C chan Event
|
||||
Dropped atomic.Int64
|
||||
Resync atomic.Bool
|
||||
}
|
||||
|
||||
func NewHub() *Hub { return &Hub{subs: map[*Sub]struct{}{}} }
|
||||
|
||||
func (h *Hub) Subscribe() *Sub {
|
||||
s := &Sub{C: make(chan Event, 64)}
|
||||
h.mu.Lock()
|
||||
h.subs[s] = struct{}{}
|
||||
h.mu.Unlock()
|
||||
return s
|
||||
}
|
||||
|
||||
func (h *Hub) Unsubscribe(s *Sub) {
|
||||
h.mu.Lock()
|
||||
delete(h.subs, s)
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *Hub) Publish(e Event) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for s := range h.subs {
|
||||
select {
|
||||
case s.C <- e:
|
||||
default:
|
||||
s.Dropped.Add(1)
|
||||
s.Resync.Store(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Node is one line of the protocol tree sent to the browser.
|
||||
type Node struct {
|
||||
Label string `json:"label"`
|
||||
Field string `json:"field,omitempty"`
|
||||
Pos int `json:"pos"`
|
||||
Size int `json:"size"`
|
||||
Verified bool `json:"verified"` // bytes at pos..pos+size are proven to belong to this field
|
||||
Children []Node `json:"children,omitempty"`
|
||||
}
|
||||
|
||||
type pdmlNode struct {
|
||||
XMLName xml.Name
|
||||
Name string `xml:"name,attr"`
|
||||
Show string `xml:"show,attr"`
|
||||
Showname string `xml:"showname,attr"`
|
||||
Value string `xml:"value,attr"`
|
||||
Unmasked string `xml:"unmaskedvalue,attr"`
|
||||
Pos string `xml:"pos,attr"`
|
||||
Size string `xml:"size,attr"`
|
||||
Hide string `xml:"hide,attr"`
|
||||
Children []pdmlNode `xml:",any"`
|
||||
}
|
||||
|
||||
// ParsePDML converts tshark PDML for a single packet into a tree. frame holds
|
||||
// the original captured bytes, used to verify each field's byte range.
|
||||
func ParsePDML(b []byte, frame []byte) ([]Node, error) {
|
||||
var doc struct {
|
||||
Packets []struct {
|
||||
Protos []pdmlNode `xml:",any"`
|
||||
} `xml:"packet"`
|
||||
}
|
||||
d := xml.NewDecoder(bytes.NewReader(b))
|
||||
d.Strict = false
|
||||
if err := d.Decode(&doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(doc.Packets) == 0 {
|
||||
return nil, errors.New("packet not found")
|
||||
}
|
||||
var out []Node
|
||||
for _, p := range doc.Packets[0].Protos {
|
||||
if p.Name == "geninfo" || p.Hide == "yes" {
|
||||
continue
|
||||
}
|
||||
n, _ := convert(p, frame)
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// convert returns the node and whether any field in its subtree contradicts the frame bytes.
|
||||
func convert(p pdmlNode, frame []byte) (Node, bool) {
|
||||
n := Node{Field: p.Name}
|
||||
n.Label = p.Showname
|
||||
if n.Label == "" {
|
||||
n.Label = p.Show
|
||||
}
|
||||
if n.Label == "" {
|
||||
n.Label = p.Name
|
||||
}
|
||||
n.Pos, _ = strconv.Atoi(p.Pos)
|
||||
n.Size, _ = strconv.Atoi(p.Size)
|
||||
inRange := n.Size > 0 && n.Pos >= 0 && n.Pos+n.Size <= len(frame)
|
||||
raw := p.Value
|
||||
if p.Unmasked != "" {
|
||||
raw = p.Unmasked // bitfields: value is the masked number, unmaskedvalue the bytes
|
||||
}
|
||||
contradicts := false
|
||||
if v, err := hex.DecodeString(raw); err == nil && len(v) > 0 && len(v) == n.Size {
|
||||
if inRange && bytes.Equal(v, frame[n.Pos:n.Pos+n.Size]) {
|
||||
n.Verified = true
|
||||
} else {
|
||||
contradicts = true
|
||||
}
|
||||
}
|
||||
matched, mismatched := false, contradicts
|
||||
for _, c := range p.Children {
|
||||
if c.Hide == "yes" {
|
||||
continue
|
||||
}
|
||||
cn, cm := convert(c, frame)
|
||||
if c.XMLName.Local != "proto" { // a nested (e.g. reassembled) protocol is judged on its own
|
||||
matched = matched || cn.Verified
|
||||
mismatched = mismatched || cm
|
||||
}
|
||||
n.Children = append(n.Children, cn)
|
||||
}
|
||||
// Protocols carry no value: trust their range only if their own fields verify
|
||||
// against the frame bytes and none contradict it (rules out reassembled sources).
|
||||
if raw == "" && inRange && matched && !mismatched {
|
||||
n.Verified = true
|
||||
}
|
||||
if p.Name == "frame" && inRange {
|
||||
n.Verified = true
|
||||
}
|
||||
return n, mismatched
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pilens.dev/pilens/internal/pcapfile"
|
||||
)
|
||||
|
||||
const rowCols = `num,seg,off,ts,src,dst,proto,len,info,sport,dport,stream,tcpflags,ttl,vlan,color`
|
||||
|
||||
func scanRows(rows *sql.Rows) ([]Row, error) {
|
||||
defer rows.Close()
|
||||
out := []Row{}
|
||||
for rows.Next() {
|
||||
var r Row
|
||||
var flags, color sql.NullString
|
||||
if err := rows.Scan(&r.Num, &r.Seg, &r.Off, &r.TS, &r.Src, &r.Dst, &r.Proto, &r.Len, &r.Info, &r.SPort, &r.DPort, &r.Stream, &flags, &r.TTL, &r.VLAN, &color); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.TCPFlags, r.Color = flags.String, color.String
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Page is a window of packet rows. Position i in the (filtered) list is
|
||||
// Offset+i; rows may be missing where summaries are still being generated.
|
||||
type Page struct {
|
||||
Total int64 `json:"total"`
|
||||
First int64 `json:"first"` // packet number at position 0 (unfiltered views)
|
||||
Offset int64 `json:"offset"`
|
||||
Rows []Row `json:"rows"`
|
||||
Nums []int64 `json:"nums,omitempty"` // packet numbers for each position (filtered views)
|
||||
Filtered bool `json:"filtered"`
|
||||
Partial bool `json:"partial"` // filter results cover only part of the capture
|
||||
}
|
||||
|
||||
func (e *Engine) Packets(ctx context.Context, id, filter string, offset, limit int64) (Page, error) {
|
||||
if limit <= 0 || limit > 2000 {
|
||||
limit = 500
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
p := Page{Offset: offset}
|
||||
if strings.TrimSpace(filter) == "" {
|
||||
var first, last sql.NullInt64
|
||||
e.DB.QueryRow(`SELECT MIN(num),MAX(num) FROM packets WHERE session_id=?`, id).Scan(&first, &last)
|
||||
if !first.Valid {
|
||||
return p, nil
|
||||
}
|
||||
p.First = first.Int64
|
||||
p.Total = last.Int64 - first.Int64 + 1
|
||||
from := first.Int64 + offset
|
||||
rows, err := e.DB.QueryContext(ctx, `SELECT `+rowCols+` FROM packets WHERE session_id=? AND num>=? AND num<? ORDER BY num`, id, from, from+limit)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
p.Rows, err = scanRows(rows)
|
||||
return p, err
|
||||
}
|
||||
nums, err := e.filterNums(ctx, id, filter)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
p.Filtered = true
|
||||
p.Total = int64(len(nums))
|
||||
if offset >= p.Total {
|
||||
return p, nil
|
||||
}
|
||||
end := min(offset+limit, p.Total)
|
||||
p.Nums = nums[offset:end]
|
||||
if len(p.Nums) == 0 {
|
||||
return p, nil
|
||||
}
|
||||
rows, err := e.DB.QueryContext(ctx, `SELECT `+rowCols+` FROM packets WHERE session_id=? AND num>=? AND num<=? ORDER BY num`, id, p.Nums[0], p.Nums[len(p.Nums)-1])
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
all, err := scanRows(rows)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
want := map[int64]bool{}
|
||||
for _, n := range p.Nums {
|
||||
want[n] = true
|
||||
}
|
||||
for _, r := range all {
|
||||
if want[r.Num] {
|
||||
p.Rows = append(p.Rows, r)
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- display filter cache
|
||||
|
||||
type filterEntry struct {
|
||||
perSeg map[int]segResult
|
||||
used time.Time
|
||||
}
|
||||
|
||||
type segResult struct {
|
||||
count int64 // segment packet count when evaluated
|
||||
nums []int64
|
||||
}
|
||||
|
||||
func (e *Engine) dropFilterCache(id string) {
|
||||
e.fcMu.Lock()
|
||||
defer e.fcMu.Unlock()
|
||||
for k := range e.fcache {
|
||||
if strings.HasPrefix(k, id+"\x00") {
|
||||
delete(e.fcache, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// filterNums evaluates a display filter with tshark segment by segment. Complete
|
||||
// segments are cached; the segment being written is re-evaluated when it grows.
|
||||
func (e *Engine) filterNums(ctx context.Context, id, filter string) ([]int64, error) {
|
||||
if err := sanityFilter(filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
segs, err := e.Segments(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := id + "\x00" + filter
|
||||
e.fcMu.Lock()
|
||||
if e.fcache == nil {
|
||||
e.fcache = map[string]*filterEntry{}
|
||||
}
|
||||
ent := e.fcache[key]
|
||||
if ent == nil {
|
||||
if len(e.fcache) >= 16 { // ponytail: evict oldest of a tiny cache
|
||||
var oldK string
|
||||
var oldT time.Time
|
||||
for k, v := range e.fcache {
|
||||
if oldK == "" || v.used.Before(oldT) {
|
||||
oldK, oldT = k, v.used
|
||||
}
|
||||
}
|
||||
delete(e.fcache, oldK)
|
||||
}
|
||||
ent = &filterEntry{perSeg: map[int]segResult{}}
|
||||
e.fcache[key] = ent
|
||||
}
|
||||
ent.used = time.Now()
|
||||
e.fcMu.Unlock()
|
||||
|
||||
var out []int64
|
||||
for _, s := range segs {
|
||||
if s.Deleted {
|
||||
continue
|
||||
}
|
||||
count := s.Count
|
||||
if !s.Complete {
|
||||
count = -1 // live: always re-evaluate
|
||||
e.flushLive(id)
|
||||
}
|
||||
e.fcMu.Lock()
|
||||
r, ok := ent.perSeg[s.Seq]
|
||||
e.fcMu.Unlock()
|
||||
if !ok || r.count != count || count == -1 {
|
||||
local, err := e.Tools.MatchFrames(ctx, filepath.Join(e.sessionDir(id), s.File), filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r = segResult{count: count}
|
||||
for _, n := range local {
|
||||
r.nums = append(r.nums, s.Base+n-1)
|
||||
}
|
||||
e.fcMu.Lock()
|
||||
ent.perSeg[s.Seq] = r
|
||||
e.fcMu.Unlock()
|
||||
}
|
||||
out = append(out, r.nums...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (e *Engine) flushLive(id string) {
|
||||
e.mu.Lock()
|
||||
l := e.live
|
||||
e.mu.Unlock()
|
||||
if l != nil && l.id == id {
|
||||
l.mu.Lock()
|
||||
if l.seg != nil {
|
||||
l.seg.bw.Flush()
|
||||
}
|
||||
l.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- packet detail
|
||||
|
||||
type Detail struct {
|
||||
Row Row `json:"row"`
|
||||
Bytes string `json:"bytes"` // hex of the original captured bytes
|
||||
Tree []Node `json:"tree"`
|
||||
Segment int `json:"segment"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
func (e *Engine) segmentFor(id string, seq int) (Segment, error) {
|
||||
segs, err := e.Segments(id)
|
||||
if err != nil {
|
||||
return Segment{}, err
|
||||
}
|
||||
for _, s := range segs {
|
||||
if s.Seq == seq && !s.Deleted {
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
return Segment{}, errors.New("segment no longer stored (ring buffer or retention)")
|
||||
}
|
||||
|
||||
func (e *Engine) Packet(ctx context.Context, id string, num int64) (Detail, error) {
|
||||
rows, err := e.DB.QueryContext(ctx, `SELECT `+rowCols+` FROM packets WHERE session_id=? AND num=?`, id, num)
|
||||
if err != nil {
|
||||
return Detail{}, err
|
||||
}
|
||||
rs, err := scanRows(rows)
|
||||
if err != nil {
|
||||
return Detail{}, err
|
||||
}
|
||||
if len(rs) == 0 {
|
||||
return Detail{}, ErrNotFound
|
||||
}
|
||||
d := Detail{Row: rs[0], Segment: rs[0].Seg}
|
||||
seg, err := e.segmentFor(id, d.Row.Seg)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
path := filepath.Join(e.sessionDir(id), seg.File)
|
||||
e.flushLive(id)
|
||||
data, err := pcapfile.ReadPacketAt(path, seg.Format, d.Row.Off)
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("reading packet bytes: %w", err)
|
||||
}
|
||||
d.Bytes = hex.EncodeToString(data)
|
||||
pdml, err := e.Tools.PDML(ctx, path, num-seg.Base+1)
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("dissection failed: %w", err)
|
||||
}
|
||||
d.Tree, err = ParsePDML(pdml, data)
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("dissection output: %w", err)
|
||||
}
|
||||
if seg.Seq > 1 || !seg.Complete {
|
||||
d.Note = "Dissected within segment " + fmt.Sprint(seg.Seq) + ": TCP stream numbers and reassembly restart at segment boundaries."
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- export
|
||||
|
||||
// ExportFormat reports the container format an export of id will use.
|
||||
func (e *Engine) ExportFormat(id string) string {
|
||||
segs, _ := e.Segments(id)
|
||||
for _, s := range segs {
|
||||
if s.Format == "pcap" {
|
||||
return "pcap"
|
||||
}
|
||||
}
|
||||
return "pcapng"
|
||||
}
|
||||
|
||||
// Export writes original packet records (all, or those matching filter) to w.
|
||||
// Packet data is copied byte-for-byte from the stored segments.
|
||||
func (e *Engine) Export(ctx context.Context, w io.Writer, id, filter string) (int, error) {
|
||||
segs, err := e.Segments(id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var keep map[int64]bool
|
||||
if strings.TrimSpace(filter) != "" {
|
||||
nums, err := e.filterNums(ctx, id, filter)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
keep = make(map[int64]bool, len(nums))
|
||||
for _, n := range nums {
|
||||
keep[n] = true
|
||||
}
|
||||
}
|
||||
e.flushLive(id)
|
||||
bw := bufio.NewWriterSize(w, 1<<20)
|
||||
total, first := 0, true
|
||||
for _, s := range segs {
|
||||
if s.Deleted {
|
||||
continue
|
||||
}
|
||||
base := s.Base
|
||||
n, err := pcapfile.CopySelected(bw, filepath.Join(e.sessionDir(id), s.File), s.Format, first, func(idx int) bool {
|
||||
return keep == nil || keep[base+int64(idx)]
|
||||
})
|
||||
total += n
|
||||
first = false
|
||||
if err != nil && !s.Complete && errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
continue // live segment: the last block is still being written
|
||||
}
|
||||
if err != nil {
|
||||
bw.Flush()
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
return total, bw.Flush()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- integrity
|
||||
|
||||
type Manifest struct {
|
||||
Session Session `json:"session"`
|
||||
Segments []Segment `json:"segments"`
|
||||
Generated string `json:"generated_utc"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
func (e *Engine) Manifest(id string) (Manifest, error) {
|
||||
s, err := e.Get(id)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
segs, err := e.Segments(id)
|
||||
return Manifest{Session: s, Segments: segs, Generated: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Note: "Segment files are original capture data written verbatim from dumpcap output (live) or the uploaded file (import). SHA-256 covers each complete file. Summary rows and dissections are derived data and are not part of the evidence."}, err
|
||||
}
|
||||
|
||||
type VerifyResult struct {
|
||||
Seq int `json:"seq"`
|
||||
File string `json:"file"`
|
||||
Status string `json:"status"` // ok | mismatch | missing | incomplete | deleted
|
||||
Actual string `json:"actual,omitempty"`
|
||||
}
|
||||
|
||||
func (e *Engine) Verify(id string) ([]VerifyResult, error) {
|
||||
segs, err := e.Segments(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []VerifyResult
|
||||
for _, s := range segs {
|
||||
r := VerifyResult{Seq: s.Seq, File: s.File}
|
||||
switch {
|
||||
case s.Deleted:
|
||||
r.Status = "deleted"
|
||||
case !s.Complete:
|
||||
r.Status = "incomplete"
|
||||
default:
|
||||
f, err := os.Open(filepath.Join(e.sessionDir(id), s.File))
|
||||
if err != nil {
|
||||
r.Status = "missing"
|
||||
break
|
||||
}
|
||||
h := sha256.New()
|
||||
io.Copy(h, f)
|
||||
f.Close()
|
||||
r.Actual = hex.EncodeToString(h.Sum(nil))
|
||||
r.Status = "ok"
|
||||
if r.Actual != s.SHA256 {
|
||||
r.Status = "mismatch"
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Seq < out[j].Seq })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- import
|
||||
|
||||
// Import stores an uploaded capture unchanged and indexes it in the background.
|
||||
// The upload is hostile input: size-limited, magic-checked, and parsed only by
|
||||
// bounds-checked code and resource-limited tshark.
|
||||
func (e *Engine) Import(ctx context.Context, name string, r io.Reader) (Session, error) {
|
||||
lim := e.Limits()
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" || len(name) > 200 {
|
||||
name = "Import " + time.Now().Format("2006-01-02 15:04")
|
||||
}
|
||||
if err := e.enforceStorage(""); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
id := newID()
|
||||
dir := e.sessionDir(id)
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
br := bufio.NewReader(io.LimitReader(r, lim.ImportBytes+1))
|
||||
head, _ := br.Peek(4)
|
||||
format, err := pcapfile.Sniff(head)
|
||||
if err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return Session{}, errors.New("file is not pcap or pcapng (compressed captures are not accepted)")
|
||||
}
|
||||
file := "original." + format
|
||||
f, err := os.OpenFile(filepath.Join(dir, file), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
|
||||
if err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return Session{}, err
|
||||
}
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(f, h), br)
|
||||
f.Sync()
|
||||
f.Close()
|
||||
if err != nil || n > lim.ImportBytes {
|
||||
os.RemoveAll(dir)
|
||||
if err == nil {
|
||||
err = fmt.Errorf("file exceeds import limit of %d bytes", lim.ImportBytes)
|
||||
}
|
||||
return Session{}, err
|
||||
}
|
||||
sum := hex.EncodeToString(h.Sum(nil))
|
||||
meta := e.meta(map[string]any{"import_sha256": sum, "import_bytes": n})
|
||||
tx, err := e.DB.Begin()
|
||||
if err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
tx.Exec(`INSERT INTO capture_sessions(id,name,kind,state,started_at,dir,meta,bytes) VALUES(?,?,?,?,?,?,?,?)`, id, name, "import", "importing", time.Now().UnixMilli(), dir, meta, n)
|
||||
tx.Exec(`INSERT INTO segments(session_id,seq,file,format,base,bytes,sha256,complete) VALUES(?,?,?,?,?,?,?,1)`, id, 1, file, format, 1, n, sum)
|
||||
if err := tx.Commit(); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
go e.indexImport(id, filepath.Join(dir, file), format)
|
||||
return e.Get(id)
|
||||
}
|
||||
|
||||
func (e *Engine) indexImport(id, path, format string) {
|
||||
fail := func(err error) { e.finishSession(id, "failed", err.Error(), "") }
|
||||
var offs []int64
|
||||
if err := pcapfile.Scan(path, format, func(idx int, off int64, _ []byte) error {
|
||||
if len(offs) >= 50_000_000 {
|
||||
return errors.New("too many packets")
|
||||
}
|
||||
offs = append(offs, off)
|
||||
return nil
|
||||
}); err != nil {
|
||||
fail(fmt.Errorf("malformed capture after packet %d: %v", len(offs), err))
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour)
|
||||
defer cancel()
|
||||
cmd := e.Tools.Command(ctx, 3600, e.Tools.SummaryArgs(path)...)
|
||||
out, err := cmd.StdoutPipe()
|
||||
var stderr limitedBuffer
|
||||
cmd.Stderr = &stderr
|
||||
if err == nil {
|
||||
err = cmd.Start()
|
||||
}
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
var batch []Row
|
||||
var count int64
|
||||
flush := func() {
|
||||
tx, err := e.DB.Begin()
|
||||
if err == nil {
|
||||
insertRows(tx, id, batch)
|
||||
tx.Commit()
|
||||
}
|
||||
batch = batch[:0]
|
||||
}
|
||||
ReadSummaries(out, func(r Row) {
|
||||
if r.Num < 1 || r.Num > int64(len(offs)) {
|
||||
return
|
||||
}
|
||||
r.Off, r.Seg = offs[r.Num-1], 1
|
||||
batch = append(batch, r)
|
||||
count++
|
||||
if len(batch) >= 2000 {
|
||||
flush()
|
||||
}
|
||||
})
|
||||
flush()
|
||||
werr := cmd.Wait()
|
||||
e.DB.Exec(`UPDATE segments SET count=? WHERE session_id=? AND seq=1`, len(offs), id)
|
||||
e.DB.Exec(`UPDATE capture_sessions SET packets=? WHERE id=?`, len(offs), id)
|
||||
msg := ""
|
||||
if werr != nil {
|
||||
msg = "tshark: " + strings.TrimSpace(stderr.String())
|
||||
}
|
||||
if count < int64(len(offs)) {
|
||||
msg = strings.TrimSpace(fmt.Sprintf("%s; summarised %d of %d packets", msg, count, len(offs)))
|
||||
}
|
||||
e.finishSession(id, "ready", msg, "")
|
||||
data, _ := json.Marshal(map[string]string{"session": id})
|
||||
e.Hub.Publish(Event{Type: "resync", Data: data})
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Tools locates and runs Wireshark command-line tools. All tshark children run
|
||||
// through `pilens rlimit-exec` so hostile input cannot exhaust the appliance.
|
||||
type Tools struct {
|
||||
Self string // path to the pilens binary (for rlimit-exec)
|
||||
TShark string
|
||||
Dumpcap string
|
||||
MemMB int // RLIMIT_AS for tshark children
|
||||
|
||||
cols []string // summary -e fields, version dependent
|
||||
emptyNG string // path to an empty pcapng used for filter validation
|
||||
}
|
||||
|
||||
// summaryFields; the column fields are filled in by Init (names changed in Wireshark 4.2).
|
||||
var summaryTail = []string{
|
||||
"frame.len", "tcp.srcport", "udp.srcport", "tcp.dstport", "udp.dstport",
|
||||
"tcp.stream", "tcp.flags.str", "ip.ttl", "ipv6.hlim", "vlan.id", "frame.coloring_rule.name",
|
||||
}
|
||||
|
||||
// EmptyPcapng is a valid pcapng with one Ethernet interface and no packets.
|
||||
func EmptyPcapng() []byte {
|
||||
var b bytes.Buffer
|
||||
le := binary.LittleEndian
|
||||
w32 := func(v uint32) { binary.Write(&b, le, v) }
|
||||
// SHB
|
||||
w32(0x0A0D0D0A)
|
||||
w32(28)
|
||||
w32(0x1A2B3C4D)
|
||||
binary.Write(&b, le, uint16(1))
|
||||
binary.Write(&b, le, uint16(0))
|
||||
binary.Write(&b, le, int64(-1))
|
||||
w32(28)
|
||||
// IDB: linktype 1 (Ethernet), snaplen 262144
|
||||
w32(1)
|
||||
w32(20)
|
||||
binary.Write(&b, le, uint16(1))
|
||||
binary.Write(&b, le, uint16(0))
|
||||
w32(262144)
|
||||
w32(20)
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func (t *Tools) Init(stateDir string) error {
|
||||
if t.MemMB == 0 {
|
||||
t.MemMB = 1536
|
||||
}
|
||||
p := stateDir + "/empty.pcapng"
|
||||
if err := os.WriteFile(p, EmptyPcapng(), 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
t.emptyNG = p
|
||||
newCols := []string{"_ws.col.def_src", "_ws.col.def_dst", "_ws.col.protocol", "_ws.col.info"}
|
||||
oldCols := []string{"_ws.col.Source", "_ws.col.Destination", "_ws.col.Protocol", "_ws.col.Info"}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := t.run(ctx, "-n", "-r", p, "-T", "fields", "-e", newCols[3]); err == nil {
|
||||
t.cols = newCols
|
||||
} else if _, err2 := t.run(ctx, "-n", "-r", p, "-T", "fields", "-e", oldCols[3]); err2 == nil {
|
||||
t.cols = oldCols
|
||||
} else {
|
||||
return fmt.Errorf("tshark unusable: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Command builds a resource-limited tshark command. cpuSec 0 = no CPU limit.
|
||||
func (t *Tools) Command(ctx context.Context, cpuSec int, args ...string) *exec.Cmd {
|
||||
var cmd *exec.Cmd
|
||||
if t.Self == "" { // tests only
|
||||
cmd = exec.CommandContext(ctx, t.TShark, args...)
|
||||
} else {
|
||||
a := append([]string{"rlimit-exec", strconv.Itoa(t.MemMB), strconv.Itoa(cpuSec), "--", t.TShark}, args...)
|
||||
cmd = exec.CommandContext(ctx, t.Self, a...)
|
||||
}
|
||||
cmd.Env = []string{"PATH=/usr/bin:/bin", "HOME=/nonexistent", "LANG=C.UTF-8"}
|
||||
cmd.WaitDelay = 5 * time.Second
|
||||
return cmd
|
||||
}
|
||||
|
||||
func (t *Tools) run(ctx context.Context, args ...string) ([]byte, error) {
|
||||
cmd := t.Command(ctx, 300, args...)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
msg := strings.TrimSpace(stderr.String())
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
return out, errors.New(msg)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SummaryArgs returns tshark arguments that print one tab-separated summary row per packet.
|
||||
func (t *Tools) SummaryArgs(input string) []string {
|
||||
a := []string{"-n", "-l", "--color", "-r", input, "-T", "fields", "-E", "separator=/t", "-E", "occurrence=f", "-E", "quote=n",
|
||||
"-e", "frame.number", "-e", "frame.time_epoch", "-e", t.cols[0], "-e", t.cols[1], "-e", t.cols[2]}
|
||||
for _, f := range summaryTail {
|
||||
a = append(a, "-e", f)
|
||||
}
|
||||
return append(a, "-e", t.cols[3]) // info last: it may contain anything
|
||||
}
|
||||
|
||||
// Row is one packet summary.
|
||||
type Row struct {
|
||||
Num int64 `json:"n"`
|
||||
TS float64 `json:"t"`
|
||||
Src string `json:"s"`
|
||||
Dst string `json:"d"`
|
||||
Proto string `json:"p"`
|
||||
Len int `json:"l"`
|
||||
Info string `json:"i"`
|
||||
SPort *int `json:"sp,omitempty"`
|
||||
DPort *int `json:"dp,omitempty"`
|
||||
Stream *int `json:"st,omitempty"`
|
||||
TCPFlags string `json:"tf,omitempty"`
|
||||
TTL *int `json:"ttl,omitempty"`
|
||||
VLAN *int `json:"vl,omitempty"`
|
||||
Color string `json:"c,omitempty"`
|
||||
Seg int `json:"-"`
|
||||
Off int64 `json:"-"`
|
||||
}
|
||||
|
||||
func atoiPtr(s string) *int {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
v, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return &v
|
||||
}
|
||||
|
||||
func firstNonEmpty(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// ParseSummaryLine parses one line of SummaryArgs output. Frame numbers are local to the input.
|
||||
func ParseSummaryLine(line string) (Row, error) {
|
||||
f := strings.SplitN(line, "\t", 17)
|
||||
if len(f) != 17 {
|
||||
return Row{}, fmt.Errorf("unexpected tshark summary line (%d fields)", len(f))
|
||||
}
|
||||
var r Row
|
||||
var err error
|
||||
if r.Num, err = strconv.ParseInt(f[0], 10, 64); err != nil {
|
||||
return r, err
|
||||
}
|
||||
r.TS, _ = strconv.ParseFloat(f[1], 64)
|
||||
r.Src, r.Dst, r.Proto = f[2], f[3], f[4]
|
||||
r.Len, _ = strconv.Atoi(f[5])
|
||||
r.SPort = atoiPtr(firstNonEmpty(f[6], f[7]))
|
||||
r.DPort = atoiPtr(firstNonEmpty(f[8], f[9]))
|
||||
r.Stream = atoiPtr(f[10])
|
||||
r.TCPFlags = f[11]
|
||||
r.TTL = atoiPtr(firstNonEmpty(f[12], f[13]))
|
||||
r.VLAN = atoiPtr(f[14])
|
||||
r.Color = f[15]
|
||||
r.Info = strings.TrimSpace(f[16])
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// ReadSummaries parses tshark summary output, calling fn per row.
|
||||
func ReadSummaries(r io.Reader, fn func(Row)) error {
|
||||
sc := bufio.NewScanner(r)
|
||||
sc.Buffer(make([]byte, 64*1024), 4<<20)
|
||||
for sc.Scan() {
|
||||
row, err := ParseSummaryLine(sc.Text())
|
||||
if err != nil {
|
||||
continue // ponytail: skip malformed lines rather than abort a capture; count if it ever matters
|
||||
}
|
||||
fn(row)
|
||||
}
|
||||
return sc.Err()
|
||||
}
|
||||
|
||||
// ValidateDisplayFilter asks tshark to compile the filter against an empty file.
|
||||
func (t *Tools) ValidateDisplayFilter(ctx context.Context, f string) error {
|
||||
if err := sanityFilter(f); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := t.run(ctx, "-n", "-r", t.emptyNG, "-Y", f)
|
||||
return err
|
||||
}
|
||||
|
||||
// ValidateCaptureFilter asks dumpcap to compile a BPF filter for iface.
|
||||
func (t *Tools) ValidateCaptureFilter(ctx context.Context, iface, f string) error {
|
||||
if err := sanityFilter(f); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, t.Dumpcap, "-d", "-i", iface, "-f", f).CombinedOutput()
|
||||
if err != nil {
|
||||
return errors.New(strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanityFilter(f string) error {
|
||||
if len(f) > 2048 {
|
||||
return errors.New("filter too long")
|
||||
}
|
||||
for _, r := range f {
|
||||
if r < 0x20 && r != '\t' {
|
||||
return errors.New("filter contains control characters")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MatchFrames returns the local frame numbers in file that match a display filter.
|
||||
// Truncated trailing data (a segment still being written) is tolerated.
|
||||
func (t *Tools) MatchFrames(ctx context.Context, file, filter string) ([]int64, error) {
|
||||
cmd := t.Command(ctx, 600, "-n", "-r", file, "-Y", filter, "-T", "fields", "-e", "frame.number")
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil && !strings.Contains(stderr.String(), "cut short") {
|
||||
return nil, errors.New(strings.TrimSpace(stderr.String() + " " + err.Error()))
|
||||
}
|
||||
var nums []int64
|
||||
for _, l := range strings.Split(string(out), "\n") {
|
||||
if n, err := strconv.ParseInt(strings.TrimSpace(l), 10, 64); err == nil {
|
||||
nums = append(nums, n)
|
||||
}
|
||||
}
|
||||
return nums, nil
|
||||
}
|
||||
|
||||
// PDML returns the PDML dissection for one local frame of file.
|
||||
func (t *Tools) PDML(ctx context.Context, file string, local int64) ([]byte, error) {
|
||||
cmd := t.Command(ctx, 120, "-n", "-r", file, "-Y", "frame.number=="+strconv.FormatInt(local, 10), "-T", "pdml")
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil && !strings.Contains(stderr.String(), "cut short") {
|
||||
return nil, errors.New(strings.TrimSpace(stderr.String() + " " + err.Error()))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (t *Tools) Version() string {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, t.TShark, "--version").Output()
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
l, _, _ := strings.Cut(string(out), "\n")
|
||||
return l
|
||||
}
|
||||
Reference in New Issue
Block a user