v3 candidate

internal/leviathan/analysis/entropy.go

@@ -0,0 +1,125 @@
+package analysis
+
+import (
+	"context"
+	"math"
+
+	"github.com/andydixon/nexus/internal/leviathan/document"
+)
+
+// Entropy computes Shannon entropy per sliding window over the range.
+//
+// The returned map has:
+//
+//	windowSize:   int
+//	step:         int
+//	samples:      []float64  — entropy per window (0..8)
+//	offsets:      []int64    — start offset of each window
+//	mean/min/max: summary stats
+//	highCount:    number of windows with entropy >= 7.5 (candidate compressed/encrypted)
+func Entropy(d *document.Document, start, length, windowSize, step int64) (map[string]any, error) {
+	return EntropyContext(context.Background(), d, start, length, windowSize, step)
+}
+
+func EntropyContext(ctx context.Context, d *document.Document, start, length, windowSize, step int64) (map[string]any, error) {
+	if windowSize <= 0 {
+		windowSize = 4096
+	}
+	if step <= 0 {
+		step = windowSize
+	}
+	if length < 0 {
+		length = d.Size() - start
+	}
+	if length <= 0 {
+		return map[string]any{
+			"windowSize": windowSize, "step": step,
+			"samples": []float64{}, "offsets": []int64{},
+			"mean": 0.0, "min": 0.0, "max": 0.0, "highCount": 0,
+		}, nil
+	}
+
+	// We run in a streaming fashion: advance through chunks, refill
+	// a rolling window, and emit a sample whenever the cursor hits
+	// a step boundary.
+	samples := make([]float64, 0, length/step+1)
+	offsets := make([]int64, 0, length/step+1)
+
+	// For bounded memory we read the document range chunk by chunk
+	// but we still need a window of size `windowSize`. Windows can be
+	// as small as 256 or as large as 1MB; we cap to length.
+	if windowSize > length {
+		windowSize = length
+	}
+
+	// Simple approach: iterate in step-sized hops, reading window
+	// bytes at each hop. This scales O(n * (w/step)) but window and
+	// step are usually the same for UI.
+	mean := 0.0
+	minE := math.Inf(1)
+	maxE := math.Inf(-1)
+	high := 0
+
+	for off := start; off < start+length; off += step {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		winLen := windowSize
+		if off+winLen > start+length {
+			winLen = start + length - off
+		}
+		buf, err := d.Range(off, winLen)
+		if err != nil {
+			return nil, err
+		}
+		e := shannon(buf)
+		samples = append(samples, e)
+		offsets = append(offsets, off)
+		mean += e
+		if e < minE {
+			minE = e
+		}
+		if e > maxE {
+			maxE = e
+		}
+		if e >= 7.5 {
+			high++
+		}
+	}
+	if len(samples) > 0 {
+		mean /= float64(len(samples))
+	} else {
+		minE, maxE = 0, 0
+	}
+
+	return map[string]any{
+		"windowSize": windowSize,
+		"step":       step,
+		"samples":    samples,
+		"offsets":    offsets,
+		"mean":       mean,
+		"min":        minE,
+		"max":        maxE,
+		"highCount":  high,
+	}, nil
+}
+
+func shannon(buf []byte) float64 {
+	if len(buf) == 0 {
+		return 0
+	}
+	var counts [256]int
+	for _, b := range buf {
+		counts[b]++
+	}
+	n := float64(len(buf))
+	e := 0.0
+	for _, c := range counts {
+		if c == 0 {
+			continue
+		}
+		p := float64(c) / n
+		e -= p * math.Log2(p)
+	}
+	return e
+}