Freeze the Go implementation as a test-only oracle, with vectors

oracle/go is the Go code at 4c1507c, moved unchanged, plus test-only
generators that record what it computes. tests/vectors holds the
deterministic crypto/store vectors and synthetic fixtures they produce.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andy Dixon
2026-10-08 09:55:17 +01:00
co-authored by Claude
parent 4c1507c463
commit 69f4b90777
60 changed files with 4843 additions and 0 deletions
+75
View File
@@ -0,0 +1,75 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"errors"
"fmt"
"io"
"os"
"path"
"strings"
)
// absorb reads a tar archive, gzipped or not, from r and adds every regular
// file in it beneath dest, streaming each straight into the vault so no
// plaintext touches the disk. Other entries, and names that would escape
// dest, are reported and skipped.
func absorb(s *store, cat map[string][]*record, r io.Reader, dest string) error {
br := bufio.NewReader(r)
if magic, _ := br.Peek(2); bytes.Equal(magic, []byte{0x1f, 0x8b}) {
zr, err := gzip.NewReader(br)
if err != nil {
return err
}
defer zr.Close()
r = zr
} else {
r = br
}
tr := tar.NewReader(r)
n := 0
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("reading archive: %w", err)
}
if hdr.Typeflag != tar.TypeReg {
if hdr.Typeflag != tar.TypeDir {
fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", hdr.Name)
}
continue
}
name := strings.TrimPrefix(path.Clean("/"+hdr.Name), "/")
if strings.Contains(hdr.Name, "\\") || name == "" || name != path.Clean(strings.TrimPrefix(strings.TrimPrefix(hdr.Name, "./"), "/")) {
fmt.Fprintf(os.Stderr, "skipping %s: unsafe name\n", hdr.Name)
continue
}
name = path.Join(dest, name)
replaced, err := s.addReader(cat, tr, hdr.Size, hdr.FileInfo().Mode(), hdr.ModTime, name)
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
err = fmt.Errorf("%s: archive ends early", hdr.Name)
}
if err != nil {
return err
}
verb := "added"
if replaced {
verb = "replaced"
}
fmt.Fprintf(os.Stderr, "%s %s\n", verb, name)
n++
}
if n == 0 {
return errors.New("no files in archive")
}
return nil
}
+233
View File
@@ -0,0 +1,233 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"fmt"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/andydixon/vault/internal/pqcrypt"
"github.com/andydixon/vault/internal/term"
)
// The agent remembers an unlocked key for a while, like ssh-agent, so
// commands skip the passphrase. `vault unlock` checks the passphrase, then
// starts a copy of itself (agentCmd) and hands it the suite ID and master
// seed over a pipe; it never touches the disk. The agent serves them on a
// Unix socket in a directory only this user can enter, keeps them in locked,
// undumpable memory where the platform allows, and wipes them and exits at
// a deadline, or on `vault lock`. The deadline is fixed, or with --idle it
// moves on each time a command uses the key.
const (
agentCmd = "__agent" // hidden: the agent process itself
agentGet = 'g'
agentQuit = 'q'
agentReply = 2 + seedSize
agentTimeout = 2 * time.Second
)
// agentSocket is the socket of the agent for the key file at keyPath, one
// per key file, in a private per-user directory it creates if needed.
func agentSocket(keyPath string) (string, error) {
abs, err := filepath.Abs(keyPath)
if err != nil {
return "", err
}
base := os.Getenv("XDG_RUNTIME_DIR")
if base == "" {
base = os.TempDir()
}
dir := filepath.Join(base, fmt.Sprintf("vault-%d", os.Getuid()))
if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
return "", err
}
if err := privateDir(dir); err != nil {
return "", err
}
sum := sha256.Sum256([]byte(abs))
return filepath.Join(dir, "agent-"+hex.EncodeToString(sum[:8])), nil
}
// agentDial connects to the agent for keyPath and sends it op.
func agentDial(keyPath string, op byte) (net.Conn, error) {
sock, err := agentSocket(keyPath)
if err != nil {
return nil, err
}
c, err := net.DialTimeout("unix", sock, agentTimeout)
if err != nil {
return nil, err
}
c.SetDeadline(time.Now().Add(agentTimeout))
if _, err := c.Write([]byte{op}); err != nil {
c.Close()
return nil, err
}
return c, nil
}
// agentIdentity asks a running agent for the key at keyPath.
func agentIdentity(keyPath string) (*pqcrypt.PQIdentity, error) {
c, err := agentDial(keyPath, agentGet)
if err != nil {
return nil, err
}
defer c.Close()
buf := make([]byte, agentReply)
defer clear(buf)
if _, err := io.ReadFull(c, buf); err != nil {
return nil, err
}
return pqcrypt.NewPQIdentityFromSeed(binary.BigEndian.Uint16(buf), buf[2:])
}
// agentStop tells the agent for keyPath to forget its key, reporting
// whether one was running.
func agentStop(keyPath string) bool {
c, err := agentDial(keyPath, agentQuit)
if err != nil {
return false
}
defer c.Close()
io.Copy(io.Discard, c) // returns once the agent has wiped the key
return true
}
// agentStart replaces any agent for keyPath with one holding id until ttl
// has passed, since it started or, with idle, since the key was last used.
func agentStart(keyPath string, id *pqcrypt.PQIdentity, ttl time.Duration, idle bool) error {
sock, err := agentSocket(keyPath)
if err != nil {
return err
}
agentStop(keyPath)
self, err := os.Executable()
if err != nil {
return err
}
mode := "fixed"
if idle {
mode = "idle"
}
cmd := exec.Command(self, agentCmd, sock, ttl.String(), mode)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
if err := cmd.Start(); err != nil {
return err
}
secret := binary.BigEndian.AppendUint16(make([]byte, 0, agentReply), id.SuiteID())
seed := id.Seed()
secret = append(secret, seed...)
clear(seed)
_, err = stdin.Write(secret)
clear(secret)
stdin.Close()
// The agent closes its standard output once listening, after saying
// "ok" or why not.
msg, _ := io.ReadAll(stdout)
if err != nil || string(msg) != "ok" {
cmd.Process.Kill()
cmd.Wait()
if err == nil {
err = fmt.Errorf("agent did not start: %s", strings.TrimSpace(string(msg)))
}
return err
}
return cmd.Process.Release()
}
// runAgent is the agent process: it reads the key from in, says "ok" or
// why not on out and closes it, then serves the key on sock until ttl has
// passed, since it started or with mode "idle" since the key was last
// handed out, or until it is told to quit.
func runAgent(sock, ttlArg, mode string, in io.Reader, out io.WriteCloser) {
report := func(msg string) { io.WriteString(out, msg); out.Close() }
ttl, err := time.ParseDuration(ttlArg)
if err != nil || ttl <= 0 || mode != "fixed" && mode != "idle" {
report("bad duration or mode")
return
}
secret := make([]byte, agentReply)
protectMemory(secret)
if _, err := io.ReadFull(in, secret); err != nil {
report("no key received")
return
}
// A socket left by an agent that died stops Listen; agentStart has
// already stopped any live one.
os.Remove(sock)
l, err := net.Listen("unix", sock)
if err != nil {
clear(secret)
report(err.Error())
return
}
os.Chmod(sock, 0o600)
report("ok")
expiry := time.AfterFunc(ttl, func() { l.Close() })
for {
c, err := l.Accept()
if err != nil {
break
}
op := serveAgent(c, secret)
c.Close()
if op == agentQuit {
break
}
if op == agentGet && mode == "idle" {
expiry.Reset(ttl)
}
}
l.Close()
clear(secret)
os.Remove(sock)
}
// serveAgent answers one request and returns its operation.
func serveAgent(c net.Conn, secret []byte) byte {
c.SetDeadline(time.Now().Add(agentTimeout))
op := make([]byte, 1)
if _, err := io.ReadFull(c, op); err != nil {
return 0
}
switch op[0] {
case agentGet:
c.Write(secret)
case agentQuit:
clear(secret)
}
return op[0]
}
// keyIdentity unlocks the key at keyPath: from the agent if one is running
// and no passphrase is in the environment, else with the passphrase.
func keyIdentity(keyPath string) (*pqcrypt.PQIdentity, error) {
if _, ok := os.LookupEnv(passphraseEnv); !ok {
if id, err := agentIdentity(keyPath); err == nil {
return id, nil
}
}
pass, err := term.ReadPassphrase("Enter passphrase:", passphraseEnv)
if err != nil {
return nil, err
}
return unlockKeyFile(keyPath, pass)
}
+11
View File
@@ -0,0 +1,11 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import "golang.org/x/sys/unix"
// Not dumpable also means not ptrace-able by the same user.
func init() {
undumpable = func() { unix.Prctl(unix.PR_SET_DUMPABLE, 0, 0, 0, 0) }
}
+13
View File
@@ -0,0 +1,13 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build !unix
package main
// ponytail: no memory locking off unix; VirtualLock on Windows if wanted.
func protectMemory([]byte) {}
// privateDir trusts the per-user temporary directory; Windows keeps it
// private with its ACL.
func privateDir(string) error { return nil }
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build unix
package main
import (
"fmt"
"os"
"syscall"
"golang.org/x/sys/unix"
)
// undumpable stops other processes of this user reading the agent's memory,
// where the platform can.
var undumpable = func() {}
// protectMemory keeps b out of swap and the agent out of core dumps.
func protectMemory(b []byte) {
unix.Mlock(b)
unix.Setrlimit(unix.RLIMIT_CORE, &unix.Rlimit{})
undumpable()
}
// privateDir checks dir is a real directory that only this user can use,
// so nobody else can reach, swap or plant the agent's socket.
func privateDir(dir string) error {
fi, err := os.Lstat(dir)
if err != nil {
return err
}
st, ok := fi.Sys().(*syscall.Stat_t)
if !fi.IsDir() || !ok || int(st.Uid) != os.Getuid() || fi.Mode().Perm() != 0o700 {
return fmt.Errorf("%s is not a private directory of this user", dir)
}
return nil
}
+13
View File
@@ -0,0 +1,13 @@
module github.com/andydixon/vault
// Go 1.26.6 or newer is a security requirement inherited from the crypto
// library: the standard-library fixes it relies on exist only from that
// release.
go 1.26.6
require (
github.com/cloudflare/circl v1.6.4
golang.org/x/crypto v0.53.0
golang.org/x/sys v0.46.0
golang.org/x/term v0.44.0
)
+8
View File
@@ -0,0 +1,8 @@
github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U=
github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+356
View File
@@ -0,0 +1,356 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// Package format implements the fury file format.
package format
import (
"bufio"
"bytes"
"encoding/base64"
"errors"
"fmt"
"io"
"strings"
)
const MaxHeaderSize = 16 << 20 // 16 MiB
const MaxStanzaSize = 1 << 20 // 1 MiB
type Header struct {
Recipients []*Stanza
MAC []byte
}
// Stanza is the serialised recipient block used in a fury header.
type Stanza struct {
Type string
Args []string
Body []byte
}
var b64 = base64.RawStdEncoding.Strict()
func DecodeString(s string) ([]byte, error) {
// CR and LF are ignored by DecodeString, but we don't want any malleability.
if strings.ContainsAny(s, "\n\r") {
return nil, errors.New(`unexpected newline character`)
}
return b64.DecodeString(s)
}
var EncodeToString = b64.EncodeToString
const ColumnsPerLine = 64
const BytesPerLine = ColumnsPerLine / 4 * 3
// NewWrappedBase64Encoder returns a WrappedBase64Encoder that writes to dst.
func NewWrappedBase64Encoder(enc *base64.Encoding, dst io.Writer) *WrappedBase64Encoder {
w := &WrappedBase64Encoder{dst: dst}
w.enc = base64.NewEncoder(enc, WriterFunc(w.writeWrapped))
return w
}
type WriterFunc func(p []byte) (int, error)
func (f WriterFunc) Write(p []byte) (int, error) { return f(p) }
// WrappedBase64Encoder is a standard base64 encoder that inserts an LF
// character every ColumnsPerLine bytes. It does not insert a newline neither at
// the beginning nor at the end of the stream, but it ensures the last line is
// shorter than ColumnsPerLine, which means it might be empty.
type WrappedBase64Encoder struct {
enc io.WriteCloser
dst io.Writer
written int
buf bytes.Buffer
}
func (w *WrappedBase64Encoder) Write(p []byte) (int, error) { return w.enc.Write(p) }
func (w *WrappedBase64Encoder) Close() error {
return w.enc.Close()
}
func (w *WrappedBase64Encoder) writeWrapped(p []byte) (int, error) {
if w.buf.Len() != 0 {
panic("fury: internal error: non-empty WrappedBase64Encoder.buf")
}
total := len(p)
for len(p) > 0 {
toWrite := min(ColumnsPerLine-(w.written%ColumnsPerLine), len(p))
n, _ := w.buf.Write(p[:toWrite])
w.written += n
p = p[n:]
if w.written%ColumnsPerLine == 0 {
w.buf.Write([]byte("\n"))
}
}
if _, err := w.buf.WriteTo(w.dst); err != nil {
// We always return n = 0 on error because it's hard to work back to the
// input length that ended up written out. Not ideal, but Write errors
// are not recoverable anyway.
return 0, err
}
return total, nil
}
// LastLineIsEmpty returns whether the last output line was empty, either
// because no input was written, or because a multiple of BytesPerLine was.
//
// Calling LastLineIsEmpty before Close is meaningless.
func (w *WrappedBase64Encoder) LastLineIsEmpty() bool {
return w.written%ColumnsPerLine == 0
}
const intro = "fury-encryption.org/v1\n"
var stanzaPrefix = []byte("->")
var footerPrefix = []byte("---")
func (r *Stanza) Marshal(w io.Writer) error {
if _, err := w.Write(stanzaPrefix); err != nil {
return err
}
for _, a := range append([]string{r.Type}, r.Args...) {
if _, err := io.WriteString(w, " "+a); err != nil {
return err
}
}
if _, err := io.WriteString(w, "\n"); err != nil {
return err
}
ww := NewWrappedBase64Encoder(b64, w)
if _, err := ww.Write(r.Body); err != nil {
return err
}
if err := ww.Close(); err != nil {
return err
}
_, err := io.WriteString(w, "\n")
return err
}
func (h *Header) MarshalWithoutMAC(w io.Writer) error {
if _, err := io.WriteString(w, intro); err != nil {
return err
}
for _, r := range h.Recipients {
if err := r.Marshal(w); err != nil {
return err
}
}
_, err := fmt.Fprintf(w, "%s", footerPrefix)
return err
}
func (h *Header) Marshal(w io.Writer) error {
if err := h.MarshalWithoutMAC(w); err != nil {
return err
}
mac := b64.EncodeToString(h.MAC)
_, err := fmt.Fprintf(w, " %s\n", mac)
return err
}
type StanzaReader struct {
r *bufio.Reader
err error
lastBytes int64
}
func NewStanzaReader(r *bufio.Reader) *StanzaReader {
return &StanzaReader{r: r}
}
func (r *StanzaReader) ReadStanza() (s *Stanza, err error) {
// Read errors are unrecoverable.
if r.err != nil {
return nil, r.err
}
var bytesRead int64
defer func() {
r.lastBytes = bytesRead
r.err = err
}()
s = &Stanza{}
line, err := readLimitedLine(r.r, &bytesRead, MaxStanzaSize)
if errors.Is(err, errSizeLimitExceeded) {
return nil, fmt.Errorf("stanza exceeds %d byte limit", MaxStanzaSize)
}
if err != nil {
return nil, fmt.Errorf("failed to read line: %w", err)
}
if !bytes.HasPrefix(line, stanzaPrefix) {
return nil, fmt.Errorf("malformed stanza opening line: %q", line)
}
prefix, args := splitArgs(line)
if prefix != string(stanzaPrefix) || len(args) < 1 {
return nil, fmt.Errorf("malformed stanza: %q", line)
}
for _, a := range args {
if !isValidString(a) {
return nil, fmt.Errorf("malformed stanza: %q", line)
}
}
s.Type = args[0]
s.Args = args[1:]
for {
line, err := readLimitedLine(r.r, &bytesRead, MaxStanzaSize)
if errors.Is(err, errSizeLimitExceeded) {
return nil, fmt.Errorf("stanza exceeds %d byte limit", MaxStanzaSize)
}
if err != nil {
return nil, fmt.Errorf("failed to read line: %w", err)
}
b, err := DecodeString(strings.TrimSuffix(string(line), "\n"))
if err != nil {
if bytes.HasPrefix(line, footerPrefix) || bytes.HasPrefix(line, stanzaPrefix) {
return nil, fmt.Errorf("malformed body line %q: stanza ended without a short line", line)
}
return nil, errorf("malformed body line %q: %v", line, err)
}
if len(b) > BytesPerLine {
return nil, errorf("malformed body line %q: too long", line)
}
s.Body = append(s.Body, b...)
if len(b) < BytesPerLine {
// A stanza body always ends with a short line.
return s, nil
}
}
}
var errSizeLimitExceeded = errors.New("size limit exceeded")
func readLimitedLine(r *bufio.Reader, bytesRead *int64, limit int64) ([]byte, error) {
var line []byte
for {
frag, err := r.ReadSlice('\n')
if len(frag) > 0 {
if limit > 0 && *bytesRead+int64(len(frag)) > limit {
return nil, errSizeLimitExceeded
}
*bytesRead += int64(len(frag))
line = append(line, frag...)
}
if err == nil {
return line, nil
}
if err != bufio.ErrBufferFull {
return nil, err
}
}
}
type ParseError struct {
err error
}
func (e *ParseError) Error() string {
return "parsing fury header: " + e.err.Error()
}
func (e *ParseError) Unwrap() error {
return e.err
}
func errorf(format string, a ...any) error {
return &ParseError{fmt.Errorf(format, a...)}
}
// Parse returns the header and a Reader that begins at the start of the
// payload.
func Parse(input io.Reader) (*Header, io.Reader, error) {
h := &Header{}
rr := bufio.NewReader(input)
var headerBytes int64
lineBytes, err := readLimitedLine(rr, &headerBytes, MaxHeaderSize)
line := string(lineBytes)
if errors.Is(err, errSizeLimitExceeded) {
return nil, nil, errorf("header exceeds %d byte limit", MaxHeaderSize)
}
if err == io.EOF {
return nil, nil, errorf("file is empty")
} else if err != nil {
return nil, nil, errorf("failed to read intro: %w", err)
}
if line != intro {
return nil, nil, errorf("unexpected intro: %q", line)
}
sr := NewStanzaReader(rr)
for {
peek, err := rr.Peek(len(footerPrefix))
if err != nil {
return nil, nil, errorf("failed to read header: %w", err)
}
if bytes.Equal(peek, footerPrefix) {
line, err := readLimitedLine(rr, &headerBytes, MaxHeaderSize)
if errors.Is(err, errSizeLimitExceeded) {
return nil, nil, errorf("header exceeds %d byte limit", MaxHeaderSize)
}
if err != nil {
return nil, nil, fmt.Errorf("failed to read header: %w", err)
}
prefix, args := splitArgs(line)
if prefix != string(footerPrefix) || len(args) != 1 {
return nil, nil, errorf("malformed closing line: %q", line)
}
h.MAC, err = DecodeString(args[0])
if err != nil || len(h.MAC) != 32 {
return nil, nil, errorf("malformed closing line %q: %v", line, err)
}
break
}
s, err := sr.ReadStanza()
if err != nil {
return nil, nil, fmt.Errorf("failed to parse header: %w", err)
}
headerBytes += sr.lastBytes
if headerBytes > MaxHeaderSize {
return nil, nil, errorf("header exceeds %d byte limit", MaxHeaderSize)
}
h.Recipients = append(h.Recipients, s)
}
// If input is a bufio.Reader, rr might be equal to input because
// bufio.NewReader short-circuits. In this case we can just return it (and
// we would end up reading the buffer twice if we prepended the peek below).
if rr == input {
return h, rr, nil
}
// Otherwise, unwind the bufio overread and return the unbuffered input.
buf, err := rr.Peek(rr.Buffered())
if err != nil {
return nil, nil, errorf("internal error: %v", err)
}
payload := io.MultiReader(bytes.NewReader(buf), input)
return h, payload, nil
}
func splitArgs(line []byte) (string, []string) {
l := strings.TrimSuffix(string(line), "\n")
parts := strings.Split(l, " ")
return parts[0], parts[1:]
}
func isValidString(s string) bool {
if len(s) == 0 {
return false
}
for _, c := range s {
if c < 33 || c > 126 {
return false
}
}
return true
}
+90
View File
@@ -0,0 +1,90 @@
// SPDX-License-Identifier: GPL-3.0-or-later
package format_test
import (
"bufio"
"bytes"
"io"
"testing"
"github.com/andydixon/vault/internal/format"
)
func TestStanzaMarshal(t *testing.T) {
s := &format.Stanza{
Type: "test",
Args: []string{"1", "2", "3"},
Body: nil, // empty
}
buf := &bytes.Buffer{}
s.Marshal(buf)
if exp := "-> test 1 2 3\n\n"; buf.String() != exp {
t.Errorf("wrong empty stanza encoding: expected %q, got %q", exp, buf.String())
}
buf.Reset()
s.Body = []byte("AAA")
s.Marshal(buf)
if exp := "-> test 1 2 3\nQUFB\n"; buf.String() != exp {
t.Errorf("wrong normal stanza encoding: expected %q, got %q", exp, buf.String())
}
buf.Reset()
s.Body = bytes.Repeat([]byte("A"), format.BytesPerLine)
s.Marshal(buf)
if exp := "-> test 1 2 3\nQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB\n\n"; buf.String() != exp {
t.Errorf("wrong 64 columns stanza encoding: expected %q, got %q", exp, buf.String())
}
}
func FuzzMalleability(f *testing.F) {
f.Fuzz(func(t *testing.T, data []byte) {
h, payload, err := format.Parse(bytes.NewReader(data))
if err != nil {
if h != nil {
t.Error("h != nil on error")
}
if payload != nil {
t.Error("payload != nil on error")
}
t.Skip()
}
w := &bytes.Buffer{}
if err := h.Marshal(w); err != nil {
t.Fatal(err)
}
if _, err := io.Copy(w, payload); err != nil {
t.Fatal(err)
}
if !bytes.Equal(w.Bytes(), data) {
t.Error("Marshal output different from input")
}
})
}
// FuzzStanza exercises the recipient-stanza reader directly. Parse reaches it
// only through a well-formed intro line, so fuzzing it on its own covers
// stanza shapes a whole-file seed cannot easily reach.
func FuzzStanza(f *testing.F) {
f.Fuzz(func(t *testing.T, data []byte) {
reader := format.NewStanzaReader(bufio.NewReader(bytes.NewReader(data)))
for range 64 {
stanza, err := reader.ReadStanza()
if err != nil {
if stanza != nil {
t.Fatal("stanza returned alongside an error")
}
return
}
if stanza == nil {
t.Fatal("no stanza and no error")
}
// Parser limits must hold: nothing an attacker supplies may make
// the reader materialise more than the stanza size limit.
if len(stanza.Body) > format.MaxStanzaSize {
t.Fatalf("stanza body of %d bytes exceeds the %d byte limit", len(stanza.Body), format.MaxStanzaSize)
}
}
})
}
+41
View File
@@ -0,0 +1,41 @@
// SPDX-License-Identifier: GPL-3.0-or-later
package format
import (
"bufio"
"bytes"
"strings"
"testing"
)
func TestStanzaReaderRejectsOversizedStanza(t *testing.T) {
var input bytes.Buffer
input.WriteString("-> test ")
input.WriteString(strings.Repeat("a", MaxStanzaSize))
input.WriteString("\n\n")
sr := NewStanzaReader(bufio.NewReader(&input))
if _, err := sr.ReadStanza(); err == nil {
t.Fatal("ReadStanza accepted an oversized stanza")
} else if !strings.Contains(err.Error(), "stanza exceeds") {
t.Fatalf("ReadStanza error = %v, want stanza size limit error", err)
}
}
func TestParseRejectsOversizedHeader(t *testing.T) {
var input bytes.Buffer
input.WriteString(intro)
for input.Len() <= MaxHeaderSize {
input.WriteString("-> test ")
input.WriteString(strings.Repeat("a", MaxStanzaSize-1024))
input.WriteString("\n\n")
}
input.WriteString("--- AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n")
if _, _, err := Parse(&input); err == nil {
t.Fatal("Parse accepted an oversized header")
} else if !strings.Contains(err.Error(), "header exceeds") {
t.Fatalf("Parse error = %v, want header size limit error", err)
}
}
@@ -0,0 +1,298 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// Test-only vector generator for the Rust port. It adds nothing to the
// production code: it only calls the frozen implementation and records what
// it computes. Run through oracle/gen-vectors.sh.
package pqcrypt
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"io"
"os"
"path/filepath"
"testing"
"github.com/andydixon/vault/internal/format"
"github.com/cloudflare/circl/kem"
"github.com/cloudflare/circl/kem/frodo/frodo640shake"
"github.com/cloudflare/circl/kem/mlkem/mlkem768"
"golang.org/x/crypto/hkdf"
)
type kemVector struct {
Seed string `json:"seed"`
PubSHA256 string `json:"pub_sha256"`
PrivSHA256 string `json:"priv_sha256"`
EncapSeed string `json:"encap_seed"`
CtSHA256 string `json:"ct_sha256"`
SharedSecret string `json:"ss"`
RejectFlipIdx int `json:"reject_flip_index"`
RejectSS string `json:"reject_ss"`
}
type hybridVector struct {
MasterSeed string `json:"master_seed"`
FrodoSeed string `json:"frodo_seed"`
MLKEMSeed string `json:"mlkem_seed"`
PubSHA256 string `json:"pub_sha256"`
PrivSHA256 string `json:"priv_sha256"`
PubPrefix string `json:"pub_prefix"`
}
type wrapVector struct {
MasterSeed string `json:"master_seed"`
FrodoEncap string `json:"frodo_encap_seed"`
MLKEMEncap string `json:"mlkem_encap_seed"`
FileKey string `json:"file_key"`
KEK string `json:"kek"`
Header string `json:"header_hex"` // full serialised header, MAC line included
MAC string `json:"mac"`
Nonce string `json:"nonce"`
PayloadKey string `json:"payload_key"`
StanzaArg0 string `json:"stanza_arg0_sha256"`
StanzaBody string `json:"stanza_body"`
}
type headerCase struct {
Input string `json:"input_hex"`
Accept bool `json:"accept"`
Rest string `json:"rest_hex,omitempty"` // payload bytes after the header
}
func sum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
func pattern(n int, salt byte) []byte {
b := make([]byte, n)
for i := range b {
b[i] = byte(i*7) ^ salt
}
return b
}
func kemVectors(t *testing.T, s kem.Scheme, n int) []kemVector {
var out []kemVector
for i := range n {
seed := pattern(s.SeedSize(), byte(i*31))
pk, sk := s.DeriveKeyPair(seed)
pkb, _ := pk.MarshalBinary()
skb, _ := sk.MarshalBinary()
es := pattern(s.EncapsulationSeedSize(), byte(i*17+5))
ct, ss, err := s.EncapsulateDeterministically(pk, es)
if err != nil {
t.Fatal(err)
}
ss2, err := s.Decapsulate(sk, ct)
if err != nil || !bytes.Equal(ss, ss2) {
t.Fatal("decapsulation mismatch")
}
flip := (i * 997) % len(ct)
bad := append([]byte(nil), ct...)
bad[flip] ^= 0x01
rss, err := s.Decapsulate(sk, bad)
if err != nil {
t.Fatal(err)
}
out = append(out, kemVector{
Seed: hex.EncodeToString(seed), PubSHA256: sum(pkb), PrivSHA256: sum(skb),
EncapSeed: hex.EncodeToString(es), CtSHA256: sum(ct), SharedSecret: hex.EncodeToString(ss),
RejectFlipIdx: flip, RejectSS: hex.EncodeToString(rss),
})
}
return out
}
func TestOracleVectors(t *testing.T) {
dir := os.Getenv("VAULT_ORACLE_OUT")
if dir == "" {
t.Skip("set VAULT_ORACLE_OUT to write vectors")
}
suite := frodoMLKEM768{}
frodo, ml := frodo640shake.Scheme(), mlkem768.Scheme()
var hybrids []hybridVector
for i := range 6 {
master := pattern(32, byte(i*53+1))
if i == 5 {
master = bytes.Repeat([]byte{0xff}, 32)
}
r := hkdf.New(sha256.New, master, nil, []byte("fury/v1 seed-expand "+suite.tag()))
fs := make([]byte, frodo.SeedSize())
ms := make([]byte, ml.SeedSize())
io.ReadFull(r, fs)
io.ReadFull(r, ms)
pub, priv, err := suite.deriveKeypair(master)
if err != nil {
t.Fatal(err)
}
hybrids = append(hybrids, hybridVector{
MasterSeed: hex.EncodeToString(master), FrodoSeed: hex.EncodeToString(fs), MLKEMSeed: hex.EncodeToString(ms),
PubSHA256: sum(pub), PrivSHA256: sum(priv), PubPrefix: hex.EncodeToString(pub[:64]),
})
}
// A wrap with controlled encapsulation randomness, built from the real
// deriveKEK, aeadEncrypt, header marshalling, headerMAC and streamKey.
var wraps []wrapVector
for i := range 3 {
master := pattern(32, byte(i*53+1))
id, err := NewPQIdentityFromSeed(suite.id(), master)
if err != nil {
t.Fatal(err)
}
pub := id.Recipient().Bytes()
fpk, _ := frodo.UnmarshalBinaryPublicKey(pub[:frodo.PublicKeySize()])
mpk, _ := ml.UnmarshalBinaryPublicKey(pub[frodo.PublicKeySize():])
fe, me := pattern(16, byte(i+100)), pattern(32, byte(i+150))
ct1, ss1, _ := frodo.EncapsulateDeterministically(fpk, fe)
ct2, ss2, _ := ml.EncapsulateDeterministically(mpk, me)
kek := suite.deriveKEK(ss1, ss2, ct1, ct2)
fileKey := pattern(32, byte(i+200))
body, _ := aeadEncrypt(kek, fileKey)
st := &format.Stanza{Type: suite.tag(), Args: []string{format.EncodeToString(ct1), format.EncodeToString(ct2)}, Body: body}
hdr := &format.Header{Recipients: []*format.Stanza{st}}
mac, err := headerMAC(fileKey, hdr)
if err != nil {
t.Fatal(err)
}
hdr.MAC = mac
var hb bytes.Buffer
hdr.Marshal(&hb)
// The real identity must open what was built.
got, err := id.Unwrap([]*Stanza{(*Stanza)(st)})
if err != nil || !bytes.Equal(got, fileKey) {
t.Fatal("identity did not unwrap the vector")
}
nonce := pattern(16, byte(i+250))
wraps = append(wraps, wrapVector{
MasterSeed: hex.EncodeToString(master), FrodoEncap: hex.EncodeToString(fe), MLKEMEncap: hex.EncodeToString(me),
FileKey: hex.EncodeToString(fileKey), KEK: hex.EncodeToString(kek), Header: hex.EncodeToString(hb.Bytes()),
MAC: hex.EncodeToString(mac), Nonce: hex.EncodeToString(nonce), PayloadKey: hex.EncodeToString(streamKey(fileKey, nonce)),
StanzaArg0: sum([]byte(st.Args[0])), StanzaBody: hex.EncodeToString(body),
})
}
// Whole files from the real Encrypt, with fresh randomness, for the
// first hybrid identity. Plaintexts are pattern(size, 0x5a).
id, _ := NewPQIdentityFromSeed(suite.id(), pattern(32, 1))
fdir := filepath.Join(dir, "fury")
os.MkdirAll(fdir, 0o755)
for _, size := range []int{0, 1, 65535, 65536, 65537, 131072, 200000} {
var buf bytes.Buffer
w, err := Encrypt(&buf, id.Recipient())
if err != nil {
t.Fatal(err)
}
w.Write(pattern(size, 0x5a))
if err := w.Close(); err != nil {
t.Fatal(err)
}
os.WriteFile(filepath.Join(fdir, "pattern-"+itoa(size)+".fury"), buf.Bytes(), 0o644)
}
// Header parser verdicts on valid and malformed headers.
var cases []headerCase
valid, _ := hex.DecodeString(wraps[0].Header)
add := func(b []byte) {
h, payload, err := format.Parse(bytes.NewReader(b))
c := headerCase{Input: hex.EncodeToString(b), Accept: err == nil}
if err == nil {
rest, _ := io.ReadAll(payload)
c.Rest = hex.EncodeToString(rest)
var again bytes.Buffer
h.Marshal(&again)
}
cases = append(cases, c)
}
for _, m := range headerMutations(valid) {
add(m)
}
write(t, filepath.Join(dir, "crypto.json"), map[string]any{
"provenance": "oracle/go at 4c1507c, internal/pqcrypt/oracle_vectors_test.go",
"frodo640shake": kemVectors(t, frodo, 8),
"mlkem768": kemVectors(t, ml, 8),
"hybrid": hybrids,
"wrap": wraps,
"header_cases": cases,
})
}
func itoa(n int) string { return string(appendInt(nil, n)) }
func appendInt(b []byte, n int) []byte {
if n >= 10 {
b = appendInt(b, n/10)
}
return append(b, byte('0'+n%10))
}
// headerMutations returns valid headers plus a range of malformed ones.
func headerMutations(valid []byte) [][]byte {
s := string(valid)
introEnd := bytes.IndexByte(valid, '\n') + 1
footer := bytes.LastIndex(valid, []byte("\n---")) + 1
lines := bytes.SplitAfter(valid, []byte("\n"))
join := func(ls [][]byte) []byte { return bytes.Join(ls, nil) }
var out [][]byte
push := func(b []byte) { out = append(out, append([]byte(nil), b...)) }
push(valid)
push(append(append([]byte(nil), valid...), []byte("payload\x00\x01")...))
push([]byte(""))
push([]byte("fury-encryption.org/v1\n--- " + s[len(s)-44:])) // no stanzas
push([]byte("fury-encryption.org/v2\n" + s[introEnd:])) // wrong intro
push([]byte("fury-encryption.org/v1" + s[introEnd:])) // intro without newline
push(valid[:footer]) // no footer
push(valid[:len(valid)-1]) // footer without newline
push(append(append([]byte(nil), valid[:len(valid)-1]...), '=', '\n')) // padded MAC
push([]byte(s[:footer] + "--- " + s[footer+4:])) // double space
push([]byte(s[:footer] + "---x " + s[footer+4:])) // bad footer prefix
push([]byte(s[:footer] + "--- " + s[footer+4:len(s)-2] + "\n")) // short MAC
push([]byte(s[:footer] + "--- " + s[footer+4:len(s)-1] + " x\n"))
// Stanza line edits.
stanza := lines[1]
for _, edit := range []string{"-> \n", "->\n", "-> x\n", "-> x\x7f\n", "-> x é\n", "-x a\n", "-> x y\n", "-> x\r\n"} {
ls := append([][]byte(nil), lines...)
ls[1] = []byte(edit)
push(join(ls))
_ = stanza
}
// Body edits: the body is the two lines after the stanza line.
body := 2
for _, edit := range []string{"\n", "AAAA\n", "AAAA=\n", "A\n", "AB\n", "AAB\n", "AA==\n", " AAAA\n", "AAAA\r\n",
string(bytes.Repeat([]byte("A"), 64)) + "\n", string(bytes.Repeat([]byte("A"), 68)) + "\n", string(bytes.Repeat([]byte("A"), 65)) + "\n"} {
ls := append([][]byte(nil), lines...)
ls[body] = []byte(edit)
push(join(ls))
}
{
// Drop the empty line that ends a 48-byte body: the stanza then runs
// into the footer.
ls := append([][]byte(nil), lines...)
ls = append(ls[:3], ls[4:]...)
push(join(ls))
}
{
// A second, foreign stanza ahead of the real one.
ls := append([][]byte(nil), lines[:1]...)
ls = append(ls, []byte("-> other-suite arg\n"), []byte("AAAA\n"))
ls = append(ls, lines[1:]...)
push(join(ls))
}
// The 1 MiB stanza and 16 MiB header limits are exercised by the Rust
// tests directly; vectors that size would bloat the repository.
return out
}
func write(t *testing.T, path string, v any) {
b, err := json.MarshalIndent(v, "", " ")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, append(b, '\n'), 0o644); err != nil {
t.Fatal(err)
}
}
+405
View File
@@ -0,0 +1,405 @@
// Copyright 2026 The fury Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//
// Package fury implements post-quantum file encryption. It provides a
// post-quantum-only file format, hybrid KEM key agreement, and a 256-bit file key.
//
// For most use cases, generate a [PQIdentity] with [GeneratePQIdentity], obtain
// its public [PQRecipient] with [PQIdentity.Recipient], and use the [Encrypt]
// and [Decrypt] functions.
//
// fury encrypted files are binary and not malleable. For encoding them as text,
// use the github.com/andydixon/fury/armor package.
//
// # Cryptographic agility
//
// Key agreement is provided by registered suites (see the suite registry).
// Each suite embeds its identifier in the file, so files remain decryptable as
// newer, stronger suites are added and made the default. The default suite is a
// hybrid of FrodoKEM-640 and ML-KEM-768: the file key stays secret unless both
// KEMs are broken.
//
// # Key management
//
// fury does not have a global keyring; generate dedicated keys per task. Public
// recipients are shared as "fury1..." strings (or, being large, as files), or
// referenced by a short "furyfp1..." fingerprint. Private identities are always
// encrypted, with passphrase recovery and optional external unlock slots; there
// is no API to serialise an unencrypted private key.
package pqcrypt
import (
"crypto/hmac"
"crypto/rand"
"errors"
"fmt"
"io"
"runtime"
"slices"
"sort"
"github.com/andydixon/vault/internal/format"
"github.com/andydixon/vault/internal/stream"
)
// An Identity is passed to [Decrypt] to unwrap an opaque file key from a
// recipient stanza. It can be for example a secret key like [HybridIdentity], a
// plugin, or a custom implementation.
type Identity interface {
// Unwrap must return an error wrapping [ErrIncorrectIdentity] if none of
// the recipient stanzas match the identity, any other error will be
// considered fatal.
//
// Most fury API users won't need to interact with this method directly, and
// should instead pass [Identity] implementations to [Decrypt].
Unwrap(stanzas []*Stanza) (fileKey []byte, err error)
}
// ErrIncorrectIdentity is returned by [Identity.Unwrap] if none of the
// recipient stanzas match the identity.
var ErrIncorrectIdentity = errors.New("incorrect identity for recipient block")
// A Recipient is passed to [Encrypt] to wrap an opaque file key to one or more
// recipient stanza(s). It can be for example a public key like [HybridRecipient],
// a plugin, or a custom implementation.
type Recipient interface {
// Most fury API users won't need to interact with this method directly, and
// should instead pass [Recipient] implementations to [Encrypt].
Wrap(fileKey []byte) ([]*Stanza, error)
}
// RecipientWithLabels can be optionally implemented by a [Recipient], in which
// case [Encrypt] will use WrapWithLabels instead of [Recipient.Wrap].
//
// Encrypt will succeed only if the labels returned by all the recipients
// (assuming the empty set for those that don't implement RecipientWithLabels)
// are the same.
//
// This can be used to ensure a recipient is only used with other recipients
// with equivalent properties (for example by setting a "postquantum" label) or
// to ensure a recipient is always used alone (by returning a random label, for
// example to preserve its authentication properties).
type RecipientWithLabels interface {
WrapWithLabels(fileKey []byte) (s []*Stanza, labels []string, err error)
}
// A Stanza is a section of the fury header that encapsulates the file key as
// encrypted to a specific recipient.
//
// Most fury API users won't need to interact with this type directly, and should
// instead pass [Recipient] implementations to [Encrypt] and [Identity]
// implementations to [Decrypt].
type Stanza struct {
Type string
Args []string
Body []byte
}
// fileKeySize is 32 bytes (256-bit) so the symmetric layer's post-quantum
// (Grover) security floor of ~128 bits matches the KEMs, rather than the 64-bit
// floor a 128-bit file key would impose. See suite.go and the payload STREAM.
const fileKeySize = 32
const streamNonceSize = 16
// maxAutoWorkers bounds the automatic worker count for parallel STREAM
// chunk encryption/decryption. ChaCha20-Poly1305 is memory-bandwidth-bound
// past a modest number of cores, so an unbounded runtime.NumCPU() on very
// large machines would just add goroutine/channel overhead for no benefit.
const maxAutoWorkers = 32
// MaxWorkers is the largest value [Options.Workers] will accept.
//
// The limit is set by what a worker actually costs, not by taste. Each worker
// may have two chunks in flight (see the stream package's pipeline depth), and
// a chunk is 64 KiB of plaintext plus its ciphertext buffer, so the pipeline
// holds roughly 256 KiB per worker: about 256 MiB at this limit, plus 1024
// goroutines and channels sized to match. That is already far past the point
// where throughput stops improving, and comfortably above the thread count of
// the largest machines currently shipping. A request beyond it is a
// configuration error rather than a tuning choice, so Fury rejects it instead
// of quietly attempting it or silently substituting another number.
const MaxWorkers = 1024
// Options configures [EncryptWithOptions] and [DecryptWithOptions] beyond
// the defaults used by [Encrypt] and [Decrypt].
type Options struct {
// Workers controls how many goroutines seal or open STREAM chunks in
// parallel. STREAM chunk nonces are a pure counter independent of chunk
// content, so parallelising this work never changes the resulting
// ciphertext or plaintext, only how fast it's produced.
//
// Valid values are 0 to [MaxWorkers]. Zero (the default) means
// automatic: runtime.NumCPU(), capped at a value past which
// ChaCha20-Poly1305 stops getting faster. One forces the sequential
// code path. A negative value, or one above MaxWorkers, is rejected.
Workers int
}
// ErrInvalidWorkers is returned when [Options.Workers] is outside 0 to
// [MaxWorkers].
var ErrInvalidWorkers = errors.New("fury: worker count out of range")
// Validate reports whether the options are usable. Every entry point that
// accepts [Options] calls it, so an out-of-range worker count is refused by
// the library itself and not only by the command-line tool.
func (o Options) Validate() error {
if o.Workers < 0 {
return fmt.Errorf("%w: %d is negative", ErrInvalidWorkers, o.Workers)
}
if o.Workers > MaxWorkers {
return fmt.Errorf("%w: %d exceeds the maximum of %d", ErrInvalidWorkers, o.Workers, MaxWorkers)
}
return nil
}
func (o Options) workers() int {
if o.Workers > 0 {
return o.Workers
}
if n := runtime.NumCPU(); n < maxAutoWorkers {
return n
}
return maxAutoWorkers
}
// encryptHdr wraps a file key to every recipient and authenticates the result.
//
// INVARIANT (no silent downgrade). All recipients must return identical label
// sets. A recipient that does not implement [RecipientWithLabels] contributes
// the empty set, so it can never be mixed with a "postquantum" recipient. Any
// future recipient type with weaker properties fails closed rather than
// quietly lowering the security of the whole file.
func encryptHdr(fileKey []byte, recipients ...Recipient) (*format.Header, error) {
if len(recipients) == 0 {
return nil, errors.New("no recipients specified")
}
hdr := &format.Header{}
var labels []string
for i, r := range recipients {
stanzas, l, err := wrapWithLabels(r, fileKey)
if err != nil {
return nil, fmt.Errorf("failed to wrap key for recipient #%d: %w", i, err)
}
sort.Strings(l)
if i == 0 {
labels = l
} else if !slices.Equal(labels, l) {
return nil, incompatibleLabelsError(labels, l)
}
for _, s := range stanzas {
hdr.Recipients = append(hdr.Recipients, (*format.Stanza)(s))
}
}
if mac, err := headerMAC(fileKey, hdr); err != nil {
return nil, fmt.Errorf("failed to compute header MAC: %v", err)
} else {
hdr.MAC = mac
}
return hdr, nil
}
// Encrypt encrypts a file to one or more recipients. Every recipient will be
// able to decrypt the file.
//
// Writes to the returned WriteCloser are encrypted and written to dst as a fury
// file. The caller must call Close on the WriteCloser when done for the last
// chunk to be encrypted and flushed to dst.
//
// The payload is sealed using multiple goroutines automatically; see
// [EncryptWithOptions] to control that.
func Encrypt(dst io.Writer, recipients ...Recipient) (io.WriteCloser, error) {
return EncryptWithOptions(dst, Options{}, recipients...)
}
// EncryptWithOptions is like [Encrypt], with explicit [Options].
//
// INVARIANT (fresh file key). Every call generates a new 32-byte file key
// from crypto/rand. Nothing derives a file key from the plaintext, the
// recipients or a counter, so no two encryptions share one.
func EncryptWithOptions(dst io.Writer, opts Options, recipients ...Recipient) (io.WriteCloser, error) {
if err := opts.Validate(); err != nil {
return nil, err
}
fileKey := make([]byte, fileKeySize)
if _, err := rand.Read(fileKey); err != nil {
return nil, fmt.Errorf("failed to generate file key: %w", err)
}
defer wipe(fileKey)
hdr, err := encryptHdr(fileKey, recipients...)
if err != nil {
return nil, err
}
if err := hdr.Marshal(dst); err != nil {
return nil, fmt.Errorf("failed to write header: %w", err)
}
nonce := make([]byte, streamNonceSize)
if _, err := rand.Read(nonce); err != nil {
return nil, fmt.Errorf("failed to generate nonce: %w", err)
}
if _, err := dst.Write(nonce); err != nil {
return nil, fmt.Errorf("failed to write nonce: %w", err)
}
key := streamKey(fileKey, nonce)
w, err := stream.NewParallelEncryptWriter(key, dst, opts.workers())
wipe(key)
return w, err
}
func wrapWithLabels(r Recipient, fileKey []byte) (s []*Stanza, labels []string, err error) {
if r, ok := r.(RecipientWithLabels); ok {
return r.WrapWithLabels(fileKey)
}
s, err = r.Wrap(fileKey)
return
}
func incompatibleLabelsError(l1, l2 []string) error {
hasPQ1 := slices.Contains(l1, "postquantum")
hasPQ2 := slices.Contains(l2, "postquantum")
if hasPQ1 != hasPQ2 {
return fmt.Errorf("incompatible recipients: can't mix post-quantum and classic recipients, or the file would be vulnerable to quantum computers")
}
return fmt.Errorf("incompatible recipients: %q and %q can't be mixed", l1, l2)
}
// NoIdentityMatchError is returned by [Decrypt] when none of the supplied
// identities match the encrypted file.
type NoIdentityMatchError struct {
// Errors is a slice of all the errors returned to Decrypt by the Unwrap
// calls it made. They all wrap [ErrIncorrectIdentity].
Errors []error
// StanzaTypes are the first argument of each recipient stanza in the
// encrypted file's header.
StanzaTypes []string
}
func (e *NoIdentityMatchError) Error() string {
if len(e.Errors) == 1 {
return "identity did not match any of the recipients: " + e.Errors[0].Error()
}
return "no identity matched any of the recipients"
}
// Decrypt decrypts a file encrypted to one or more identities.
// All identities will be tried until one successfully decrypts the file.
// Native, non-interactive identities are tried before any other identities.
//
// Decrypt returns a Reader reading the decrypted plaintext of the fury file read
// from src. If no identity matches the encrypted file, the returned error will
// be of type [NoIdentityMatchError].
//
// The payload is opened using multiple goroutines automatically; see
// [DecryptWithOptions] to control that.
func Decrypt(src io.Reader, identities ...Identity) (io.Reader, error) {
return DecryptWithOptions(src, Options{}, identities...)
}
// DecryptWithOptions is like [Decrypt], with explicit [Options].
func DecryptWithOptions(src io.Reader, opts Options, identities ...Identity) (io.Reader, error) {
if err := opts.Validate(); err != nil {
return nil, err
}
hdr, payload, err := format.Parse(src)
if err != nil {
return nil, fmt.Errorf("failed to read header: %w", err)
}
fileKey, err := decryptHdr(hdr, identities...)
if err != nil {
return nil, err
}
defer wipe(fileKey)
nonce := make([]byte, streamNonceSize)
if _, err := io.ReadFull(payload, nonce); err != nil {
return nil, fmt.Errorf("failed to read nonce: %w", err)
}
key := streamKey(fileKey, nonce)
r, err := stream.NewParallelDecryptReader(key, payload, opts.workers())
wipe(key)
return r, err
}
func decryptHdr(hdr *format.Header, identities ...Identity) ([]byte, error) {
if len(identities) == 0 {
return nil, errors.New("no identities specified")
}
slices.SortStableFunc(identities, func(a, b Identity) int {
var aIsNative, bIsNative bool
switch a.(type) {
case *PQIdentity:
aIsNative = true
}
switch b.(type) {
case *PQIdentity:
bIsNative = true
}
if aIsNative && !bIsNative {
return -1
}
if !aIsNative && bIsNative {
return 1
}
return 0
})
stanzas := make([]*Stanza, 0, len(hdr.Recipients))
errNoMatch := &NoIdentityMatchError{}
for _, s := range hdr.Recipients {
errNoMatch.StanzaTypes = append(errNoMatch.StanzaTypes, s.Type)
stanzas = append(stanzas, (*Stanza)(s))
}
var fileKey []byte
// The unwrapped key is copied out below; clear this copy on every exit
// path so it does not outlive the call. A closure is required: fileKey is
// assigned inside the loop, and a bare defer would capture today's nil.
defer func() { wipe(fileKey) }()
for _, id := range identities {
var err error
fileKey, err = id.Unwrap(stanzas)
if errors.Is(err, ErrIncorrectIdentity) {
errNoMatch.Errors = append(errNoMatch.Errors, err)
continue
}
if err != nil {
return nil, err
}
break
}
if fileKey == nil {
return nil, errNoMatch
}
if mac, err := headerMAC(fileKey, hdr); err != nil {
return nil, fmt.Errorf("failed to compute header MAC: %v", err)
} else if !hmac.Equal(mac, hdr.MAC) {
return nil, errors.New("bad header MAC")
}
return append([]byte(nil), fileKey...), nil
}
// multiUnwrap is a helper that implements Identity.Unwrap in terms of a
// function that unwraps a single recipient stanza.
func multiUnwrap(unwrap func(*Stanza) ([]byte, error), stanzas []*Stanza) ([]byte, error) {
for _, s := range stanzas {
fileKey, err := unwrap(s)
if errors.Is(err, ErrIncorrectIdentity) {
// If we ever start returning something interesting wrapping
// ErrIncorrectIdentity, we should let it make its way up through
// Decrypt into NoIdentityMatchError.Errors.
continue
}
if err != nil {
return nil, err
}
return fileKey, nil
}
return nil, ErrIncorrectIdentity
}
@@ -0,0 +1,70 @@
// Copyright 2026 The fury Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package pqcrypt
import (
"bytes"
"crypto/rand"
"fmt"
"io"
"testing"
)
// TestEncryptWithOptionsRoundTrip exercises the parallel STREAM path end to
// end, at a size spanning many chunks, across a few explicit worker counts
// (including 1, which forces the original sequential code path).
func TestEncryptWithOptionsRoundTrip(t *testing.T) {
id, err := GeneratePQIdentity()
if err != nil {
t.Fatal(err)
}
r := id.Recipient()
plaintext := make([]byte, 4<<20) // 4 MiB: many STREAM chunks
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
for _, workers := range []int{1, 2, 8} {
t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) {
buf := &bytes.Buffer{}
w, err := EncryptWithOptions(buf, Options{Workers: workers}, r)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
out, err := DecryptWithOptions(bytes.NewReader(buf.Bytes()), Options{Workers: workers}, id)
if err != nil {
t.Fatal(err)
}
got, err := io.ReadAll(out)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, plaintext) {
t.Fatal("round-trip mismatch")
}
})
}
}
func TestOptionsWorkers(t *testing.T) {
if got := (Options{}).workers(); got <= 0 {
t.Fatalf("default workers = %d, want > 0", got)
}
if got := (Options{Workers: 3}).workers(); got != 3 {
t.Fatalf("explicit workers = %d, want 3", got)
}
// An explicit request is honoured even above the automatic cap; only the
// zero-value automatic path is capped.
if got := (Options{Workers: 1000}).workers(); got != 1000 {
t.Fatalf("explicit workers = %d, want 1000", got)
}
}
+111
View File
@@ -0,0 +1,111 @@
// SPDX-License-Identifier: GPL-3.0-or-later
package pqcrypt
import (
"crypto/hmac"
"crypto/sha256"
"errors"
"io"
"github.com/andydixon/vault/internal/format"
"golang.org/x/crypto/chacha20poly1305"
"golang.org/x/crypto/hkdf"
)
func wipe(b []byte) {
for i := range b {
b[i] = 0
}
}
// aeadEncrypt encrypts a message with a one-time key.
//
// INVARIANT (fixed nonce). The all-zero nonce below is safe only because
// every key reaching this function is a fresh, unique, single-use derived
// KEK. It is not a shortcut and must not be reused anywhere that condition
// does not hold: repeating a (key, nonce) pair under ChaCha20-Poly1305
// destroys both confidentiality and authenticity.
//
// The condition holds today because aeadEncrypt has exactly one caller, the
// default suite's wrap, whose KEK is
// HKDF-SHA-256(ss1 || ss2, info = label || ct1 || ct2). Both KEM
// encapsulations are performed freshly for each wrap, so ss1, ss2, ct1 and
// ct2 differ for every call even when the same file key is wrapped to the
// same recipient twice. Any new caller must establish the same property, or
// carry its own nonce.
//
// For the code that encrypts the actual payload, which uses a counter nonce
// rather than a fixed one, see the
// github.com/andydixon/vault/internal/stream package.
func aeadEncrypt(key, plaintext []byte) ([]byte, error) {
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
nonce := make([]byte, chacha20poly1305.NonceSize)
return aead.Seal(nil, nonce, plaintext, nil), nil
}
var errIncorrectCiphertextSize = errors.New("encrypted value has unexpected length")
// aeadDecrypt decrypts a message of an expected fixed size.
//
// The message size is limited to mitigate multi-key attacks, where a ciphertext
// can be crafted that decrypts successfully under multiple keys. Short
// ciphertexts can only target two keys, which has limited impact.
func aeadDecrypt(key []byte, size int, ciphertext []byte) ([]byte, error) {
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
if len(ciphertext) != size+aead.Overhead() {
return nil, errIncorrectCiphertextSize
}
nonce := make([]byte, chacha20poly1305.NonceSize)
return aead.Open(nil, nonce, ciphertext, nil)
}
// headerMAC computes the header authentication tag.
//
// INVARIANT (header authentication). The MAC covers the complete header
// except the MAC line itself: the intro line, every recipient stanza in
// order, and the footer prefix. Nothing that a decryptor acts upon is left
// unauthenticated, so a stanza cannot be added, removed, reordered or edited
// without detection by anyone who can recover the file key.
//
// INVARIANT (domain separation). The MAC key and the payload key are both
// HKDF-SHA-256 expansions of the same file key under distinct info strings
// ("header" and "payload"), so neither can be substituted for the other.
func headerMAC(fileKey []byte, hdr *format.Header) ([]byte, error) {
h := hkdf.New(sha256.New, fileKey, nil, []byte("header"))
hmacKey := make([]byte, 32)
if _, err := io.ReadFull(h, hmacKey); err != nil {
wipe(hmacKey)
return nil, err
}
// The MAC key is derived per call and is of no use once the hash is
// keyed, so it does not need to outlive this function.
defer wipe(hmacKey)
hh := hmac.New(sha256.New, hmacKey)
if err := hdr.MarshalWithoutMAC(hh); err != nil {
return nil, err
}
return hh.Sum(nil), nil
}
// streamKey derives the payload key.
//
// INVARIANT (payload key uniqueness). The 16-byte file nonce is fresh
// randomness written into every file and used here as the HKDF salt, so two
// encryptions never share a payload key even if a file key were somehow
// repeated. Combined with the STREAM counter nonce, no (key, nonce) pair is
// ever reused.
func streamKey(fileKey, nonce []byte) []byte {
h := hkdf.New(sha256.New, fileKey, nonce, []byte("payload"))
streamKey := make([]byte, chacha20poly1305.KeySize)
if _, err := io.ReadFull(h, streamKey); err != nil {
panic("fury: internal error: failed to read from HKDF: " + err.Error())
}
return streamKey
}
+236
View File
@@ -0,0 +1,236 @@
// Copyright 2026 The fury Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package pqcrypt
import (
"crypto/rand"
"errors"
"fmt"
)
// A pqSuite is a registered post-quantum KEM suite. Each suite wraps the file
// key to a recipient and unwraps it with the corresponding identity. Suites are
// self-describing: their stanza tag is written into the header, so a file made
// with one suite keeps decrypting even after newer suites are added and made the
// default. This is the "use the best now, add better later, still decrypt older"
// property.
//
// Recipient public keys and identity private keys are opaque byte slices whose
// layout is defined by the suite. Identities are derived deterministically from
// a compact master seed so they can be stored and passphrase-wrapped cheaply.
type pqSuite interface {
// id is a stable numeric identifier embedded in recipient encodings.
id() uint16
// tag is the header stanza Type for this suite.
tag() string
// rank orders suites when picking a default; higher wins.
rank() int
// seedSize is the length of the master seed accepted by deriveKeypair.
seedSize() int
// deriveKeypair deterministically expands a master seed into a suite public
// key and private key.
deriveKeypair(seed []byte) (pub, priv []byte, err error)
// wrap encapsulates fileKey to pub, returning one or more header stanzas.
wrap(pub, fileKey []byte) ([]*Stanza, error)
// unwrap recovers fileKey from a stanza using priv. It must return an error
// wrapping ErrIncorrectIdentity if the stanza does not match.
unwrap(priv []byte, s *Stanza) ([]byte, error)
// publicKeySize is the expected length of a suite public key.
publicKeySize() int
}
var (
pqSuitesByTag = map[string]pqSuite{}
pqSuitesByID = map[uint16]pqSuite{}
)
// registerSuite adds a suite to the registry. It is called from suite files'
// init functions.
func registerSuite(s pqSuite) {
if _, ok := pqSuitesByTag[s.tag()]; ok {
panic("fury: duplicate suite tag " + s.tag())
}
if _, ok := pqSuitesByID[s.id()]; ok {
panic(fmt.Sprintf("fury: duplicate suite id %#04x", s.id()))
}
pqSuitesByTag[s.tag()] = s
pqSuitesByID[s.id()] = s
}
// defaultSuite returns the highest-ranked registered suite, used for key
// generation and encryption.
func defaultSuite() pqSuite {
var best pqSuite
for _, s := range pqSuitesByTag {
if best == nil || s.rank() > best.rank() {
best = s
}
}
if best == nil {
panic("fury: no PQ suites registered")
}
return best
}
// PQRecipient is a post-quantum public key. Messages encrypted to it can be
// decrypted with the corresponding [PQIdentity].
//
// It is anonymous: an observer can't tell from a message alone whether it is
// encrypted to a given recipient.
type PQRecipient struct {
suite pqSuite
pub []byte
}
var _ Recipient = &PQRecipient{}
var _ RecipientWithLabels = &PQRecipient{}
func (r *PQRecipient) Wrap(fileKey []byte) ([]*Stanza, error) {
return r.suite.wrap(r.pub, fileKey)
}
// WrapWithLabels returns a "postquantum" label so a PQRecipient can only be
// combined with other post-quantum recipients, never with a recipient type that
// would defeat its security.
func (r *PQRecipient) WrapWithLabels(fileKey []byte) ([]*Stanza, []string, error) {
s, err := r.Wrap(fileKey)
if err != nil {
return nil, nil, err
}
return s, []string{"postquantum"}, nil
}
// SuiteID returns the numeric identifier of the recipient's suite.
func (r *PQRecipient) SuiteID() uint16 { return r.suite.id() }
// Bytes returns the raw suite public key.
func (r *PQRecipient) Bytes() []byte { return append([]byte(nil), r.pub...) }
// PQIdentity is a post-quantum private key, derived from a compact master seed,
// which can decrypt messages encrypted to the corresponding [PQRecipient].
type PQIdentity struct {
suite pqSuite
seed []byte
pub []byte
priv []byte
}
var _ Identity = &PQIdentity{}
// GeneratePQIdentity randomly generates a new identity using the default suite.
func GeneratePQIdentity() (*PQIdentity, error) {
s := defaultSuite()
seed := make([]byte, s.seedSize())
if _, err := rand.Read(seed); err != nil {
return nil, fmt.Errorf("failed to generate seed: %w", err)
}
id, err := newPQIdentityFromSeed(s, seed)
wipe(seed)
return id, err
}
// newPQIdentityFromSeed derives an identity from a suite and master seed.
func newPQIdentityFromSeed(s pqSuite, seed []byte) (*PQIdentity, error) {
if len(seed) != s.seedSize() {
return nil, fmt.Errorf("invalid seed size %d for suite %s", len(seed), s.tag())
}
seed = append([]byte(nil), seed...)
pub, priv, err := s.deriveKeypair(seed)
if err != nil {
return nil, fmt.Errorf("failed to derive keypair: %w", err)
}
return &PQIdentity{suite: s, seed: seed, pub: pub, priv: priv}, nil
}
// usable reports whether the identity still holds its secret material. A
// closed identity, and one that was never fully initialised, both report
// false rather than panicking on a nil slice.
func (i *PQIdentity) usable() bool {
return i != nil && i.suite != nil && len(i.seed) == i.suite.seedSize() && len(i.priv) != 0
}
// ErrIdentityClosed is returned by operations on an identity whose secret
// material has been cleared by Close.
var ErrIdentityClosed = errors.New("fury identity is closed or uninitialised")
// Close overwrites the secret material this identity owns: its master seed
// and its expanded private key. Repeated calls are safe, and a partially
// initialised identity closes without panicking. The public key is retained,
// so [PQIdentity.Recipient] keeps working on a closed identity.
//
// Explicit overwriting shortens the lifetime of buffers Fury knows about. It
// cannot guarantee removal from physical memory: the Go compiler and runtime
// may have copied a value during allocation, garbage collection, stack growth
// or escape analysis, and Fury has no way to find or clear those copies. Treat
// this as defence in depth, not as erasure.
func (i *PQIdentity) Close() {
if i == nil {
return
}
wipe(i.seed)
wipe(i.priv)
i.seed, i.priv = nil, nil
}
func (i *PQIdentity) Unwrap(stanzas []*Stanza) ([]byte, error) {
if !i.usable() {
return nil, ErrIdentityClosed
}
return multiUnwrap(i.unwrap, stanzas)
}
func (i *PQIdentity) unwrap(s *Stanza) ([]byte, error) {
su, ok := pqSuitesByTag[s.Type]
if !ok || su.tag() != i.suite.tag() {
return nil, ErrIncorrectIdentity
}
fileKey, err := su.unwrap(i.priv, s)
if err != nil {
return nil, err
}
// Enforce the file key length centrally, so no future suite can feed a
// wrong-sized key into the payload STREAM by forgetting its own check.
if len(fileKey) != fileKeySize {
return nil, fmt.Errorf("suite %s returned a file key of %d bytes, want %d", su.tag(), len(fileKey), fileKeySize)
}
return fileKey, nil
}
// Recipient returns the public [PQRecipient] corresponding to i. It remains
// valid after [PQIdentity.Close], because a public key is not secret.
func (i *PQIdentity) Recipient() *PQRecipient {
return &PQRecipient{suite: i.suite, pub: append([]byte(nil), i.pub...)}
}
// Seed returns a copy of the compact master seed, or nil if the identity is
// closed.
//
// The returned slice is secret-equivalent: it regenerates both the private and
// the public key, so anything that can read it can decrypt everything the
// identity can. It must never be shared as a recipient, written unencrypted,
// logged, or passed to a subprocess.
//
// The copy is deliberate: it is never an alias into the identity's own buffer,
// so closing the identity does not clear it. From the moment it is returned it
// is the caller's responsibility to wipe it when finished and to keep its
// lifetime short.
func (i *PQIdentity) Seed() []byte {
if !i.usable() {
return nil
}
return append([]byte(nil), i.seed...)
}
// SuiteID returns the numeric identifier of the identity's suite.
func (i *PQIdentity) SuiteID() uint16 { return i.suite.id() }
// NewPQIdentityFromSeed derives an identity of the given suite from its master
// seed, as returned by [PQIdentity.Seed].
func NewPQIdentityFromSeed(suiteID uint16, seed []byte) (*PQIdentity, error) {
s, ok := pqSuitesByID[suiteID]
if !ok {
return nil, fmt.Errorf("unknown suite id %#04x", suiteID)
}
return newPQIdentityFromSeed(s, seed)
}
@@ -0,0 +1,215 @@
// Copyright 2026 The fury Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package pqcrypt
import (
"crypto/sha256"
"errors"
"fmt"
"io"
"slices"
"github.com/andydixon/vault/internal/format"
"github.com/cloudflare/circl/kem"
"github.com/cloudflare/circl/kem/frodo/frodo640shake"
"github.com/cloudflare/circl/kem/mlkem/mlkem768"
"golang.org/x/crypto/hkdf"
)
// frodoMLKEM768 is the default post-quantum suite: a hybrid of FrodoKEM-640
// (unstructured-lattice) and ML-KEM-768 (module-lattice). The file key stays
// secret unless both KEMs are broken. It is pure Go, so fury keeps building as a
// static binary with no cgo. (A code-based suite such as HQC or Classic McEliece
// via liboqs can be registered later with a higher rank; see suite.go.)
type frodoMLKEM768 struct{}
func init() { registerSuite(frodoMLKEM768{}) }
func (frodoMLKEM768) id() uint16 { return 0x0001 }
func (frodoMLKEM768) tag() string { return "frodo640shake+mlkem768" }
func (frodoMLKEM768) rank() int { return 10 }
func (frodoMLKEM768) seedSize() int { return 32 }
func (frodoMLKEM768) schemes() (frodo, ml kem.Scheme) {
return frodo640shake.Scheme(), mlkem768.Scheme()
}
func (s frodoMLKEM768) publicKeySize() int {
frodo, ml := s.schemes()
return frodo.PublicKeySize() + ml.PublicKeySize()
}
// deriveKeypair expands a 32-byte master seed into per-KEM seeds via HKDF, then
// derives each keypair deterministically. The public key is frodoPub||mlkemPub
// and the private key is frodoPriv||mlkemPriv.
func (s frodoMLKEM768) deriveKeypair(seed []byte) (pub, priv []byte, err error) {
frodo, ml := s.schemes()
r := hkdf.New(sha256.New, seed, nil, []byte("fury/v1 seed-expand "+s.tag()))
frodoSeed := make([]byte, frodo.SeedSize())
mlSeed := make([]byte, ml.SeedSize())
// The per-KEM seeds regenerate the private keys, so they are as sensitive
// as the master seed and are cleared as soon as the keys are derived.
defer wipe(frodoSeed)
defer wipe(mlSeed)
if _, err := io.ReadFull(r, frodoSeed); err != nil {
return nil, nil, err
}
if _, err := io.ReadFull(r, mlSeed); err != nil {
return nil, nil, err
}
fpk, fsk := frodo.DeriveKeyPair(frodoSeed)
mpk, msk := ml.DeriveKeyPair(mlSeed)
fpkb, err := fpk.MarshalBinary()
if err != nil {
return nil, nil, err
}
mpkb, err := mpk.MarshalBinary()
if err != nil {
return nil, nil, err
}
fskb, err := fsk.MarshalBinary()
if err != nil {
return nil, nil, err
}
defer wipe(fskb)
mskb, err := msk.MarshalBinary()
if err != nil {
return nil, nil, err
}
defer wipe(mskb)
// fpkb and mpkb are public key bytes; wiping them would be theatre.
return slices.Concat(fpkb, mpkb), slices.Concat(fskb, mskb), nil
}
// wrap encapsulates to both KEMs and seals the file key under the combined
// KEK.
//
// INVARIANT (fresh encapsulation). Both Encapsulate calls draw fresh
// randomness on every invocation. Wrapping the same file key to the same
// recipient twice therefore produces two independent KEKs, never a repeat.
func (s frodoMLKEM768) wrap(pub, fileKey []byte) ([]*Stanza, error) {
frodo, ml := s.schemes()
if len(pub) != s.publicKeySize() {
return nil, fmt.Errorf("invalid %s public key size", s.tag())
}
fpk, err := frodo.UnmarshalBinaryPublicKey(pub[:frodo.PublicKeySize()])
if err != nil {
return nil, fmt.Errorf("invalid FrodoKEM public key: %v", err)
}
mpk, err := ml.UnmarshalBinaryPublicKey(pub[frodo.PublicKeySize():])
if err != nil {
return nil, fmt.Errorf("invalid ML-KEM public key: %v", err)
}
ct1, ss1, err := frodo.Encapsulate(fpk)
if err != nil {
return nil, fmt.Errorf("FrodoKEM encapsulation failed: %v", err)
}
ct2, ss2, err := ml.Encapsulate(mpk)
if err != nil {
return nil, fmt.Errorf("ML-KEM encapsulation failed: %v", err)
}
kek := s.deriveKEK(ss1, ss2, ct1, ct2)
defer wipe(ss1)
defer wipe(ss2)
defer wipe(kek)
body, err := aeadEncrypt(kek, fileKey)
if err != nil {
return nil, fmt.Errorf("failed to wrap file key: %v", err)
}
return []*Stanza{{
Type: s.tag(),
Args: []string{format.EncodeToString(ct1), format.EncodeToString(ct2)},
Body: body,
}}, nil
}
func (s frodoMLKEM768) unwrap(priv []byte, st *Stanza) ([]byte, error) {
frodo, ml := s.schemes()
if st.Type != s.tag() {
return nil, ErrIncorrectIdentity
}
if len(st.Args) != 2 {
return nil, fmt.Errorf("invalid %s stanza: expected 2 arguments", s.tag())
}
ct1, err := format.DecodeString(st.Args[0])
if err != nil {
return nil, fmt.Errorf("invalid FrodoKEM ciphertext: %v", err)
}
ct2, err := format.DecodeString(st.Args[1])
if err != nil {
return nil, fmt.Errorf("invalid ML-KEM ciphertext: %v", err)
}
if len(ct1) != frodo.CiphertextSize() || len(ct2) != ml.CiphertextSize() {
return nil, errIncorrectCiphertextSize
}
if len(priv) != frodo.PrivateKeySize()+ml.PrivateKeySize() {
return nil, errors.New("invalid private key size")
}
fsk, err := frodo.UnmarshalBinaryPrivateKey(priv[:frodo.PrivateKeySize()])
if err != nil {
return nil, fmt.Errorf("invalid FrodoKEM private key: %v", err)
}
// priv itself belongs to the caller's identity and is cleared by
// PQIdentity.Close; the parsed key objects are owned by CIRCL and expose
// no buffer Fury can clear.
msk, err := ml.UnmarshalBinaryPrivateKey(priv[frodo.PrivateKeySize():])
if err != nil {
return nil, fmt.Errorf("invalid ML-KEM private key: %v", err)
}
ss1, err := frodo.Decapsulate(fsk, ct1)
if err != nil {
return nil, fmt.Errorf("FrodoKEM decapsulation failed: %v", err)
}
ss2, err := ml.Decapsulate(msk, ct2)
if err != nil {
return nil, fmt.Errorf("ML-KEM decapsulation failed: %v", err)
}
// Both KEMs use implicit rejection, so a mismatched key yields a wrong
// shared secret rather than an error; detection happens here at aeadDecrypt.
kek := s.deriveKEK(ss1, ss2, ct1, ct2)
defer wipe(ss1)
defer wipe(ss2)
defer wipe(kek)
fileKey, err := aeadDecrypt(kek, fileKeySize, st.Body)
if err != nil {
return nil, ErrIncorrectIdentity
}
return fileKey, nil
}
// deriveKEK combines the two KEM shared secrets and both ciphertexts into a
// 256-bit key-encryption key.
//
// INVARIANT (both secrets contribute). ss1 and ss2 are concatenated as the
// HKDF input keying material, so the KEK stays unpredictable unless both KEMs
// are broken for the same target. Dropping either would silently reduce the
// construction to a single KEM.
//
// INVARIANT (transcript binding). Both ciphertexts are bound into the HKDF
// info string, preventing a mix-and-match of encapsulations across the two
// KEMs: an attacker cannot pair ct1 from one stanza with ct2 from another and
// obtain a KEK either party would compute.
//
// INVARIANT (single use). Because both encapsulations are performed freshly
// for every wrap, the resulting KEK is unique to that wrap. That is what
// makes the fixed AEAD nonce in aeadEncrypt safe; see primitives.go.
func (s frodoMLKEM768) deriveKEK(ss1, ss2, ct1, ct2 []byte) []byte {
secret := slices.Concat(ss1, ss2)
defer wipe(secret)
info := slices.Concat([]byte("fury/v1 kek "+s.tag()), ct1, ct2)
r := hkdf.New(sha256.New, secret, nil, info)
kek := make([]byte, 32)
if _, err := io.ReadFull(r, kek); err != nil {
panic("fury: internal error: failed to read from HKDF: " + err.Error())
}
return kek
}
+135
View File
@@ -0,0 +1,135 @@
// Copyright 2026 The fury Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package pqcrypt
import (
"bytes"
"io"
"testing"
)
// TestPQRoundTrip exercises the full Encrypt -> Decrypt path through the default
// post-quantum suite with a 256-bit file key.
func TestPQRoundTrip(t *testing.T) {
id, err := GeneratePQIdentity()
if err != nil {
t.Fatal(err)
}
r := id.Recipient()
plaintext := []byte("fury: post-quantum only, hard g like GIF")
buf := &bytes.Buffer{}
w, err := Encrypt(buf, r)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
out, err := Decrypt(bytes.NewReader(buf.Bytes()), id)
if err != nil {
t.Fatal(err)
}
got, err := io.ReadAll(out)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, plaintext) {
t.Fatalf("round-trip mismatch: got %q want %q", got, plaintext)
}
}
func TestPQFileKeyIs256Bit(t *testing.T) {
if fileKeySize != 32 {
t.Fatalf("fileKeySize = %d, want 32 (256-bit)", fileKeySize)
}
}
// TestPQWrongIdentity confirms a different identity cannot decrypt.
func TestPQWrongIdentity(t *testing.T) {
id, _ := GeneratePQIdentity()
other, _ := GeneratePQIdentity()
buf := &bytes.Buffer{}
w, err := Encrypt(buf, id.Recipient())
if err != nil {
t.Fatal(err)
}
w.Write([]byte("secret"))
w.Close()
if _, err := Decrypt(bytes.NewReader(buf.Bytes()), other); err == nil {
t.Fatal("decryption with wrong identity unexpectedly succeeded")
}
}
// TestSuiteRegistry checks the default suite is registered and self-describing.
func TestSuiteRegistry(t *testing.T) {
s := defaultSuite()
if s.tag() != "frodo640shake+mlkem768" {
t.Fatalf("unexpected default suite %q", s.tag())
}
if pqSuitesByID[s.id()] != s {
t.Fatal("suite not registered by id")
}
if pqSuitesByTag[s.tag()] != s {
t.Fatal("suite not registered by tag")
}
}
// TestSeedDeterministic confirms an identity round-trips through its seed, which
// is what makes compact passphrase-wrapped identities possible.
func TestSeedDeterministic(t *testing.T) {
id, err := GeneratePQIdentity()
if err != nil {
t.Fatal(err)
}
id2, err := newPQIdentityFromSeed(defaultSuite(), id.Seed())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(id.pub, id2.pub) || !bytes.Equal(id.priv, id2.priv) {
t.Fatal("identity not deterministic from seed")
}
}
func TestIdentityAndRecipientAccessorsReturnCopies(t *testing.T) {
id, err := GeneratePQIdentity()
if err != nil {
t.Fatal(err)
}
seed := id.Seed()
seed[0] ^= 0xff
if id.seed[0] == seed[0] {
t.Fatal("Seed returned mutable identity internals")
}
inputSeed := id.Seed()
id2, err := newPQIdentityFromSeed(defaultSuite(), inputSeed)
if err != nil {
t.Fatal(err)
}
inputSeed[0] ^= 0xff
if id2.seed[0] == inputSeed[0] {
t.Fatal("newPQIdentityFromSeed retained caller-owned seed slice")
}
recipient := id.Recipient()
pub := recipient.Bytes()
pub[0] ^= 0xff
if recipient.pub[0] == pub[0] {
t.Fatal("Bytes returned mutable recipient internals")
}
recipient.pub[0] ^= 0xff
if id.pub[0] == recipient.pub[0] {
t.Fatal("Recipient returned mutable identity public-key internals")
}
}
@@ -0,0 +1,65 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// Test-only vector generator for the Rust port; see oracle/gen-vectors.sh.
package stream
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"io"
"os"
"path/filepath"
"testing"
)
func TestOracleVectors(t *testing.T) {
dir := os.Getenv("VAULT_ORACLE_OUT")
if dir == "" {
t.Skip("set VAULT_ORACLE_OUT to write vectors")
}
key := make([]byte, 32)
for i := range key {
key[i] = byte(i + 1)
}
type vec struct {
Size int `json:"size"`
CtLen int `json:"ct_len"`
CtSHA256 string `json:"ct_sha256"`
CtHex string `json:"ct_hex,omitempty"`
}
var vs []vec
for _, size := range []int{0, 1, 15, 16, 17, ChunkSize - 1, ChunkSize, ChunkSize + 1, 2 * ChunkSize, 2*ChunkSize + 1, 3*ChunkSize - 16, 300000} {
pt := make([]byte, size)
for i := range pt {
pt[i] = byte(i*13 + 7)
}
var buf bytes.Buffer
w, _ := NewEncryptWriter(key, &buf)
w.Write(pt)
w.Close()
// The reader-side encryptor must agree.
r, _ := NewEncryptReader(key, bytes.NewReader(pt))
alt, _ := io.ReadAll(r)
if !bytes.Equal(alt, buf.Bytes()) {
t.Fatal("EncryptReader disagrees with EncryptWriter")
}
s := sha256.Sum256(buf.Bytes())
v := vec{Size: size, CtLen: buf.Len(), CtSHA256: hex.EncodeToString(s[:])}
if size <= 17 {
v.CtHex = hex.EncodeToString(buf.Bytes())
}
vs = append(vs, v)
}
b, _ := json.MarshalIndent(map[string]any{
"provenance": "oracle/go at 4c1507c, internal/stream/oracle_vectors_test.go",
"key": hex.EncodeToString(key),
"plaintext": "byte(i*13+7)",
"vectors": vs,
}, "", " ")
if err := os.WriteFile(filepath.Join(dir, "stream.json"), append(b, '\n'), 0o644); err != nil {
t.Fatal(err)
}
}
@@ -0,0 +1,271 @@
// Copyright 2026 The fury Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package stream
import (
"crypto/cipher"
"errors"
"fmt"
"io"
"sync"
"golang.org/x/crypto/chacha20poly1305"
)
// NewParallelDecryptReader is equivalent to [NewDecryptReader], except that
// chunks are opened by a pool of worker goroutines instead of inline on the
// calling goroutine.
//
// The sequential DecryptReader disambiguates a full-length final chunk by
// trying to open it with the non-last nonce first and, on authentication
// failure, retrying with the last-chunk nonce. That trick is inherently
// sequential (try, then fall back), so this reader instead keeps a single
// raw chunk of read-ahead: the one goroutine that reads from src (reads must
// stay sequential, there is only one underlying stream) always reads one
// chunk further than it has dispatched, so by the time chunk i is handed to
// the worker pool, whether it is the last chunk is already known for
// certain from the shape of the next read. Each worker then does exactly
// one Open call, with a predetermined nonce, no retry.
//
// For well-formed files this produces byte-for-byte identical plaintext and
// the same errors as NewDecryptReader. For adversarially malformed files,
// the accept/reject verdict is always the same, but the exact error message
// in some corner cases (a corrupted chunk that happens to be full-length)
// may differ, because the classification strategy differs; see the package
// tests for the cases this was checked against.
//
// workers <= 1 returns a plain [NewDecryptReader] with no pipeline overhead.
//
// The returned Reader starts background goroutines that run until the
// stream is fully consumed or a fatal error occurs; abandoning it before
// reading to EOF leaks those goroutines (bounded to a small multiple of
// workers) until they unblock or the process exits.
func NewParallelDecryptReader(key []byte, src io.Reader, workers int) (io.Reader, error) {
if workers <= 1 {
return NewDecryptReader(key, src)
}
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
depth := workers * pipelineDepthPerWorker
r := &parallelDecryptReader{
a: aead,
src: src,
jobs: make(chan decJob, depth),
order: make(chan chan decResult, depth),
}
r.wg.Add(workers)
for i := 0; i < workers; i++ {
go r.work()
}
go r.pump()
return r, nil
}
type decJob struct {
nonce [chacha20poly1305.NonceSize]byte
ciphertext []byte
result chan decResult
}
// decResult is either a successfully decrypted chunk (err == nil) or a
// terminal event: a decryption failure, a raw read error, or a trailing-data
// violation. There is no explicit "clean EOF" value: a clean end of stream
// is represented by the order channel closing with nothing left to receive.
type decResult struct {
plaintext []byte
err error
}
type parallelDecryptReader struct {
a cipher.AEAD
src io.Reader
jobs chan decJob
order chan chan decResult
wg sync.WaitGroup
unread []byte
err error
}
func (r *parallelDecryptReader) Read(p []byte) (int, error) {
if len(r.unread) > 0 {
n := copy(p, r.unread)
r.unread = r.unread[n:]
return n, nil
}
if r.err != nil {
return 0, r.err
}
if len(p) == 0 {
return 0, nil
}
ch, ok := <-r.order
if !ok {
r.err = io.EOF
return 0, r.err
}
res := <-ch
if res.err != nil {
r.err = res.err
return 0, r.err
}
r.unread = res.plaintext
n := copy(p, r.unread)
r.unread = r.unread[n:]
return n, nil
}
// dispatch hands a raw chunk, whose classification is already known for
// certain, to the worker pool.
func (r *parallelDecryptReader) dispatch(index int64, ciphertext []byte, last bool) {
nonce := nonceForChunk(index)
if last {
setLastChunkFlag(nonce)
}
ch := make(chan decResult, 1)
r.order <- ch
r.jobs <- decJob{nonce: *nonce, ciphertext: ciphertext, result: ch}
}
// emitDirect enqueues a terminal result produced by the pump goroutine
// itself (a read error, a malformed-stream error, or a trailing-data
// violation), without involving the worker pool.
func (r *parallelDecryptReader) emitDirect(res decResult) {
ch := make(chan decResult, 1)
r.order <- ch
ch <- res
}
// work opens dispatched chunks. Unlike the sequential reader, there is no
// retry here: the pump goroutine has already determined, from the shape of
// the raw stream, whether each chunk is the last one.
func (r *parallelDecryptReader) work() {
defer r.wg.Done()
for j := range r.jobs {
out := make([]byte, 0, ChunkSize)
plaintext, err := r.a.Open(out, j.nonce[:], j.ciphertext, nil)
if err != nil {
j.result <- decResult{err: errors.New("failed to decrypt and authenticate payload chunk, file may be corrupted or tampered with")}
continue
}
j.result <- decResult{plaintext: plaintext}
}
}
type chunkKind int
const (
chunkFull chunkKind = iota
chunkShort
chunkEOF
chunkErr
)
type rawChunk struct {
kind chunkKind
data []byte
err error
}
// readRawChunk reads exactly one raw encChunkSize block from src into a
// freshly allocated buffer and classifies the read, mirroring the switch at
// the top of the sequential DecryptReader.readChunk.
func readRawChunk(src io.Reader) rawChunk {
buf := make([]byte, encChunkSize)
n, err := io.ReadFull(src, buf)
switch {
case err == nil:
return rawChunk{kind: chunkFull, data: buf}
case err == io.EOF:
return rawChunk{kind: chunkEOF}
case err == io.ErrUnexpectedEOF:
return rawChunk{kind: chunkShort, data: buf[:n]}
default:
return rawChunk{kind: chunkErr, err: err}
}
}
// pump is the single goroutine that reads raw chunks from src, in order,
// always staying one chunk ahead so that every dispatched chunk's
// last-chunk status is already certain. It closes r.jobs and r.order when
// the stream ends or a fatal error is found, which is what lets Read
// observe a clean end of stream (order closed with nothing left) versus an
// explicit terminal error (an emitDirect result).
func (r *parallelDecryptReader) pump() {
defer close(r.jobs)
defer close(r.order)
first := readRawChunk(r.src)
switch first.kind {
case chunkErr:
r.emitDirect(decResult{err: first.err})
return
case chunkEOF:
// A message can't end without a marked chunk; this message is
// truncated, exactly like the sequential reader's first read.
r.emitDirect(decResult{err: io.ErrUnexpectedEOF})
return
case chunkShort:
// This is chunk index 0: an empty chunk here is the valid
// empty-plaintext-file case, not the "empty last chunk" error,
// which can only apply to a later chunk.
r.dispatch(0, first.data, true)
r.probeTrailing()
return
}
// first.kind == chunkFull: ambiguous until we see what follows it.
pendingIndex := int64(0)
pending := first.data
for {
next := readRawChunk(r.src)
switch next.kind {
case chunkErr:
r.dispatch(pendingIndex, pending, false)
r.emitDirect(decResult{err: next.err})
return
case chunkEOF:
// Nothing follows: pending was a full-length final chunk. The
// clean EOF we just observed is itself the trailing-data proof,
// no separate probe needed.
r.dispatch(pendingIndex, pending, true)
return
case chunkShort:
r.dispatch(pendingIndex, pending, false)
if len(next.data) == chacha20poly1305.Overhead {
// A short chunk here is never index 0, so an empty one is
// always the disallowed case.
r.emitDirect(decResult{err: errors.New("last chunk is empty; encrypted payload is malformed")})
return
}
r.dispatch(pendingIndex+1, next.data, true)
r.probeTrailing()
return
case chunkFull:
r.dispatch(pendingIndex, pending, false)
pendingIndex++
pending = next.data
continue
}
}
}
// probeTrailing confirms there is no data after a short final chunk, exactly
// as the sequential DecryptReader.Read does after processing a last chunk.
func (r *parallelDecryptReader) probeTrailing() {
var b [1]byte
_, err := r.src.Read(b[:])
switch {
case err == nil:
r.emitDirect(decResult{err: errors.New("trailing data after end of encrypted file")})
case err != io.EOF:
r.emitDirect(decResult{err: fmt.Errorf("non-EOF error reading after end of encrypted file: %w", err)})
}
}
@@ -0,0 +1,192 @@
// Copyright 2026 The fury Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package stream
import (
"bytes"
"crypto/cipher"
"errors"
"io"
"sync"
"golang.org/x/crypto/chacha20poly1305"
)
// pipelineDepthPerWorker bounds how many chunks may be queued for, or
// awaiting collection from, a single worker at once. It caps the pipeline's
// memory use to a small multiple of ChunkSize regardless of input size.
const pipelineDepthPerWorker = 2
// NewParallelEncryptWriter is equivalent to [NewEncryptWriter], except that
// chunks are sealed by a pool of worker goroutines instead of inline on the
// calling goroutine. Its output is byte-for-byte identical to
// NewEncryptWriter's: STREAM chunk nonces are a pure counter, so sealing
// chunk i never depends on chunk i-1. Only the write order to dst must stay
// sequential, which a dedicated collector goroutine enforces.
//
// workers <= 1 returns a plain [NewEncryptWriter] with no pipeline overhead.
func NewParallelEncryptWriter(key []byte, dst io.Writer, workers int) (io.WriteCloser, error) {
if workers <= 1 {
return NewEncryptWriter(key, dst)
}
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
depth := workers * pipelineDepthPerWorker
w := &parallelEncryptWriter{
a: aead,
dst: dst,
jobs: make(chan encJob, depth),
order: make(chan chan []byte, depth),
done: make(chan struct{}),
}
w.wg.Add(workers)
for i := 0; i < workers; i++ {
go w.work()
}
go w.collect()
return w, nil
}
type encJob struct {
nonce [chacha20poly1305.NonceSize]byte
plaintext []byte
result chan []byte
}
type parallelEncryptWriter struct {
a cipher.AEAD
dst io.Writer
buf bytes.Buffer
index int64
jobs chan encJob
order chan chan []byte
wg sync.WaitGroup
done chan struct{} // closed once the collector goroutine has returned
mu sync.Mutex
err error
}
func (w *parallelEncryptWriter) getErr() error {
w.mu.Lock()
defer w.mu.Unlock()
return w.err
}
func (w *parallelEncryptWriter) setErr(err error) {
w.mu.Lock()
defer w.mu.Unlock()
if w.err == nil {
w.err = err
}
}
func (w *parallelEncryptWriter) Write(p []byte) (n int, err error) {
if err := w.getErr(); err != nil {
return 0, err
}
if len(p) == 0 {
return 0, nil
}
total := len(p)
for len(p) > 0 {
room := ChunkSize - w.buf.Len()
nn := min(len(p), room)
w.buf.Write(p[:nn])
p = p[nn:]
// Only submit a chunk once we know more data is coming, so the very
// last chunk (submitted from Close) is always the one that gets the
// last-chunk-flag nonce, exactly like the sequential EncryptWriter.
if w.buf.Len() == ChunkSize && len(p) > 0 {
if err := w.submitChunk(false); err != nil {
w.setErr(err)
return 0, err
}
}
}
return total, nil
}
// Close flushes the last chunk and waits for every queued chunk to be
// sealed and written to dst, in order. It does not close the underlying
// Writer.
func (w *parallelEncryptWriter) Close() error {
if err := w.getErr(); err != nil {
return err
}
// submitChunk cannot fail here: we just confirmed there is no prior
// error, and channel sends to w.jobs/w.order only block, they never
// return an error.
_ = w.submitChunk(true)
close(w.jobs)
close(w.order)
w.wg.Wait()
<-w.done
if err := w.getErr(); err != nil {
return err
}
w.setErr(errors.New("stream.Writer is already closed"))
return nil
}
// submitChunk copies out the current buffer as a chunk of plaintext, assigns
// it the next sequential nonce, and hands it to the worker pool. last must be
// true only for the final call, made from Close.
func (w *parallelEncryptWriter) submitChunk(last bool) error {
if err := w.getErr(); err != nil {
return err
}
plaintext := append([]byte(nil), w.buf.Bytes()...)
w.buf.Reset()
nonce := nonceForChunk(w.index)
if last {
setLastChunkFlag(nonce)
}
w.index++
result := make(chan []byte, 1)
w.order <- result
w.jobs <- encJob{nonce: *nonce, plaintext: plaintext, result: result}
return nil
}
// work seals queued chunks. Seal cannot fail: the nonce is always the
// correct size, so the only possible failure in this pipeline is dst.Write,
// handled in collect.
func (w *parallelEncryptWriter) work() {
defer w.wg.Done()
for j := range w.jobs {
ciphertext := w.a.Seal(make([]byte, 0, len(j.plaintext)+w.a.Overhead()), j.nonce[:], j.plaintext, nil)
j.result <- ciphertext
}
}
// collect writes sealed chunks to dst strictly in submission order, even
// though workers may finish sealing them in a different order.
func (w *parallelEncryptWriter) collect() {
defer close(w.done)
var failed bool
for result := range w.order {
ciphertext := <-result
if failed {
continue
}
if _, err := w.dst.Write(ciphertext); err != nil {
w.setErr(err)
failed = true
}
}
}
+332
View File
@@ -0,0 +1,332 @@
// Copyright 2026 The fury Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package stream_test
import (
"bytes"
"crypto/cipher"
"crypto/rand"
"encoding/binary"
"fmt"
"io"
"runtime"
"testing"
"github.com/andydixon/vault/internal/stream"
"golang.org/x/crypto/chacha20poly1305"
)
func workerCounts() []int {
return []int{1, 2, 4, runtime.NumCPU() + 4}
}
func randBytes(t *testing.T, n int) []byte {
t.Helper()
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
t.Fatal(err)
}
return b
}
// TestParallelEncryptMatchesSequential is the main compatibility check: for a
// range of lengths and worker counts, the parallel encrypter must produce
// byte-identical ciphertext to the sequential one, and ciphertext from
// either encrypter must decrypt correctly with either reader.
func TestParallelEncryptMatchesSequential(t *testing.T) {
lengths := []int{0, 1000, cs - 1, cs, cs + 1, cs + 100, 2 * cs, 2*cs + 500}
for _, length := range lengths {
for _, workers := range workerCounts() {
t.Run(fmt.Sprintf("len=%d,workers=%d", length, workers), func(t *testing.T) {
src := randBytes(t, length)
key := randBytes(t, chacha20poly1305.KeySize)
seqBuf := &bytes.Buffer{}
sw, err := stream.NewEncryptWriter(key, seqBuf)
if err != nil {
t.Fatal(err)
}
if _, err := sw.Write(src); err != nil {
t.Fatal(err)
}
if err := sw.Close(); err != nil {
t.Fatal(err)
}
parBuf := &bytes.Buffer{}
pw, err := stream.NewParallelEncryptWriter(key, parBuf, workers)
if err != nil {
t.Fatal(err)
}
if _, err := pw.Write(src); err != nil {
t.Fatal(err)
}
if err := pw.Close(); err != nil {
t.Fatal(err)
}
if !bytes.Equal(seqBuf.Bytes(), parBuf.Bytes()) {
t.Fatalf("parallel ciphertext (%d bytes) differs from sequential (%d bytes)",
parBuf.Len(), seqBuf.Len())
}
t.Run("decrypt-with-sequential", func(t *testing.T) {
dr, err := stream.NewDecryptReader(key, bytes.NewReader(parBuf.Bytes()))
if err != nil {
t.Fatal(err)
}
checkRoundTrip(t, dr, src)
})
t.Run("decrypt-with-parallel", func(t *testing.T) {
dr, err := stream.NewParallelDecryptReader(key, bytes.NewReader(parBuf.Bytes()), workers)
if err != nil {
t.Fatal(err)
}
checkRoundTrip(t, dr, src)
})
t.Run("sequential-ciphertext-decrypts-with-parallel", func(t *testing.T) {
dr, err := stream.NewParallelDecryptReader(key, bytes.NewReader(seqBuf.Bytes()), workers)
if err != nil {
t.Fatal(err)
}
checkRoundTrip(t, dr, src)
})
})
}
}
}
func checkRoundTrip(t *testing.T, r io.Reader, want []byte) {
t.Helper()
got, err := io.ReadAll(r)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !bytes.Equal(got, want) {
t.Fatalf("decrypted mismatch: got %d bytes, want %d bytes", len(got), len(want))
}
}
// TestParallelEncryptWriteInSteps confirms the parallel writer's output is
// unaffected by how the caller chunks its Write calls, mirroring the
// sequential TestRoundTrip's step-size coverage but scoped to a couple of
// representative lengths to keep runtime reasonable.
func TestParallelEncryptWriteInSteps(t *testing.T) {
lengths := []int{cs + 100, 2*cs + 500}
steps := []int{1, 537, cs}
for _, length := range lengths {
for _, step := range steps {
t.Run(fmt.Sprintf("len=%d,step=%d", length, step), func(t *testing.T) {
src := randBytes(t, length)
key := randBytes(t, chacha20poly1305.KeySize)
seqBuf := &bytes.Buffer{}
sw, err := stream.NewEncryptWriter(key, seqBuf)
if err != nil {
t.Fatal(err)
}
writeInSteps(t, sw, src, step)
if err := sw.Close(); err != nil {
t.Fatal(err)
}
parBuf := &bytes.Buffer{}
pw, err := stream.NewParallelEncryptWriter(key, parBuf, 4)
if err != nil {
t.Fatal(err)
}
writeInSteps(t, pw, src, step)
if err := pw.Close(); err != nil {
t.Fatal(err)
}
if !bytes.Equal(seqBuf.Bytes(), parBuf.Bytes()) {
t.Fatalf("parallel ciphertext differs from sequential with step=%d", step)
}
})
}
}
}
func writeInSteps(t *testing.T, w io.Writer, src []byte, step int) {
t.Helper()
for len(src) > 0 {
n := min(step, len(src))
if _, err := w.Write(src[:n]); err != nil {
t.Fatal(err)
}
src = src[n:]
}
}
// TestParallelDecryptMalformed exercises the same categories of malformed
// input the sequential reader rejects. The parallel reader resolves the
// last-chunk ambiguity from the shape of the raw stream (read-ahead) rather
// than by a trial decryption, so in adversarial cases the exact error
// message can differ from the sequential reader's -- what must not differ is
// the accept/reject verdict. Where the message is deterministic under the
// new strategy too, it's asserted explicitly.
func TestParallelDecryptMalformed(t *testing.T) {
newCiphertext := func(t *testing.T, key []byte, plaintext []byte) []byte {
t.Helper()
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
t.Run("wrong key", func(t *testing.T) {
key := randBytes(t, chacha20poly1305.KeySize)
ciphertext := newCiphertext(t, key, randBytes(t, 1000))
wrongKey := randBytes(t, chacha20poly1305.KeySize)
dr, err := stream.NewParallelDecryptReader(wrongKey, bytes.NewReader(ciphertext), 4)
if err != nil {
t.Fatal(err)
}
if _, err := io.ReadAll(dr); err == nil {
t.Fatal("expected error, got nil")
}
})
t.Run("corrupted byte in non-final chunk", func(t *testing.T) {
key := randBytes(t, chacha20poly1305.KeySize)
ciphertext := newCiphertext(t, key, randBytes(t, 2*cs+500))
ciphertext[10] ^= 0xff // inside the first (non-final) chunk
dr, err := stream.NewParallelDecryptReader(key, bytes.NewReader(ciphertext), 4)
if err != nil {
t.Fatal(err)
}
if _, err := io.ReadAll(dr); err == nil {
t.Fatal("expected error, got nil")
}
})
t.Run("truncated", func(t *testing.T) {
key := randBytes(t, chacha20poly1305.KeySize)
ciphertext := newCiphertext(t, key, randBytes(t, 2*cs+500))
truncated := ciphertext[:len(ciphertext)-100]
dr, err := stream.NewParallelDecryptReader(key, bytes.NewReader(truncated), 4)
if err != nil {
t.Fatal(err)
}
if _, err := io.ReadAll(dr); err == nil {
t.Fatal("expected error, got nil")
}
})
t.Run("trailing data appended after short last chunk", func(t *testing.T) {
// With a bytes.Reader, io.ReadFull for the (undersized) chunk buffer
// greedily consumes the real short chunk AND the trailing garbage in
// one read, since everything is available at once: the result is a
// misaligned AEAD tag, not a distinguishable "trailing data" case.
// This is also true of the sequential reader (same io.ReadFull over
// the same fixed-size buffer) -- the verdict, an error, is what must
// hold, not a specific message.
key := randBytes(t, chacha20poly1305.KeySize)
ciphertext := newCiphertext(t, key, randBytes(t, 1000)) // ends in a short chunk
withTrailer := append(append([]byte(nil), ciphertext...), 1, 2, 3, 4, 5)
dr, err := stream.NewParallelDecryptReader(key, bytes.NewReader(withTrailer), 4)
if err != nil {
t.Fatal(err)
}
if _, err := io.ReadAll(dr); err == nil {
t.Fatal("expected error, got nil")
}
})
t.Run("trailing data after full-length last chunk", func(t *testing.T) {
key := randBytes(t, chacha20poly1305.KeySize)
ciphertext := newCiphertext(t, key, randBytes(t, cs)) // exact multiple: full-length last chunk
withTrailer := append(append([]byte(nil), ciphertext...), 1, 2, 3, 4, 5)
dr, err := stream.NewParallelDecryptReader(key, bytes.NewReader(withTrailer), 4)
if err != nil {
t.Fatal(err)
}
// The trailing bytes are re-interpreted as a short "final" chunk here
// (a different, but still safe, rejection path than the sequential
// reader's trailing-byte probe); the verdict must still be an error.
if _, err := io.ReadAll(dr); err == nil {
t.Fatal("expected error, got nil")
}
})
t.Run("empty chunk after a non-empty chunk is rejected", func(t *testing.T) {
key := randBytes(t, chacha20poly1305.KeySize)
aead, err := chacha20poly1305.New(key)
if err != nil {
t.Fatal(err)
}
var malformed []byte
malformed = append(malformed, sealChunkForTest(t, aead, 0, false, randBytes(t, cs))...)
malformed = append(malformed, sealChunkForTest(t, aead, 1, true, nil)...)
dr, err := stream.NewParallelDecryptReader(key, bytes.NewReader(malformed), 4)
if err != nil {
t.Fatal(err)
}
_, err = io.ReadAll(dr)
if err == nil {
t.Fatal("expected error, got nil")
}
if got, want := err.Error(), "last chunk is empty; encrypted payload is malformed"; got != want {
t.Fatalf("error = %q, want %q", got, want)
}
})
}
// sealChunkForTest manually seals a chunk using the same nonce scheme as the
// stream package (a big-endian chunk counter in bytes [3:11], a last-chunk
// flag in the final byte), to construct deliberately malformed streams that
// aren't reachable through the public encrypt API.
func sealChunkForTest(t *testing.T, aead cipher.AEAD, index int64, last bool, plaintext []byte) []byte {
t.Helper()
var nonce [chacha20poly1305.NonceSize]byte
binary.BigEndian.PutUint64(nonce[3:11], uint64(index))
if last {
nonce[len(nonce)-1] = 1
}
return aead.Seal(nil, nonce[:], plaintext, nil)
}
// TestParallelRace exercises the pipeline with oversubscribed workers on a
// multi-chunk payload; run with -race to catch data races in the ordered
// fan-in.
func TestParallelRace(t *testing.T) {
key := randBytes(t, chacha20poly1305.KeySize)
src := randBytes(t, 6*cs+12345)
buf := &bytes.Buffer{}
w, err := stream.NewParallelEncryptWriter(key, buf, runtime.NumCPU()*2)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(src); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
dr, err := stream.NewParallelDecryptReader(key, bytes.NewReader(buf.Bytes()), runtime.NumCPU()*2)
if err != nil {
t.Fatal(err)
}
checkRoundTrip(t, dr, src)
}
+468
View File
@@ -0,0 +1,468 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// Package stream implements a variant of the STREAM chunked encryption scheme.
//
// The invariants this package exists to hold:
//
// - Counter uniqueness. A chunk's nonce is a pure big-endian counter over
// bytes 3..10, incremented once per chunk and never reset within a
// stream. incNonce panics rather than wrapping, so a (key, nonce) pair
// can never repeat: the counter is 88 bits and the panic is unreachable
// for any real file.
// - Content independence. The nonce depends only on the chunk index and the
// last-chunk flag, never on chunk contents. That is what lets the parallel
// encoder seal chunks out of order and still produce byte-identical
// output.
// - Truncation resistance. The final chunk is sealed under a nonce whose
// last byte carries lastChunkFlag. A truncated stream therefore fails to
// authenticate: the attacker would have to forge a last-chunk tag for a
// chunk the writer sealed as non-final.
// - No trailing data. After a last chunk is accepted, the reader confirms
// the underlying source is at EOF and fails otherwise, so appended bytes
// are rejected rather than silently ignored.
package stream
import (
"bytes"
"crypto/cipher"
"encoding/binary"
"errors"
"fmt"
"io"
"sync/atomic"
"golang.org/x/crypto/chacha20poly1305"
)
const ChunkSize = 64 * 1024
func EncryptedChunkCount(encryptedSize int64) (int64, error) {
chunks := (encryptedSize + encChunkSize - 1) / encChunkSize
plaintextSize := encryptedSize - chunks*chacha20poly1305.Overhead
expChunks := (plaintextSize + ChunkSize - 1) / ChunkSize
// Empty plaintext, the only case that allows (and requires) an empty chunk.
if plaintextSize == 0 {
expChunks = 1
}
if expChunks != chunks {
return 0, fmt.Errorf("invalid encrypted payload size: %d", encryptedSize)
}
return chunks, nil
}
func PlaintextSize(encryptedSize int64) (int64, error) {
chunks, err := EncryptedChunkCount(encryptedSize)
if err != nil {
return 0, err
}
plaintextSize := encryptedSize - chunks*chacha20poly1305.Overhead
return plaintextSize, nil
}
type DecryptReader struct {
a cipher.AEAD
src io.Reader
unread []byte // decrypted but unread data, backed by buf
buf [encChunkSize]byte
err error
nonce [chacha20poly1305.NonceSize]byte
}
const (
encChunkSize = ChunkSize + chacha20poly1305.Overhead
lastChunkFlag = 0x01
)
func NewDecryptReader(key []byte, src io.Reader) (*DecryptReader, error) {
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
return &DecryptReader{a: aead, src: src}, nil
}
func (r *DecryptReader) Read(p []byte) (int, error) {
if len(r.unread) > 0 {
n := copy(p, r.unread)
r.unread = r.unread[n:]
return n, nil
}
if r.err != nil {
return 0, r.err
}
if len(p) == 0 {
return 0, nil
}
last, err := r.readChunk()
if err != nil {
r.err = err
return 0, err
}
n := copy(p, r.unread)
r.unread = r.unread[n:]
if last {
// Ensure there is an EOF after the last chunk as expected. In other
// words, check for trailing data after a full-length final chunk.
// Hopefully, the underlying reader supports returning EOF even if it
// had previously returned an EOF to ReadFull.
if _, err := r.src.Read(make([]byte, 1)); err == nil {
r.err = errors.New("trailing data after end of encrypted file")
} else if err != io.EOF {
r.err = fmt.Errorf("non-EOF error reading after end of encrypted file: %w", err)
} else {
r.err = io.EOF
}
}
return n, nil
}
// readChunk reads the next chunk of ciphertext from r.src and makes it available
// in r.unread. last is true if the chunk was marked as the end of the message.
// readChunk must not be called again after returning a last chunk or an error.
func (r *DecryptReader) readChunk() (last bool, err error) {
if len(r.unread) != 0 {
panic("stream: internal error: readChunk called with dirty buffer")
}
in := r.buf[:]
n, err := io.ReadFull(r.src, in)
switch {
case err == io.EOF:
// A message can't end without a marked chunk. This message is truncated.
return false, io.ErrUnexpectedEOF
case err == io.ErrUnexpectedEOF:
// The last chunk can be short, but not empty unless it's the first and
// only chunk.
if !nonceIsZero(&r.nonce) && n == r.a.Overhead() {
return false, errors.New("last chunk is empty; encrypted payload is malformed")
}
in = in[:n]
last = true
setLastChunkFlag(&r.nonce)
case err != nil:
return false, err
}
outBuf := make([]byte, 0, ChunkSize)
out, err := r.a.Open(outBuf, r.nonce[:], in, nil)
if err != nil && !last {
// Check if this was a full-length final chunk.
last = true
setLastChunkFlag(&r.nonce)
out, err = r.a.Open(outBuf, r.nonce[:], in, nil)
}
if err != nil {
return false, errors.New("failed to decrypt and authenticate payload chunk, file may be corrupted or tampered with")
}
incNonce(&r.nonce)
r.unread = r.buf[:copy(r.buf[:], out)]
return last, nil
}
func incNonce(nonce *[chacha20poly1305.NonceSize]byte) {
for i := len(nonce) - 2; i >= 0; i-- {
nonce[i]++
if nonce[i] != 0 {
return
}
}
// The counter is 88 bits, this is unreachable.
panic("stream: chunk counter wrapped around")
}
func nonceForChunk(chunkIndex int64) *[chacha20poly1305.NonceSize]byte {
var nonce [chacha20poly1305.NonceSize]byte
binary.BigEndian.PutUint64(nonce[3:11], uint64(chunkIndex))
return &nonce
}
func setLastChunkFlag(nonce *[chacha20poly1305.NonceSize]byte) {
nonce[len(nonce)-1] = lastChunkFlag
}
func nonceIsZero(nonce *[chacha20poly1305.NonceSize]byte) bool {
return *nonce == [chacha20poly1305.NonceSize]byte{}
}
type EncryptWriter struct {
a cipher.AEAD
dst io.Writer
buf bytes.Buffer
nonce [chacha20poly1305.NonceSize]byte
err error
}
func NewEncryptWriter(key []byte, dst io.Writer) (*EncryptWriter, error) {
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
return &EncryptWriter{a: aead, dst: dst}, nil
}
func (w *EncryptWriter) Write(p []byte) (n int, err error) {
if w.err != nil {
return 0, w.err
}
if len(p) == 0 {
return 0, nil
}
total := len(p)
for len(p) > 0 {
n := min(len(p), ChunkSize-w.buf.Len())
w.buf.Write(p[:n])
p = p[n:]
// Only flush if there's a full chunk with bytes still to write, or we
// can't know if this is the last chunk yet.
if w.buf.Len() == ChunkSize && len(p) > 0 {
if err := w.flushChunk(notLastChunk); err != nil {
w.err = err
return 0, err
}
}
}
return total, nil
}
// Close flushes the last chunk. It does not close the underlying Writer.
func (w *EncryptWriter) Close() error {
if w.err != nil {
return w.err
}
w.err = w.flushChunk(lastChunk)
if w.err != nil {
return w.err
}
w.err = errors.New("stream.Writer is already closed")
return nil
}
const (
lastChunk = true
notLastChunk = false
)
func (w *EncryptWriter) flushChunk(last bool) error {
if !last && w.buf.Len() != ChunkSize {
panic("stream: internal error: flush called with partial chunk")
}
if last {
setLastChunkFlag(&w.nonce)
}
w.buf.Grow(chacha20poly1305.Overhead)
ciphertext := w.a.Seal(w.buf.Bytes()[:0], w.nonce[:], w.buf.Bytes(), nil)
_, err := w.dst.Write(ciphertext)
incNonce(&w.nonce)
w.buf.Reset()
return err
}
type EncryptReader struct {
a cipher.AEAD
src io.Reader
// The first ready bytes of buf are already encrypted. This may be less than
// buf.Len(), because we need to over-read to know if a chunk is the last.
ready int
buf bytes.Buffer
nonce [chacha20poly1305.NonceSize]byte
err error
}
func NewEncryptReader(key []byte, src io.Reader) (*EncryptReader, error) {
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
return &EncryptReader{a: aead, src: src}, nil
}
func (r *EncryptReader) Read(p []byte) (int, error) {
if r.ready > 0 {
n, err := r.buf.Read(p[:min(len(p), r.ready)])
r.ready -= n
return n, err
}
if r.err != nil {
return 0, r.err
}
if len(p) == 0 {
return 0, nil
}
if err := r.feedBuffer(); err != nil {
r.err = err
return 0, err
}
n, err := r.buf.Read(p[:min(len(p), r.ready)])
r.ready -= n
return n, err
}
// feedBuffer reads and encrypts the next chunk from r.src and appends it to
// r.buf. It sets r.ready to the number of newly available bytes in r.buf.
func (r *EncryptReader) feedBuffer() error {
if r.ready > 0 {
panic("stream: internal error: feedBuffer called with dirty buffer")
}
// CopyN will use r.buf.ReadFrom/WriteTo to fill the buffer directly.
// We need ChunkSize + 1 bytes to determine if this is the last chunk.
_, err := io.CopyN(&r.buf, r.src, int64(ChunkSize-r.buf.Len()+1))
if err != nil && err != io.EOF {
return err
}
if last := r.buf.Len() <= ChunkSize; last {
setLastChunkFlag(&r.nonce)
// After Grow, we know r.buf.Bytes() has enough capacity for the
// overhead. We encrypt in place and then do a Write to include the
// overhead in the buffer.
r.buf.Grow(chacha20poly1305.Overhead)
plaintext := r.buf.Bytes()
r.a.Seal(plaintext[:0], r.nonce[:], plaintext, nil)
incNonce(&r.nonce)
r.buf.Write(plaintext[len(plaintext) : len(plaintext)+chacha20poly1305.Overhead])
r.ready = r.buf.Len()
r.err = io.EOF
return nil
}
// Same, but accounting for the tail byte which will remain unencrypted and
// needs to be shifted past the overhead.
if r.buf.Len() != ChunkSize+1 {
panic("stream: internal error: unexpected buffer length")
}
tailByte := r.buf.Bytes()[ChunkSize]
r.buf.Grow(chacha20poly1305.Overhead)
plaintext := r.buf.Bytes()[:ChunkSize]
r.a.Seal(plaintext[:0], r.nonce[:], plaintext, nil)
incNonce(&r.nonce)
r.buf.Write(plaintext[len(plaintext)+1 : len(plaintext)+chacha20poly1305.Overhead])
r.buf.WriteByte(tailByte)
r.ready = ChunkSize + chacha20poly1305.Overhead
return nil
}
type DecryptReaderAt struct {
a cipher.AEAD
src io.ReaderAt
size int64
chunks int64
cache atomic.Pointer[cachedChunk]
}
type cachedChunk struct {
off int64
data []byte
}
func NewDecryptReaderAt(key []byte, src io.ReaderAt, size int64) (*DecryptReaderAt, error) {
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
// Check that size is valid by decrypting the final chunk.
chunks, err := EncryptedChunkCount(size)
if err != nil {
return nil, err
}
finalChunkIndex := chunks - 1
finalChunkOff := finalChunkIndex * encChunkSize
finalChunkSize := size - finalChunkOff
finalChunk := make([]byte, finalChunkSize)
if _, err := src.ReadAt(finalChunk, finalChunkOff); err != nil {
return nil, fmt.Errorf("failed to read final chunk: %w", err)
}
nonce := nonceForChunk(finalChunkIndex)
setLastChunkFlag(nonce)
plaintext, err := aead.Open(finalChunk[:0], nonce[:], finalChunk, nil)
if err != nil {
return nil, fmt.Errorf("failed to decrypt and authenticate final chunk: %w", err)
}
cache := &cachedChunk{off: finalChunkOff, data: plaintext}
plaintextSize := size - chunks*chacha20poly1305.Overhead
r := &DecryptReaderAt{a: aead, src: src, size: plaintextSize, chunks: chunks}
r.cache.Store(cache)
return r, nil
}
func (r *DecryptReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
if off < 0 || off > r.size {
return 0, fmt.Errorf("offset out of range [0:%d]: %d", r.size, off)
}
if len(p) == 0 {
return 0, nil
}
var cacheUpdate *cachedChunk
chunk := make([]byte, encChunkSize)
for len(p) > 0 && off < r.size {
chunkIndex := off / ChunkSize
chunkOff := chunkIndex * encChunkSize
encSize := r.size + r.chunks*chacha20poly1305.Overhead
chunkSize := min(encSize-chunkOff, encChunkSize)
cached := r.cache.Load()
var plaintext []byte
if cached != nil && cached.off == chunkOff {
plaintext = cached.data
cacheUpdate = nil
} else {
nn, err := r.src.ReadAt(chunk[:chunkSize], chunkOff)
if err == io.EOF {
if int64(nn) != chunkSize {
err = io.ErrUnexpectedEOF
} else {
err = nil
}
}
if err != nil {
return n, fmt.Errorf("failed to read chunk at offset %d: %w", chunkOff, err)
}
nonce := nonceForChunk(chunkIndex)
if chunkIndex == r.chunks-1 {
setLastChunkFlag(nonce)
}
plaintext, err = r.a.Open(chunk[:0], nonce[:], chunk[:chunkSize], nil)
if err != nil {
return n, fmt.Errorf("failed to decrypt and authenticate chunk at offset %d: %w", chunkOff, err)
}
cacheUpdate = &cachedChunk{off: chunkOff, data: plaintext}
}
plainChunkOff := int(off - chunkIndex*ChunkSize)
copySize := min(len(plaintext)-plainChunkOff, len(p))
copy(p, plaintext[plainChunkOff:plainChunkOff+copySize])
p = p[copySize:]
off += int64(copySize)
n += copySize
}
if cacheUpdate != nil {
r.cache.Store(cacheUpdate)
}
if off == r.size {
return n, io.EOF
}
return n, nil
}
+950
View File
@@ -0,0 +1,950 @@
// SPDX-License-Identifier: GPL-3.0-or-later
package stream_test
import (
"bytes"
"crypto/rand"
"fmt"
"io"
"testing"
"testing/iotest"
"github.com/andydixon/vault/internal/stream"
"golang.org/x/crypto/chacha20poly1305"
)
const cs = stream.ChunkSize
func TestRoundTrip(t *testing.T) {
for _, length := range []int{0, 1000, cs - 1, cs, cs + 1, cs + 100, 2 * cs, 2*cs + 500} {
for _, stepSize := range []int{512, 600, 1000, cs - 1, cs, cs + 1} {
t.Run(fmt.Sprintf("len=%d,step=%d", length, stepSize), func(t *testing.T) {
testRoundTrip(t, stepSize, length)
})
}
}
length, stepSize := 2*cs+500, 1
t.Run(fmt.Sprintf("len=%d,step=%d", length, stepSize), func(t *testing.T) {
testRoundTrip(t, stepSize, length)
})
}
func testRoundTrip(t *testing.T, stepSize, length int) {
src := make([]byte, length)
if _, err := rand.Read(src); err != nil {
t.Fatal(err)
}
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
var ciphertext []byte
t.Run("EncryptWriter", func(t *testing.T) {
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
var n int
for n < length {
b := min(length-n, stepSize)
nn, err := w.Write(src[n : n+b])
if err != nil {
t.Fatal(err)
}
if nn != b {
t.Errorf("Write returned %d, expected %d", nn, b)
}
n += nn
nn, err = w.Write(src[n:n])
if err != nil {
t.Fatal(err)
}
if nn != 0 {
t.Errorf("Write returned %d, expected 0", nn)
}
}
if err := w.Close(); err != nil {
t.Error("Close returned an error:", err)
}
ciphertext = buf.Bytes()
})
t.Run("DecryptReader", func(t *testing.T) {
r, err := stream.NewDecryptReader(key, bytes.NewReader(ciphertext))
if err != nil {
t.Fatal(err)
}
var n int
readBuf := make([]byte, stepSize)
for n < length {
nn, err := r.Read(readBuf)
if err != nil {
t.Fatalf("Read error at index %d: %v", n, err)
}
if !bytes.Equal(readBuf[:nn], src[n:n+nn]) {
t.Errorf("wrong data at indexes %d - %d", n, n+nn)
}
n += nn
}
t.Run("TestReader", func(t *testing.T) {
if length > 1000 && testing.Short() {
t.Skip("skipping slow iotest.TestReader on long input")
}
r, _ := stream.NewDecryptReader(key, bytes.NewReader(ciphertext))
if err := iotest.TestReader(r, src); err != nil {
t.Error("iotest.TestReader error on DecryptReader:", err)
}
})
})
t.Run("DecryptReaderAt", func(t *testing.T) {
rAt, err := stream.NewDecryptReaderAt(key, bytes.NewReader(ciphertext), int64(len(ciphertext)))
if err != nil {
t.Fatal(err)
}
rr := io.NewSectionReader(rAt, 0, int64(len(ciphertext)))
var n int
readBuf := make([]byte, stepSize)
for n < length {
nn, err := rr.Read(readBuf)
if n+nn == length && err == io.EOF {
err = nil
}
if err != nil {
t.Fatalf("ReadAt error at index %d: %v", n, err)
}
if !bytes.Equal(readBuf[:nn], src[n:n+nn]) {
t.Errorf("wrong data at indexes %d - %d", n, n+nn)
}
n += nn
}
t.Run("TestReader", func(t *testing.T) {
if length > 1000 && testing.Short() {
t.Skip("skipping slow iotest.TestReader on long input")
}
rr := io.NewSectionReader(rAt, 0, int64(len(src)))
if err := iotest.TestReader(rr, src); err != nil {
t.Error("iotest.TestReader error on DecryptReaderAt:", err)
}
})
})
t.Run("EncryptReader", func(t *testing.T) {
er, err := stream.NewEncryptReader(key, bytes.NewReader(src))
if err != nil {
t.Fatal(err)
}
var n int
readBuf := make([]byte, stepSize)
for {
nn, err := er.Read(readBuf)
if nn == 0 && err == io.EOF {
break
} else if err != nil {
t.Fatalf("EncryptReader Read error at index %d: %v", n, err)
}
if !bytes.Equal(readBuf[:nn], ciphertext[n:n+nn]) {
t.Errorf("EncryptReader wrong data at indexes %d - %d", n, n+nn)
}
n += nn
}
if n != len(ciphertext) {
t.Errorf("EncryptReader read %d bytes, expected %d", n, len(ciphertext))
}
t.Run("TestReader", func(t *testing.T) {
if length > 1000 && testing.Short() {
t.Skip("skipping slow iotest.TestReader on long input")
}
er, _ := stream.NewEncryptReader(key, bytes.NewReader(src))
if err := iotest.TestReader(er, ciphertext); err != nil {
t.Error("iotest.TestReader error on EncryptReader:", err)
}
})
})
}
// trackingReaderAt wraps an io.ReaderAt and tracks whether ReadAt was called.
type trackingReaderAt struct {
r io.ReaderAt
called bool
}
func (t *trackingReaderAt) ReadAt(p []byte, off int64) (int, error) {
t.called = true
return t.r.ReadAt(p, off)
}
func (t *trackingReaderAt) reset() {
t.called = false
}
func TestDecryptReaderAt(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
// Create plaintext spanning exactly 3 chunks: 2 full chunks + partial third
// Chunk 0: [0, cs)
// Chunk 1: [cs, 2*cs)
// Chunk 2: [2*cs, 2*cs+500)
plaintextSize := 2*cs + 500
plaintext := make([]byte, plaintextSize)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
// Encrypt
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
// Create tracking ReaderAt
tracker := &trackingReaderAt{r: bytes.NewReader(ciphertext)}
// Create DecryptReaderAt (this reads and caches the final chunk)
ra, err := stream.NewDecryptReaderAt(key, tracker, int64(len(ciphertext)))
if err != nil {
t.Fatal(err)
}
tracker.reset()
// Helper to check reads
checkRead := func(name string, off int64, size int, wantN int, wantEOF bool, wantSrcRead bool) {
t.Helper()
tracker.reset()
p := make([]byte, size)
n, err := ra.ReadAt(p, off)
if wantEOF {
if err != io.EOF {
t.Errorf("%s: got err=%v, want EOF", name, err)
}
} else {
if err != nil {
t.Errorf("%s: got err=%v, want nil", name, err)
}
}
if n != wantN {
t.Errorf("%s: got n=%d, want %d", name, n, wantN)
}
if tracker.called != wantSrcRead {
t.Errorf("%s: src.ReadAt called=%v, want %v", name, tracker.called, wantSrcRead)
}
// Verify data correctness
if n > 0 && off >= 0 && off < int64(plaintextSize) {
end := int(off) + n
if end > plaintextSize {
end = plaintextSize
}
if !bytes.Equal(p[:n], plaintext[off:end]) {
t.Errorf("%s: data mismatch", name)
}
}
}
// Test 1: Read from final chunk (cached by constructor)
checkRead("final chunk (cached)", int64(2*cs+100), 100, 100, false, false)
// Test 2: Read spanning second and third chunk
checkRead("span chunks 1-2", int64(cs+cs-50), 100, 100, false, true)
// Test 3: Read from final chunk again (cached from test 2)
// When reading across chunks 1-2 in test 2, the loop processes chunk 1 then chunk 2,
// so chunk 2 ends up in the cache.
checkRead("final chunk after span", int64(2*cs+200), 100, 100, false, false)
// Test 4: Read from final chunk again (now cached)
checkRead("final chunk (cached again)", int64(2*cs+50), 50, 50, false, false)
// Test 5: Read from first chunk (not cached)
checkRead("first chunk", 0, 100, 100, false, true)
// Test 6: Read from first chunk again (now cached)
checkRead("first chunk (cached)", 50, 100, 100, false, false)
// Test 7: Read spanning all chunks
tracker.reset()
p := make([]byte, plaintextSize)
n, err := ra.ReadAt(p, 0)
if err != io.EOF {
t.Errorf("span all: got err=%v, want EOF", err)
}
if n != plaintextSize {
t.Errorf("span all: got n=%d, want %d", n, plaintextSize)
}
if !bytes.Equal(p, plaintext) {
t.Errorf("span all: data mismatch")
}
// Test 8: Read beyond the end (offset > size)
tracker.reset()
p = make([]byte, 100)
n, err = ra.ReadAt(p, int64(plaintextSize+100))
if err == nil {
t.Error("beyond end: expected error, got nil")
}
if n != 0 {
t.Errorf("beyond end: got n=%d, want 0", n)
}
// Test 9: Read with off = size (should return 0, EOF)
tracker.reset()
p = make([]byte, 100)
n, err = ra.ReadAt(p, int64(plaintextSize))
if err != io.EOF {
t.Errorf("off=size: got err=%v, want EOF", err)
}
if n != 0 {
t.Errorf("off=size: got n=%d, want 0", n)
}
// Test 10: Read spanning last chunk and beyond
tracker.reset()
p = make([]byte, 1000) // request more than available
n, err = ra.ReadAt(p, int64(2*cs+400))
if err != io.EOF {
t.Errorf("span last+beyond: got err=%v, want EOF", err)
}
wantN := 500 - 400 // only 100 bytes available from offset 2*cs+400
if n != wantN {
t.Errorf("span last+beyond: got n=%d, want %d", n, wantN)
}
if !bytes.Equal(p[:n], plaintext[2*cs+400:]) {
t.Error("span last+beyond: data mismatch")
}
// Test 11: Read spanning second+last chunk and beyond
tracker.reset()
p = make([]byte, cs+1000) // request more than available
n, err = ra.ReadAt(p, int64(cs+100))
if err != io.EOF {
t.Errorf("span 1-2+beyond: got err=%v, want EOF", err)
}
wantN = plaintextSize - (cs + 100)
if n != wantN {
t.Errorf("span 1-2+beyond: got n=%d, want %d", n, wantN)
}
if !bytes.Equal(p[:n], plaintext[cs+100:]) {
t.Error("span 1-2+beyond: data mismatch")
}
// Test 12: Negative offset
tracker.reset()
p = make([]byte, 100)
n, err = ra.ReadAt(p, -1)
if err == nil {
t.Error("negative offset: expected error, got nil")
}
if n != 0 {
t.Errorf("negative offset: got n=%d, want 0", n)
}
// Test 13: Zero-length read in the middle
tracker.reset()
p = make([]byte, 0)
n, err = ra.ReadAt(p, 100)
if err != nil {
t.Errorf("zero-length middle: got err=%v, want nil", err)
}
if n != 0 {
t.Errorf("zero-length middle: got n=%d, want 0", n)
}
// Test 14: Zero-length read at end
tracker.reset()
p = make([]byte, 0)
n, err = ra.ReadAt(p, int64(plaintextSize))
if err != nil {
t.Errorf("zero-length end: got err=%v, want nil", err)
}
if n != 0 {
t.Errorf("zero-length end: got n=%d, want 0", n)
}
// Test 15: Read exactly one chunk at chunk boundary
checkRead("exact chunk at boundary", int64(cs), cs, cs, false, true)
// Test 16: Read one byte at each chunk boundary
checkRead("one byte at start", 0, 1, 1, false, true)
checkRead("one byte at cs-1", int64(cs-1), 1, 1, false, false) // cached from test 15
checkRead("one byte at cs", int64(cs), 1, 1, false, true)
checkRead("one byte at 2*cs-1", int64(2*cs-1), 1, 1, false, false) // same chunk
checkRead("one byte at 2*cs", int64(2*cs), 1, 1, false, true)
checkRead("last byte", int64(plaintextSize-1), 1, 1, true, false) // same chunk, EOF because we reach end
// Test 17: Read crossing exactly one chunk boundary
checkRead("cross boundary 0-1", int64(cs-50), 100, 100, false, true)
checkRead("cross boundary 1-2", int64(2*cs-50), 100, 100, false, true)
}
func TestDecryptReaderAtEmpty(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
// Create empty encrypted file
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
tracker := &trackingReaderAt{r: bytes.NewReader(ciphertext)}
ra, err := stream.NewDecryptReaderAt(key, tracker, int64(len(ciphertext)))
if err != nil {
t.Fatal(err)
}
tracker.reset()
// Test 1: Read from empty file at offset 0
p := make([]byte, 100)
n, err := ra.ReadAt(p, 0)
if err != io.EOF {
t.Errorf("empty read: got err=%v, want EOF", err)
}
if n != 0 {
t.Errorf("empty read: got n=%d, want 0", n)
}
// Test 2: Zero-length read from empty file
p = make([]byte, 0)
n, err = ra.ReadAt(p, 0)
if err != nil {
t.Errorf("empty zero-length: got err=%v, want nil", err)
}
if n != 0 {
t.Errorf("empty zero-length: got n=%d, want 0", n)
}
// Test 3: Read beyond empty file
p = make([]byte, 100)
n, err = ra.ReadAt(p, 1)
if err == nil {
t.Error("empty beyond: expected error, got nil")
}
if n != 0 {
t.Errorf("empty beyond: got n=%d, want 0", n)
}
}
func TestDecryptReaderAtSingleChunk(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
// Single chunk, not full
plaintext := make([]byte, 1000)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
tracker := &trackingReaderAt{r: bytes.NewReader(ciphertext)}
ra, err := stream.NewDecryptReaderAt(key, tracker, int64(len(ciphertext)))
if err != nil {
t.Fatal(err)
}
tracker.reset()
// All reads should use cache (final chunk = only chunk)
p := make([]byte, 100)
n, err := ra.ReadAt(p, 0)
if err != nil {
t.Errorf("single chunk start: got err=%v, want nil", err)
}
if n != 100 {
t.Errorf("single chunk start: got n=%d, want 100", n)
}
if tracker.called {
t.Error("single chunk start: unexpected src.ReadAt call")
}
if !bytes.Equal(p[:n], plaintext[:100]) {
t.Error("single chunk start: data mismatch")
}
// Read at end
n, err = ra.ReadAt(p, 900)
if err != io.EOF {
t.Errorf("single chunk end: got err=%v, want EOF", err)
}
if n != 100 {
t.Errorf("single chunk end: got n=%d, want 100", n)
}
if tracker.called {
t.Error("single chunk end: unexpected src.ReadAt call")
}
if !bytes.Equal(p[:n], plaintext[900:]) {
t.Error("single chunk end: data mismatch")
}
}
func TestDecryptReaderAtFullChunks(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
// Exactly 2 full chunks
plaintext := make([]byte, 2*cs)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
tracker := &trackingReaderAt{r: bytes.NewReader(ciphertext)}
ra, err := stream.NewDecryptReaderAt(key, tracker, int64(len(ciphertext)))
if err != nil {
t.Fatal(err)
}
tracker.reset()
// Read last byte of second chunk (cached)
p := make([]byte, 1)
n, err := ra.ReadAt(p, int64(2*cs-1))
if err != io.EOF {
t.Errorf("last byte: got err=%v, want EOF", err)
}
if n != 1 {
t.Errorf("last byte: got n=%d, want 1", n)
}
if tracker.called {
t.Error("last byte: unexpected src.ReadAt call (should be cached)")
}
if p[0] != plaintext[2*cs-1] {
t.Error("last byte: data mismatch")
}
// Read at exactly the boundary between chunks
p = make([]byte, 100)
n, err = ra.ReadAt(p, int64(cs-50))
if err != nil {
t.Errorf("boundary: got err=%v, want nil", err)
}
if n != 100 {
t.Errorf("boundary: got n=%d, want 100", n)
}
if !bytes.Equal(p, plaintext[cs-50:cs+50]) {
t.Error("boundary: data mismatch")
}
}
func TestDecryptReaderAtWrongKey(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
plaintext := make([]byte, 1000)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
// Try to decrypt with wrong key
wrongKey := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(wrongKey); err != nil {
t.Fatal(err)
}
_, err = stream.NewDecryptReaderAt(wrongKey, bytes.NewReader(ciphertext), int64(len(ciphertext)))
if err == nil {
t.Error("wrong key: expected error, got nil")
}
}
func TestDecryptReaderAtInvalidSize(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
plaintext := make([]byte, 1000)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
// Wrong size (too small)
_, err = stream.NewDecryptReaderAt(key, bytes.NewReader(ciphertext), int64(len(ciphertext)-1))
if err == nil {
t.Error("wrong size (small): expected error, got nil")
}
// Wrong size (too large)
_, err = stream.NewDecryptReaderAt(key, bytes.NewReader(ciphertext), int64(len(ciphertext)+1))
if err == nil {
t.Error("wrong size (large): expected error, got nil")
}
// Size that would imply empty final chunk (invalid)
// This would be: one full encrypted chunk + just overhead
invalidSize := int64(cs + chacha20poly1305.Overhead + chacha20poly1305.Overhead)
_, err = stream.NewDecryptReaderAt(key, bytes.NewReader(make([]byte, invalidSize)), invalidSize)
if err == nil {
t.Error("invalid size (empty final chunk): expected error, got nil")
}
}
func TestDecryptReaderAtTruncated(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
plaintext := make([]byte, 2*cs+500)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
// Truncate ciphertext but lie about size
truncated := ciphertext[:len(ciphertext)-100]
_, err = stream.NewDecryptReaderAt(key, bytes.NewReader(truncated), int64(len(ciphertext)))
if err == nil {
t.Error("truncated: expected error, got nil")
}
}
func TestDecryptReaderAtTruncatedChunk(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
// Create 4 chunks: 3 full + 1 partial
plaintext := make([]byte, 3*cs+500)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
// Truncate to 3 chunks (remove the actual final chunk)
// The third chunk was NOT encrypted with the last chunk flag,
// so decryption should fail when we try to use it as the final chunk.
encChunkSize := cs + 16 // ChunkSize + Overhead
truncatedSize := int64(3 * encChunkSize)
truncated := ciphertext[:truncatedSize]
_, err = stream.NewDecryptReaderAt(key, bytes.NewReader(truncated), truncatedSize)
if err == nil {
t.Error("truncated at chunk boundary: expected error, got nil")
}
}
func TestDecryptReaderAtConcurrent(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
// Create plaintext spanning 3 chunks: 2 full + partial
plaintextSize := 2*cs + 500
plaintext := make([]byte, plaintextSize)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
// Encrypt
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := buf.Bytes()
ra, err := stream.NewDecryptReaderAt(key, bytes.NewReader(ciphertext), int64(len(ciphertext)))
if err != nil {
t.Fatal(err)
}
t.Run("same chunk", func(t *testing.T) {
t.Parallel()
const goroutines = 10
const iterations = 100
errc := make(chan error, goroutines)
for g := range goroutines {
go func(id int) {
for i := range iterations {
off := int64((id*iterations + i) % 500)
p := make([]byte, 100)
n, err := ra.ReadAt(p, off)
if err != nil {
errc <- fmt.Errorf("goroutine %d iter %d: %v", id, i, err)
return
}
if n != 100 {
errc <- fmt.Errorf("goroutine %d iter %d: n=%d, want 100", id, i, n)
return
}
if !bytes.Equal(p, plaintext[off:off+100]) {
errc <- fmt.Errorf("goroutine %d iter %d: data mismatch", id, i)
return
}
}
errc <- nil
}(g)
}
for range goroutines {
if err := <-errc; err != nil {
t.Error(err)
}
}
})
t.Run("different chunks", func(t *testing.T) {
t.Parallel()
const goroutines = 10
const iterations = 100
errc := make(chan error, goroutines)
for g := range goroutines {
go func(id int) {
for i := range iterations {
// Each goroutine reads from a different chunk based on id
chunkIdx := id % 3
off := int64(chunkIdx*cs + (i % 400))
size := 100
if off+int64(size) > int64(plaintextSize) {
size = plaintextSize - int(off)
}
p := make([]byte, size)
n, err := ra.ReadAt(p, off)
if n == size && err == io.EOF {
err = nil // EOF at end is acceptable
}
if err != nil {
errc <- fmt.Errorf("goroutine %d iter %d: off=%d: %v", id, i, off, err)
return
}
if n != size {
errc <- fmt.Errorf("goroutine %d iter %d: n=%d, want %d", id, i, n, size)
return
}
if !bytes.Equal(p[:n], plaintext[off:off+int64(n)]) {
errc <- fmt.Errorf("goroutine %d iter %d: data mismatch", id, i)
return
}
}
errc <- nil
}(g)
}
for range goroutines {
if err := <-errc; err != nil {
t.Error(err)
}
}
})
t.Run("across chunks", func(t *testing.T) {
t.Parallel()
const goroutines = 10
const iterations = 100
errc := make(chan error, goroutines)
for g := range goroutines {
go func(id int) {
for i := range iterations {
// Read across chunk boundaries
boundary := (id%2 + 1) * cs // either cs or 2*cs
off := int64(boundary - 50 + (i % 30))
size := 100
if off+int64(size) > int64(plaintextSize) {
size = plaintextSize - int(off)
}
if size <= 0 {
continue
}
p := make([]byte, size)
n, err := ra.ReadAt(p, off)
if n == size && err == io.EOF {
err = nil
}
if err != nil {
errc <- fmt.Errorf("goroutine %d iter %d: off=%d size=%d: %v", id, i, off, size, err)
return
}
if n != size {
errc <- fmt.Errorf("goroutine %d iter %d: n=%d, want %d", id, i, n, size)
return
}
if !bytes.Equal(p[:n], plaintext[off:off+int64(n)]) {
errc <- fmt.Errorf("goroutine %d iter %d: data mismatch", id, i)
return
}
}
errc <- nil
}(g)
}
for range goroutines {
if err := <-errc; err != nil {
t.Error(err)
}
}
})
}
func TestDecryptReaderAtCorrupted(t *testing.T) {
key := make([]byte, chacha20poly1305.KeySize)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
plaintext := make([]byte, 2*cs+500)
if _, err := rand.Read(plaintext); err != nil {
t.Fatal(err)
}
buf := &bytes.Buffer{}
w, err := stream.NewEncryptWriter(key, buf)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write(plaintext); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
ciphertext := bytes.Clone(buf.Bytes())
// Corrupt final chunk - should fail in constructor
corruptedFinal := bytes.Clone(ciphertext)
corruptedFinal[len(corruptedFinal)-10] ^= 0xFF
_, err = stream.NewDecryptReaderAt(key, bytes.NewReader(corruptedFinal), int64(len(corruptedFinal)))
if err == nil {
t.Error("corrupted final: expected error, got nil")
}
// Corrupt first chunk - should fail on read
corruptedFirst := bytes.Clone(ciphertext)
corruptedFirst[10] ^= 0xFF
ra, err := stream.NewDecryptReaderAt(key, bytes.NewReader(corruptedFirst), int64(len(corruptedFirst)))
if err != nil {
t.Fatalf("corrupted first constructor: unexpected error: %v", err)
}
p := make([]byte, 100)
_, err = ra.ReadAt(p, 0)
if err == nil {
t.Error("corrupted first read: expected error, got nil")
}
}
+108
View File
@@ -0,0 +1,108 @@
package term
import (
"bufio"
"fmt"
"os"
"runtime"
"strings"
"golang.org/x/term"
)
// enableVirtualTerminalProcessing tries to enable virtual terminal processing
// on Windows. If it fails, avoid using escape sequences to prevent weird
// characters being printed to the console.
var enableVirtualTerminalProcessing func(out *os.File) error
// clearLine clears the current line on the terminal, or opens a new line if
// terminal escape codes don't work.
func clearLine(out *os.File) {
const (
CUI = "\033[" // Control Sequence Introducer
CPL = CUI + "F" // Cursor Previous Line
EL = CUI + "K" // Erase in Line
)
// First, open a new line, which is guaranteed to work everywhere. Then, try
// to erase the line above with escape codes, if possible.
//
// (We use CRLF instead of LF to work around an apparent bug in WSL2's
// handling of CONOUT$. Only when running a Windows binary from WSL2, the
// cursor would not go back to the start of the line with a simple LF.
// Honestly, it's impressive CONIN$ and CONOUT$ work at all inside WSL2.)
fmt.Fprintf(out, "\r\n")
if enableVirtualTerminalProcessing == nil || enableVirtualTerminalProcessing(out) == nil {
fmt.Fprintf(out, CPL+EL)
}
}
// WithTerminal runs f with the terminal input and output files, if available.
// WithTerminal does not open a non-terminal stdin, so the caller does not need
// to check if stdin is in use.
func WithTerminal(f func(in, out *os.File) error) error {
if runtime.GOOS == "windows" {
in, err := os.OpenFile("CONIN$", os.O_RDWR, 0)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile("CONOUT$", os.O_WRONLY, 0)
if err != nil {
return err
}
defer out.Close()
return f(in, out)
} else if tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0); err == nil {
defer tty.Close()
return f(tty, tty)
} else if term.IsTerminal(int(os.Stdin.Fd())) {
return f(os.Stdin, os.Stdin)
} else {
return fmt.Errorf("standard input is not a terminal, and /dev/tty is not available: %v", err)
}
}
// ReadSecret reads a value from the terminal with no echo. The prompt is ephemeral.
func ReadSecret(prompt string) (s []byte, err error) {
err = WithTerminal(func(in, out *os.File) error {
fmt.Fprintf(out, "%s ", prompt)
defer clearLine(out)
s, err = term.ReadPassword(int(in.Fd()))
return err
})
return
}
// Confirm asks a yes/no question on the terminal. Only "y" or "yes" is yes.
func Confirm(prompt string) (yes bool, err error) {
err = WithTerminal(func(in, out *os.File) error {
fmt.Fprintf(out, "%s ", prompt)
line, err := bufio.NewReader(in).ReadString('\n')
if err != nil && line == "" {
return err
}
switch strings.ToLower(strings.TrimSpace(line)) {
case "y", "yes":
yes = true
}
return nil
})
return
}
// ReadPassphrase returns the passphrase from the environment variable env if
// it is set, otherwise prompts on the terminal.
func ReadPassphrase(prompt, env string) (string, error) {
if p, ok := os.LookupEnv(env); ok {
if p == "" {
return "", fmt.Errorf("%s is set but empty", env)
}
return p, nil
}
p, err := ReadSecret(prompt)
if err != nil {
return "", fmt.Errorf("could not read passphrase (set %s for non-interactive use): %v", env, err)
}
return string(p), nil
}
+46
View File
@@ -0,0 +1,46 @@
// SPDX-License-Identifier: GPL-3.0-or-later
package term
import (
"errors"
"os"
"syscall"
"golang.org/x/sys/windows"
)
func init() {
enableVirtualTerminalProcessing = func(out *os.File) error {
// Some instances of the Windows Console (e.g., cmd.exe and Windows PowerShell)
// do not have the virtual terminal processing enabled, which is necessary to
// make terminal escape sequences work. For this reason the clearLine function
// may not properly work. Here we enable the virtual terminal processing, if
// possible.
//
// See https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences.
const (
ENABLE_PROCESSED_OUTPUT uint32 = 0x1
ENABLE_VIRTUAL_TERMINAL_PROCESSING uint32 = 0x4
)
kernel32DLL := windows.NewLazySystemDLL("Kernel32.dll")
setConsoleMode := kernel32DLL.NewProc("SetConsoleMode")
var mode uint32
if err := syscall.GetConsoleMode(syscall.Handle(out.Fd()), &mode); err != nil {
return err
}
mode |= ENABLE_PROCESSED_OUTPUT
mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING
// If the SetConsoleMode function fails, the return value is zero.
// See https://learn.microsoft.com/en-us/windows/console/setconsolemode#return-value.
if ret, _, _ := setConsoleMode.Call(out.Fd(), uintptr(mode)); ret == 0 {
return errors.New("SetConsoleMode failed")
}
return nil
}
}
+159
View File
@@ -0,0 +1,159 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"bytes"
"crypto/rand"
"encoding/binary"
"encoding/pem"
"errors"
"os"
"path/filepath"
"github.com/andydixon/vault/internal/pqcrypt"
"golang.org/x/crypto/argon2"
"golang.org/x/crypto/chacha20poly1305"
)
// A key file holds exactly two passphrase slots. Each slot seals a
// (suite ID, master seed) pair under an Argon2id-derived key. When no decoy
// passphrase is set, the second slot is random bytes of identical length, so
// the file never reveals whether a decoy exists, and the real slot's position
// is chosen at random so it never reveals which one is real.
const (
keyPEMType = "VAULT KEY"
argonTime = 3
argonMemory = 64 * 1024 // KiB
argonThreads = 4
slotSaltSize = 16
seedSize = 32
slotPlainSize = 2 + seedSize
slotSize = slotSaltSize + chacha20poly1305.NonceSizeX + slotPlainSize + chacha20poly1305.Overhead
)
var keyMagic = []byte("VAULTKEY1")
// keyHeader is the authenticated, non-secret prefix of every key file.
func keyHeader() []byte {
h := append([]byte(nil), keyMagic...)
h = binary.BigEndian.AppendUint32(h, argonTime)
h = binary.BigEndian.AppendUint32(h, argonMemory)
return append(h, argonThreads)
}
// sealSlot encrypts the identity's seed under passphrase.
func sealSlot(id *pqcrypt.PQIdentity, passphrase string) ([]byte, error) {
slot := make([]byte, slotSaltSize+chacha20poly1305.NonceSizeX, slotSize)
if _, err := rand.Read(slot); err != nil {
return nil, err
}
aead, err := slotAEAD(passphrase, slot[:slotSaltSize])
if err != nil {
return nil, err
}
plain := binary.BigEndian.AppendUint16(nil, id.SuiteID())
seed := id.Seed()
plain = append(plain, seed...)
clear(seed)
defer clear(plain)
return aead.Seal(slot, slot[slotSaltSize:], plain, keyHeader()), nil
}
func slotAEAD(passphrase string, salt []byte) (interface {
Seal(dst, nonce, plaintext, ad []byte) []byte
Open(dst, nonce, ciphertext, ad []byte) ([]byte, error)
}, error) {
key := argon2.IDKey([]byte(passphrase), salt, argonTime, argonMemory, argonThreads, chacha20poly1305.KeySize)
defer clear(key)
return chacha20poly1305.NewX(key)
}
// writeKeyFile creates a new key file. decoy may be empty, in which case the
// second slot is filled with random bytes.
func writeKeyFile(path, passphrase, decoy string) error {
var slots [2][]byte
var b [1]byte
if _, err := rand.Read(b[:]); err != nil {
return err
}
realSlot := int(b[0] & 1)
for i, pass := range [2]string{passphrase, decoy} {
idx := realSlot ^ i
if pass == "" {
slots[idx] = make([]byte, slotSize)
if _, err := rand.Read(slots[idx]); err != nil {
return err
}
continue
}
id, err := pqcrypt.GeneratePQIdentity()
if err != nil {
return err
}
slots[idx], err = sealSlot(id, pass)
id.Close()
if err != nil {
return err
}
}
body := append(keyHeader(), slots[0]...)
body = append(body, slots[1]...)
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return err
}
if err := pem.Encode(f, &pem.Block{Type: keyPEMType, Bytes: body}); err != nil {
f.Close()
os.Remove(path)
return err
}
if err := f.Sync(); err != nil {
f.Close()
return err
}
return f.Close()
}
// unlockKeyFile returns the identity whose slot the passphrase opens. Both
// slots are always tried, so timing does not reveal which slot matched.
func unlockKeyFile(path, passphrase string) (*pqcrypt.PQIdentity, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
block, rest := pem.Decode(data)
header := keyHeader()
if block == nil || block.Type != keyPEMType || len(bytes.TrimSpace(rest)) != 0 ||
len(block.Bytes) != len(header)+2*slotSize || !bytes.Equal(block.Bytes[:len(header)], header) {
return nil, errors.New("not a vault key file")
}
var found *pqcrypt.PQIdentity
for i := range 2 {
slot := block.Bytes[len(header)+i*slotSize:][:slotSize]
aead, err := slotAEAD(passphrase, slot[:slotSaltSize])
if err != nil {
return nil, err
}
plain, err := aead.Open(nil, slot[slotSaltSize:][:chacha20poly1305.NonceSizeX],
slot[slotSaltSize+chacha20poly1305.NonceSizeX:], header)
if err != nil || found != nil {
clear(plain)
continue
}
found, err = pqcrypt.NewPQIdentityFromSeed(binary.BigEndian.Uint16(plain), plain[2:])
clear(plain)
if err != nil {
return nil, err
}
}
if found == nil {
return nil, errors.New("incorrect passphrase")
}
return found, nil
}
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"fmt"
"os"
"path/filepath"
)
const lockName = ".lock"
// lockVault takes the vault lock: exclusive for commands that change the
// vault, shared for commands that only read it. If another vault process
// holds a conflicting lock it says so and waits. The returned function
// releases the lock.
func lockVault(root string, exclusive bool) (func(), error) {
release, err := acquireLock(filepath.Join(root, lockName), exclusive, func() {
fmt.Fprintln(os.Stderr, "waiting for another vault process to finish")
})
if err != nil && !exclusive {
// A reader on read-only storage cannot create the lock file, and
// nothing can be writing there either.
return func() {}, nil
}
return release, err
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build !unix && !windows
package main
import (
"errors"
"fmt"
"io/fs"
"os"
"time"
)
// acquireLock falls back to an exclusive lock file where the OS offers no
// file locks. Every lock is exclusive here.
// ponytail: a crash leaves the lock file behind; the error names it for manual removal.
func acquireLock(path string, _ bool, waiting func()) (func(), error) {
for i := 0; ; i++ {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err == nil {
f.Close()
return func() { os.Remove(path) }, nil
}
if !errors.Is(err, fs.ErrExist) {
return nil, err
}
if i == 0 {
waiting()
}
if i == 600 {
return nil, fmt.Errorf("vault still locked after a minute; if no other vault process is running, remove %s", path)
}
time.Sleep(100 * time.Millisecond)
}
}
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build unix
package main
import (
"errors"
"io"
"os"
"golang.org/x/sys/unix"
)
// acquireLock uses fcntl record locks, which the kernel releases if the
// process dies, and which also work on NFS.
func acquireLock(path string, exclusive bool, waiting func()) (func(), error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
if err != nil {
return nil, err
}
lk := unix.Flock_t{Type: unix.F_RDLCK, Whence: io.SeekStart}
if exclusive {
lk.Type = unix.F_WRLCK
}
err = unix.FcntlFlock(f.Fd(), unix.F_SETLK, &lk)
if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EACCES) {
waiting()
for err = unix.EINTR; errors.Is(err, unix.EINTR); {
err = unix.FcntlFlock(f.Fd(), unix.F_SETLKW, &lk)
}
}
if err != nil {
f.Close()
return nil, err
}
return func() { f.Close() }, nil
}
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build windows
package main
import (
"errors"
"os"
"golang.org/x/sys/windows"
)
// acquireLock uses LockFileEx, which Windows releases if the process dies.
func acquireLock(path string, exclusive bool, waiting func()) (func(), error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
if err != nil {
return nil, err
}
h := windows.Handle(f.Fd())
var flags uint32
if exclusive {
flags = windows.LOCKFILE_EXCLUSIVE_LOCK
}
err = windows.LockFileEx(h, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped))
if errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
waiting()
err = windows.LockFileEx(h, flags, 0, 1, 0, new(windows.Overlapped))
}
if err != nil {
f.Close()
return nil, err
}
return func() { f.Close() }, nil
}
+728
View File
@@ -0,0 +1,728 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
// vault is a post-quantum encrypted file vault.
package main
import (
"bufio"
"errors"
"flag"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/andydixon/vault/internal/term"
)
const usage = `Usage:
vault genkey create a new passphrase-protected key
vault init [--filler SIZE] create a new, empty vault, with at least
SIZE of filler (e.g. 500M, 2G)
vault add FILE|DIR|URL [FOLDER] encrypt a file, a directory (recursively)
or a downloaded URL into the vault,
optionally under a folder inside the vault
vault migrate FILE|DIR [FOLDER] like add, then delete each original once
[--yes] it is stored; asks before overwriting
anything already in the vault, unless
--yes is given
vault mirror URL [FOLDER] crawl a website (same host only, one request
at a time) into FOLDER/HOST, rewriting its
links to relative paths
vault absorb [FOLDER] read a tar archive (optionally gzipped) from
standard input and add every file in it,
optionally under a folder inside the vault
vault ls [FOLDER] list one folder: its files, and its
sub-folders (ending in /) with their
total size; -r lists everything under it
vault list [FOLDER] list every stored file under FOLDER,
with full names
vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name)
vault get -r FOLDER [-o DIR] decrypt a folder and everything under it
into DIR (default: the folder's base name)
vault mv OLD NEW rename or move a file or folder; NEW
ending in / or naming an existing folder
moves OLD into it
vault cp [-r] OLD NEW copy a file, or with -r a folder, like mv;
copies share their encrypted data
vault rm NAME delete a stored file
vault rm -r FOLDER [-f] delete a folder and everything under it;
asks first unless -f (or -rf) is given
vault unlock [--for DURATION] remember the passphrase for DURATION
[--idle] (default 5m), so commands skip asking;
with --idle, DURATION since last used
vault lock forget it now
Options (any command):
--key PATH key file (default $VAULT_KEY, else ~/.vault/key)
--vault DIR vault directory (default $VAULT_DIR, else ~/.vault/store)
The passphrase is read from the terminal, or from $VAULT_PASSPHRASE. After
vault unlock, commands use the remembered key instead, unless
$VAULT_PASSPHRASE is set.
Decoy passphrase: genkey offers a second, optional passphrase
($VAULT_DECOY_PASSPHRASE when scripted). It unlocks a separate key, so
every command run with it sees and changes only the decoy files, and the
real files stay invisible. The key file does not reveal whether a decoy
exists, and init fills each vault with unreadable filler, so hidden
files look like filler to anyone holding only the decoy passphrase.`
const (
passphraseEnv = "VAULT_PASSPHRASE"
decoyEnv = "VAULT_DECOY_PASSPHRASE"
)
// unlock releases the vault lock taken by open. fail calls it too, because
// os.Exit skips deferred calls.
var unlock = func() {}
func main() {
defer func() { unlock() }()
if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, usage)
os.Exit(2)
}
cmd, args := os.Args[1], os.Args[2:]
if cmd == agentCmd && len(args) == 3 {
runAgent(args[0], args[1], args[2], os.Stdin, os.Stdout)
return
}
fset := flag.NewFlagSet("vault "+cmd, flag.ContinueOnError)
fset.SetOutput(io.Discard)
keyPath := fset.String("key", defaultPath("VAULT_KEY", "key"), "")
vaultDir := fset.String("vault", defaultPath("VAULT_DIR", "store"), "")
output := fset.String("o", "", "")
filler := fset.String("filler", "0", "")
yes := fset.Bool("yes", false, "")
ttl := fset.Duration("for", 5*time.Minute, "")
idle := fset.Bool("idle", false, "")
// rm's -r and -f; Go's flag package has no combined short flags, so
// -rf and -fr are spelled out.
recurse, force := fset.Bool("r", false, ""), fset.Bool("f", false, "")
rf, fr := fset.Bool("rf", false, ""), fset.Bool("fr", false, "")
pos, err := parseInterleaved(fset, args)
if err != nil {
fail("%v\n\n%s", err, usage)
}
want := func(min, max int) {
if len(pos) < min || len(pos) > max {
fail("wrong number of arguments for %s\n\n%s", cmd, usage)
}
}
switch cmd {
case "genkey":
want(0, 0)
if _, err := os.Stat(*keyPath); err == nil {
fail("%s already exists", *keyPath)
}
pass := newPassphrase("passphrase")
decoy := decoyPassphrase()
if decoy == pass {
fail("decoy passphrase must differ from the passphrase")
}
check(writeKeyFile(*keyPath, pass, decoy))
agentStop(*keyPath) // one left from an old key at this path would serve it
fmt.Fprintf(os.Stderr, "key written to %s\n", *keyPath)
case "init":
want(0, 0)
size, err := parseSize(*filler)
check(err)
check(initStore(*vaultDir, size))
fmt.Fprintf(os.Stderr, "vault created at %s\n", *vaultDir)
case "add", "migrate":
want(1, 2)
dest := ""
if len(pos) == 2 {
dest = strings.Trim(pos[1], "/")
}
s := open(*keyPath, *vaultDir, true)
cat, err := s.catalogue()
check(err)
if isURL(pos[0]) {
if cmd == "migrate" {
fail("cannot migrate a URL; use add")
}
check(addURL(s, cat, pos[0], dest))
break
}
// Never ingest, or delete, the vault or its key.
var skip []os.FileInfo
for _, p := range []string{*vaultDir, *keyPath} {
if fi, err := os.Stat(p); err == nil {
skip = append(skip, fi)
}
}
overwrite := confirmOverwrite
if *yes {
overwrite = func([]string) bool { return true }
}
check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite))
case "mirror":
want(1, 2)
if !isURL(pos[0]) {
fail("mirror needs an http:// or https:// URL")
}
dest := ""
if len(pos) == 2 {
dest = strings.Trim(pos[1], "/")
}
s := open(*keyPath, *vaultDir, true)
cat, err := s.catalogue()
check(err)
check(mirror(s, cat, pos[0], dest))
case "absorb":
want(0, 1)
if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
fail("absorb reads a tar archive from standard input; pipe one in")
}
dest := ""
if len(pos) == 1 {
dest = strings.Trim(pos[0], "/")
}
// Wait for the archive to start arriving before asking for the
// passphrase, so a program feeding the pipe (gpg, ssh) has finished
// with the terminal and cannot bury the prompt.
in := bufio.NewReader(os.Stdin)
if _, err := in.Peek(1); err != nil {
fail("nothing to absorb on standard input")
}
s := open(*keyPath, *vaultDir, true)
cat, err := s.catalogue()
check(err)
check(absorb(s, cat, in, dest))
case "list", "ls":
want(0, 1)
prefix := ""
if len(pos) == 1 {
prefix = strings.Trim(pos[0], "/")
}
cat, err := open(*keyPath, *vaultDir, false).catalogue()
check(err)
if cmd == "ls" && !*recurse {
check(ls(os.Stdout, cat, prefix))
break
}
list(cat, prefix)
case "get":
want(1, 1)
out := *output
if out == "" {
out = path.Base(strings.TrimRight(pos[0], "/"))
}
s := open(*keyPath, *vaultDir, false)
if !*recurse {
check(s.get(pos[0], out))
break
}
cat, err := s.catalogue()
check(err)
folder := strings.Trim(pos[0], "/")
names := under(cat, folder)
if folder == "" || len(names) == 0 || len(cat[folder]) > 0 {
fail("%s: not a folder in the vault", pos[0])
}
for _, n := range names {
dst := filepath.Join(out, filepath.FromSlash(n[len(folder)+1:]))
check(os.MkdirAll(filepath.Dir(dst), 0o700))
check(s.getRecord(cat[n][0], dst))
fmt.Fprintf(os.Stderr, "extracted %s\n", dst)
}
case "mv", "rename", "cp":
want(2, 2)
s := open(*keyPath, *vaultDir, true)
cat, err := s.catalogue()
check(err)
pairs, err := transfers(cat, pos[0], pos[1], cmd != "cp" || *recurse)
check(err)
verb := "copied"
if cmd != "cp" {
verb = "moved"
}
check(s.copyAll(cat, pairs, cmd != "cp", func(from, to string) {
fmt.Fprintf(os.Stderr, "%s %s -> %s\n", verb, from, to)
}))
case "rm", "delete":
want(1, 1)
recursive, forced := *recurse || *rf || *fr, *force || *rf || *fr
name := pos[0]
if recursive {
name = strings.Trim(name, "/")
if name == "" {
fail("refusing to remove the whole vault")
}
}
s := open(*keyPath, *vaultDir, true)
cat, err := s.catalogue()
check(err)
var names []string
if recursive {
names = under(cat, name)
} else if len(cat[name]) > 0 {
names = []string{name}
}
if len(names) == 0 {
if forced {
break
}
fail("%s: not in the vault", name)
}
var size int64
for _, n := range names {
size += cat[n][0].Size
}
if recursive && !forced && !confirm("these files will be deleted, with no undo:", names,
fmt.Sprintf("delete %d file(s), %s? [y/N]", len(names), humanSize(size))) {
fail("rm cancelled; nothing changed (-f skips the question)")
}
check(s.removeAll(cat, names, func(n string) { fmt.Fprintf(os.Stderr, "removed %s\n", n) }))
if recursive {
fmt.Fprintf(os.Stderr, "removed %d file(s), %s\n", len(names), humanSize(size))
}
case "unlock":
want(0, 0)
if *ttl <= 0 {
fail("--for needs a positive duration, such as 5m or 1h")
}
pass, err := term.ReadPassphrase("Enter passphrase:", passphraseEnv)
check(err)
id, err := unlockKeyFile(*keyPath, pass)
check(err)
check(agentStart(*keyPath, id, *ttl, *idle))
id.Close()
if *idle {
fmt.Fprintf(os.Stderr, "unlocked until unused for %s; vault lock forgets the key sooner\n", *ttl)
} else {
fmt.Fprintf(os.Stderr, "unlocked until %s; vault lock forgets the key sooner\n",
time.Now().Add(*ttl).Format("15:04:05"))
}
case "lock":
want(0, 0)
if agentStop(*keyPath) {
fmt.Fprintln(os.Stderr, "locked")
} else {
fmt.Fprintln(os.Stderr, "nothing was unlocked")
}
case "help", "-h", "--help":
fmt.Println(usage)
default:
fail("unknown command %q\n\n%s", cmd, usage)
}
}
// transfers works out, like mv(1) and cp(1), which stored names src and dst
// mean: each file to copy or move, paired with its new name. src is a file,
// or with recursive a folder. dst ending in / or naming an existing folder
// means into it. No new name may already exist.
func transfers(cat map[string][]*record, src, dst string, recursive bool) ([][2]string, error) {
into := dst == "" || strings.HasSuffix(dst, "/")
src, dst = strings.Trim(src, "/"), strings.Trim(dst, "/")
if src == "" {
return nil, errors.New("name a file or folder to copy or move")
}
if dst == "" || len(cat[dst]) == 0 && len(under(cat, dst)) > 0 {
into = true
}
if into {
dst = path.Join(dst, path.Base(src))
}
var pairs [][2]string
if len(cat[src]) > 0 {
pairs = [][2]string{{src, dst}}
} else {
names := under(cat, src)
if len(names) == 0 {
return nil, fmt.Errorf("%s: not in the vault", src)
}
if !recursive {
return nil, fmt.Errorf("%s is a folder; use -r", src)
}
if dst == src || strings.HasPrefix(dst, src+"/") {
return nil, fmt.Errorf("cannot put %s inside itself", src)
}
for _, n := range names {
pairs = append(pairs, [2]string{n, dst + n[len(src):]})
}
}
for _, p := range pairs {
if !validName(p[1]) {
return nil, fmt.Errorf("invalid vault path %q", p[1])
}
if len(cat[p[1]]) > 0 {
return nil, fmt.Errorf("%s already exists", p[1])
}
}
return pairs, nil
}
// parseInterleaved lets flags appear before, between or after positional
// arguments.
func parseInterleaved(fset *flag.FlagSet, args []string) ([]string, error) {
var pos []string
for {
if err := fset.Parse(args); err != nil {
return nil, err
}
if fset.NArg() == 0 {
return pos, nil
}
pos = append(pos, fset.Arg(0))
args = fset.Args()[1:]
}
}
// parseSize reads a byte count with an optional K, M, G or T suffix, in
// powers of 1024.
func parseSize(s string) (int64, error) {
num, shift := strings.TrimSuffix(strings.ToUpper(s), "B"), 0
for i, unit := range "KMGT" {
if strings.HasSuffix(num, string(unit)) {
num, shift = strings.TrimSuffix(num, string(unit)), 10*(i+1)
}
}
n, err := strconv.ParseInt(num, 10, 64)
if err != nil || n < 0 || n > fileLimit>>shift {
return 0, fmt.Errorf("invalid size %q", s)
}
return n << shift, nil
}
func defaultPath(env, name string) string {
if v := os.Getenv(env); v != "" {
return v
}
home, err := os.UserHomeDir()
if err != nil {
fail("cannot find home directory; set %s", env)
}
return filepath.Join(home, ".vault", name)
}
// open unlocks the key and opens the vault, holding the vault lock
// exclusively for writers and shared for readers.
func open(keyPath, vaultDir string, exclusive bool) *store {
id, err := keyIdentity(keyPath)
check(err)
s, err := openStore(vaultDir, id)
check(err)
unlock, err = lockVault(vaultDir, exclusive)
check(err)
return s
}
func newPassphrase(what string) string {
if p, ok := os.LookupEnv(passphraseEnv); ok && p != "" {
return p
}
p, err := term.ReadPassphrase("Enter "+what+":", passphraseEnv)
check(err)
if p == "" {
fail("%s must not be empty", what)
}
again, err := term.ReadPassphrase("Confirm "+what+":", passphraseEnv)
check(err)
if p != again {
fail("%ss do not match", what)
}
return p
}
// decoyPassphrase asks for an optional second passphrase. Empty means none.
func decoyPassphrase() string {
if p, ok := os.LookupEnv(decoyEnv); ok {
return p
}
if _, ok := os.LookupEnv(passphraseEnv); ok {
return "" // non-interactive run without a decoy
}
p, err := term.ReadSecret("Enter decoy passphrase (empty for none):")
check(err)
if len(p) == 0 {
return ""
}
again, err := term.ReadSecret("Confirm decoy passphrase:")
check(err)
if string(p) != string(again) {
fail("decoy passphrases do not match")
}
return string(p)
}
// addPath adds a file, or every regular file under a directory, beneath dest,
// keeping the directory structure. A directory named "." or ".." adds its
// contents directly beneath dest. With migrate, each original is deleted
// once stored, then any directories left empty, and nothing is stored
// unless overwrite approves the vault names it would replace, if any.
func addPath(s *store, cat map[string][]*record, src, dest string, migrate bool, skip []os.FileInfo, overwrite func([]string) bool) error {
info, err := os.Stat(src)
if err != nil {
return err
}
base := filepath.Base(filepath.Clean(src))
if base == "." || base == ".." || base == string(filepath.Separator) {
base = ""
}
skipped := func(fi os.FileInfo) bool {
for _, s := range skip {
if os.SameFile(fi, s) {
return true
}
}
return false
}
if skipped(info) {
return fmt.Errorf("%s is the vault or its key", src)
}
type item struct{ src, name string }
var items []item
var dirs []string
if !info.IsDir() {
items = append(items, item{src, path.Join(dest, filepath.Base(src))})
} else {
err = filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if fi, err := d.Info(); err == nil && skipped(fi) {
fmt.Fprintf(os.Stderr, "skipping %s: the vault or its key\n", p)
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if d.IsDir() {
dirs = append(dirs, p)
return nil
}
if !d.Type().IsRegular() {
fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", p)
return nil
}
rel, err := filepath.Rel(src, p)
if err != nil {
return err
}
items = append(items, item{p, path.Join(dest, base, filepath.ToSlash(rel))})
return nil
})
if err != nil {
return err
}
}
if migrate {
var existing []string
for _, it := range items {
if len(cat[it.name]) > 0 {
existing = append(existing, it.name)
}
}
if len(existing) > 0 && !overwrite(existing) {
return errors.New("migrate cancelled; nothing changed")
}
}
for _, it := range items {
if err := addOne(s, cat, it.src, it.name, migrate); err != nil {
return err
}
}
if migrate {
// Children come after their parents in walk order, so go backwards.
// Directories still holding skipped files stay.
for i := len(dirs) - 1; i >= 0; i-- {
os.Remove(dirs[i])
}
}
return nil
}
// confirmOverwrite lists the vault names a migrate would replace and asks
// on the terminal whether to go ahead. No terminal means no.
func confirmOverwrite(names []string) bool {
return confirm("these files are already in the vault and would be overwritten:", names,
fmt.Sprintf("overwrite %d file(s)? [y/N]", len(names)))
}
// confirm prints intro and names, the first 20 of them, then asks question
// on the terminal. No terminal means no.
func confirm(intro string, names []string, question string) bool {
fmt.Fprintln(os.Stderr, intro)
for i, name := range names {
if i == 20 {
fmt.Fprintf(os.Stderr, " … and %d more\n", len(names)-i)
break
}
fmt.Fprintf(os.Stderr, " %s\n", name)
}
ok, err := term.Confirm(question)
if err != nil {
fmt.Fprintf(os.Stderr, "cannot ask for confirmation: %v\n", err)
}
return ok
}
func isURL(s string) bool {
return strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://")
}
const chromeUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
// addURL downloads rawURL to a private temporary file and adds it beneath
// dest, named after the last element of the final (post-redirect) URL path.
func addURL(s *store, cat map[string][]*record, rawURL, dest string) error {
req, err := http.NewRequest("GET", rawURL, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", chromeUA)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%s: %s", rawURL, resp.Status)
}
name := path.Base(resp.Request.URL.Path)
if name == "/" || name == "." {
name = resp.Request.URL.Hostname()
}
tmp, err := os.CreateTemp("", "vault-download-")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
if _, err := io.Copy(tmp, resp.Body); err != nil {
return fmt.Errorf("%s: %w", rawURL, err)
}
if t, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil {
os.Chtimes(tmp.Name(), t, t)
}
return addOne(s, cat, tmp.Name(), path.Join(dest, name), false)
}
func addOne(s *store, cat map[string][]*record, src, name string, migrate bool) error {
replaced, err := s.add(cat, src, name)
if err != nil {
return err
}
verb := "added"
if replaced {
verb = "replaced"
}
if migrate {
if err := os.Remove(src); err != nil {
return err
}
verb = "migrated"
}
fmt.Fprintf(os.Stderr, "%s %s\n", verb, name)
return nil
}
// ls prints one level of the vault, like ls(1): the files directly in
// folder, and each sub-folder once, marked with a trailing slash and shown
// with the total size and newest date of everything under it. Folders come
// first. A folder that is a file's name lists just that file.
func ls(w io.Writer, cat map[string][]*record, folder string) error {
type entry struct {
size int64
modified time.Time
}
entries := map[string]*entry{}
for name, rs := range cat {
rel := name
switch {
case folder == "":
case name == folder:
rel = path.Base(name)
case strings.HasPrefix(name, folder+"/"):
rel = name[len(folder)+1:]
default:
continue
}
if i := strings.IndexByte(rel, '/'); i >= 0 {
rel = rel[:i+1]
}
e := entries[rel]
if e == nil {
e = &entry{}
entries[rel] = e
}
r := rs[0]
e.size += r.Size
if r.Modified.After(e.modified) {
e.modified = r.Modified
}
}
if len(entries) == 0 && folder != "" {
return fmt.Errorf("%s: not in the vault", folder)
}
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Slice(names, func(i, j int) bool {
di, dj := strings.HasSuffix(names[i], "/"), strings.HasSuffix(names[j], "/")
if di != dj {
return di
}
return names[i] < names[j]
})
for _, name := range names {
e := entries[name]
fmt.Fprintf(w, "%10s %s %s\n", humanSize(e.size), e.modified.Local().Format("2006-01-02 15:04"), name)
}
return nil
}
// list prints every file at or under prefix, with its full name.
func list(cat map[string][]*record, prefix string) {
var names []string
if prefix == "" {
for name := range cat {
names = append(names, name)
}
sort.Strings(names)
} else {
names = under(cat, prefix)
}
for _, name := range names {
r := cat[name][0]
fmt.Printf("%10s %s %s\n", humanSize(r.Size), r.Modified.Local().Format("2006-01-02 15:04"), name)
}
}
func humanSize(n int64) string {
const unit = 1024
if n < unit {
return fmt.Sprintf("%d B", n)
}
div, exp := int64(unit), 0
for m := n / unit; m >= unit; m /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}
func check(err error) {
if err != nil {
fail("%v", err)
}
}
func fail(format string, args ...any) {
unlock()
fmt.Fprintf(os.Stderr, "vault: "+format+"\n", args...)
os.Exit(1)
}
+150
View File
@@ -0,0 +1,150 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"fmt"
"html"
"io"
"net/http"
"net/url"
"os"
"path"
"regexp"
"strings"
)
var (
// Attribute values in HTML, and url(...) in CSS: group 1 is everything
// up to the URL, group 2 is the URL.
htmlURLs = regexp.MustCompile(`(?i)(\b(?:href|src|action|poster)\s*=\s*["'])([^"']*)`)
cssURLs = regexp.MustCompile(`(?i)(url\(\s*["']?)([^"')\s]+)`)
)
// mirror crawls the site at rawURL, one request at a time, staying on its
// host, and adds every page and asset beneath dest/host. Links in HTML and
// CSS that point into the mirror are rewritten to relative paths, so the
// copy browses correctly once fetched out of the vault. A page that fails
// is reported and skipped.
// ponytail: regex rewriting, so links built by scripts or in srcset are left
// as they were; swap in golang.org/x/net/html if that matters.
func mirror(s *store, cat map[string][]*record, rawURL, dest string) error {
start, err := url.Parse(rawURL)
if err != nil {
return err
}
host := strings.ToLower(start.Host)
root := path.Join(dest, host)
seen := map[string]bool{}
queue := []*url.URL{start}
seen[key(start)] = true
for len(queue) > 0 {
u := queue[0]
queue = queue[1:]
err := mirrorOne(s, cat, u, host, root, func(l *url.URL) {
if k := key(l); !seen[k] {
seen[k] = true
queue = append(queue, l)
}
})
if err != nil {
fmt.Fprintf(os.Stderr, "skipping %s: %v\n", u, err)
}
}
return nil
}
// key identifies a page independent of its fragment.
func key(u *url.URL) string {
c := *u
c.Fragment = ""
return c.String()
}
// localPath is where u lives inside the mirror, relative to its root.
func localPath(u *url.URL) string {
p := strings.TrimPrefix(u.Path, "/")
if p == "" || strings.HasSuffix(p, "/") {
p += "index.html"
}
if u.RawQuery != "" {
p += "_" + url.QueryEscape(u.RawQuery)
}
return p
}
// relative gives the link from the file "from" to the file "to", both
// slash-separated paths relative to the same root.
func relative(from, to string) string {
f := strings.Split(path.Dir(from), "/")
t := strings.Split(to, "/")
if f[0] == "." {
f = nil
}
for len(f) > 0 && len(t) > 1 && f[0] == t[0] {
f, t = f[1:], t[1:]
}
return strings.Repeat("../", len(f)) + strings.Join(t, "/")
}
func mirrorOne(s *store, cat map[string][]*record, u *url.URL, host, root string, enqueue func(*url.URL)) error {
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", chromeUA)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%s", resp.Status)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
self := localPath(u)
// rewrite maps a link found in the page to a relative one, queueing it.
rewrite := func(re *regexp.Regexp, text string) string {
return re.ReplaceAllStringFunc(text, func(m string) string {
sub := re.FindStringSubmatch(m)
l, err := u.Parse(html.UnescapeString(sub[2]))
if err != nil || (l.Scheme != "http" && l.Scheme != "https") || strings.ToLower(l.Host) != host {
return m
}
frag := l.Fragment
l.Fragment = ""
if l.Path == "" && l.RawQuery == "" {
l.Path = "/"
}
enqueue(l)
r := relative(self, localPath(l))
if frag != "" {
r += "#" + frag
}
return sub[1] + r
})
}
switch ct := strings.ToLower(resp.Header.Get("Content-Type")); {
case strings.HasPrefix(ct, "text/html"):
body = []byte(rewrite(cssURLs, rewrite(htmlURLs, string(body))))
case strings.HasPrefix(ct, "text/css"):
body = []byte(rewrite(cssURLs, string(body)))
}
tmp, err := os.CreateTemp("", "vault-mirror-")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
if _, err := tmp.Write(body); err != nil {
return err
}
return addOne(s, cat, tmp.Name(), path.Join(root, self), false)
}
+208
View File
@@ -0,0 +1,208 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// Test-only vector and fixture generator for the Rust port. It calls the
// frozen implementation and records what it computes; it changes nothing in
// the vault binary. Run through oracle/gen-vectors.sh.
package main
import (
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path"
"path/filepath"
"testing"
"time"
"github.com/andydixon/vault/internal/pqcrypt"
"golang.org/x/crypto/argon2"
)
// Synthetic, public test passphrases. Never use them for anything real.
const (
oracleReal = "oracle-real passphrase"
oracleDecoy = "oracle-decoy passphrase"
)
func TestOracleVectors(t *testing.T) {
dir := os.Getenv("VAULT_ORACLE_OUT")
if dir == "" {
t.Skip("set VAULT_ORACLE_OUT to write vectors")
}
out := map[string]any{"provenance": "oracle/go at 4c1507c, oracle_vectors_test.go"}
// Key files.
keys := map[string]any{}
for _, k := range []struct{ name, real, decoy string }{
{"key-with-decoy.pem", oracleReal, oracleDecoy},
{"key-no-decoy.pem", oracleReal, ""},
} {
p := filepath.Join(dir, k.name)
os.Remove(p)
if err := writeKeyFile(p, k.real, k.decoy); err != nil {
t.Fatal(err)
}
entry := map[string]any{}
for label, pass := range map[string]string{"real": k.real, "decoy": k.decoy} {
if pass == "" {
continue
}
id, err := unlockKeyFile(p, pass)
if err != nil {
t.Fatal(err)
}
entry[label] = map[string]any{"passphrase": pass, "suite": id.SuiteID(), "seed": hex.EncodeToString(id.Seed())}
}
keys[k.name] = entry
}
out["keys"] = keys
var argon []map[string]string
for i, pass := range []string{"", "a", "correct horse", "pässwörd ü", "x\x00y"} {
salt := make([]byte, 16)
for j := range salt {
salt[j] = byte(i*16 + j)
}
k := argon2.IDKey([]byte(pass), salt, argonTime, argonMemory, argonThreads, 32)
argon = append(argon, map[string]string{"passphrase_hex": hex.EncodeToString([]byte(pass)), "salt": hex.EncodeToString(salt), "key": hex.EncodeToString(k)})
}
out["argon2id"] = argon
out["key_header_hex"] = hex.EncodeToString(keyHeader())
// Object IDs and sizes.
seed := "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"
var ids []string
for i := range int64(4) {
ids = append(ids, objectID(seed, i))
}
out["object_ids"] = map[string]any{"seed": seed, "ids": ids, "big_index": objectID(seed, 1<<40), "big_index_value": int64(1 << 40)}
padded := map[string]int64{}
counts := map[string]int64{}
for _, n := range []int64{0, 1, 16, 65535, 65536, 65537, 1 << 20, chunkSize - 1, chunkSize, chunkData - 1, chunkData, chunkData + 1, 2 * chunkData, fileLimit} {
padded[fmt.Sprint(n)] = paddedSize(n)
counts[fmt.Sprint(n)] = objectCount(n)
}
out["padded_size"] = padded
out["object_count"] = counts
names := map[string]bool{}
for _, n := range []string{"", "a", "a/b", "/a", "a/", "a//b", "./a", "a/./b", "../a", "a/..", "..", ".", "a/../b", "a\\b", "a\x00b",
"a\nb", "a\x7fb", "a\u0085b", "ü/ñ", "\xff", "a b", " a", "...", "a/...", "..a", "a/.b", string(make([]byte, 1024)), longName(1024), longName(1025), "a​b", "
"} {
names[hex.EncodeToString([]byte(n))] = validName(n)
}
out["valid_name_hex"] = names
sizes := map[string]any{}
for _, s := range []string{"0", "512", "2k", "500M", "2G", "1TB", "", "G", "-1", "2X", "1.5G", "2T", "+5", "1KB", "1MK", "1kb", "B", "1T", "1099511627776", "1099511627777", " 1", "0x10", "1024G", "1025G", "-0", "1bb"} {
n, err := parseSize(s)
if err != nil {
sizes[s] = nil
} else {
sizes[s] = n
}
}
out["parse_size"] = sizes
human := map[string]string{}
for _, n := range []int64{0, 1, 1023, 1024, 1025, 1536, 1101, 10239, 10240, 1048575, 1048576, 1 << 30, 1<<40 - 1, 1 << 40, 1<<62 + 12345, 1<<63 - 1, 1075, 1126, 1177, 1638, 1689} {
human[fmt.Sprint(n)] = humanSize(n)
}
out["human_size"] = human
clean := map[string]string{}
for _, p := range []string{"", ".", "..", "/", "//", "a/b/../c", "/../a", "a/./b/", "../../a", "a//b", "./", "/a/b/..", "a/b/../../..", "abc/def/ghi/../.."} {
clean[p] = path.Clean(p)
}
out["path_clean"] = clean
durs := map[string]any{}
for _, d := range []string{"5m", "1h", "90s", "1.5h", "300ms", "1h30m", "0", "-5m", "", "5", "1d", "2h45m30.5s", "1us", "1µs", "1ns", ".5s", "5.s", "+5m", "9223372036854775807ns", "9223372036854775808ns"} {
v, err := time.ParseDuration(d)
if err != nil {
durs[d] = nil
} else {
durs[d] = map[string]any{"ns": int64(v), "string": v.String()}
}
}
out["durations"] = durs
// Record JSON, exactly as written.
var recs []map[string]any
for _, r := range []record{
{Name: "plain.txt", Seed: seed, Size: 5, Mode: 0o644, Modified: time.Unix(1700000000, 123456789).UTC(), Added: time.Unix(1700000001, 0).UTC()},
{Name: "<a&b>/

'\"\\x", Seed: seed, Size: 0, Mode: 0o755, Modified: time.Unix(0, 0).UTC(), Added: time.Unix(1, 100).UTC()},
{Name: "ü/日本語", Seed: seed, Size: 1 << 40, Mode: 0o600, Modified: time.Unix(-1, 500000000).UTC(), Added: time.Unix(253402300799, 999999999).UTC()},
{Name: "tab\there", Seed: seed, Size: 1, Mode: 0, Modified: time.Unix(1234567890, 120000000).UTC(), Added: time.Unix(1234567890, 1000).UTC()},
} {
b, err := json.Marshal(&r)
if err != nil {
t.Fatal(err)
}
recs = append(recs, map[string]any{"json": string(b), "name": r.Name, "modified_unix_nano": r.Modified.UnixNano(), "added_unix_nano": r.Added.UnixNano()})
}
out["records"] = recs
writeMiniVault(t, filepath.Join(dir, "mini-vault"), filepath.Join(dir, "key-with-decoy.pem"))
b, _ := json.MarshalIndent(out, "", " ")
if err := os.WriteFile(filepath.Join(dir, "store.json"), append(b, '\n'), 0o644); err != nil {
t.Fatal(err)
}
}
func longName(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = 'a' + byte(i%26)
}
return string(b)
}
// writeMiniVault builds a small vault through the real store code, without
// init's filler (which would make the fixture megabytes): three real files,
// one decoy file and one record sealed to a throwaway key.
func writeMiniVault(t *testing.T, root, keyPath string) {
os.RemoveAll(root)
for _, d := range []string{recordsDir, objectsDir} {
for i := range 256 {
if err := os.MkdirAll(filepath.Join(root, d, fmt.Sprintf("%02x", i)), 0o700); err != nil {
t.Fatal(err)
}
}
}
os.WriteFile(filepath.Join(root, markerName), []byte(markerContent), 0o600)
src := filepath.Join(t.TempDir(), "src")
put := func(s *store, name string, body []byte, mode os.FileMode, mtime time.Time) {
os.WriteFile(src, body, 0o600)
os.Chmod(src, mode)
os.Chtimes(src, mtime, mtime)
cat, err := s.catalogue()
if err != nil {
t.Fatal(err)
}
if _, err := s.add(cat, src, name); err != nil {
t.Fatal(err)
}
}
real, _ := unlockKeyFile(keyPath, oracleReal)
decoy, _ := unlockKeyFile(keyPath, oracleDecoy)
throwaway, _ := pqcrypt.GeneratePQIdentity()
rs := &store{root: root, id: real}
boundary := make([]byte, minPadding-idSize)
for i := range boundary {
boundary[i] = byte(i * 7)
}
put(rs, "hello.txt", []byte("hello"), 0o644, time.Unix(1700000000, 123456789))
put(rs, "dir/empty", nil, 0o600, time.Unix(1600000000, 0))
put(rs, "dir/ünïcode.bin", boundary, 0o755, time.Unix(1500000000, 999999999))
put(&store{root: root, id: decoy}, "shopping.txt", []byte("milk\n"), 0o600, time.Unix(1700000000, 0))
put(&store{root: root, id: throwaway}, "foreign", []byte("not yours"), 0o600, time.Unix(1700000000, 0))
for _, d := range []string{recordsDir, objectsDir} {
shards, _ := filepath.Glob(filepath.Join(root, d, "??"))
for _, p := range append(shards, filepath.Join(root, d)) {
os.Chtimes(p, epoch, epoch)
}
}
}
+683
View File
@@ -0,0 +1,683 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"bytes"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"math"
mrand "math/rand/v2"
"os"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"unicode"
"unicode/utf8"
"github.com/andydixon/vault/internal/pqcrypt"
)
// On disk a vault is:
//
// DIR/.vault marker
// DIR/records/<xx>/<id> one encrypted catalogue record per file
// DIR/objects/<xx>/<id> the encrypted body of a file, in one or more chunks
//
// <xx> is the first two hex digits of <id>, spreading entries over 256
// shards. IDs look random: record IDs are, and a file's object IDs derive
// from a secret seed in its record, so nothing on disk links a file's
// chunks. Records are padded to one fixed size. A body is cut into
// chunkSize objects and the last is padded to a power-of-two bucket, so
// every object is 64 KiB to 4 MiB whatever the file size. Every ciphertext's
// and shard's mtime is pinned to the epoch.
const (
markerName = ".vault"
markerContent = "VAULT 2\n"
recordsDir = "records"
objectsDir = "objects"
recordSize = 4096
nameLimit = 1024
fileLimit = int64(1) << 40
minPadding = int64(64 << 10)
chunkSize = minPadding << 6 // 4 MiB, the largest bucket
chunkData = chunkSize - idSize
idSize = 16
objectSeedSize = 32
fillerMin = 8
fillerMax = 32
)
var epoch = time.Unix(0, 0)
type record struct {
Name string `json:"name"`
Seed string `json:"seed"` // derives the file's object IDs
Size int64 `json:"size"`
Mode uint32 `json:"mode"`
Modified time.Time `json:"modified"`
Added time.Time `json:"added"`
id string // record file ID; not stored
}
type store struct {
root string
id *pqcrypt.PQIdentity
noSync bool // filler needs no per-file durability
}
// initStore creates a vault at root holding at least filler bytes of filler.
func initStore(root string, filler int64) error {
if _, err := os.Stat(filepath.Join(root, markerName)); err == nil {
return fmt.Errorf("%s is already a vault", root)
}
// Every shard exists from the start, so the tree's shape says nothing.
for _, dir := range []string{recordsDir, objectsDir} {
for i := range 256 {
if err := os.MkdirAll(filepath.Join(root, dir, fmt.Sprintf("%02x", i)), 0o700); err != nil {
return err
}
}
}
if err := writeFiller(root, filler); err != nil {
return err
}
for _, dir := range []string{recordsDir, objectsDir} {
shards, _ := filepath.Glob(filepath.Join(root, dir, "??"))
for _, p := range append(shards, filepath.Join(root, dir)) {
if err := os.Chtimes(p, epoch, epoch); err != nil {
return err
}
}
}
return os.WriteFile(filepath.Join(root, markerName), []byte(markerContent), 0o600)
}
// writeFiller stores random entries sealed to a key that is thrown away.
// Every vault therefore holds entries nobody can open, so a file hidden from
// a decoy passphrase is indistinguishable from filler. Filler entries are
// real entries, written by the same code as add, of log-uniformly random
// sizes: fillerMin to fillerMax of them, or more until they total target
// bytes.
func writeFiller(root string, target int64) error {
throwaway, err := pqcrypt.GeneratePQIdentity()
if err != nil {
return err
}
defer throwaway.Close()
s := &store{root: root, id: throwaway, noSync: true}
// Bigger targets allow bigger files, so the filler holds multi-chunk
// files in about the proportion a real collection would.
upper := float64(min(max(target/8, chunkSize), 1<<30))
count := fillerMin + mrand.IntN(fillerMax-fillerMin+1)
for i, total := 0, int64(0); i < count || total < target; i++ {
size := int64(1024 * math.Exp(mrand.Float64()*math.Log(upper/1024)))
seed, err := randomHex(objectSeedSize)
if err != nil {
return err
}
if err := s.writeObjects(rand.Reader, seed, size); err != nil {
return err
}
name, err := randomHex(8)
if err != nil {
return err
}
now := time.Now().UTC()
if err := s.writeRecord(&record{Name: name, Seed: seed, Size: size, Mode: 0o600, Modified: now, Added: now}); err != nil {
return err
}
total += size
}
return nil
}
func openStore(root string, id *pqcrypt.PQIdentity) (*store, error) {
marker, err := os.ReadFile(filepath.Join(root, markerName))
if err != nil || string(marker) != markerContent {
return nil, fmt.Errorf("%s is not a vault; run vault init", root)
}
return &store{root: root, id: id}, nil
}
func randomHex(n int) (string, error) {
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
func validHex(s string, n int) bool {
b, err := hex.DecodeString(s)
return err == nil && len(b) == n && hex.EncodeToString(b) == s
}
func validID(s string) bool { return validHex(s, idSize) }
// objectID derives the ID of a file's i-th object from its record's seed.
// Without the seed the IDs are unlinkable, so a file's chunks can't be told
// apart from any other object.
func objectID(seed string, i int64) string {
key, _ := hex.DecodeString(seed)
m := hmac.New(sha256.New, key)
binary.Write(m, binary.BigEndian, i)
return hex.EncodeToString(m.Sum(nil)[:idSize])
}
// objectCount is how many objects hold a file of size bytes.
func objectCount(size int64) int64 {
return max(1, (size+chunkData-1)/chunkData)
}
func (s *store) path(dir, id string) string {
return filepath.Join(s.root, dir, id[:2], id)
}
// removeFile deletes dir/id and re-pins its shard's mtime.
func (s *store) removeFile(dir, id string) error {
p := s.path(dir, id)
if err := os.Remove(p); err != nil {
return err
}
return os.Chtimes(filepath.Dir(p), epoch, epoch)
}
func validName(name string) bool {
if name == "" || len(name) > nameLimit || !utf8.ValidString(name) || path.Clean(name) != name ||
path.IsAbs(name) || name == "." || name == ".." || strings.HasPrefix(name, "../") || strings.Contains(name, "\\") {
return false
}
for _, r := range name {
if unicode.IsControl(r) {
return false
}
}
return true
}
// paddedSize rounds size up to the next power-of-two bucket.
func paddedSize(size int64) int64 {
p := minPadding
for p < size {
p *= 2
}
return p
}
// writeCiphertext encrypts to dir/id via a temporary file, so a partial
// ciphertext never appears under a real ID.
func (s *store) writeCiphertext(dir, id string, write func(io.Writer) error) error {
dst := s.path(dir, id)
dir = filepath.Dir(dst)
tmp, err := os.CreateTemp(dir, ".pending-")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
w, err := pqcrypt.Encrypt(tmp, s.id.Recipient())
if err != nil {
return err
}
if err := write(w); err != nil {
return err
}
if err := w.Close(); err != nil {
return err
}
if !s.noSync {
if err := tmp.Sync(); err != nil {
return err
}
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Chtimes(tmp.Name(), epoch, epoch); err != nil {
return err
}
if err := os.Rename(tmp.Name(), dst); err != nil {
return err
}
if err := os.Chtimes(dir, epoch, epoch); err != nil || s.noSync {
return err
}
return syncDir(dir)
}
func syncDir(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
defer d.Close()
return d.Sync()
}
func (s *store) writeRecord(r *record) error {
data, err := json.Marshal(r)
if err != nil {
return err
}
if len(data) > recordSize-4 {
return errors.New("catalogue record too large")
}
plain := make([]byte, recordSize)
binary.BigEndian.PutUint32(plain, uint32(len(data)))
copy(plain[4:], data)
if _, err := rand.Read(plain[4+len(data):]); err != nil {
return err
}
if r.id, err = randomHex(idSize); err != nil {
return err
}
return s.writeCiphertext(recordsDir, r.id, func(w io.Writer) error {
_, err := w.Write(plain)
return err
})
}
// catalogue decrypts every record this identity can open. Records sealed to
// a different key are skipped silently: they are simply not ours.
func (s *store) catalogue() (map[string][]*record, error) {
shards, err := os.ReadDir(filepath.Join(s.root, recordsDir))
if err != nil {
return nil, err
}
byName := map[string][]*record{}
for _, shard := range shards {
if !shard.IsDir() {
continue
}
entries, err := os.ReadDir(filepath.Join(s.root, recordsDir, shard.Name()))
if err != nil {
return nil, err
}
for _, e := range entries {
if !validID(e.Name()) || e.Name()[:2] != shard.Name() {
continue
}
r, err := s.readRecord(e.Name())
var noMatch *pqcrypt.NoIdentityMatchError
if errors.As(err, &noMatch) {
continue
}
if err != nil {
fmt.Fprintf(os.Stderr, "vault: skipping unreadable record %s: %v\n", e.Name(), err)
continue
}
byName[r.Name] = append(byName[r.Name], r)
}
}
// Newest first, so [0] is current. Duplicates only exist after an
// interrupted replace; the next write of that name cleans them up.
for _, rs := range byName {
sort.Slice(rs, func(i, j int) bool { return rs[i].Added.After(rs[j].Added) })
}
return byName, nil
}
func (s *store) readRecord(id string) (*record, error) {
f, err := os.Open(s.path(recordsDir, id))
if err != nil {
return nil, err
}
defer f.Close()
pr, err := pqcrypt.Decrypt(f, s.id)
if err != nil {
return nil, err
}
plain, err := io.ReadAll(io.LimitReader(pr, recordSize+1))
if err != nil {
return nil, err
}
if len(plain) != recordSize {
return nil, errors.New("wrong record size")
}
n := binary.BigEndian.Uint32(plain)
if n > recordSize-4 {
return nil, errors.New("malformed record")
}
r := &record{id: id}
dec := json.NewDecoder(bytes.NewReader(plain[4 : 4+n]))
dec.DisallowUnknownFields()
if err := dec.Decode(r); err != nil {
return nil, err
}
if !validName(r.Name) || !validHex(r.Seed, objectSeedSize) || r.Size < 0 || r.Size > fileLimit {
return nil, errors.New("malformed record")
}
return r, nil
}
// add stores the file at src as name, replacing any existing file of that
// name, and updates cat to match. It reports whether something was replaced.
func (s *store) add(cat map[string][]*record, src, name string) (bool, error) {
if !validName(name) {
return false, fmt.Errorf("invalid vault path %q", name)
}
f, err := os.Open(src)
if err != nil {
return false, err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return false, err
}
if !info.Mode().IsRegular() || info.Size() > fileLimit {
return false, fmt.Errorf("%s: not a regular file of at most 1 TiB", src)
}
replaced, err := s.addReader(cat, f, info.Size(), info.Mode().Perm(), info.ModTime(), name)
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
err = fmt.Errorf("%s changed while being added", src)
}
return replaced, err
}
// addReader stores size bytes from r as name, like add. Reading fewer than
// size bytes stores nothing and fails with io.ErrUnexpectedEOF or io.EOF.
func (s *store) addReader(cat map[string][]*record, r io.Reader, size int64, mode fs.FileMode, mtime time.Time, name string) (bool, error) {
if !validName(name) {
return false, fmt.Errorf("invalid vault path %q", name)
}
if size < 0 || size > fileLimit {
return false, fmt.Errorf("%s: larger than 1 TiB", name)
}
seed, err := randomHex(objectSeedSize)
if err != nil {
return false, err
}
if err := s.writeObjects(r, seed, size); err != nil {
s.dropObjects(seed, size)
return false, err
}
rec := &record{
Name: name, Seed: seed, Size: size, Mode: uint32(mode.Perm()),
Modified: mtime.UTC(), Added: time.Now().UTC(),
}
if err := s.writeRecord(rec); err != nil {
s.dropObjects(seed, size)
return false, err
}
replaced := len(cat[name]) > 0
if err := s.remove(cat, name, true); err != nil {
return replaced, err
}
cat[name] = []*record{rec}
return replaced, nil
}
// writeObjects reads size bytes from f into the objects derived from seed.
// Chunks are read in order but encrypted, written and synced by a few
// workers at once, so their fsyncs overlap instead of queueing.
func (s *store) writeObjects(f io.Reader, seed string, size int64) error {
var (
wg sync.WaitGroup
mu sync.Mutex
firstErr error
)
fail := func(err error) {
mu.Lock()
defer mu.Unlock()
if firstErr == nil {
firstErr = err
}
}
failed := func() bool {
mu.Lock()
defer mu.Unlock()
return firstErr != nil
}
slots := make(chan struct{}, 8) // at most 8 chunks, 32 MiB, in memory
remaining := size
for i := range objectCount(size) {
if failed() {
break
}
// The object ID leads the plaintext so a record can only ever
// open the objects it derives, each in its place, not one
// swapped in under its ID.
id := objectID(seed, i)
part := min(remaining, chunkData)
remaining -= part
slots <- struct{}{}
buf := make([]byte, idSize+part)
hex.Decode(buf, []byte(id))
if _, err := io.ReadFull(f, buf[idSize:]); err != nil {
<-slots
fail(err)
break
}
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-slots }()
err := s.writeCiphertext(objectsDir, id, func(w io.Writer) error {
if _, err := w.Write(buf); err != nil {
return err
}
_, err := io.CopyN(w, rand.Reader, paddedSize(int64(len(buf)))-int64(len(buf)))
return err
})
if err != nil {
fail(err)
}
}()
}
wg.Wait()
return firstErr
}
// remove deletes every record for name and, when dropObjects is set, any of
// their objects no other record still uses. Records go first, so an
// interruption only ever leaves an orphaned object.
func (s *store) remove(cat map[string][]*record, name string, dropObjects bool) error {
inUse := map[string]bool{}
for n, rs := range cat {
for _, r := range rs {
if n != name {
inUse[r.Seed] = true
}
}
}
for _, r := range cat[name] {
if err := s.removeFile(recordsDir, r.id); err != nil {
return err
}
if dropObjects && !inUse[r.Seed] {
if err := s.dropObjects(r.Seed, r.Size); err != nil {
return err
}
}
}
return nil
}
// dropObjects deletes the objects holding a file of size bytes, skipping any
// already gone.
func (s *store) dropObjects(seed string, size int64) error {
for i := range objectCount(size) {
if err := s.removeFile(objectsDir, objectID(seed, i)); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
func (s *store) lookup(name string) (map[string][]*record, *record, error) {
cat, err := s.catalogue()
if err != nil {
return nil, nil, err
}
rs := cat[name]
if len(rs) == 0 {
return nil, nil, fmt.Errorf("%s: not in the vault", name)
}
return cat, rs[0], nil
}
func (s *store) delete(name string) error {
cat, _, err := s.lookup(name)
if err != nil {
return err
}
return s.removeAll(cat, []string{name}, func(string) {})
}
// under returns the names in cat at or beneath the folder prefix, sorted.
func under(cat map[string][]*record, prefix string) []string {
var names []string
for name := range cat {
if name == prefix || strings.HasPrefix(name, prefix+"/") {
names = append(names, name)
}
}
sort.Strings(names)
return names
}
// removeAll deletes every record for names and the objects they hold, then
// drops them from cat, calling done after each name. Like remove, but it
// works out once which objects other records still use, so deleting a large
// folder stays linear.
func (s *store) removeAll(cat map[string][]*record, names []string, done func(name string)) error {
gone := map[string]bool{}
for _, name := range names {
gone[name] = true
}
inUse := map[string]bool{}
for name, rs := range cat {
if !gone[name] {
for _, r := range rs {
inUse[r.Seed] = true
}
}
}
for _, name := range names {
for _, r := range cat[name] {
if err := s.removeFile(recordsDir, r.id); err != nil {
return err
}
if !inUse[r.Seed] {
if err := s.dropObjects(r.Seed, r.Size); err != nil {
return err
}
}
}
delete(cat, name)
done(name)
}
return nil
}
// copyAll stores each pair's first name again under its second, sharing its
// objects, then with move deletes the first names. New records go first, so
// an interruption leaves a file under both names, never neither.
func (s *store) copyAll(cat map[string][]*record, pairs [][2]string, move bool, done func(from, to string)) error {
var old []string
for _, p := range pairs {
r := *cat[p[0]][0]
r.Name = p[1]
if !move {
r.Added = time.Now().UTC()
}
if err := s.writeRecord(&r); err != nil {
return err
}
cat[p[1]] = []*record{&r}
old = append(old, p[0])
done(p[0], p[1])
}
if !move {
return nil
}
return s.removeAll(cat, old, func(string) {})
}
// get decrypts name to out. Nothing appears at out until every object,
// padding included, has authenticated.
func (s *store) get(name, out string) error {
_, r, err := s.lookup(name)
if err != nil {
return err
}
return s.getRecord(r, out)
}
func (s *store) getRecord(r *record, out string) error {
if _, err := os.Lstat(out); err == nil {
return fmt.Errorf("%s already exists", out)
}
tmp, err := os.CreateTemp(filepath.Dir(out), ".vault-get-")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
remaining := r.Size
for i := range objectCount(r.Size) {
part := min(remaining, chunkData)
if err := s.readObject(tmp, objectID(r.Seed, i), part); err != nil {
return err
}
remaining -= part
}
if err := tmp.Chmod(os.FileMode(r.Mode) & 0o777); err != nil {
return err
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Chtimes(tmp.Name(), r.Modified, r.Modified); err != nil {
return err
}
if _, err := os.Lstat(out); err == nil {
return fmt.Errorf("%s already exists", out)
}
return os.Rename(tmp.Name(), out)
}
// readObject decrypts object id to w, checking it leads with its own ID and
// holds part bytes padded to their bucket.
func (s *store) readObject(w io.Writer, id string, part int64) error {
f, err := os.Open(s.path(objectsDir, id))
if err != nil {
return err
}
defer f.Close()
pr, err := pqcrypt.Decrypt(f, s.id)
if err != nil {
return err
}
head := make([]byte, idSize)
if _, err := io.ReadFull(pr, head); err != nil || hex.EncodeToString(head) != id {
return errors.New("object does not match its catalogue record")
}
if _, err := io.CopyN(w, pr, part); err != nil {
return err
}
pad, err := io.Copy(io.Discard, pr)
if err != nil {
return err
}
if idSize+part+pad != paddedSize(idSize+part) {
return errors.New("object size does not match its catalogue record")
}
return nil
}
+728
View File
@@ -0,0 +1,728 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"crypto/rand"
"encoding/binary"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"github.com/andydixon/vault/internal/pqcrypt"
)
func TestKeyFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "key")
if err := writeKeyFile(path, "pass", ""); err != nil {
t.Fatal(err)
}
if _, err := unlockKeyFile(path, "wrong"); err == nil {
t.Fatal("wrong passphrase unlocked the key")
}
if _, err := unlockKeyFile(path, "pass"); err != nil {
t.Fatal(err)
}
}
func newTestStore(t *testing.T, keyPath, pass string) *store {
t.Helper()
id, err := unlockKeyFile(keyPath, pass)
if err != nil {
t.Fatal(err)
}
s, err := openStore(filepath.Join(filepath.Dir(keyPath), "store"), id)
if err != nil {
t.Fatal(err)
}
return s
}
func TestStoreLifecycle(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
src := filepath.Join(dir, "in")
want := bytes.Repeat([]byte("x"), 70000)
os.WriteFile(src, want, 0o640)
cat, _ := s.catalogue()
if _, err := s.add(cat, src, "a/b"); err != nil {
t.Fatal(err)
}
if replaced, err := s.add(cat, src, "a/b"); err != nil || !replaced {
t.Fatalf("replace: %v %v", replaced, err)
}
if _, err := s.add(cat, src, "c"); err != nil {
t.Fatal(err)
}
if cat, _ = s.catalogue(); len(cat) != 2 || len(cat["a/b"]) != 1 {
t.Fatalf("catalogue = %v", cat)
}
pairs, err := transfers(cat, "a/b", "d", false)
if err != nil {
t.Fatal(err)
}
if err := s.copyAll(cat, pairs, true, func(string, string) {}); err != nil {
t.Fatal(err)
}
out := filepath.Join(dir, "out")
if err := s.get("d", out); err != nil {
t.Fatal(err)
}
if got, _ := os.ReadFile(out); !bytes.Equal(got, want) {
t.Fatal("round trip mismatch")
}
if err := s.get("d", out); err == nil {
t.Fatal("get overwrote an existing file")
}
// Swapping one object for another must be detected.
cat, _ = s.catalogue()
d, c := cat["d"][0], cat["c"][0]
data, _ := os.ReadFile(s.path(objectsDir, objectID(c.Seed, 0)))
os.WriteFile(s.path(objectsDir, objectID(d.Seed, 0)), data, 0o600)
if err := s.get("d", filepath.Join(dir, "swapped")); err == nil {
t.Fatal("swapped object was accepted")
}
if err := s.delete("c"); err != nil {
t.Fatal(err)
}
if cat, _ = s.catalogue(); len(cat) != 1 {
t.Fatalf("after delete: %v", cat)
}
}
func TestDecoy(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "real", "decoy"); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
src := filepath.Join(dir, "in")
os.WriteFile(src, []byte("data"), 0o600)
real, decoy := newTestStore(t, keyPath, "real"), newTestStore(t, keyPath, "decoy")
for s, name := range map[*store]string{real: "secret", decoy: "shopping"} {
cat, err := s.catalogue()
if err != nil || len(cat) != 0 {
t.Fatalf("fresh vault catalogue = %v, %v", cat, err)
}
if _, err := s.add(cat, src, name); err != nil {
t.Fatal(err)
}
}
for s, name := range map[*store]string{real: "secret", decoy: "shopping"} {
if cat, _ := s.catalogue(); len(cat) != 1 || cat[name] == nil {
t.Fatalf("catalogue = %v, want only %s", cat, name)
}
}
if _, err := unlockKeyFile(keyPath, "neither"); err == nil {
t.Fatal("wrong passphrase unlocked the key")
}
if n := len(ciphertexts(t, filepath.Join(dir, "store", recordsDir))); n < fillerMin+2 {
t.Fatalf("only %d records; filler missing", n)
}
}
// ciphertexts maps every file under dir to its size.
func ciphertexts(t *testing.T, dir string) map[string]int64 {
t.Helper()
sizes := map[string]int64{}
err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
info, err := d.Info()
sizes[p] = info.Size()
return err
})
if err != nil {
t.Fatal(err)
}
return sizes
}
// TestChunks checks a file bigger than one chunk becomes unlinkable 4 MiB
// objects plus a bucket-sized tail, all in their shards, and that reordering
// the chunks is caught.
func TestChunks(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
objects := filepath.Join(s.root, objectsDir)
before := ciphertexts(t, objects)
src := filepath.Join(dir, "in")
want := make([]byte, 2*chunkData+100)
rand.Read(want)
os.WriteFile(src, want, 0o600)
cat, _ := s.catalogue()
if _, err := s.add(cat, src, "big"); err != nil {
t.Fatal(err)
}
cat, _ = s.catalogue()
r := cat["big"][0]
var sizes []int64
for i := range objectCount(r.Size) {
p := s.path(objectsDir, objectID(r.Seed, i))
if filepath.Base(filepath.Dir(p)) != filepath.Base(p)[:2] {
t.Fatalf("%s is in the wrong shard", p)
}
sizes = append(sizes, ciphertexts(t, p)[p])
}
if len(sizes) != 3 || sizes[0] != sizes[1] || sizes[2] >= sizes[0] {
t.Fatalf("object sizes = %v, want two full chunks and a smaller tail", sizes)
}
if n := len(ciphertexts(t, objects)) - len(before); n != 3 {
t.Fatalf("add wrote %d objects, want 3", n)
}
for p, size := range ciphertexts(t, objects) {
if size > sizes[0] {
t.Fatalf("%s is %d bytes, larger than a chunk", p, size)
}
}
out := filepath.Join(dir, "out")
if err := s.get("big", out); err != nil {
t.Fatal(err)
}
if got, _ := os.ReadFile(out); !bytes.Equal(got, want) {
t.Fatal("round trip mismatch")
}
a, b := s.path(objectsDir, objectID(r.Seed, 0)), s.path(objectsDir, objectID(r.Seed, 1))
os.Rename(a, a+".x")
os.Rename(b, a)
os.Rename(a+".x", b)
if err := s.get("big", filepath.Join(dir, "swapped")); err == nil {
t.Fatal("reordered chunks were accepted")
}
if err := s.delete("big"); err != nil {
t.Fatal(err)
}
if n := len(ciphertexts(t, objects)); n != len(before) {
t.Fatalf("%d objects after rm, want %d", n, len(before))
}
}
// TestLockExcludesOtherProcess runs a copy of the test binary that holds the
// exclusive lock, then checks this process has to wait for it.
func TestLockExcludesOtherProcess(t *testing.T) {
if dir := os.Getenv("VAULT_TEST_LOCK_HOLDER"); dir != "" {
release, err := lockVault(dir, true)
if err != nil {
os.Exit(1)
}
os.Stdout.WriteString("locked\n")
io.Copy(io.Discard, os.Stdin) // hold until the parent closes stdin
release()
os.Exit(0)
}
dir := t.TempDir()
holder := exec.Command(os.Args[0], "-test.run=^TestLockExcludesOtherProcess$")
holder.Env = append(os.Environ(), "VAULT_TEST_LOCK_HOLDER="+dir)
stdin, _ := holder.StdinPipe()
stdout, _ := holder.StdoutPipe()
if err := holder.Start(); err != nil {
t.Fatal(err)
}
defer holder.Wait()
if line, _ := bufio.NewReader(stdout).ReadString('\n'); line != "locked\n" {
t.Fatalf("holder said %q", line)
}
waited := make(chan bool, 1)
go func() {
time.Sleep(200 * time.Millisecond)
stdin.Close() // let the holder go only after we have started waiting
}()
for _, exclusive := range []bool{false, true} {
release, err := acquireLock(filepath.Join(dir, lockName), exclusive, func() { waited <- true })
if err != nil {
t.Fatal(err)
}
release()
if !exclusive && len(waited) == 0 {
t.Fatal("shared lock did not wait for the exclusive holder")
}
}
}
func TestParseSize(t *testing.T) {
for in, want := range map[string]int64{"0": 0, "512": 512, "2k": 2 << 10, "500M": 500 << 20, "2G": 2 << 30, "1TB": 1 << 40} {
if got, err := parseSize(in); err != nil || got != want {
t.Errorf("parseSize(%q) = %d, %v; want %d", in, got, err, want)
}
}
for _, in := range []string{"", "G", "-1", "2X", "1.5G", "2T"} {
if _, err := parseSize(in); err == nil {
t.Errorf("parseSize(%q) accepted", in)
}
}
}
// TestFiller checks a filler target is met with entries shaped like real
// files: fixed-size records and objects no bigger than a chunk.
func TestFiller(t *testing.T) {
root := filepath.Join(t.TempDir(), "store")
if err := initStore(root, 24<<20); err != nil {
t.Fatal(err)
}
if n := len(ciphertexts(t, filepath.Join(root, recordsDir))); n < fillerMin {
t.Fatalf("%d filler records, want at least %d", n, fillerMin)
}
var total, largest int64
for _, size := range ciphertexts(t, filepath.Join(root, objectsDir)) {
total, largest = total+size, max(largest, size)
}
if total < 24<<20 || largest > chunkSize+chunkSize/64 {
t.Fatalf("filler objects total %d, largest %d", total, largest)
}
}
// TestAddTree migrates "." holding the vault and key: structure is kept,
// originals and emptied directories go, the vault and key are left alone.
// A URL download is then added under a folder.
func TestAddTree(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
os.MkdirAll(filepath.Join(dir, "a", "b"), 0o755)
os.WriteFile(filepath.Join(dir, "x"), []byte("x"), 0o600)
os.WriteFile(filepath.Join(dir, "a", "b", "y"), []byte("y"), 0o600)
t.Chdir(dir)
var skip []os.FileInfo
for _, p := range []string{"store", "key"} {
fi, _ := os.Stat(p)
skip = append(skip, fi)
}
cat, _ := s.catalogue()
if err := addPath(s, cat, "x", "d", false, skip, nil); err != nil {
t.Fatal(err)
}
// Migrating over d/x must ask first, and a no must leave everything.
var asked []string
no := func(names []string) bool { asked = names; return false }
if err := addPath(s, cat, ".", "d", true, skip, no); err == nil || !slices.Equal(asked, []string{"d/x"}) {
t.Fatalf("migrate over d/x: err %v, asked %v", err, asked)
}
if _, err := os.Stat("x"); err != nil || len(cat) != 1 {
t.Fatalf("declined migrate changed things: %v, %d stored", err, len(cat))
}
if err := addPath(s, cat, ".", "d", true, skip, func([]string) bool { return true }); err != nil {
t.Fatal(err)
}
var got []string
for name := range cat {
got = append(got, name)
}
slices.Sort(got)
if !slices.Equal(got, []string{"d/a/b/y", "d/x"}) {
t.Fatalf("stored %v", got)
}
var left []string
entries, _ := os.ReadDir(".")
for _, e := range entries {
left = append(left, e.Name())
}
if !slices.Equal(left, []string{"key", "store"}) {
t.Fatalf("left behind %v", left)
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.UserAgent() != chromeUA {
http.Error(w, "bad agent", http.StatusForbidden)
return
}
io.WriteString(w, "body")
}))
defer srv.Close()
if err := addURL(s, cat, srv.URL+"/files/page.html", "web"); err != nil {
t.Fatal(err)
}
if err := s.get("web/page.html", "out"); err != nil {
t.Fatal(err)
}
if b, _ := os.ReadFile("out"); string(b) != "body" {
t.Fatalf("downloaded %q", b)
}
}
// TestMirror crawls a two-page site: links in HTML and CSS become relative,
// off-host links stay, and everything is stored under host/.
func TestMirror(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.UserAgent() != chromeUA {
http.Error(w, "bad agent", http.StatusForbidden)
return
}
switch r.URL.Path {
case "/":
w.Header().Set("Content-Type", "text/html")
io.WriteString(w, `<a href="`+srv.URL+`/sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='/s.css'>`)
case "/sub/p.html":
w.Header().Set("Content-Type", "text/html")
io.WriteString(w, `<a href="/">home</a> <img src="../img/a.png">`)
case "/s.css":
w.Header().Set("Content-Type", "text/css")
io.WriteString(w, `body{background:url(/img/a.png)}`)
case "/img/a.png":
io.WriteString(w, "png")
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
cat, _ := s.catalogue()
if err := mirror(s, cat, srv.URL, "m"); err != nil {
t.Fatal(err)
}
host := srv.Listener.Addr().String()
for name, want := range map[string]string{
"m/" + host + "/index.html": `<a href="sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='s.css'>`,
"m/" + host + "/sub/p.html": `<a href="../index.html">home</a> <img src="../img/a.png">`,
"m/" + host + "/s.css": `body{background:url(img/a.png)}`,
"m/" + host + "/img/a.png": "png",
} {
out := filepath.Join(dir, "out")
if err := s.get(name, out); err != nil {
t.Fatal(name, err)
}
if b, _ := os.ReadFile(out); string(b) != want {
t.Errorf("%s = %q, want %q", name, b, want)
}
os.Remove(out)
}
}
func TestAbsorb(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
mtime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
for _, h := range []*tar.Header{
{Name: "./d/", Typeflag: tar.TypeDir, Mode: 0o755},
{Name: "./d/a.txt", Size: 5, Mode: 0o600, ModTime: mtime},
{Name: "../escape", Size: 5, Mode: 0o600},
{Name: "./d/link", Typeflag: tar.TypeSymlink, Linkname: "a.txt"},
} {
tw.WriteHeader(h)
if h.Size > 0 {
io.WriteString(tw, "hello")
}
}
tw.Close()
zw.Close()
cat, _ := s.catalogue()
if err := absorb(s, cat, &buf, "in"); err != nil {
t.Fatal(err)
}
if len(cat) != 1 || len(cat["in/d/a.txt"]) != 1 {
t.Fatalf("catalogue = %v, want only in/d/a.txt", cat)
}
if r := cat["in/d/a.txt"][0]; !r.Modified.Equal(mtime) || r.Mode != 0o600 {
t.Errorf("record = %v, %o", r.Modified, r.Mode)
}
out := filepath.Join(dir, "out")
if err := s.get("in/d/a.txt", out); err != nil {
t.Fatal(err)
}
if b, _ := os.ReadFile(out); string(b) != "hello" {
t.Errorf("got %q", b)
}
}
func TestRemoveTree(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
objects := filepath.Join(dir, "store", "objects")
before := len(ciphertexts(t, objects))
cat, _ := s.catalogue()
src := filepath.Join(dir, "in")
big := make([]byte, chunkData+1) // two objects
rand.Read(big)
for name, body := range map[string][]byte{"a/x": big, "a/b/y": []byte("y"), "ab/z": []byte("z"), "b/w": []byte("w")} {
os.WriteFile(src, body, 0o600)
if _, err := s.add(cat, src, name); err != nil {
t.Fatal(err)
}
}
if n := len(ciphertexts(t, objects)) - before; n != 5 {
t.Fatalf("%d objects added, want 5", n)
}
names := under(cat, "a")
if !slices.Equal(names, []string{"a/b/y", "a/x"}) {
t.Fatalf("under a: %v", names)
}
var done []string
if err := s.removeAll(cat, names, func(n string) { done = append(done, n) }); err != nil {
t.Fatal(err)
}
if !slices.Equal(done, names) || len(cat) != 2 || len(cat["ab/z"]) != 1 || len(cat["b/w"]) != 1 {
t.Fatalf("done %v, left %v", done, cat)
}
if n := len(ciphertexts(t, objects)) - before; n != 2 {
t.Fatalf("%d objects left, want 2: the space was not freed", n)
}
fresh, _ := s.catalogue()
if len(fresh) != 2 {
t.Fatalf("on disk: %v", fresh)
}
out := filepath.Join(dir, "out")
if err := s.get("ab/z", out); err != nil {
t.Fatal(err)
}
}
func TestLs(t *testing.T) {
at := func(m int) time.Time { return time.Date(2026, 1, 1, 12, m, 0, 0, time.UTC) }
cat := map[string][]*record{
"notes.md": {{Size: 10, Modified: at(1)}},
"a/x": {{Size: 100, Modified: at(2)}},
"a/b/y": {{Size: 1000, Modified: at(5)}},
"a/b/c/z": {{Size: 1, Modified: at(3)}},
"ab/z": {{Size: 7, Modified: at(4)}},
"photos/2025/img1": {{Size: 2048, Modified: at(6)}},
}
names := func(folder string) []string {
t.Helper()
var b bytes.Buffer
if err := ls(&b, cat, folder); err != nil {
t.Fatal(err)
}
var got []string
for _, line := range strings.Split(strings.TrimRight(b.String(), "\n"), "\n") {
got = append(got, strings.TrimSpace(line[10:]))
}
return got
}
day := func(m int) string { return at(m).Local().Format("2006-01-02 15:04") + " " }
if got, want := names(""), []string{day(5) + "a/", day(4) + "ab/", day(6) + "photos/", day(1) + "notes.md"}; !slices.Equal(got, want) {
t.Fatalf("ls: %q, want %q", got, want)
}
if got, want := names("a"), []string{day(5) + "b/", day(2) + "x"}; !slices.Equal(got, want) {
t.Fatalf("ls a: %q, want %q", got, want)
}
if got, want := names("a/b"), []string{day(3) + "c/", day(5) + "y"}; !slices.Equal(got, want) {
t.Fatalf("ls a/b: %q, want %q", got, want)
}
if got, want := names("a/x"), []string{day(2) + "x"}; !slices.Equal(got, want) {
t.Fatalf("ls a/x: %q, want %q", got, want)
}
var b bytes.Buffer
ls(&b, cat, "")
if !strings.HasPrefix(b.String(), " 1.1 KiB ") {
t.Fatalf("folder a/ should total 1101 bytes: %q", b.String())
}
if err := ls(&b, cat, "nope"); err == nil {
t.Fatal("ls of a missing folder succeeded")
}
}
func TestCopyMove(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
cat, _ := s.catalogue()
src := filepath.Join(dir, "in")
for _, name := range []string{"a/x", "a/b/y", "ab/z", "c/w"} {
os.WriteFile(src, []byte(name), 0o600)
if _, err := s.add(cat, src, name); err != nil {
t.Fatal(err)
}
}
run := func(src, dst string, recursive, move bool) error {
pairs, err := transfers(cat, src, dst, recursive)
if err != nil {
return err
}
return s.copyAll(cat, pairs, move, func(string, string) {})
}
for _, bad := range [][2]string{{"a", "d"}, {"a/x", "ab/z"}, {"nope", "d"}, {"", "d"}} {
if run(bad[0], bad[1], false, false) == nil {
t.Errorf("cp %s %s succeeded", bad[0], bad[1])
}
}
if run("a", "a/b", true, true) == nil {
t.Error("moved a folder into itself")
}
if err := run("a", "d", true, false); err != nil { // cp -r a d
t.Fatal(err)
}
if err := run("d", "c/", true, true); err != nil { // mv d c/
t.Fatal(err)
}
if err := run("ab/z", "c", false, true); err != nil { // mv ab/z into existing folder c
t.Fatal(err)
}
if err := s.removeAll(cat, under(cat, "a"), func(string) {}); err != nil {
t.Fatal(err)
}
fresh, _ := s.catalogue()
var names []string
for n := range fresh {
names = append(names, n)
}
slices.Sort(names)
if want := []string{"c/d/b/y", "c/d/x", "c/w", "c/z"}; !slices.Equal(names, want) {
t.Fatalf("names %v, want %v", names, want)
}
out := filepath.Join(dir, "out")
if err := s.get("c/d/x", out); err != nil {
t.Fatal("copy lost its data when the original went: ", err)
}
if b, _ := os.ReadFile(out); string(b) != "a/x" {
t.Errorf("got %q", b)
}
}
func TestAgent(t *testing.T) {
dir := t.TempDir()
t.Setenv("XDG_RUNTIME_DIR", dir)
t.Setenv(passphraseEnv, "") // restored after the test
os.Unsetenv(passphraseEnv) // so keyIdentity asks the agent
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
id, err := unlockKeyFile(keyPath, "pass")
if err != nil {
t.Fatal(err)
}
if _, err := agentIdentity(keyPath); err == nil {
t.Fatal("identity from an agent that is not running")
}
sock, err := agentSocket(keyPath)
if err != nil {
t.Fatal(err)
}
secret := binary.BigEndian.AppendUint16(nil, id.SuiteID())
secret = append(secret, id.Seed()...)
pr, pw := io.Pipe()
stopped := make(chan struct{})
go func() { runAgent(sock, "1m", "fixed", bytes.NewReader(secret), pw); close(stopped) }()
if msg, _ := io.ReadAll(pr); string(msg) != "ok" {
t.Fatalf("agent said %q", msg)
}
got, err := keyIdentity(keyPath)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got.Seed(), id.Seed()) || got.SuiteID() != id.SuiteID() {
t.Fatal("agent returned a different key")
}
if !agentStop(keyPath) {
t.Fatal("agent was not running")
}
<-stopped
if _, err := os.Stat(sock); !os.IsNotExist(err) {
t.Fatal("socket left behind")
}
if agentStop(keyPath) {
t.Fatal("agent still running")
}
}
func TestAgentIdle(t *testing.T) {
dir := t.TempDir()
t.Setenv("XDG_RUNTIME_DIR", dir)
keyPath := filepath.Join(dir, "key")
sock, err := agentSocket(keyPath)
if err != nil {
t.Fatal(err)
}
id, err := pqcrypt.GeneratePQIdentity()
if err != nil {
t.Fatal(err)
}
secret := binary.BigEndian.AppendUint16(nil, id.SuiteID())
secret = append(secret, id.Seed()...)
pr, pw := io.Pipe()
stopped := make(chan struct{})
go func() { runAgent(sock, "400ms", "idle", bytes.NewReader(secret), pw); close(stopped) }()
if msg, _ := io.ReadAll(pr); string(msg) != "ok" {
t.Fatalf("agent said %q", msg)
}
// Used every 150ms, it outlives its 400ms by well over twice.
for range 7 {
time.Sleep(150 * time.Millisecond)
if _, err := agentIdentity(keyPath); err != nil {
t.Fatal("idle agent expired while in use: ", err)
}
}
select {
case <-stopped:
case <-time.After(2 * time.Second):
t.Fatal("idle agent outlived its idle time")
}
}