fix(planning): a stale step copy could reset a claimed step and run it twice

The pending → ready and pending → cancelled transitions, and cancellation
of open steps, saved a step unconditionally. Under load an executor
holding a stale "pending" copy could write "ready" over a step another
executor had already claimed and was running, after which the step could
be claimed again: the full race suite caught two restarts from one plan.
These transitions now use SaveIf, which writes only when the stored status
is still the expected one. A deterministic regression test hands the
executor a stale copy and fails without the fix; the concurrency test now
races eight executors ticking repeatedly.
This commit is contained in:
Andy Dixon
2026-09-20 22:54:24 +01:00
parent e7dc6dfe65
commit 672884cd22
3 changed files with 58 additions and 6 deletions
+4 -3
View File
@@ -247,14 +247,14 @@ func (e *Executor) advanceStep(ctx context.Context, p *Plan, it volition.Intenti
}
if dep.Status == StepFailed || dep.Status == StepCancelled {
st.Status, st.StatusReason = StepCancelled, "dependency "+d+" did not complete"
return true, e.Plans.Save(ctx, nil, *st)
return e.Plans.SaveIf(ctx, *st, StepPending)
}
if !StepDone(dep.Status) {
return false, nil
}
}
st.Status = StepReady
return true, e.Plans.Save(ctx, nil, *st)
return e.Plans.SaveIf(ctx, *st, StepPending)
case StepReady:
if !due || it.Status != volition.IntentionActive {
return false, nil
@@ -857,8 +857,9 @@ func (e *Executor) cancelOpenSteps(ctx context.Context, p Plan, reason string) {
e.Caps.WithdrawCapabilityRequest(ctx, p.MindID, s.RequestID, reason) //nolint:errcheck
}
e.dropSubordinate(ctx, &p, &s, reason)
from := s.Status
s.Status, s.StatusReason = StepCancelled, reason
e.Plans.Save(ctx, nil, s) //nolint:errcheck
e.Plans.SaveIf(ctx, s, from) //nolint:errcheck // a step that moved on meanwhile keeps its newer state
}
}