fix(planning): a stale step copy could reset a claimed step and run it twice

The pending → ready and pending → cancelled transitions, and cancellation
of open steps, saved a step unconditionally. Under load an executor
holding a stale "pending" copy could write "ready" over a step another
executor had already claimed and was running, after which the step could
be claimed again: the full race suite caught two restarts from one plan.
These transitions now use SaveIf, which writes only when the stored status
is still the expected one. A deterministic regression test hands the
executor a stale copy and fails without the fix; the concurrency test now
races eight executors ticking repeatedly.
This commit is contained in:
Andy Dixon
2026-09-20 22:54:24 +01:00
parent e7dc6dfe65
commit 672884cd22
3 changed files with 58 additions and 6 deletions
+18
View File
@@ -367,6 +367,24 @@ func (s *Store) Save(ctx context.Context, q storage.Querier, st Step) error {
return err
}
// SaveIf writes a step only if its stored status is still expected, so a
// caller holding a stale copy can never move a step backwards (for example
// resetting a step another executor is running to "ready"). It reports
// whether the write happened.
func (s *Store) SaveIf(ctx context.Context, st Step, expected string) (bool, error) {
now := time.Now()
var finished *time.Time
if StepTerminal(st.Status) {
finished = &now
}
tag, err := s.db.Exec(ctx, `UPDATE plan_steps SET status=$2, status_reason=$3, finished_at=coalesce($4, finished_at), updated_at=$5 WHERE id=$1 AND status=$6`,
st.ID, st.Status, trunc(st.StatusReason, 500), finished, now, expected)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
}
// SetStatus writes a plan's status and reason.
func (s *Store) SetStatus(ctx context.Context, q storage.Querier, id ids.ID, status, reason string) error {
if q == nil {