fix(planning): a stale step copy could reset a claimed step and run it twice
The pending → ready and pending → cancelled transitions, and cancellation of open steps, saved a step unconditionally. Under load an executor holding a stale "pending" copy could write "ready" over a step another executor had already claimed and was running, after which the step could be claimed again: the full race suite caught two restarts from one plan. These transitions now use SaveIf, which writes only when the stored status is still the expected one. A deterministic regression test hands the executor a stale copy and fails without the fix; the concurrency test now races eight executors ticking repeatedly.
This commit is contained in:
@@ -367,6 +367,24 @@ func (s *Store) Save(ctx context.Context, q storage.Querier, st Step) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// SaveIf writes a step only if its stored status is still expected, so a
|
||||
// caller holding a stale copy can never move a step backwards (for example
|
||||
// resetting a step another executor is running to "ready"). It reports
|
||||
// whether the write happened.
|
||||
func (s *Store) SaveIf(ctx context.Context, st Step, expected string) (bool, error) {
|
||||
now := time.Now()
|
||||
var finished *time.Time
|
||||
if StepTerminal(st.Status) {
|
||||
finished = &now
|
||||
}
|
||||
tag, err := s.db.Exec(ctx, `UPDATE plan_steps SET status=$2, status_reason=$3, finished_at=coalesce($4, finished_at), updated_at=$5 WHERE id=$1 AND status=$6`,
|
||||
st.ID, st.Status, trunc(st.StatusReason, 500), finished, now, expected)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
// SetStatus writes a plan's status and reason.
|
||||
func (s *Store) SetStatus(ctx context.Context, q storage.Querier, id ids.ID, status, reason string) error {
|
||||
if q == nil {
|
||||
|
||||
Reference in New Issue
Block a user