fix(planning): a stale step copy could reset a claimed step and run it twice
The pending → ready and pending → cancelled transitions, and cancellation of open steps, saved a step unconditionally. Under load an executor holding a stale "pending" copy could write "ready" over a step another executor had already claimed and was running, after which the step could be claimed again: the full race suite caught two restarts from one plan. These transitions now use SaveIf, which writes only when the stored status is still the expected one. A deterministic regression test hands the executor a stale copy and fails without the fix; the concurrency test now races eight executors ticking repeatedly.
This commit is contained in:
@@ -415,13 +415,17 @@ func TestConcurrentExecutorsRunAStepOnce(t *testing.T) {
|
||||
f.host.fixOnRestart = true
|
||||
it := f.intention(false)
|
||||
p := f.start(it)
|
||||
other := f.newExecutor(f.exec.Predictions, f.exec.Planner, f.exec.Log)
|
||||
// Many executors race over the same plan from its first transition, each
|
||||
// ticking repeatedly, as duplicate schedulers or several daemons would.
|
||||
var wg sync.WaitGroup
|
||||
for _, e := range []*Executor{f.exec, other, f.newExecutor(f.exec.Predictions, f.exec.Planner, f.exec.Log)} {
|
||||
for i := 0; i < 8; i++ {
|
||||
e := f.newExecutor(f.exec.Predictions, f.exec.Planner, f.exec.Log)
|
||||
wg.Add(1)
|
||||
go func(e *Executor) {
|
||||
defer wg.Done()
|
||||
e.TickMind(f.ctx, f.mind.ID) //nolint:errcheck
|
||||
for j := 0; j < 3; j++ {
|
||||
e.TickMind(f.ctx, f.mind.ID) //nolint:errcheck
|
||||
}
|
||||
}(e)
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -505,3 +509,32 @@ func TestInvalidModelPlanNeverRuns(t *testing.T) {
|
||||
t.Fatal("invalid plan executed")
|
||||
}
|
||||
}
|
||||
|
||||
// Regression: an executor holding a stale "pending" copy of a step must not
|
||||
// move it back to "ready" after another executor has claimed it, or the
|
||||
// step can be claimed and run a second time.
|
||||
func TestStaleStepCopyCannotMoveStepBackwards(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.allow("service.restart")
|
||||
it := f.intention(false)
|
||||
p := f.start(it)
|
||||
stale := f.step(p, "P1") // pending
|
||||
fresh := stale
|
||||
fresh.Status = StepReady
|
||||
if ok, err := f.plans.SaveIf(f.ctx, fresh, StepPending); err != nil || !ok {
|
||||
t.Fatalf("pending → ready: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := f.plans.Claim(f.ctx, &fresh, StepReady, StepExecuting, time.Minute); err != nil || !ok {
|
||||
t.Fatalf("claim: %v %v", ok, err)
|
||||
}
|
||||
byKey := map[string]*Step{}
|
||||
for i := range p.Steps {
|
||||
byKey[p.Steps[i].Key] = &p.Steps[i]
|
||||
}
|
||||
if _, err := f.exec.advanceStep(f.ctx, &p, it, &stale, byKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := f.step(p, "P1"); got.Status != StepExecuting {
|
||||
t.Fatalf("stale copy overwrote a claimed step: %s", got.Status)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user