From a083c851b3535fe4e10c6e0c6b6a61ecc9477a4e Mon Sep 17 00:00:00 2001 From: Timothy Date: Sat, 5 Sep 2026 14:13:56 +0200 Subject: [PATCH] fix(916): the harness counts lens completions per round, so the runner assertion can fail The runner observer reset the shared counter to zero one line before reading it, so its assertion held under the very mutant it existed to reject and the fallback assertion caught that mutant for the wrong reason. Completions are now keyed by the round in each agent's own label; no stub resets shared state. Measured: re-serialising the runner reddens the runner assertion (expected [2] to equal [0]) in both scripts; moving the fallback beside the lenses reddens the fallback assertion and the round-one-failure case. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015QqCpYFsKgnAnx6jVwrKiV --- .../orchestration-workflow-loop.test.mjs | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/web/scripts/orchestration-workflow-loop.test.mjs b/web/scripts/orchestration-workflow-loop.test.mjs index db9aa63cd..6dca2fb44 100644 --- a/web/scripts/orchestration-workflow-loop.test.mjs +++ b/web/scripts/orchestration-workflow-loop.test.mjs @@ -36,13 +36,19 @@ const IMPL_COMMITS = 'aaaaaaa feat(1): the implementer commit'; const finding = (severity) => ({ severity, file: 'f', summary: `${severity} finding`, evidence: 'e' }); // Every stub lens returns the same findings, so a finding counts once per lens (two lenses → 2). -// reviewRounds: array indexed by round-1 of {findings, lensesNull?}; landOverrides: extra fields on the finisher report; -// postRebase: findings returned by the review after a patch-changing rebase, or 'lensesNull'; fixerNull: the fixer agent dies. +// reviewRounds: array indexed by round-1 of {findings, lensesNull?, runnerRan?} — every review-stage stub reads +// its round from its own label (review::rN), never from a shared counter; landOverrides: extra fields on +// the finisher report; postRebase: findings returned by the review after a patch-changing rebase, or 'lensesNull'; +// fixerNull / fixerDone / fixerNoCommit: fixer shapes; fallbackNull: true, or an array of rounds in which the +// fallback agent dies. function run(script, args, { reviewRounds, landOverrides = {}, postRebase = null, fixerNull = false, fixerDone = true, fixerNoCommit = false, fallbackNull = false }) { const calls = []; - let round = 0; let fixCount = 0; - let lensesReturned = 0; + // Lens completions are counted PER ROUND, keyed by the round in each agent's label (review::rN), + // so an observer never depends on another stub resetting shared state. + const lensesReturnedByRound = new Map(); + const roundOf = (l) => Number(/:r(\d+)$/.exec(l)[1]); + const returnedIn = (l) => lensesReturnedByRound.get(roundOf(l)) || 0; const lensesReturnedWhenFallbackStarted = []; const lensesReturnedWhenRunnerStarted = []; const agent = async (prompt, opts) => { @@ -50,21 +56,20 @@ function run(script, args, { reviewRounds, landOverrides = {}, postRebase = null calls.push({ label, prompt, opts }); if (label.startsWith('impl:') || label.startsWith('fix:#')) return { done: true, summary: '', verified: '', left: '', commits: IMPL_COMMITS, head_sha: 'aaaaaaa' }; if (label.startsWith('review:codex:')) { - lensesReturned = 0; - lensesReturnedWhenRunnerStarted.push(lensesReturned); - const spec = reviewRounds[round] || {}; + lensesReturnedWhenRunnerStarted.push(returnedIn(label)); + const spec = reviewRounds[roundOf(label) - 1] || {}; return { verdict: 'merge', findings: [], ran: spec.runnerRan === true }; } if (label.startsWith('review:fallback:')) { - lensesReturnedWhenFallbackStarted.push(lensesReturned); - if (fallbackNull === true || (Array.isArray(fallbackNull) && fallbackNull.includes(round))) return null; - const spec = reviewRounds[round - 1] || { findings: [] }; + lensesReturnedWhenFallbackStarted.push(returnedIn(label)); + if (fallbackNull === true || (Array.isArray(fallbackNull) && fallbackNull.includes(roundOf(label)))) return null; + const spec = reviewRounds[roundOf(label) - 1] || { findings: [] }; return { verdict: 'merge', findings: spec.findings }; } if (label.startsWith('review:')) { - const spec = reviewRounds[round - 1] || { findings: [] }; + const spec = reviewRounds[roundOf(label) - 1] || { findings: [] }; await new Promise((resolve) => globalThis.setTimeout(resolve, 5)); - lensesReturned += 1; + lensesReturnedByRound.set(roundOf(label), returnedIn(label) + 1); if (spec.lensesNull) return null; return { verdict: 'merge', findings: spec.findings }; } @@ -78,8 +83,6 @@ function run(script, args, { reviewRounds, landOverrides = {}, postRebase = null throw new Error(`unexpected agent label ${label}`); }; const parallel = async (thunks) => { - round += 1; - lensesReturned = 0; if (postRebase !== null && calls.some((c) => c.label.startsWith('land:'))) { return Promise.all(thunks.map(() => Promise.resolve(postRebase === 'lensesNull' ? null : { verdict: 'merge', findings: postRebase }))); }