diff --git a/.gitea/workflows/review-verdict.yml b/.gitea/workflows/review-verdict.yml index defddaac3..ffd206401 100644 --- a/.gitea/workflows/review-verdict.yml +++ b/.gitea/workflows/review-verdict.yml @@ -452,7 +452,7 @@ jobs: exit 1 ;; 0) ;; *) - echo "::error::${CONTEXT} was not on page 1 of the statuses for ${SHA:0:7}, but page 2 carries ${more_len} more row(s) — the list is longer than one page and an existing verdict may be beyond it. Refusing to post anything rather than overwrite a verdict this job cannot see." + echo "::error::${CONTEXT} was not on page 1 of the statuses for ${SHA:0:7}, but page 2 carries ${more_len} more row(s) — the list is longer than one page and an existing verdict may be beyond it. Refusing to post anything rather than overwrite a verdict this job cannot see. A human verdict clears this: scripts/post-review-verdict.sh ${PR} MERGEABLE." exit 1 ;; esac ;; *) @@ -602,9 +602,13 @@ jobs: # workflow once at jq 1.6 (ersatztv#647). A `case` over `jq -r 'type'` cannot be read # two ways. Anything that is neither `null` nor `array` is still "cannot tell". kind=$(printf '%s' "$raw" | jq -r 'type' 2>/dev/null) || kind="" - # `null` counts as exhaustion only from page 2 ON. Every real PR's first page carries - # events — measured across #752/#753/#749/#739/#717 (4, 2, 9, 5, 10) — so a `null` - # FIRST page is anomalous rather than empty, and trusting a zero count from it would + # `null` counts as exhaustion only from page 2 ON. THE INVARIANT, not a figure that + # rots: a real PR's timeline always carries at least one event on page 1 (it is created + # by a push, which is itself an event), so a `null` FIRST page is anomalous rather than + # empty. Spot-checked across #752/#753/#749/#739/#717, all non-empty; the counts + # themselves are deliberately not recorded here because timelines grow and an earlier + # version of this comment cited five numbers of which three were stale within days. + # Trusting a zero count from an anomalous first page would # mean trusting that no retarget happened on the strength of a response we cannot # explain. Requiring one real page keeps the property the walk is for: something was # actually read. A PR that somehow has an empty first page falls through to `rt_ok=no`, @@ -942,9 +946,21 @@ jobs: # live head), `/statuses/{sha}` a BARE ARRAY (24 rows on the same head) — hence the different # `type == "array"` guard here. # - # No claim is made about the order rows come back in, because the check does not depend on - # it: it selects by id against the high-water mark rather than inspecting the top of the - # list. A verdict older than the mark is invisible to it no matter where it sits. + # ORDER IS NOT RELIED ON *WITHIN* A PAGE — the check selects by id against the high-water + # mark rather than inspecting the top of the list, so a verdict older than the mark is + # invisible to it no matter where it sits. But this read is ONE PAGE, and `limit=100` clamps + # to the server-wide `MAX_RESPONSE_ITEMS`, measured at 50 on this instance (ersatztv#751). + # An earlier version of this comment claimed order-independence flatly; that is false the + # moment a head carries more rows than the clamp, because a raced row can then sit on a page + # this never reads. Measured: a probe head reached 33 rows after ~5 runs, and the ordering is + # only coarsely newest-first (ids came back `33,32,31,30,28,29,27,…`), so >50 is ordinarily + # reachable on a PR with a few CI reruns. + # + # That matters more here than anywhere else in this job, because this is the ONE path whose + # failure direction is toward SUCCESS: missing a raced human `failure` leaves a forged green + # standing over a rejection. So the page-2 probe below treats "there are rows I did not read" + # as "assume raced" — the conservative direction — rather than as "no race found". + # Full paging of this endpoint, including for the high-water mark above, is ersatztv#763. # # The repair is `pending`, NEVER a copy of the human's state. Re-posting their `failure` # would attribute a human verdict to this job — the exact provenance confusion the @@ -989,6 +1005,30 @@ jobs: and (((.description // "") | startswith("Review-verdict:")))) or ((.creator == null) and ((.description // "") == $rd)) )] | length') + # "NO RACE FOUND ON PAGE 1" IS NOT "NO RACE" (ersatztv#751). If page 1 was not the whole + # list, a raced row can be beyond it, so an unread page is treated as a race rather than + # as absence. Only checked when page 1 looked clean — a race already found needs no + # further evidence. Unreadable page 2 also counts as raced: this is the fail-toward- + # SUCCESS path, so uncertainty must resolve to `pending`, never to leaving green. + if [ "$raced" = "0" ]; then + more_hist=$(gh "$BASE_URL/repos/$REPO/statuses/$SHA?limit=100&page=2") || more_hist="UNREADABLE" + if [ "$more_hist" = "UNREADABLE" ]; then + echo "::warning::Could not read page 2 of the status history for ${SHA:0:7}; treating this as a possible raced verdict and repairing to pending rather than leaving an exemption green on an unverified head." + raced=1 + else + mh_kind=$(printf '%s' "$more_hist" | jq -r 'type' 2>/dev/null) || mh_kind="" + mh_len=$(printf '%s' "$more_hist" | jq -r 'if type == "array" then length else 0 end' 2>/dev/null) || mh_len="" + case "${mh_kind}:${mh_len}" in + null:*|array:0) ;; + array:*) + echo "::warning::The status history for ${SHA:0:7} runs past page 1 (${mh_len} more row(s)), so a raced human verdict could be on a page this job did not read. Repairing to pending rather than leaving the exemption green." + raced=1 ;; + *) + echo "::warning::Page 2 of the status history for ${SHA:0:7} was '${mh_kind:-unparseable}'; treating as a possible raced verdict and repairing to pending." + raced=1 ;; + esac + fi + fi case "$raced" in ''|*[!0-9]*) echo "::warning::Post-write verification for ${SHA:0:7} returned '${raced}' instead of a count; not acting on it." diff --git a/docs/ci-cd.md b/docs/ci-cd.md index a8fa92ede..83cd4732e 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -1072,14 +1072,19 @@ Two things kept it invisible, and both are worth generalising: "Real shapes, measured on this instance and deliberately mirrored" and it printed `[]` for a page past the end. Every fence test was green against a response the server never produces, so the `array`-only gate was never exercised by the suite either. With the double corrected and the old - gate restored, **18 tests fail** — they had all been passing for the wrong reason. When a double - claims fidelity, that claim is a test assertion and needs re-measuring like any other. + gate restored, **most of the fence suite fails** — 18 tests when first measured at `c710db4a1`, 21 + once three more fence-dependent tests existed. The invariant is the point, not the count: they had + all been passing for the wrong reason. (Given as a range on purpose — an earlier draft cited a bare + "18", which was stale two commits later, inside a section about stale claims.) When a double claims + fidelity, that claim is a test assertion and needs re-measuring like any other. The type is now read as a value (`case` over `jq -r 'type'`) rather than through `jq -e`, whose exit-status semantics already bit this workflow once at jq 1.6, and both `null` and `[]` terminate the walk. The regression test is parameterised over both shapes because both are live on this server. `null` is accepted as exhaustion only from **page 2 on** — every real PR's first page carries events -(measured: 4, 2, 9, 5, 10 across #752/#753/#749/#739/#717), so a `null` first page is anomalous rather +(spot-checked non-empty across #752/#753/#749/#739/#717; the counts are deliberately not recorded here +because timelines grow and an earlier draft's five figures were stale within days), so a `null` first +page is anomalous rather than empty, and the walk should not certify "no retarget happened" from a response it cannot explain. **The same nil-slice shape bites `/commits/{sha}/status`** — a third instance, found by cold review of diff --git a/docs/decisions/records/ci/verdict-write-retarget-fence.md b/docs/decisions/records/ci/verdict-write-retarget-fence.md index c93f4ef8a..ef95fcbe1 100644 --- a/docs/decisions/records/ci/verdict-write-retarget-fence.md +++ b/docs/decisions/records/ci/verdict-write-retarget-fence.md @@ -87,8 +87,10 @@ Two reasons it read as deliberate rather than broken, both worth carrying forwar - **The double asserted the wrong shape while claiming to be measured.** The stub's comment read "Real shapes, measured on this instance and deliberately mirrored" and it printed `[]` past the end. So the `array`-only gate was never exercised by the suite either. Correcting the double and restoring the - old gate turns **18 tests red**; every one of them had been green for the wrong reason. A fidelity - claim in a test double is an assertion, and it decays like any other. + old gate reddens most of the fence suite — 18 tests when first measured, 21 once three more + fence-dependent tests existed. The invariant, not the number, is that every one of them had been + green for the wrong reason. A fidelity claim in a test double is an assertion, and it decays like any + other. The type is now read as a value (`case` over `jq -r 'type'`) rather than through `jq -e`, whose exit-status semantics already bit this workflow at jq 1.6 (`ci.jq-version-contract`), and both `null` diff --git a/scripts/tests/test_pr_changed_files.py b/scripts/tests/test_pr_changed_files.py index 8094f9402..eeec10e6b 100644 --- a/scripts/tests/test_pr_changed_files.py +++ b/scripts/tests/test_pr_changed_files.py @@ -969,6 +969,38 @@ if "/timeline" in url: # endpoint no longer shows. if "/statuses/" in url: mode = os.environ.get("STUB_HISTORY_MODE", "none") + + # PAGE, honoured, and BEFORE the read counters — same reason as the combined endpoint (#751). The + # post-write check now reads page 2 to decide whether page 1 was the whole list, and the + # `*-after-post` modes key on how many times the job has LOOKED at the history; the page-2 probe is + # part of the same look. Letting it bump those counters would shift "raced row appears on read N". + # + # PAST THE END THIS ENDPOINT RETURNS `[]`, NOT `null` — measured at 1.27.1: + # `/statuses/{sha}?limit=100&page=99` is the two bytes `[]`, while `/commits/{sha}/status` past the + # end returns `{"statuses": null}`. A THIRD distinct empty shape on one server; the job tolerates + # both here precisely because guessing per endpoint has been wrong twice. + hist_page = 1 + for part in url.split("?", 1)[-1].split("&"): + if part.startswith("page="): + try: + hist_page = int(part.split("=", 1)[1]) + except ValueError: + hist_page = 1 + if hist_page > 1 and mode == "second-page-garbage": + print("502 Bad Gateway") + sys.exit(0) + if hist_page > 1 and mode == "second-page-error": + sys.exit(22) + if hist_page > 1 and mode not in ("second-page", "second-page-garbage", "second-page-error"): + print("[]") + sys.exit(0) + if mode == "second-page": + # The history runs past page 1, so "no raced row on page 1" does not establish "no race". The + # rows are deliberately ORDINARY (no verdict, no sentinel): the point is that unread rows EXIST, + # not that a verdict was found, so the repair must fire on uncertainty alone. + print(json.dumps([{"id": 7000 + i, "context": "ci/other", "status": "success", + "creator": None, "description": "unrelated"} for i in range(3)])) + sys.exit(0) rows = [] if mode.startswith("human-after-post"): # The raced verdict: absent when the high-water mark is taken, present afterwards. Its id is @@ -1039,6 +1071,11 @@ if "/status" in url: # and broke three mid-run-race tests. Page 2 also has to terminate here for every mode that # describes page 1 only, or the job would re-read page 1's rows as a second page and conclude the # list is longer than it is. + if status_page > 1 and mode == "page2-garbage": + print("502 Bad Gateway") + sys.exit(0) + if status_page > 1 and mode == "page2-error": + sys.exit(22) if status_page > 1 and mode != "twopage": print(empty_statuses()) sys.exit(0) @@ -2285,9 +2322,12 @@ def test_the_verdict_workflow_has_NO_expression_delimiter_in_any_run_body(): The rule is also what keeps the dropped-step guard trustworthy. A guard that the guarded mechanism can silently delete is worse than no guard, because its absence is silent too. - RAW TEXT, never `_code_lines` — see the block comment above. A comment is not inert here. + The `run:` SCALAR AS PARSED, comments and all — never `_code_lines`, which strips them. The + distinction matters and is easy to garble: what must not be filtered is the SHELL comments inside + the body, because those are what the runner scans. Ordinary YAML comments outside a `run:` body are + genuinely inert and are handled by the repo-wide test below, which is why neither test reads the + file as flat text any more. """ - raw = WORKFLOW.read_text() offenders = [] for job_name, step in _iter_workflow_steps(WORKFLOW): for m in _EXPR.finditer(step.get("run") or ""): @@ -2323,9 +2363,13 @@ def test_the_verdict_workflow_has_NO_expression_delimiter_in_any_run_body(): # classifier is ~700 lines; a walk that returned nothing, or only the short steps, is the failure # to catch. Both are asserted on content, which no spelling change can spoof. bodies = [s["run"] for _, s in _iter_workflow_steps(WORKFLOW) if s.get("run")] - assert len(bodies) >= 3, ( - f"the YAML walk reached only {len(bodies)} run: bodies in review-verdict.yml; the assertion " - "above cannot have examined the classifier, so a green here proves nothing") + # NOT `>= 3`. The file has exactly three `run:` bodies today, so that bound had zero slack: cold + # review deleted the optional jq-preflight step — a legitimate simplification — and this redded + # with a message claiming the classifier had not been examined, which was untrue. The length + # assertion below already carries the whole property, so the count only needs to be non-empty. + assert bodies, ( + "the YAML walk reached NO run: bodies in review-verdict.yml, so the assertion above examined " + "nothing and a green here proves nothing") assert max(len(b) for b in bodies) > 5000, ( "the YAML walk did not reach a substantial run: body — the ~700-line classifier is the one " "that must be scanned, so this test would be vacuous") @@ -2339,19 +2383,6 @@ def _iter_workflow_steps(path: Path): yield job_name, step -def _workflow_expression_fields_text(path: Path) -> str: - """Every `with:`/`env:` VALUE in the file, concatenated — the places an expression belongs.""" - chunks = [] - import yaml - doc = yaml.safe_load(path.read_text()) or {} - for job in (doc.get("jobs") or {}).values(): - for holder in (job, *(job.get("steps") or [])): - for key in ("env", "with"): - for v in (holder.get(key) or {}).values(): - chunks.append(str(v)) - return "\n".join(chunks) - - def test_every_workflow_expression_names_a_REAL_context_or_function(): """The general form of #751, across every workflow and every field. @@ -2592,7 +2623,7 @@ def test_the_dropped_step_guard_BEHAVIOURALLY_fails_without_the_marker_and_passe # assertion makes the intent explicit and fails loudly rather than subtly. body_before = "\n".join(ln for ln in prologue.splitlines() if not ln.lstrip().startswith("#")) - assert not re.search(r"^\s*\w+\s*\(\)\s*\{", body_before, re.M), ( + assert not re.search(r"^\s*(\w+\s*\(\)|function\s+\w+)\s*\{", body_before, re.M), ( "a function is defined before the marker write, so the write may be inside it and unreached " f"in production while this test still passes:\n{body_before}") @@ -2655,8 +2686,11 @@ def test_the_GOVERNANCE_docs_are_protected_and_get_no_docs_only_exemption(tmp_pa def test_a_terminator_on_PAGE_ONE_does_not_certify_a_zero_retarget_count(tmp_path): """`null` means "exhausted" only after a real page has been read. - Every real PR's first page carries events — measured 4, 2, 9, 5, 10 on #752/#753/#749/#739/#717 — - so a terminator on page 1 is anomalous, not empty. Trusting a zero count from it would mean + THE INVARIANT, not a figure that rots: a real PR's timeline always carries at least one event on + page 1, because the PR is created by a push and that is itself an event. Spot-checked non-empty + across #752/#753/#749/#739/#717; the counts are not recorded, because an earlier version of this + docstring cited five and three were stale within days. So a terminator on page 1 is anomalous, not + empty. Trusting a zero count from it would mean certifying that no retarget happened on the strength of a response we cannot explain, which is the one thing the fence exists to refuse. Withholding the exemption is the safe direction: the PR asks for a human verdict instead. @@ -2722,3 +2756,73 @@ def test_a_STRING_total_count_is_not_accepted_as_numeric_zero(tmp_path): assert posted is None, ( f"a string total_count was accepted as numeric zero, so a body that merely lost its statuses " f"array reads as 'no verdict exists': {posted}\n{r.stdout[-800:]}") + + +@pytest.mark.parametrize("mode,why", [ + ("page2-garbage", "a non-JSON page 2"), + ("page2-error", "an HTTP error on page 2"), +], ids=["garbage", "transport-error"]) +def test_an_UNREADABLE_page_2_refuses_to_conclude_that_no_verdict_exists(tmp_path, mode, why): + """The two refuse branches of the completeness probe, which cold review found untested. + + Worth a test rather than trusting the shape: this file's history is two consecutive guards that + were UNREACHABLE — the `count_retargets` type gate that never saw a real terminator, and a + full-page check written against a limit of 100 on a server that caps at 50. An unexercised branch + here has a track record. + + Both must fail CLOSED: the point of reading page 2 is to justify "no verdict exists", so a page 2 + that cannot be read justifies nothing. + """ + posted, r = _run_classify(tmp_path, _emitting("docs/a.md"), status_mode=mode) + assert posted is None, ( + f"an exemption was posted despite {why}, so 'no verdict exists' was concluded without " + f"evidence: {posted}\n{r.stdout[-800:]}") + assert "page 2" in (r.stdout + r.stderr).lower(), ( + f"nothing was posted, but not because of the page-2 probe:\n{r.stdout[-800:]}") + + +def test_a_status_history_RUNNING_PAST_PAGE_1_repairs_rather_than_leaving_green(tmp_path): + """The post-write race check reads ONE page of `/statuses/{sha}`, and `limit=100` clamps to the + server-wide cap (measured 50). So "no raced row on page 1" does not establish "no race" — a raced + human `failure` can sit on a page this job never reads. + + This is the ONE path in the design whose failure direction is toward SUCCESS: missing a raced + verdict leaves a forged green over a human rejection, permanently. So unread rows are treated as a + race and the exemption is repaired to `pending`, which is the conservative direction — a stall a + reviewer can clear, rather than a rejection silently turned green. + + The stub's second page carries ORDINARY rows, no verdict and no sentinel: what must trigger the + repair is the mere existence of rows this job did not read, not the discovery of a verdict. + """ + posted, r = _run_classify(tmp_path, _emitting("docs/a.md"), history_mode="second-page") + assert r.returncode == 0, r.stderr + seq = _posted_sequence(tmp_path) + assert len(seq) == 2, ( + "the exemption was posted and left standing even though the status history ran past page 1, so " + f"a raced verdict beyond it would be buried. Posts: {seq}\n{r.stdout[-900:]}") + assert seq[0]["state"] == "success" and seq[1]["state"] == "pending", ( + f"expected an exemption then a repair to pending; got {seq}") + assert seq[1]["description"] == "Human verdict raced this exemption write — re-post the verdict", ( + f"repaired, but not to the sentinel, so the fixed point is broken: {seq[1]}") + + +@pytest.mark.parametrize("mode,why", [ + ("second-page-garbage", "a non-JSON page 2"), + ("second-page-error", "an HTTP error on page 2"), +], ids=["garbage", "transport-error"]) +def test_an_UNREADABLE_history_page_2_also_repairs_rather_than_leaving_green(tmp_path, mode, why): + """Found by mutating the branch and watching nothing go red — my own coverage gap, in the same + class the review had just flagged twice. + + This is the fail-toward-SUCCESS path, so uncertainty must resolve to `pending`. "I could not read + the rest of the history" is not evidence that no verdict raced this write; treating it as such is + exactly how a forged green survives over a human rejection. + """ + posted, r = _run_classify(tmp_path, _emitting("docs/a.md"), history_mode=mode) + assert r.returncode == 0, r.stderr + seq = _posted_sequence(tmp_path) + assert len(seq) == 2, ( + f"the exemption was left standing despite {why} — a raced verdict beyond page 1 would be " + f"buried. Posts: {seq}\n{r.stdout[-900:]}") + assert seq[0]["state"] == "success" and seq[1]["state"] == "pending", ( + f"expected an exemption then a repair to pending; got {seq}")