ersatztv/scripts/tests/test_review_verdict_vocabulary.py

"""The H10 verdict vocabulary is declared ONCE and both sides derive from it (ersatztv#788).

`scripts/post-review-verdict.sh` (WRITE: word -> commit-status state) and
`scripts/check-review-verdict.sh` (READ: comment -> classification for the merge-consent hook) must
not carry two hand-written copies of the word list. A word the write side treats as positive but the
read side does not sends `review-verdict/h10` green while the hook still denies.

WHAT THIS FILE DELIBERATELY DOES NOT DO. It does not compare the two scripts' sources, and nothing
here scrapes a vocabulary out of shell text. ersatztv#774 shipped exactly that and withdrew it after
six rounds, each finding another shell construction that escaped the extractor or
reddened it on a correct tree; a regex over shell source is not a shell parser and the sequence
converged on nothing. See `docs/decisions/records/testing/guard-derives-population-from-source.md`.

So the proof here is BEHAVIOURAL and runs against the real scripts: edit the one declaration in a
sandbox copy and require the new word to reach both sides, which is only possible if both derive.
The negative half is declared in `scripts/tests/mutation_manifest.py` and executed by the clause
mutation harness on every run: it puts the pre-#788 hardcoded `POS_RE` back into the read side and
requires `test_a_word_added_to_the_shared_source_reaches_BOTH_sides` to go RED. So a future refactor
that quietly reintroduces a second copy cannot leave this suite green.
"""

from __future__ import annotations

import json
import os
import shutil
import subprocess
from pathlib import Path

import pytest

from scripts.tests.test_post_review_verdict import CURL_SHIM, SHA_A

REPO_ROOT = Path(__file__).resolve().parents[2]
VOCABULARY = REPO_ROOT / "scripts" / "lib" / "review-verdict-vocabulary.sh"

# The declaration line the sandbox rewrites. Named once here so a rename of the array is a single
# edit and an obvious red, rather than four silently-not-matching `str.replace` calls.
POSITIVE_DECL = "ETV_VERDICT_POSITIVE_WORDS=(mergeable approved lgtm)"
NEGATIVE_DECL = "ETV_VERDICT_NEGATIVE_WORDS=(blocked not-mergeable)"


class Sandbox:
    """A writable copy of `scripts/`, so a proof may edit the vocabulary without touching the repo.

    The whole directory is copied rather than the three files of interest, because both scripts
    resolve the vocabulary RELATIVE TO THEMSELVES (`dirname "${BASH_SOURCE[0]}"/lib/...`). Copying
    them individually would leave them sourcing the repo's real vocabulary and every mutation below
    would silently prove nothing.
    """

    def __init__(self, root: Path):
        self.scripts = root / "scripts"
        shutil.copytree(REPO_ROOT / "scripts", self.scripts)
        # The gate workflow comes too. `post-review-verdict.sh` derives the H10 reviewer allow-list
        # from it (ersatztv#845), resolving it relative to its own location — so a sandbox holding
        # only `scripts/` would leave the writer unable to answer whether its verdict is inheritable,
        # and every write-side assertion below would fail for a reason that has nothing to do with
        # the vocabulary under test.
        workflow = root / ".gitea" / "workflows" / "review-verdict.yml"
        workflow.parent.mkdir(parents=True)
        shutil.copy2(REPO_ROOT / ".gitea" / "workflows" / "review-verdict.yml", workflow)
        self.vocabulary = self.scripts / "lib" / "review-verdict-vocabulary.sh"
        self.reader = self.scripts / "check-review-verdict.sh"
        self.writer = self.scripts / "post-review-verdict.sh"

        bindir = root / "bin"
        bindir.mkdir()
        shim = bindir / "curl"
        shim.write_text(CURL_SHIM)
        shim.chmod(0o755)

        self.state = root / "state"
        self.state.mkdir()
        (self.state / "pr_shas").write_text(SHA_A)
        (self.state / "pr_bases").write_text("main")
        (self.state / "pr_state").write_text("open")

        self.env = dict(os.environ)
        self.env["PATH"] = f"{bindir}{os.pathsep}{self.env['PATH']}"
        self.env["STUB_DIR"] = str(self.state)
        self.env["ETV_GITEA_TOKEN"] = "stub-token"  # noqa: S105 - fake; real creds come from env
        self.env["ETV_GITEA_URL"] = "http://gitea.example"
        self.env["ETV_GITEA_REPO"] = "timothy/ersatztv"
        self.env.pop("ETV_GITEA_BASICAUTH", None)

    # --- editing the one declaration -----------------------------------------------------------
    def edit_vocabulary(self, old: str, new: str) -> None:
        text = self.vocabulary.read_text()
        assert text.count(old) == 1, f"expected exactly one {old!r} in the shared vocabulary"
        self.vocabulary.write_text(text.replace(old, new))

    # --- driving the two real scripts ----------------------------------------------------------
    def read_side(self, token: str, head: str = SHA_A) -> subprocess.CompletedProcess[str]:
        payload = json.dumps([{"body": f"Review-verdict: {token} @ {head}"}])
        return subprocess.run(
            ["bash", str(self.reader), "--head", head],
            input=payload,
            env=self.env,
            capture_output=True,
            text=True,
        )

    def write_side(self, token: str) -> subprocess.CompletedProcess[str]:
        return subprocess.run(
            ["bash", str(self.writer), "42", token],
            env=self.env,
            capture_output=True,
            text=True,
        )

    def posted_status_state(self) -> str | None:
        log = self.state / "posts.jsonl"
        if not log.exists():
            return None
        for line in log.read_text().splitlines():
            if not line.strip():
                continue
            post = json.loads(line)
            if "/statuses/" in post["url"]:
                return post["payload"]["state"]
        return None


@pytest.fixture
def sandbox(tmp_path):
    return Sandbox(tmp_path)


# --- The load-bearing proof: one declaration, both sides. ---------------------------------------


def test_a_word_added_to_the_shared_source_reaches_BOTH_sides(sandbox):
    """The Done-when of #788, asserted as behaviour rather than as textual agreement.

    `shipit` is in neither list to begin with, so the two `before` assertions are a real negative
    control: they establish that the word is genuinely unknown to both sides, which is what makes
    the `after` assertions evidence of derivation rather than of a word that already worked.
    """
    assert sandbox.read_side("SHIPIT").stdout.strip() == "unknown"
    before = sandbox.write_side("SHIPIT")
    assert before.returncode != 0, "an unknown word must be refused by the write side"
    assert "unknown verdict" in before.stderr

    # ONE line, in ONE file.
    sandbox.edit_vocabulary(POSITIVE_DECL, POSITIVE_DECL[:-1] + " shipit)")

    assert sandbox.read_side("SHIPIT").stdout.strip() == "positive", (
        "the read side did not pick up a word added to the shared vocabulary — it is restating the "
        "word list rather than deriving it"
    )
    after = sandbox.write_side("SHIPIT")
    assert after.returncode == 0, f"the write side still refuses the new word: {after.stderr}"
    assert sandbox.posted_status_state() == "success", (
        "the write side did not map the newly-added POSITIVE word onto a success status"
    )


def test_a_word_added_to_the_NEGATIVE_list_reaches_both_sides_as_a_block(sandbox):
    """The positive list proving derivation would not prove the negative list does too.

    Both are separate arrays and separate call sites; #685 is on record here for two guards on one
    condition where deleting either left the suite green.
    """
    assert sandbox.read_side("VETOED").stdout.strip() == "unknown"

    sandbox.edit_vocabulary(NEGATIVE_DECL, NEGATIVE_DECL[:-1] + " vetoed)")

    assert sandbox.read_side("VETOED").stdout.strip() == "negative"
    assert sandbox.write_side("VETOED").returncode == 0
    assert sandbox.posted_status_state() == "failure", (
        "a newly-added NEGATIVE word must block, not merely be recognised"
    )


# --- Fail-closed: a vocabulary that cannot be loaded is never a pass. ---------------------------


def test_a_MISSING_vocabulary_makes_the_read_side_fail_closed(sandbox):
    sandbox.vocabulary.unlink()
    result = sandbox.read_side("MERGEABLE")
    assert result.returncode == 2, "callers fail closed on exit 2; anything else is a silent pass"
    assert result.stdout.strip() != "positive"
    assert "vocabulary" in result.stderr.lower()


def test_a_MISSING_vocabulary_makes_the_write_side_refuse_to_post(sandbox):
    sandbox.vocabulary.unlink()
    result = sandbox.write_side("MERGEABLE")
    assert result.returncode != 0
    assert sandbox.posted_status_state() is None, "no status may be written without a vocabulary"


@pytest.mark.skipif(
    hasattr(os, "geteuid") and os.geteuid() == 0,
    reason="mode 000 does not make a file unreadable to root, and script-tests runs in a container",
)
def test_an_UNREADABLE_vocabulary_is_refused_rather_than_skipped(sandbox):
    sandbox.vocabulary.chmod(0o000)
    try:
        assert sandbox.read_side("MERGEABLE").returncode == 2
        assert sandbox.write_side("MERGEABLE").returncode != 0
    finally:
        sandbox.vocabulary.chmod(0o644)


# --- The list is interpolated into an ERE, so it is an injection surface. -----------------------


def test_a_REGEX_METACHARACTER_in_the_word_list_is_refused_not_interpolated(sandbox):
    """`.*` in the positive list would make POS_RE match every verdict-shaped line.

    The failure direction is the dangerous one: an explicit BLOCKED would classify `positive` and
    the gate would grant consent nobody gave. The validator must refuse the vocabulary outright
    rather than build a pattern from it.
    """
    sandbox.edit_vocabulary(POSITIVE_DECL, "ETV_VERDICT_POSITIVE_WORDS=(mergeable '.*')")

    result = sandbox.read_side("BLOCKED")
    assert result.returncode == 2, "a vocabulary that cannot be safely interpolated must be fatal"
    assert result.stdout.strip() != "positive", (
        "an explicit BLOCKED verdict classified as positive — the word list reached the regex unvalidated"
    )


def test_a_GLOB_in_the_word_list_is_not_expanded_against_the_filesystem(sandbox):
    """A `*` entry must be reported as `*`, not as whatever files happen to sit alongside.

    An unquoted expansion of a string word-list did exactly that: with `.*` declared, the validator
    reported on `.claude`. Pathname expansion makes the vocabulary depend on the working directory,
    and a `*` expanding to lowercase filenames (`scripts`, `docs`) would VALIDATE and enter the
    vocabulary as real verdict words.

    Run with cwd=REPO_ROOT deliberately: that directory contains `scripts`, `docs` and `.claude`,
    all of which match `[a-z][a-z-]*` and would therefore pass validation if they ever reached it.
    """
    sandbox.edit_vocabulary(POSITIVE_DECL, "ETV_VERDICT_POSITIVE_WORDS=(mergeable '*')")

    result = subprocess.run(
        ["bash", str(sandbox.reader), "--head", SHA_A],
        input=json.dumps([{"body": f"Review-verdict: MERGEABLE @ {SHA_A}"}]),
        env=sandbox.env,
        capture_output=True,
        text=True,
        cwd=str(REPO_ROOT),
    )
    assert result.returncode == 2
    # POSITIVE assertion first: the diagnostic must name the literal word that was written. Without
    # this the test would pass on any exit-2 for any reason, which is the vacuous shape.
    assert "* is not a usable verdict word" in result.stderr, (
        f"the validator did not report on the literal '*': {result.stderr!r}"
    )
    for leaked in ("scripts", "docs", ".claude"):
        assert f"{leaked} is not a usable" not in result.stderr, (
            f"the glob was expanded against the filesystem and reported on {leaked!r}"
        )


def test_the_SAME_word_in_both_lists_is_refused_rather_than_silently_picking_a_side(sandbox):
    sandbox.edit_vocabulary(NEGATIVE_DECL, "ETV_VERDICT_NEGATIVE_WORDS=(blocked not-mergeable lgtm)")
    result = sandbox.read_side("LGTM")
    assert result.returncode == 2
    assert "appears twice" in result.stderr


def test_an_EMPTY_list_is_refused_rather_than_matching_nothing(sandbox):
    """An empty positive list would build a pattern no verdict matches — every PR reads `unknown`.

    That direction is safe but unexplainable, and the empty NEGATIVE list is not safe at all: a
    BLOCKED verdict would stop classifying as negative.
    """
    sandbox.edit_vocabulary(NEGATIVE_DECL, "ETV_VERDICT_NEGATIVE_WORDS=()")
    result = sandbox.read_side("BLOCKED")
    assert result.returncode == 2
    assert result.stdout.strip() != "positive"


# --- The unchanged grammar still holds, now that the words arrive from elsewhere. ---------------


@pytest.mark.parametrize(
    ("token", "expected"),
    [
        ("MERGEABLE", "positive"),
        ("mergeable", "positive"),
        ("Approved", "positive"),
        ("LGTM", "positive"),
        ("BLOCKED", "negative"),
        ("NOT-MERGEABLE", "negative"),
        ("MERGEABLE-LATER", "unknown"),
        ("LGTMish", "unknown"),
        ("SHIPIT", "unknown"),
    ],
)
def test_the_shipped_vocabulary_classifies_as_before(sandbox, token, expected):
    """#788 is a refactor: the classification of every shipped word is unchanged.

    Whole-word matching (`MERGEABLE-LATER`, `LGTMish`) is the #629 false-open and is asserted here
    because the boundary now sits next to an interpolated alternation rather than a literal one.
    """
    assert sandbox.read_side(token).stdout.strip() == expected


@pytest.mark.parametrize(
    ("token", "state"),
    [
        ("MERGEABLE", "success"),
        ("APPROVED", "success"),
        ("LGTM", "success"),
        ("BLOCKED", "failure"),
        ("NOT-MERGEABLE", "failure"),
    ],
)
def test_the_write_side_maps_the_shipped_words_onto_the_same_states(sandbox, token, state):
    """Every shipped word, both polarities. A presence test cannot see POLARITY: a positive-only
    matrix stays green if the negative arm maps onto `success` too."""
    assert sandbox.write_side(token).returncode == 0
    assert sandbox.posted_status_state() == state


# --- A control-flow gate can be SKIPPED; the sentinel is what actually holds. --------------------
#
# Under `set -u`, an unbound-variable error inside a FUNCTION aborts the function but not the
# script, and in an `if ! validate` form NEITHER branch is then taken — so the `exit 2` never runs
# and execution continues past the gate. `check-review-verdict.sh` runs without `set -e`, so the
# words reached the regex unvalidated: with `.*` in the positive list and the negative list written
# as a scalar, an explicit `BLOCKED @ <head>` classified `positive`, exit 0. The calling hook
# discards stderr, so the only signal was invisible.
#
# The fix is a DATA dependency rather than a control-flow one: validation sets
# `ETV_VERDICT_VOCABULARY_OK=1` on its last line, and the derived views refuse without it. These
# tests pin the malformation classes that produce the abort.


def test_a_SCALAR_word_list_cannot_reach_the_regex_unvalidated(sandbox):
    """The originally-measured fail-open, pinned end to end.

    Two faults at once, because that is the shape that produced `positive`: a metacharacter in the
    positive list (which only matters if validation is skipped) and a scalar negative list (which is
    what skips it).
    """
    sandbox.edit_vocabulary(POSITIVE_DECL, 'ETV_VERDICT_POSITIVE_WORDS=(mergeable ".*")')
    sandbox.edit_vocabulary(NEGATIVE_DECL, 'ETV_VERDICT_NEGATIVE_WORDS="blocked"')

    result = sandbox.read_side("BLOCKED")
    assert result.stdout.strip() != "positive", (
        "an explicit BLOCKED verdict classified as positive — the word list reached the regex "
        "without validation completing"
    )
    assert result.returncode == 2


def test_a_SCALAR_word_list_is_refused_even_with_no_metacharacter(sandbox):
    """The single-fault case, which is the realistic wrong edit.

    Writing the list as a string is the natural mistake, and on its own it silently degraded every
    negative verdict from `negative` to `unknown` — DENY downgraded to ASK — with no non-zero exit.
    """
    sandbox.edit_vocabulary(NEGATIVE_DECL, 'ETV_VERDICT_NEGATIVE_WORDS="blocked not-mergeable"')

    result = sandbox.read_side("BLOCKED")
    assert result.returncode == 2
    assert result.stdout.strip() != "negative", "expected a refusal, not a lucky classification"
    assert "is not an array" in result.stderr


def test_an_UNSET_word_list_is_refused(sandbox):
    sandbox.edit_vocabulary(NEGATIVE_DECL, "unset ETV_VERDICT_NEGATIVE_WORDS")
    result = sandbox.read_side("BLOCKED")
    assert result.returncode == 2
    assert result.stdout.strip() != "positive"


def test_a_library_that_EXITS_at_top_level_does_not_return_exit_0_with_no_verdict(sandbox):
    """A top-level `exit` in a SOURCED file terminates the sourcing script where it stands.

    No check written after the `source` can run, so the reader returned exit 0 with EMPTY stdout —
    violating its own "exactly one classification word" contract. A caller testing only the exit
    code would read that as a successful classification. The subshell probe moves the failure
    somewhere observable.
    """
    sandbox.vocabulary.write_text("exit 0\n")

    result = sandbox.read_side("BLOCKED")
    assert result.returncode == 2, f"exit {result.returncode} with stdout {result.stdout!r}"
    assert result.stdout.strip() == ""

    write = sandbox.write_side("MERGEABLE")
    assert write.returncode != 0
    assert sandbox.posted_status_state() is None


def test_a_TRUNCATED_library_is_refused_rather_than_half_used(sandbox):
    """A file cut short can define every function it got to and still be incomplete.

    So the marker the probe requires is the LAST line of the library, not merely a function it
    defines somewhere.
    """
    text = sandbox.vocabulary.read_text()
    marker = "etv_verdict_vocabulary_loaded()"
    assert marker in text, "the end-of-file marker moved; this test no longer truncates anything"
    # Cut exactly at the marker: everything before it is a file that defines every function it got
    # to and is still incomplete. A fixed line count would drift as the library grows.
    sandbox.vocabulary.write_text(text[: text.index(marker)])

    result = sandbox.read_side("BLOCKED")
    assert result.returncode == 2
    assert result.stdout.strip() != "positive"


# The array assertion and the sentinel are NOT redundant, and this is where that is proved.
#
# `etv_verdict__is_array` removes the `set -u` abort, so with it in place validation returns 1
# cleanly and the ordinary `if ! validate; then exit 2; fi` fires — which means every other refusal
# test in this file stays green even with the sentinel deleted. MEASURED:
# all three sentinel guards removed, 150 tests still passing.
#
# So the sentinel is proved by DISARMING THE OTHER GUARD FIRST. With the array assertion gone the
# abort route reopens, the `if` is skipped, and the sentinel is the only thing left standing between
# a malformed word list and the regex.
#
# EXACTLY WHICH COPY THAT PROVES, stated because the obvious reading overclaims. There are three
# sentinel guards, one per derived view, and only the READ side's (`etv_verdict_alternation`) has a
# behavioural alone-deletion red — measured: deleting it alone reddens the three tests below,
# deleting either of the other two reddens only the `count == 3` tripwire further down. The write
# side's two are defence-in-depth: `post-review-verdict.sh` runs under `set -e`, so the abort that
# defeats the read side kills the writer outright (measured: rc=1, nothing posted). They are kept
# because the writer should not depend on `set -e` staying, but their current proof is a COUNT, not
# a behaviour, and saying otherwise would be that same overclaim one level down.
ARRAY_ASSERTION = (
    "  for name in ETV_VERDICT_POSITIVE_WORDS ETV_VERDICT_NEGATIVE_WORDS; do\n"
    '    if ! etv_verdict__is_array "$name"; then\n'
    "      printf 'review-verdict vocabulary: %s is not an array — declare it as "
    '%s=(word word), not as a string\\n\' "$name" "$name" >&2\n'
    "      return 1\n"
    "    fi\n"
    "  done\n"
)

SENTINEL_GUARD = """  if [ "${ETV_VERDICT_VOCABULARY_OK:-0}" != 1 ]; then
    printf 'review-verdict vocabulary: not validated — refusing to hand out verdict words\\n' >&2
    return 1
  fi
"""


def _disarm_array_assertion(sandbox):
    """Reopen the `set -u` abort route by removing the assertion that prevents it."""
    text = sandbox.vocabulary.read_text()
    assert ARRAY_ASSERTION in text, "the array assertion moved; this proof no longer disarms it"
    sandbox.vocabulary.write_text(text.replace(ARRAY_ASSERTION, ""))


def test_MUTATION_with_the_array_assertion_disarmed_the_SENTINEL_still_refuses(sandbox):
    """The sentinel is load-bearing, not decoration.

    Disarm the array assertion, then declare the negative list as a scalar: `${#arr[@]}` aborts the
    validator mid-way, `if ! validate` takes neither branch, and the `exit 2` never runs. Only the
    sentinel — never assigned, because validation did not reach its last line — stops the words.
    """
    _disarm_array_assertion(sandbox)
    sandbox.edit_vocabulary(NEGATIVE_DECL, 'ETV_VERDICT_NEGATIVE_WORDS="blocked"')

    result = sandbox.read_side("BLOCKED")
    assert result.stdout.strip() != "positive"
    assert result.returncode == 2
    assert "not validated" in result.stderr, (
        "expected the SENTINEL's refusal specifically; a refusal from some other clause would not "
        f"prove the sentinel does anything. stderr={result.stderr!r}"
    )


def test_MUTATION_removing_BOTH_guards_reopens_the_original_fail_open(sandbox):
    """The negative control for the test above — without it, that test proves only 'it refuses'.

    With the array assertion AND the sentinel both gone, this is the tree as first committed, and
    the measured fail-open comes back: an explicit BLOCKED classifies `positive`, exit 0. Pinning it
    here means the pair cannot both be deleted without a red.
    """
    _disarm_array_assertion(sandbox)
    text = sandbox.vocabulary.read_text()
    assert text.count(SENTINEL_GUARD) == 3, f"expected 3 sentinel guards, found {text.count(SENTINEL_GUARD)}"
    sandbox.vocabulary.write_text(text.replace(SENTINEL_GUARD, ""))

    sandbox.edit_vocabulary(POSITIVE_DECL, 'ETV_VERDICT_POSITIVE_WORDS=(mergeable ".*")')
    sandbox.edit_vocabulary(NEGATIVE_DECL, 'ETV_VERDICT_NEGATIVE_WORDS="blocked"')

    result = sandbox.read_side("BLOCKED")
    assert (result.returncode, result.stdout.strip()) == (0, "positive"), (
        "the historical fail-open no longer reproduces with both guards removed — this control is "
        f"no longer measuring what it claims. got rc={result.returncode} out={result.stdout!r}"
    )


def test_an_inherited_sentinel_cannot_stand_in_for_validation(sandbox):
    """Reset-at-load is what makes the sentinel unforgeable from the environment.

    Tested with the array assertion disarmed, so the abort route is open and the inherited value is
    the ONLY thing that could carry a malformed vocabulary through.
    """
    _disarm_array_assertion(sandbox)
    sandbox.edit_vocabulary(NEGATIVE_DECL, 'ETV_VERDICT_NEGATIVE_WORDS="blocked"')

    env = dict(sandbox.env)
    env["ETV_VERDICT_VOCABULARY_OK"] = "1"
    result = subprocess.run(
        ["bash", str(sandbox.reader), "--head", SHA_A],
        input=json.dumps([{"body": f"Review-verdict: BLOCKED @ {SHA_A}"}]),
        env=env,
        capture_output=True,
        text=True,
    )
    assert result.returncode == 2, "an inherited sentinel must not stand in for validation"
    assert result.stdout.strip() != "positive"
    assert "not validated" in result.stderr


def test_a_VALID_vocabulary_still_classifies(sandbox):
    """The positive control for every refusal above: the guards must not simply always refuse."""
    result = sandbox.read_side("BLOCKED")
    assert result.returncode == 0
    assert result.stdout.strip() == "negative"