Compare commits
159
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7fcb5e9b28 | ||
|
|
884ac8a7e9 | ||
|
|
9a5d34e888 | ||
|
|
20b117dabf | ||
|
|
e133c11fde | ||
|
|
951dae26a9 | ||
|
|
46ec532745 | ||
|
|
edd8d3d9c9 | ||
|
|
40b3747434 | ||
|
|
2bdb6c44e4 | ||
|
|
9881d1ff81 | ||
|
|
3aed43c6de | ||
|
|
f822e4737c | ||
|
|
6af65ba5c5 | ||
|
|
6d80343320 | ||
|
|
b91707b707 | ||
|
|
691a7acc77 | ||
|
|
08e95f9ec1 | ||
|
|
b91939e5c4 | ||
|
|
e298bb291e | ||
|
|
d7647b6104 | ||
|
|
7be42654fe | ||
|
|
d7725c274c | ||
|
|
b6bf94f129 | ||
|
|
4be3f247d8 | ||
|
|
28ce8c4dfe | ||
|
|
e46e2cfe68 | ||
|
|
3a6174c953 | ||
|
|
5fa672e2e5 | ||
|
|
a2b3a56d93 | ||
|
|
772277e255 | ||
|
|
efc34a3481 | ||
|
|
57ad5efb3f | ||
|
|
0ff9671393 | ||
|
|
56afa4652d | ||
|
|
2ff52d4236 | ||
|
|
b24c51ab51 | ||
|
|
c56dfdd539 | ||
|
|
0db56c3ebd | ||
|
|
b3a8826281 | ||
|
|
80818aa294 | ||
|
|
9d2b30dc3b | ||
|
|
aa79ec59c9 | ||
|
|
fe3d29276a | ||
|
|
a8bbd74a64 | ||
|
|
5077408528 | ||
|
|
63040296f4 | ||
|
|
8f6d4f4432 | ||
|
|
7cd71327b0 | ||
|
|
53e6be8390 | ||
|
|
b99812eb8b | ||
|
|
03662dcdfd | ||
|
|
aaa4e869b8 | ||
|
|
9dc360c9fa | ||
|
|
95f36d1c45 | ||
|
|
ff8b0bf984 | ||
|
|
016a05ced8 | ||
|
|
9928be805f | ||
|
|
fbbdaeca3c | ||
|
|
f9cbd152bc | ||
|
|
980da6db00 | ||
|
|
81be685df9 | ||
|
|
1eca9b0c11 | ||
|
|
a3458e6e2c | ||
|
|
57e33f9937 | ||
|
|
fe00e0d71f | ||
|
|
ef92b46dd2 | ||
|
|
e7bae06385 | ||
|
|
d4c600149d | ||
|
|
d8bd1dcba9 | ||
|
|
bafb487eaa | ||
|
|
f523fc535d | ||
|
|
0c492defac | ||
|
|
4e2ea61674 | ||
|
|
ceef16081d | ||
|
|
20b7171fba | ||
|
|
b2a5c72bfe | ||
|
|
dd7b58232c | ||
|
|
35a8ea8aef | ||
|
|
8b73234d78 | ||
|
|
a4700185b2 | ||
|
|
cf907f0988 | ||
|
|
036bcfc5a0 | ||
|
|
2249a806c9 | ||
|
|
34eee753b2 | ||
|
|
572737a29e | ||
|
|
017ef988d0 | ||
|
|
8523088ceb | ||
|
|
d4ea1584c0 | ||
|
|
f2d9c0dc8e | ||
|
|
07723e418b | ||
|
|
dda98efcc4 | ||
|
|
1f6802bb62 | ||
|
|
7ca058f83b | ||
|
|
ce215be590 | ||
|
|
ac67c9ee74 | ||
|
|
05542946ad | ||
|
|
61aa8a902a | ||
|
|
aa1f504e02 | ||
|
|
689451161e | ||
|
|
fc8353c75c | ||
|
|
ac0f65c743 | ||
|
|
c794a48462 | ||
|
|
aeff810cad | ||
|
|
cb7da865b6 | ||
|
|
1d76a088c6 | ||
|
|
d751f5e01d | ||
|
|
400e30a278 | ||
|
|
7ed0a59c56 | ||
|
|
b83e965994 | ||
|
|
2a2dcacd58 | ||
|
|
31f2a927a2 | ||
|
|
66c8500e94 | ||
|
|
27867e03cf | ||
|
|
39c4e8df0a | ||
|
|
e605e4006a | ||
|
|
a973fc48e2 | ||
|
|
78cd9e0ebf | ||
|
|
f601d957a6 | ||
|
|
5b0ba08aab | ||
|
|
ba52219a9a | ||
|
|
e7e425fa25 | ||
|
|
fad6805b91 | ||
|
|
fc3ede09bc | ||
|
|
5f73cd4482 | ||
|
|
b93a7d33ff | ||
|
|
373956fcee | ||
|
|
fbc7b2a1dd | ||
|
|
a37847e509 | ||
|
|
1641ca8305 | ||
|
|
cd6f36185c | ||
|
|
17c25e75fa | ||
|
|
5b46214774 | ||
|
|
937ee92a3f | ||
|
|
1c86a1c1fc | ||
|
|
ca99bedb1a | ||
|
|
2d049e9a28 | ||
|
|
ad4ac6c7e0 | ||
|
|
8de02d5bde | ||
|
|
8dcd4f3602 | ||
|
|
e960d5b918 | ||
|
|
ed8de77e10 | ||
|
|
3885fd6aea | ||
|
|
d51255a8ef | ||
|
|
322dd43d10 | ||
|
|
f0f8708a6e | ||
|
|
00e623c066 | ||
|
|
9114a7e8af | ||
|
|
8103e34fff | ||
|
|
256cb0221b | ||
|
|
3684fd7ef6 | ||
|
|
b255b7ffdc | ||
|
|
807ebbd38e | ||
|
|
4e094637c6 | ||
|
|
5e7623b8d5 | ||
|
|
2c10f057b8 | ||
|
|
63fa81fbb5 | ||
|
|
2fd798cccf | ||
|
|
06e8181dee |
@@ -12,6 +12,38 @@ set -uo pipefail
|
||||
[ "${ETV_SKIP_REBASE_CHECK:-}" = "1" ] && exit 0
|
||||
git rev-parse --git-dir >/dev/null 2>&1 || exit 0
|
||||
|
||||
# Tag-only push exemption (ersatztv#719): the release cut tags a commit on main while the local
|
||||
# branch sits 1 commit behind origin/main, so H11 blocked EVERY release -- and its "rebase first"
|
||||
# advice did not even apply, since no branch was being pushed. A tag push cannot revert anyone's
|
||||
# merged work, which is the failure mode H11 exists to prevent, so skip the freshness check when
|
||||
# EVERY ref being pushed is under refs/tags/. (See #719 for the observed flow.)
|
||||
#
|
||||
# Read pushed refs from stdin: git feeds pre-push hooks one line per ref, "<local ref> <local sha>
|
||||
# <remote ref> <remote sha>" (.husky/pre-push forwards the lines it already captured). Ignore blank
|
||||
# lines. VACUOUS-TRUTH GUARD: "all refs are tags" is trivially true when there are zero ref lines
|
||||
# (hook run manually, stdin not forwarded, etc.) -- that would silently disable H11 for every push.
|
||||
# Require at least one parsed ref line before granting the exemption; with zero lines, fall through
|
||||
# to the existing branch-freshness check below (current behavior preserved).
|
||||
#
|
||||
# `[ -t 0 ] ||` so an interactive run does not hang waiting on a terminal: this script had no stdin
|
||||
# reader before #719, and its own docs call "run by hand" a supported case. A TTY yields no ref
|
||||
# lines, which is exactly the zero-line fall-through.
|
||||
_h11_refs_seen=0
|
||||
_h11_all_tags=1
|
||||
[ -t 0 ] || while IFS=' ' read -r _h11_local_ref _h11_local_sha _h11_remote_ref _h11_remote_sha \
|
||||
|| [ -n "${_h11_local_ref:-}" ]; do # `|| [ -n ... ]` also processes a final line with no trailing newline
|
||||
[ -z "${_h11_local_ref:-}" ] && continue
|
||||
_h11_refs_seen=1
|
||||
case "${_h11_remote_ref:-}" in
|
||||
refs/tags/*) ;;
|
||||
*) _h11_all_tags=0 ;;
|
||||
esac
|
||||
_h11_local_ref=''
|
||||
done
|
||||
if [ "$_h11_refs_seen" = "1" ] && [ "$_h11_all_tags" = "1" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Best-effort fetch of the latest main; offline / no network -> don't block.
|
||||
git fetch origin main --quiet 2>/dev/null || exit 0
|
||||
git rev-parse --verify --quiet origin/main >/dev/null 2>&1 || exit 0
|
||||
|
||||
@@ -85,110 +85,58 @@ sha=$(printf '%s' "$prjson" | jq -r '.head.sha // ""' 2>/dev/null || true)
|
||||
body=$(printf '%s' "$prjson" | jq -r '.body // ""' 2>/dev/null || true)
|
||||
|
||||
# --- Docs-only exemption: if every changed file is docs/process, skip the gate. ---
|
||||
# The file list must be enumerated EXHAUSTIVELY or the exemption is unsafe. Gitea caps this
|
||||
# endpoint at 50 rows per page and silently ignores a larger `limit` (verified: PR #619 has 194
|
||||
# changed files and `?limit=100` returns exactly 50), so the previous single-page read could see 50
|
||||
# docs files, miss the code in positions 51+, and exempt a PR that is not remotely docs-only.
|
||||
# Page until a short page proves the end; anything else leaves `files_complete=no`, which withholds
|
||||
# the exemption and falls through to the full gate (ersatztv#622).
|
||||
files=""; files_complete=no; page=1
|
||||
while [ "$page" -le 40 ]; do
|
||||
raw=$(gq "repos/$owner/$repo/pulls/$pr/files?limit=50&page=$page")
|
||||
# A transport/parse failure must not look like a legitimate short final page: `gq` returns empty
|
||||
# on any error, which counts as zero rows and would set files_complete=yes over a PARTIAL list —
|
||||
# failing OPEN into the exemption.
|
||||
#
|
||||
# Checking only the top-level type leaves the same hole one level down: `[{}]` is a valid array
|
||||
# whose rows carry no `filename`, so it yields no paths, looks like a short page, and completes
|
||||
# the enumeration from a partial list. Require every row to carry a non-empty string `filename`
|
||||
# (an empty array is still valid — that is a genuine end-of-pagination). This also rejects arrays
|
||||
# of scalars, which would otherwise make the `.filename` extraction below fail under `set -e`.
|
||||
#
|
||||
# An EMPTY body is rejected EXPLICITLY here rather than left to jq's exit status, because that
|
||||
# status is not portable: `jq -e` over empty input exits 4 on jq >= 1.7 but **0 on jq 1.6**
|
||||
# (verified against both binaries — ersatztv#631). Relying on it made this guard fail OPEN on any
|
||||
# host with the older jq, including the CI runner, which ships jq 1.6. The chain: a transport
|
||||
# failure makes `gq` return empty, the jq guard wrongly passes, `n` is empty so `[ "$n" -lt 50 ]`
|
||||
# errors into false, the loop walks PAST the failed page, the NEXT page legitimately returns `[]`,
|
||||
# and `files_complete=yes` is set over a PARTIAL list — exempting a PR whose unread pages may be
|
||||
# pure code. That is the very defect the paragraph above describes, reintroduced one layer down.
|
||||
if [ -z "${raw//[[:space:]]/}" ]; then
|
||||
files_complete=no; break
|
||||
fi
|
||||
#
|
||||
# CR/LF in a path is REJECTED outright (ersatztv#643 review). `chunk` below flattens paths into
|
||||
# newline-delimited text, so a filename containing a newline splits into TWO lines that are each
|
||||
# matched against the allow-list separately: `"safe.md\ndocs/Program.cs"` yields `safe.md` and
|
||||
# `docs/Program.cs`, both of which pass, while the actual single path ends in `.cs`. Git permits
|
||||
# newlines in filenames, so this is reachable, and it was reproduced against this hook. Failing
|
||||
# closed on control characters is the cheap fix; no decision/docs path ever contains one.
|
||||
#
|
||||
# VALIDATE EVERY FIELD THE EXTRACTION BELOW CONSUMES. `chunk` emits `(.previous_filename //
|
||||
# empty)` for EVERY row regardless of `.status`, so validating that field only on `renamed` rows
|
||||
# left a hole one predicate wide: a row with `status: "modified"` (or Gitea's distinct `copied`)
|
||||
# carrying a newline in `previous_filename` was reproducibly exempted. The rule this encodes:
|
||||
# the validation domain must match the CONSUMPTION domain, not the domain the field is
|
||||
# semantically "supposed to" appear in. The `renamed` => REQUIRED clause is kept on top of the
|
||||
# unconditional if-present check.
|
||||
#
|
||||
# `..` is rejected for the same reason: the allow-list anchors `^docs/`, so
|
||||
# `docs/../ErsatzTV/Program.cs` matches it. Git will not produce such a path, but this guard's
|
||||
# whole job is to fail closed on unexpected 2xx shapes rather than to assume a well-behaved peer.
|
||||
#
|
||||
# `.status` is checked against a CLOSED set, verified against live Gitea 1.25.4 output:
|
||||
# added|deleted|changed|renamed|copied. Without it, the `renamed => previous_filename REQUIRED`
|
||||
# clause could be dodged by any other value — `"Renamed"` with a capital R, or an absent status —
|
||||
# letting a `git mv ErsatzTV/Program.cs -> docs/a.md` drop its source path and read as docs-only.
|
||||
# An unknown status now fails closed rather than silently taking the `else true` branch.
|
||||
#
|
||||
# `modified` is accepted ALONGSIDE `changed` deliberately. Live Gitea 1.25.4 emits `changed`, but
|
||||
# a closed allow-list built from the wrong vocabulary is a worse failure than the hole it closes:
|
||||
# it would gate every genuine docs-only PR, on every version that spells it differently. The
|
||||
# security property here is "reject values we do not recognise", not "enumerate one version
|
||||
# exactly", so the set errs toward accepting plausible synonyms.
|
||||
if ! printf '%s' "$raw" \
|
||||
| jq -e 'def ok: type == "string" and length > 0
|
||||
and (test("[\\r\\n]") | not)
|
||||
and (split("/") | index("..") | not);
|
||||
type == "array" and all(.[];
|
||||
(.filename | ok)
|
||||
and (.previous_filename == null or (.previous_filename | ok))
|
||||
and ((.status // "") as $s | ($s | type) == "string"
|
||||
and (["added","deleted","changed","modified","renamed","copied"] | index($s)) != null)
|
||||
and (if .status == "renamed"
|
||||
then (.previous_filename | type == "string" and length > 0)
|
||||
else true end))' \
|
||||
>/dev/null 2>&1; then
|
||||
files_complete=no; break
|
||||
fi
|
||||
# BOTH sides of a rename: Gitea reports a `git mv` as ONE row whose `filename` is the DESTINATION,
|
||||
# with the source in `previous_filename`. Reading only `filename` would let a PR move code into
|
||||
# docs/ and claim the docs-only exemption. Page size is measured in ROWS, not paths — one renamed
|
||||
# row is one row but two paths.
|
||||
n=$(printf '%s' "$raw" | jq -r 'length')
|
||||
chunk=$(printf '%s' "$raw" | jq -r '.[] | (.filename // empty), (.previous_filename // empty)')
|
||||
[ -n "$chunk" ] && files=$(printf '%s\n%s' "$files" "$chunk")
|
||||
# Terminate ONLY on an explicitly validated EMPTY page — never on a merely SHORT one
|
||||
# (ersatztv#643 review). "Fewer than 50 rows means last page" assumes the server's page size is
|
||||
# the 50 we asked for, but Gitea caps `limit` at the server-wide `MAX_RESPONSE_ITEMS` (default 50,
|
||||
# configurable) and is free to return fewer. A 30-row page followed by a page of code would set
|
||||
# files_complete=yes over a PARTIAL list — the same fail-open, reached without any transport error.
|
||||
# Costs one extra request per enumeration; the `page <= 40` cap still fails closed.
|
||||
if [ "$n" -eq 0 ]; then files_complete=yes; break; fi
|
||||
page=$((page + 1))
|
||||
done
|
||||
files=$(printf '%s\n' "$files" | grep -v '^$' || true)
|
||||
# Bind the enumeration to ONE head (ersatztv#643 review). Paging is several round-trips; a
|
||||
# force-push between them means page 1 came from head A and page 2 from head B, so the assembled
|
||||
# list belongs to no single commit — B's code page can be skipped entirely while B's docs page
|
||||
# reads as a clean short tail. Re-read the head and refuse the exemption if it moved.
|
||||
if [ "$files_complete" = yes ]; then
|
||||
sha_after=$(printf '%s' "$(gq "repos/$owner/$repo/pulls/$pr")" | jq -r '.head.sha // ""' 2>/dev/null || true)
|
||||
if [ -z "$sha_after" ] || [ "$sha_after" != "$sha" ]; then
|
||||
files_complete=no
|
||||
fi
|
||||
# The file list must be enumerated EXHAUSTIVELY, validated row by row, and bound to ONE head, or the
|
||||
# exemption is unsafe. ALL of that now lives in scripts/pr-changed-files.sh — the single shared
|
||||
# implementation, also called by .gitea/workflows/review-verdict.yml (ersatztv#649).
|
||||
#
|
||||
# Why it moved: this logic was written twice. This copy is ADVISORY (a failure produces a human
|
||||
# prompt); the workflow's copy is ENFORCED (it writes the branch-protection-required
|
||||
# `review-verdict/h10` status). Four rounds of ersatztv#643 hardening landed here and never reached
|
||||
# there, leaving the copy with real authority strictly weaker than the copy without — and its safe
|
||||
# behaviour resting on a bash arithmetic error rather than an intentional guard. Two copies of a
|
||||
# security predicate drift; one cannot.
|
||||
#
|
||||
# What is NOT shared, deliberately: the docs-only allow-list below. This one also lets .claude/,
|
||||
# .gitea/ and .husky/ through, which is safe HERE only because a match falls through to a human
|
||||
# prompt rather than auto-granting. The workflow's list is narrower for exactly that reason. Sharing
|
||||
# the enumeration fixes the drift; sharing the classification would erase an intended difference.
|
||||
#
|
||||
# A non-zero exit means "could not tell" and MUST withhold the exemption — never read stdout without
|
||||
# checking the status. An empty `$sha` (unparseable PR JSON) reaches the script as an empty argument
|
||||
# and is rejected there, so that path also fails closed.
|
||||
#
|
||||
# The 5th argument binds the enumeration to a base branch (ersatztv#698 route 1), because
|
||||
# `/pulls/{n}/files` diffs against the PR's LIVE base and retargeting moves that without moving the
|
||||
# head. Be precise about what it buys HERE, which is less than what it buys in the workflow: the
|
||||
# workflow passes the base from a `pull_request_target` event payload, fixed at event time and beyond
|
||||
# a retarget's reach, so it detects a retarget outright. This hook has no such trusted snapshot — it
|
||||
# passes the base it just read from the live PR, so what it asserts is that the base did not move
|
||||
# between that read and the enumeration. Narrower, and still worth having: without it the hook cannot
|
||||
# tell a mid-flight retarget from an honest read at all. An empty/unparseable `.base.ref` reaches the
|
||||
# script as an empty argument and is rejected there, so that path fails closed too.
|
||||
base_ref=$(printf '%s' "$prjson" | jq -r '.base.ref // ""' 2>/dev/null || true)
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
files=""; files_complete=no
|
||||
if files=$("$repo_root/scripts/pr-changed-files.sh" "$owner" "$repo" "$pr" "$sha" "$base_ref" 2>/dev/null); then
|
||||
files_complete=yes
|
||||
fi
|
||||
if [ "$files_complete" = yes ] && [ -n "$files" ] && ! printf '%s\n' "$files" | grep -qvE '^(docs/|\.claude/|\.husky/|\.gitea/|.*\.md$)'; then
|
||||
|
||||
# HOW THIS PREDICATE IS EVALUATED, matching the enforced gate (ersatztv#698,
|
||||
# `ci.grep-q-pipefail-inversion`). `printf … | grep -q` INVERTS under `set -o pipefail`: grep -q exits
|
||||
# at its first match, printf then takes SIGPIPE (141), and a MATCH is reported as a failed pipeline —
|
||||
# so this negated test would grant a spurious docs-only exemption for any PR whose path list exceeds
|
||||
# the pipe buffer. A here-string fixes that but is materialised via temporary storage for large inputs,
|
||||
# so it can fail when temp space is full or unwritable and flip the predicate the same way. Counting
|
||||
# with `grep -c` drains stdin (no SIGPIPE) over an ordinary pipe (no temp file); `grep -c` exits 1 for
|
||||
# a zero count, which is a legitimate answer, so only a status >1 is a real error and is treated as
|
||||
# "cannot tell" -> no exemption.
|
||||
# Advisory here, so the blast radius is a missing prompt rather than a green required check; the
|
||||
# construct is identical on purpose, because the two copies drifting is what ersatztv#649 was about.
|
||||
docs_nonmatching=$(printf '%s\n' "$files" | grep -cvE '^(docs/|\.claude/|\.husky/|\.gitea/|.*\.md$)') || docs_grep_status=$?
|
||||
if [ "${docs_grep_status:-0}" -gt 1 ]; then
|
||||
docs_nonmatching=1 # grep itself failed: cannot tell, so withhold the exemption
|
||||
fi
|
||||
if [ "$files_complete" = yes ] && [ -n "$files" ] && [ "${docs_nonmatching:-1}" -eq 0 ]; then
|
||||
# Docs/process-only PR: the Done-when + review-verdict gate doesn't apply — but this exemption is a
|
||||
# file-TYPE bypass, NOT the a+b+c "provably reviewed & ready" proof, so it does NOT auto-grant. It
|
||||
# passes through to normal permissioning (one prompt). This deliberately keeps a human in the loop for
|
||||
@@ -198,6 +146,76 @@ if [ "$files_complete" = yes ] && [ -n "$files" ] && ! printf '%s\n' "$files" |
|
||||
decide allow "" # passthrough (exit 0 → normal prompt), NOT grant
|
||||
fi
|
||||
|
||||
# --- Base-change detection: a verdict is bound to a head AND to a base (ersatztv#632). ---
|
||||
# `review-verdict/h10` is per-sha, which makes "the head moved under a fixed verdict" impossible by
|
||||
# construction. Retargeting a PR's base is the mirror case and slips through: it changes neither the
|
||||
# head sha nor the status, so a verdict formed while the PR targeted `main` still reads green after
|
||||
# the PR is pointed at a branch with a very different merge-base. The diff moves while the verdict
|
||||
# and the head both hold still.
|
||||
#
|
||||
# DETECTION, NOT PREVENTION, and only on this path. A commit status carries no base, so the
|
||||
# server-side required check cannot see this; a merge driven through the Gitea UI or API is
|
||||
# unaffected. That is the accepted exposure — base changes are rare, manual, and this is a
|
||||
# two-account repo — but it is now recorded in a place that fails LOUD rather than only in a doc.
|
||||
#
|
||||
# GRACEFUL ADOPTION, mirroring (b) and (c): a description with no `(base: …)` field is a verdict
|
||||
# posted before ersatztv#632 and gets NO opinion, rather than denying every in-flight PR the day
|
||||
# this lands. The window closes on its own — verdicts are per-head and short-lived, so every verdict
|
||||
# posted after this carries the field.
|
||||
# "Could not check" is a THIRD outcome, distinct from both "matches" and "no base recorded". Cold
|
||||
# review found the first draft collapsing it into the latter: an unreadable status response yielded
|
||||
# an empty `recorded_base`, which took the graceful-adoption path and skipped validation silently —
|
||||
# after which a later, successful status read could still auto-grant. A transient failure would then
|
||||
# have produced a "merge gate: satisfied" message for a comparison that never happened. Every
|
||||
# unreadable input here therefore falls through to a human (`ask`), never to silence.
|
||||
live_base=$(printf '%s' "$prjson" | jq -r '.base.ref // ""' 2>/dev/null || true)
|
||||
if [ -z "$live_base" ]; then
|
||||
decide ask "H10 merge gate: PR #$pr reports no base branch (.base.ref), so the verdict cannot be checked against the branch it was formed for (ersatztv#632). Confirm the PR still targets the branch it was reviewed against before merging."
|
||||
fi
|
||||
if [ -n "$sha" ]; then
|
||||
# This is the THIRD read of this endpoint in a worst-case hook run (the ordinary-CI branch and the
|
||||
# scheduled-auto-merge branch each do their own). Sharing one snapshot would close a narrow
|
||||
# same-run window where two reads disagree, but the later branches derive different decisions from
|
||||
# a failed read than this one does, so threading a shared response through them is a change to
|
||||
# pre-existing logic rather than to ersatztv#632's. Left deliberately, noted so it is not
|
||||
# rediscovered as an oversight: every `decide` exits immediately, so the reads cannot produce a
|
||||
# single self-contradictory message — only a later decision made on a fresher snapshot.
|
||||
vjson_base=$(gq "repos/$owner/$repo/commits/$sha/status?limit=100")
|
||||
# Same jq-1.6 rule as everywhere else in this file: check emptiness in SHELL first, never via
|
||||
# `jq -e`'s exit status over empty input.
|
||||
# VALIDATE EVERY FIELD THE EXTRACTION CONSUMES, on EVERY row — the same rule the file-enumeration
|
||||
# guard learned the hard way. Checking only that `.statuses` is an array left a hole one level
|
||||
# down: `{"statuses":[1]}` passes a top-level type check, then `.context` on a number errors, and
|
||||
# a `|| true` on the extraction turned that error into an empty `vdesc` — i.e. straight back onto
|
||||
# the graceful-adoption path this block exists to distinguish from. That is the identical
|
||||
# swallow-the-error shape fixed a few lines up, surviving one level deeper.
|
||||
if [ -z "${vjson_base//[[:space:]]/}" ] \
|
||||
|| ! printf '%s' "$vjson_base" \
|
||||
| jq -e '.statuses | type == "array"
|
||||
and all(.[]; type == "object"
|
||||
and (.context | type == "string")
|
||||
and (.description == null or (.description | type == "string")))' \
|
||||
>/dev/null 2>&1; then
|
||||
decide ask "H10 merge gate: could not read the commit statuses for PR #$pr head ${sha:0:7}, so the verdict could not be checked against the PR's base branch (ersatztv#632). Confirm the review covered the branch this PR currently targets ('$live_base') before merging."
|
||||
fi
|
||||
# No `|| true` here. The validation above makes an error unreachable, but a swallowed error would
|
||||
# be indistinguishable from "no base recorded" — the exact confusion this block removes — so the
|
||||
# failure is handled explicitly rather than left to a fallback that reads as a benign result.
|
||||
if ! vdesc=$(printf '%s' "$vjson_base" \
|
||||
| jq -r '[.statuses[] | select(.context == "review-verdict/h10")] | first | .description // ""' \
|
||||
2>/dev/null); then
|
||||
decide ask "H10 merge gate: the commit statuses for PR #$pr head ${sha:0:7} could not be parsed to find the review verdict, so it could not be checked against the PR's base branch (ersatztv#632). Confirm the review covered the branch this PR currently targets ('$live_base') before merging."
|
||||
fi
|
||||
# The field is written by scripts/post-review-verdict.sh as a trailing `(base: <ref>)`. Its
|
||||
# ABSENCE is the one benign case: a verdict posted before ersatztv#632 could not have carried it,
|
||||
# and denying those would block every in-flight PR the day this lands. The window closes on its
|
||||
# own, since verdicts are per-head and short-lived.
|
||||
recorded_base=$(printf '%s' "$vdesc" | sed -n 's/.*(base: \(.*\))$/\1/p')
|
||||
if [ -n "$recorded_base" ] && [ "$recorded_base" != "$live_base" ]; then
|
||||
decide deny "H10 merge gate: BLOCKED — the review verdict on head ${sha:0:7} was formed while PR #$pr targeted '$recorded_base', but it now targets '$live_base'. Retargeting a base does not move the head sha, so the per-sha verdict status still reads green even though the effective diff has changed (ersatztv#632). Re-review against the new base and run: scripts/post-review-verdict.sh $pr MERGEABLE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Linked issue: Gitea auto-close keywords in the PR body. ---
|
||||
issues=$(printf '%s' "$body" | grep -ioE '(close[sd]?|fix(e[sd])?|resolve[sd]?) +#[0-9]+' | grep -oE '[0-9]+' | sort -u || true)
|
||||
[ -n "$issues" ] || decide ask "H6 merge gate: PR #$pr has no linked issue (no 'fixes #N' / 'closes #N' in its body), so there is no Done-when checklist to derive consent from. Confirm the work is complete + reviewed, then approve."
|
||||
|
||||
@@ -331,12 +331,15 @@ docker start ersatztv
|
||||
|
||||
## FFmpeg & Hardware
|
||||
|
||||
- **VAAPI on Intel (iHD)** hardware acceleration — ErsatzTV runs on **jazz** (i7-10700K, Intel iGPU) since #633. The single `FFmpegProfile` row (`Id = 1`, referenced by all 43 channels) has `HardwareAcceleration = 3` (Vaapi), `VaapiDevice = /dev/dri/renderD128`, `VaapiDriver = 0` (auto → iHD), `VaapiDisplay = drm`.
|
||||
- **Do NOT set QSV (1) here, despite the Intel hardware.** It was tried on 2026-07-20 and **regressed**: QSV's *decoder* is far stricter than VAAPI about malformed NAL units and failed **3 of 6 channel cold-starts** (`Error splitting the input into NAL units` → `dec:h264_qsv Error while opening decoder: Invalid data found`). ErsatzTV has a **single** `HardwareAcceleration` column governing *both* decode and encode, so it cannot express Jellyfin's working combination of VAAPI-decode + QSV-encode. Tracked upstream: timothy/ersatztv#498. Jellyfin **does** use QSV successfully, because it splits the two.
|
||||
- **QSV encode + VA-API decode on Intel (iHD)** — ErsatzTV runs on **jazz** (i7-10700K, Intel iGPU) since #633. The single `FFmpegProfile` row (`Id = 1`, referenced by all 43 channels) has `HardwareAcceleration = 1` (**Qsv**), `QsvPreferNativeDecoder = 1` (ON), `QsvExtraHardwareFrames = 64`, `VaapiDevice = /dev/dri/renderD128`. Verified live 2026-07-26. The profile is still *named* "1080p VAAPI h264 aac" — cosmetic, ignore the name.
|
||||
- **The old "do NOT set QSV" rule is RETIRED — #498 fixed the blocker it was based on.** The 2026-07-20 regression was real (QSV's *decoder* is far stricter than VAAPI about malformed NAL units and failed 3 of 6 cold-starts: `Error splitting the input into NAL units`), and the stated cause was that one `HardwareAcceleration` column governed both decode and encode. **#498 added `QsvPreferNativeDecoder` (default ON, Linux-only)**, which splits them exactly like Jellyfin: decode with the tolerant VA-API decoder, encode with QSV. That is what prod runs now. Do not "fix" prod back to `3` (Vaapi) on the strength of the old note.
|
||||
- **Two QSV traps already paid for, both fixed in code — don't re-derive them:**
|
||||
- `QsvExtraHardwareFrames` must never be `0`: the software→QSV `hwupload` bridge has no headroom and the transcode writes **zero segments** on any unthrottled read (#523/#529). Code now floors it at 64 (`ffmpeg.qsv-extra-hw-frames-floor`).
|
||||
- **HDR tonemapping never uses `vpp_qsv=tonemap`** — on this Gen9.5 iGPU that filter is a *silent no-op* (byte-identical output, exit 0, no warning), so it looked like GPU tonemapping while doing nothing. ErsatzTV now tonemaps via VA-API→OpenCL (#505, `ffmpeg.qsv-hdr-tonemap-opencl`). Same trap applies to Jellyfin's `EnableVppTonemapping` on this host — keep it off.
|
||||
- Fallback if VAAPI also misbehaves (see #631, VAAPI `hwupload -22` on 10-bit): `HardwareAcceleration = 0` (software). jazz has 16 threads at load ~2, so it is affordable and maximally tolerant of imperfect sources.
|
||||
- Resolution: 1920x1080, H264, AAC stereo
|
||||
- Device: `/dev/dri` passed through (`renderD128`)
|
||||
- HardwareAccelerationKind: 0=None, 1=Qsv, 2=Nvenc, 3=Vaapi, 4=VideoToolbox, 5=Amf — **use 3 (Vaapi)** on jazz (not Qsv — see above)
|
||||
- HardwareAccelerationKind: 0=None, 1=Qsv, 2=Nvenc, 3=Vaapi, 4=VideoToolbox, 5=Amf — **jazz uses 1 (Qsv)** with `QsvPreferNativeDecoder` ON (see above)
|
||||
- jazz's iGPU is shared with Jellyfin only (Frigate stayed on bumblebee); render GID is 992 on both hosts, so `group_add: '992'` carried over unchanged
|
||||
|
||||
## Jellyfin Integration
|
||||
@@ -360,7 +363,7 @@ docker start ersatztv
|
||||
```
|
||||
- **Dispatcharr caches ErsatzTV's XMLTV.** Repointing its DB rows is not enough — it keeps serving a stale EPG full of dead `ersatztv:8409` artwork URLs (breaks Kodi artwork). Force a refresh (EPG source 9):
|
||||
```bash
|
||||
ssh timothy@192.168.1.99 'docker exec dispatcharr python manage.py shell -c \
|
||||
ssh timothy@192.168.1.29 'docker exec dispatcharr python manage.py shell -c \
|
||||
"from apps.epg.tasks import refresh_epg_data; refresh_epg_data(9)"'
|
||||
```
|
||||
- **`/api/health` returns 401** (needs an API key). The Telegraf probe has no `response_string_match`, so ErsatzTV reads as **unhealthy in Grafana** — a false alarm, and **pre-existing**, not caused by the move. The container healthcheck uses the unauthenticated internal `/health` and is unaffected.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"isRoot": true,
|
||||
"tools": {
|
||||
"jetbrains.resharper.globaltools": {
|
||||
"version": "2025.3.4.1",
|
||||
"version": "2025.3.5",
|
||||
"commands": [
|
||||
"jb"
|
||||
],
|
||||
|
||||
@@ -122,14 +122,23 @@ jobs:
|
||||
# ersatztv#416: is this a docs-only change? If so, every heavy step below is skipped and this
|
||||
# REQUIRED job reports success in seconds. It still RUNS (never `if:`-skipped) so the required
|
||||
# context keeps reporting — see the workflow header and docs/ci-cd.md -> "Docs-only skip".
|
||||
# EVERY consequential `run:` step in this job marks itself as its FIRST act (ersatztv#756),
|
||||
# and the trailing `Assert every expected step executed` guard fails the job when one is
|
||||
# missing. This is a REQUIRED context on `main`, and a step the runner drops takes the job
|
||||
# GREEN having done no work — see scripts/ci-step-ran.sh for why that is fail-OPEN here while
|
||||
# the same drop in review-verdict.yml is fail-CLOSED.
|
||||
- name: Detect docs-only changes
|
||||
id: detect
|
||||
run: scripts/ci-detect-docs-only.sh
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark detect
|
||||
scripts/ci-detect-docs-only.sh
|
||||
- name: Detect already-validated tree (#420)
|
||||
id: revalidate
|
||||
env:
|
||||
ETV_STATUS_AUTH: ${{ secrets.REGISTRY_USER }}:${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: scripts/ci-detect-already-validated.sh
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark revalidate
|
||||
scripts/ci-detect-already-validated.sh
|
||||
|
||||
- name: Cache NuGet packages
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
@@ -141,7 +150,9 @@ jobs:
|
||||
|
||||
- name: Restore
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: dotnet restore
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark restore
|
||||
dotnet restore
|
||||
|
||||
# Replaces setup-node's built-in `cache: npm`. The toolchain image supplies node/npm, but
|
||||
# the SPA's package downloads are project deps, so they stay cached per lockfile.
|
||||
@@ -156,36 +167,50 @@ jobs:
|
||||
- name: Install SPA dependencies
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm ci
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark npm-ci
|
||||
npm ci
|
||||
|
||||
- name: Check generated SPA API client
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm run check:api
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark check-api
|
||||
npm run check:api
|
||||
|
||||
- name: Lint SPA
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm run lint
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark lint
|
||||
npm run lint
|
||||
|
||||
- name: Typecheck SPA
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm run typecheck
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark typecheck
|
||||
npm run typecheck
|
||||
|
||||
- name: Test SPA
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm test -- --run
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark web-test
|
||||
npm test -- --run
|
||||
|
||||
- name: Build SPA
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm run build
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark web-build
|
||||
npm run build
|
||||
|
||||
- name: Strip Scanner project ref (matches Docker build)
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: sed -i '/Scanner/d' ErsatzTV/ErsatzTV.csproj
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark strip-scanner
|
||||
sed -i '/Scanner/d' ErsatzTV/ErsatzTV.csproj
|
||||
|
||||
# Start the true peak-anon sampler just before the memory-heavy dotnet Build/Test/Coverage so
|
||||
# its high-water mark spans them (SPA build/test above are comparatively light). Paired with the
|
||||
@@ -199,13 +224,16 @@ jobs:
|
||||
|
||||
- name: Build
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: dotnet build --configuration Release --no-restore
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark build
|
||||
dotnet build --configuration Release --no-restore
|
||||
|
||||
- name: Test
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: >-
|
||||
dotnet test --configuration Release --no-build --blame-hang-timeout "2m" --verbosity normal
|
||||
--collect:"XPlat Code Coverage" --settings coverlet.runsettings --results-directory ./coverage
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark dotnet-test
|
||||
dotnet test --configuration Release --no-build --blame-hang-timeout "2m" --verbosity normal \
|
||||
--collect:"XPlat Code Coverage" --settings coverlet.runsettings --results-directory ./coverage
|
||||
|
||||
# Coverage reporting (ersatztv#15 scope item 4): coverlet.collector emits a Cobertura report
|
||||
# per test project (via --collect above); ReportGenerator merges them into a human-readable
|
||||
@@ -258,6 +286,43 @@ jobs:
|
||||
continue-on-error: true
|
||||
run: scripts/ci-peak-anon.sh report
|
||||
|
||||
# THE DROPPED-STEP GUARD (ersatztv#756). Every `run:` step above records that it began; this
|
||||
# asserts the whole expected SET was recorded. A step the runner declines to interpolate is
|
||||
# DROPPED and still concludes `success` (ersatztv#751), so without this a REQUIRED context
|
||||
# reports green having done no work — fail-OPEN, and strictly worse than the fail-CLOSED
|
||||
# version of the same bug that #751 fixed in review-verdict.yml.
|
||||
#
|
||||
# NO `if:` HERE, WHICH IS A DELIBERATE DEPARTURE FROM THE #751 GUARD and the one decision in
|
||||
# this block that is easy to "fix" wrongly. That guard uses `if: always()` because its job has
|
||||
# exactly one real step, so there is no ordinary red for it to talk over. Here there are
|
||||
# twelve, and a genuine failure in an early one (a lint error, a failing test) SKIPS every
|
||||
# later step — an `always()` guard would then announce "these steps never executed: typecheck
|
||||
# web-test build dotnet-test" on top of every normal red build. That is not a dropped step, it
|
||||
# is the runner doing what it is told, and a guard that cries wolf on every red build is a
|
||||
# guard that gets deleted.
|
||||
#
|
||||
# The default `if:` is `success()`, which is exactly the condition wanted, and the invariant it
|
||||
# rests on is worth stating because it is what makes the omission safe rather than lucky: this
|
||||
# step is skipped ONLY when an earlier step failed, and an earlier step failing already fails
|
||||
# the job. So `guard skipped => job red`, and the only path to a green job runs the guard. A
|
||||
# dropped step is invisible precisely because it concludes `success`, which keeps the job green
|
||||
# and therefore reaches here.
|
||||
#
|
||||
# ITS OWN BODY CANNOT BE DROPPED BY THE MECHANISM IT GUARDS AGAINST: it is a single command
|
||||
# with no expression delimiter anywhere in the scalar, so the runner has nothing to rewrite.
|
||||
# The two gate values come in through `env:`, which is interpolated PER VALUE — a bad payload
|
||||
# there cannot take the body with it (`ci.workflow-run-body-no-expressions`), and both paths
|
||||
# are held to naming a real context by
|
||||
# test_every_workflow_expression_names_a_REAL_context_or_function.
|
||||
- name: Assert every expected step executed (ersatztv#756)
|
||||
env:
|
||||
ETV_DOCS_ONLY: ${{ steps.detect.outputs.docs_only }}
|
||||
ETV_REVALIDATE_SKIP: ${{ steps.revalidate.outputs.skip }}
|
||||
run: >-
|
||||
scripts/ci-step-ran.sh assert
|
||||
--always detect revalidate
|
||||
--gated restore npm-ci check-api lint typecheck web-test web-build strip-scanner build dotnet-test
|
||||
|
||||
migrations:
|
||||
name: EF migration integrity (SQLite + MySql)
|
||||
runs-on: ubuntu-latest
|
||||
@@ -328,14 +393,21 @@ jobs:
|
||||
|
||||
# ersatztv#416: docs-only? Skip the build + migration replay; the job still reports success in
|
||||
# seconds. REQUIRED context, so it always RUNS (never `if:`-skipped). See the workflow header.
|
||||
# Same per-step marker contract as the `test` job above (ersatztv#756) — this is the other
|
||||
# REQUIRED context, so a dropped migration-replay step would report EF integrity green having
|
||||
# replayed nothing.
|
||||
- name: Detect docs-only changes
|
||||
id: detect
|
||||
run: scripts/ci-detect-docs-only.sh
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark detect
|
||||
scripts/ci-detect-docs-only.sh
|
||||
- name: Detect already-validated tree (#420)
|
||||
id: revalidate
|
||||
env:
|
||||
ETV_STATUS_AUTH: ${{ secrets.REGISTRY_USER }}:${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: scripts/ci-detect-already-validated.sh
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark revalidate
|
||||
scripts/ci-detect-already-validated.sh
|
||||
|
||||
- name: Cache NuGet packages
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
@@ -347,11 +419,15 @@ jobs:
|
||||
|
||||
- name: Restore
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: dotnet restore
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark restore
|
||||
dotnet restore
|
||||
|
||||
- name: Build
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: dotnet build --configuration Release --no-restore
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark build
|
||||
dotnet build --configuration Release --no-restore
|
||||
|
||||
# dotnet-ef is baked into the CI toolchain image (docker/ci/Dockerfile) and already on PATH
|
||||
# — no per-run `dotnet tool install`. Bump its version there (ersatztv#390).
|
||||
@@ -361,6 +437,7 @@ jobs:
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark sqlite
|
||||
echo "::group::SQLite model drift (has-pending-model-changes)"
|
||||
dotnet ef migrations has-pending-model-changes --no-build --configuration Release \
|
||||
--context TvContext --startup-project ErsatzTV --project ErsatzTV.Infrastructure.Sqlite -- --provider Sqlite
|
||||
@@ -384,6 +461,7 @@ jobs:
|
||||
MySql__ConnectionString: "Server=mysql;Port=3306;Database=ersatztv_migrations;Uid=root;Pwd=ersatztv;DefaultCommandTimeout=300;"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark mysql
|
||||
echo "::group::MySql model drift (has-pending-model-changes)"
|
||||
dotnet ef migrations has-pending-model-changes --no-build --configuration Release \
|
||||
--context TvContext --startup-project ErsatzTV --project ErsatzTV.Infrastructure.MySql -- --provider MySql
|
||||
@@ -418,6 +496,42 @@ jobs:
|
||||
# how the original defects escaped. The fixture itself is retained and is opt-in via
|
||||
# ETV_TEST_MYSQL_CONNECTION (skipped, visibly, without it). Re-arming it here is tracked by #627.
|
||||
|
||||
# THE DROPPED-STEP GUARD (ersatztv#756). Every `run:` step above records that it began; this
|
||||
# asserts the whole expected SET was recorded. A step the runner declines to interpolate is
|
||||
# DROPPED and still concludes `success` (ersatztv#751), so without this a REQUIRED context
|
||||
# reports green having done no work — fail-OPEN, and strictly worse than the fail-CLOSED
|
||||
# version of the same bug that #751 fixed in review-verdict.yml.
|
||||
#
|
||||
# NO `if:` HERE, WHICH IS A DELIBERATE DEPARTURE FROM THE #751 GUARD and the one decision in
|
||||
# this block that is easy to "fix" wrongly. That guard uses `if: always()` because its job has
|
||||
# exactly one real step, so there is no ordinary red for it to talk over. Here a genuine
|
||||
# failure in an early step (a failing `dotnet build`, a MySql replay error) SKIPS every later
|
||||
# step — an `always()` guard would then announce "these steps never executed: sqlite mysql" on
|
||||
# top of every normal red build. That is not a dropped step, it is the runner doing what it is
|
||||
# told, and a guard that cries wolf on every red build is a guard that gets deleted.
|
||||
#
|
||||
# The default `if:` is `success()`, which is exactly the condition wanted, and the invariant it
|
||||
# rests on is worth stating because it is what makes the omission safe rather than lucky: this
|
||||
# step is skipped ONLY when an earlier step failed, and an earlier step failing already fails
|
||||
# the job. So `guard skipped => job red`, and the only path to a green job runs the guard. A
|
||||
# dropped step is invisible precisely because it concludes `success`, which keeps the job green
|
||||
# and therefore reaches here.
|
||||
#
|
||||
# ITS OWN BODY CANNOT BE DROPPED BY THE MECHANISM IT GUARDS AGAINST: it is a single command
|
||||
# with no expression delimiter anywhere in the scalar, so the runner has nothing to rewrite.
|
||||
# The two gate values come in through `env:`, which is interpolated PER VALUE — a bad payload
|
||||
# there cannot take the body with it (`ci.workflow-run-body-no-expressions`), and both paths
|
||||
# are held to naming a real context by
|
||||
# test_every_workflow_expression_names_a_REAL_context_or_function.
|
||||
- name: Assert every expected step executed (ersatztv#756)
|
||||
env:
|
||||
ETV_DOCS_ONLY: ${{ steps.detect.outputs.docs_only }}
|
||||
ETV_REVALIDATE_SKIP: ${{ steps.revalidate.outputs.skip }}
|
||||
run: >-
|
||||
scripts/ci-step-ran.sh assert
|
||||
--always detect revalidate
|
||||
--gated restore build sqlite mysql
|
||||
|
||||
functional-e2e:
|
||||
name: Functional E2E (curl + UI contracts)
|
||||
runs-on: ubuntu-latest
|
||||
@@ -529,6 +643,61 @@ jobs:
|
||||
# server. Its exit status is Playwright's.
|
||||
scripts/e2e-ui.sh
|
||||
|
||||
# THE DELIMITER BAN, RE-CHECKED ON THE RELEASE PATH ITSELF (ersatztv#767).
|
||||
#
|
||||
# The ban that keeps `build`'s `Smoke + IPTV E2E` from being silently dropped was enforced only by
|
||||
# `test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body` in the
|
||||
# `script-tests` job of pr-checks.yml — `on: pull_request`, and NOT a required context. So the ban
|
||||
# was REVIEW-TIME only: nothing re-checked it on a `v*` tag push, which is precisely when the
|
||||
# candidate image is published and `DeployStack jazz-media` promotes it.
|
||||
#
|
||||
# WHY A JOB AND NOT A STEP INSIDE `build`. A step cannot protect the thing it shares a job with:
|
||||
# `build` is what publishes, so a guard step there fails OPEN if the runner drops it, and "my body
|
||||
# has no opener so I cannot be dropped" is circular when the only thing enforcing that property is
|
||||
# the same PR-only test being backstopped. As a `needs:` of `build`, a red here means `build` never
|
||||
# runs at all — the image is not built, let alone pushed. Fail-closed by dependency, not by
|
||||
# assertion.
|
||||
#
|
||||
# WHY IT RUNS THE REAL PYTEST rather than a bespoke scanner. The first cut of #767 hand-parsed the
|
||||
# workflow YAML in stdlib Python, to avoid provisioning PyYAML on `build`'s bare runner. Two
|
||||
# independent reviews found ~10 false NEGATIVES in that parser within one round (flow mappings
|
||||
# `{run: …}`, a quoted `"run":` key, aliases, multiline quoted scalars) — i.e. it was strictly
|
||||
# WEAKER than the check it was meant to backstop, in the one direction that matters for a security
|
||||
# gate. Running the existing PyYAML-based test needs no second implementation of "what is a `run:`
|
||||
# body" and therefore has no drift surface. `small` is git-only, so Python is provisioned here the
|
||||
# same way `script-tests` does it.
|
||||
#
|
||||
# This job's OWN steps carry #756 markers and a trailing assert, so a drop inside THIS job is
|
||||
# caught too. That terminates the regress at the same axiom the sibling guards already rest on —
|
||||
# to fail open you must now drop the pytest step AND the assert step, rather than either one.
|
||||
scan:
|
||||
name: Delimiter ban (release path)
|
||||
runs-on: small
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- name: Install test dependencies
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark deps
|
||||
python3 -m pip install --disable-pip-version-check --quiet pytest pyyaml
|
||||
# The ban test plus the structural tests that hold this job's own shape. NOT the whole
|
||||
# scripts/tests suite: that is `script-tests`'s job, it needs jq/git preflights, and an
|
||||
# unrelated pytest regression must not be able to block a release.
|
||||
- name: Run the delimiter-ban tests
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark ban
|
||||
PYTHONPATH=. python3 -m pytest scripts/tests/test_ci_dropped_step_guard.py scripts/tests/test_ci_release_path_scan_job.py -q
|
||||
# No `if:` — see the sibling guards in `test`/`migrations` for why the default `success()` is
|
||||
# the wanted condition. Both keys are `--always`: every step in this job is unconditional.
|
||||
- name: Assert every expected step executed (ersatztv#756)
|
||||
run: >-
|
||||
scripts/ci-step-ran.sh assert
|
||||
--always deps ban
|
||||
|
||||
build:
|
||||
name: Build & push image (amd64)
|
||||
# Moved back off `small` (server-management#639). This is the one HEAVY job that
|
||||
@@ -545,7 +714,10 @@ jobs:
|
||||
# was queueing behind has drained. Real builds (main/tags) get the full
|
||||
# ubuntu-latest allotment: 4 CPUs / 10g on ci-runner (.127).
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, migrations]
|
||||
# `scan` (ersatztv#767) re-checks the delimiter ban on the release path. As a `needs:` its red
|
||||
# SKIPS this job outright, so a delimiter in `Smoke + IPTV E2E` can no longer reach the point
|
||||
# where an image is published and never booted.
|
||||
needs: [test, migrations, scan]
|
||||
if: github.event_name != 'pull_request'
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -616,11 +788,33 @@ jobs:
|
||||
cache-from: type=registry,ref=192.168.1.95:3000/timothy/ersatztv:buildcache
|
||||
cache-to: type=registry,ref=192.168.1.95:3000/timothy/ersatztv:buildcache,mode=max,ignore-error=true
|
||||
|
||||
# THE TWO VALUES COME IN THROUGH `env:`, NOT INLINE (ersatztv#756). This step runs AFTER
|
||||
# `Build and push`, so on a `v*` tag the image is already in the registry as the release
|
||||
# candidate — and it is this smoke run that decides whether the candidate was ever booted at
|
||||
# all. A stray expression delimiter anywhere in this body (a comment is not inert — #751) would
|
||||
# DROP the step and conclude the job `success`: a candidate published, never smoke-tested, and
|
||||
# `DeployStack jazz-media` promotes exactly that image. `env:` is interpolated PER VALUE, so a
|
||||
# bad payload there fails that value instead of taking the whole body with it, and with the
|
||||
# body delimiter-free the class is unreachable here — held by
|
||||
# test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body.
|
||||
#
|
||||
# The ban IS re-checked on the release path now (ersatztv#767): the `scan` job above runs the
|
||||
# PyYAML-based ban test and is a `needs:` of this job, so a delimiter here means `build` never
|
||||
# runs and no image is published. Do not re-add the note that once stood here saying the ban is
|
||||
# "review-time only, tracked as #767" — that was true before the `scan` job existed.
|
||||
#
|
||||
# This step still carries no per-step markers, and that is a genuine (smaller) residual rather
|
||||
# than a dismissal: markers would additionally catch a drop caused by something OTHER than a
|
||||
# delimiter. Adding them needs a bucket modelling this step's publish-ref `if:`, which the
|
||||
# guard's always/gated buckets do not express. The delimiter class itself is covered.
|
||||
- name: Smoke + IPTV E2E (assert key endpoints)
|
||||
if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && steps.detect.outputs.docs_only != 'true' }}
|
||||
env:
|
||||
SMOKE_SHORT_SHA: ${{ steps.meta.outputs.short }}
|
||||
SMOKE_RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
IMG="${IMAGE}:${{ steps.meta.outputs.short }}"
|
||||
NAME="etv-smoke-${{ github.run_id }}"
|
||||
IMG="${IMAGE}:${SMOKE_SHORT_SHA}"
|
||||
NAME="etv-smoke-${SMOKE_RUN_ID}"
|
||||
trap 'docker rm -f "$NAME" >/dev/null 2>&1 || true' EXIT
|
||||
echo "Pulling ${IMG}"
|
||||
docker pull "$IMG"
|
||||
|
||||
@@ -239,14 +239,24 @@ jobs:
|
||||
# as ~20 opaque assertion errors — this turns that into one actionable line.
|
||||
- name: Preflight external tools
|
||||
run: |
|
||||
missing=()
|
||||
for t in jq git; do command -v "$t" >/dev/null 2>&1 || missing+=("$t"); done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
echo "::error::script-tests needs these on PATH but they are absent: ${missing[*]}." \
|
||||
"The suite execs real shell scripts that use them. Bake them into the runner" \
|
||||
"image rather than apt-get installing here (see ersatztv#390)."
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "::error::script-tests needs git on PATH but it is absent. The suite execs real" \
|
||||
"shell scripts that use it. Bake it into the runner image rather than apt-get" \
|
||||
"installing here (see ersatztv#390)."
|
||||
exit 1
|
||||
fi
|
||||
echo "Preflight OK: $(jq --version), $(git --version)"
|
||||
echo "Preflight OK: $(git --version)"
|
||||
# jq gets its OWN step because its VERSION, not merely its presence, is load-bearing
|
||||
# (ersatztv#648). `--expect` makes this a TRIPWIRE: scripts/tests exercises the jq 1.6 code path
|
||||
# only because this runner ships 1.6, so an upgrade would silently delete that coverage — and
|
||||
# the three divergences found in ersatztv#643/#647 all lived exactly there. Going red forces an
|
||||
# explicit human decision instead of letting the coverage evaporate.
|
||||
#
|
||||
# The pin lives HERE and deliberately NOT in review-verdict.yml: that workflow writes the
|
||||
# branch-protection-required `review-verdict/h10` status, so pinning a version there would turn
|
||||
# any jq bump on the runner into a repo-wide merge deadlock. It gets the floor-only mode.
|
||||
# See docs/ci-cd.md -> "The jq contract".
|
||||
- name: Preflight jq version
|
||||
run: ./scripts/jq-preflight.sh --expect 1.6
|
||||
- name: Run scripts/tests
|
||||
run: PYTHONPATH=. python3 -m pytest scripts/tests -q
|
||||
|
||||
+1001
-83
File diff suppressed because it is too large
Load Diff
@@ -80,3 +80,9 @@ web/playwright-report/
|
||||
|
||||
# Per-session worktree-ownership marker (H7, ersatztv#303) — local, never committed
|
||||
.claude-worktree-owner
|
||||
|
||||
# Codex CLI project scaffolding — a machine-local mirror of the .claude hooks, generated by
|
||||
# `codex exec`. Deliberately NOT tracked even though `.claude/` is: its config.toml embeds a
|
||||
# plaintext Gitea credential and absolute /Users paths, so it is neither portable nor safe to
|
||||
# commit. See ersatztv#711 for the related merge-gate gap.
|
||||
.codex/
|
||||
|
||||
+3
-2
@@ -12,8 +12,9 @@ unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE
|
||||
|
||||
# H11 (ersatztv#311): refuse to push a branch that is BEHIND origin/main — rebase, don't merge
|
||||
# main in (a merge drags in files you never touched, e.g. legacy-BOM .cs, and trips the format
|
||||
# hook on code that isn't yours). Fail-open; escape with ETV_SKIP_REBASE_CHECK=1.
|
||||
./.claude/hooks/prepush-rebase-check.sh || exit 1
|
||||
# hook on code that isn't yours). Fail-open; escape with ETV_SKIP_REBASE_CHECK=1. Exempts a
|
||||
# tag-only push (ersatztv#719) — forward the ref lines captured above so it can tell.
|
||||
printf '%s\n' "$_prepush_refs" | ./.claude/hooks/prepush-rebase-check.sh || exit 1
|
||||
|
||||
# H13 (ersatztv#416 session): refuse to push when a file in the pushed diff still has uncommitted
|
||||
# working-tree/index changes — the pushed commit wouldn't match what you built/reviewed (the #416
|
||||
|
||||
@@ -52,20 +52,44 @@ docker build -f docker/Dockerfile -t ersatztv:dev .
|
||||
- Test with **NUnit** + Shouldly + NSubstitute (the existing `*.Tests` projects); xUnit is **not** used here
|
||||
- **Dependencies use Central Package Management**: versions live in the repo-root `Directory.Packages.props`; csproj reference packages by name only. Add/upgrade by editing the central `<PackageVersion>` — never put `Version=` back on a `<PackageReference>` (trips `NU1008`). See `docs/ci-cd.md` → Dependency management.
|
||||
- **DB migrations target BOTH providers**: a `TvContext` model change needs a migration in `ErsatzTV.Infrastructure.Sqlite` **and** `ErsatzTV.Infrastructure.MySql` — run `scripts/add-migration.sh <Name>` (does both). CI's `migrations` job enforces model-drift + apply-to-fresh-DB per provider. See `docs/ci-cd.md` → Migration integrity.
|
||||
- **Renovate** is live (`.gitea/workflows/renovate.yml`, weekly + `workflow_dispatch`): opens dependency-update + OSV vuln-fix PRs and a Dependency Dashboard issue; patch bumps to test/dev-only packages auto-merge once `Build & test` passes (their `review-verdict/h10` required check is auto-passed as a bot PR — unless they touch `.claude/`/`.gitea/`/`.husky/`/`scripts/`/`docker/ci/`, which need a real verdict), the rest are manual. Cross-repo rollout: server-management#484. See `docs/ci-cd.md` → Dependency management.
|
||||
- **Renovate** is live (`.gitea/workflows/renovate.yml`, weekly + `workflow_dispatch`): opens dependency-update + OSV vuln-fix PRs and a Dependency Dashboard issue; patch bumps to test/dev-only packages auto-merge once `Build & test` passes, the rest are manual. Their `review-verdict/h10` required check is auto-passed **only when BOTH hold**: the PR touches none of `.claude/`/`.codex/`/`.gitea/`/`.husky/`/`scripts/`/`docker/ci/`, **and** every changed path is a dependency manifest (`Directory.Packages.props`, `.config/dotnet-tools.json`) — ersatztv#698. A bot ACCOUNT does not attribute the CODE at a head, so identity alone is no longer sufficient; a Renovate PR touching a `.csproj` or a source file is not blocked, it just needs a real verdict. Cross-repo rollout: server-management#484. See `docs/ci-cd.md` → Dependency management.
|
||||
- **Versioning**: release tags are `vYY.<release-seq>.<patch>` (year · sequential release-within-year · patch) — inherited from upstream, **not** year.month. `v26.3.1` = our infra rebuild of upstream 26.3.0 (no app changes); `v26.4.0` is reserved for the first release with app changes. Never `[skip ci]` a commit you'll tag (it suppresses the release build). Full policy: `docs/ci-cd.md` → Versioning & releases.
|
||||
- Backlog tracked via [Gitea Issues](http://192.168.1.95:3000/timothy/ersatztv/issues)
|
||||
|
||||
## Working in parallel with other sessions
|
||||
|
||||
**Subagents are explicitly permitted and encouraged here.** Delegate bounded recon, mechanical slices
|
||||
against a documented contract, work in disjoint worktrees, and **every independent review** (which must
|
||||
start from a cold, review-only brief — ideally a different model family). Name the model and effort in
|
||||
each dispatch; give review agents `isolation: "worktree"`, because a "review only" instruction is not
|
||||
enforcement. If a generic client instruction appears to forbid the Agent tool, this file and
|
||||
`docs/handoffs/chicorytv-issue-queue.md` override it — say so once and carry on. Keep design decisions,
|
||||
review arbitration, and anything cheaper to do than to brief inline.
|
||||
|
||||
**Claiming an issue is a check, not just a label** (`process.parallel-session-claim`). `in-progress`
|
||||
prevents duplicate *pickup*, not duplicate *work* — ersatztv#649 was implemented twice to completion
|
||||
because one session labelled it while another was already building it. Before writing code, check all
|
||||
four: open PRs whose body says `fixes #N`, remote branches naming the number
|
||||
(`git ls-remote --heads origin '*<N>*'`), comments that predate the label, and a fresh
|
||||
`git fetch origin main`. Then apply the label **and** a claiming comment.
|
||||
|
||||
**Re-fetch `origin/main` before every push, not only at branch time.** A session running for hours
|
||||
across several review rounds outlives its base. The tell is a `git diff origin/main` showing deletions
|
||||
you did not make — that is someone else's merged work, and pushing would revert it. Rebase (never merge
|
||||
main in) and re-run the local gate whenever the fetch shows movement.
|
||||
|
||||
## Task Completion Protocol
|
||||
|
||||
Every task that closes a Gitea issue MUST complete ALL of these before it is considered done. Use `/done <issue>` to run through this automatically.
|
||||
|
||||
**Merge-consent is derived from state, not asserted (`## Done-when` convention — ersatztv#303 H6 + H10).** Any issue whose PR will merge to `main` should carry a `## Done-when` section in its **issue body** — a checklist of completion criteria (always include an "adversarial review passed" box; add per-issue criteria like tests-green, docs-updated, live-E2E). Two hooks derive merge-consent from it so a premature merge is blocked *by construction*, not by memory:
|
||||
- `pretooluse-merge-consent.sh` (Claude PreToolUse on the Gitea merge tool) — **auto-grants** a merge (emits `permissionDecision: allow`, so **no** redundant mechanical prompt fires) only when the PR's CI is green **and** every `## Done-when` box on the linked issue (`fixes #N`) is ticked **and** a `Review-verdict:` comment references the PR's *current head sha* (**H10**); **denies** on an unticked box, red CI, or a stale/negative review verdict; **asks** (falls back to a human prompt) when it can't derive state (no linked issue, no `## Done-when` section, no `Review-verdict:` comment yet, no creds, Gitea down). On the auto-grant (satisfied) path the derived state **is** the consent — do not also ask conversationally to merge; a separate human confirmation is warranted only when the gate **asks** (ersatztv#314). **The H10 review-verdict convention**: after an adversarial/Codex review of a PR (or its latest fix commit), run **`scripts/post-review-verdict.sh <pr> <MERGEABLE|APPROVED|BLOCKED|NOT-MERGEABLE> [note]`** — it posts both the `Review-verdict: … @ <head-sha>` comment and the sha-bound `review-verdict/h10` commit status, proving the *latest* commit was reviewed rather than a stale earlier diff (ersatztv#242). Do not hand-write the comment: the **status** is the required check branch protection enforces, and a comment alone leaves it absent.
|
||||
- **The gate is enforced server-side, per sha (ersatztv#622).** `review-verdict/h10` is a required status check on `main`. Because a commit status belongs to one sha, a commit pushed *after* an auto-merge is scheduled clears it and blocks the merge — closing the hole where `merge_when_checks_succeed` froze consent at scheduling time and Gitea later merged an unreviewed head. Renovate-authored and docs-only PRs are auto-passed by `.gitea/workflows/review-verdict.yml`, **except** when they touch `.claude/`, `.gitea/`, `.husky/`, `scripts/` or `docker/ci/`. See `docs/ci-cd.md` → Review-verdict gate.
|
||||
- `.husky/pre-push` → `prepush-donewhen.sh` — a fail-open backstop that blocks a direct `git push origin main` whose commits `fix #N` an issue with unticked boxes.
|
||||
- **The gate is enforced server-side, per sha (ersatztv#622).** `review-verdict/h10` is a required status check on `main`. Because a commit status belongs to one sha, a commit pushed *after* an auto-merge is scheduled clears it and blocks the merge — closing the hole where `merge_when_checks_succeed` froze consent at scheduling time and Gitea later merged an unreviewed head. Renovate-authored and docs-only PRs are auto-passed by `.gitea/workflows/review-verdict.yml`, **except** when they touch `.claude/`, `.codex/`, `.gitea/`, `.husky/`, `scripts/` or `docker/ci/`. See `docs/ci-cd.md` → Review-verdict gate.
|
||||
- `.husky/pre-push` → `prepush-donewhen.sh` — a fail-open backstop that blocks a direct `git push origin main` whose commits `fix #N` an issue with unticked boxes. **Since ersatztv#743 that push can no longer happen at all** (see below), so this hook is now belt-and-braces for a path the server refuses.
|
||||
|
||||
Both need Gitea read creds in the env to enforce (**`ETV_GITEA_BASICAUTH=user:pass`** or `ETV_GITEA_TOKEN`; `ETV_GITEA_URL` overrides the base). Without them the merge hook asks and the push backstop is a no-op — the gate degrades to today's manual confirmation, never a silent pass. Docs-only PRs/pushes are exempt.
|
||||
**`main` is PR-only — there is no direct-push path any more (ersatztv#743, `release.main-direct-push-disabled`).** Branch protection carries `enable_push: false` **and** `block_admin_merge_override: true`: a direct `git push origin HEAD:main` is refused server-side at pre-receive for every account including a site admin, the contents API is refused too, and an admin cannot `force_merge` past a missing or red required context. This is what makes `review-verdict/h10` load-bearing rather than conventional — Gitea only evaluates `status_check_contexts` on the PR merge path, so before this the whole gate was skippable with no forgery. Practically: **every** change to `main` goes through a PR, including a one-line docs fix. Tag pushes are unaffected (separate mechanism), so the release cut is unchanged.
|
||||
|
||||
Both need Gitea read creds in the env to enforce (**`ETV_GITEA_BASICAUTH=user:pass`** or `ETV_GITEA_TOKEN`; `ETV_GITEA_URL` overrides the base). Without them the merge hook asks and the push backstop is a no-op — the gate degrades to today's manual confirmation, never a silent pass. Docs-only PRs are exempt from the *review-verdict* gate; the direct-push exemption is moot now that direct pushes are refused outright.
|
||||
|
||||
**The 7 mandatory completion steps and the `## Closing record` comment template** live in the
|
||||
`closing-an-issue` skill (`.claude/skills/closing-an-issue/SKILL.md`) — invoke it (or `/done`)
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<ItemGroup>
|
||||
<PackageVersion Include="AsyncFixer" Version="2.1.0" />
|
||||
<PackageVersion Include="Blurhash.SkiaSharp" Version="2.0.0" />
|
||||
<PackageVersion Include="CliWrap" Version="3.10.2" />
|
||||
<PackageVersion Include="CliWrap" Version="3.10.4" />
|
||||
<PackageVersion Include="coverlet.collector" Version="6.0.4" />
|
||||
<PackageVersion Include="Dapper" Version="2.1.79" />
|
||||
<PackageVersion Include="Destructurama.Attributed" Version="5.2.0" />
|
||||
@@ -29,7 +29,7 @@
|
||||
<PackageVersion Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00017" />
|
||||
<PackageVersion Include="Lucene.Net.QueryParser" Version="4.8.0-beta00017" />
|
||||
<PackageVersion Include="MediatR" Version="[12.5.0]" />
|
||||
<PackageVersion Include="Meziantou.Analyzer" Version="3.0.115" />
|
||||
<PackageVersion Include="Meziantou.Analyzer" Version="3.0.129" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.2" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="10.0.2" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Identity.Core" Version="10.0.2" />
|
||||
@@ -93,8 +93,8 @@
|
||||
<PackageVersion Include="SonarAnalyzer.CSharp" Version="10.27.0.140913" />
|
||||
<!-- Direct pin to override EF Core 9's transitive SQLitePCLRaw 2.1.10 (vulnerable
|
||||
bundled SQLite, GHSA-2m69-gcr7-jv3q). The 3.x line ships the patched native
|
||||
(lib.e_sqlite3 3.50.3); core 3.0.3 satisfies Microsoft.Data.Sqlite's `>= 2.1.10`. (#8) -->
|
||||
<PackageVersion Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.3" />
|
||||
(lib.e_sqlite3 3.50.3); core 3.0.4 satisfies Microsoft.Data.Sqlite's `>= 2.1.10`. (#8) -->
|
||||
<PackageVersion Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.4" />
|
||||
<PackageVersion Include="System.CommandLine" Version="2.0.2" />
|
||||
<PackageVersion Include="TagLibSharp" Version="2.3.0" />
|
||||
<PackageVersion Include="Testably.Abstractions" Version="10.0.0" />
|
||||
|
||||
@@ -37,23 +37,43 @@ internal static class Mapper
|
||||
collection.Collection is not null ? ProjectToViewModel(collection.Collection) : null,
|
||||
collection.MultiCollection is not null ? ProjectToViewModel(collection.MultiCollection) : null,
|
||||
collection.SmartCollection is not null ? ProjectToViewModel(collection.SmartCollection) : null,
|
||||
collection.MediaItem switch
|
||||
{
|
||||
Show show => MediaItems.Mapper.ProjectToViewModel(show),
|
||||
Season season => MediaItems.Mapper.ProjectToViewModel(season),
|
||||
Artist artist => MediaItems.Mapper.ProjectToViewModel(artist),
|
||||
Movie movie => MediaItems.Mapper.ProjectToViewModel(movie),
|
||||
Episode episode => MediaItems.Mapper.ProjectToViewModel(episode),
|
||||
MusicVideo musicVideo => MediaItems.Mapper.ProjectToViewModel(musicVideo),
|
||||
OtherVideo otherVideo => MediaItems.Mapper.ProjectToViewModel(otherVideo),
|
||||
Song song => MediaItems.Mapper.ProjectToViewModel(song),
|
||||
Image image => MediaItems.Mapper.ProjectToViewModel(image),
|
||||
_ => null
|
||||
},
|
||||
ProjectMediaItemToViewModel(collection.MediaItem),
|
||||
collection.FirstRunPlaybackOrder,
|
||||
collection.RerunPlaybackOrder,
|
||||
collection.Version);
|
||||
|
||||
/// <summary>
|
||||
/// Flattens the <see cref="MediaItem" /> half of a selection tagged union to a named view model.
|
||||
/// Shared by <see cref="RerunCollection" /> and <see cref="PlaylistItem" />, which select from an
|
||||
/// identical set of media types; one copy is what stops the two drifting apart again (issue #671
|
||||
/// — the same rationale as <c>ProgramScheduleItemQueryExtensions.IncludeScheduleItemDetails</c>
|
||||
/// on the query side).
|
||||
/// A null <paramref name="mediaItem" /> is the legitimate "this selection is not a media item"
|
||||
/// case (the selection is a Collection/MultiCollection/SmartCollection instead) and maps to null.
|
||||
/// An unrecognized non-null subtype keeps its id and takes a deliberately conspicuous name rather
|
||||
/// than falling through to null: the id is what the editor round-trips, so returning null there
|
||||
/// silently clears the user's stored selection — while throwing would fail an entire paged GET
|
||||
/// over one unreadable row.
|
||||
/// </summary>
|
||||
private static MediaItems.NamedMediaItemViewModel ProjectMediaItemToViewModel(MediaItem mediaItem) =>
|
||||
mediaItem switch
|
||||
{
|
||||
null => null,
|
||||
Show show => MediaItems.Mapper.ProjectToViewModel(show),
|
||||
Season season => MediaItems.Mapper.ProjectToViewModel(season),
|
||||
Artist artist => MediaItems.Mapper.ProjectToViewModel(artist),
|
||||
Movie movie => MediaItems.Mapper.ProjectToViewModel(movie),
|
||||
Episode episode => MediaItems.Mapper.ProjectToViewModel(episode),
|
||||
MusicVideo musicVideo => MediaItems.Mapper.ProjectToViewModel(musicVideo),
|
||||
OtherVideo otherVideo => MediaItems.Mapper.ProjectToViewModel(otherVideo),
|
||||
Song song => MediaItems.Mapper.ProjectToViewModel(song),
|
||||
Image image => MediaItems.Mapper.ProjectToViewModel(image),
|
||||
RemoteStream remoteStream => MediaItems.Mapper.ProjectToNamedViewModel(remoteStream),
|
||||
_ => new MediaItems.NamedMediaItemViewModel(
|
||||
mediaItem.Id,
|
||||
$"[unsupported media type: {mediaItem.GetType().Name}]")
|
||||
};
|
||||
|
||||
internal static TraktListViewModel ProjectToViewModel(TraktList traktList) =>
|
||||
new(
|
||||
traktList.Id,
|
||||
@@ -108,19 +128,7 @@ internal static class Mapper
|
||||
playlistItem.SmartCollection is not null
|
||||
? ProjectToViewModel(playlistItem.SmartCollection)
|
||||
: null,
|
||||
playlistItem.MediaItem switch
|
||||
{
|
||||
Show show => MediaItems.Mapper.ProjectToViewModel(show),
|
||||
Season season => MediaItems.Mapper.ProjectToViewModel(season),
|
||||
Artist artist => MediaItems.Mapper.ProjectToViewModel(artist),
|
||||
Movie movie => MediaItems.Mapper.ProjectToViewModel(movie),
|
||||
Episode episode => MediaItems.Mapper.ProjectToViewModel(episode),
|
||||
MusicVideo musicVideo => MediaItems.Mapper.ProjectToViewModel(musicVideo),
|
||||
OtherVideo otherVideo => MediaItems.Mapper.ProjectToViewModel(otherVideo),
|
||||
Song song => MediaItems.Mapper.ProjectToViewModel(song),
|
||||
Image image => MediaItems.Mapper.ProjectToViewModel(image),
|
||||
_ => null
|
||||
},
|
||||
ProjectMediaItemToViewModel(playlistItem.MediaItem),
|
||||
playlistItem.PlaybackOrder,
|
||||
playlistItem.Count,
|
||||
playlistItem.PlayAll,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using static ErsatzTV.Application.MediaCollections.Mapper;
|
||||
@@ -15,13 +15,15 @@ public class GetPagedRerunCollectionsHandler(IDbContextFactory<TvContext> dbCont
|
||||
await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
int count = await dbContext.RerunCollections.CountAsync(cancellationToken);
|
||||
|
||||
IQueryable<RerunCollection> query = dbContext.RerunCollections.AsNoTracking();
|
||||
IQueryable<RerunCollection> query = dbContext.RerunCollections.AsNoTracking().IncludeSelectionDetails();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(request.Query))
|
||||
{
|
||||
query = query.Where(rc => EF.Functions.Like(rc.Name, $"%{request.Query}%"));
|
||||
}
|
||||
|
||||
// EF applies the includes to the paged subquery, so the selection graph is loaded for at most
|
||||
// PageSize rows — the per-request cost is bounded by the page, not by the table (issue #671).
|
||||
List<RerunCollectionViewModel> page = await query
|
||||
.OrderBy(rc => rc.Name)
|
||||
.Skip(request.PageNum * request.PageSize)
|
||||
|
||||
@@ -55,6 +55,10 @@ public class GetPlaylistItemsHandler(IDbContextFactory<TvContext> dbContextFacto
|
||||
.Include(i => i.MediaItem)
|
||||
.ThenInclude(i => (i as Image).ImageMetadata)
|
||||
.ThenInclude(mm => mm.Artwork)
|
||||
// RemoteStream is projected by the shared ProjectMediaItemToViewModel switch as of #671;
|
||||
// without its metadata the name would degrade to "???" here while every sibling type resolves.
|
||||
.Include(i => i.MediaItem)
|
||||
.ThenInclude(i => (i as RemoteStream).RemoteStreamMetadata)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return allItems.Map(Mapper.ProjectToViewModel).ToList();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
using ErsatzTV.Infrastructure.Extensions;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
@@ -16,20 +16,7 @@ public class GetRerunCollectionByIdHandler(IDbContextFactory<TvContext> dbContex
|
||||
await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
return await dbContext.RerunCollections
|
||||
.AsNoTracking()
|
||||
.Include(c => c.Collection)
|
||||
.Include(c => c.MultiCollection)
|
||||
.Include(c => c.SmartCollection)
|
||||
.Include(i => i.MediaItem)
|
||||
.ThenInclude(i => (i as Movie).MovieMetadata)
|
||||
.Include(i => i.MediaItem)
|
||||
.ThenInclude(i => (i as Season).SeasonMetadata)
|
||||
.Include(i => i.MediaItem)
|
||||
.ThenInclude(i => (i as Season).Show)
|
||||
.ThenInclude(s => s.ShowMetadata)
|
||||
.Include(i => i.MediaItem)
|
||||
.ThenInclude(i => (i as Show).ShowMetadata)
|
||||
.Include(i => i.MediaItem)
|
||||
.ThenInclude(i => (i as Artist).ArtistMetadata)
|
||||
.IncludeSelectionDetails()
|
||||
.SelectOneAsync(c => c.Id, c => c.Id == request.Id, cancellationToken)
|
||||
.MapT(ProjectToViewModel);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
using ErsatzTV.Core.Domain;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ErsatzTV.Application.MediaCollections;
|
||||
|
||||
internal static class RerunCollectionQueryExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// The single source of truth for the navigation graph a <see cref="RerunCollection" /> needs before it
|
||||
/// can be projected via <see cref="Mapper.ProjectToViewModel(RerunCollection)" />. Both the paged-list
|
||||
/// and by-id handlers reload through this chain so the two cannot drift apart again (see #671 — the list
|
||||
/// handler had no includes at all, so every row projected a null selection, while the by-id handler
|
||||
/// covered only Movie/Season/Show/Artist and so returned a null selection for Song/OtherVideo/Image and
|
||||
/// a 500 for Episode/MusicVideo).
|
||||
/// Because the id and the display name are both read off these navigations, an un-included type does not
|
||||
/// merely lose its label — it loses the selected id too, which is what silently cleared a stored
|
||||
/// selection in the editor.
|
||||
/// Deliberately narrower than the analogous playlist-item chain in <c>GetPlaylistItemsHandler</c>: the
|
||||
/// rerun projection reads only each selection's id and title, never its artwork, so the
|
||||
/// <c>.ThenInclude(… => …Artwork)</c> legs are omitted rather than paid for on every page.
|
||||
/// </summary>
|
||||
public static IQueryable<RerunCollection> IncludeSelectionDetails(this IQueryable<RerunCollection> query) =>
|
||||
query
|
||||
.Include(c => c.Collection)
|
||||
.Include(c => c.MultiCollection)
|
||||
.Include(c => c.SmartCollection)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as Movie).MovieMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as Show).ShowMetadata)
|
||||
// No (i as Season).SeasonMetadata leg on purpose: ProjectToViewModel(Season) builds its name
|
||||
// from Show.ShowMetadata and the scalar SeasonNumber, and never reads SeasonMetadata.
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as Season).Show)
|
||||
.ThenInclude(s => s.ShowMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as Artist).ArtistMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as Episode).EpisodeMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as Episode).Season)
|
||||
.ThenInclude(s => s.Show)
|
||||
.ThenInclude(s => s.ShowMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as MusicVideo).Artist)
|
||||
.ThenInclude(a => a.ArtistMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as OtherVideo).OtherVideoMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as Song).SongMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as Image).ImageMetadata)
|
||||
.Include(c => c.MediaItem)
|
||||
.ThenInclude(i => (i as RemoteStream).RemoteStreamMetadata);
|
||||
}
|
||||
@@ -1,18 +1,24 @@
|
||||
using System.Globalization;
|
||||
using System.Globalization;
|
||||
using ErsatzTV.Core.Domain;
|
||||
|
||||
namespace ErsatzTV.Application.MediaItems;
|
||||
|
||||
internal static class Mapper
|
||||
{
|
||||
// Every metadata navigation below is read through Optional(...).Flatten() rather than a bare
|
||||
// dereference: these projections are reached from several handlers whose Include chains differ,
|
||||
// and an un-included navigation must degrade to the "???" placeholder instead of throwing an
|
||||
// NRE that surfaces as a 500 on a GET (issue #671).
|
||||
internal static NamedMediaItemViewModel ProjectToViewModel(Show show) =>
|
||||
new(show.Id, show.ShowMetadata.HeadOrNone().Map(sm => $"{sm?.Title} ({sm?.Year})").IfNone("???"));
|
||||
new(
|
||||
show.Id,
|
||||
Optional(show.ShowMetadata).Flatten().HeadOrNone().Map(sm => $"{sm?.Title} ({sm?.Year})").IfNone("???"));
|
||||
|
||||
internal static NamedMediaItemViewModel ProjectToViewModel(Season season) =>
|
||||
new(season.Id, $"{ShowTitle(season)} - {SeasonDescription(season)}");
|
||||
|
||||
internal static NamedMediaItemViewModel ProjectToViewModel(Artist artist) =>
|
||||
new(artist.Id, artist.ArtistMetadata.HeadOrNone().Match(am => am.Title, () => "???"));
|
||||
new(artist.Id, Optional(artist.ArtistMetadata).Flatten().HeadOrNone().Match(am => am.Title, () => "???"));
|
||||
|
||||
internal static NamedMediaItemViewModel ProjectToViewModel(Movie movie) =>
|
||||
new(movie.Id, MovieTitle(movie));
|
||||
@@ -24,23 +30,37 @@ internal static class Mapper
|
||||
new(musicVideo.Id, MusicVideoTitle(musicVideo));
|
||||
|
||||
internal static NamedMediaItemViewModel ProjectToViewModel(OtherVideo otherVideo) =>
|
||||
new(otherVideo.Id, otherVideo.OtherVideoMetadata.HeadOrNone().Match(ov => ov.Title, () => "???"));
|
||||
new(
|
||||
otherVideo.Id,
|
||||
Optional(otherVideo.OtherVideoMetadata).Flatten().HeadOrNone().Match(ov => ov.Title, () => "???"));
|
||||
|
||||
internal static NamedMediaItemViewModel ProjectToViewModel(Song song) =>
|
||||
new(song.Id, SongTitle(song));
|
||||
|
||||
internal static NamedMediaItemViewModel ProjectToViewModel(Image image) =>
|
||||
new(image.Id, image.ImageMetadata.HeadOrNone().Match(i => i.Title, () => "???"));
|
||||
new(image.Id, Optional(image.ImageMetadata).Flatten().HeadOrNone().Match(i => i.Title, () => "???"));
|
||||
|
||||
internal static RemoteStreamViewModel ProjectToViewModel(RemoteStream remoteStream) =>
|
||||
new(remoteStream.Id, remoteStream.Url, remoteStream.Script);
|
||||
|
||||
/// <summary>
|
||||
/// The named projection for a <see cref="RemoteStream" />. This cannot be an overload of
|
||||
/// <see cref="ProjectToViewModel(RemoteStream)" /> — that one already exists and returns a
|
||||
/// <see cref="RemoteStreamViewModel" />, and C# will not overload on return type alone. Its
|
||||
/// absence is why every selection-flattening switch dropped <c>RemoteStream</c> through a
|
||||
/// <c>_ => null</c> arm (issue #671).
|
||||
/// </summary>
|
||||
internal static NamedMediaItemViewModel ProjectToNamedViewModel(RemoteStream remoteStream) =>
|
||||
new(
|
||||
remoteStream.Id,
|
||||
Optional(remoteStream.RemoteStreamMetadata).Flatten().HeadOrNone().Match(rsm => rsm.Title, () => "???"));
|
||||
|
||||
private static string MovieTitle(Movie movie)
|
||||
{
|
||||
var title = "???";
|
||||
var year = "???";
|
||||
|
||||
foreach (MovieMetadata movieMetadata in movie.MovieMetadata.HeadOrNone())
|
||||
foreach (MovieMetadata movieMetadata in Optional(movie.MovieMetadata).Flatten().HeadOrNone())
|
||||
{
|
||||
title = movieMetadata.Title;
|
||||
foreach (int y in Optional(movieMetadata.Year))
|
||||
@@ -57,7 +77,10 @@ internal static class Mapper
|
||||
var title = "???";
|
||||
var year = "???";
|
||||
|
||||
foreach (ShowMetadata show in season.Show.ShowMetadata.HeadOrNone())
|
||||
// Season.Show and Show.ShowMetadata are only populated when the caller eager-loaded them.
|
||||
// An un-included navigation must degrade to the "???" placeholder these helpers already
|
||||
// produce for missing metadata — never an NRE, which surfaced as a 500 (issue #671).
|
||||
foreach (ShowMetadata show in Optional(season.Show?.ShowMetadata).Flatten().HeadOrNone())
|
||||
{
|
||||
title = show.Title;
|
||||
foreach (int y in Optional(show.Year))
|
||||
@@ -74,10 +97,10 @@ internal static class Mapper
|
||||
|
||||
private static string EpisodeTitle(Episode e)
|
||||
{
|
||||
string showTitle = e.Season.Show.ShowMetadata.HeadOrNone()
|
||||
string showTitle = Optional(e.Season?.Show?.ShowMetadata).Flatten().HeadOrNone()
|
||||
.Map(sm => $"{sm.Title} - ").IfNone(string.Empty);
|
||||
var episodeNumbers = e.EpisodeMetadata.Map(em => em.EpisodeNumber).ToList();
|
||||
var episodeTitles = e.EpisodeMetadata.Map(em => em.Title).ToList();
|
||||
var episodeNumbers = Optional(e.EpisodeMetadata).Flatten().Map(em => em.EpisodeNumber).ToList();
|
||||
var episodeTitles = Optional(e.EpisodeMetadata).Flatten().Map(em => em.Title).ToList();
|
||||
if (episodeNumbers.Count == 0 || episodeTitles.Count == 0)
|
||||
{
|
||||
return "[unknown episode]";
|
||||
@@ -86,24 +109,34 @@ internal static class Mapper
|
||||
var numbersString = $"e{string.Join('e', episodeNumbers.Map(n => $"{n:00}"))}";
|
||||
var titlesString = $"{string.Join('/', episodeTitles)}";
|
||||
|
||||
return $"{showTitle}s{e.Season.SeasonNumber:00}{numbersString} - {titlesString}";
|
||||
// "s00" conventionally means Specials, so an unloaded Season must not borrow it — that would
|
||||
// fabricate plausible-looking real data. Render the season as explicitly unknown instead.
|
||||
string seasonNumber = e.Season is null ? "??" : $"{e.Season.SeasonNumber:00}";
|
||||
|
||||
return $"{showTitle}s{seasonNumber}{numbersString} - {titlesString}";
|
||||
}
|
||||
|
||||
private static string MusicVideoTitle(MusicVideo mv)
|
||||
{
|
||||
string artistName = mv.Artist.ArtistMetadata.HeadOrNone()
|
||||
string artistName = Optional(mv.Artist?.ArtistMetadata).Flatten().HeadOrNone()
|
||||
.Map(am => $"{am.Title} - ").IfNone(string.Empty);
|
||||
return mv.MusicVideoMetadata.HeadOrNone()
|
||||
return Optional(mv.MusicVideoMetadata).Flatten().HeadOrNone()
|
||||
.Map(mvm => $"{artistName}{mvm.Title}")
|
||||
.IfNone("[unknown music video]");
|
||||
}
|
||||
|
||||
private static string SongTitle(Song s)
|
||||
{
|
||||
string songArtist = s.SongMetadata.HeadOrNone()
|
||||
.Map(sm => $"{string.Join(", ", sm.Artists)} - ")
|
||||
// Artists is a NULLABLE primitive collection, not a navigation: a song whose tags failed to read
|
||||
// is persisted by FallbackMetadataProvider with Artists never assigned, and string.Join throws
|
||||
// ArgumentNullException on a null sequence. Filtering the empty case too avoids prefixing an
|
||||
// artist-less song with a bare " - ".
|
||||
string songArtist = Optional(s.SongMetadata).Flatten().HeadOrNone()
|
||||
.Map(sm => Optional(sm.Artists).Flatten().ToList())
|
||||
.Filter(artists => artists.Count > 0)
|
||||
.Map(artists => $"{string.Join(", ", artists)} - ")
|
||||
.IfNone(string.Empty);
|
||||
return s.SongMetadata.HeadOrNone()
|
||||
return Optional(s.SongMetadata).Flatten().HeadOrNone()
|
||||
.Map(sm => $"{songArtist}{sm.Title ?? string.Empty}")
|
||||
.IfNone("[unknown song]");
|
||||
}
|
||||
|
||||
@@ -102,14 +102,24 @@ internal static class Mapper
|
||||
: $"{s} ({chapterTitle})")
|
||||
.IfNone("[unknown video]");
|
||||
case Song s:
|
||||
string songArtist = s.SongMetadata.HeadOrNone()
|
||||
.Map(sm => $"{string.Join(", ", sm.Artists)} - ")
|
||||
// SongMetadata.Artists is a NULLABLE primitive collection (FallbackMetadataProvider never
|
||||
// assigns it for a song whose tags failed to read) and string.Join throws
|
||||
// ArgumentNullException on a null sequence. SongMetadata IS eager-loaded on this path, so
|
||||
// this was a LIVE 500 on the playout guide, not a latent one (issue #671).
|
||||
string songArtist = Optional(s.SongMetadata).Flatten().HeadOrNone()
|
||||
.Map(sm => Optional(sm.Artists).Flatten().ToList())
|
||||
.Filter(artists => artists.Count > 0)
|
||||
.Map(artists => $"{string.Join(", ", artists)} - ")
|
||||
.IfNone(string.Empty);
|
||||
return s.SongMetadata.HeadOrNone()
|
||||
return Optional(s.SongMetadata).Flatten().HeadOrNone()
|
||||
.Map(sm => $"{songArtist}{sm.Title ?? string.Empty}")
|
||||
.Map(t => string.IsNullOrWhiteSpace(chapterTitle)
|
||||
// interpolate the composed title `t`, NOT the `case Song s` entity — Song has no
|
||||
// ToString() override, so `{s}` rendered a chaptered song as the literal type name
|
||||
// "ErsatzTV.Core.Domain.Song (Chapter 3)". The MusicVideo/OtherVideo arms above are
|
||||
// correct only because they happen to name their lambda parameter `s`.
|
||||
? t
|
||||
: $"{s} ({chapterTitle})")
|
||||
: $"{t} ({chapterTitle})")
|
||||
.IfNone("[unknown song]");
|
||||
case Image i:
|
||||
return i.ImageMetadata.HeadOrNone().Map(im => im.Title ?? string.Empty).IfNone("[unknown image]");
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Dapper;
|
||||
using ErsatzTV.Core.Api.Search;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
@@ -11,6 +14,62 @@ public class GetSearchFieldValuesHandler(IDbContextFactory<TvContext> dbContextF
|
||||
private const int DefaultLimit = 50;
|
||||
private const int MaxLimit = 50;
|
||||
|
||||
/// <summary>
|
||||
/// Rows read per round trip when walking the list-valued (JSON-array) columns on
|
||||
/// <c>SongMetadata</c>, and the ceiling on rows read per request.
|
||||
/// <para>
|
||||
/// These count ACTUAL ROWS, and arriving at that took four tries — each earlier attempt bounded a
|
||||
/// quantity that sounded like rows and was not. A fixed <c>LIMIT</c> budget bounded the RESULT, and
|
||||
/// the pre-filter (allowed to over-match) starved it with rows that could not match. Keyset paging
|
||||
/// with a <c>LIMIT</c> bounded CANDIDATES RETURNED — but a query matching nothing must evaluate
|
||||
/// every eligible row before it can return an empty page, so rows inspected stayed unbounded. A
|
||||
/// closed <c>Id</c> range bounded KEYSPACE WIDTH — but keyspace is not rows: delete 20,000
|
||||
/// historical rows, put one song at <c>Id</c> 20001, and the walk burns its whole allowance on empty
|
||||
/// ranges and inspects nothing.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// What makes this one hold is that <b>the query has no RESIDUAL predicate</b> — nothing that can
|
||||
/// discard a row the engine already produced. The only condition is the cursor
|
||||
/// <c>Id > @AfterId</c>, which is a seek on the <c>ORDER BY</c> key itself, not a filter. So the
|
||||
/// page returns exactly <see cref="ListValuedBatchRows" /> rows whenever that many logical rows
|
||||
/// remain, independent of how sparse the matches are or where the <c>Id</c> gaps fall.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Be precise about what is bounded: LOGICAL ROWS RETURNED AND MATERIALIZED, and the number of
|
||||
/// round trips. Not physical work, and not bytes.</b> Two things break the stronger reading, and an
|
||||
/// earlier version of this comment asserted it anyway:
|
||||
/// <list type="bullet">
|
||||
/// <item>
|
||||
/// MySQL purge lag. Deleted clustered-index records survive until purge runs, and a range
|
||||
/// scan still traverses them, so returning 2,000 VISIBLE rows can touch far more index
|
||||
/// records. Deletion history therefore still affects physical work — the very thing the
|
||||
/// keyspace attempt was trying to make irrelevant.
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// Row width is unbounded. These columns are <c>TEXT</c>/<c>longtext</c>, which both SQLite
|
||||
/// and InnoDB spill to overflow pages, so a row count implies neither a byte count nor a
|
||||
/// page-read count.
|
||||
/// </item>
|
||||
/// </list>
|
||||
/// The logical-row bound is still worth having — it is what makes the walk terminate and what caps
|
||||
/// the number of rows and round trips — but do not restate it as bounded I/O, and do not restate it
|
||||
/// as bounded MEMORY either: payload width is unrestricted and a single JSON array can hold
|
||||
/// arbitrarily many strings, every one of which may enter the in-memory set.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The trade is real and deliberate: no server-side narrowing, so a query with few matches transfers
|
||||
/// rows it will discard, up to <see cref="ListValuedMaxRowsRead" />. A query with enough matches
|
||||
/// stops as soon as it has <c>limit</c> distinct ones, so the dense cases — including an empty
|
||||
/// <c>q</c> — finish on the first page. See <c>api.search-field-values-sources</c> for the measured
|
||||
/// cost and for why reintroducing a <c>LIKE</c> is not an option.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal const int ListValuedBatchRows = 2000;
|
||||
|
||||
/// <inheritdoc cref="ListValuedBatchRows" />
|
||||
internal const int ListValuedMaxRowsRead = 20000;
|
||||
|
||||
|
||||
public async Task<Option<SearchFieldValuesResponseModel>> Handle(
|
||||
GetSearchFieldValues request,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -24,17 +83,22 @@ public class GetSearchFieldValuesHandler(IDbContextFactory<TvContext> dbContextF
|
||||
}
|
||||
|
||||
int limit = request.Limit <= 0 ? DefaultLimit : Math.Clamp(request.Limit, 1, MaxLimit);
|
||||
string qLower = (request.Query ?? string.Empty).ToLower();
|
||||
string query = request.Query ?? string.Empty;
|
||||
|
||||
// Invariant, not current-culture: UseRequestLocalization honours Accept-Language, so a caller can select
|
||||
// tr-TR and turn `q=I` into `ı` — which then matches nothing a Turkish-dotless-i-free library contains.
|
||||
// This feeds the EF-translated filter, which has no StringComparison overload EF can translate.
|
||||
string qLower = query.ToLowerInvariant();
|
||||
|
||||
// in-memory special cases (no DB query needed)
|
||||
switch (request.Name)
|
||||
{
|
||||
case "state":
|
||||
return new SearchFieldValuesResponseModel(
|
||||
FilterSortTake(Enum.GetNames<MediaItemState>(), qLower, limit));
|
||||
FilterSortTake(Enum.GetNames<MediaItemState>(), query, limit));
|
||||
case "video_dynamic_range":
|
||||
return new SearchFieldValuesResponseModel(
|
||||
FilterSortTake(["hdr", "sdr"], qLower, limit));
|
||||
FilterSortTake(["hdr", "sdr"], query, limit));
|
||||
}
|
||||
|
||||
await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
@@ -42,34 +106,75 @@ public class GetSearchFieldValuesHandler(IDbContextFactory<TvContext> dbContextF
|
||||
if (request.Name == "content_rating")
|
||||
{
|
||||
return new SearchFieldValuesResponseModel(
|
||||
await GetContentRatingValues(dbContext, qLower, limit, cancellationToken));
|
||||
await GetContentRatingValues(dbContext, query, limit, cancellationToken));
|
||||
}
|
||||
|
||||
IQueryable<string> source = GetSource(dbContext, request.Name);
|
||||
if (source is null)
|
||||
string listColumn = GetSongListValuedColumn(request.Name);
|
||||
if (source is null && listColumn is null)
|
||||
{
|
||||
return Option<SearchFieldValuesResponseModel>.None;
|
||||
}
|
||||
|
||||
List<string> values = await source
|
||||
.Where(v => v != null && v.ToLower().StartsWith(qLower))
|
||||
.Distinct()
|
||||
.OrderBy(v => v)
|
||||
.Take(limit)
|
||||
.ToListAsync(cancellationToken);
|
||||
var values = new List<string>();
|
||||
|
||||
return new SearchFieldValuesResponseModel(values);
|
||||
if (source is not null)
|
||||
{
|
||||
values.AddRange(
|
||||
await source
|
||||
.Where(v => v != null && v.ToLower().StartsWith(qLower))
|
||||
.Distinct()
|
||||
.OrderBy(v => v)
|
||||
.Take(limit)
|
||||
.ToListAsync(cancellationToken));
|
||||
}
|
||||
|
||||
// ersatztv#668. The query above prefix-matches through SQL LOWER(), and SQLite's LOWER() folds ASCII
|
||||
// ONLY -- lower('Édith') is 'Édith' unchanged -- so it cannot reach a stored value whose prefix
|
||||
// carries an uppercase non-ASCII character, from ANY query. It UNDER-matches, and an under-match is
|
||||
// unrecoverable downstream: no later stage can reintroduce a row SQL never returned. So for the only
|
||||
// queries that can be affected (those containing a non-ASCII character) run a second, Unicode-correct
|
||||
// pass and merge it in. This is ADDITIVE on purpose -- the SQL pass above still contributes, so a
|
||||
// value already reachable today cannot stop being reachable.
|
||||
//
|
||||
// MySQL needs none of this: its LOWER() is Unicode-aware, so LOWER('Édith') really is 'édith' and the
|
||||
// existing predicate reaches the row unaided. Measured on 8.4 -- and note the executed path does NOT
|
||||
// over-match, even though the column collation (utf8mb4_0900_ai_ci) is accent-insensitive: the driver
|
||||
// binds the LIKE pattern with a BINARY collation, so the comparison is accent-sensitive in practice.
|
||||
// A hand-typed probe using a LITERAL pattern DOES over-match; that is a different query from the one
|
||||
// this code runs, and mistaking the two is how an earlier revision of the decision record got it wrong.
|
||||
if (source is not null && ContainsNonAscii(query) && IsSqlite(dbContext))
|
||||
{
|
||||
values.AddRange(
|
||||
await GetUnicodeFoldedValues(dbContext, request.Name, query, limit, cancellationToken));
|
||||
}
|
||||
|
||||
if (listColumn is not null)
|
||||
{
|
||||
values.AddRange(await GetSongListValuedValues(dbContext, listColumn, query, limit, cancellationToken));
|
||||
}
|
||||
|
||||
// ORDERING IS BEST-EFFORT, NOT EXACT. Each source truncates using its own ordering — the EF source by the
|
||||
// database collation (SQLite's NOCASE/BINARY is ASCII-only), the list source by primary key — and neither
|
||||
// is the ordinal ordering applied here. So when a source actually truncates, a value it dropped may have
|
||||
// outranked one that survived: with "Zulu" and "apple" and limit=1 the database keeps "apple" (its
|
||||
// ordering is case-insensitive) while ordinal ranks "Zulu" first, so the merge never sees "Zulu".
|
||||
// Below the truncation points (the normal typeahead case) the result is exact.
|
||||
return new SearchFieldValuesResponseModel(FilterSortTake(values.Distinct(StringComparer.Ordinal), query, limit));
|
||||
}
|
||||
|
||||
private static IQueryable<string> GetSource(TvContext dbContext, string name) => name switch
|
||||
internal static IQueryable<string> GetSource(TvContext dbContext, string name) => name switch
|
||||
{
|
||||
"genre" or "show_genre" => dbContext.Set<Genre>().Select(g => g.Name),
|
||||
"studio" => dbContext.Set<Studio>().Select(s => s.Name),
|
||||
"director" => dbContext.Set<Director>().Select(d => d.Name),
|
||||
"writer" => dbContext.Set<Writer>().Select(w => w.Name),
|
||||
"actor" => dbContext.Actors.Select(a => a.Name),
|
||||
// entity artists only; free-text music-video/song artist credits are not included (known limitation)
|
||||
"artist" => dbContext.ArtistMetadata.Select(m => m.Title),
|
||||
// Mirrors what LuceneSearchIndex writes to the `artist` field: the music video's linked artist entity
|
||||
// (ArtistMetadata.Title) plus its free-text credits (MusicVideoArtist rows). The third contributor —
|
||||
// SongMetadata.Artists — is a JSON-array column and is handled by GetSongListValuedValues instead.
|
||||
"artist" => dbContext.ArtistMetadata.Select(m => m.Title)
|
||||
.Concat(dbContext.Set<MusicVideoArtist>().Select(a => a.Name)),
|
||||
"tag" => dbContext.Set<Tag>()
|
||||
.Where(t => t.ExternalTypeId != Tag.NfoCountryTypeId && t.ExternalTypeId != Tag.PlexNetworkTypeId)
|
||||
.Select(t => t.Name),
|
||||
@@ -87,9 +192,309 @@ public class GetSearchFieldValuesHandler(IDbContextFactory<TvContext> dbContextF
|
||||
_ => null
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// SQL name of the invariant-uppercase fold registered by <c>SqliteUnicodeFunctions</c>. Duplicated
|
||||
/// rather than referenced because Application must not depend on a provider assembly; a test asserts
|
||||
/// the two constants are equal so they cannot drift.
|
||||
/// </summary>
|
||||
internal const string UpperFunction = "etv_upper";
|
||||
|
||||
/// <summary>
|
||||
/// True when the value contains any character outside US-ASCII, which is exactly when SQLite's
|
||||
/// ASCII-only <c>LOWER()</c> can under-match. Evaluated on the RAW query, never the lowercased copy:
|
||||
/// the trigger must not be coupled to the fold.
|
||||
/// </summary>
|
||||
internal static bool ContainsNonAscii(string value)
|
||||
{
|
||||
foreach (char c in value)
|
||||
{
|
||||
if (c > 0x7F)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// Derived per-context rather than read from the TvContext.IsSqlite static on purpose. Nothing MECHANICALLY
|
||||
// stops that read -- ProviderStaticsWiringTests only parses the two composition roots for ASSIGNMENTS, not
|
||||
// readers -- but that test's scanner exemption for IsSqlite is justified in prose as "read only by
|
||||
// DbInitializer + DatabaseMigratorService, both host-only", and reading it here would make that reason
|
||||
// false while the test stayed green. Do not "simplify" this to IsSqlite.
|
||||
private static bool IsSqlite(TvContext dbContext) =>
|
||||
(dbContext.Database.ProviderName ?? string.Empty).Contains("Sqlite", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Escapes the LIKE metacharacters in a user-supplied prefix and appends the trailing wildcard. The
|
||||
/// backslash MUST be escaped first, or the escapes added for <c>%</c>/<c>_</c> would themselves be
|
||||
/// re-escaped. Paired with an explicit <c>ESCAPE '\'</c> in <see cref="UnicodeFoldSql" />, since raw
|
||||
/// SQL gets none of the escaping EF does for <c>StartsWith</c>.
|
||||
/// </summary>
|
||||
internal static string EscapeLikePrefix(string value) =>
|
||||
value
|
||||
.Replace("\\", "\\\\", StringComparison.Ordinal)
|
||||
.Replace("%", "\\%", StringComparison.Ordinal)
|
||||
.Replace("_", "\\_", StringComparison.Ordinal) + "%";
|
||||
|
||||
/// <summary>
|
||||
/// One bounded, exact prefix query using the Unicode-correct fold. Unlike the list-valued walk this
|
||||
/// KEEPS its selectivity in SQL — it is a normal indexed-or-not <c>LIMIT</c>ed query exactly like the
|
||||
/// EF one it supplements, not a paged walk, so there is no row budget to blow and no reason to strip
|
||||
/// the discriminator predicates out of it.
|
||||
/// </summary>
|
||||
internal static string UnicodeFoldSql(string table, string column, string predicate)
|
||||
{
|
||||
var match = $"{UpperFunction}({column}) LIKE @Pattern ESCAPE '\\'";
|
||||
string where = predicate is null ? match : $"({predicate}) AND {match}";
|
||||
return $"SELECT DISTINCT {column} AS Value FROM {table} WHERE {where} ORDER BY {column} LIMIT @Limit";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The tables/columns behind each EF-sourced field, mirroring <see cref="GetSource" /> 1:1.
|
||||
/// <para>
|
||||
/// The discriminator predicates must mirror EF's NULL semantics, not C#'s reading of the source.
|
||||
/// EF compiles <c>t.ExternalTypeId != Tag.NfoCountryTypeId</c> with null semantics, so a row whose
|
||||
/// <c>ExternalTypeId</c> is NULL IS included; plain SQL <c><></c> against NULL yields NULL and
|
||||
/// would silently drop it. Hence the explicit <c>IS NULL</c> arm.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private static IReadOnlyList<UnicodeFoldSource> GetUnicodeFoldSources(string name) => name switch
|
||||
{
|
||||
"genre" or "show_genre" => [new UnicodeFoldSource("Genre", "Name")],
|
||||
"studio" => [new UnicodeFoldSource("Studio", "Name")],
|
||||
"director" => [new UnicodeFoldSource("Director", "Name")],
|
||||
"writer" => [new UnicodeFoldSource("Writer", "Name")],
|
||||
"actor" => [new UnicodeFoldSource("Actor", "Name")],
|
||||
"artist" =>
|
||||
[
|
||||
new UnicodeFoldSource("ArtistMetadata", "Title"),
|
||||
new UnicodeFoldSource("MusicVideoArtist", "Name")
|
||||
],
|
||||
"tag" =>
|
||||
[
|
||||
new UnicodeFoldSource(
|
||||
"Tag",
|
||||
"Name",
|
||||
"ExternalTypeId IS NULL OR (ExternalTypeId <> @NfoCountryTypeId AND ExternalTypeId <> @PlexNetworkTypeId)",
|
||||
new Dictionary<string, object>
|
||||
{
|
||||
["NfoCountryTypeId"] = Tag.NfoCountryTypeId,
|
||||
["PlexNetworkTypeId"] = Tag.PlexNetworkTypeId
|
||||
})
|
||||
],
|
||||
"network" =>
|
||||
[
|
||||
new UnicodeFoldSource(
|
||||
"Tag",
|
||||
"Name",
|
||||
"ExternalTypeId = @PlexNetworkTypeId",
|
||||
new Dictionary<string, object> { ["PlexNetworkTypeId"] = Tag.PlexNetworkTypeId })
|
||||
],
|
||||
"collection" => [new UnicodeFoldSource("Collection", "Name")],
|
||||
"video_codec" =>
|
||||
[
|
||||
new UnicodeFoldSource(
|
||||
"MediaStream",
|
||||
"Codec",
|
||||
"MediaStreamKind = @VideoStreamKind AND Codec IS NOT NULL",
|
||||
new Dictionary<string, object> { ["VideoStreamKind"] = (int)MediaStreamKind.Video })
|
||||
],
|
||||
"album" =>
|
||||
[
|
||||
new UnicodeFoldSource("MusicVideoMetadata", "Album", "Album IS NOT NULL"),
|
||||
new UnicodeFoldSource("SongMetadata", "Album", "Album IS NOT NULL")
|
||||
],
|
||||
_ => []
|
||||
};
|
||||
|
||||
private static async Task<List<string>> GetUnicodeFoldedValues(
|
||||
TvContext dbContext,
|
||||
string name,
|
||||
string query,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
IReadOnlyList<UnicodeFoldSource> sources = GetUnicodeFoldSources(name);
|
||||
if (sources.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
// CreateFunction is per-connection, so registration happens here, at the one call site that needs
|
||||
// the function, rather than through an EF connection interceptor: Dapper opens a closed connection
|
||||
// itself and a direct ADO open does not raise EF's interceptors, so an interceptor-based seam would
|
||||
// silently miss exactly this query. Opening first makes the registration order-independent.
|
||||
await dbContext.Database.OpenConnectionAsync(cancellationToken);
|
||||
TvContext.RegisterUnicodeCaseFunctions(dbContext.Connection);
|
||||
|
||||
string pattern = EscapeLikePrefix(query.ToUpperInvariant());
|
||||
var values = new List<string>();
|
||||
|
||||
foreach (UnicodeFoldSource source in sources)
|
||||
{
|
||||
var parameters = new DynamicParameters();
|
||||
parameters.Add("Pattern", pattern);
|
||||
parameters.Add("Limit", limit);
|
||||
if (source.Parameters is not null)
|
||||
{
|
||||
foreach ((string key, object value) in source.Parameters)
|
||||
{
|
||||
parameters.Add(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
IEnumerable<string> rows = await dbContext.Connection.QueryAsync<string>(
|
||||
new CommandDefinition(
|
||||
UnicodeFoldSql(source.Table, source.Column, source.Predicate),
|
||||
parameters,
|
||||
cancellationToken: cancellationToken));
|
||||
|
||||
values.AddRange(rows.Where(v => !string.IsNullOrEmpty(v)));
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
private sealed record UnicodeFoldSource(
|
||||
string Table,
|
||||
string Column,
|
||||
string Predicate = null,
|
||||
IReadOnlyDictionary<string, object> Parameters = null);
|
||||
|
||||
/// <summary>
|
||||
/// Maps a field name onto the <c>SongMetadata</c> column that backs it as an <c>IList<string></c>.
|
||||
/// The returned value is a compile-time constant from this switch — never caller input — so it is safe
|
||||
/// to interpolate into the SQL in <see cref="ListValuedSql" />.
|
||||
/// </summary>
|
||||
private static string GetSongListValuedColumn(string name) => name switch
|
||||
{
|
||||
"artist" => "Artists",
|
||||
"album_artist" => "AlbumArtists",
|
||||
_ => null
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Reads whole values out of a <c>SongMetadata</c> <c>IList<string></c> column.
|
||||
/// <para>
|
||||
/// EF maps these as primitive collections: one JSON array per row in a single <c>TEXT</c>/
|
||||
/// <c>longtext</c> column. Neither provider can project the elements server-side — SQLite needs
|
||||
/// the SQL <c>APPLY</c> operator it doesn't have, and Pomelo MySQL doesn't implement primitive
|
||||
/// collections at all — so there is no server-side <c>SELECT DISTINCT</c> over the elements.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// So the rows are walked in primary-key order, keyset-paged by row position, and split +
|
||||
/// exact-filtered in memory. All selectivity is in memory — the query's only condition is the
|
||||
/// cursor, a seek on the ordering key that never discards a row, so its <c>LIMIT</c> bounds the
|
||||
/// LOGICAL ROWS returned. See <see cref="ListValuedBatchRows" /> for the four revisions it took to
|
||||
/// get that right, and for what that bound does and does not cover.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private static async Task<List<string>> GetSongListValuedValues(
|
||||
TvContext dbContext,
|
||||
string column,
|
||||
string query,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string sql = ListValuedSql(column);
|
||||
|
||||
var distinct = new System.Collections.Generic.HashSet<string>(StringComparer.Ordinal);
|
||||
var afterId = 0;
|
||||
var read = 0;
|
||||
|
||||
while (read < ListValuedMaxRowsRead && distinct.Count < limit)
|
||||
{
|
||||
int batch = Math.Min(ListValuedBatchRows, ListValuedMaxRowsRead - read);
|
||||
|
||||
List<ListValuedRow> rows = (await dbContext.Connection.QueryAsync<ListValuedRow>(
|
||||
new CommandDefinition(
|
||||
sql,
|
||||
new { AfterId = afterId, Batch = batch },
|
||||
cancellationToken: cancellationToken))).AsList();
|
||||
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
read += rows.Count;
|
||||
afterId = rows[^1].Id;
|
||||
|
||||
foreach (ListValuedRow row in rows)
|
||||
{
|
||||
foreach (string element in ParseElements(row.Payload))
|
||||
{
|
||||
if (element.StartsWith(query, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
distinct.Add(element);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rows.Count < batch)
|
||||
{
|
||||
// With no RESIDUAL predicate -- only the cursor, which selects a range rather than discarding
|
||||
// rows from it -- a short page can only mean the table is exhausted. It can never mean "this
|
||||
// stretch happened to match nothing", which is precisely why the residual predicate had to go.
|
||||
// Advancing from the last returned Id is safe for the same reason: nothing was filtered out
|
||||
// behind it, so no row can be skipped.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return distinct.ToList();
|
||||
}
|
||||
|
||||
private static IEnumerable<string> ParseElements(string payload)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(payload))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return (JsonSerializer.Deserialize<string[]>(payload) ?? []).Where(e => !string.IsNullOrEmpty(e));
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One keyset page of rows, by ROW POSITION rather than by <c>Id</c> value.
|
||||
/// <para>
|
||||
/// The only condition is the cursor — deliberately <b>no RESIDUAL predicate</b>: no <c>LIKE</c>, no
|
||||
/// <c>LOWER</c>, not even <c>IS NOT NULL</c>. The distinction that matters is not "no predicate"
|
||||
/// (the cursor is one); it is that <c>Id > @AfterId</c> is a <i>seekable predicate on the
|
||||
/// ordering key</i>, which positions the scan and never discards a row, whereas a residual
|
||||
/// predicate throws away rows the engine already produced. <c>LIMIT</c> only truncates what
|
||||
/// survives a residual predicate, so with one present it bounds the output rather than the row
|
||||
/// count — which is how every earlier revision scanned past its own bound. With none, <c>LIMIT n</c>
|
||||
/// yields <c>n</c> logical rows. Null payloads are dropped in memory by
|
||||
/// <see cref="ParseElements" />.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Note this pins the SQL string only. It cannot pin an execution plan, MVCC visibility work, or
|
||||
/// payload I/O — and on MySQL, using the index to satisfy <c>ORDER BY</c> is an optimizer choice,
|
||||
/// not a semantic guarantee.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static string ListValuedSql(string column) =>
|
||||
$"SELECT Id, {column} AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch";
|
||||
|
||||
private sealed class ListValuedRow
|
||||
{
|
||||
public int Id { get; init; }
|
||||
|
||||
public string Payload { get; init; }
|
||||
}
|
||||
|
||||
private static async Task<List<string>> GetContentRatingValues(
|
||||
TvContext dbContext,
|
||||
string qLower,
|
||||
string query,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -108,13 +513,22 @@ public class GetSearchFieldValuesHandler(IDbContextFactory<TvContext> dbContextF
|
||||
.Where(cr => !string.IsNullOrEmpty(cr))
|
||||
.Distinct();
|
||||
|
||||
return FilterSortTake(split, qLower, limit);
|
||||
return FilterSortTake(split, query, limit);
|
||||
}
|
||||
|
||||
private static List<string> FilterSortTake(IEnumerable<string> values, string qLower, int limit) =>
|
||||
/// <summary>
|
||||
/// The one in-memory filter/sort/take every field funnels through. Both the comparison and the ordering
|
||||
/// are ORDINAL on purpose: <c>UseRequestLocalization</c> honours <c>Accept-Language</c>, so the current
|
||||
/// culture is caller-controlled, and <c>ToLower()</c> plus the default (linguistic)
|
||||
/// <c>StartsWith(string)</c> would make the result depend on it — under <c>tr-TR</c>, <c>q=I</c> lowers
|
||||
/// to <c>ı</c> and stops matching <c>Istanbul</c>. Note this is the LAST stage only: a field sourced by
|
||||
/// a plain EF query has already been filtered and truncated by the database collation before it gets
|
||||
/// here, which ordinal semantics downstream cannot undo (ersatztv#668).
|
||||
/// </summary>
|
||||
private static List<string> FilterSortTake(IEnumerable<string> values, string query, int limit) =>
|
||||
values
|
||||
.Where(v => v.ToLower().StartsWith(qLower))
|
||||
.OrderBy(v => v)
|
||||
.Where(v => v.StartsWith(query, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(v => v, StringComparer.Ordinal)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Core.FFmpeg;
|
||||
using ErsatzTV.Core.Interfaces.FFmpeg;
|
||||
using ErsatzTV.Core.Interfaces.Images;
|
||||
using ErsatzTV.Core.Interfaces.Metadata;
|
||||
using ErsatzTV.FFmpeg.State;
|
||||
using NSubstitute;
|
||||
using NUnit.Framework;
|
||||
using Shouldly;
|
||||
|
||||
namespace ErsatzTV.Core.Tests.FFmpeg;
|
||||
|
||||
[TestFixture]
|
||||
public class SongVideoGeneratorTests
|
||||
{
|
||||
private ITempFilePool _tempFilePool;
|
||||
private IImageCache _imageCache;
|
||||
private IFFmpegProcessService _ffmpegProcessService;
|
||||
private ILocalFileSystem _localFileSystem;
|
||||
private SongVideoGenerator _songVideoGenerator;
|
||||
private string _tempSubtitleFile;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_tempSubtitleFile = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.ass");
|
||||
|
||||
_tempFilePool = Substitute.For<ITempFilePool>();
|
||||
_tempFilePool.GetNextTempFile(Arg.Any<TempFileCategory>()).Returns(_tempSubtitleFile);
|
||||
|
||||
_imageCache = Substitute.For<IImageCache>();
|
||||
_imageCache.GetPathForImage(Arg.Any<string>(), Arg.Any<ArtworkKind>(), Arg.Any<Option<int>>())
|
||||
.Returns("/fake/watermark.png");
|
||||
|
||||
_ffmpegProcessService = Substitute.For<IFFmpegProcessService>();
|
||||
_ffmpegProcessService.GenerateSongImage(
|
||||
Arg.Any<string>(),
|
||||
Arg.Any<string>(),
|
||||
Arg.Any<Option<string>>(),
|
||||
Arg.Any<Channel>(),
|
||||
Arg.Any<MediaVersion>(),
|
||||
Arg.Any<string>(),
|
||||
Arg.Any<bool>(),
|
||||
Arg.Any<Option<string>>(),
|
||||
Arg.Any<WatermarkLocation>(),
|
||||
Arg.Any<int>(),
|
||||
Arg.Any<int>(),
|
||||
Arg.Any<int>(),
|
||||
Arg.Any<CancellationToken>())
|
||||
.Returns(Either<BaseError, string>.Right("/fake/song-image.png"));
|
||||
|
||||
_localFileSystem = Substitute.For<ILocalFileSystem>();
|
||||
_localFileSystem.GetCustomOrDefaultFile(Arg.Any<string>(), Arg.Any<string>())
|
||||
.Returns("/fake/background.png");
|
||||
|
||||
_songVideoGenerator = new SongVideoGenerator(
|
||||
_tempFilePool,
|
||||
_imageCache,
|
||||
_ffmpegProcessService,
|
||||
_localFileSystem);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
if (_tempSubtitleFile is not null && File.Exists(_tempSubtitleFile))
|
||||
{
|
||||
File.Delete(_tempSubtitleFile);
|
||||
}
|
||||
}
|
||||
|
||||
private static Channel BuildChannel()
|
||||
{
|
||||
var resolution = new Resolution { Width = 1920, Height = 1080 };
|
||||
FFmpegProfile ffmpegProfile = FFmpegProfile.New("test", resolution);
|
||||
|
||||
return new Channel(Guid.NewGuid())
|
||||
{
|
||||
Number = "1",
|
||||
Name = "Test Channel",
|
||||
FFmpegProfile = ffmpegProfile,
|
||||
SongVideoMode = ChannelSongVideoMode.Default
|
||||
};
|
||||
}
|
||||
|
||||
private static Song BuildUntaggedSong()
|
||||
{
|
||||
// an untagged song: FallbackMetadataProvider.GetSongMetadata never assigns
|
||||
// Artists/AlbumArtists, so they persist (and materialize) as null (ersatztv#691)
|
||||
var metadata = new SongMetadata
|
||||
{
|
||||
MetadataKind = MetadataKind.Fallback,
|
||||
Title = "Untagged Song",
|
||||
Artwork = [],
|
||||
Artists = null,
|
||||
AlbumArtists = null
|
||||
};
|
||||
|
||||
return new Song
|
||||
{
|
||||
SongMetadata = [metadata],
|
||||
MediaVersions = []
|
||||
};
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task GenerateSongVideo_should_not_throw_when_artists_and_album_artists_are_null()
|
||||
{
|
||||
Song song = BuildUntaggedSong();
|
||||
Channel channel = BuildChannel();
|
||||
|
||||
// SongVideoGenerator randomly picks between two rendering styles (and dereferences
|
||||
// metadata.Artists/AlbumArtists differently in each); loop enough times that both
|
||||
// branches -- including the AlbumArtists.Filter(... Artists.Contains ...) branch --
|
||||
// are exercised with overwhelming probability, so the null guard is proven on both.
|
||||
for (var i = 0; i < 25; i++)
|
||||
{
|
||||
Tuple<string, MediaVersion> result = await _songVideoGenerator.GenerateSongVideo(
|
||||
song,
|
||||
channel,
|
||||
"/usr/bin/ffmpeg",
|
||||
"/usr/bin/ffprobe",
|
||||
CancellationToken.None);
|
||||
|
||||
result.ShouldNotBeNull();
|
||||
result.Item1.ShouldBe("/fake/song-image.png");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
namespace ErsatzTV.Core.Domain;
|
||||
namespace ErsatzTV.Core.Domain;
|
||||
|
||||
public class SongMetadata : Metadata
|
||||
{
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Globalization;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Core.Interfaces.FFmpeg;
|
||||
@@ -85,6 +85,9 @@ public class SongVideoGenerator : ISongVideoGenerator
|
||||
|
||||
var sb = new StringBuilder();
|
||||
|
||||
List<string> artists = Optional(metadata.Artists).Flatten().ToList();
|
||||
List<string> albumArtists = Optional(metadata.AlbumArtists).Flatten().ToList();
|
||||
|
||||
if (detailsStyle)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(metadata.Title))
|
||||
@@ -92,17 +95,17 @@ public class SongVideoGenerator : ISongVideoGenerator
|
||||
sb.Append(CultureInfo.InvariantCulture, $"{{\\fs{largeFontSize}}}{metadata.Title}");
|
||||
}
|
||||
|
||||
if (metadata.Artists.Count > 0)
|
||||
if (artists.Count > 0)
|
||||
{
|
||||
var allArtists = string.Join(", ", metadata.Artists);
|
||||
var allArtists = string.Join(", ", artists);
|
||||
sb.Append(CultureInfo.InvariantCulture, $"\\N{{\\fs{fontSize}}}{allArtists}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (metadata.Artists.Count > 0)
|
||||
if (artists.Count > 0)
|
||||
{
|
||||
var allArtists = string.Join(", ", metadata.Artists);
|
||||
var allArtists = string.Join(", ", artists);
|
||||
sb.Append(allArtists);
|
||||
}
|
||||
|
||||
@@ -111,11 +114,11 @@ public class SongVideoGenerator : ISongVideoGenerator
|
||||
sb.Append(CultureInfo.InvariantCulture, $"\\N\"{metadata.Title}\"");
|
||||
}
|
||||
|
||||
if (metadata.AlbumArtists.Count > 0)
|
||||
if (albumArtists.Count > 0)
|
||||
{
|
||||
var allAlbumArtists = string.Join(
|
||||
", ",
|
||||
metadata.AlbumArtists.Filter(aa => !metadata.Artists.Contains(aa)));
|
||||
albumArtists.Filter(aa => !artists.Contains(aa)));
|
||||
sb.Append(CultureInfo.InvariantCulture, $"\\N{allAlbumArtists}");
|
||||
}
|
||||
|
||||
|
||||
@@ -593,7 +593,97 @@ public class PipelineBuilderBaseTests
|
||||
command.ShouldNotContain("-readrate_initial_burst");
|
||||
}
|
||||
|
||||
private string BuildRealtimeCommand(IFFmpegCapabilities capabilities, bool stillImage = false)
|
||||
[Test]
|
||||
public void Realtime_Input_Should_Catch_Up_When_Option_Is_Supported()
|
||||
{
|
||||
string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities());
|
||||
|
||||
// -readrate paces an input off its furthest-behind stream, so a sparse stream sharing the
|
||||
// input pins throughput below realtime; catchup lets it recover (ersatztv#726). anchor on
|
||||
// the input path so this can't be satisfied by some other input carrying the option
|
||||
// this overlaps Bitmap_Subtitle_Burn_In_... by design: that one pins the #726 MECHANISM on a
|
||||
// bitmap pipeline, this one pins the plain no-subtitle shape plus the uniqueness guard below
|
||||
command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -readrate_catchup 6.0 -i /tmp/whatever.mkv");
|
||||
Regex.Matches(command, Regex.Escape("-readrate_catchup 6.0")).Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Realtime_Input_Should_Not_Catch_Up_A_Still_Image()
|
||||
{
|
||||
// mirrors the burst's still-image exclusion (ersatztv#350): the video input takes no readrate
|
||||
// at all, so catchup would only reach the separate audio input and run it ahead of a graph
|
||||
// that the realtime filter is already pacing. pinned so the divergence can't reappear silently
|
||||
string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities(), stillImage: true);
|
||||
|
||||
// the positive anchor keeps this from passing vacuously if the helper ever stops
|
||||
// producing a realtime audio input at all
|
||||
command.ShouldContain("-readrate 1.05");
|
||||
command.ShouldNotContain("-readrate_catchup");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Realtime_Input_Should_Not_Catch_Up_When_Option_Is_Unsupported()
|
||||
{
|
||||
// an older binary silently keeps today's behavior rather than failing to start
|
||||
string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities());
|
||||
|
||||
// the positive anchor keeps this from passing vacuously if the helper ever stops
|
||||
// producing a realtime input at all
|
||||
command.ShouldContain("-readrate 1.05");
|
||||
command.ShouldNotContain("-readrate_catchup");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Concat_Should_Never_Catch_Up()
|
||||
{
|
||||
// concat reads already-written segments from the running segmenter at a flat 1.0; it has no
|
||||
// sparse stream to lag on, and letting it catch up would gallop through the segments
|
||||
var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", new FrameSize(1920, 1080));
|
||||
|
||||
var builder = new SoftwarePipelineBuilder(
|
||||
new CatchupCapableFFmpegCapabilities(),
|
||||
HardwareAccelerationMode.None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
concatInputFile,
|
||||
Option<GraphicsEngineInput>.None,
|
||||
"",
|
||||
"",
|
||||
_logger);
|
||||
|
||||
FFmpegPipeline result = builder.Concat(concatInputFile, FFmpegState.Concat(false, "Some Channel"));
|
||||
|
||||
string command = PrintCommand(None, None, None, concatInputFile, None, result);
|
||||
|
||||
command.ShouldContain("-readrate 1.0");
|
||||
command.ShouldNotContain("-readrate_catchup");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Bitmap_Subtitle_Burn_In_Should_Catch_Up_On_The_Shared_Video_Input()
|
||||
{
|
||||
// THE #726 regression test. an embedded bitmap subtitle is read through the SAME -i as the
|
||||
// video (SubtitleInputFile carries the video's path and resolves to a stream specifier on
|
||||
// that input), and being sparse it drags that input's pacing down to ~0.53x realtime.
|
||||
// this must be built on a BITMAP subtitle: a text subtitle is fetched by the libass filter
|
||||
// outside the demuxer, so the same assertions would pass vacuously while the bug is present.
|
||||
string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities(), imageSubtitle: true);
|
||||
|
||||
// the mechanism itself: subtitle stream 2 resolves onto input 0 -- the VIDEO's input -- so it
|
||||
// is read through the throttled demuxer that catchup is being applied to. if the subtitle
|
||||
// ever moves to an input of its own this label changes and the test fails, which is the point
|
||||
command.ShouldContain("[0:0][0:2]overlay");
|
||||
|
||||
// ...so the catchup has to be on that input
|
||||
command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -readrate_catchup 6.0 -i /tmp/whatever.mkv");
|
||||
}
|
||||
|
||||
private string BuildRealtimeCommand(
|
||||
IFFmpegCapabilities capabilities,
|
||||
bool stillImage = false,
|
||||
bool imageSubtitle = false)
|
||||
{
|
||||
var videoInputFile = new VideoInputFile(
|
||||
"/tmp/whatever.mkv",
|
||||
@@ -676,13 +766,22 @@ public class PipelineBuilderBaseTests
|
||||
AudioFilter.None,
|
||||
Option<double>.None));
|
||||
|
||||
// an embedded bitmap subtitle carries the VIDEO's path, which is how it ends up sharing the
|
||||
// video's single throttled -i rather than getting one of its own (ersatztv#726)
|
||||
Option<SubtitleInputFile> subtitleInputFile = imageSubtitle
|
||||
? new SubtitleInputFile(
|
||||
"/tmp/whatever.mkv",
|
||||
new List<MediaStream> { new(2, "dvdsub", StreamKind.Subtitle) },
|
||||
SubtitleMethod.Burn)
|
||||
: Option<SubtitleInputFile>.None;
|
||||
|
||||
var builder = new SoftwarePipelineBuilder(
|
||||
capabilities,
|
||||
HardwareAccelerationMode.None,
|
||||
videoInputFile,
|
||||
audioInputFile,
|
||||
None,
|
||||
None,
|
||||
subtitleInputFile,
|
||||
None,
|
||||
Option<GraphicsEngineInput>.None,
|
||||
"",
|
||||
@@ -735,4 +834,19 @@ public class PipelineBuilderBaseTests
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string> { FFmpegKnownOption.ReadrateInitialBurst.Name },
|
||||
new System.Collections.Generic.HashSet<string>());
|
||||
|
||||
// a binary new enough for -readrate_catchup also has -readrate_initial_burst, so this models a
|
||||
// real ffmpeg rather than an impossible catchup-without-burst one
|
||||
public class CatchupCapableFFmpegCapabilities() : FFmpegCapabilities(
|
||||
string.Empty,
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>
|
||||
{
|
||||
FFmpegKnownOption.ReadrateInitialBurst.Name,
|
||||
FFmpegKnownOption.ReadrateCatchup.Name
|
||||
},
|
||||
new System.Collections.Generic.HashSet<string>());
|
||||
}
|
||||
|
||||
@@ -13,8 +13,15 @@ public record FFmpegKnownOption
|
||||
// ffmpeg 6.1+; lets a readrate-throttled input read flat out for an initial window
|
||||
public static FFmpegKnownOption ReadrateInitialBurst => new("readrate_initial_burst");
|
||||
|
||||
// ffmpeg 8.0+ (added 2025-02-15 in 6232f416b, first released in 8.0); lets a readrate-throttled
|
||||
// input read faster than its readrate *while it is behind*, so a sparse stream sharing that
|
||||
// input cannot pin throughput below realtime (ersatztv#726). verified present in 8.1.2, the
|
||||
// pinned base image — note this is NEWER than 7.1, so it is detected at runtime, never assumed
|
||||
public static FFmpegKnownOption ReadrateCatchup => new("readrate_catchup");
|
||||
|
||||
public static IList<string> AllOptions =>
|
||||
[
|
||||
ReadrateInitialBurst.Name
|
||||
ReadrateInitialBurst.Name,
|
||||
ReadrateCatchup.Name
|
||||
];
|
||||
}
|
||||
|
||||
@@ -3,10 +3,11 @@ using ErsatzTV.FFmpeg.Environment;
|
||||
|
||||
namespace ErsatzTV.FFmpeg.InputOption;
|
||||
|
||||
public class ReadrateInputOption(double readRate, Option<int> initialBurstSeconds) : IInputOption
|
||||
public class ReadrateInputOption(double readRate, Option<int> initialBurstSeconds, Option<double> catchupReadRate)
|
||||
: IInputOption
|
||||
{
|
||||
public ReadrateInputOption(double readRate)
|
||||
: this(readRate, Option<int>.None)
|
||||
: this(readRate, Option<int>.None, Option<double>.None)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -30,6 +31,17 @@ public class ReadrateInputOption(double readRate, Option<int> initialBurstSecond
|
||||
result.Add(burst.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
// -readrate paces the WHOLE input off its furthest-behind stream, so one sparse stream
|
||||
// (an embedded PGS/DVD bitmap subtitle feeding the overlay) drags the video down with it
|
||||
// and output collapses to ~0.53x realtime. catchup lets a lagging input read faster until
|
||||
// it is level again; it is a ceiling that only applies WHILE behind, never a target, so
|
||||
// caught-up input still paces at readRate and cannot race ahead (ersatztv#726)
|
||||
foreach (double catchup in catchupReadRate)
|
||||
{
|
||||
result.Add("-readrate_catchup");
|
||||
result.Add(catchup.ToString("0.0####", CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
return result.ToArray();
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,14 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
|
||||
// an operator who raises that setting above 2 gets less of the benefit (ersatztv#350)
|
||||
private const int InitialBurstSeconds = OutputFormatHls.SegmentSeconds * 2;
|
||||
|
||||
// how fast a LAGGING realtime input may read until it is level again. measured on the #726
|
||||
// repro (embedded dvd_subtitle -> overlay, QSV encode): 1.05 alone sustains 0.53x, catchup 2.0
|
||||
// reaches 0.711x, and 6.0 restores the full 1.067x that the same pipeline achieves with no
|
||||
// subtitle at all. 20.0 also measures 1.067x — i.e. the value is not a throughput dial above
|
||||
// the point where the input catches up, so 6.0 is chosen as the smallest measured-sufficient
|
||||
// ceiling rather than the largest that works (ersatztv#726)
|
||||
private const double CatchupReadRate = 6.0;
|
||||
|
||||
private readonly Option<AudioInputFile> _audioInputFile;
|
||||
private readonly Option<ConcatInputFile> _concatInputFile;
|
||||
private readonly IFFmpegCapabilities _ffmpegCapabilities;
|
||||
@@ -871,8 +879,26 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
|
||||
? InitialBurstSeconds
|
||||
: Option<int>.None;
|
||||
|
||||
_audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds)));
|
||||
videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds));
|
||||
// -readrate paces an input off its furthest-behind stream. an embedded bitmap subtitle is
|
||||
// read through the SAME -i as the video (its SubtitleInputFile carries the video's path and
|
||||
// resolves to a stream specifier on that input), and being sparse it falls further behind
|
||||
// every second, dragging video throughput to ~0.53x — well under the 1.0x a live client
|
||||
// consumes at. catchup lets the lagging input recover instead of pinning the whole process.
|
||||
// applied to every realtime input, not just subtitle pipelines: it is inert unless an input
|
||||
// is actually behind, and any sparse stream can cause this (ersatztv#726).
|
||||
//
|
||||
// a still image is excluded for the SAME reason the burst above excludes it: its video input
|
||||
// takes no readrate at all, so this would reach only the separate audio input and let it run
|
||||
// ahead of the video, which is exactly what #350 declined. for a non-still-image item both
|
||||
// inputs carry identical options, so the symmetry is preserved there. and an image-based
|
||||
// subtitle always rides the video path, so this shape cannot suffer the starvation anyway
|
||||
Option<double> catchupReadRate =
|
||||
!isStillImage && _ffmpegCapabilities.HasOption(FFmpegKnownOption.ReadrateCatchup)
|
||||
? CatchupReadRate
|
||||
: Option<double>.None;
|
||||
|
||||
_audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds, catchupReadRate)));
|
||||
videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds, catchupReadRate));
|
||||
}
|
||||
|
||||
protected static void SetStillImageLoop(
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
using System.Data;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace ErsatzTV.Infrastructure.Sqlite.Data;
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. SQLite's built-in <c>lower()</c>/<c>upper()</c> fold ASCII ONLY — <c>lower('Édith')</c>
|
||||
/// returns <c>'Édith'</c> unchanged — so a facet value whose prefix carries an uppercase non-ASCII
|
||||
/// character can never be matched by the prefix predicate the facet-value endpoint emits. Registering a
|
||||
/// managed scalar gives that one query a Unicode-correct fold. Wired to
|
||||
/// <see cref="ErsatzTV.Infrastructure.Data.TvContext.RegisterUnicodeCaseFunctions" /> at startup.
|
||||
/// </summary>
|
||||
public static class SqliteUnicodeFunctions
|
||||
{
|
||||
/// <summary>
|
||||
/// SQL name of the invariant-uppercase fold. The facet-value handler interpolates this constant into
|
||||
/// its SQL, so the two cannot drift apart.
|
||||
/// </summary>
|
||||
public const string UpperInvariantFunction = "etv_upper";
|
||||
|
||||
/// <summary>
|
||||
/// Registers <see cref="UpperInvariantFunction" /> on <paramref name="connection" /> when it is a
|
||||
/// SQLite connection, and does nothing otherwise. Idempotent — a repeat registration replaces the
|
||||
/// previous delegate with an identical one — so the single call site may call it unconditionally.
|
||||
/// <para>
|
||||
/// The property this fold has to satisfy is ONE-SIDED: the SQL stage may over-match freely,
|
||||
/// because the endpoint applies an exact <see cref="StringComparison.OrdinalIgnoreCase" /> filter
|
||||
/// in memory afterwards, but it must never UNDER-match — no later stage can reintroduce a row SQL
|
||||
/// never returned. <see cref="string.ToUpperInvariant" /> satisfies it because
|
||||
/// <c>OrdinalIgnoreCase</c> equality is a strict SUBSET of invariant-uppercase equality, so
|
||||
/// folding both sides with it yields a superset of the final filter's matches.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Do not restate that as "<c>OrdinalIgnoreCase</c> IS invariant-uppercase-then-ordinal" — it is
|
||||
/// not, and the difference is measurable: <c>char.ToUpperInvariant('ſ')</c> (U+017F) is <c>'S'</c>,
|
||||
/// yet <c>"ſweet".StartsWith("S", OrdinalIgnoreCase)</c> is <b>false</b>. That gap is precisely
|
||||
/// the harmless direction — SQL returns the row, the in-memory filter drops it. The containment,
|
||||
/// not any identity of the two foldings, is what makes this safe.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Registration is per-connection and therefore done at the one call site that uses the function,
|
||||
/// not through an EF connection interceptor: Dapper opens a closed connection itself, and a direct
|
||||
/// ADO open does not raise EF's interceptors — so an interceptor-based seam would silently miss
|
||||
/// exactly the query that needs it.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static void Register(IDbConnection connection)
|
||||
{
|
||||
if (connection is SqliteConnection sqlite)
|
||||
{
|
||||
sqlite.CreateFunction(
|
||||
UpperInvariantFunction,
|
||||
(string? value) => value?.ToUpperInvariant(),
|
||||
isDeterministic: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1144,7 +1144,7 @@ public class MediaCollectionRepository : IMediaCollectionRepository
|
||||
|
||||
var allArtists = items.OfType<Song>()
|
||||
.SelectMany(s => s.SongMetadata)
|
||||
.Map(sm => sm.AlbumArtists.HeadOrNone().Match(aa => aa, string.Empty))
|
||||
.Map(sm => Optional(sm.AlbumArtists).Flatten().HeadOrNone().Match(aa => aa, string.Empty))
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
@@ -1157,7 +1157,7 @@ public class MediaCollectionRepository : IMediaCollectionRepository
|
||||
foreach (Song song in items.OfType<Song>())
|
||||
{
|
||||
string firstArtist = song.SongMetadata
|
||||
.SelectMany(sm => sm.AlbumArtists)
|
||||
.SelectMany(sm => Optional(sm.AlbumArtists).Flatten())
|
||||
.HeadOrNone()
|
||||
.Match(aa => aa, string.Empty);
|
||||
|
||||
|
||||
@@ -36,6 +36,18 @@ public class TvContext : DbContext
|
||||
/// </summary>
|
||||
public static Func<DbUpdateException, bool> IsUniqueConstraintViolation { get; set; } = static _ => false;
|
||||
|
||||
/// <summary>
|
||||
/// Registers provider-specific SQL scalar functions on a connection, called immediately before a raw
|
||||
/// query that needs them. Set at startup by the active provider's wiring, mirroring
|
||||
/// <see cref="IsUniqueConstraintViolation" />: SQLite points this at
|
||||
/// <c>SqliteUnicodeFunctions.Register</c>, MySQL leaves it a no-op because its own <c>LOWER()</c> is
|
||||
/// already Unicode-aware and needs no help. Defaults to a no-op, which is safe because the sole
|
||||
/// caller invokes it only on the SQLite branch that requires it, and an unwired provider then fails
|
||||
/// LOUDLY ("no such function: etv_upper") rather than returning silently wrong results. See
|
||||
/// ersatztv#668.
|
||||
/// </summary>
|
||||
public static Action<IDbConnection> RegisterUnicodeCaseFunctions { get; set; } = static _ => { };
|
||||
|
||||
public IDbConnection Connection => Database.GetDbConnection();
|
||||
|
||||
public DbSet<ConfigElement> ConfigElements { get; set; }
|
||||
|
||||
@@ -21,4 +21,16 @@
|
||||
<ProjectReference Include="..\ErsatzTV.Mcp\ErsatzTV.Mcp.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<!--
|
||||
The generated OpenAPI document is the wire contract the MCP catalog wraps. Copying it into the
|
||||
test output lets ToolCatalogTests assert that every write tool declares exactly the request-body
|
||||
fields its endpoint accepts, so a new DTO property cannot drift out of a tool schema unnoticed
|
||||
(issue #754). Regenerated by scripts/update-openapi.sh.
|
||||
-->
|
||||
<ItemGroup>
|
||||
<Content Include="..\ErsatzTV\wwwroot\openapi\v1.json"
|
||||
Link="openapi\v1.json"
|
||||
CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -77,9 +77,22 @@ public class ToolCatalogTests
|
||||
// Enums must NOT be forced required (they have server-side defaults).
|
||||
createRequired.ShouldNotContain("streamingMode");
|
||||
|
||||
// Update carries the same body fields plus the route id.
|
||||
update.InputSchema.RootElement.GetProperty("properties").TryGetProperty("id", out _).ShouldBeTrue();
|
||||
update.InputSchema.RootElement.GetProperty("properties").TryGetProperty("showInEpg", out _).ShouldBeTrue();
|
||||
// Update carries the create body fields plus the route id...
|
||||
JsonElement updateProps = update.InputSchema.RootElement.GetProperty("properties");
|
||||
updateProps.TryGetProperty("id", out _).ShouldBeTrue();
|
||||
updateProps.TryGetProperty("showInEpg", out _).ShouldBeTrue();
|
||||
|
||||
// ...plus graphicsElementIds, which is on UpdateChannelRequest only. PUT is a full replace, so
|
||||
// while the tool could not express this field an agent following the tool's own "send the full
|
||||
// desired state" instruction silently detached every graphics element (issue #754).
|
||||
updateProps.TryGetProperty("graphicsElementIds", out JsonElement graphicsElementIds).ShouldBeTrue();
|
||||
graphicsElementIds.GetProperty("type").GetString().ShouldBe("array");
|
||||
graphicsElementIds.GetProperty("items").GetProperty("type").GetString().ShouldBe("integer");
|
||||
|
||||
// Create must NOT send it: CreateChannelRequest has no such property, and the tool schema is
|
||||
// additionalProperties:false. This is why it is declared on the update tool rather than in the
|
||||
// shared ChannelFields().
|
||||
createProps.TryGetProperty("graphicsElementIds", out _).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Test]
|
||||
@@ -256,4 +269,248 @@ public class ToolCatalogTests
|
||||
tool.QueryParameters.ShouldNotBeNull();
|
||||
tool.QueryParameters!.ShouldContain("deep");
|
||||
}
|
||||
|
||||
// #754: ToolCatalog declared 27 of UpdateChannelRequest's 28 properties. The missing one was
|
||||
// graphicsElementIds, and because PUT /api/v1/channels/{id} is a FULL REPLACE the omission was not
|
||||
// merely "one field you cannot set" — an agent that GET-edit-PUT the channel, exactly as the tool's
|
||||
// description tells it to, detached every graphics element (including the On Now/Next overlay) with
|
||||
// a 200 and no error. The same shape was live on ersatztv_update_schedule, which omitted
|
||||
// padToNearestMinute and silently cleared a configured pad.
|
||||
//
|
||||
// Neither is fixable by counting fields once: the defect is that nothing tied the tool schema to the
|
||||
// contract it wraps. So this test asserts the tie for EVERY write tool against the generated OpenAPI
|
||||
// document (the actual wire contract, linked into the test output by the csproj). A new property on
|
||||
// any request DTO now fails here until the catalog declares it.
|
||||
[Test]
|
||||
public void Every_Write_Tool_Should_Declare_Exactly_Its_OpenApi_Request_Body_Fields()
|
||||
{
|
||||
using JsonDocument spec = LoadOpenApiDocument();
|
||||
JsonElement paths = spec.RootElement.GetProperty("paths");
|
||||
|
||||
ToolDefinition[] writeTools = ToolCatalog.All
|
||||
.Where(t => t.HttpMethod == HttpMethod.Post
|
||||
|| t.HttpMethod == HttpMethod.Put
|
||||
|| t.HttpMethod == HttpMethod.Patch)
|
||||
.ToArray();
|
||||
|
||||
// Pin the covered set rather than trusting the filter. A tool that stopped being a write verb,
|
||||
// or a new write tool, must show up as a change here — a bare loop over a filtered set passes
|
||||
// just as happily when the set silently shrinks to nothing.
|
||||
string[] expectedWriteTools =
|
||||
[
|
||||
"ersatztv_add_collection_items",
|
||||
"ersatztv_create_channel",
|
||||
"ersatztv_create_collection",
|
||||
"ersatztv_create_playout",
|
||||
"ersatztv_create_schedule",
|
||||
"ersatztv_create_smart_collection",
|
||||
"ersatztv_enable_jellyfin_library_sync",
|
||||
"ersatztv_refresh_jellyfin_libraries",
|
||||
"ersatztv_reset_channel_playout",
|
||||
"ersatztv_scan_jellyfin_collections",
|
||||
"ersatztv_scan_library",
|
||||
"ersatztv_update_channel",
|
||||
"ersatztv_update_collection",
|
||||
"ersatztv_update_collection_custom_order",
|
||||
"ersatztv_update_playout",
|
||||
"ersatztv_update_schedule",
|
||||
"ersatztv_update_smart_collection"
|
||||
];
|
||||
|
||||
writeTools.Select(t => t.Name).OrderBy(n => n, StringComparer.Ordinal)
|
||||
.ShouldBe(expectedWriteTools.OrderBy(n => n, StringComparer.Ordinal));
|
||||
|
||||
foreach (ToolDefinition tool in writeTools)
|
||||
{
|
||||
paths.TryGetProperty(tool.PathTemplate, out JsonElement pathItem)
|
||||
.ShouldBeTrue($"{tool.Name}: {tool.PathTemplate} is not in the OpenAPI document");
|
||||
|
||||
string verb = tool.HttpMethod.Method.ToLowerInvariant();
|
||||
pathItem.TryGetProperty(verb, out JsonElement operation)
|
||||
.ShouldBeTrue($"{tool.Name}: {verb.ToUpperInvariant()} {tool.PathTemplate} is not in the OpenAPI document");
|
||||
|
||||
Dictionary<string, string> declared = DeclaredBodyArguments(tool);
|
||||
Dictionary<string, string> accepted = RequestBodyProperties(spec, operation, tool.Name);
|
||||
|
||||
// Compare name AND type. Names alone would let a field drift to the wrong JSON type: the
|
||||
// tool would advertise "string" for an int?, the agent would send "30", and the API would
|
||||
// 400 — green test, broken tool.
|
||||
declared.Select(p => $"{p.Key}: {p.Value}").OrderBy(s => s, StringComparer.Ordinal)
|
||||
.ShouldBe(
|
||||
accepted.Select(p => $"{p.Key}: {p.Value}").OrderBy(s => s, StringComparer.Ordinal),
|
||||
customMessage:
|
||||
$"{tool.Name} declares body fields that do not match {verb.ToUpperInvariant()} {tool.PathTemplate}. "
|
||||
+ "A field the endpoint accepts but the tool omits is silently dropped on a full-replace "
|
||||
+ "write (#754); a field the tool sends but the endpoint does not accept is rejected; "
|
||||
+ "a field declared with the wrong type is rejected at the API.");
|
||||
}
|
||||
}
|
||||
|
||||
// #757, the sibling of the body guard above. Query parameters drift the same way and are WORSE for
|
||||
// reads: ToolArgumentValidator rejects undeclared arguments, so a parameter the tool omits is not
|
||||
// merely undocumented, it is unreachable — the caller cannot pass it at all. That is how #616's
|
||||
// paging omission hard-capped two tools at the first page. This covers EVERY tool, not just the
|
||||
// write verbs, because the drift that existed when this was written was entirely on reads.
|
||||
[Test]
|
||||
public void Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters()
|
||||
{
|
||||
using JsonDocument spec = LoadOpenApiDocument();
|
||||
JsonElement paths = spec.RootElement.GetProperty("paths");
|
||||
|
||||
// Every tool is covered, so an emptiness guard is enough here — there is no filter to escape.
|
||||
ToolCatalog.All.Count.ShouldBeGreaterThan(30);
|
||||
|
||||
// Accumulate rather than throwing on the first mismatch, so one run reports the WHOLE drift set.
|
||||
// Failing fast here would hand back one tool at a time and invite fixing them one at a time,
|
||||
// which is how the #754 twin stayed hidden in the first place.
|
||||
List<string> drift = [];
|
||||
|
||||
foreach (ToolDefinition tool in ToolCatalog.All)
|
||||
{
|
||||
paths.TryGetProperty(tool.PathTemplate, out JsonElement pathItem)
|
||||
.ShouldBeTrue($"{tool.Name}: {tool.PathTemplate} is not in the OpenAPI document");
|
||||
|
||||
string verb = tool.HttpMethod.Method.ToLowerInvariant();
|
||||
pathItem.TryGetProperty(verb, out JsonElement operation)
|
||||
.ShouldBeTrue($"{tool.Name}: {verb.ToUpperInvariant()} {tool.PathTemplate} is not in the OpenAPI document");
|
||||
|
||||
IReadOnlySet<string> declared = tool.QueryParameters ?? new HashSet<string>(StringComparer.Ordinal);
|
||||
HashSet<string> accepted = QueryParameterNames(operation);
|
||||
|
||||
string[] missing = accepted.Except(declared, StringComparer.Ordinal).OrderBy(n => n, StringComparer.Ordinal).ToArray();
|
||||
string[] phantom = declared.Except(accepted, StringComparer.Ordinal).OrderBy(n => n, StringComparer.Ordinal).ToArray();
|
||||
|
||||
if (missing.Length > 0 || phantom.Length > 0)
|
||||
{
|
||||
drift.Add(
|
||||
$"{tool.Name} ({verb.ToUpperInvariant()} {tool.PathTemplate}): "
|
||||
+ $"unreachable={string.Join(",", missing)} phantom={string.Join(",", phantom)}");
|
||||
}
|
||||
}
|
||||
|
||||
// A parameter the endpoint accepts but the tool omits is UNREACHABLE, not merely undocumented:
|
||||
// ToolArgumentValidator rejects undeclared arguments, so the caller cannot pass it at all
|
||||
// (#616 hard-capped two paged tools exactly this way). A phantom is the reverse — the tool
|
||||
// advertises something the endpoint ignores.
|
||||
drift.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
private static HashSet<string> QueryParameterNames(JsonElement operation)
|
||||
{
|
||||
if (!operation.TryGetProperty("parameters", out JsonElement parameters))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return parameters.EnumerateArray()
|
||||
.Where(p => p.TryGetProperty("in", out JsonElement location)
|
||||
&& string.Equals(location.GetString(), "query", StringComparison.Ordinal))
|
||||
.Select(p => p.GetProperty("name").GetString())
|
||||
.OfType<string>()
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
// The body is every declared argument that is not routed elsewhere — mirroring exactly how
|
||||
// ErsatzTvApiClient builds the request, so this test cannot disagree with the code it guards.
|
||||
// DELETE is not compared: ErsatzTvApiClient sets hasBody for POST/PUT/PATCH only, so a body
|
||||
// argument on a DELETE tool would be silently dropped. No DELETE tool has one today.
|
||||
private static Dictionary<string, string> DeclaredBodyArguments(ToolDefinition tool)
|
||||
{
|
||||
var pathParameters = Regex.Matches(tool.PathTemplate, @"\{([^}]+)\}")
|
||||
.Select(m => m.Groups[1].Value)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
IReadOnlySet<string> queryParameters = tool.QueryParameters ?? new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
if (!tool.InputSchema.RootElement.TryGetProperty("properties", out JsonElement properties))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return properties.EnumerateObject()
|
||||
.Where(p => !pathParameters.Contains(p.Name)
|
||||
&& !queryParameters.Contains(p.Name)
|
||||
&& !string.Equals(p.Name, "ifMatch", StringComparison.Ordinal))
|
||||
.ToDictionary(p => p.Name, p => DeclaredType(p.Value), StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
// The tool schema's own shape: a plain "type", plus the array element type where there is one.
|
||||
private static string DeclaredType(JsonElement property)
|
||||
{
|
||||
string type = property.GetProperty("type").GetString().ShouldNotBeNull();
|
||||
|
||||
return type == "array" && property.TryGetProperty("items", out JsonElement items)
|
||||
? $"array<{items.GetProperty("type").GetString()}>"
|
||||
: type;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> RequestBodyProperties(JsonDocument spec, JsonElement operation, string toolName)
|
||||
{
|
||||
// No request body at all (queue/scan POSTs) — the tool must send none either.
|
||||
if (!operation.TryGetProperty("requestBody", out JsonElement requestBody))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
JsonElement schema = requestBody
|
||||
.GetProperty("content")
|
||||
.GetProperty("application/json")
|
||||
.GetProperty("schema");
|
||||
|
||||
// Every request body in this document is a plain $ref to a component schema. Anything else
|
||||
// (allOf/inline/oneOf) is a contract shape this guard has not been taught to read, so fail
|
||||
// loudly rather than comparing against an empty set and reporting a false pass.
|
||||
schema.TryGetProperty("$ref", out JsonElement reference)
|
||||
.ShouldBeTrue($"{toolName}: request body schema is not a $ref; teach this test the new shape");
|
||||
|
||||
JsonElement schemas = spec.RootElement.GetProperty("components").GetProperty("schemas");
|
||||
string componentName = reference.GetString().ShouldNotBeNull().Split('/')[^1];
|
||||
|
||||
return schemas
|
||||
.GetProperty(componentName)
|
||||
.GetProperty("properties")
|
||||
.EnumerateObject()
|
||||
.ToDictionary(p => p.Name, p => SpecType(schemas, p.Value, toolName, p.Name), StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
// Normalize the generator's shapes onto the catalog's vocabulary. Two forms appear in this
|
||||
// document: a nullable type as ["null", T] (the catalog has no nullable notion — optionality is
|
||||
// carried by `required`), and a $ref to a component, which for the enum fields is a string enum
|
||||
// and for `logo` is an object.
|
||||
private static string SpecType(JsonElement schemas, JsonElement property, string toolName, string fieldName)
|
||||
{
|
||||
if (property.TryGetProperty("$ref", out JsonElement reference))
|
||||
{
|
||||
string componentName = reference.GetString().ShouldNotBeNull().Split('/')[^1];
|
||||
return SpecType(schemas, schemas.GetProperty(componentName), toolName, fieldName);
|
||||
}
|
||||
|
||||
JsonElement type = property.GetProperty("type");
|
||||
|
||||
string[] types = type.ValueKind == JsonValueKind.Array
|
||||
? type.EnumerateArray().Select(t => t.GetString()).OfType<string>().Where(t => t != "null").ToArray()
|
||||
: [type.GetString().ShouldNotBeNull()];
|
||||
|
||||
// More than one non-null type is a shape this guard has not been taught to read; fail rather
|
||||
// than picking one and reporting a comparison that means nothing.
|
||||
types.Length.ShouldBe(1, $"{toolName}.{fieldName}: unexpected OpenAPI type union [{string.Join(", ", types)}]");
|
||||
|
||||
// The element schema is resolved through the same normalization: an array's items can itself be
|
||||
// a $ref to a component (ReplaceRemoteLibraryPreferencesRequest.libraries), which the catalog
|
||||
// declares as an object array.
|
||||
return types[0] == "array" && property.TryGetProperty("items", out JsonElement items)
|
||||
? $"array<{SpecType(schemas, items, toolName, fieldName)}>"
|
||||
: types[0];
|
||||
}
|
||||
|
||||
private static JsonDocument LoadOpenApiDocument()
|
||||
{
|
||||
string path = Path.Combine(AppContext.BaseDirectory, "openapi", "v1.json");
|
||||
|
||||
// A missing spec would make every assertion above vacuous, so it is an explicit failure.
|
||||
File.Exists(path).ShouldBeTrue(
|
||||
$"OpenAPI document not found at {path}; the test project links it from ErsatzTV/wwwroot/openapi/v1.json");
|
||||
|
||||
return JsonDocument.Parse(File.ReadAllText(path));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,14 +29,32 @@ public static class ToolCatalog
|
||||
Get("ersatztv_list_schedules", "List schedules.", "/api/v1/schedules"),
|
||||
Get("ersatztv_get_schedule", "Get a schedule by id.", "/api/v1/schedules/{id}", IdPath("Schedule id.")),
|
||||
Get("ersatztv_get_schedule_items", "Get a schedule's items. Emits the schedule ETag.", "/api/v1/schedules/{id}/items", IdPath("Schedule id.")),
|
||||
Get("ersatztv_list_playouts", "List playouts (paged).", "/api/v1/playouts", [], Page()),
|
||||
Get(
|
||||
"ersatztv_list_playouts",
|
||||
"List playouts (paged), optionally filtered by channel name.",
|
||||
"/api/v1/playouts",
|
||||
[],
|
||||
[
|
||||
Str(
|
||||
"query",
|
||||
"Optional case-insensitive substring match on the CHANNEL name (not the playout or schedule name); omit for all playouts.",
|
||||
arg: In.Query),
|
||||
.. Page()
|
||||
]),
|
||||
Get("ersatztv_get_playout", "Get a playout by id.", "/api/v1/playouts/{id}", IdPath("Playout id.")),
|
||||
Get(
|
||||
"ersatztv_get_playout_items",
|
||||
"Get upcoming items (and unscheduled gaps) for a playout (paged).",
|
||||
"/api/v1/playouts/{id}/items",
|
||||
[IdPath("Playout id.")],
|
||||
Page()),
|
||||
[
|
||||
Bool(
|
||||
"showFiller",
|
||||
"Include items whose filler kind is not None (pre/mid/post-roll, tail, fallback, guide-mode, deco); "
|
||||
+ "default false returns only non-filler items.",
|
||||
arg: In.Query),
|
||||
.. Page()
|
||||
]),
|
||||
Get("ersatztv_list_ffmpeg_profiles", "List FFmpeg profiles.", "/api/v1/ffmpeg/profiles"),
|
||||
Get("ersatztv_get_ffmpeg_profile", "Get an FFmpeg profile by id.", "/api/v1/ffmpeg/profiles/{id}", IdPath("FFmpeg profile id.")),
|
||||
Get(
|
||||
@@ -132,7 +150,8 @@ public static class ToolCatalog
|
||||
[Str("name", "Schedule name.", required: true), .. ScheduleFlags()]),
|
||||
Put(
|
||||
"ersatztv_update_schedule",
|
||||
"Update a program schedule's settings.",
|
||||
"Update a program schedule. Send the full desired state: every field is applied, so omitting "
|
||||
+ "padToNearestMinute CLEARS a configured pad (GET the schedule first to copy current values).",
|
||||
"/api/v1/schedules/{id}",
|
||||
[IdPath("Schedule id."), Str("name", "Schedule name.", required: true), .. ScheduleFlags()]),
|
||||
Delete("ersatztv_delete_schedule", "Delete a program schedule.", "/api/v1/schedules/{id}", IdPath("Schedule id.")),
|
||||
@@ -159,9 +178,22 @@ public static class ToolCatalog
|
||||
ChannelFields()),
|
||||
Put(
|
||||
"ersatztv_update_channel",
|
||||
"Update a channel. Send the full desired state; enum fields take the enum name (GET the channel first to copy current values).",
|
||||
"Update a channel. Send the full desired state; enum fields take the enum name (GET the channel first to copy current values). "
|
||||
+ "graphicsElementIds is part of that state: omitting it DETACHES every graphics element (e.g. the On Now/Next overlay), "
|
||||
+ "so copy it from ersatztv_get_channel unless you mean to clear it.",
|
||||
"/api/v1/channels/{id}",
|
||||
[IdPath("Channel id."), .. ChannelFields()]),
|
||||
[
|
||||
IdPath("Channel id."),
|
||||
.. ChannelFields(),
|
||||
|
||||
// Update-only: UpdateChannelRequest carries GraphicsElementIds, CreateChannelRequest does
|
||||
// not, so this cannot move into the shared ChannelFields() without making create send an
|
||||
// unknown property. PUT is a full replace, so omitting it detaches every attached element
|
||||
// with no error — issue #754.
|
||||
IntArray(
|
||||
"graphicsElementIds",
|
||||
"Ids of the graphics elements attached to the channel. Full replace: omit or send [] to detach all.")
|
||||
]),
|
||||
Post(
|
||||
"ersatztv_reset_channel_playout",
|
||||
"Queue a rebuild of a channel's playout (202 Accepted; 409 if a build is already running).",
|
||||
@@ -297,7 +329,12 @@ public static class ToolCatalog
|
||||
Bool("treatCollectionsAsShows", "Treat collections as shows."),
|
||||
Bool("shuffleScheduleItems", "Shuffle schedule items."),
|
||||
Bool("randomStartPoint", "Use a random start point."),
|
||||
Str("fixedStartTimeBehavior", "Fixed start-time behavior (enum name; GET a schedule to see valid values).")
|
||||
Str("fixedStartTimeBehavior", "Fixed start-time behavior (enum name; GET a schedule to see valid values)."),
|
||||
|
||||
// Both Create- and UpdateScheduleRequest carry this, so it belongs in the shared helper. The
|
||||
// update PUT is a full replace that writes the value unconditionally, so omitting it used to
|
||||
// clear a configured pad silently — the same #754 shape as channel graphicsElementIds.
|
||||
Int("padToNearestMinute", "Pad each item to the nearest N minutes; omit or send null for no padding.")
|
||||
];
|
||||
|
||||
// ---- Tool factories ----
|
||||
|
||||
@@ -162,6 +162,7 @@ public class Program
|
||||
TvContext.LastInsertedRowId = "last_insert_rowid()";
|
||||
TvContext.CaseInsensitiveCollation = "NOCASE";
|
||||
TvContext.IsUniqueConstraintViolation = SqliteErrorClassifier.IsUniqueConstraintViolation;
|
||||
TvContext.RegisterUnicodeCaseFunctions = SqliteUnicodeFunctions.Register;
|
||||
|
||||
SqlMapper.AddTypeHandler(new DateTimeOffsetHandler());
|
||||
SqlMapper.AddTypeHandler(new GuidHandler());
|
||||
@@ -173,6 +174,10 @@ public class Program
|
||||
TvContext.LastInsertedRowId = "last_insert_id()";
|
||||
TvContext.CaseInsensitiveCollation = "utf8mb4_general_ci";
|
||||
TvContext.IsUniqueConstraintViolation = MySqlErrorClassifier.IsUniqueConstraintViolation;
|
||||
|
||||
// MySQL's LOWER() is already Unicode-aware; assigned explicitly for the same reason as
|
||||
// the host — a provider switch must not inherit SQLite's registration.
|
||||
TvContext.RegisterUnicodeCaseFunctions = static _ => { };
|
||||
}
|
||||
|
||||
services.AddHttpClient();
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using ErsatzTV.Application.MediaCollections;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
using ErsatzTV.Tests.Support;
|
||||
using NUnit.Framework;
|
||||
using Shouldly;
|
||||
|
||||
namespace ErsatzTV.Tests.Application.MediaCollections;
|
||||
|
||||
/// <summary>
|
||||
/// The second consumer of the shared <c>ProjectMediaItemToViewModel</c> switch (issue #671).
|
||||
/// <c>GetPlaylistItemsHandler</c> had no handler-level test — the controller tests stub the mediator
|
||||
/// and never execute the query — so the only symptom of a missing include here was a silent "???"
|
||||
/// name that nothing in the suite could see. Widening the shared switch with a RemoteStream arm
|
||||
/// obliged this handler to gain a matching include; proving that by inspection would have repeated
|
||||
/// the very method that produced #671, so it gets the same full matrix the rerun handlers get.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public class GetPlaylistItemsHandlerTests : MediaCollectionHandlerTestBase
|
||||
{
|
||||
private static IEnumerable<CollectionType> SupportedSelectionTypes => SelectionSeedData.SupportedSelectionTypes;
|
||||
|
||||
[TestCaseSource(nameof(SupportedSelectionTypes))]
|
||||
public async Task GetPlaylistItems_Should_Resolve_The_Selection(CollectionType collectionType)
|
||||
{
|
||||
await SeedSelection(collectionType);
|
||||
await SeedPlaylistItem(collectionType);
|
||||
|
||||
var handler = new GetPlaylistItemsHandler(Db.Factory);
|
||||
|
||||
List<PlaylistItemViewModel> items =
|
||||
await handler.Handle(new GetPlaylistItems(1), CancellationToken.None);
|
||||
|
||||
items.Count.ShouldBe(1);
|
||||
|
||||
PlaylistItemViewModel item = items[0];
|
||||
|
||||
int? selectedId = item.Collection?.Id
|
||||
?? item.MultiCollection?.Id
|
||||
?? item.SmartCollection?.Id
|
||||
?? item.MediaItem?.MediaItemId;
|
||||
|
||||
string selectedName = item.Collection?.Name
|
||||
?? item.MultiCollection?.Name
|
||||
?? item.SmartCollection?.Name
|
||||
?? item.MediaItem?.Name;
|
||||
|
||||
selectedId.ShouldBe(SelectionSeedData.SelectedId, $"{collectionType} lost its selected id");
|
||||
selectedName.ShouldBe(
|
||||
SelectionSeedData.ExpectedName(collectionType),
|
||||
$"{collectionType} projected the wrong name");
|
||||
}
|
||||
|
||||
private async Task SeedSelection(CollectionType collectionType)
|
||||
{
|
||||
await using TvContext context = Db.CreateContext();
|
||||
await SelectionSeedData.SeedSelection(context, collectionType);
|
||||
}
|
||||
|
||||
private async Task SeedPlaylistItem(CollectionType collectionType)
|
||||
{
|
||||
await using TvContext context = Db.CreateContext();
|
||||
|
||||
var item = new PlaylistItem
|
||||
{
|
||||
Id = 1,
|
||||
Index = 0,
|
||||
PlaylistId = 1,
|
||||
CollectionType = collectionType,
|
||||
PlaybackOrder = PlaybackOrder.Chronological
|
||||
};
|
||||
|
||||
SelectionSeedData.ApplySelection(
|
||||
collectionType,
|
||||
v => item.CollectionId = v,
|
||||
v => item.MultiCollectionId = v,
|
||||
v => item.SmartCollectionId = v,
|
||||
v => item.MediaItemId = v);
|
||||
|
||||
context.Playlists.Add(new Playlist
|
||||
{
|
||||
Id = 1,
|
||||
Name = "Playlist",
|
||||
Items = [item]
|
||||
});
|
||||
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
using ErsatzTV.Application.MediaCollections;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
using ErsatzTV.Tests.Support;
|
||||
using LanguageExt;
|
||||
using NUnit.Framework;
|
||||
using Shouldly;
|
||||
|
||||
namespace ErsatzTV.Tests.Application.MediaCollections;
|
||||
|
||||
/// <summary>
|
||||
/// Read-path coverage for the two rerun-collection query handlers (issue #671). The defect was
|
||||
/// precisely that nobody enumerated the selection types: the list handler eager-loaded nothing, and
|
||||
/// the by-id handler loaded metadata for only four of the ten media types. So the matrix is derived
|
||||
/// from the production predicate (see <see cref="SelectionSeedData" />) rather than hand-listed.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public class RerunCollectionQueryHandlerTests : MediaCollectionHandlerTestBase
|
||||
{
|
||||
private static IEnumerable<CollectionType> SupportedSelectionTypes => SelectionSeedData.SupportedSelectionTypes;
|
||||
|
||||
/// <summary>
|
||||
/// Completeness guard. Without it, a change that narrowed <c>IsSupportedSelectionType</c> would
|
||||
/// shrink the matrix silently and every remaining case would still pass — the "filters on the
|
||||
/// property it asserts" failure mode. Set equality, so it fails on widening too.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void Supported_Selection_Types_Should_Be_The_Full_Documented_Set()
|
||||
{
|
||||
SupportedSelectionTypes.ShouldBe(
|
||||
[
|
||||
CollectionType.Collection,
|
||||
CollectionType.TelevisionShow,
|
||||
CollectionType.TelevisionSeason,
|
||||
CollectionType.Artist,
|
||||
CollectionType.MultiCollection,
|
||||
CollectionType.SmartCollection,
|
||||
CollectionType.Movie,
|
||||
CollectionType.Episode,
|
||||
CollectionType.MusicVideo,
|
||||
CollectionType.OtherVideo,
|
||||
CollectionType.Song,
|
||||
CollectionType.Image,
|
||||
CollectionType.RemoteStream
|
||||
],
|
||||
ignoreOrder: true);
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(SupportedSelectionTypes))]
|
||||
public async Task GetById_Should_Resolve_The_Selection(CollectionType collectionType)
|
||||
{
|
||||
await SeedSelection(collectionType);
|
||||
await SeedRerunCollection(1, collectionType);
|
||||
|
||||
var handler = new GetRerunCollectionByIdHandler(Db.Factory);
|
||||
|
||||
Option<RerunCollectionViewModel> result =
|
||||
await handler.Handle(new GetRerunCollectionById(1), CancellationToken.None);
|
||||
|
||||
RerunCollectionViewModel vm = result.IfNone(() => throw new AssertionException("Expected a result"));
|
||||
AssertSelectionResolved(vm, collectionType);
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(SupportedSelectionTypes))]
|
||||
public async Task GetPaged_Should_Resolve_The_Selection(CollectionType collectionType)
|
||||
{
|
||||
await SeedSelection(collectionType);
|
||||
await SeedRerunCollection(1, collectionType);
|
||||
|
||||
var handler = new GetPagedRerunCollectionsHandler(Db.Factory);
|
||||
|
||||
PagedRerunCollectionsViewModel result = await handler.Handle(
|
||||
new GetPagedRerunCollections(string.Empty, 0, 10),
|
||||
CancellationToken.None);
|
||||
|
||||
result.Page.Count.ShouldBe(1);
|
||||
AssertSelectionResolved(result.Page[0], collectionType);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>SongMetadata.Artists</c> is a NULLABLE primitive collection, and a song whose tags failed to
|
||||
/// read is persisted with it never assigned. Before #671 the rerun list did not load SongMetadata
|
||||
/// at all, so this was unreachable there; eager-loading it made a latent `string.Join` throw into a
|
||||
/// live 500 that would take down the whole page.
|
||||
/// </summary>
|
||||
[TestCase(null, "Selected song", TestName = "GetById_Song_With_Null_Artists_Should_Not_Throw")]
|
||||
[TestCase(new string[] { }, "Selected song", TestName = "GetById_Song_With_No_Artists_Should_Not_Prefix")]
|
||||
public async Task GetById_Should_Tolerate_Song_Artists(string[] artists, string expectedName)
|
||||
{
|
||||
await using (TvContext context = Db.CreateContext())
|
||||
{
|
||||
context.Songs.Add(new Song
|
||||
{
|
||||
Id = SelectionSeedData.SelectedId,
|
||||
SongMetadata = [new SongMetadata { Title = "Selected song", Artists = artists?.ToList() }]
|
||||
});
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await SeedRerunCollection(1, CollectionType.Song);
|
||||
|
||||
var handler = new GetRerunCollectionByIdHandler(Db.Factory);
|
||||
|
||||
Option<RerunCollectionViewModel> result =
|
||||
await handler.Handle(new GetRerunCollectionById(1), CancellationToken.None);
|
||||
|
||||
RerunCollectionViewModel vm = result.IfNone(() => throw new AssertionException("Expected a result"));
|
||||
vm.MediaItem.ShouldNotBeNull();
|
||||
vm.MediaItem.MediaItemId.ShouldBe(SelectionSeedData.SelectedId);
|
||||
vm.MediaItem.Name.ShouldBe(expectedName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mirrors <c>RerunCollectionController.ProjectToResponseModel</c>, which flattens the tagged
|
||||
/// union to the single <c>selectedId</c> / <c>selectedName</c> pair the SPA consumes. The id is
|
||||
/// the load-bearing half: the editor round-trips it, so a null there silently clears the user's
|
||||
/// stored selection.
|
||||
/// </summary>
|
||||
private static void AssertSelectionResolved(RerunCollectionViewModel vm, CollectionType collectionType)
|
||||
{
|
||||
int? selectedId = vm.Collection?.Id
|
||||
?? vm.MultiCollection?.Id
|
||||
?? vm.SmartCollection?.Id
|
||||
?? vm.MediaItem?.MediaItemId;
|
||||
|
||||
string selectedName = vm.Collection?.Name
|
||||
?? vm.MultiCollection?.Name
|
||||
?? vm.SmartCollection?.Name
|
||||
?? vm.MediaItem?.Name;
|
||||
|
||||
selectedId.ShouldBe(SelectionSeedData.SelectedId, $"{collectionType} lost its selected id");
|
||||
selectedName.ShouldBe(
|
||||
SelectionSeedData.ExpectedName(collectionType),
|
||||
$"{collectionType} projected the wrong name");
|
||||
}
|
||||
|
||||
private async Task SeedSelection(CollectionType collectionType)
|
||||
{
|
||||
await using TvContext context = Db.CreateContext();
|
||||
await SelectionSeedData.SeedSelection(context, collectionType);
|
||||
}
|
||||
|
||||
private async Task SeedRerunCollection(int id, CollectionType collectionType)
|
||||
{
|
||||
await using TvContext context = Db.CreateContext();
|
||||
|
||||
var rerunCollection = new RerunCollection
|
||||
{
|
||||
Id = id,
|
||||
Name = "Rerun",
|
||||
CollectionType = collectionType,
|
||||
FirstRunPlaybackOrder = PlaybackOrder.Chronological,
|
||||
RerunPlaybackOrder = PlaybackOrder.Chronological
|
||||
};
|
||||
|
||||
SelectionSeedData.ApplySelection(
|
||||
collectionType,
|
||||
v => rerunCollection.CollectionId = v,
|
||||
v => rerunCollection.MultiCollectionId = v,
|
||||
v => rerunCollection.SmartCollectionId = v,
|
||||
v => rerunCollection.MediaItemId = v);
|
||||
|
||||
context.RerunCollections.Add(rerunCollection);
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using ErsatzTV.Core.Domain;
|
||||
using LanguageExt;
|
||||
using NUnit.Framework;
|
||||
using Shouldly;
|
||||
using Mapper = ErsatzTV.Application.Playouts.Mapper;
|
||||
|
||||
namespace ErsatzTV.Tests.Application.Playouts;
|
||||
|
||||
/// <summary>
|
||||
/// <c>SongMetadata.Artists</c> is a nullable EF primitive collection that
|
||||
/// <c>FallbackMetadataProvider</c> leaves unassigned for a song whose tags failed to read, and
|
||||
/// <c>string.Join</c> throws <see cref="ArgumentNullException" /> on a null sequence. Because
|
||||
/// <c>SongMetadata</c> IS eager-loaded on the playout paths, this was a LIVE 500 rather than a
|
||||
/// latent one — and <c>GetDisplayTitle</c> feeds the playout guide, troubleshooting, media-item
|
||||
/// info and channel states alike (issue #671).
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public class PlayoutMapperDisplayTitleTests
|
||||
{
|
||||
[Test]
|
||||
public void GetDisplayTitle_Should_Not_Throw_When_Song_Artists_Is_Null()
|
||||
{
|
||||
var song = new Song
|
||||
{
|
||||
Id = 1,
|
||||
SongMetadata = [new SongMetadata { Title = "Untagged", Artists = null }]
|
||||
};
|
||||
|
||||
string title = Mapper.GetDisplayTitle(song, Option<string>.None);
|
||||
|
||||
title.ShouldBe("Untagged");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetDisplayTitle_Should_Not_Prefix_When_Song_Has_No_Artists()
|
||||
{
|
||||
var song = new Song
|
||||
{
|
||||
Id = 1,
|
||||
SongMetadata = [new SongMetadata { Title = "Untagged", Artists = [] }]
|
||||
};
|
||||
|
||||
string title = Mapper.GetDisplayTitle(song, Option<string>.None);
|
||||
|
||||
title.ShouldBe("Untagged");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetDisplayTitle_Should_Prefix_The_Artists_When_Present()
|
||||
{
|
||||
var song = new Song
|
||||
{
|
||||
Id = 1,
|
||||
SongMetadata = [new SongMetadata { Title = "Tagged", Artists = ["A", "B"] }]
|
||||
};
|
||||
|
||||
string title = Mapper.GetDisplayTitle(song, Option<string>.None);
|
||||
|
||||
title.ShouldBe("A, B - Tagged");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The chapter branch interpolated the `case Song s` ENTITY rather than the composed title, and
|
||||
/// <see cref="Song" /> has no <c>ToString()</c> override — so a chaptered song rendered as the
|
||||
/// literal "ErsatzTV.Core.Domain.Song (Chapter 1)". Pre-existing; the sibling MusicVideo and
|
||||
/// OtherVideo arms are correct only because they name their lambda parameter `s` too.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void GetDisplayTitle_Should_Compose_The_Title_Not_The_Entity_When_Chaptered()
|
||||
{
|
||||
var song = new Song
|
||||
{
|
||||
Id = 1,
|
||||
SongMetadata = [new SongMetadata { Title = "Tagged", Artists = ["A"] }]
|
||||
};
|
||||
|
||||
string title = Mapper.GetDisplayTitle(song, Option<string>.Some("Chapter 1"));
|
||||
|
||||
title.ShouldBe("A - Tagged (Chapter 1)");
|
||||
title.ShouldNotContain("ErsatzTV.Core.Domain");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetDisplayTitle_Should_Not_Throw_When_Chaptered_Song_Has_Null_Artists()
|
||||
{
|
||||
var song = new Song
|
||||
{
|
||||
Id = 1,
|
||||
SongMetadata = [new SongMetadata { Title = "Untagged", Artists = null }]
|
||||
};
|
||||
|
||||
string title = Mapper.GetDisplayTitle(song, Option<string>.Some("Chapter 2"));
|
||||
|
||||
title.ShouldBe("Untagged (Chapter 2)");
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
using System.Globalization;
|
||||
using ErsatzTV.Application.Search.Queries;
|
||||
using ErsatzTV.Core.Api.Search;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
using ErsatzTV.Tests.Support;
|
||||
using LanguageExt;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using NUnit.Framework;
|
||||
using Shouldly;
|
||||
|
||||
@@ -176,6 +178,461 @@ public class GetSearchFieldValuesHandlerTests
|
||||
networkResult.IfSome(r => r.Values.ShouldBe(new List<string> { "HBO" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Artist_Merges_Entity_Artists_Music_Video_Credits_And_Song_Credits()
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.ArtistMetadata.Add(Artist("Alpha Entity"));
|
||||
|
||||
// negative control: an entity artist that must NOT match the "al" prefix
|
||||
context.ArtistMetadata.Add(Artist("Zeta Entity"));
|
||||
|
||||
context.MusicVideoMetadata.AddRange(
|
||||
MusicVideo("MV One", "Alpha Credit", "Alpha Shared"),
|
||||
// "Alpha Shared" appears in two rows, so DISTINCT has something to collapse
|
||||
MusicVideo("MV Two", "Alpha Shared"),
|
||||
MusicVideo("MV Three", "Zeta Credit"));
|
||||
|
||||
context.SongMetadata.AddRange(
|
||||
Song("Song One", ["Alpha Song", "Zeta Song"]),
|
||||
Song("Song Two", ["Alpha Song"]),
|
||||
Song("Song Three", ["Zeta Only"]));
|
||||
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("artist", "al", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(
|
||||
new List<string> { "Alpha Credit", "Alpha Entity", "Alpha Shared", "Alpha Song" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Artist_Returns_Every_Source_For_Empty_Query()
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.ArtistMetadata.Add(Artist("Entity"));
|
||||
context.MusicVideoMetadata.Add(MusicVideo("MV", "Credit"));
|
||||
context.SongMetadata.Add(Song("Song", ["SongArtist"]));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("artist", string.Empty, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { "Credit", "Entity", "SongArtist" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Album_Artist_Returns_Song_Album_Artists_Instead_Of_NotFound()
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.SongMetadata.AddRange(
|
||||
Song("One", ["Performer"], ["Alpha Album Artist", "Beta Album Artist"]),
|
||||
// repeated across rows so DISTINCT is exercised
|
||||
Song("Two", ["Performer"], ["Alpha Album Artist"]),
|
||||
// negative control: a row whose album artists are absent entirely
|
||||
Song("Three", ["Performer"], null));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("album_artist", string.Empty, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { "Alpha Album Artist", "Beta Album Artist" }));
|
||||
|
||||
// the performers on the same rows must not leak into album_artist
|
||||
result.IfSome(r => r.Values.ShouldNotContain("Performer"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task List_Valued_Fields_Match_Whole_Elements_Not_Substrings_And_Ignore_Neighbours()
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.SongMetadata.AddRange(
|
||||
// "Neighbour" arrives on the same row as "Radiohead" -- rows are read whole -- and must be
|
||||
// dropped by the in-memory exact prefix filter.
|
||||
Song("One", ["Radiohead", "Neighbour"]),
|
||||
// "The Radio Dept." contains "radio" but does not start with it
|
||||
Song("Two", ["The Radio Dept."]),
|
||||
Song("Three", ["Radio Birdman"]));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("artist", "radio", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { "Radio Birdman", "Radiohead" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task List_Valued_Fields_Match_Literally_Including_Json_Escaped_And_Wildcard_Characters()
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.SongMetadata.AddRange(
|
||||
// non-ASCII: stored on disk JSON-escaped as \u00E9, and must survive the round trip
|
||||
Song("One", ["Beyoncé"]),
|
||||
// an embedded quote is stored as \u0022
|
||||
Song("Two", ["\"Weird Al\" Yankovic"]),
|
||||
// SQL wildcards must be ordinary characters here, matched literally
|
||||
Song("Three", ["50% Off"]),
|
||||
Song("Four", ["50 Cent"]));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", "beyoncé", 50), CancellationToken.None))
|
||||
.IfSome(r => r.Values.ShouldBe(new List<string> { "Beyoncé" }));
|
||||
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", "\"weird", 50), CancellationToken.None))
|
||||
.IfSome(r => r.Values.ShouldBe(new List<string> { "\"Weird Al\" Yankovic" }));
|
||||
|
||||
// "50%" must not behave as the wildcard "50<anything>" — "50 Cent" must not come back
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", "50%", 50), CancellationToken.None))
|
||||
.IfSome(r => r.Values.ShouldBe(new List<string> { "50% Off" }));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeds <paramref name="fillerRows" /> non-matching songs through raw SQL — 20k rows via the change
|
||||
/// tracker is minutes, this is milliseconds.
|
||||
/// </summary>
|
||||
private static Task SeedFiller(TvContext context, int fillerRows) =>
|
||||
context.Database.ExecuteSqlRawAsync(
|
||||
$"""
|
||||
WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < {fillerRows})
|
||||
INSERT INTO SongMetadata (SongId, MetadataKind, Title, Artists, DateAdded, DateUpdated)
|
||||
SELECT 0, 0, 'Filler ' || n, '["zzz-filler"]', '2026-01-01', '2026-01-01' FROM seq
|
||||
""");
|
||||
|
||||
[Test]
|
||||
public async Task List_Valued_Walk_Reads_At_Most_20000_Rows()
|
||||
{
|
||||
// Pinned in both directions so the ceiling itself is nailed down: a match in row 20000 is read, the same
|
||||
// match in row 20001 is not. The query has no RESIDUAL predicate -- only the cursor -- so "rows read" is
|
||||
// what LIMIT returns. That bounds LOGICAL rows, not physical work: the engine may still traverse more
|
||||
// index records than it returns (MySQL purge lag), and row width is unbounded.
|
||||
const string needle = "\u00E9clair-the-needle";
|
||||
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
await SeedFiller(context, 19999);
|
||||
context.SongMetadata.Add(Song("Needle", [needle]));
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
(await context.SongMetadata.CountAsync()).ShouldBe(20000);
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", "\u00E9", 50), CancellationToken.None))
|
||||
.IfSome(r => r.Values.ShouldBe(new List<string> { needle }, "row 20000 is inside the ceiling"));
|
||||
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
SongMetadata existing = await context.SongMetadata.SingleAsync(m => m.Title == "Needle");
|
||||
context.SongMetadata.Remove(existing);
|
||||
await SeedFiller(context, 1);
|
||||
await context.SaveChangesAsync();
|
||||
context.SongMetadata.Add(Song("Needle", [needle]));
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
(await context.SongMetadata.CountAsync()).ShouldBe(20001);
|
||||
}
|
||||
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", "\u00E9", 50), CancellationToken.None))
|
||||
.IfSome(
|
||||
r => r.Values.ShouldBeEmpty(
|
||||
"row 20001 is past the ceiling; this false negative is the documented bounded-best-effort "
|
||||
+ "contract, deliberately pinned rather than papered over"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task List_Valued_Walk_Reads_Live_Rows_Regardless_Of_Id_Density()
|
||||
{
|
||||
// THE round-4 killer. That revision bounded the Id KEYSPACE, and keyspace is not rows: with 20,000
|
||||
// historical rows deleted and one live song at Id 20001, the walk spent its whole allowance on empty
|
||||
// ranges and returned [] for a table containing exactly one row. Capacity degraded linearly with
|
||||
// deletion ratio, and no ratio was safe -- one placed gap hid the next match.
|
||||
//
|
||||
// Paging by row position rather than Id value makes density irrelevant: LIMIT @Batch returns @Batch
|
||||
// ROWS, wherever they sit in the keyspace.
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
await SeedFiller(context, 20000);
|
||||
await context.Database.ExecuteSqlRawAsync("DELETE FROM SongMetadata");
|
||||
context.SongMetadata.Add(Song("Survivor", ["Queen"]));
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
// one live row, sitting past the old keyspace allowance
|
||||
(await context.SongMetadata.CountAsync()).ShouldBe(1);
|
||||
(await context.SongMetadata.Select(m => m.Id).SingleAsync()).ShouldBeGreaterThan(20000);
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", "que", 50), CancellationToken.None))
|
||||
.IfSome(
|
||||
r => r.Values.ShouldBe(
|
||||
new List<string> { "Queen" },
|
||||
"a one-row table must be fully readable no matter where its Id sits"));
|
||||
|
||||
// and a leading gap must not hide a later match either
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.SongMetadata.Add(Song("Second", ["Queens of the Stone Age"]));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", "que", 50), CancellationToken.None))
|
||||
.IfSome(r => r.Values.ShouldBe(new List<string> { "Queen", "Queens of the Stone Age" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase("é", "\u00C9dith Piaf")]
|
||||
[TestCase("\u00C9", "\u00C9dith Piaf")]
|
||||
[TestCase("\u00E9dith", "\u00C9dith Piaf")]
|
||||
[TestCase("bj", "Bj\u00F6rk")]
|
||||
[TestCase("bj\u00F6", "Bj\u00F6rk")]
|
||||
[TestCase("BJ\u00D6RK", "Bj\u00F6rk")]
|
||||
[TestCase("beyonc\u00E9", "Beyonc\u00E9")]
|
||||
[TestCase("sigur r", "Sigur R\u00F3s")]
|
||||
[TestCase("\u00D6", "\u00D6zdemir")]
|
||||
public async Task Matches_NonAscii_Values_In_Any_Casing(string query, string stored)
|
||||
{
|
||||
// Accented artists are the common case in a music library, so non-ASCII matching is pinned end to
|
||||
// end, in both casings of the query.
|
||||
//
|
||||
// Historical note, because it is why this suite exists: revision 1b78dc9e narrowed rows in SQL
|
||||
// with a LIKE built by JSON-encoding the query, which cannot work -- non-ASCII is stored escaped
|
||||
// (\u00C9) and SQL LOWER() folds the escape TEXT, not the codepoint it denotes. THREE of these nine
|
||||
// cases fail against that revision (the ones where query and stored casing differ, so \u00e9 and
|
||||
// \u00C9 diverge); the other six pass it, because when the casings agree the escape texts line up.
|
||||
// The SQL now has no residual predicate at all -- matching happens in memory, where a string is just
|
||||
// a string -- so these cases pin current behaviour rather than guard that revision.
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.SongMetadata.AddRange(
|
||||
Song("Hit", [stored]),
|
||||
// negative control: a row that must never come back for any of these queries
|
||||
Song("Other", ["Nothing Relevant"]));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("artist", query, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { stored }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Results_Do_Not_Depend_On_The_Request_Culture()
|
||||
{
|
||||
// UseRequestLocalization honours Accept-Language, so CurrentCulture is caller-controlled. Under tr-TR
|
||||
// the old `q.ToLower()` turned "I" into "\u0131" and the default linguistic StartsWith(string) compounded
|
||||
// it, so the same library answered differently per caller. The contract is ordinal: "I" matches
|
||||
// "Istanbul" and does NOT match "\u0131pek", in every culture.
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.SongMetadata.Add(Song("One", ["Istanbul Orkestrasi", "\u0131pek"]));
|
||||
context.ArtistMetadata.Add(Artist("Idil Biret"));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
var expected = new List<string> { "Idil Biret", "Istanbul Orkestrasi" };
|
||||
|
||||
CultureInfo original = CultureInfo.CurrentCulture;
|
||||
try
|
||||
{
|
||||
foreach (string culture in new[] { "en-US", "tr-TR", "az-AZ", "lt-LT" })
|
||||
{
|
||||
CultureInfo.CurrentCulture = new CultureInfo(culture);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("artist", "I", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(expected, $"culture {culture} changed the result"));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
CultureInfo.CurrentCulture = original;
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Ordering_Is_Ordinal_And_Culture_Independent()
|
||||
{
|
||||
// The merge sorts ordinally rather than by culture, so the response order does not depend on the caller
|
||||
// either. Ordinal puts all ASCII uppercase before ASCII lowercase, and non-ASCII last.
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.SongMetadata.Add(Song("One", ["Zulu", "apple", "\u00C9clair", "Apple"]));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
var expected = new List<string> { "Apple", "Zulu", "apple", "\u00C9clair" };
|
||||
|
||||
CultureInfo original = CultureInfo.CurrentCulture;
|
||||
try
|
||||
{
|
||||
foreach (string culture in new[] { "en-US", "sv-SE" })
|
||||
{
|
||||
CultureInfo.CurrentCulture = new CultureInfo(culture);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("artist", string.Empty, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IfSome(r => r.Values.ShouldBe(expected, $"culture {culture} changed the order"));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
CultureInfo.CurrentCulture = original;
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Ordering_Is_Best_Effort_When_A_Source_Truncates()
|
||||
{
|
||||
// Documents the acknowledged imprecision rather than claiming exactness the code does not have. The EF
|
||||
// source truncates by the DATABASE collation, which is NOT the ordinal ordering the merge then applies —
|
||||
// so a value the database ranked outside its first `limit` never reaches the merge, even if the merge
|
||||
// would have ranked it first.
|
||||
//
|
||||
// "Zulu" vs "apple" is the pair that actually diverges: ordinal puts every ASCII uppercase letter before
|
||||
// every lowercase one, so ordinal ranks "Zulu" first, while a case-insensitive database ordering ranks
|
||||
// "apple" first. (An earlier version used "Zulu"/"Éclair", where BOTH orderings pick "Zulu" — it could
|
||||
// not have told the two apart, and the divergence it claimed to show did not exist.)
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.ArtistMetadata.Add(Artist("Zulu"));
|
||||
context.ArtistMetadata.Add(Artist("apple"));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
// with room for both, the ordinal merge ranks "Zulu" first
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", string.Empty, 50), CancellationToken.None))
|
||||
.IfSome(r => r.Values.ShouldBe(new List<string> { "Zulu", "apple" }));
|
||||
|
||||
// with limit=1 the database picks the survivor by ITS ordering, and the merge only ever sees that one
|
||||
(await handler.Handle(new GetSearchFieldValues("artist", string.Empty, 1), CancellationToken.None))
|
||||
.IfSome(r => r.Values.ShouldBe(new List<string> { "apple" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task A_Match_Behind_Many_NonMatching_Rows_Is_Still_Found()
|
||||
{
|
||||
// Fails a883e5f0, which capped rows at a fixed 1000 AFTER a deliberately over-matching SQL pre-filter:
|
||||
// the 1001st row -- the only exact match -- was discarded before the in-memory filter ever saw it and
|
||||
// the endpoint returned []. The pre-filter is gone, and the property it broke now holds for any match
|
||||
// within the read ceiling: preceding non-matching rows do not hide it. Past the ceiling it is still
|
||||
// lost by design -- see List_Valued_Walk_Reads_At_Most_20000_Rows, which pins that boundary.
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
for (var i = 0; i < 1000; i++)
|
||||
{
|
||||
context.SongMetadata.Add(Song($"Filler {i}", ["zzz-filler"], ["zzz-filler-album"]));
|
||||
}
|
||||
|
||||
context.SongMetadata.Add(Song("Needle", ["\u00E9clair"], ["\u00E9clair"]));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> albumArtist = await handler.Handle(
|
||||
new GetSearchFieldValues("album_artist", "\u00E9", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
albumArtist.IsSome.ShouldBeTrue();
|
||||
albumArtist.IfSome(r => r.Values.ShouldBe(new List<string> { "\u00E9clair" }));
|
||||
|
||||
// same starvation shape on the merged `artist` field
|
||||
Option<SearchFieldValuesResponseModel> artist = await handler.Handle(
|
||||
new GetSearchFieldValues("artist", "\u00E9", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
artist.IsSome.ShouldBeTrue();
|
||||
artist.IfSome(r => r.Values.ShouldBe(new List<string> { "\u00E9clair" }));
|
||||
|
||||
// ... and for a prefix beginning with a character that JSON escapes on disk. That used to collapse the
|
||||
// SQL pattern to the bare anchor; there is no prefix predicate at all now, so it is simply an ordinary
|
||||
// prefix -- kept because it is the input shape that broke the old scheme.
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.SongMetadata.Add(Song("Ampersand", ["&Me"]));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
Option<SearchFieldValuesResponseModel> escapedPrefix = await handler.Handle(
|
||||
new GetSearchFieldValues("artist", "&M", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
escapedPrefix.IsSome.ShouldBeTrue();
|
||||
escapedPrefix.IfSome(r => r.Values.ShouldBe(new List<string> { "&Me" }));
|
||||
}
|
||||
|
||||
private static ArtistMetadata Artist(string title) => new()
|
||||
{
|
||||
MetadataKind = MetadataKind.External,
|
||||
DateAdded = DateTime.UtcNow,
|
||||
DateUpdated = DateTime.UtcNow,
|
||||
Title = title
|
||||
};
|
||||
|
||||
private static MusicVideoMetadata MusicVideo(string title, params string[] artists) => new()
|
||||
{
|
||||
MetadataKind = MetadataKind.External,
|
||||
DateAdded = DateTime.UtcNow,
|
||||
DateUpdated = DateTime.UtcNow,
|
||||
Title = title,
|
||||
Artists = artists.Map(a => new MusicVideoArtist { Name = a }).ToList()
|
||||
};
|
||||
|
||||
private static SongMetadata Song(string title, IList<string> artists, IList<string> albumArtists = null) => new()
|
||||
{
|
||||
MetadataKind = MetadataKind.External,
|
||||
DateAdded = DateTime.UtcNow,
|
||||
DateUpdated = DateTime.UtcNow,
|
||||
Title = title,
|
||||
Artists = artists,
|
||||
AlbumArtists = albumArtists
|
||||
};
|
||||
|
||||
[Test]
|
||||
public async Task Dedupes_Repeated_Values()
|
||||
{
|
||||
@@ -196,4 +653,226 @@ public class GetSearchFieldValuesHandlerTests
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { "Action" }));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. The EF-sourced fields prefix-match through SQL <c>LOWER()</c>, which on SQLite folds
|
||||
/// ASCII only: <c>lower('Édith')</c> returns <c>'Édith'</c> unchanged, so a stored value whose
|
||||
/// prefix carries an uppercase non-ASCII character is unreachable from any query long enough to reach it.
|
||||
/// The stored-LOWERCASE case already worked (the handler lowercases the query before it reaches SQL, so
|
||||
/// both casings of the query fold to the same pattern) and is pinned alongside it, because the fix must
|
||||
/// SUPPLEMENT that path rather than replace it.
|
||||
/// </summary>
|
||||
[TestCase("genre", "é")]
|
||||
[TestCase("genre", "É")]
|
||||
public async Task Ef_Sourced_Stored_Uppercase_Accent_Is_Reachable(string field, string query)
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.Set<Genre>().AddRange(
|
||||
new Genre { Name = "Édith" },
|
||||
new Genre { Name = "Zulu" });
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues(field, query, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { "Édith" }));
|
||||
}
|
||||
|
||||
/// <inheritdoc cref="Ef_Sourced_Stored_Uppercase_Accent_Is_Reachable" />
|
||||
[TestCase("genre", "é")]
|
||||
[TestCase("genre", "É")]
|
||||
public async Task Ef_Sourced_Stored_Lowercase_Accent_Stays_Reachable(string field, string query)
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.Set<Genre>().AddRange(
|
||||
new Genre { Name = "édith" },
|
||||
new Genre { Name = "Zulu" });
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues(field, query, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { "édith" }));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. The Unicode fold added for the non-ASCII branch may OVER-match — the in-memory
|
||||
/// <see cref="StringComparison.OrdinalIgnoreCase" /> filter runs afterwards and drops the extras —
|
||||
/// but it must never UNDER-match. Each case pins the endpoint's answer against what that filter
|
||||
/// alone would say, so a fold that starts dropping rows fails here. It does NOT catch removal of the
|
||||
/// in-memory filter — every case here is either a positive that SQL alone returns, or an ASCII-query
|
||||
/// negative that SQL alone rejects. That direction is
|
||||
/// <see cref="Unicode_Fold_Over_Match_Is_Discarded_By_The_Ordinal_Filter" />'s job.
|
||||
/// <para>
|
||||
/// The negative cases here have ASCII queries, so they exercise the FAST PATH (the fold is
|
||||
/// skipped entirely) and pin that it is exact: <c>"ſweet".StartsWith("S", OrdinalIgnoreCase)</c>
|
||||
/// is false even though <c>char.ToUpperInvariant('ſ')</c> IS <c>'S'</c>. The over-match the fold
|
||||
/// itself produces is a different path and is covered by
|
||||
/// <see cref="Unicode_Fold_Over_Match_Is_Discarded_By_The_Ordinal_Filter" />.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[TestCase("Édith", "é", true, TestName = "Fold_UppercaseAccent_LowercaseQuery")]
|
||||
[TestCase("Édith", "É", true, TestName = "Fold_UppercaseAccent_UppercaseQuery")]
|
||||
[TestCase("Özdemir", "ö", true, TestName = "Fold_Umlaut")]
|
||||
[TestCase("Sigur Rós", "sigur", true, TestName = "Fold_AsciiPrefix_NonAsciiLater")]
|
||||
[TestCase("Straße", "stra", true, TestName = "Fold_Eszett_AsciiQuery")]
|
||||
// explicit escapes: these three are visually indistinguishable from their ASCII lookalikes in a diff,
|
||||
// and an ASCII 'K' here would silently turn the KELVIN SIGN case into a trivially-true one
|
||||
[TestCase("\u017Fweet", "S", false, TestName = "Fold_LongS_IsNotOrdinalEqualToS")]
|
||||
[TestCase("\u212Aelvin", "k", false, TestName = "Fold_KelvinSign_IsNotOrdinalEqualToK")]
|
||||
[TestCase("\u0130stanbul", "i", false, TestName = "Fold_DottedCapitalI_IsNotOrdinalEqualToI")]
|
||||
public async Task Unicode_Fold_Agrees_With_The_Ordinal_Filter(string stored, string query, bool expected)
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.Set<Genre>().Add(new Genre { Name = stored });
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
// the oracle: what the endpoint's own final filter says, computed independently of the database
|
||||
stored.StartsWith(query, StringComparison.OrdinalIgnoreCase).ShouldBe(expected);
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("genre", query, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(expected ? new List<string> { stored } : []));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. Drives a row THROUGH the fold that the ordinal filter must then discard — the
|
||||
/// harmless over-match direction the whole design rests on, which the ASCII-query negative cases
|
||||
/// above cannot reach. q="ſ" is non-ASCII so the fold runs; <c>ToUpperInvariant('ſ')</c> is 'S', so
|
||||
/// the SQL pattern is <c>S%</c> and SQLite genuinely returns "Sword" — and the response must still
|
||||
/// be empty, because <c>"Sword".StartsWith("ſ", OrdinalIgnoreCase)</c> is false.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public async Task Unicode_Fold_Over_Match_Is_Discarded_By_The_Ordinal_Filter()
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.Set<Genre>().Add(new Genre { Name = "Sword" });
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
// Premises, asserted because the expectation is an EMPTY list and would otherwise pass for the
|
||||
// wrong reason -- e.g. if the branch stopped running, or a hand-rolled fold stopped mapping ſ to S,
|
||||
// SQL would return nothing and this test would still be green.
|
||||
GetSearchFieldValuesHandler.ContainsNonAscii("\u017F").ShouldBeTrue();
|
||||
char.ToUpperInvariant('\u017F').ShouldBe('S');
|
||||
"Sword".StartsWith("\u017F", StringComparison.OrdinalIgnoreCase).ShouldBeFalse();
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("genre", "\u017F", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBeEmpty());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. The escaping's load-bearing role is NOT filtering — the in-memory ordinal filter
|
||||
/// already drops an over-match, which is why a plain count assertion stays green even with the
|
||||
/// escaping removed. It is preventing LIMIT CROWDING: an unescaped <c>_</c> also matches the space,
|
||||
/// binary ORDER BY ranks "100 Édith" first, LIMIT 1 returns only that, the filter discards it, and
|
||||
/// the genuine "100_Édith" is never returned at all. This case fails if the escaping is removed.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public async Task Unicode_Fold_Escaping_Prevents_Limit_Crowding()
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.Set<Genre>().AddRange(
|
||||
new Genre { Name = "100 \u00C9dith" },
|
||||
new Genre { Name = "100_\u00C9dith" });
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("genre", "100_\u00C9", 1),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { "100_\u00C9dith" }));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. The non-ASCII branch is raw SQL, so it gets none of the LIKE-wildcard escaping EF
|
||||
/// does for <c>StartsWith</c>. An unescaped <c>%</c> or <c>_</c> in the query would match anything.
|
||||
/// </summary>
|
||||
[TestCase("100%É", 1, TestName = "Escapes_Percent")]
|
||||
[TestCase("100_É", 0, TestName = "Escapes_Underscore")]
|
||||
public async Task Unicode_Fold_Escapes_Like_Wildcards(string query, int expectedCount)
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.Set<Genre>().AddRange(
|
||||
new Genre { Name = "100%Édith" },
|
||||
new Genre { Name = "100XÉdith" });
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("genre", query, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.Count.ShouldBe(expectedCount));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. The non-ASCII branch duplicates each field's discriminator predicate in raw SQL, so
|
||||
/// it must reproduce EF's NULL semantics: EF compiles <c>ExternalTypeId != NfoCountryTypeId</c> with
|
||||
/// null semantics, which INCLUDES a NULL-typed row. Plain SQL <c><></c> would silently drop it.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public async Task Unicode_Fold_Tag_Discriminator_Matches_Ef_Null_Semantics()
|
||||
{
|
||||
await using (TvContext context = _db.CreateContext())
|
||||
{
|
||||
context.Set<Tag>().AddRange(
|
||||
new Tag { Name = "Édith", ExternalTypeId = null },
|
||||
new Tag { Name = "Éclair", ExternalTypeId = Tag.PlexNetworkTypeId },
|
||||
new Tag { Name = "Ézra", ExternalTypeId = Tag.NfoCountryTypeId });
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(_db.Factory);
|
||||
|
||||
Option<SearchFieldValuesResponseModel> tags = await handler.Handle(
|
||||
new GetSearchFieldValues("tag", "é", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
// the NULL-typed row is a tag; the network- and country-typed rows are excluded
|
||||
tags.IsSome.ShouldBeTrue();
|
||||
tags.IfSome(r => r.Values.ShouldBe(new List<string> { "Édith" }));
|
||||
|
||||
Option<SearchFieldValuesResponseModel> networks = await handler.Handle(
|
||||
new GetSearchFieldValues("network", "é", 50),
|
||||
CancellationToken.None);
|
||||
|
||||
networks.IsSome.ShouldBeTrue();
|
||||
networks.IfSome(r => r.Values.ShouldBe(new List<string> { "Éclair" }));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
using ErsatzTV.Application.Search.Queries;
|
||||
using ErsatzTV.Infrastructure;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
using ErsatzTV.Infrastructure.Sqlite.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NUnit.Framework;
|
||||
using Shouldly;
|
||||
|
||||
namespace ErsatzTV.Tests.Application.Search;
|
||||
|
||||
/// <summary>
|
||||
/// Provider-shape guards for the <c>artist</c> / <c>album_artist</c> facet-value sources (#578).
|
||||
/// <para>
|
||||
/// <see cref="GetSearchFieldValuesHandlerTests" /> runs against in-memory SQLite, so it structurally
|
||||
/// cannot see a MySQL translation or collation difference. These tests build the same LINQ against the
|
||||
/// Pomelo MySQL provider and assert the generated SQL — <c>ToQueryString</c> compiles the query without
|
||||
/// touching a server, so no MySQL instance is needed.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
[NonParallelizable]
|
||||
public class SearchFieldValuesQueryShapeTests
|
||||
{
|
||||
private bool _wasSqlite;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp() => _wasSqlite = TvContext.IsSqlite;
|
||||
|
||||
[TearDown]
|
||||
public void TearDown() => TvContext.IsSqlite = _wasSqlite;
|
||||
|
||||
[Test]
|
||||
public void Artist_Entity_Union_Translates_On_Both_Providers_With_Lower_And_A_Row_Limit()
|
||||
{
|
||||
foreach ((string provider, Func<TvContext> create) in Providers())
|
||||
{
|
||||
using TvContext context = create();
|
||||
|
||||
// calls the handler's own source builder (internal, via InternalsVisibleTo) rather than rebuilding
|
||||
// the LINQ here — a copy would keep passing after the handler's query changed underneath it
|
||||
string sql = GetSearchFieldValuesHandler.GetSource(context, "artist")
|
||||
.Where(v => v != null && v.ToLower().StartsWith("a"))
|
||||
.Distinct()
|
||||
.OrderBy(v => v)
|
||||
.Take(50)
|
||||
.ToQueryString();
|
||||
|
||||
// case-insensitivity comes from LOWER() on the column, not from the provider's LIKE collation
|
||||
sql.ShouldContain("LOWER(", Case.Insensitive, $"{provider}: {sql}");
|
||||
sql.ShouldContain("LIKE", Case.Insensitive, $"{provider}: {sql}");
|
||||
sql.ShouldContain("MusicVideoArtist", Case.Insensitive, $"{provider}: {sql}");
|
||||
// the whole thing is one bounded server-side query, never a client-side scan
|
||||
sql.ShouldContain("LIMIT", Case.Insensitive, $"{provider}: {sql}");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Regression_Pin_Song_List_Columns_Cannot_Be_Projected_Server_Side_On_Either_Provider()
|
||||
{
|
||||
// REGRESSION PIN, not coverage of #578: this asserts pre-existing EF/provider behaviour and passes
|
||||
// against the code before this change.
|
||||
//
|
||||
// Documents WHY the handler drops to raw SQL for SongMetadata.Artists / .AlbumArtists rather than
|
||||
// SelectMany-ing them: EF maps them as JSON primitive collections and neither provider can translate
|
||||
// the projection (SQLite needs APPLY; Pomelo has no primitive-collection support). If a provider
|
||||
// upgrade ever makes this translate, this test fails and the raw-SQL path can be retired.
|
||||
foreach ((string provider, Func<TvContext> create) in Providers())
|
||||
{
|
||||
using TvContext context = create();
|
||||
|
||||
Should.Throw<InvalidOperationException>(
|
||||
() => context.SongMetadata.SelectMany(m => m.Artists).Distinct().Take(50).ToQueryString(),
|
||||
$"{provider} unexpectedly translated a primitive-collection projection");
|
||||
|
||||
Should.Throw<InvalidOperationException>(
|
||||
() => context.SongMetadata.SelectMany(m => m.AlbumArtists).Distinct().Take(50).ToQueryString(),
|
||||
$"{provider} unexpectedly translated a primitive-collection projection");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void List_Valued_Page_Query_Has_No_Predicate_Beyond_The_Keyset_Cursor()
|
||||
{
|
||||
// This is the whole basis of the row bound, so it is asserted rather than assumed. LIMIT truncates what
|
||||
// survives a RESIDUAL predicate — one that discards rows the engine already produced — so with such a
|
||||
// predicate present it bounds the output rather than the row count, and the engine may produce and
|
||||
// discard arbitrarily many rows first. That is how four successive revisions scanned past their own
|
||||
// bound. The cursor `Id > @AfterId` is NOT such a predicate: it is a seek on the ordering key, which
|
||||
// positions the scan without discarding anything, so LIMIT n yields n logical rows.
|
||||
//
|
||||
// What this test can and cannot do: it pins the SQL STRING. It cannot pin an execution plan, MVCC
|
||||
// visibility work or payload I/O -- physical work is NOT bounded (see the record: MySQL traverses
|
||||
// deleted-but-unpurged index records, and TEXT payloads spill to overflow pages).
|
||||
string sql = GetSearchFieldValuesHandler.ListValuedSql("Artists");
|
||||
|
||||
sql.ShouldBe(
|
||||
"SELECT Id, Artists AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch");
|
||||
|
||||
// named explicitly so a future "optimization" that reintroduces server-side selectivity fails here
|
||||
sql.ShouldNotContain("LIKE");
|
||||
sql.ShouldNotContain("LOWER");
|
||||
sql.ShouldNotContain("IS NOT NULL");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. The SQL function name is duplicated — the handler lives in Application, which must
|
||||
/// not reference a provider assembly, so it cannot use the constant the registration side defines. A
|
||||
/// rename on one side alone would compile cleanly and fail only at runtime, only on SQLite, only for
|
||||
/// non-ASCII queries; this pins the two together instead.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void Unicode_Fold_Function_Name_Matches_The_Registration() =>
|
||||
GetSearchFieldValuesHandler.UpperFunction.ShouldBe(SqliteUnicodeFunctions.UpperInvariantFunction);
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668. Unlike the list-valued walk, this query KEEPS its selectivity in SQL — it is a
|
||||
/// bounded <c>LIMIT</c>ed prefix query exactly like the EF one it supplements, so a <c>LIKE</c> here
|
||||
/// is correct rather than the trap the walk's shape test guards against. What must hold is that the
|
||||
/// fold is the registered Unicode-correct one and NOT the provider's ASCII-only builtin, and that the
|
||||
/// wildcard escape is declared.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void Unicode_Fold_Query_Uses_The_Registered_Fold_And_Declares_Its_Escape()
|
||||
{
|
||||
string sql = GetSearchFieldValuesHandler.UnicodeFoldSql("Genre", "Name", null);
|
||||
|
||||
sql.ShouldBe(
|
||||
"SELECT DISTINCT Name AS Value FROM Genre "
|
||||
+ "WHERE etv_upper(Name) LIKE @Pattern ESCAPE '\\' ORDER BY Name LIMIT @Limit");
|
||||
|
||||
// The point of the whole change: SQLite's BUILTIN lower()/upper() fold ASCII only, so quietly falling
|
||||
// back to one reinstates #668. Checked by removing the qualified call first — Shouldly's string
|
||||
// assertions are case-INSENSITIVE by default, so a bare ShouldNotContain("UPPER(") matches inside
|
||||
// "etv_upper(" and fails against correct SQL.
|
||||
sql.ShouldNotContain("LOWER(");
|
||||
sql.Replace($"{GetSearchFieldValuesHandler.UpperFunction}(", "", StringComparison.Ordinal)
|
||||
.ShouldNotContain("UPPER(");
|
||||
|
||||
// a discriminator predicate is parenthesised and ANDed, so an OR inside it cannot swallow the match
|
||||
GetSearchFieldValuesHandler.UnicodeFoldSql("Tag", "Name", "ExternalTypeId IS NULL OR X")
|
||||
.ShouldContain("WHERE (ExternalTypeId IS NULL OR X) AND etv_upper(Name) LIKE @Pattern");
|
||||
}
|
||||
|
||||
private static IEnumerable<(string Provider, Func<TvContext> Create)> Providers() =>
|
||||
[
|
||||
("sqlite", Sqlite),
|
||||
("mysql", MySql)
|
||||
];
|
||||
|
||||
private static TvContext Sqlite()
|
||||
{
|
||||
TvContext.IsSqlite = true;
|
||||
var builder = new DbContextOptionsBuilder<TvContext>();
|
||||
builder.UseSqlite("Data Source=:memory:");
|
||||
return Create(builder.Options);
|
||||
}
|
||||
|
||||
private static TvContext MySql()
|
||||
{
|
||||
TvContext.IsSqlite = false;
|
||||
var builder = new DbContextOptionsBuilder<TvContext>();
|
||||
builder.UseMySql(
|
||||
"Server=localhost;Database=ersatztv_query_shape;User=root;Password=ersatztv;",
|
||||
new MySqlServerVersion(new Version(8, 0, 36)));
|
||||
return Create(builder.Options);
|
||||
}
|
||||
|
||||
private static TvContext Create(DbContextOptions<TvContext> options) =>
|
||||
new(
|
||||
options,
|
||||
NullLoggerFactory.Instance,
|
||||
new SlowQueryInterceptor(NullLogger<SlowQueryInterceptor>.Instance));
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
using ErsatzTV.Application.Search.Queries;
|
||||
using ErsatzTV.Core.Api.Search;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Infrastructure;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
using ErsatzTV.Infrastructure.MySql.Data;
|
||||
using ErsatzTV.Infrastructure.Sqlite.Data;
|
||||
using LanguageExt;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MySqlConnector;
|
||||
using NUnit.Framework;
|
||||
using Shouldly;
|
||||
|
||||
namespace ErsatzTV.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// ersatztv#668, EXECUTED on both providers. The bug was a collation/fold difference, so it lives exactly
|
||||
/// where a single-provider test cannot see it: SQLite's <c>LOWER()</c> folds ASCII only and UNDER-matched
|
||||
/// a stored <c>Édith</c>, while MySQL's is Unicode-aware and reaches it unaided. (Its column collation
|
||||
/// is accent-INsensitive, but the executed comparison is not — see the method docstring below.)
|
||||
/// <para>
|
||||
/// <see cref="ErsatzTV.Tests.Application.Search.GetSearchFieldValuesHandlerTests" /> covers the
|
||||
/// SQLite semantics in depth against in-memory SQLite, and
|
||||
/// <c>SearchFieldValuesQueryShapeTests</c> pins the generated SQL for both providers without a
|
||||
/// server. Neither can show that a REAL MySQL server returns the accented value — the fix's central
|
||||
/// claim is "on both providers", and on MySQL that rests on the server's Unicode-aware
|
||||
/// <c>LOWER()</c> rather than on any code this repo owns — explicitly NOT on its collation, which
|
||||
/// the executed comparison bypasses. That is precisely the kind of assumption worth executing.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// MySQL needs a live server via <c>ETV_TEST_MYSQL_CONNECTION</c>. Without it the MySQL fixture
|
||||
/// <b>ignores</b> — a visible skip, never a silent pass. Setting <c>ETV_REQUIRE_MYSQL_TESTS=1</c>
|
||||
/// turns that skip into a hard failure, so an ARMED lane cannot degrade into "connected to nothing
|
||||
/// and passed".
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>CI does not currently arm it</b>, so in CI this half SKIPS. Running MySQL fixtures against the
|
||||
/// live service was implemented and then removed as non-deterministic — see the note in
|
||||
/// <c>.gitea/workflows/docker-build.yml</c>; re-arming is tracked by ersatztv#627. Do not read the
|
||||
/// REQUIRE variable above as a guarantee that something enforces this today: nothing does. This
|
||||
/// mirrors <see cref="LibraryFolderDedupeMigrationTests" /> deliberately; the two fixtures share the
|
||||
/// contract, not code, because their setup needs differ.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[TestFixture(TestProvider.Sqlite)]
|
||||
[TestFixture(TestProvider.MySql)]
|
||||
[NonParallelizable]
|
||||
public class SearchFieldValuesProviderTests(TestProvider provider)
|
||||
{
|
||||
private const string MySqlConnectionVariable = "ETV_TEST_MYSQL_CONNECTION";
|
||||
private const string MySqlRequiredVariable = "ETV_REQUIRE_MYSQL_TESTS";
|
||||
|
||||
private string _databasePath = null!;
|
||||
private string? _mySqlConnectionString;
|
||||
private DbContextOptions<TvContext> _options = null!;
|
||||
|
||||
[SetUp]
|
||||
public async Task SetUp()
|
||||
{
|
||||
if (provider is TestProvider.Sqlite)
|
||||
{
|
||||
TvContext.IsSqlite = true;
|
||||
TvContext.LastInsertedRowId = "last_insert_rowid()";
|
||||
TvContext.CaseInsensitiveCollation = "NOCASE";
|
||||
TvContext.IsUniqueConstraintViolation = SqliteErrorClassifier.IsUniqueConstraintViolation;
|
||||
TvContext.RegisterUnicodeCaseFunctions = SqliteUnicodeFunctions.Register;
|
||||
|
||||
_databasePath = Path.Combine(Path.GetTempPath(), $"etv668-{Guid.NewGuid():N}.sqlite3");
|
||||
_options = new DbContextOptionsBuilder<TvContext>()
|
||||
.UseSqlite($"Data Source={_databasePath}")
|
||||
.Options;
|
||||
}
|
||||
else
|
||||
{
|
||||
string? baseConnectionString = Environment.GetEnvironmentVariable(MySqlConnectionVariable);
|
||||
if (string.IsNullOrWhiteSpace(baseConnectionString))
|
||||
{
|
||||
string message =
|
||||
$"{MySqlConnectionVariable} is not set, so the MySql half of the #668 facet-value fixture "
|
||||
+ "cannot run. This endpoint's correctness is collation-dependent and therefore "
|
||||
+ "provider-specific, so the coverage is not optional in CI.";
|
||||
|
||||
if (IsTrue(Environment.GetEnvironmentVariable(MySqlRequiredVariable)))
|
||||
{
|
||||
Assert.Fail($"{message} {MySqlRequiredVariable} is set, so this is a failure, not a skip.");
|
||||
}
|
||||
|
||||
Assert.Ignore($"{message} Set it to run this locally.");
|
||||
}
|
||||
|
||||
// A database of our own with a name that has never been used, so isolation does not depend on a
|
||||
// wipe succeeding. Dropped and its pool cleared in TearDown.
|
||||
_mySqlConnectionString =
|
||||
new MySqlConnectionStringBuilder(baseConnectionString) { Database = $"etv668_{Guid.NewGuid():N}" }
|
||||
.ConnectionString;
|
||||
|
||||
TvContext.IsSqlite = false;
|
||||
TvContext.LastInsertedRowId = "last_insert_id()";
|
||||
TvContext.CaseInsensitiveCollation = "utf8mb4_general_ci";
|
||||
TvContext.IsUniqueConstraintViolation = MySqlErrorClassifier.IsUniqueConstraintViolation;
|
||||
|
||||
// Explicitly the no-op: MySQL's own LOWER() is Unicode-aware, so the handler must reach the
|
||||
// accented value WITHOUT any custom fold. Wiring SQLite's here would mask that.
|
||||
TvContext.RegisterUnicodeCaseFunctions = static _ => { };
|
||||
|
||||
_options = new DbContextOptionsBuilder<TvContext>()
|
||||
.UseMySql(_mySqlConnectionString, ServerVersion.AutoDetect(_mySqlConnectionString))
|
||||
.Options;
|
||||
}
|
||||
|
||||
// Schema creation deliberately does NOT happen here: NUnit skips [TearDown] when [SetUp] throws, so
|
||||
// a failure part-way through EnsureCreatedAsync would strand the created database (and its pooled
|
||||
// connection) with nothing to drop it. The test body creates it instead, matching the sibling
|
||||
// fixture, whose SetUp likewise cannot strand one.
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public async Task TearDown()
|
||||
{
|
||||
if (provider is TestProvider.Sqlite)
|
||||
{
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
foreach (string path in new[] { _databasePath, $"{_databasePath}-wal", $"{_databasePath}-shm" })
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (_mySqlConnectionString is not null)
|
||||
{
|
||||
await using (TvContext context = Create(_options))
|
||||
{
|
||||
await context.Database.EnsureDeletedAsync();
|
||||
}
|
||||
|
||||
// MySqlConnector keys pools by connection string; a fresh database name means a fresh pool, and
|
||||
// leaving it uncleared leaks a server thread per test until max_connections is exhausted.
|
||||
await using var probe = new MySqlConnection(_mySqlConnectionString);
|
||||
await MySqlConnection.ClearPoolAsync(probe);
|
||||
|
||||
_mySqlConnectionString = null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The #668 headline, executed: a stored value whose prefix carries an UPPERCASE non-ASCII character
|
||||
/// is reachable from both casings of the query, on whichever provider this fixture is running.
|
||||
/// <para>
|
||||
/// Negative controls: "Zulu" (trivially unrelated) and "Edith" (unaccented, the near miss).
|
||||
/// <b>Be precise about what "Edith" does and does not prove.</b> It was added expecting MySQL to
|
||||
/// OVER-match it — the column collation is <c>utf8mb4_0900_ai_ci</c>, so <c>é</c> equals <c>e</c>
|
||||
/// — which would have made the in-memory ordinal filter load-bearing here. Measured against a
|
||||
/// live 8.4 server, it does not: deleting that filter leaves this test green, because the driver
|
||||
/// binds the LIKE pattern with a BINARY collation and the executed comparison is therefore
|
||||
/// accent-SENSITIVE. (A literal pattern typed by hand DOES over-match — a different query from
|
||||
/// the one the handler runs.) So the row pins the accent-sensitive result on both providers and
|
||||
/// documents the near miss; it does NOT exercise an over-match correction, because with the
|
||||
/// CURRENT driver there is nothing to correct. That is a driver-contingent fact, not a law: a
|
||||
/// driver or protocol change that made the pattern ci-collated would restore the over-match, and
|
||||
/// the ordinal filter — which stays regardless — would then be doing real work here.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[TestCase("é", TestName = "Uppercase_Accent_Reachable_From_Lowercase_Query")]
|
||||
[TestCase("É", TestName = "Uppercase_Accent_Reachable_From_Uppercase_Query")]
|
||||
public async Task Stored_Uppercase_Accent_Is_Reachable(string query)
|
||||
{
|
||||
await using (TvContext context = Create(_options))
|
||||
{
|
||||
await context.Database.EnsureCreatedAsync();
|
||||
context.Set<Genre>().AddRange(
|
||||
new Genre { Name = "Édith" },
|
||||
new Genre { Name = "Edith" },
|
||||
new Genre { Name = "Zulu" });
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var handler = new GetSearchFieldValuesHandler(new TestDbContextFactory(_options));
|
||||
|
||||
Option<SearchFieldValuesResponseModel> result = await handler.Handle(
|
||||
new GetSearchFieldValues("genre", query, 50),
|
||||
CancellationToken.None);
|
||||
|
||||
result.IsSome.ShouldBeTrue();
|
||||
result.IfSome(r => r.Values.ShouldBe(new List<string> { "Édith" }));
|
||||
}
|
||||
|
||||
private static bool IsTrue(string? value) =>
|
||||
value is "1" || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static TvContext Create(DbContextOptions<TvContext> options) =>
|
||||
new(
|
||||
options,
|
||||
NullLoggerFactory.Instance,
|
||||
new SlowQueryInterceptor(NullLogger<SlowQueryInterceptor>.Instance));
|
||||
|
||||
private sealed class TestDbContextFactory(DbContextOptions<TvContext> options) : IDbContextFactory<TvContext>
|
||||
{
|
||||
public TvContext CreateDbContext() => Create(options);
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ public sealed class InMemoryTvContext : IAsyncDisposable
|
||||
{
|
||||
TvContext.IsSqlite = true;
|
||||
TvContext.IsUniqueConstraintViolation = SqliteErrorClassifier.IsUniqueConstraintViolation;
|
||||
TvContext.RegisterUnicodeCaseFunctions = SqliteUnicodeFunctions.Register;
|
||||
|
||||
var connection = new SqliteConnection("Data Source=:memory:;Foreign Keys=False");
|
||||
await connection.OpenAsync();
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
using ErsatzTV.Controllers.Api.Requests;
|
||||
using ErsatzTV.Core.Domain;
|
||||
using ErsatzTV.Infrastructure.Data;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace ErsatzTV.Tests.Support;
|
||||
|
||||
/// <summary>
|
||||
/// One selection-type matrix, shared by every fixture that exercises a tagged-union selection
|
||||
/// (rerun collections and playlist items). Both consumers of
|
||||
/// <c>MediaCollections.Mapper.ProjectMediaItemToViewModel</c> are proved against the SAME data, so
|
||||
/// widening the shared switch cannot be discharged for the second consumer by inspection alone —
|
||||
/// which is the method that produced #671 in the first place.
|
||||
/// </summary>
|
||||
internal static class SelectionSeedData
|
||||
{
|
||||
public const int SelectedId = 42;
|
||||
|
||||
/// <summary>
|
||||
/// Derived from production rather than hand-listed, so a newly-supported type joins the matrix
|
||||
/// automatically and trips the <c>default:</c> arms below until someone teaches them about it.
|
||||
/// Note this is the RERUN-COLLECTION predicate, used for playlist items as a deliberate
|
||||
/// SUPERSET: <c>ReplacePlaylistItemsHandler.CollectionTypeMustBeValid</c> has no
|
||||
/// <c>RemoteStream</c> case, so a RemoteStream playlist item cannot be created through the write
|
||||
/// API today and the playlist fixture seeds that row directly. Covering it is forward-looking,
|
||||
/// not a claim that the two sets are equivalent — split this if they ever legitimately diverge.
|
||||
/// </summary>
|
||||
public static IEnumerable<CollectionType> SupportedSelectionTypes =>
|
||||
Enum.GetValues<CollectionType>().Where(RerunCollectionRequestMapping.IsSupportedSelectionType);
|
||||
|
||||
/// <summary>
|
||||
/// The exact projected name per type. Pinning the whole string — rather than merely asserting
|
||||
/// "not a placeholder" — is what makes a missing NESTED include leg visible: dropping
|
||||
/// Episode → Season → Show still yields the placeholder-free "s??e04 - Selected episode", and
|
||||
/// dropping MusicVideo → Artist still yields "Selected music video". Both would sail past a
|
||||
/// looser assertion while having lost real data.
|
||||
/// </summary>
|
||||
public static string ExpectedName(CollectionType collectionType) =>
|
||||
collectionType switch
|
||||
{
|
||||
CollectionType.Collection => "Selected collection",
|
||||
CollectionType.MultiCollection => "Selected multi collection",
|
||||
CollectionType.SmartCollection => "Selected smart collection",
|
||||
CollectionType.TelevisionShow => "Selected show (2020)",
|
||||
CollectionType.TelevisionSeason => "Parent show (2020) - Season 3",
|
||||
CollectionType.Artist => "Selected artist",
|
||||
CollectionType.Movie => "Selected movie (2019)",
|
||||
CollectionType.Episode => "Episode's show - s02e04 - Selected episode",
|
||||
CollectionType.MusicVideo => "Video's artist - Selected music video",
|
||||
CollectionType.OtherVideo => "Selected other video",
|
||||
CollectionType.Song => "Song artist - Selected song",
|
||||
CollectionType.Image => "Selected image",
|
||||
CollectionType.RemoteStream => "Selected remote stream",
|
||||
_ => throw new AssertionException($"No expected name pinned for {collectionType}")
|
||||
};
|
||||
|
||||
public static async Task SeedSelection(TvContext context, CollectionType collectionType)
|
||||
{
|
||||
switch (collectionType)
|
||||
{
|
||||
case CollectionType.Collection:
|
||||
context.Collections.Add(new Collection
|
||||
{
|
||||
Id = SelectedId,
|
||||
Name = "Selected collection",
|
||||
MediaItems = []
|
||||
});
|
||||
break;
|
||||
case CollectionType.MultiCollection:
|
||||
context.MultiCollections.Add(new MultiCollection
|
||||
{
|
||||
Id = SelectedId,
|
||||
Name = "Selected multi collection"
|
||||
});
|
||||
break;
|
||||
case CollectionType.SmartCollection:
|
||||
context.SmartCollections.Add(new SmartCollection
|
||||
{
|
||||
Id = SelectedId,
|
||||
Name = "Selected smart collection",
|
||||
Query = "tag:family"
|
||||
});
|
||||
break;
|
||||
case CollectionType.TelevisionShow:
|
||||
context.Shows.Add(new Show
|
||||
{
|
||||
Id = SelectedId,
|
||||
ShowMetadata = [new ShowMetadata { Title = "Selected show", Year = 2020 }]
|
||||
});
|
||||
break;
|
||||
case CollectionType.TelevisionSeason:
|
||||
context.Seasons.Add(new Season
|
||||
{
|
||||
Id = SelectedId,
|
||||
SeasonNumber = 3,
|
||||
Show = new Show
|
||||
{
|
||||
Id = 900,
|
||||
ShowMetadata = [new ShowMetadata { Title = "Parent show", Year = 2020 }]
|
||||
}
|
||||
});
|
||||
break;
|
||||
case CollectionType.Artist:
|
||||
context.Artists.Add(new Artist
|
||||
{
|
||||
Id = SelectedId,
|
||||
ArtistMetadata = [new ArtistMetadata { Title = "Selected artist" }]
|
||||
});
|
||||
break;
|
||||
case CollectionType.Movie:
|
||||
context.Movies.Add(new Movie
|
||||
{
|
||||
Id = SelectedId,
|
||||
MovieMetadata = [new MovieMetadata { Title = "Selected movie", Year = 2019 }]
|
||||
});
|
||||
break;
|
||||
case CollectionType.Episode:
|
||||
context.Episodes.Add(new Episode
|
||||
{
|
||||
Id = SelectedId,
|
||||
EpisodeMetadata = [new EpisodeMetadata { Title = "Selected episode", EpisodeNumber = 4 }],
|
||||
Season = new Season
|
||||
{
|
||||
Id = 901,
|
||||
SeasonNumber = 2,
|
||||
Show = new Show
|
||||
{
|
||||
Id = 902,
|
||||
ShowMetadata = [new ShowMetadata { Title = "Episode's show", Year = 2018 }]
|
||||
}
|
||||
}
|
||||
});
|
||||
break;
|
||||
case CollectionType.MusicVideo:
|
||||
context.MusicVideos.Add(new MusicVideo
|
||||
{
|
||||
Id = SelectedId,
|
||||
MusicVideoMetadata = [new MusicVideoMetadata { Title = "Selected music video" }],
|
||||
Artist = new Artist
|
||||
{
|
||||
Id = 903,
|
||||
ArtistMetadata = [new ArtistMetadata { Title = "Video's artist" }]
|
||||
}
|
||||
});
|
||||
break;
|
||||
case CollectionType.OtherVideo:
|
||||
context.OtherVideos.Add(new OtherVideo
|
||||
{
|
||||
Id = SelectedId,
|
||||
OtherVideoMetadata = [new OtherVideoMetadata { Title = "Selected other video" }]
|
||||
});
|
||||
break;
|
||||
case CollectionType.Song:
|
||||
context.Songs.Add(new Song
|
||||
{
|
||||
Id = SelectedId,
|
||||
SongMetadata =
|
||||
[new SongMetadata { Title = "Selected song", Artists = ["Song artist"] }]
|
||||
});
|
||||
break;
|
||||
case CollectionType.Image:
|
||||
context.Images.Add(new Image
|
||||
{
|
||||
Id = SelectedId,
|
||||
ImageMetadata = [new ImageMetadata { Title = "Selected image" }]
|
||||
});
|
||||
break;
|
||||
case CollectionType.RemoteStream:
|
||||
context.RemoteStreams.Add(new RemoteStream
|
||||
{
|
||||
Id = SelectedId,
|
||||
Url = "http://example.invalid/stream",
|
||||
RemoteStreamMetadata = [new RemoteStreamMetadata { Title = "Selected remote stream" }]
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw new AssertionException(
|
||||
$"{collectionType} is a supported selection type but this suite does not know how " +
|
||||
"to seed it — teach SeedSelection about it rather than narrowing the matrix.");
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Assigns the one foreign key the tagged union uses for this type. Shared so the rerun and
|
||||
/// playlist fixtures cannot disagree about which slot a type occupies.
|
||||
/// </summary>
|
||||
public static void ApplySelection(
|
||||
CollectionType collectionType,
|
||||
Action<int> setCollectionId,
|
||||
Action<int> setMultiCollectionId,
|
||||
Action<int> setSmartCollectionId,
|
||||
Action<int> setMediaItemId)
|
||||
{
|
||||
switch (collectionType)
|
||||
{
|
||||
case CollectionType.Collection:
|
||||
setCollectionId(SelectedId);
|
||||
break;
|
||||
case CollectionType.MultiCollection:
|
||||
setMultiCollectionId(SelectedId);
|
||||
break;
|
||||
case CollectionType.SmartCollection:
|
||||
setSmartCollectionId(SelectedId);
|
||||
break;
|
||||
default:
|
||||
setMediaItemId(SelectedId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -205,7 +205,11 @@ public class SearchController(IMediator mediator) : ControllerBase
|
||||
"Returns distinct whole values from the database for the given text field, filtered by an " +
|
||||
"optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. " +
|
||||
"404 when the field is unknown, is not a text field, or is a text field with no distinct-value " +
|
||||
"source.")]
|
||||
"source. The final filter, dedup and ordering applied to the response are ordinal and not " +
|
||||
"culture-dependent; note that fields sourced by a plain database query are additionally " +
|
||||
"pre-filtered by the database collation first, which on SQLite is ASCII-only. The list-valued " +
|
||||
"music fields (artist, album_artist) are bounded best-effort: the server reads a bounded number " +
|
||||
"of song rows per request, so a library larger than that bound may yield a subset of the matches.")]
|
||||
[EndpointGroupName("general")]
|
||||
[ProducesResponseType(typeof(SearchFieldValuesResponseModel), StatusCodes.Status200OK)]
|
||||
[ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)]
|
||||
|
||||
@@ -649,6 +649,7 @@ public class Startup
|
||||
TvContext.LastInsertedRowId = "last_insert_rowid()";
|
||||
TvContext.CaseInsensitiveCollation = "NOCASE";
|
||||
TvContext.IsUniqueConstraintViolation = SqliteErrorClassifier.IsUniqueConstraintViolation;
|
||||
TvContext.RegisterUnicodeCaseFunctions = SqliteUnicodeFunctions.Register;
|
||||
|
||||
SqlMapper.AddTypeHandler(new DateTimeOffsetHandler());
|
||||
SqlMapper.AddTypeHandler(new GuidHandler());
|
||||
@@ -660,6 +661,10 @@ public class Startup
|
||||
TvContext.LastInsertedRowId = "last_insert_id()";
|
||||
TvContext.CaseInsensitiveCollation = "utf8mb4_general_ci";
|
||||
TvContext.IsUniqueConstraintViolation = MySqlErrorClassifier.IsUniqueConstraintViolation;
|
||||
|
||||
// MySQL's LOWER() is already Unicode-aware, so the facet-value handler never takes the
|
||||
// custom-fold branch here; assigned explicitly so a provider switch cannot inherit SQLite's.
|
||||
TvContext.RegisterUnicodeCaseFunctions = static _ => { };
|
||||
}
|
||||
|
||||
Log.Logger.Information("Transcode folder is {Folder}", FileSystemLayout.TranscodeFolder);
|
||||
|
||||
@@ -18063,7 +18063,7 @@
|
||||
"Search"
|
||||
],
|
||||
"summary": "List distinct database values for a text search field",
|
||||
"description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source.",
|
||||
"description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source. The final filter, dedup and ordering applied to the response are ordinal and not culture-dependent; note that fields sourced by a plain database query are additionally pre-filtered by the database collation first, which on SQLite is ASCII-only. The list-valued music fields (artist, album_artist) are bounded best-effort: the server reads a bounded number of song rows per request, so a library larger than that bound may yield a subset of the matches.",
|
||||
"operationId": "GetSearchFieldValues",
|
||||
"parameters": [
|
||||
{
|
||||
|
||||
+78
-1
@@ -146,6 +146,41 @@ Exemplars:
|
||||
`Brief`. `Remediation.Kind` is a mapped **string** ("ExternalDoc"/"AppRoute"), not a wire enum —
|
||||
same pattern as `Status`. See `decisions.md` 2026-07-17 (#164).
|
||||
|
||||
### 2a. Flattening a tagged-union selection (read path)
|
||||
|
||||
Several DTOs flatten a "exactly one of these navigations is populated" tagged union to a single
|
||||
`selectedId` + `selectedName` pair (`RerunCollectionResponseModel`, and the playlist-item shape).
|
||||
Two rules, both learned from #671, where the list endpoint returned a null selection for **every**
|
||||
row and the detail GET 500'd for two of its media types:
|
||||
|
||||
- **One include chain per projected aggregate, shared by every handler that projects it.** Put it in
|
||||
a `<Aggregate>QueryExtensions` extension method and call it from the list handler *and* the by-id
|
||||
handler. Exemplars: `RerunCollectionQueryExtensions.IncludeSelectionDetails()`,
|
||||
`ProgramScheduleItemQueryExtensions.IncludeScheduleItemDetails()`. Two hand-maintained chains
|
||||
drift, and the one that drifts is usually the paged list, whose rows are individually less
|
||||
obviously wrong. Applying it before `Skip`/`Take` is fine — EF applies the includes to the paged
|
||||
subquery, so the cost is bounded by `PageSize`, not by the table.
|
||||
- **The id and the name must not share a single point of failure.** When both are read off the same
|
||||
eager-loaded navigation, the id is only ever as available as the name — so an un-included type
|
||||
doesn't merely render an unlabelled badge, it drops the selected id, and an editor that
|
||||
round-trips that id silently clears the user's stored selection. Accordingly a media-item
|
||||
flattening switch never ends in `_ => null`: an unrecognized subtype keeps its id and takes a
|
||||
conspicuous `[unsupported media type: X]` name. Throwing is the wrong lever — it would fail an
|
||||
entire paged GET over one unreadable row. The shared switch is
|
||||
`MediaCollections.Mapper.ProjectMediaItemToViewModel`.
|
||||
|
||||
Corollary for the mappers themselves: `MediaItems.Mapper`'s projections are reached from handlers
|
||||
whose include chains differ, so every metadata navigation is read through `Optional(...).Flatten()`
|
||||
and degrades to the `"???"` placeholder rather than throwing. A bare `x.Season.Show.ShowMetadata`
|
||||
inside a projection is a latent 500 on some other caller's GET.
|
||||
|
||||
**And it is not only navigations.** `SongMetadata.Artists` is a nullable EF *primitive collection*
|
||||
(a JSON array in one column), which `FallbackMetadataProvider` leaves unassigned for a song whose
|
||||
tags failed to read — and `string.Join` throws `ArgumentNullException` on a null sequence, not a
|
||||
`NullReferenceException`. Adding an include is therefore not automatically safe: it can promote a
|
||||
latent throw on a previously-unloaded member into a live 500 that fails the whole page. When you
|
||||
widen an include chain, audit what the newly-reachable projection dereferences.
|
||||
|
||||
## 3. Error mapping
|
||||
|
||||
Central helper: `ErsatzTV/Extensions/ApiResults.cs`. Use these extension methods instead of
|
||||
@@ -444,7 +479,7 @@ standard credential (catalog-read tier — no `[RequiresAuthentication]`):
|
||||
Query params: `q` (optional prefix filter, case-insensitive, default empty) and `limit` (optional,
|
||||
clamped `1..50`, default 50). `{name}` is allow-listed to `SearchFieldCatalog` fields with
|
||||
`type: "text"` AND a distinct-value source in the database — an unknown field, a non-text field (e.g.
|
||||
an enum), or a text field without a source (`title`, `show_title`, `album_artist`) 404s rather than
|
||||
an enum), or a text field without a source (`title`, `show_title`) 404s rather than
|
||||
returning an empty list, since enum fields already ship their values inline on
|
||||
`GET /api/v1/search/fields` and never need this endpoint. Returns `SearchFieldValuesResponseModel`
|
||||
(`{ values: string[] }`), sourced from a per-field distinct-values DB query (`IDbContextFactory<TvContext>`),
|
||||
@@ -452,6 +487,48 @@ not the Lucene term dictionary — analyzed text fields store lowercased word to
|
||||
No server-side caching. Powers the visual rule builder's value-input combobox for text fields; see
|
||||
`docs/decisions.md` 2026-07-23 (#434) and `spa-conventions.md` §12.
|
||||
|
||||
**Bounded best-effort for list-valued fields (#578)**: `artist` and `album_artist` are backed (wholly
|
||||
or partly) by `SongMetadata.Artists`/`AlbumArtists`, which EF maps as **primitive collections** — one
|
||||
JSON array per row in a single column, with no server-side projection on either provider. `album_artist`
|
||||
therefore no longer 404s, and `artist` now also covers free-text music-video (`MusicVideoArtist`) and
|
||||
song credits, not only entity artists. Those rows are read by a keyset page whose only condition is the
|
||||
**cursor** — no residual predicate that could discard a row — and filtered in memory, bounded at 20,000
|
||||
logical rows per request; so on a larger library the
|
||||
response may be a bounded subset of the matches — bounded in LOGICAL ROWS, which is not the same as
|
||||
bounded work or bytes. Say so in the `[EndpointDescription]` of any endpoint that adopts this shape.
|
||||
|
||||
Three rules generalize beyond this endpoint.
|
||||
|
||||
1. **`LIMIT` bounds the OUTPUT, not the row count, whenever a RESIDUAL predicate is present.** The
|
||||
distinction is not "predicate vs none" — a keyset cursor is a predicate. It is that a *seekable
|
||||
predicate on the ordering key* positions the scan and never discards a row, while a *residual*
|
||||
predicate (`LIKE`, `LOWER`, `IS NOT NULL`) throws away rows the engine already produced, so `LIMIT`
|
||||
truncates the survivors and says nothing about how many were produced — a query matching nothing
|
||||
must examine every eligible row before it can return an empty page. To bound rows, drop the residual
|
||||
predicate, page by row position over the primary key, and filter in memory. This endpoint got it
|
||||
wrong four times: bounding the result, then candidates returned, then `Id` keyspace width (keyspace
|
||||
is not rows — one live row at `Id` 20001 behind 20,000 deleted ones reads nothing), before arriving
|
||||
at "cursor only".
|
||||
**And scope the resulting claim to LOGICAL ROWS.** It is not bounded physical work: MySQL still
|
||||
traverses deleted-but-unpurged index records, so deletion history keeps affecting cost, and an
|
||||
unrestricted `TEXT` column spills to overflow pages so a row count implies no byte or page-read
|
||||
count. A SQL-string assertion pins none of that — not a plan, not visibility work, not I/O.
|
||||
2. **A SQL pre-filter under an in-memory exact filter may over-match but must never under-match — and
|
||||
that licence is void the moment the candidate set is truncated.** Widening the predicate then starves
|
||||
the budget with rows that cannot match. If you find yourself proving a superset property to keep a
|
||||
pre-filter honest, consider deleting the pre-filter instead: here it removed a JSON-escaping bug
|
||||
class, an exhaustive Unicode sweep and an `ESCAPE` portability workaround along with it.
|
||||
3. **Prefix matching, dedup and ordering must be ordinal, not current-culture** (`OrdinalIgnoreCase`,
|
||||
`StringComparer.Ordinal`): `UseRequestLocalization` honours `Accept-Language`, so `ToLower()` and the
|
||||
default linguistic `StartsWith(string)` let a caller change the result by changing a header. **Scope
|
||||
the claim to the stage that actually holds it** — a value set that a database `LOWER`/`DISTINCT`/
|
||||
`ORDER BY`/`LIMIT` already filtered and truncated is not ordinal no matter what runs after it, and
|
||||
saying otherwise in an `[EndpointDescription]` publishes a false contract (ersatztv#668).
|
||||
|
||||
Full rationale, the measured transfer cost, the four-attempts table and the rejected
|
||||
normalized-side-table alternative (ersatztv#669): `api.search-field-values-sources` (supersedes
|
||||
`api.search-field-values`).
|
||||
|
||||
**Param + DTO expansion (#293, cap `search/all-items`)**: no new endpoint — `GET /api/v1/search/all-items`
|
||||
gained two **optional** query params (`pageSize` default 500, clamped 1–1000 via the §1 Logs `Math.Clamp`
|
||||
precedent; `pageNum` 0-based, clamped `0..2_000_000` so `pageNum * pageSize` can't overflow `int` to a 500)
|
||||
|
||||
+581
-13
@@ -39,6 +39,8 @@ Upstream's final release was **`v26.3.0`** (archived). Our line continues from t
|
||||
| `v26.10.0` | Auto-Tune channel workflow (#69) + weighted content distribution (#70); scheduling refactors, health-check remediation UX (#164), HLS cold-start instrumentation (#350), security hardening (#293/#376/#308). |
|
||||
| `v26.11.0` | **QSV profiles decode via VA-API** — `QsvPreferNativeDecoder`, default **on**, fixes ~50% channel cold-start failures on Intel (#498); unified logo/on-screen bug via a shared watermark preset (#67). Media-scanner resilience: Jellyfin mixed-content libraries (#489), music-video scan correctness (#488/#494/#497), remote-stream probing before ffmpeg (#473/#480); weighted-distribution SPA (#404). **First release deployed to `jazz`** (server-management#633). |
|
||||
| `v26.12.0` | **`ErsatzTV.Mcp` MCP server** — read + cautious-write over `/api/v1`, `ERSATZTV_ALLOW_WRITES`-gated (#58). **External channel-logo URLs download + cache at save time** (#525), with the on-screen bug now rendered for external-URL logos (#502). HLS cold-start hardening: burst-read the first segments so start isn't `-readrate`-bound (#350) and floor QSV extra hardware frames so an unthrottled read can't exhaust the pool (#529); remote graphics-engine image fetches bounded — timeout, size cap, decode cap, redirects, pooling (#511). Decision-lifecycle tooling + parallel-orientation startup rewrite (#520/#521); CI `docker build` lane rebalance (#508). |
|
||||
| `v26.13.0` | **RuleBuilder maturation** — arbitrary-depth group nesting (#436), inline smart-query authoring in Channel Builder (#437), DB-sourced facet typeahead + relative-date operators + validation (#434/#435/#438), and an artist typeahead covering music-video/song credits with `album_artist` no longer 404ing (#578). **Per-channel On Now/Next transient overlay** (#74/#570) and **per-schedule clock-boundary padding** (#392); in-browser channel preview (#60); Auto-Tune per-source weight steppers + exclude/add-untagged (#440). Library-browse pickers now resolve by search instead of a 100-row window, closing several silent at-cap truncations (#644/#650/#651/#634). Correctness: one watermark resolver for all four attachment points, incl. `MiddleCenter` (#503/#510); QSV HDR tonemaps through OpenCL because `vpp_qsv=tonemap` is a silent no-op (#505); `LibraryFolder` unique index + concurrent-insert tolerance (#491); per-library music-video identity with soft trash (#496); Jellyfin Album/Track music-video projection (#177); metadata-collection dedup (#500); accented facet values via a registered Unicode fold on SQLite (#668); `WorkAheadSlots` atomic slot claim, never a negative count (#536/#539); on-demand guide rebuild on thaw (#68). Process/CI: the H10 review-verdict gate became a sha-bound **required** commit status and was hardened through its false-open chain (#622/#629/#632/#648/#649/#672/#698), the decision corpus split to one YAML-frontmatter record per file (#610/#620), and headless Playwright UI-E2E flows landed (#445/#533). Five dual-provider migrations. |
|
||||
| `v26.14.0` | **Live TV no longer starves on embedded bitmap subtitles** — `-readrate` paces an input off its *furthest-behind* stream, and a PGS/DVD subtitle read through the video's own `-i` is sparse enough to drag the whole process to **0.53x realtime** against the 1.0x a client consumes, draining the buffer until the channel stalls. Fixed with a capability-gated `-readrate_catchup` (ffmpeg 8.0+) on realtime inputs, keeping `-readrate` on the frame-producing path so the `ffmpeg.qsv-extra-hw-frames-floor` bound is untouched; measured 0.533x → 1.067x on QSV and software, with a 240s QSV soak clean of allocation errors (#726). Affects items carrying an embedded bitmap subtitle matching the channel's subtitle mode — 3,182 of 24,646 media versions on prod, and a property of the *item*, not the channel, which is why the stall presented as random. Process/CI: the H10 review-verdict gate's repair sentinel became a fixed point and its write is now fenced on the timeline retarget count, closing a raced-sentinel false-open (#706/#707/#711). **The decisions validator now cross-checks its dependency-free frontmatter parse against PyYAML** and reports both the truncating unquoted `` #`` and the scalar-closing bare apostrophe as errors, so a record whose `rule:` silently halves under PyYAML fails the local gate instead of CI (#674/#688) — the ceiling-calibration claim was also split so the suite pins what the derivation MEANS rather than live-corpus order statistics. Dependencies: CliWrap 3.10.4, JetBrains.ReSharper.GlobalTools 2025.3.5. |
|
||||
|
||||
**Before cutting a release — sweep `docs/decisions.md` + `docs/decisions/`** (ersatztv#521, supersedes
|
||||
the ersatztv#303 H9 append-only ritual). Supersession/retirement is now a same-PR act (add the new
|
||||
@@ -46,7 +48,12 @@ active record, relocate the predecessor to `docs/decisions/archive/` with recipr
|
||||
`supersedes`/`superseded-by` links), not a release-boundary batch job — most of the old "consolidate"
|
||||
step is now continuous. The release boundary is instead where you:
|
||||
1. Run `PYTHONPATH=. python3 scripts/decisions_validate.py` — confirms lifecycle metadata is
|
||||
well-formed and every `supersedes`/`superseded-by` link resolves both ways.
|
||||
well-formed and every `supersedes`/`superseded-by` link resolves both ways. Since **ersatztv#674**
|
||||
it also cross-checks its dependency-free frontmatter parse against **PyYAML when PyYAML is
|
||||
importable**, failing on any file PyYAML rejects (a bare apostrophe in a single-quoted value) or
|
||||
reads differently (an unquoted ` #`, which YAML truncates as a comment). Where PyYAML is absent —
|
||||
the `decisions-guard` job, the Husky hooks — the cross-check is **skipped with a `::notice::`**
|
||||
and every other check still runs; the read path stays dependency-free.
|
||||
2. Confirm every record already classified `superseded`/`retired` actually lives under
|
||||
`docs/decisions/archive/` (the validator fails this, but eyeball it at the boundary too).
|
||||
3. Regenerate the active catalog: `PYTHONPATH=. python3 scripts/build_decisions_catalog.py` and
|
||||
@@ -55,10 +62,24 @@ step is now continuous. The release boundary is instead where you:
|
||||
- a **per-record prose ceiling** (`decisions_validate.py --record-ceiling <n>`, default **60**)
|
||||
— a **non-blocking `::warning::`** naming every record over it. This is the actionable signal:
|
||||
it points at a file. The 60 is derived from the distribution, not picked as a round number.
|
||||
A test pins what that derivation MEANS rather than any particular numbers: the ceiling must sit
|
||||
between the **90th and 95th percentile** of record lengths, i.e. at the tail boundary. Stated
|
||||
as percentiles it is scale-free, so ordinary corpus growth cannot ratchet it — it fires only
|
||||
when the ceiling genuinely stops marking the tail and should be re-derived.
|
||||
Its **calibration is guarded in two pieces of different robustness** (ersatztv#688), because
|
||||
four earlier single-assertion versions all failed — the first two by being vacuous or
|
||||
accepting an absurd ceiling, the last two by ratcheting:
|
||||
- **blocking** (`script-tests`) — only the coarse property that the ceiling flags a
|
||||
**meaningful minority** of records (`0.02 <= fraction_over <= 0.25`). One record moves a
|
||||
fraction by at most 1/N, so no SINGLE ordinary addition can cross it. This is measured
|
||||
headroom, not immunity: from today's 18/183 it takes 38 consecutive over-ceiling additions to
|
||||
breach the cap, 718 short ones to dilute below the floor, or — the tightest arm —
|
||||
consolidating 15 of the 18 offenders away. The floor is
|
||||
a fraction rather than "at least one record", which would accept any ceiling up to 229 on the
|
||||
live corpus; as a fraction the accepted range is 43..180.
|
||||
- **reported, never asserted against the LIVE corpus** — the fine claim that the ceiling sits
|
||||
between the **90th and 95th percentile**, i.e. at the tail boundary. `main()` prints a
|
||||
`::notice::` when it drifts; the tests assert it only on distributions they own.
|
||||
It is an order statistic over a sparse distribution, so a single new record could move p90 by
|
||||
21 lines and red the blocking job for whoever wrote it; a ceiling going out of date is
|
||||
the passage of corpus growth, not a defect in the commit under test, so it is treated like
|
||||
`stale-after`. Re-derive the constant when the notice says so.
|
||||
- the **aggregate prose total**, printed every run as an unthresholded `::notice::` **trend**.
|
||||
It has no pass/fail. A total over a monotonically growing corpus can only ratchet: the old
|
||||
4800→5600 budget went quiet at 5228 after #610 changed the metric and was back over at 5658
|
||||
@@ -392,6 +413,14 @@ the image build.
|
||||
`insecure-registries`**, so without this, cache/base-image/push over the HTTP
|
||||
registry fails (`http: server gave HTTP response to HTTPS client`).
|
||||
3. `docker/login-action` with repo secrets `REGISTRY_USER` / `REGISTRY_PASSWORD`.
|
||||
**`REGISTRY_PASSWORD` is a scoped PAT (`write:package` + `read:repository`), not an account
|
||||
password** — deliberately, so head-resolved PR code cannot use it to forge a commit status
|
||||
(`ci.actions-credential-scoping`, ersatztv#697). If a job ever fails with `token does not have at
|
||||
least one of required scope(s)`, the fix is to narrow what the job does, **never** to widen the
|
||||
token to `write:repository` or to put the admin password back. Note what the scope still reaches:
|
||||
`write:package` covers `ersatztv:prod` (the tag prod's stack follows) and `ersatztv-ci:<sha>` (the
|
||||
toolchain image five `container:` jobs execute), so this is the deployment supply chain, not an
|
||||
inert endpoint — see `ci.actions-credential-scoping`.
|
||||
4. `docker/build-push-action@v6`: amd64-only, `docker/Dockerfile`, `INFO_VERSION`
|
||||
build-arg, registry layer cache (`type=registry,ref=…:buildcache`,
|
||||
`cache-to … ignore-error=true`).
|
||||
@@ -576,6 +605,167 @@ CI-validated, so the tree-match check correctly declines. So the skip is a genui
|
||||
win (clean, up-to-date, un-rebased merges in quiet periods) — correct-but-conservative by
|
||||
construction, not a general dedup. It never fires unsafely; when in doubt it runs the full matrix.
|
||||
|
||||
### Dropped-step guard on the required jobs (ersatztv#756)
|
||||
|
||||
`test` and `migrations` write the only two `docker-build.yml` contexts branch protection requires on
|
||||
`main`. A step the runner declines to interpolate is **dropped, and the job still concludes
|
||||
`success`** (ersatztv#751), so in these two jobs that failure is **fail-OPEN**: a required check
|
||||
reports green having done no work. In `review-verdict.yml` the same drop is fail-closed — the status
|
||||
is simply absent and the merge is blocked — which is why #751 fixed the safe direction first.
|
||||
|
||||
Two independent mechanisms hold it, and neither is redundant:
|
||||
|
||||
- **A static ban on expression delimiters** in any `run:` body of `test`, `migrations` **and
|
||||
`build`**. The drop mechanism *requires* an opener in the scalar, so this makes the class
|
||||
unreachable rather than merely detected — and it is the raw `${{` opener that is banned, not a
|
||||
well-formed pair, because an unclosed one triggers the same rewrite. When a step genuinely needs a
|
||||
value, pass it through the step's `env:` block, which is interpolated **per value**, so a bad
|
||||
payload there cannot take the body with it.
|
||||
|
||||
**Why `build` is in the ban although it is not a required context.** Its one delimiter-bearing body
|
||||
was `Smoke + IPTV E2E`, which runs *after* `Build and push` — so on a `v*` tag the image is already
|
||||
in the registry as the release candidate and that step is what decides whether the candidate was
|
||||
ever booted. A drop there publishes an unsmoked candidate, reports green, and `DeployStack
|
||||
jazz-media` promotes exactly that image. Its two payloads moved into the step's `env:`, so the ban
|
||||
cost nothing.
|
||||
|
||||
**The ban is re-checked on the release path itself (ersatztv#767).** It used to be enforced only by
|
||||
the `script-tests` job, which lives in `pr-checks.yml` (`on: pull_request`) and is **not** a
|
||||
required context — a *review-time* check on the PR that would introduce a delimiter, not a gate on
|
||||
the release. `pr-checks.yml` does not run on a `v*` tag push at all, so a delimiter that ever
|
||||
reached `main` would still drop `Smoke` on the tag build and go green; `main` being PR-only (#743)
|
||||
meant such a change had to pass through a PR where `script-tests` reddens, but a red on a
|
||||
non-required check does not block the merge server-side.
|
||||
|
||||
There is now a **`scan` job** (`Delimiter ban (release path)`) that runs the PyYAML-based ban test,
|
||||
and **`build` lists it in `needs:`**. That single edge is the fail-closed property: a red `scan`
|
||||
means `build` is skipped outright, so the image is never built, let alone pushed.
|
||||
|
||||
**Why a job and not a step inside `build`.** A step cannot protect the job it lives in. `build` is
|
||||
what publishes, so a guard step there fails **open** if the runner drops it — and the defence
|
||||
("the guard's own body has no opener, so it cannot be dropped") is circular when the only thing
|
||||
enforcing that property is the same PR-only test being backstopped. This was the first design and
|
||||
two independent reviews rejected it for exactly that.
|
||||
|
||||
**Why it runs the real pytest and not a bespoke scanner.** The same first cut hand-parsed the
|
||||
workflow YAML in stdlib Python, to avoid provisioning PyYAML on `build`'s bare runner. Review found
|
||||
~10 **false negatives** in that parser in one round — flow mappings (`{run: …}`), a quoted
|
||||
`"run":` key, aliases, multiline quoted scalars — making it strictly *weaker* than the check it
|
||||
backstopped, in the only direction that matters for a security gate. Running the existing test
|
||||
needs no second definition of "what is a `run:` body", so it has no drift surface at all. `scan`
|
||||
runs on `small` and provisions Python the same way `script-tests` does.
|
||||
|
||||
The wiring is held by `scripts/tests/test_ci_release_path_scan_job.py` — `build` depends on it, it
|
||||
carries **no job-level `if:`** (one that excluded the tag push would restore the hole; one that
|
||||
skipped the job would skip `build` too), no step is `continue-on-error`, it actually invokes the
|
||||
ban test, and every one of its own `run:` bodies is delimiter-free. Its steps also carry #756
|
||||
markers and a trailing assert, so a drop *inside this job* is caught as well.
|
||||
|
||||
What this does **not** claim: that no step can ever fail to run for a reason other than the
|
||||
interpolation drop. It moves the terminal assumption — to fail open you must now drop the pytest
|
||||
step **and** the assert step, rather than either one alone.
|
||||
|
||||
Measuring a guard on this path does **not** require cutting a release, and an earlier draft here
|
||||
claiming it would was simply wrong: `build` runs on every push to `main`
|
||||
(`if: github.event_name != 'pull_request'`), and a `workflow_dispatch` on any other ref runs the
|
||||
job while `Build and push` publishes nothing (its `push:` is gated on `main`/`v*`). That is how
|
||||
#767 was verified — see the decision record for the run ids.
|
||||
|
||||
`functional-e2e` is delimiter-free too but is deliberately **not** banned: it is
|
||||
advisory by declaration, and the rule is "ban where a drop is consequential", not "ban wherever it
|
||||
is currently free". `api-docs` and `format` keep one `github.base_ref` each in a detect step and
|
||||
gate nothing that ships.
|
||||
- **Runtime per-step markers**, for a step that fails to run for any *other* reason. Every `run:`
|
||||
step that is not `continue-on-error: true` calls
|
||||
`"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark <key>` as its **first act**, and the job's last
|
||||
step calls `ci-step-ran.sh assert --always … --gated …`, which fails the job when an expected key
|
||||
was never recorded.
|
||||
|
||||
**Per step, not per job.** A marker written by the first step only proves the job *began*, which was
|
||||
never in doubt. The drop that costs something is `Test`, `Build` or a migration replay — all well
|
||||
past step one — so a job-level marker would have been a guard that cannot see the case it exists for.
|
||||
|
||||
**The guard carries no `if:`, and that is deliberate.** The #751 guard uses `if: always()` because its
|
||||
job has one real step. These have a dozen, and a genuine failure in an early step legitimately skips
|
||||
every later one — an `always()` guard would then announce a false *"these steps never executed:
|
||||
typecheck web-test build dotnet-test"* on top of every ordinary red build, and a guard that cries wolf
|
||||
gets deleted. (`migrations` is smaller — six marked steps — but the same argument applies, and its
|
||||
guard comment is worded for its own keys rather than copied from `test`'s.) The default `if:` is `success()`, which is the wanted condition, and the invariant that
|
||||
makes relying on it safe rather than lucky is: **the guard is skipped only when an earlier step
|
||||
failed, and that already fails the job**. So *guard skipped ⇒ job red*, and every path to a green job
|
||||
runs the guard. A dropped step is invisible precisely *because* it concludes `success` — which keeps
|
||||
the job green and therefore reaches the guard.
|
||||
|
||||
That invariant has one path where it could plausibly be false and where being wrong would be silent:
|
||||
a step marked `continue-on-error: true` that FAILS. If that flipped `success()`, the guard would be
|
||||
skipped on a job that still concluded green — the guard rendered a no-op by exactly the failure mode
|
||||
it exists to catch, with no signal. The `test` job has three `continue-on-error` steps and two of them sit immediately before the guard,
|
||||
so this is a live path, not a theoretical one. **Measured** (scratch PR #766, run 1913 job 8075): the last
|
||||
advisory step was made to `exit 1`, the log carries `❌ Failure - Main Report peak container memory`,
|
||||
and the guard **still ran**, reported `All 12 expected step(s) executed`, and the job concluded
|
||||
`success`. A failing `continue-on-error` step does not flip `success()` on this runner, so the
|
||||
invariant holds where it mattered most. That run is also the `test` job's full twelve-key positive
|
||||
control on the build lane.
|
||||
|
||||
**Adding a step to either job?** Mark it, and add its key to that job's guard list in the right
|
||||
bucket (`--always` for the two detect steps, `--gated` for anything carrying the docs-only /
|
||||
already-validated `if:`). `scripts/tests/test_ci_dropped_step_guard.py` derives the expected set from
|
||||
the workflow, so an unmarked step or a bucket mismatch is a red — it does not rely on anyone
|
||||
remembering. One caveat, since this section is careful about it elsewhere: that red is `script-tests`,
|
||||
the same non-required, PR-only check discussed above. For the delimiter ban on `test`/`migrations`
|
||||
that hardly matters, because the runtime guard is the fail-closed backstop — but a **newly added,
|
||||
unmarked** step is caught by the static test *alone*, since the runtime guard cannot expect a key
|
||||
nobody declared.
|
||||
|
||||
**The marker path is keyed on job + run id + attempt** — and be precise about why, because the
|
||||
obvious justification is a #751 measurement that does *not* transfer. #751 found `RUNNER_TEMP` to be
|
||||
`/tmp` and called it "not a private per-job directory", but that was taken on `review-verdict.yml`,
|
||||
which runs *without* a `container:`. These two jobs run **inside** the CI toolchain image, so their
|
||||
`/tmp` is the job container's own and starts empty. The fresh container is therefore what actually
|
||||
rules out a stale marker here; the keying is defence in depth against a lane change nobody would
|
||||
think to re-check this against. `GITHUB_JOB` and `GITHUB_RUN_ID` are *measured* present and the
|
||||
script refuses without them rather than falling back to a name other runs share.
|
||||
`GITHUB_RUN_ATTEMPT` is required too — but **how** that was established is the part worth keeping,
|
||||
because the first two attempts at it were both worthless. Grepping a job log for the variable *name*
|
||||
proves nothing: logs do not dump the environment. Inferring it from the *absence* of the script's
|
||||
"not set" warning proves nothing either, because that warning goes to **stderr**, and whether step
|
||||
stderr reaches a job log here was itself never established — the control offered for that turned out
|
||||
to be an `::error::` line this script writes to *stdout*. So the script was made to **report its
|
||||
resolved identity on stdout**, where capture is not in question, and the answer was simply read off
|
||||
this change's own run: `Marker identity: job=test run=1916 attempt=1 (from the runner)`, and the same
|
||||
for `migrations`. Both required jobs, on the lane that matters.
|
||||
|
||||
That measurement is what promoted it from warn-and-default to required, and it is why the residual
|
||||
this paragraph used to describe — a rerun inheriting attempt 1's markers — no longer exists. The
|
||||
identity line stays, as the standing evidence a future reader checks first if the keying is ever
|
||||
doubted again.
|
||||
|
||||
**The premise was re-measured on the build lane.** The whole thing rests on the runner still executing
|
||||
a later step after dropping an earlier one. #751 established that on the `small` lane; these jobs run
|
||||
in a `container:` on `ubuntu-latest`, so it was measured there rather than assumed — scratch PR #765
|
||||
(Gitea 1.27.1, 2026-08-10) reintroduced the exact #751 defect in the `test` job's `revalidate` step.
|
||||
Recorded outcome (job `test`, run 1910, 20:03:49→20:13:31Z — a full 9m42s heavy run, so `Build` and
|
||||
`Test` really executed):
|
||||
|
||||
- `Unable to interpolate expression 'format('# PROBE ONLY … {0}\n…', pr number)'` at 20:04:06 — the
|
||||
step was **dropped**, exactly as #751 describes, and it reported conclusion `success`.
|
||||
- **Every other marked step still ran** — eleven markers were recorded, ten of them AFTER the drop
|
||||
(`restore npm-ci check-api lint typecheck web-test web-build strip-scanner build dotnet-test`),
|
||||
`detect` being the eleventh and earlier. The premise holds on this lane.
|
||||
- The guard ran at 20:13:29, reported `These steps of job 'test' never executed: revalidate`, and was
|
||||
the **only** ❌ in the entire job log — every other step succeeded. Without it this run would have
|
||||
concluded `success` having never executed that step, which is precisely the fail-open being closed.
|
||||
- Incidental but kept: the dropped step's output arrived as `ETV_REVALIDATE_SKIP:` **empty**, not
|
||||
`false` — the case the guard must read as "widen what is required", never as a skip.
|
||||
|
||||
**The positive control is the same run's `migrations` job**, which the probe did not touch: it marked
|
||||
all six steps, the guard reported `All 6 expected step(s) executed: detect revalidate restore build
|
||||
sqlite mysql`, and the job concluded **success**. So one run demonstrates both directions on the build
|
||||
lane — a drop caught and reddened, and a clean job passing. The twelve-step `test` positive control is
|
||||
this change's own CI run.
|
||||
|
||||
Full rationale: `docs/decisions/records/ci/required-job-step-execution-markers.md`.
|
||||
|
||||
### `docs-reminder` job (non-blocking, PR-only — in `pr-checks.yml`)
|
||||
|
||||
A lightweight nudge that enforces the CLAUDE.md "docs-update is part of done" rule for the
|
||||
@@ -655,11 +845,53 @@ which is itself a small demonstration of why the suite needed to run in CI at al
|
||||
`small`-lane Python jobs it adds `actions/setup-python@v5` first. Checkout is at default depth: every `git` call in the suite runs
|
||||
against a temp repo it creates itself, never this repository's history.
|
||||
|
||||
A **preflight step** asserts `jq` and `git` are on PATH before running the suite. Those two tests
|
||||
exec the real shell scripts, which shell out to `jq` ~26 times; the tests shim `curl` on PATH but
|
||||
not `jq`, so a runner image without it would surface as ~20 opaque assertion failures instead of one
|
||||
diagnosis. It deliberately **checks** rather than installs — ersatztv#390 removed run-time
|
||||
`apt-get` from CI; the fix for a genuine miss is to bake the tool into the runner image.
|
||||
Two **preflight steps** run before the suite. The first asserts `git` is on PATH; the second runs
|
||||
`scripts/jq-preflight.sh --expect 1.6`, which checks jq's **version**, not merely its presence (see
|
||||
"The jq contract" below). Those two tests exec the real shell scripts, which shell out to `jq` ~26
|
||||
times; the tests shim `curl` on PATH but not `jq`, so a runner image without it would surface as ~20
|
||||
opaque assertion failures instead of one diagnosis. Both deliberately **check** rather than install —
|
||||
ersatztv#390 removed run-time `apt-get` from CI; the fix for a genuine miss is to bake the tool into
|
||||
the runner image.
|
||||
|
||||
### The jq contract (ersatztv#648)
|
||||
|
||||
> Full rationale: `docs/decisions/records/ci/jq-version-contract.md`.
|
||||
|
||||
Every shell gate in this repo — `decisions-guard`, `script-tests`'s own harness,
|
||||
`pretooluse-merge-consent.sh`, `review-verdict.yml`, `scripts/pr-changed-files.sh` — is authored and
|
||||
tested on a developer Mac shipping **jq 1.8.x**. The CI runner ships **jq 1.6**. Author to the
|
||||
1.6-compatible subset; three concrete constructs diverge between the two and each one produced a real
|
||||
bug when it hit CI for the first time:
|
||||
|
||||
- **`jq -e` over EMPTY input.** Exits 4 on jq >= 1.7, but **0** on jq 1.6. A guard that infers
|
||||
"transport failure" from that exit status silently passes an empty/failed page on 1.6.
|
||||
- **`` contains("\u0000") `` (or any NUL literal).** The NUL escape truncates to `""` on jq 1.6, so
|
||||
the containment test is vacuously true for **every** string, not just ones containing a NUL. Use
|
||||
`explode | index(0)` instead — it is version-stable.
|
||||
- **Parse-error exit code.** `jq empty` exits 5 on jq >= 1.7 but **4** on jq 1.6 — the same code 1.6
|
||||
uses for "no output produced". Reading that exit code as a specific failure mode conflates garbage
|
||||
input with an empty-but-valid response.
|
||||
|
||||
`scripts/jq-preflight.sh` makes the running version **observable** in every gate job's log (it prints
|
||||
the parsed version and asserts a floor of 1.6) so a future divergence can be diagnosed from the log
|
||||
alone instead of guessing at the runner image.
|
||||
|
||||
**Pin vs floor is deliberately asymmetric.** `scripts/jq-preflight.sh --expect 1.6` additionally pins
|
||||
the version and fails loudly if it drifts, but that mode is used **only** by `script-tests`
|
||||
(`.gitea/workflows/pr-checks.yml`) — advisory, not a required check. `review-verdict.yml` runs the
|
||||
no-args floor-only mode and never pins, because that workflow writes `review-verdict/h10`, the
|
||||
branch-protection-**required** status check on `main`: a hard pin there would mean the day the
|
||||
runner's jq version changes (a base-image bump, a host reimage — nothing this repo controls), every
|
||||
PR on `main` stops merging until someone notices and re-pins. A required merge gate cannot fail
|
||||
because an upstream package manager did its job. The narrower pin on `script-tests` exists precisely
|
||||
because that job is the suite's only 1.6 coverage — if the runner's jq silently changed, that coverage
|
||||
would evaporate with no signal, so failing loudly there forces a human decision instead.
|
||||
|
||||
Baking a pinned jq into `docker/ci/Dockerfile` was considered and rejected: `review-verdict.yml` is
|
||||
`runs-on: small` with no toolchain-image pin, and per `ci.small-lane-git-only` the small lane is
|
||||
git-only, so it gets the **host's** jq regardless of what the toolchain image contains — a pin in the
|
||||
image provably cannot reach the gate that broke. This was checked against the running binary, not
|
||||
assumed.
|
||||
|
||||
## PR gates workflow
|
||||
|
||||
@@ -702,6 +934,40 @@ from `Build ErsatzTV Image / …` to `PR Gates / …`) does not affect merges. T
|
||||
unreviewed commit from merging** (ersatztv#622). It is not produced by a job's success/failure; it
|
||||
is a commit status that `scripts/post-review-verdict.sh` POSTs onto one specific sha.
|
||||
|
||||
**`main` is PR-only AND admin-override-proof, and it takes both to make the check load-bearing**
|
||||
(ersatztv#743, `release.main-direct-push-disabled`). Gitea evaluates `status_check_contexts` when it
|
||||
**merges a PR** — a direct `git push origin HEAD:main` never consults them. So until 2026-08-05 the
|
||||
entire gate was skippable with no forgery at all, which was cheaper than every route enumerated in
|
||||
#697. `main` now carries **two** fields, and citing either alone is a mistake:
|
||||
|
||||
- `enable_push: false` — a direct push is refused server-side at pre-receive (`Not allowed to push to
|
||||
protected branch main`), for every account including a site admin. The contents API is refused too
|
||||
— measured, HTTP 403 `user cannot commit to repo`. The web editor, upload, apply-patch, revert and
|
||||
cherry-pick paths share that same `CanUserPush` predicate and are therefore expected to refuse as
|
||||
well, but were not probed (source-attested only).
|
||||
- `block_admin_merge_override: true` — without it (the default is `false`), a repo admin could
|
||||
`POST /pulls/{n}/merge` with `force_merge: true` and merge straight past a missing or red
|
||||
`review-verdict/h10`. Disabling push alone just moves the bypass from the push path to the merge
|
||||
path, since `timothy` is admin and is the identity every session already uses. **Source-attested,
|
||||
not probed** (Gitea 1.27 `CanBypassBranchProtection`): verifying it by experiment means merging an
|
||||
unreviewed PR, so the field was set rather than measured. Setting it is safe under either
|
||||
semantics; re-confirming the bypass itself rides with ersatztv#747.
|
||||
|
||||
**Operator recovery when a required context gets stuck.** `block_admin_merge_override: true` removes
|
||||
the "Merge (admin)" / `force_merge: true` escape that used to unstick a PR whose required context was
|
||||
absent or wrongly red — a recurring situation here (a killed run overwriting a newer green, an
|
||||
advisory red counted into the combined status, a gate workflow that cannot post). That escape is gone
|
||||
*by design*: it was also the bypass. The supported recovery is to fix the status
|
||||
(re-run the job, or re-post the verdict with `scripts/post-review-verdict.sh`); the last resort is to
|
||||
`PATCH .../branch_protections/main` setting `block_admin_merge_override: false`, merge, and set it
|
||||
straight back. Do the last one deliberately and say so in the PR — it is the one action that
|
||||
re-opens the hole this section exists to close.
|
||||
|
||||
Practical consequences: **every** change to `main` goes through a PR, including a one-line docs fix;
|
||||
and the client-side Husky guards (H6/H11/H13) remain useful friction but were never the control —
|
||||
they are fail-open and `--no-verify` bypasses them. Tag pushes are unaffected (separate mechanism;
|
||||
`tag_protections` is empty), so the release cut in "Cutting a release" still works unchanged.
|
||||
|
||||
**The hole it closes.** `pretooluse-merge-consent.sh` proves its three consent conditions at the
|
||||
moment the merge tool is called. Pass `merge_when_checks_succeed=true` and Gitea performs the merge
|
||||
*later*, against whatever head is green then — while the Done-when and review-verdict checks were
|
||||
@@ -736,20 +1002,322 @@ hook's condition (c)) and the `review-verdict/h10` status on the same sha. `BLOC
|
||||
a commit landed mid-flight it writes **no** status and exits non-zero rather than retargeting your
|
||||
verdict at a commit you never read.
|
||||
|
||||
**Exemptions** are handled by `review-verdict.yml` on every `pull_request` event, which posts the
|
||||
The status description also records the base branch — `Review-verdict: MERGEABLE @ abc1234 (base:
|
||||
main)` — and the merge-consent hook denies when that no longer matches the PR's live `base.ref`
|
||||
(ersatztv#632). Retargeting a PR changes the effective diff without moving the head sha, so the
|
||||
per-sha binding alone cannot see it. This is **detection on the hook path only**: a commit status
|
||||
carries no base of its own, so a merge driven through the Gitea UI or API is unaffected. The
|
||||
comparator is the base *branch*, never its tip sha — a base that merely advances is ordinary churn,
|
||||
and comparing tips would invalidate every open verdict on every unrelated merge to `main`.
|
||||
|
||||
**Exemptions** are handled by `review-verdict.yml` on every `pull_request_target` event, which posts the
|
||||
status as `success` for **Renovate-authored** PRs (it uses `platformAutomerge: true`, so a required
|
||||
verdict with no exemption would stall every dependency bump) and for **docs-only** PRs, and as
|
||||
`pending` for everything else so the block has a visible reason. Both exemptions are **void when the
|
||||
PR touches `.claude/`, `.gitea/`, `.husky/`, `scripts/` or `docker/ci/`** — a PR that can weaken the
|
||||
PR touches `.claude/`, `.codex/`, `.gitea/`, `.husky/`, `scripts/` or `docker/ci/`** — a PR that can weaken the
|
||||
gate must not be able to exempt itself from the gate. That includes Renovate's `docker/ci` base
|
||||
bumps, which already need the manual publish-then-pin two-step anyway.
|
||||
|
||||
The Renovate exemption additionally requires **every** changed path to be a dependency manifest —
|
||||
`Directory.Packages.props` or `.config/dotnet-tools.json`, and only those (ersatztv#698). The npm
|
||||
manifests are deliberately excluded: `renovate.json` enables only `nuget`/`github-actions`/`dockerfile`,
|
||||
so npm is unmanaged here, while `package.json` `scripts` are executed by CI (`npm ci`, `npm run build`)
|
||||
— exempting it would put a code-execution path inside the allow-list for no benefit. An author match alone is not enough, because `pull_request.user.login` is the PR's
|
||||
*immutable creator* while its head is not: pushing application code onto an open Renovate branch
|
||||
leaves the PR still "authored by renovate" and, previously, still exempt. A Renovate PR touching
|
||||
anything else — a `.csproj`, a source file — is not blocked, it just needs a real verdict. **If a
|
||||
dependency PR is unexpectedly asking for a verdict, this is why**; the status description says so.
|
||||
|
||||
The two exemptions are evaluated as **independent predicates**, never as an `elif` chain: a Renovate
|
||||
PR touching only `docs/` still gets the docs-only exemption on its own merits.
|
||||
|
||||
An existing `review-verdict/h10` on the head is **only** left alone when it is positively identifiable
|
||||
as a human verdict — a non-null `.creator.login` **and** a `Review-verdict:` description, which is what
|
||||
`post-review-verdict.sh` writes. Anything else, including any shape the workflow does not recognise, is
|
||||
**re-derived** rather than inherited. (Measured: a status POSTed with a user credential carries a
|
||||
creator; one POSTed by an Actions job carries `"creator": null`.) Without this, an exemption obtained
|
||||
once was accepted unchanged on every later run. This is a *provenance* check, not an authentication
|
||||
one — someone who can POST statuses directly can still impersonate a verdict (ersatztv#697). That
|
||||
provenance asymmetry is *why* the credential scoping in `ci.actions-credential-scoping` mattered: a
|
||||
forgery through a **user** credential inherits as a human verdict, while one through a job's
|
||||
`GITEA_TOKEN` carries `creator: null` and is re-derived, so it must win a race. CI's registry secret
|
||||
was a user credential — the admin account — and no longer carries status-write. **`RENOVATE_TOKEN`
|
||||
still is one** (`write:repository`, a real bot account), and secrets are a per-repo store any
|
||||
PR-added workflow can reference, so that route is narrowed rather than closed; tightening this check
|
||||
from "non-null creator" to an allow-list of approved reviewers is what would close it
|
||||
(ersatztv#742). A collaborator's own personal token still can, and no repo-side change closes that.
|
||||
Note also that re-derivation is **not** a race the attacker can lose: it fires only on the trigger's
|
||||
`types`, and posting a status is not one of them, so a POST timed after the last PR event stands
|
||||
until the next one.
|
||||
|
||||
Deciding either exemption requires the PR's **complete** changed-file list, which the workflow does
|
||||
not compute itself: it calls `scripts/pr-changed-files.sh`, the single shared implementation also
|
||||
used by the advisory hook `.claude/hooks/pretooluse-merge-consent.sh` (ersatztv#649). The workflow
|
||||
reads that script's **exit status** — a non-zero exit means "could not tell" and withholds the
|
||||
exemption; its stdout is meaningless on any failure path and is never consumed.
|
||||
|
||||
**Never write a classification guard as `producer | grep -q…` here.** Under `set -o pipefail`, `grep -q`
|
||||
exits at its first match, the producer takes SIGPIPE (141), and a MATCH is reported as a failed
|
||||
pipeline — inverting the guard for any PR whose path list exceeds the pipe buffer. That let a large PR
|
||||
be classified docs-only, and let one editing `.gitea/` skip the protected-path check entirely. A
|
||||
here-string is **also** wrong (bash spills a large one to temp storage, which fails the same way when
|
||||
temp is full). **Count** instead — `grep -c` drains stdin over an ordinary pipe — evaluate the counts
|
||||
once at top level rather than inline in an `if`, and fail closed on a non-numeric result. Full detail:
|
||||
`ci.grep-q-pipefail-inversion`.
|
||||
|
||||
That script takes the expected base branch as a **required 5th argument** and refuses to enumerate when
|
||||
the PR's live base does not match it, checked both before and after paging (ersatztv#698).
|
||||
`/pulls/{n}/files` diffs against the PR's *live* base, so retargeting changes the answer without moving
|
||||
the head sha — a PR opened into `main` and retargeted mid-run was granted a docs-only exemption while
|
||||
its diff against `main` carried a C# file. The workflow passes the base from the `pull_request_target`
|
||||
payload, which a retarget cannot rewrite, and `edited` is in `types:` so a retarget reclassifies.
|
||||
`edited` gives **detection, not atomicity**: runs are not serialized, so a stale run could still post
|
||||
`success` after the reclassifying run posted `pending`.
|
||||
|
||||
**That residual is now fenced (ersatztv#706).** Runs are still not serialized — instead a run that was
|
||||
overtaken *declines to write*. The job counts `change_target_branch` events on the PR's issue timeline
|
||||
at start and again immediately before its POST, and posts **nothing** if the count moved. The count is
|
||||
the key precisely because the branch *name* is ABA-vulnerable: `main → scratch → main` reads `main` at
|
||||
both ends, which is how the forged exemption was obtained in the first place. Abstaining never strands
|
||||
a PR, because every retarget fires `edited` — the event that makes one run abstain has already queued
|
||||
its successor.
|
||||
|
||||
If the count can't be established (unreadable timeline, paging that never reached a validated empty
|
||||
page), only the exemption `success` is withheld; `pending` still posts, since `pending` cannot turn an
|
||||
unreviewed head green and withholding it would strand ordinary PRs for nothing. **If an exempt PR is
|
||||
unexpectedly missing its status after a retarget, this is why** — the job log names the counts.
|
||||
|
||||
Worth knowing before reaching for the obvious alternative: **a concurrency group does not work here**,
|
||||
measured rather than assumed. Gitea 1.25.4 auto-cancels superseded `push` runs on a branch, but *not*
|
||||
`pull_request_target` runs — two runs for one PR genuinely overlap, and adding
|
||||
`concurrency: {…, cancel-in-progress: false}` changed nothing (probe runs still overlapped by 36s).
|
||||
`cancel-in-progress: true` is deliberately untried, because a cancelled run leaves an exempt PR
|
||||
statusless with nothing left to re-trigger it. Full measurements and the two surviving residuals:
|
||||
`ci.verdict-write-retarget-fence`.
|
||||
|
||||
Separately, after posting an exemption `success` the job re-reads the per-POST status history and, if
|
||||
a human `Review-verdict:` row appeared during the write window, overwrites its own status with
|
||||
`pending` and logs an error — so a human `BLOCKED` can never be silently turned green. The repair is
|
||||
`pending`, never a copy of the human's verdict, which would attribute a human decision to the job.
|
||||
|
||||
Three properties of this workflow are security-relevant and are **structurally** asserted by tests in
|
||||
`scripts/tests/test_pr_changed_files.py` — those tests pin the workflow's shape, which is not the same
|
||||
as establishing that the gate cannot be forged (see the residual below, and ersatztv#697/#698):
|
||||
|
||||
- **The trigger is `pull_request_target`, scoped to `branches: [main]`** — never plain
|
||||
`pull_request` (ersatztv#672). Gitea resolves a `pull_request` workflow *definition* from the PR's
|
||||
own head, so under that trigger a PR editing `review-verdict.yml` ran its own rewritten copy and
|
||||
could post `review-verdict/h10=success` for itself. The base-ref checkout below binds the scripts
|
||||
this job runs; only the trigger binds the definition. The `branches` filter is half the fix, not a
|
||||
refinement of it: base resolution means the *base branch* supplies the gate, so an unfiltered
|
||||
trigger merely moves the rewrite to an attacker-pushed base — and a status forged there is
|
||||
inherited by any later PR carrying the same head sha (ersatztv#663). `pull_request_target` is safe
|
||||
here **only** because this job never checks out or executes head-supplied code. Verified on this
|
||||
instance with four scratch PRs rather than inferred from GitHub; full rationale in
|
||||
`docs/decisions/records/ci/gate-trigger-base-resolved.md`. **This closes the rewrite route through
|
||||
this workflow, not the class:** `docker-build.yml` is also head-resolved and must stay on
|
||||
`pull_request` because it builds the PR's code, so it got the read-only status identity instead —
|
||||
its `ETV_STATUS_AUTH` is now a PAT scoped `write:package` + `read:repository`, which the status
|
||||
endpoint refuses (`ci.actions-credential-scoping`, ersatztv#697). The inventory was never that one
|
||||
workflow, though: Gitea injects a write-capable `GITEA_TOKEN` into every job and branch protection
|
||||
binds the *context*, not its issuer. Gitea >=1.26 with the Actions default set to **Restricted**
|
||||
(server-management#714) binds the injected token, but does not close the class either — not against
|
||||
a personal token, and not against `RENOVATE_TOKEN` (ersatztv#742). **And none of it was necessary:
|
||||
direct pushes to `main` were server-side permitted, so the gate could be skipped without any forgery
|
||||
(ersatztv#743). That is now CLOSED — `main` carries `enable_push: false` **and**
|
||||
`block_admin_merge_override: true`, so it is reachable only through the PR merge path, the one path
|
||||
on which Gitea evaluates `status_check_contexts`, and an admin cannot `force_merge` past them
|
||||
(`release.main-direct-push-disabled` — neither field is citable alone).** Note the fix is *disabling* push, not whitelisting it: a
|
||||
push whitelist naming `timothy` was measured to still admit the push, and `timothy` is the identity
|
||||
every session, PAT and injected `GITEA_TOKEN` already acts as, so the whitelist form would have
|
||||
closed nothing. The block binds a site admin at pre-receive but not a credential that can first
|
||||
PATCH branch protection off — an accepted residual, recorded in that decision. The
|
||||
exemption path has separate defects of its own (ersatztv#698). One operational
|
||||
consequence of the trigger change: a PR whose base is not `main` now gets **no**
|
||||
`review-verdict/h10` at all. That is fail-closed. `edited` **is** now among the trigger's `types`
|
||||
(ersatztv#698), so a PR retargeted onto `main` reclassifies instead of staying statusless until its
|
||||
next push — but note that only gives *detection*: runs are not serialized, so a stale run can still
|
||||
post `success` after the reclassifying run posts `pending` (ersatztv#706).
|
||||
- **The checkout takes the PR's BASE ref**, `ref: ${{ github.event.pull_request.base.sha }}` with
|
||||
`persist-credentials: false` — never the head. This job judges the PR, so the PR must not supply
|
||||
the code that judges it; a head checkout would let a PR edit the enumeration to return an empty
|
||||
list and exempt itself.
|
||||
- **`scripts/jq-preflight.sh` runs in floor-only mode**, never `--expect`. This job writes a
|
||||
branch-protection-**required** status, so an exact version pin would turn any jq upgrade on the
|
||||
runner into a repo-wide merge deadlock.
|
||||
|
||||
A PR whose base predates ersatztv#658 has no such script on its base ref; that case posts `pending`
|
||||
with the reason rather than dying with no status at all.
|
||||
|
||||
⚠️ **Changing `review-verdict.yml` itself: it is not exercised by its own PR.** Base resolution cuts
|
||||
both ways — the PR editing this workflow runs the version already on `main`, so an edit goes live
|
||||
**only on merge**, repo-wide, having never run. A broken edit merges green and then breaks the gate
|
||||
for every subsequent PR, and the PR that would repair it is gated by the same broken workflow. Do not
|
||||
trust the editing PR's own checks. Verify the way ersatztv#672 did:
|
||||
|
||||
1. Push a scratch **base** branch carrying the candidate workflow.
|
||||
2. Open a throwaway PR from a scratch head *into that base*, so the candidate is the definition that
|
||||
runs. Have it post a **probe-named** context (e.g. `review-verdict/h10-PROBE`), never the real
|
||||
`review-verdict/h10` — a probe must not be able to forge the gate it is testing.
|
||||
3. Read the resulting commit statuses to see which definition actually ran, then delete both
|
||||
branches.
|
||||
|
||||
The same shape is what makes a `branches:`/`types:` change verifiable at all, since neither can be
|
||||
observed from the editing PR. Note step 2 requires the scratch **base**'s own `branches:` filter to
|
||||
name that base — the definition comes from the base, so a base the filter does not admit produces no
|
||||
run at all.
|
||||
|
||||
⚠️ **Never write an expression delimiter inside a `run:` body here — a comment is NOT inert**
|
||||
(ersatztv#751, `ci.workflow-run-body-no-expressions`). A `run:` body is not shell when the runner
|
||||
reads it. The runner scans the whole scalar for the expression opener and, on finding one, rewrites
|
||||
the **entire** body into a single `format(...)` call so the result can be spliced back in. That
|
||||
rewrite is all-or-nothing: a payload that does not evaluate fails the interpolation of the whole
|
||||
scalar, and **the runner then drops the step and concludes the job `success`**.
|
||||
|
||||
That is not hypothetical. From 8f6d4f443 (2026-08-03) to 2026-08-06 the classify step **never ran**.
|
||||
The #706 note above, explaining why a concurrency group does not work here, quoted a `concurrency:`
|
||||
snippet containing a PR-number expression *as an illustration*, in a shell comment. `pr number` is not
|
||||
a valid expression. So `review-verdict/h10` was posted by nothing but a human hand for three days,
|
||||
both exemption classes silently stopped working, and every run reported success. The prose documenting
|
||||
a fix disabled the fix.
|
||||
|
||||
**The silent green is the real defect.** An absent required status reads as "not reviewed yet", which
|
||||
is indistinguishable from the correct pending state — so an ordinary PR looked ordinary while the gate
|
||||
was dead, and the cost landed only where no human was in the loop. PR #739 (docs-only) merged
|
||||
2026-08-05 with **zero** commit statuses on its head, and got in only because admin force-merge was
|
||||
still enabled; ersatztv#743 removed that escape the next day, so a docs-only or Renovate-manifest PR
|
||||
arriving after that would simply have been stuck with no bypass. The two Renovate PRs in the window
|
||||
escaped by timing, merging minutes before the bad commit.
|
||||
|
||||
Three things now hold the line, and they are deliberately different in kind:
|
||||
|
||||
- **The prose names expressions instead of quoting them** — write "a
|
||||
`github.event.pull_request.number` expression", not the delimiters. Pass values in through the
|
||||
step's `env:` block, which is interpolated per value, so a bad payload there cannot take the body
|
||||
with it.
|
||||
- **A start-marker guard turns a dropped step RED.** The classifier writes a marker as its first act
|
||||
and an `if: always()` step fails the job when it is missing. It asserts execution *started*, never
|
||||
that it completed — the classifier has several legitimate `exit 0` abstention paths. The guard's own
|
||||
body must stay expression-free, or the mechanism it guards against can delete the guard too, and
|
||||
that absence would be silent as well.
|
||||
- **Two static guards**, in `scripts/tests/test_pr_changed_files.py`: no delimiter in *any* `run:`
|
||||
body of this file (absolute — a dropped step here is a dead merge gate, and its bodies are ~700
|
||||
lines of prose), and repo-wide, every expression payload's **head token** must name a context or
|
||||
function the runner can resolve (permissive, because the other workflows interpolate into `run:`
|
||||
legitimately — 5 occurrences today, in `ci-image.yml`, `docker-build.yml`'s `api-docs`/`format`
|
||||
and `pr-checks.yml`'s two git-diff gates; #756 removed `build`'s two and banned that job as
|
||||
well, so the ban now covers `test`, `migrations` and `build`). Be precise about the second
|
||||
one's reach: it catches the
|
||||
historical defect (`pr number`) and a nonexistent context, but **not** a syntactically invalid
|
||||
payload whose tokens are all known (`${{ github.ref == }}` passes), nor a renamed output
|
||||
(`steps.metadata.outputs.shortsha` passes — every token after the first is preceded by `.` and is
|
||||
skipped), nor an unclosed opener. Catching those needs an expression parser. An earlier draft of
|
||||
this section claimed it caught "a payload that cannot evaluate, wherever it sits"; that was false,
|
||||
and the corrected claim is the one to rely on.
|
||||
|
||||
Worth knowing why nothing caught this for three days: every *other* workflow-shape test in that file
|
||||
reads `_code_lines()`, which strips comments. That is correct for what it was for, but it encodes the
|
||||
assumption this bug falsifies. The strict test reads the raw scalar, and must never adopt
|
||||
`_code_lines`.
|
||||
|
||||
⚠️ **A page past the end of `/issues/{n}/timeline` is JSON `null`, not `[]`** — and this instance is
|
||||
not consistent between endpoints (`/issues/{n}/comments` returns `[]` when empty). The retarget
|
||||
fence's `count_retargets` gated on `type == "array"`, so it read the real terminator as *unreadable*:
|
||||
the walk never reached a validated empty page, `rt_ok` was never `yes` for **any** PR, and the fence
|
||||
therefore withheld **every** exemption `success`. Renovate and docs-only PRs got no status at all —
|
||||
the same user-visible outcome as the dropped step above, by a completely unrelated route. So fixing
|
||||
the interpolation alone would not have restored the exemptions.
|
||||
|
||||
Two things kept it invisible, and both are worth generalising:
|
||||
|
||||
- It shipped in the **same commit** (8f6d4f443) that stopped the step executing, so the fence had
|
||||
never once run in production. A guard's first real execution is not the same event as its merge.
|
||||
- The **test double asserted the wrong shape while claiming measured fidelity.** Its comment read
|
||||
"Real shapes, measured on this instance and deliberately mirrored" and it printed `[]` for a page
|
||||
past the end. Every fence test was green against a response the server never produces, so the
|
||||
`array`-only gate was never exercised by the suite either. With the double corrected and the old
|
||||
gate restored, **most of the fence suite fails** — 18 tests when first measured at `c710db4a1`, 21
|
||||
once three more fence-dependent tests existed. The invariant is the point, not the count: they had
|
||||
all been passing for the wrong reason. (Given as a range on purpose — an earlier draft cited a bare
|
||||
"18", which was stale two commits later, inside a section about stale claims.) When a double claims
|
||||
fidelity, that claim is a test assertion and needs re-measuring like any other.
|
||||
|
||||
The type is now read as a value (`case` over `jq -r 'type'`) rather than through `jq -e`, whose
|
||||
exit-status semantics already bit this workflow once at jq 1.6, and both `null` and `[]` terminate the
|
||||
walk. The regression test is parameterised over both shapes because both are live on this server.
|
||||
`null` is accepted as exhaustion only from **page 2 on** — every real PR's first page carries events
|
||||
(spot-checked non-empty across #752/#753/#749/#739/#717; the counts are deliberately not recorded here
|
||||
because timelines grow and an earlier draft's five figures were stale within days), so a `null` first
|
||||
page is anomalous rather
|
||||
than empty, and the walk should not certify "no retarget happened" from a response it cannot explain.
|
||||
|
||||
**The same nil-slice shape bites `/commits/{sha}/status`** — a third instance, found by cold review of
|
||||
the fix for the second. A head with no statuses yet returns
|
||||
`{"state":"pending","total_count":0,"statuses":null}` (measured on PR #739's head). `read_existing_verdict`
|
||||
gated on `.statuses | type == "array"`, so it hit its `exit 1` and posted nothing at all — fail-closed,
|
||||
same user-visible outcome. `null` is now accepted there only when `total_count` is 0, so a body that
|
||||
merely lost its array is still refused and an existing verdict is still protected from a transient
|
||||
error. `scripts/pr-changed-files.sh` was swept and is unaffected (`pulls/{n}/files` returns `[]`).
|
||||
**The generalisable rule: a nil Go slice serialises to `null`, so every list-shaped field on this API
|
||||
is suspect and only a per-endpoint measurement settles it.**
|
||||
|
||||
**Establishing that "no verdict exists" needs a second page, and both arithmetic guards for it are
|
||||
no-ops here.** `read_existing_verdict` concluding absence is what licenses posting an exemption over a
|
||||
verdict the job cannot see, so that conclusion has to be earned. Two obvious checks were tried and both
|
||||
proved empty:
|
||||
|
||||
- **`.statuses | length` vs `.total_count`** — `total_count` is the count for the **page returned**, not
|
||||
for the commit. Measured at 1.27.1 on `3aed43c6` (6 contexts): `?limit=1` returns
|
||||
`len=1, total_count=1`, `?limit=3` returns `len=3, total_count=3`. Equal by construction, so the check
|
||||
reads as a completeness proof while proving nothing.
|
||||
- **"refuse when the page comes back full at the requested `limit=100`"** — this instance caps `limit`
|
||||
at the server-wide `MAX_RESPONSE_ITEMS`, **measured at 50** (`/issues?limit=100` returns 50). A
|
||||
response can therefore never carry 100 rows, and the comparison was **dead code**. The repo already
|
||||
documented that cap in `scripts/pr-changed-files.sh`, two test files and `ci.script-tests-job`; the
|
||||
guard was written against 100 anyway, and a cold review caught it. Hardcoding 50 instead would
|
||||
re-break the day the setting changes.
|
||||
|
||||
So the job **asks the server, and only when it matters**: if the `review-verdict/h10` row is on page 1
|
||||
there is nothing further to learn (this endpoint returns the latest status per *context*, and a context
|
||||
cannot recur on a later page). When the row is absent it reads **page 2** — any rows there mean the list
|
||||
runs longer than one page and a verdict could be beyond it, so it refuses instead of concluding absence.
|
||||
Cap-independent by construction. Paging is real here: measured `?limit=3&page=2` returning three further
|
||||
rows, and `page=9` returning the same `statuses: null` terminator.
|
||||
|
||||
The `total_count` zero-check also requires the JSON **type** to be a number: `jq -r` renders `0` and
|
||||
`"0"` identically, so a text compare would accept a schema-corrupted `"total_count": "0"` as "no
|
||||
statuses".
|
||||
|
||||
**The repo-wide expression guard scans PARSED scalars, not raw file text.** A delimiter in an ordinary
|
||||
top-level YAML comment is inert — the runner never evaluates it — so redding on it is a false positive,
|
||||
and this file has now produced that false red twice. PyYAML drops those comments. A `run:` body is
|
||||
itself a scalar and keeps its *shell* comments, which is the point: inside a `run:` scalar a comment is
|
||||
not inert. Verified both directions by mutation — an inert top-level comment passes; the same payload
|
||||
in a run-body comment still reds.
|
||||
|
||||
**`CLAUDE.md` and `AGENTS.md` are now PROTECTED paths.** `DOCS_ONLY` matched them, so the documents
|
||||
that *define* the completion protocol, the merge-consent convention and the H10 rule were themselves
|
||||
docs-only-exemptible while `.claude/` was protected — the same self-exemption the gate rules out, one
|
||||
directory over. Driving the real classify body with a lone `CLAUDE.md` change produced
|
||||
`review-verdict/h10=success`. It is fixed here rather than deferred because restoring the exemptions is
|
||||
what makes it reachable: no exemption `success` was writable at all while the classify step was
|
||||
dropped. `README.md` is deliberately not listed — ordinary prose, no enforcement. For the same reason,
|
||||
#706's known residual returns with the working fence: while `rt_ok` was never `yes`, route 1 was closed
|
||||
by accident.
|
||||
|
||||
**That gap is now closed** — `docker-build.yml`'s `test` and `migrations` jobs are also required
|
||||
contexts, and there a dropped step is **fail-OPEN**: the required check goes green having done no work,
|
||||
which is strictly worse than an absent status (compare #684). ersatztv#756 gave those two jobs
|
||||
per-**step** execution markers and extended the delimiter ban to them; see
|
||||
"Dropped-step guard on the required jobs" above.
|
||||
|
||||
It lives in its **own workflow file** on purpose: `pr-checks.yml` sets `cancel-in-progress: true`,
|
||||
and a cancelled run there would leave an exempt PR with no status and no further push to
|
||||
re-trigger it. Its own job context (`Review verdict / Set review-verdict status`) is **not** the
|
||||
required check — a workflow must not satisfy the gate merely by running successfully.
|
||||
|
||||
Full rationale: `docs/decisions/records/release/verdict-status-check.md`.
|
||||
Full rationale: `docs/decisions/records/release/verdict-status-check.md` and
|
||||
`docs/decisions/records/ci/shared-pr-file-enumeration.md`.
|
||||
|
||||
## CI toolchain image (`docker/ci/Dockerfile`, `.gitea/workflows/ci-image.yml`)
|
||||
|
||||
|
||||
+2
-1
@@ -206,7 +206,7 @@ another doc or an old issue comment should land here and then follow the link.
|
||||
- 2026-07-22 — per-schedule clock-boundary padding is a synthetic content-less Pad over the existing per-episode machinery (#392) — [`sched.clock-padding-schedule-toggle`](decisions/records/sched/clock-padding-schedule-toggle.md)
|
||||
- 2026-07-23 — Channel health = a server-derived `health` object on the channel DTOs, built-timeline detection (#415) — [`api.channel-health-object`](decisions/records/api/channel-health-object.md)
|
||||
- 2026-07-23 — Channel origin is immutable creation-provenance, stamped at insert, not a health signal (#414) — [`channel.origin-marker`](decisions/records/channel/origin-marker.md)
|
||||
- 2026-07-23 — Facet-value typeahead is a new endpoint, allow-listed to text fields, no caching (#434) — [`api.search-field-values`](decisions/records/api/search-field-values.md)
|
||||
- 2026-07-23 — Facet-value typeahead is a new endpoint, allow-listed to text fields, no caching (#434) — [`api.search-field-values`](decisions/archive/api/search-field-values.md) (superseded by `api.search-field-values-sources`)
|
||||
- 2026-07-23 — Relative-date rule builder operators are a frontend-only mapping onto existing Lucene macros (#435) — [`rulebuilder.relative-date-macros`](decisions/records/rulebuilder/relative-date-macros.md)
|
||||
- 2026-07-25 — A media-server sweep also refuses when the api client silently dropped items whose projection threw; the ratio threshold is rejected (#484) — [`scan.projection-failure-sweep-guard`](decisions/records/scan/projection-failure-sweep-guard.md)
|
||||
- 2026-07-25 — LibraryFolder identity is enforced by a unique index on `(LibraryPathId, PathHash)`, not an in-process lock (#491) — [`scan.libraryfolder-unique-identity`](decisions/records/scan/libraryfolder-unique-identity.md)
|
||||
@@ -215,3 +215,4 @@ another doc or an old issue comment should land here and then follow the link.
|
||||
- 2026-07-25 — Rule-builder group nesting is bounded-arbitrary depth (`MAX_GROUP_DEPTH`), not one level (#436) — [`spa.rulebuilder-nesting`](decisions/records/spa/rulebuilder-nesting.md)
|
||||
- 2026-07-25 — The rationale-edit marker is a git trailer, not a substring anywhere in the commit range (#609) — [`ci.decisions-edit-trailer`](decisions/records/ci/decisions-edit-trailer.md)
|
||||
- 2026-07-25 — UI-E2E: headless Playwright flows in the existing `functional-e2e` job, browser baked into the CI image (#445) — [`ci.ui-e2e-harness`](decisions/records/ci/ui-e2e-harness.md)
|
||||
- 2026-07-26 — Facet-value typeahead restated: every artist source covered; the JSON-column source is paged by row position with no residual SQL predicate (#578) — [`api.search-field-values-sources`](decisions/records/api/search-field-values-sources.md)
|
||||
|
||||
@@ -28,12 +28,15 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `api.schedule-item-flat-dto` | Schedule-item GET/POST/PUT use a flat, non-polymorphic `ScheduleItemResponseModel` (every subtype field promoted to a nullable top-level member) instead of the polymorphic Application VM hierarchy, with mutation field names matching `ScheduleItemRequest` 1:1 for a lossless round-trip. | 2026-07-10 | [link](records/api/schedule-item-flat-dto.md) |
|
||||
| `api.scheduling-hardening` | Create/Replace handlers guard against null/whitespace `name` (`IsNullOrWhiteSpace`, not just `Length`) to prevent NRE-500s, template-item overlap validation compares by index (not record value-equality) to catch exact-duplicate items, and unreachable 404 `ProducesResponseType` attributes on create-only actions are trimmed. | 2026-07-13 | [link](records/api/scheduling-hardening.md) |
|
||||
| `api.search-allitems-paging` | `GET /api/v1/search/all-items` is paginated (capped page size, `Totals` field) to bound DoS exposure; the SPA add-all flow pages to completeness instead of relying on an unbounded response. | 2026-07-18 | [link](records/api/search-allitems-paging.md) |
|
||||
| `api.search-field-values` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for one of a narrow allow-list of catalog fields (not the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a typeahead suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source; case-insensitive prefix-filtered on `q`, `limit` clamped to `[1, 50]` (default 50). | 2026-07-23 | [link](records/api/search-field-values.md) |
|
||||
| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort, and its rows are read by a keyset page carrying **NO RESIDUAL predicate** — `SELECT Id, <col> AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL`. The cursor is itself a predicate, but a SEEKABLE one on the ordering key: it positions the scan and never discards a row. A RESIDUAL predicate discards rows the engine already produced, and `LIMIT` truncates only the survivors — so with one present it bounds the OUTPUT rather than the row count. All selectivity is in memory. The guarantee is scoped: **at most 20,000 LOGICAL rows returned/materialized and at most 10 round trips (11 for `artist`)** — NOT bounded physical work and NOT bounded bytes, because MySQL traverses deleted-but-unpurged index records and the `TEXT`/`longtext` payload width is unrestricted. The walk pages 2,000 rows at a time, stopping on the first of enough distinct matches, a short page, or the ceiling. | 2026-07-26 | [link](records/api/search-field-values-sources.md) |
|
||||
| `api.search-field-values-unicode-fold` | The EF-sourced facet fields (`genre`, `show_genre`, `studio`, `director`, `writer`, `actor`, `tag`, `network`, `collection`, `video_codec`, `album`, and `artist`'s entity half) reach stored values whose prefix carries an uppercase non-ASCII character, on BOTH providers, with no row budget and no accepted loss. The defect was SQLite-only and ONE-SIDED: SQLite's `LOWER()` folds ASCII only (`lower('Édith')` is `'Édith'` unchanged), so the predicate UNDER-matched, which no later stage can repair. MySQL was already correct — its `LOWER()` is Unicode-aware, so `LOWER('Édith')` really is `'édith'` and the existing predicate reaches the row unaided. The fix is a SECOND, ADDITIVE query taken only when `isSqlite && q contains a non-ASCII character`: raw Dapper SQL `SELECT DISTINCT <col> AS Value FROM <table> WHERE [<discriminator> AND] etv_upper(<col>) LIKE @Pattern ESCAPE '\' ORDER BY <col> LIMIT @Limit`, where `etv_upper` is a `SqliteConnection.CreateFunction` scalar implementing `ToUpperInvariant`. Every other case — all-ASCII `q`, and MySQL for all `q` — runs today's EF query BYTE-IDENTICALLY. Keeping selectivity in SQL here is NOT the refuted family from `api.search-field-values-sources`: those four attempts bounded a walk around a predicate that could not be made correct over JSON escape text, whereas this is a correct fold on a plain column in an ordinary `LIMIT`ed query. It narrows that record's "Known limitation inherited, not introduced" clause; everything else it settles still holds. | 2026-07-27 | [link](records/api/search-field-values-unicode-fold.md) |
|
||||
| `api.search-paging-cap` | Search stays capped at 100 items per media kind; an overflowing kind's "See all" reuses library-browse paging instead of adding new API surface. | 2026-07-11 | [link](records/api/search-paging-cap.md) |
|
||||
| `api.selection-projection-include-chain` | Every handler that projects an aggregate carrying a tagged-union selection loads it through ONE shared `<Aggregate>QueryExtensions` include chain — `RerunCollectionQueryExtensions.IncludeSelectionDetails()`, joining the existing `ProgramScheduleItemQueryExtensions.IncludeScheduleItemDetails()` — called by the paged-list handler and the by-id handler alike, so the two cannot drift. The media-item flattening switch is likewise ONE shared helper, `MediaCollections.Mapper.ProjectMediaItemToViewModel`, covering all ten selectable media types including `RemoteStream`, whose named projection is `MediaItems.Mapper.ProjectToNamedViewModel` (it cannot be an overload of `ProjectToViewModel(RemoteStream)`, which already exists returning the unrelated `RemoteStreamViewModel`; C# will not overload on return type). That switch NEVER ends in `_ => null`: a null MediaItem is the legitimate not-a-media-item case, while an unrecognized non-null subtype keeps its id and takes a conspicuous `[unsupported media type: X]` name. Fail-soft is deliberate — throwing would fail an entire paged GET over one unreadable row. Finally, every metadata navigation inside `MediaItems.Mapper` is read through `Optional(...).Flatten()` and degrades to the `"???"` placeholder, because those projections are reached from handlers whose include chains differ and a bare `x.Season.Show.ShowMetadata` is a latent 500 on some other caller GET. | 2026-07-28 | [link](records/api/selection-projection-include-chain.md) |
|
||||
| `api.versioning-v1` | The entire `/api` surface is versioned to `/api/v1` uniformly (no unversioned corner); legacy unversioned callers are rewritten in-pipeline (not redirected) with Deprecation/Link/Sunset headers, and post-freeze `/api/v1` is additive-only — a breaking change requires `/api/v2`. | 2026-07-13 | [link](records/api/versioning-v1.md) |
|
||||
| `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) |
|
||||
| `blazor.ui-removed` | The legacy Blazor Server UI (`Pages/`, `Shared/`, `ViewModels/`, `Validators/`, MudBlazor + 8 other packages, Blazor Startup wiring) is fully deleted now that the SPA has parity; the legacy `MapWhen` branch is kept only for controllers/docs/OpenAPI/`LegacyUiRedirects`, and the catch-all fallback 302s any unmatched non-api/artwork/docs/openapi path to `/app`. | 2026-07-11 | [link](records/blazor/ui-removed.md) |
|
||||
| `channel.origin-marker` | A new `Channel.Origin` (`ChannelOrigin` enum — `Unknown`/`UserCreated`/`AutoTuned`) records how a channel row was created and is stamped exactly once at insert (`AutoTuned` in `CreateChannelFromLineupHandler`, `UserCreated` in `CreateChannelHandler`), and is never mutated on a later edit. It is surfaced as a raw `origin` field on `ChannelResponseModel`; the SPA badges only `AutoTuned`. Rows predating the column read `Unknown` — provenance is **not** back-filled. | 2026-07-23 | [link](records/channel/origin-marker.md) |
|
||||
| `ci.actions-credential-scoping` | Any credential reachable from an Actions job is scoped to what that job needs. The container-registry secret `REGISTRY_PASSWORD` is a personal access token scoped `write:package` + `read:repository` — never an account PASSWORD. This matters because Gitea has NO `status` token scope: `POST /repos/{o}/{r}/statuses/{sha}` is gated by `reqRepoWriter(unit.TypeCode)`, so ANY credential that can write the repository can forge `review-verdict/h10`, the required context that is supposed to make merge-consent derived rather than assertable. Package-write IS a separate scope, so the registry credential can be made status-incapable at no cost: `scripts/ci-detect-already-validated.sh` only GETs. Do NOT add a `permissions:` key to constrain the injected `GITEA_TOKEN` on the assumption that it binds — below Gitea 1.26.0 it is silently a NO-OP, which is worse than absent because it reads in review as a constraint. That version precondition NO LONGER HOLDS: this instance was upgraded 1.25.4 -> 1.27.1 on 2026-08-05. What has NOT changed is that the consequence is unverified — whether `permissions:` is honored here, and what this instance's default Actions token permission is, were both left UNPROBED (there is still no API surface: `/api/v1/settings/actions` 404s at 1.27.1). Probe before relying on it; do not read the upgrade alone as the constraint now working. Scoping is necessary and not sufficient: it bounds what a job may DO, never whether attacker YAML runs at all, so a self-referencing trigger needs its own filter (`ci-image.yml`, tracked in #744 — deliberately NOT bundled here, because editing that file re-points `ci-image-pin` at the editing commit and reddens a blocking job). This record closes ONE route. It does not close the class, and four later sections say exactly what survives — read them before citing this record as a mitigation. | 2026-08-05 | [link](records/ci/actions-credential-scoping.md) |
|
||||
| `ci.batch-pushes-no-cancel-route` | Hold review fixes, doc corrections and format fixes locally and push **once** — a superseded run cannot be cancelled from the agent side and holds a runner slot until it finishes. | 2026-07-21 | [link](records/ci/batch-pushes-no-cancel-route.md) |
|
||||
| `ci.build-once-rejected` | CI build-once (a shared compile artifact across jobs) was implemented, measured, and rejected for a 40-85% wall-clock regression; keep the #420 cross-run tree-identity skip instead. | 2026-07-18 | [link](records/ci/build-once-rejected.md) |
|
||||
| `ci.cancelled-is-not-a-verdict` | Treat a `cancelled` conclusion as "no verdict" — never as pass or fail — and report FAILED and CANCELLED counts separately in any CI monitor. | 2026-07-21 | [link](records/ci/cancelled-is-not-a-verdict.md) |
|
||||
@@ -41,21 +44,29 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `ci.decisions-lifecycle-flake` | When `decisions lifecycle` is the **only** red job, do not investigate and do not create a new run to clear it — no rebase, no `--amend`, no no-op push; the operator reruns that single job from the Gitea UI. | 2026-07-21 | [link](records/ci/decisions-lifecycle-flake.md) |
|
||||
| `ci.docs-only-detect-shallow-safe` | The docs-only detect script must diff against `FETCH_HEAD` (always resolves after `git fetch`, even shallow) using a two-dot tree diff — not `origin/<base>` with three-dot — because a `fetch-depth: 1` shallow clone has no remote-tracking ref and no merge-base, which silently fails the original detect into `docs_only=false` (full matrix, no functional error). A CI-behavior change must be verified by measuring the effect (job durations), not just a green check. | 2026-07-17 | [link](records/ci/docs-only-detect-shallow-safe.md) |
|
||||
| `ci.docs-only-skip-steps` | A docs-only change must still run every required job (`test`, `migrations`) so their commit-status contexts always report; each heavy job runs `scripts/ci-detect-docs-only.sh` first and gates its real STEPS on `if: steps.detect.outputs.docs_only != 'true'`, never `if:`-skips the whole job (an `if:`-skipped job reports `skipped`, not `success`, which branch protection may never unblock on). Detection biases toward running more on any doubt. | 2026-07-17 | [link](records/ci/docs-only-skip-steps.md) |
|
||||
| `ci.exemption-provenance` | The three inputs the exemption decision rests on must each be bound to something the judged PR cannot mutate. (1) BASE — `scripts/pr-changed-files.sh` takes the expected base BRANCH as a REQUIRED 5th argument and re-reads it before and after paging, because `/pulls/{n}/files` diffs against the PR's live base and retargeting moves the answer without moving the head sha; the workflow passes `github.event.pull_request.base.ref` from the `pull_request_target` payload, which a retarget cannot rewrite. (2) BOT EXEMPTION — an author match is necessary but never sufficient: `pull_request.user.login` is the PR's immutable CREATOR while its head is not, so the exemption additionally requires EVERY changed path to be a dependency manifest (`Directory.Packages.props` or `.config/dotnet-tools.json`, and ONLY those — the npm manifests are excluded because `package.json` `scripts` are executed by CI). (3) INHERITED SUCCESS — the never-overwrite short-circuit fires only for a status POSITIVELY identified as a human verdict for THIS base, meaning a non-null `.creator.login` AND a `Review-verdict:` description AND, when that description records a base (`(base: …)`, `release.verdict-status-check`), a base matching the PR's — tested by requiring the description to END with the exact literal `(base: <base>)` and to contain exactly ONE such marker, never by extracting a value (see below); a present-but-different base is rejected, an absent one is not, since verdicts predating that convention carry none; every other shape, including any unrecognised one, is re-derived rather than trusted. The bot and docs-only exemptions are evaluated as INDEPENDENT predicates and the decision made afterwards, never as an `elif` chain. `edited` is in the workflow's `types:` so a retarget reclassifies — which gives DETECTION, not atomicity: status writes are not serialized, so a stale run can still post over a fresher one. That residual is now FENCED rather than merely tracked — the job refuses to write at all if the PR's timeline retarget COUNT moved while it was classifying (`ci.verdict-write-retarget-fence`, #706) — leaving only the sub-round-trip window that no API without compare-and-set can close. The PROTECTED path list additionally covers `CLAUDE.md` and `AGENTS.md` (#751) — they are not prose but the documents DEFINING the completion protocol, the merge-consent convention and the H10 rule, so protecting `.claude/` while the file specifying what it enforces stayed docs-only-exempt was the same self-exemption one directory over; driving the real classify body with a lone `CLAUDE.md` change produced an exemption `success`. `README.md` is deliberately not listed. It also covers `.codex/` (#711), which mirrors `.claude/hooks/` byte for byte including the merge-consent hook — latent while that directory is untracked, live the moment it is tracked; the list stays ENUMERATIVE rather than derived, because a derived rule would have to be evaluated against the very file list being classified. Reading the CURRENT status for input (3) must tolerate `statuses: null`: `GET /commits/{sha}/status` serialises a nil slice as `null`, not `[]`, on a head with no statuses yet, and an `array`-only gate made `read_existing_verdict` `exit 1` and post nothing at all (#751, `ci.workflow-run-body-no-expressions`) — `null` is accepted only when `total_count` is 0, so a body that merely lost its array is still refused. Path predicates are evaluated by COUNTING with `grep -c`, never `\| grep -q` (SIGPIPE inversion) and never a here-string (temp-space failure) — see `ci.grep-q-pipefail-inversion`. | 2026-07-29 | [link](records/ci/exemption-provenance.md) |
|
||||
| `ci.format-gate-folder-mode` | The blocking `format` CI job (and matching pre-commit hook) runs `dotnet format whitespace . --folder --include <files>` instead of loading the full MSBuild/Roslyn solution, cutting the gate from ~480s to ~0.5s with unchanged whitespace/charset coverage. | 2026-07-19 | [link](records/ci/format-gate-folder-mode.md) |
|
||||
| `ci.functional-e2e-harness` | The `functional-e2e` CI job boots the PR's own code from source via `dotnet run` (`scripts/e2e-local.sh`) and runs deterministic assertions (`scripts/e2e-functional.sh`) as an advisory (non-blocking) job, not a `build` dependency or required check. Originally curl-only; since #445 the same job carries a second, headless-browser step for the contracts curl cannot express — see `ci.ui-e2e-harness`. | 2026-07-16 | [link](records/ci/functional-e2e-harness.md) |
|
||||
| `ci.gate-trigger-base-resolved` | The workflow that writes the branch-protection-required `review-verdict/h10` status triggers on `pull_request_target` with `branches: [main]`, never on plain `pull_request`. Gitea resolves a `pull_request` workflow DEFINITION from the PR's own head commit, so under that trigger a PR editing `.gitea/workflows/review-verdict.yml` ran its own rewritten copy and could post `h10=success` for itself; `pull_request_target` resolves the definition from the base instead. The `branches: [main]` filter is part of the rule, not a refinement of it: base resolution only relocates the rewrite from the head to the base, so without the filter a PR opened into an attacker-pushed base branch runs that branch's gate. `pull_request_target` is safe HERE only because this job never checks out or executes head-supplied code — it checks out `base.sha` and runs only that tree's scripts (`ci.shared-pr-file-enumeration`); reintroducing a head checkout under this trigger would be worse than the bug it fixed. This closes the rewrite route through THIS workflow and does NOT close the class: Gitea injects a write-capable `GITEA_TOKEN` into EVERY job, so any ref-resolved workflow — and a collaborator's own API token, since branch protection binds the context and not its issuer — can still forge `review-verdict/h10`. The credential half is now RESOLVED in `ci.actions-credential-scoping` (#697): CI's registry secret was the ADMIN account's basic auth and is now a PAT that cannot post a status, which removes the ADMIN escalation and that credential's route (a user credential's forgery carries a real `creator` and is inherited as a human verdict; an Actions job's carries `creator: null` and is re-derived — but do NOT read that asymmetry as protection: re-derivation fires only on the trigger's `types`, and posting a status is not one of them, so a POST timed after the last PR event simply stands). It does not remove EVERY route: `RENOVATE_TOKEN` is a `write:repository` bot PAT in the same secret store, reachable by any PR-added workflow. The injected token stays write-capable until Gitea >=1.26 with a Restricted default (server-management#714), and a collaborator's own token remains unfixable; the exemption path has its own separate defects in #698. | 2026-07-28 | [link](records/ci/gate-trigger-base-resolved.md) |
|
||||
| `ci.gitea-milestone-filter-noop` | Never filter issues with the server-side `?milestones=<name>` parameter — fetch all open issues once and filter LOCALLY on each issue's `.milestone.title`. | 2026-07-21 | [link](records/ci/gitea-milestone-filter-noop.md) |
|
||||
| `ci.grep-q-pipefail-inversion` | In any script running under `set -o pipefail`, a security or classification predicate of the form `producer \| grep -q…` is FORBIDDEN: `grep -q` exits at its first match, the producer then takes SIGPIPE and exits 141 once the data exceeds the pipe buffer (~64K), so `pipefail` reports the pipeline as FAILED even though grep MATCHED — inverting the predicate exactly when the input is large. A here-string (`grep -q… <<< "$data"`) is ALSO forbidden: bash materialises a large here-string via temporary storage, so it fails when temp space is full or unwritable, and inside an `if`/`!` that failure flips the predicate the same way. COUNT instead — `n=$(printf '%s\n' "$data" \| grep -cE "$re")` — because `grep -c` drains stdin (no early exit, no SIGPIPE) over an ordinary pipe (no temp file). Read grep's status honestly: exit 1 means a zero count and is a legitimate answer, anything >1 is a real error. Evaluate the counts ONCE at TOP LEVEL, never inline inside an `if`/`elif` condition: inside `$( )` an `exit` leaves only the subshell and `set -e` does not fire, so an error silently reads as "no match". Validate that each result is numeric and fail closed if not. This applies to both the enforced gate `.gitea/workflows/review-verdict.yml` and the advisory hook `.claude/hooks/pretooluse-merge-consent.sh`. | 2026-07-29 | [link](records/ci/grep-q-pipefail-inversion.md) |
|
||||
| `ci.infra-shaped-red-under-load` | When a job dies inside a setup/cache step before your code compiles, check the runner host's load before diagnosing the diff, and never file a CI bug off one sample under pressure. | 2026-07-21 | [link](records/ci/infra-shaped-red-under-load.md) |
|
||||
| `ci.jq-version-contract` | Every shell gate that shells out to `jq` is authored to the jq 1.6-compatible subset, because the CI runner ships jq 1.6 while every developer Mac ships 1.8.x. `scripts/jq-preflight.sh` (no args) prints the parsed version and asserts a floor of 1.6 in every gate job's log; `scripts/jq-preflight.sh --expect 1.6` additionally pins and fails loudly, but ONLY in the `script-tests` job. `review-verdict.yml` never pins — it writes the branch-protection-required `review-verdict/h10` status, so a hard pin there would turn any jq bump into a repo-wide merge deadlock. | 2026-07-26 | [link](records/ci/jq-version-contract.md) |
|
||||
| `ci.killed-job-triage` | Never trust a job's `conclusion` field alone — read the log tail and require an `❌ Failure - Main …` marker before treating a red as a real failure. | 2026-07-21 | [link](records/ci/killed-job-triage.md) |
|
||||
| `ci.monitor-armed-at-pr-open` | Arm a CI monitor on the PR head sha the moment the PR opens, polling the commit-status endpoint — not at the end of the work. | 2026-07-21 | [link](records/ci/monitor-armed-at-pr-open.md) |
|
||||
| `ci.no-host-health-gating` | Push when your work is validated — never SSH to bumblebee to sample load/RAM first, and never hand-schedule around other sessions' runs. | 2026-07-21 | [link](records/ci/no-host-health-gating.md) |
|
||||
| `ci.peak-anon-measurement` | The `test` job's headline memory figure is a sampled high-water mark of cgroup `anon`, produced by `scripts/ci-peak-anon.sh`; `memory.peak` and the end-of-job `anon`/`file` split are kept only as a cache-inflated reference. | 2026-07-19 | [link](records/ci/peak-anon-measurement.md) |
|
||||
| `ci.required-job-step-execution-markers` | A step the runner declines to interpolate is DROPPED and the job still concludes `success` (`ci.workflow-run-body-no-expressions`). In `review-verdict.yml` that is fail-CLOSED — the required status is absent and the merge is blocked. In `docker-build.yml`'s `test` and `migrations` it is fail-OPEN: those are the other two required contexts on `main`, so the check reports green having done no work. So in those two jobs every `run:` step that is not `continue-on-error: true` calls `"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark <key>` as its FIRST act, and the job's LAST step calls `ci-step-ran.sh assert --always <keys> --gated <keys>`, which fails the job when an expected key was never recorded. PER STEP, not per job: a marker written by the first step only proves the job started, while the drop that costs something is `Test` or the migration replay. The guard carries NO `if:` — the default `success()` is the wanted condition, because a genuine failure in an early step legitimately skips every later one and an `always()` guard would announce a false "these steps never executed" on every ordinary red build; the invariant that makes the omission safe is that the guard is skipped only when an earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every path to a green job runs the guard. Separately and independently, no `${{` OPENER may appear in any `run:` body of those two jobs OR of `build` — the drop mechanism requires the opener, so banning it makes the class unreachable rather than merely caught, and an UNCLOSED opener triggers the same rewrite as a well-formed pair. Pass values in through the step's `env:`, which is interpolated per value. The two halves have DIFFERENT scopes on purpose: markers cover the required pair, while the ban also covers `build`, whose `Smoke + IPTV E2E` step runs AFTER the image is pushed, so a drop there publishes a release candidate that was never booted and that `DeployStack jazz-media` then promotes. `functional-e2e` is delimiter-free but deliberately excluded (advisory by declaration), and `api-docs`/`format` keep one `github.base_ref` each and gate nothing that ships. The ban is enforced on the RELEASE PATH itself, not only in review (#767): a `scan` job runs the PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and no image is published. A guard STEP inside `build` was tried first and is wrong — a step cannot protect the job it publishes from, and "my body has no opener so I cannot be dropped" is circular when only the PR-only test enforces that. The pytest in `script-tests` remains, but it is `on: pull_request` and not a required context, so it alone left the tag path unchecked. | 2026-08-10 | [link](records/ci/required-job-step-execution-markers.md) |
|
||||
| `ci.root-screenshot-guard` | The Husky `pre-commit` hook refuses a staged root-level `*.png` (belt-and-suspenders with the `.gitignore` rule); nested `*.png` real assets are unaffected. | 2026-07-12 | [link](records/ci/root-screenshot-guard.md) |
|
||||
| `ci.runner-placement` | No persistent Roslyn compiler server survives a CI build (`UseSharedCompilation=false` etc., runner env + Dockerfile `ENV`); every `services:` container gets its own explicit `--memory`/`--memory-swap`/`--cpus` cap (it does not inherit the job container's). | 2026-07-17 | [link](records/ci/runner-placement.md) |
|
||||
| `ci.script-tests-job` | The `scripts/tests/` pytest suite runs on every PR as a dedicated `script-tests` job in `pr-checks.yml` (`runs-on: small`, `setup-python` + `pip install pytest`, `PYTHONPATH=. python3 -m pytest scripts/tests -q`), unconditionally rather than behind a `scripts/**` path filter, and **never as a step inside `decisions-guard`** — a job whose reds a standing rule instructs sessions to ignore must never host a gate whose reds are real. Any new CI gate must be reachable by a failure that is unambiguously attributable to it. | 2026-07-26 | [link](records/ci/script-tests-job.md) |
|
||||
| `ci.shared-pr-file-enumeration` | A PR's complete set of changed file paths is computed by exactly one implementation, `scripts/pr-changed-files.sh`, called by both `.claude/hooks/pretooluse-merge-consent.sh` (advisory — a failure falls through to a human prompt) and `.gitea/workflows/review-verdict.yml` (enforced — a failure must fail closed, because a match here posts the branch-protection-required `review-verdict/h10` status with nobody in the loop). The script owns exhaustiveness (pagination, rename/path validation, head-sha binding, base-ref binding — see `ci.exemption-provenance` — and base-TIP binding, #707: the ref answers "did this PR RETARGET", the tip answers "did the base ADVANCE mid-enumeration", and only the second can see `/pulls/{n}/files` recomputing each offset-paged page against a moved base and dropping a path out of an already-consumed range; both ends of the window are bound, and an advance BEFORE the window is deliberately not an error, or ordinary churn on `main` would fail every open PR) and returns exit 0 only for a verified-complete list; it does NOT classify paths — each caller keeps its own docs-only allow-list, and the two allow-lists differ on purpose and stay separate. | 2026-07-26 | [link](records/ci/shared-pr-file-enumeration.md) |
|
||||
| `ci.small-lane-git-only` | `runs-on: small` is defined by what a job does (git-only), not its usual runtime; the two `docker build` jobs (docker-build.yml, ci-image.yml) move to `ubuntu-latest` because their worst-case memory, not median runtime, was pinning the small lane's per-slot cap. | 2026-07-20 | [link](records/ci/small-lane-git-only.md) |
|
||||
| `ci.ui-e2e-harness` | The UI-interactive E2E flows run as headless Playwright specs (`web/e2e/*.spec.ts`, driven by `scripts/e2e-ui.sh`) in a **second step of the existing advisory `functional-e2e` job**, never their own job; the browser is `chromium-headless-shell` **baked into the CI toolchain image** (`docker/ci/Dockerfile`, `PLAYWRIGHT_VERSION` kept equal to `web/package.json`'s EXACT `@playwright/test` pin), never installed per run; specs are `serial` with `retries: 0` and assert only contracts the curl harness structurally cannot reach. | 2026-07-25 | [link](records/ci/ui-e2e-harness.md) |
|
||||
| `ci.verdict-write-retarget-fence` | The `review-verdict/h10` job counts `change_target_branch` events on the PR's issue timeline at run start and again immediately before its POST, and writes NOTHING if the count moved. The COUNT is the key because the branch NAME is ABA-vulnerable — `main -> S -> main` reads `main` at both ends, which is how #698 route 1 obtained a forged exemption — while the event count is monotonic and cannot alias. Abstaining is a handoff, not a stall, and that is the property the design rests on: every retarget fires `edited`, which is in this workflow's `types:`, so the event that makes a run abstain has already queued a successor whose window opens after it; the induction terminates when retargeting stops and the last run writes the final answer. `updated_at` was REJECTED as the key because it also moves for comments and labels, which fire none of this workflow's `types:` — a run could abstain with no successor coming, which is a real stall. The count is trusted only when paging reached a validated EMPTY page; an untrusted count (unreadable page, non-array body, non-numeric length, page cap hit) blocks the exemption `success` ONLY and still lets `pending` through, because `pending` cannot turn an unreviewed head green while withholding it would strand ordinary PRs for no safety gain. SEPARATELY, and for the human-verdict race the fence does nothing about: after posting an exemption `success` the job re-reads `/statuses/{sha}` and, if a human `Review-verdict:` row appeared with an id ABOVE a high-water mark taken just before the POST, overwrites its own status with `pending` and logs an error. The repair is `pending`, NEVER a copy of the human's state, since re-posting their `failure` under the machine credential would attribute a human verdict to the job; its description is a SENTINEL that the classification refuses to grant an exemption over AND re-writes verbatim on every later run, so the block is a FIXED POINT rather than decaying — writing the generic `pending` description there instead erases the marker and the exemption simply returns one event later. The mark is captured BEFORE the last-moment re-read, not merely before the POST — a later mark leaves a multi-round-trip blind gap in which a verdict is neither seen by the re-read nor repaired afterwards. The id comparison is load-bearing: a mere presence test would fire forever on a base-mismatched verdict that `read_existing_verdict` deliberately declines to honour, deadlocking that PR's exemption permanently. Finally, a run whose last-moment re-read finds a sentinel it did not see at its FIRST read ABSTAINS instead of posting: that can only mean an overlapping run repaired a raced verdict mid-flight, and this run's `success` — frozen at classification time, with the human row below its own mark, so neither the fence nor the post-write check would catch it — would otherwise bury the rejection. That is the one path in this design that failed toward SUCCESS rather than `pending`. The post-write check counts TWO row shapes above the mark, not one — a human `Review-verdict:` row AND a machine sentinel — because with two overlapping runs the human row can sit BELOW the second run's mark while the first masks it and only then writes the sentinel, leaving the second to post its own `success` on top; counting the sentinel converges both runs on the fixed point instead. | 2026-08-03 | [link](records/ci/verdict-write-retarget-fence.md) |
|
||||
| `ci.verify-locally-ci-confirms` | Treat the local build/verify/review pass as the decision point and CI as confirmation — don't idle waiting on a run you have no reason to doubt. | 2026-07-21 | [link](records/ci/verify-locally-ci-confirms.md) |
|
||||
| `ci.web-test-per-test-timeouts` | Give heavy-render web tests an explicit per-test vitest timeout (e.g. 15s); never raise the global default to fix one slow test. | 2026-07-21 | [link](records/ci/web-test-per-test-timeouts.md) |
|
||||
| `ci.workflow-run-body-no-expressions` | A `run:` body is not shell when the runner reads it: the runner scans the whole scalar for the expression opener and, on finding one, rewrites the ENTIRE body into a single `format(...)` call. That rewrite is all-or-nothing, so a payload that does not evaluate fails the interpolation of the whole scalar — and the runner then DROPS THE STEP AND CONCLUDES THE JOB `success`. A shell comment is therefore NOT inert. In `.gitea/workflows/review-verdict.yml` no expression delimiter may appear in ANY `run:` body, in code or in prose, because a dropped step there is a dead merge gate rather than a failed build; pass values in through the step's `env:` block, which is interpolated per value so a bad payload cannot take the body with it, and describe an expression in prose by NAMING it (`a github.event.pull_request.number expression`) rather than quoting the delimiters. Repo-wide the rule is weaker and its reach must be stated precisely rather than generously: every expression payload in every workflow field must have a HEAD TOKEN naming a context or function the runner can resolve. That catches the defect above and a nonexistent context; it does NOT catch a syntactically invalid payload whose tokens are all known (`${{ github.ref == }}`), a renamed output (every token after the first is skipped), or an unclosed opener — those need an expression parser, and the guard is kept permissive on purpose because a red here blocks every merge through the combined status. In `review-verdict.yml` specifically, any step whose non-execution is consequential is paired with a start-marker guard that FAILS the job when the marker is absent, and that guard's own body must be expression-free — a guard the guarded mechanism can silently delete is worse than none. That pairing now also covers `docker-build.yml`'s `test` and `migrations` jobs, where a dropped step is fail-OPEN (the required check goes green having done no work) rather than fail-closed as it is here — see `ci.required-job-step-execution-markers`, which adds per-STEP markers there and extends this file's delimiter ban to those two jobs. It is still not a repo-wide property, but the remaining exceptions are narrower than this record originally said: `build` was brought into the ban too (its `Smoke + IPTV E2E` runs AFTER the image is pushed, so a drop there ships an unsmoked release candidate — its two payloads moved to `env:`, so the ban was free), leaving only `api-docs` and `format`, whose one `github.base_ref` each sits in a detect step that gates nothing that ships. | 2026-08-06 | [link](records/ci/workflow-run-body-no-expressions.md) |
|
||||
| `concurrency.diff-scalar-fanout` | The frozen Block optimistic-concurrency recipe (api-conventions §7a) fans out to Collection/Playout×2/MultiCollection/RerunCollection, keeping a guard-returned `PreconditionFailedError` out of any handler's generic `catch(Exception)`→422 mapping, and preserving each aggregate's existing `SaveChangesAsync() > 0` gate semantics under the new unconditional `Version++`. | 2026-07-11 | [link](records/concurrency/diff-scalar-fanout.md) |
|
||||
| `concurrency.etag-rotation-completion` | Every handler that mutates a versioned root's editor-visible config state must bump `Version` (rotating the ETag) with no per-aggregate carve-outs, short-circuiting on a genuine no-op before the bump so idempotent re-submits don't fire spurious rebuild fan-out; `SaveChangesForcingVersion` rebases the retry (stored + pending delta), never adopts the stored token verbatim. | 2026-07-12 | [link](records/concurrency/etag-rotation-completion.md) |
|
||||
| `concurrency.force-write-non-ifmatch` | Any handler that leaves a versioned root `Modified` or `Deleted` but takes no `If-Match` (deletes, item add/remove bumpers, scalar-config writers) must save through `ConcurrencyExtensions.SaveChangesForcingVersion` — force-write past a concurrent `Version` bump rather than throw an unhandled `DbUpdateConcurrencyException` (500). | 2026-07-12 | [link](records/concurrency/force-write-non-ifmatch.md) |
|
||||
@@ -64,10 +75,11 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `concurrency.replace-all-contract` | Replace-all aggregate PUTs carry a uniform plain `int Version` concurrency token (EF `.IsConcurrencyToken()`), checked pre-save and enforced by the EF UPDATE guard, returning 412 (not 409) on a stale `If-Match`. | 2026-07-11 | [link](records/concurrency/replace-all-contract.md) |
|
||||
| `concurrency.schedule-item-child-identity` | `PUT /api/schedules/{id}/items` reconciles by an optional round-tripped child `Id` (null/absent/0 ⇒ new item), never by array position, so fill-group/shuffle state follows the logical item across reorders; an unknown or duplicate id is rejected 422 (checked after the §7a `CheckVersion`, so 412 precedes 422). | 2026-07-11 | [link](records/concurrency/schedule-item-child-identity.md) |
|
||||
| `docs.convention-docs-session-start` | Docs-first, not source-first: conventions (api-conventions, spa-conventions, e2e-local, blazor-route-parity, domain-model, decisions, README) are read from docs, not reverse-engineered from code, via `docs/README.md`'s task-signal map — only the sections it points to for the task at hand, not the whole set. Each doc is updated in the same PR that changes what it documents, replacing deferred/follow-up doc updates. | 2026-07-07 | [link](records/docs/convention-docs-session-start.md) |
|
||||
| `docs.corpus-size-signal` | The corpus's size signal is a per-record prose ceiling (`decisions_validate.py --record-ceiling`, default 60, chosen at a natural gap in the distribution), reported as a NON-BLOCKING `::warning::` naming each record over it. The aggregate prose total is still printed every run but carries NO threshold — it is a `::notice::` trend only — because a total over a monotonically growing corpus can only ratchet, and the generated catalog (`docs/decisions/README.md`) is no longer counted at all since it gains one row per record and cannot be consolidated away. Being listed by the ceiling is an invitation to check for REDUNDANCY, never an instruction to cut: a long record that is all distinct findings is a legitimate decline, and should be recorded as one. | 2026-07-26 | [link](records/docs/corpus-size-signal.md) |
|
||||
| `docs.corpus-size-signal` | The corpus's size signal is a per-record prose ceiling (`decisions_validate.py --record-ceiling`, default 60, chosen at a natural gap in the distribution), reported as a NON-BLOCKING `::warning::` naming each record over it. The aggregate prose total is still printed every run but carries NO threshold — it is a `::notice::` trend only — because a total over a monotonically growing corpus can only ratchet, and the generated catalog (`docs/decisions/README.md`) is no longer counted at all since it gains one row per record and cannot be consolidated away. Being listed by the ceiling is an invitation to check for REDUNDANCY, never an instruction to cut: a long record that is all distinct findings is a legitimate decline, and should be recorded as one. The ceiling's CALIBRATION is guarded in two pieces of different robustness (#688): the blocking test asserts only the coarse, non-ratcheting property that the ceiling flags a MEANINGFUL MINORITY of records (`0.02 <= fraction_over <= 0.25`), while the fine claim — that it sits between p90 and p95 — is REPORTED by `main()` as a `::notice::` and never asserted against the live corpus. A ceiling drifting out of date is the passage of corpus growth, not a defect in the commit under test, so it gets `stale_records`' treatment rather than a red in the blocking `script-tests` job. | 2026-07-26 | [link](records/docs/corpus-size-signal.md) |
|
||||
| `docs.decision-lifecycle` | every decision `##` record (active or archived) carries a 5-field metadata block (`key`, `status`, `since`, `supersedes`, `superseded-by`) checked by `scripts/decisions_validate.py`; a record is never deleted or line-edited to reverse a call — it is moved to `docs/decisions/archive/` with `status: superseded`/`retired` and a reciprocal `superseded-by`/`supersedes` key pair to its replacement. | 2026-07-21 | [link](records/docs/decision-lifecycle.md) |
|
||||
| `docs.decision-one-file-per-record` | Each decision record is its own file at `docs/decisions/records/<area>/<topic>.md` (archived ones at `docs/decisions/archive/<area>/<topic>.md`) with YAML frontmatter; the filename IS the key, so one-active-record-per-key is a filesystem property rather than a validator check, and supersession is a `git mv`. | 2026-07-25 | [link](records/docs/decision-one-file-per-record.md) |
|
||||
| `docs.decision-optional-provenance` | Decision records gain two OPTIONAL fields — `stale-after: YYYY-MM-DD` on the metadata line and a `**Sources:**` line in the metadata block; the Open Knowledge Format (OKF) itself is NOT adopted as the record format. | 2026-07-25 | [link](records/docs/decision-optional-provenance.md) |
|
||||
| `docs.frontmatter-pyyaml-crosscheck` | `decisions_validate.py` runs `pyyaml_frontmatter_faults()` over every record-wing file: it loads the frontmatter with PyYAML and reports an ERROR when PyYAML rejects the document OR when any key's value differs from what the dependency-free `dl._read_frontmatter` read. PyYAML is the WRITER of these files (`migrate_decisions_split.render_record` emits them with `yaml.safe_dump`), so on any disagreement PyYAML is authoritative and the defect is in the FILE, not in either parser. The check is strictly additive: when PyYAML is not importable it is SKIPPED and `main()` says so with a `::notice::`, never silently — the read path stays dependency-free because `decisions-guard`, the Husky hooks and contributor machines install nothing. The comparison has exactly ONE implementation, called by both the validator and `test_frontmatter_reader_matches_pyyaml_on_every_real_record`, so the suite and the tool cannot drift on what "matches PyYAML" means. | 2026-08-04 | [link](records/docs/frontmatter-pyyaml-crosscheck.md) |
|
||||
| `docs.record-wing-parse-guard` | `decisions_validate.py` asserts, per PATH, that every `*.md` under `docs/decisions/records/**` and `docs/decisions/archive/**` parses to exactly one record carrying a `key` — an ERROR, not a warning, since a file in the record wings that is not a record is a mistake by definition. A file sitting DIRECTLY in `archive/` is exempt only when it actually looks like a #610 stripped index — exactly one keyless record with a known generated heading — never merely by living there. The one other exemption, `archive/README.md`, is by exact RELATIVE PATH; nothing is ever exempt by BASENAME, since that would exempt the same filename in the active wing too. `_read_frontmatter` is deliberately NOT extended to accept YAML block scalars: every record value goes on ONE line, and the structural check is what makes that limitation loud instead of silent. | 2026-07-26 | [link](records/docs/record-wing-parse-guard.md) |
|
||||
| `docs.tracker-comment-retrofit` | When the knowledge exporter flags an over-cap tracker issue and excludes it from ingestion, triage its comments instead of assuming a retrofit is owed — and for each decision-shaped item check the **worked issue first**, because a tracker session comment is by construction a précis of the fuller closing record posted on the issue it narrates. Applied to #237 (111 comments) this yielded **zero** records, so server-management#642's "a fact found only in a #237 comment" retrieval row has no valid subject and its interim target (an already-migrated record) is permanent. | 2026-07-21 | [link](records/docs/tracker-comment-retrofit.md) |
|
||||
| `ffmpeg.external-logo-graphics-engine` | External-URL channel logos pass through to the graphics engine like any other watermark source; `WatermarkSelector` must never gate them on `File.Exists` (always false for a URL) and never route them through the ffmpeg-native overlay shortcut. | 2026-07-20 | [link](records/ffmpeg/external-logo-graphics-engine.md) |
|
||||
@@ -75,6 +87,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `ffmpeg.qsv-decode-encode-split` | QSV decode is decoupled from QSV encode via a single `FFmpegProfile.QsvPreferNativeDecoder` bool (default ON, Linux-only), so a QSV encode profile can decode with the more tolerant native VA-API decoder instead of the QSV decoder, mirroring Jellyfin's hybrid decode/encode toggle instead of a general decode-family enum. | 2026-07-20 | [link](records/ffmpeg/qsv-decode-encode-split.md) |
|
||||
| `ffmpeg.qsv-extra-hw-frames-floor` | a QSV upload never emits `extra_hw_frames` below `FFmpegState.MinimumQsvExtraHardwareFrames` (64); a stored `0` or negative value is treated as "no pool configured" rather than honored literally, because with no headroom any unthrottled read exhausts the pool and the transcode writes nothing at all. | 2026-07-21 | [link](records/ffmpeg/qsv-extra-hw-frames-floor.md) |
|
||||
| `ffmpeg.qsv-hdr-tonemap-opencl` | the QSV pipeline never emits `vpp_qsv=tonemap=1`, which is a SILENT no-op on pre-Gen11 Intel graphics; HDR is tonemapped on the GPU via `hwupload=derive_device=vaapi` → `scale_vaapi` → `hwmap=derive_device=opencl` → `tonemap_opencl` when a VA-API device exists, the frames are still in software, and `tonemap_opencl` is available, and by the software `TonemapFilter` otherwise. The scale runs BEFORE the tonemap, and any hardware filter on the path forces the output to be re-tagged bt709. | 2026-07-26 | [link](records/ffmpeg/qsv-hdr-tonemap-opencl.md) |
|
||||
| `ffmpeg.readrate-catchup-sparse-streams` | a realtime video/audio input also gets `-readrate_catchup` (6.0) when the binary supports it — but NOT a still-image input (mirroring the #350 exclusion) and NOT a concat input, which keep at most bare `-readrate` (a still image's video input takes none at all). Reason: `-readrate` paces the whole input off its furthest-behind stream, so a sparse stream sharing that input (an embedded PGS/DVD bitmap subtitle feeding the overlay) otherwise pins output at ~0.53x realtime. Catchup is a ceiling that applies only WHILE an input is behind, never a target, so it does not let a caught-up input race ahead. | 2026-08-04 | [link](records/ffmpeg/readrate-catchup-sparse-streams.md) |
|
||||
| `ffmpeg.remote-image-fetcher-bounded` | remote graphics-engine images are fetched through `IRemoteImageFetcher` with a pooled `HttpClientFactory` client, a body-covering deadline, a wire-transfer size cap, and a decoder-enforced `DecoderOptions.MaxFrames` bound re-verified post-decode — never cached, re-fetched per element init. | 2026-07-20 | [link](records/ffmpeg/remote-image-fetcher-bounded.md) |
|
||||
| `ffmpeg.watermark-resolution-unified` | Every watermark `WatermarkSelector` resolves goes through one shared `ResolveWatermark` — the playout-item, channel and global precedence levels AND the deco path, for all three `ChannelWatermarkImageSource` values. An unresolvable watermark (missing file, un-migrated external URL, or no logo artwork) resolves to no on-screen bug plus a warning, never a dead path or a URL handed downstream; the one deliberate exception is a playout-item `Custom` with a blank image, which still falls THROUGH to channel/global. The generated-initials fallback is therefore off everywhere, including the deco path where it demonstrably rendered. Watermarks built OUTSIDE the selector (the song-progress overlay, #653) are not covered and remain unchecked. | 2026-07-26 | [link](records/ffmpeg/watermark-resolution-unified.md) |
|
||||
| `ffmpeg.work-ahead-slot-atomic` | `workAheadSegmenterLimit` is enforced by a single compare-exchange claim on a shared `WorkAheadSlots` pool taken by the *caller* of `Transcode`, which then passes ownership in and gets the release in `Transcode`'s `finally` — never a `Volatile.Read` compare in one place and an `Interlocked.Increment` in another. | 2026-07-21 | [link](records/ffmpeg/work-ahead-slot-atomic.md) |
|
||||
@@ -85,6 +98,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `iptv.logo-drives-bug-preset` | One uploaded channel logo drives both the listing logo and the on-screen bug via a shared, seeded `ChannelLogo`-sourced watermark preset (`Channel Bug`), not new per-channel schema. | 2026-07-20 | [link](records/iptv/logo-drives-bug-preset.md) |
|
||||
| `locking.entitylocker-atomic-flags` | `EntityLocker` uses `Interlocked.CompareExchange`-guarded atomic flags plus a documented single-owner-release discipline (no owner tokens/leases); `Unlock*` on an already-unlocked slot returns `false` and logs a Warning rather than throwing. | 2026-07-11 | [link](records/locking/entitylocker-atomic-flags.md) |
|
||||
| `mcp.server-foundation` | `ErsatzTV.Mcp` is a fresh stdio JSON-RPC server wrapping frozen `/api/v1` with explicit narrow per-endpoint tools, read-only-by-default enforced at runtime (`ERSATZTV_ALLOW_WRITES`), machine-key auth, and opt-in `If-Match`. | 2026-07-20 | [link](records/mcp/server-foundation.md) |
|
||||
| `mcp.tool-schema-openapi-parity` | Every POST/PUT/PATCH tool in `ToolCatalog` declares exactly the request-body properties its endpoint accepts, each with a matching type, and EVERY tool (read and write) declares exactly its endpoint's query parameters, both asserted against the generated `ErsatzTV/wwwroot/openapi/v1.json` (linked into `ErsatzTV.Mcp.Tests`) by `Every_Write_Tool_Should_Declare_Exactly_Its_OpenApi_Request_Body_Fields` and `Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters`. A field the endpoint accepts but the tool omits is a DEFECT, not a deferral: on the full-replace tools (channel update, schedule update, custom-order) the omission is silently applied as a clear. The write tools are NOT uniformly full-replace — add-collection-items is additive, and several leave an omitted field unchanged — so each tool description states its own semantics. An omitted query parameter is UNREACHABLE, not merely undocumented, because `ToolArgumentValidator` rejects undeclared arguments. | 2026-08-06 | [link](records/mcp/tool-schema-openapi-parity.md) |
|
||||
| `media.lastscan-null-boundary` | A never-scanned `LastScan` surfaces as `null` at the API/MCP boundary, not the `0001-01-01` MinValue sentinel — enforced by an ongoing read-boundary coercion plus a one-time data migration cleanup. | 2026-07-18 | [link](records/media/lastscan-null-boundary.md) |
|
||||
| `media.remote-stream-probe` | `ValidatePlayoutItemPath` probes the Plex/Jellyfin/Emby remote-stream URL via `IRemoteStreamProber` before returning it; only a redirected 404 fails closed (`PlayoutItemNotAvailableFromMediaServer`), everything else fails open, and there is no toggle. | 2026-07-19 | [link](records/media/remote-stream-probe.md) |
|
||||
| `media.remote-stream-probe-externaljson` | External-JSON playout channels' `StreamRemotely` now probes the remote-stream URL through the same `IRemoteStreamProber` seam as the generated-playout path, closing the #473 scope gap for a channel kind with no DB `PlayoutItem` rows. | 2026-07-20 | [link](records/media/remote-stream-probe-externaljson.md) |
|
||||
@@ -102,7 +116,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `process.local-gate-before-push` | Run the local build/test gate and a cold-context, scoped "review only" adversarial review over the diff, fold the fixes, and only then push or open the PR. | 2026-07-21 | [link](records/process/local-gate-before-push.md) |
|
||||
| `process.lock-ownership-enumerate-producers` | Before trusting any "single owner / no double release / no cross-release" claim, grep the whole host project for every writer of that channel message (or acquirer of that lock) — the background scheduler/worker is the usual missing producer. | 2026-07-21 | [link](records/process/lock-ownership-enumerate-producers.md) |
|
||||
| `process.one-worktree-one-committing-agent` | Never run two committing agents concurrently on one worktree — give each parallel slice its own worktree branched off the feature branch and merge back. | 2026-07-21 | [link](records/process/one-worktree-one-committing-agent.md) |
|
||||
| `process.parallel-session-claim` | Apply the `in-progress` label before starting an issue, and still read its dependency notes before touching shared surfaces — a claim prevents duplicate pickup, not overlapping code changes. | 2026-07-21 | [link](records/process/parallel-session-claim.md) |
|
||||
| `process.parallel-session-claim` | Before starting an issue, check for an existing claim four ways — open PRs referencing it, remote branches naming it, recent comments (a claim can precede the label), and a fresh `git fetch origin main` — then claim with the `in-progress` label plus a comment. A claim prevents duplicate PICKUP, not duplicate WORK. Re-fetch `origin/main` before every push, not only at branch time. | 2026-07-21 | [link](records/process/parallel-session-claim.md) |
|
||||
| `process.per-agent-model-routing` | State the model tier (and effort, where the client exposes it) in the dispatch itself for every delegated agent — bounded recon → cheapest fast tier at `low`; mechanical slice against a documented contract → mid tier; judgment-heavy work → orchestrator tier; independent review → a different model family than the implementer. | 2026-07-25 | [link](records/process/per-agent-model-routing.md) |
|
||||
| `process.pr-routine-sequence` | Worktree off origin/main → implement → regenerate API artifacts → full local tests + cold review + live-E2E ALL before the push → push, open PR, arm the CI monitor at open → fixes after the push are follow-up commits, never amend/force-push. | 2026-07-21 | [link](records/process/pr-routine-sequence.md) |
|
||||
| `process.review-disagreement-frontier-judge` | When independent reviews disagree on a gate PR, escalate to the frontier judge, and put the proposed fix approach in front of it — not just the disputed finding. | 2026-07-21 | [link](records/process/review-disagreement-frontier-judge.md) |
|
||||
@@ -110,14 +124,15 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `process.subagent-drop-resume` | Treat a subagent connection drop as laptop sleep or transient network and re-resume via SendMessage — the work survives. | 2026-07-21 | [link](records/process/subagent-drop-resume.md) |
|
||||
| `release.api-contract-ci-gate` | A PR touching `ErsatzTV/Controllers/Api/**` or `ErsatzTV.Core/Api/**` must ship regenerated OpenAPI artifacts (`v1.json`, `v1.d.ts`, `endpoint-index.md`) in the same diff, enforced by a blocking `api-docs` CI job that regenerates-and-diffs against a fresh build. | 2026-07-12 | [link](records/release/api-contract-ci-gate.md) |
|
||||
| `release.done-when-merge-consent` | A PR may merge only when its linked issue's `## Done-when` checklist is fully ticked and the PR's CI is green, enforced by a PreToolUse hook on the Gitea merge tool (deny/allow/ask) plus a pre-push backstop for direct pushes to main. | 2026-07-12 | [link](records/release/done-when-merge-consent.md) |
|
||||
| `release.format-as-you-touch-rebase` | A blocking `format` CI job runs `dotnet format --verify-no-changes` scoped only to the PR's changed `.cs` files (never the legacy BOM backlog), and a PR branch must be kept current by rebasing on `origin/main` (never merging main in), enforced by `.husky/pre-push` → `prepush-rebase-check.sh`. | 2026-07-12 | [link](records/release/format-as-you-touch-rebase.md) |
|
||||
| `release.format-as-you-touch-rebase` | A blocking `format` CI job runs `dotnet format --verify-no-changes` scoped only to the PR's changed `.cs` files (never the legacy BOM backlog), and a PR branch must be kept current by rebasing on `origin/main` (never merging main in), enforced by `.husky/pre-push` → `prepush-rebase-check.sh`. H11 has ONE always-on carve-out, #719 — a push in which EVERY ref is under `refs/tags/` skips the freshness check, because a tag push cannot revert merged work, which is the failure mode H11 exists to prevent, and the release cut tags from a branch that is behind `origin/main` (observed on the v26.13.0 cut, #719). A push mixing branch and tag refs is still blocked, and so is a push with zero parsed ref lines (the exemption requires at least one, so empty stdin cannot vacuously disable H11). | 2026-07-12 | [link](records/release/format-as-you-touch-rebase.md) |
|
||||
| `release.live-e2e-required` | A PR that changes an API write-path handler must include a live-E2E pass (driving the real endpoint/screen and confirming the round-trip through a subsequent read), not only unit/characterization tests, and must state whether live-E2E ran or wasn't required. | 2026-07-12 | [link](records/release/live-e2e-required.md) |
|
||||
| `release.main-direct-push-disabled` | Branch protection on `main` carries `enable_push: false` AND `block_admin_merge_override: true`. Both halves are required and neither is sufficient. `enable_push: false` removes the direct-push path, leaving the PR merge path — the only path on which Gitea evaluates `status_check_contexts`, and therefore the only path on which `review-verdict/h10` is consulted at all. `block_admin_merge_override: true` then closes the force-merge bypass on that remaining path: with it false (the default), `CanBypassBranchProtection` returns true for a repo admin, so `POST /pulls/{n}/merge` with `force_merge: true` merges a PR whose `h10` is missing or red — one API call, no forgery, no PATCH. Do NOT "soften" the push half to a push WHITELIST: measured here, a whitelist naming `timothy` still admits the push, and `timothy` is the identity every agent session, PAT and injected `GITEA_TOKEN` already acts as, so the whitelist form closes nothing while reading in review as a control. Same reasoning is why the admin-override half is needed: an admin-shaped control that exempts the only admin exempts everybody. What remains open: a credential that can PATCH branch protection off can still undo either half — an accepted residual, not a closed route. Tag pushes are unaffected (`tag_protections` governs those separately), so the release cut still works. | 2026-08-05 | [link](records/release/main-direct-push-disabled.md) |
|
||||
| `release.merge-consent-autogrant` | When Done-when boxes are ticked, CI is green, and a fresh positive Review-verdict references head, the merge-consent hook emits `permissionDecision: allow` to actually suppress the redundant mechanical prompt — the derived state IS the consent, no separate conversational confirmation on that path. | 2026-07-12 | [link](records/release/merge-consent-autogrant.md) |
|
||||
| `release.migration-rehearsal-prodcopy` | Before promoting a migration-bearing release, rehearse the new image's migrations against a throwaway copy of the latest prod backup (`scripts/migration-smoke.sh`), gating PASS on the migrator's completion log line rather than HTTP readiness alone. | 2026-07-12 | [link](records/release/migration-rehearsal-prodcopy.md) |
|
||||
| `release.prepush-clean-worktree-guard` | A fail-open pre-push hook blocks a push when any file in the branch's diff vs `origin/main` also has uncommitted working-tree or index changes, since a stale-index commit (e.g. `git reset --soft` + `git add` over an edited-but-unstaged fix) can silently push, CI-test, and get reviewed a different tree than the one on disk. Scope is precise to pushed-diff files; escape hatch `ETV_ALLOW_DIRTY_PUSH=1`. | 2026-07-17 | [link](records/release/prepush-clean-worktree-guard.md) |
|
||||
| `release.promotion-floating-prod` | Prod tracks the floating `:prod` image reference; a tag build's immutable `:<version>` image is scanned first, then promotion happens via a separate manual `DeployStack`, with daily auto-update only as a fallback — tag with enough runway before 03:00 to avoid an unscanned promotion. | 2026-07-13 | [link](records/release/promotion-floating-prod.md) |
|
||||
| `release.review-verdict-gate` | A PR may not merge until a `Review-verdict: <MERGEABLE\|APPROVED\|BLOCKED\|NOT-MERGEABLE> @ <head-sha>` comment references the PR's current head sha (short-sha prefix match against the verdict's OWN `@ <sha>` field, marker at COLUMN 0 (no indent, so indented code blocks cannot self-approve), whole-word verdict token, fenced code blocks stripped with markdown fence-length semantics, negative wins over positive on the same head); folds into the H6 merge-consent hook as condition (c). The grammar lives in ONE tested place, `scripts/check-review-verdict.sh` — #629 found three false-opens that survived because it was implemented inline and untested while this record described stricter behaviour than the code had. | 2026-07-12 | [link](records/release/review-verdict-gate.md) |
|
||||
| `release.verdict-status-check` | The H10 review verdict is written as a `review-verdict/h10` Gitea **commit status** on the exact reviewed sha by `scripts/post-review-verdict.sh`, and that context is a REQUIRED status check on `main`. Because a status belongs to one sha, a later commit cannot inherit it, so Gitea's own `merge_when_checks_succeed` refuses to merge a head no one reviewed. The PreToolUse hook additionally refuses to SCHEDULE an auto-merge unless that status is already green on head. A `pull_request` workflow auto-passes the two exempt classes (Renovate-authored, docs-only) unless the PR touches a protected path (`.claude/`, `.gitea/`, `.husky/`, `scripts/`, `docker/ci/`). This extends — does not supersede — `release.review-verdict-gate` (#303 H10), whose comment convention remains the human-readable artifact and the hook's condition (c). | 2026-07-25 | [link](records/release/verdict-status-check.md) |
|
||||
| `release.verdict-status-check` | The H10 review verdict is written as a `review-verdict/h10` Gitea **commit status** on the exact reviewed sha by `scripts/post-review-verdict.sh`, and that context is a REQUIRED status check on `main`. Because a status belongs to one sha, a later commit cannot inherit it, so Gitea's own `merge_when_checks_succeed` refuses to merge a head no one reviewed. The PreToolUse hook additionally refuses to SCHEDULE an auto-merge unless that status is already green on head. A `pull_request_target` workflow auto-passes the two exempt classes (Renovate-authored, docs-only) unless the PR touches a protected path (`.claude/`, `.codex/`, `.gitea/`, `.husky/`, `scripts/`, `docker/ci/`). This extends — does not supersede — `release.review-verdict-gate` (#303 H10), whose comment convention remains the human-readable artifact and the hook's condition (c). | 2026-07-25 | [link](records/release/verdict-status-check.md) |
|
||||
| `rulebuilder.relative-date-macros` | The visual rule builder's `inLast`/`notInLast` date operators compile to/parse from the pre-existing `CustomMultiFieldQueryParser` macros `released_inthelast`/`released_notinthelast` and `added_inthelast`/`added_notinthelast`, value form `"<n> day\|week\|month\|year"`; there is no backend change. | 2026-07-23 | [link](records/rulebuilder/relative-date-macros.md) |
|
||||
| `scan.collections-scan-status` | `GET /api/v1/media-sources/collections-scan-status` reports a family-global (not per-source), boolean-only active-scan set read from `IEntityLocker`; the SPA reconciles authoritatively against it (with a grace-tick helper) instead of a fixed client-side timeout. | 2026-07-12 | [link](records/scan/collections-scan-status.md) |
|
||||
| `scan.getoraddfolder-db-lookup` | `ILibraryRepository.GetOrAddFolder` resolves the existing folder via a DB query on `(LibraryPathId, Path)`, not the caller's `LibraryPath.LibraryFolders` in-memory navigation, since that navigation is only eager-loaded on the local scan path and is null on remote (Jellyfin) callers. | 2026-07-20 | [link](records/scan/getoraddfolder-db-lookup.md) |
|
||||
@@ -162,7 +177,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `spa.deco-templates-table` | The deco-templates editor also renders its day/deco assignment as a table, extending (not replacing) the templates-editor-table convention. | 2026-07-09 | [link](records/spa/deco-templates-table.md) |
|
||||
| `spa.download-sample-gate` | The SPA disables both Download Media Sample and Download Results while a troubleshooting session is starting/running (Blazor only gated Download Results). | 2026-07-09 | [link](records/spa/download-sample-gate.md) |
|
||||
| `spa.legacy-redirect-matcher` | `LegacyUiRedirects.TryGetRedirect` is a two-tier matcher — an exact `OrdinalIgnoreCase` `Map` (Tier 1) then an ordered segment-template pattern list (Tier 2, first-match-wins) — collision-free by construction, with a guard invariant that no rule may prefix-match `/api`, `/artwork`, `/docs`, `/openapi`, `/iptv`, `/app`, or `/media/sources`. | 2026-07-11 | [link](records/spa/legacy-redirect-matcher.md) |
|
||||
| `spa.list-completeness-vs-bounded-pickers` | The shared `loadAllPages` helper (`web/src/api/paging.ts`) pages a `/api/v1` list to completeness against `totalCount` and is used ONLY for lists that are bounded by construction (rerun collections, multi-collections, playlists — admin-created, hundreds of rows at most). A `getLibraryBrowseItems` picker over a media-library table (Episode/Song/Image/Movie/MusicVideo, tens of thousands of rows possible) must NOT page to completeness — it fetches ONE bounded page (the server cap) and surfaces the truncation (a `ctv-field-help` hint wired to the real `totalCount`) instead of silently dropping the rest. | 2026-07-26 | [link](records/spa/list-completeness-vs-bounded-pickers.md) |
|
||||
| `spa.library-pickers-resolve-by-search` | A picker over a media-library table (Episode/Song/Image/Movie/MusicVideo/TelevisionShow/TelevisionSeason/Artist/OtherVideo/RemoteStream) resolves its options by SEARCH — a debounced `SearchPicker` calling `searchLibraryPickerOptions`, which issues at most ONE `getLibraryBrowseItems` request per settled query, bounded to `LIBRARY_PICKER_RESULTS` (25) rows — CLAMPED inside the helper, not merely defaulted — and gated on `LIBRARY_PICKER_MIN_QUERY` (2) characters. It list-loads NOTHING on mount or on a type switch, so there is no truncation to surface and no truncation hint. The typed text is COMPILED (`titleContainsQuery` → `title:*<escaped>*`), never forwarded raw. The current selection renders from the OWNING RECORD, not from the result set (`selectedName` on a rerun collection / playlist item; a single by-id detail read — `getShow`/`getSeason`/`getArtist` — for a filler preset, which stores only the id), and an edit draft is INITIALIZED ONCE from the detail read — never seeded from the list row, never reconciled against a late response — with the form withheld until it lands, the editor failing CLOSED when the response carries no USABLE concurrency token — absent, empty and whitespace-only ETags are ONE case, normalized in one place, so a PUT without `If-Match` is unreachable, and a deadline plus a route back so a hung request cannot strand it. An id NEVER travels without its namespace: search results are cached against `(source, query)` and list-backed options carry the type they were loaded for, so no id from one type can be offered under another; and every id entering editor state — search result, list-backed option, or a selection restored from a detail read — passes ONE shared `isSelectionId` (int32) predicate at that boundary, an unbindable id being treated as ABSENT rather than coerced. Conflicts are detected at SAVE time via `If-Match` -> 412 -> Reload, and Reload simply drops the draft back to null and re-runs the same initialize-once load, so the form is unmounted while the replacement is in flight; an asynchronously-resolved name is keyed to the id it was resolved for and never overwrites a label naming a different id. The typeahead implements the full ARIA combobox keyboard contract, because it replaces a natively keyboard-operable `<select>`. The other half of the superseded record is UNCHANGED: bounded-by-construction admin lists (collections, multi-collections, smart collections, playlists) still page to completeness via `loadAllPages` and still report `complete`/`hint: incomplete`. Server-side caps are not raised — this is a web-only change. | 2026-07-26 | [link](records/spa/library-pickers-resolve-by-search.md) |
|
||||
| `spa.logs-page-size-local` | The Logs page rows-per-page preference is stored in `window.localStorage` (`ctv-logs-page-size`), not a server `ConfigElement`. | 2026-07-11 | [link](records/spa/logs-page-size-local.md) |
|
||||
| `spa.playback-troubleshoot-poll` | The playback-troubleshooting screen reports FFmpeg completion by polling `GET /api/troubleshoot/playback/status` (~2s) rather than a server push channel. | 2026-07-09 | [link](records/spa/playback-troubleshoot-poll.md) |
|
||||
| `spa.playout-reset-button` | The SPA keeps a single Reset action (server picks the default build mode) and drops Blazor's separate "Schedule reset" button since its capability already exists via the playout's Edit-details flow. | 2026-07-09 | [link](records/spa/playout-reset-button.md) |
|
||||
@@ -177,6 +192,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `startup.parallel-orientation` | A fresh session runs two concurrent tracks at startup — Orientation (`AGENTS.md`/`CLAUDE.md` → `docs/README.md` task-signal map → the active decisions catalog `docs/decisions/README.md`) and, only when no issue is named, Selection (`scripts/select-queue.sh N`, deterministic live-Gitea ranking). A named issue skips Selection entirely. ersatztv#237, the closed pickup tracker this replaces, is reduced to a single archival breadcrumb and MUST NOT be read for live state. | 2026-07-21 | [link](records/startup/parallel-orientation.md) |
|
||||
| `testing.e2e-cleanup-scope-by-pid` | An E2E harness or agent may only kill processes whose PIDs it captured at launch — capture the PID; whoever owns the lifecycle releases it from a `trap ... EXIT INT TERM`. Never `pkill -f "dotnet ErsatzTV.dll"` (or any pattern that can match a process this run did not start). A foreign listener is reported, not reaped. | 2026-07-25 | [link](records/testing/e2e-cleanup-scope-by-pid.md) |
|
||||
| `testing.e2e-local-fresh-config-dir` | Always point `scripts/e2e-local.sh` at a fresh config dir — leftover channels/schedules/DB rows bleed state between runs and corrupt assertions. (The *readiness-probe hang* this record was originally written about was fixed in #533; the fresh-dir rule stands on state-bleed grounds alone.) | 2026-07-21 | [link](records/testing/e2e-local-fresh-config-dir.md) |
|
||||
| `testing.enumerating-guard-identity-not-position` | A guard that cross-checks a hand-reviewed registry against call sites discovered across the whole repo must key each entry on properties INTRINSIC to the site — file, kind, and the value source text — and never on its absolute line or column. A registry keyed on position is a function of every other file in the repo, so a branch that never touches the guard can invalidate it; and because each PR is green against its own base, that failure is structurally invisible pre-merge and lands on `main` after review and after the merge gate. Dropping the position keeps every mutation the guard exists for — a NEW site, a REMOVED site and a CHANGED value each still fail, since each changes the identity multiset — and costs exactly ONE case, which must be stated rather than implied: a SAME-IDENTITY SUBSTITUTION within one file (delete a registered site, add a different unreviewed one with the same kind and value token, net-zero count) now passes. A REPORTED failure still prints the discovered line:column, because identity and diagnostics need not share a format. Comparison stays a MULTISET count rather than set membership, so two sites in one file sharing an identity must be discovered exactly that many times and a third occurrence still fails. A SCANNER test that asserts real AST positions against FIXED inline fixtures is the opposite case and keeps its line/column identity — it has no churn, because its input does not move. | 2026-07-27 | [link](records/testing/enumerating-guard-identity-not-position.md) |
|
||||
| `testing.live-e2e-prepush-timing` | Run live-E2E via `scripts/e2e-local.sh` before pushing a write-path or UI change, and exercise download endpoints with curl, never a browser tab. | 2026-07-21 | [link](records/testing/live-e2e-prepush-timing.md) |
|
||||
| `testing.playwright-mcp-download-and-recovery` | In Playwright-MCP E2E, fetch file-download endpoints with curl — never a browser tab or `window.open` — and if browser tools stall repeatedly, `pkill -f ms-playwright-mcp` and drive a fresh session. | 2026-07-21 | [link](records/testing/playwright-mcp-download-and-recovery.md) |
|
||||
| `testing.scripted-playout-golden-deferred` | The `PlayoutBuildGoldenTests` in-memory golden net covers Sequential (YAML) as of #381. Scripted's *end-to-end pipeline* is excluded — `ScriptedPlayoutBuilder` runs a user-authored external program that drives the engine over HTTP loopback, which the in-memory harness can't pin — so that full-pipeline (integration) harness is deferred to #563. But the scheduling *behavior* those scripts drive lives entirely in the in-process `SchedulingEngine` (the `ScriptedScheduleController` is a 1:1 pass-through to it), which IS directly unit/golden-testable; the earlier "Scripted is un-golden-able by construction" framing overstated the constraint by conflating transport with engine. #395 extracts that shared switch to `ContentEnumeratorBuilder` and adds a direct regression net (`ContentEnumeratorBuilderTests`) over it. | 2026-07-22 | [link](records/testing/scripted-playout-golden-deferred.md) |
|
||||
|
||||
+4
-4
@@ -1,13 +1,13 @@
|
||||
---
|
||||
key: api.search-field-values
|
||||
title: 2026-07-23 — Facet-value typeahead is a new endpoint, allow-listed to text fields, no caching (#434)
|
||||
status: active
|
||||
status: superseded
|
||||
since: '2026-07-23'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for one of a narrow allow-list of catalog fields (not the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a typeahead suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source; case-insensitive prefix-filtered on `q`, `limit` clamped to `[1, 50]` (default 50).'
|
||||
superseded-by: api.search-field-values-sources@2026-07-26
|
||||
rule: '(superseded) `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for one of a narrow allow-list of catalog fields (not the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a typeahead suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source; case-insensitive prefix-filtered on `q`, `limit` clamped to `[1, 50]` (default 50).'
|
||||
signals: 'facet-value typeahead, rule builder value combobox, distinct field values, GetSearchFieldValues, text field allow-list, DB-sourced distinct values, content_rating split · paths: `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Application/Search/Queries/GetSearchFieldValues.cs`, `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `web/src/api/search.ts` · issues: #434, #176'
|
||||
mechanics: '`SearchController.GetSearchFieldValues`; `GetSearchFieldValuesHandler`; api-conventions.md; spa-conventions.md §12'
|
||||
mechanics: superseded by `api.search-field-values-sources` (ersatztv#578), which keeps this endpoint contract and reverses the "no distinct-value source" call for the list-valued music fields
|
||||
---
|
||||
|
||||
Enum fields (e.g. `type`, `content_rating` group) already ship their allowed values inline on
|
||||
+4
-4
@@ -1,13 +1,13 @@
|
||||
---
|
||||
key: spa.list-completeness-vs-bounded-pickers
|
||||
title: '2026-07-26 — `loadAllPages` is for bounded-by-construction lists only; media-library pickers stay bounded and show truncation (#644 follow-up)'
|
||||
status: active
|
||||
status: superseded
|
||||
since: '2026-07-26'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The shared `loadAllPages` helper (`web/src/api/paging.ts`) pages a `/api/v1` list to completeness against `totalCount` and is used ONLY for lists that are bounded by construction (rerun collections, multi-collections, playlists — admin-created, hundreds of rows at most). A `getLibraryBrowseItems` picker over a media-library table (Episode/Song/Image/Movie/MusicVideo, tens of thousands of rows possible) must NOT page to completeness — it fetches ONE bounded page (the server cap) and surfaces the truncation (a `ctv-field-help` hint wired to the real `totalCount`) instead of silently dropping the rest.'
|
||||
superseded-by: spa.library-pickers-resolve-by-search@2026-07-26
|
||||
rule: '(superseded) The shared `loadAllPages` helper (`web/src/api/paging.ts`) pages a `/api/v1` list to completeness against `totalCount` and is used ONLY for lists that are bounded by construction (rerun collections, multi-collections, playlists — admin-created, hundreds of rows at most). A `getLibraryBrowseItems` picker over a media-library table (Episode/Song/Image/Movie/MusicVideo, tens of thousands of rows possible) must NOT page to completeness — it fetches ONE bounded page (the server cap) and surfaces the truncation (a `ctv-field-help` hint wired to the real `totalCount`) instead of silently dropping the rest.'
|
||||
signals: '`loadAllPages`, Class A vs Class B picker, LuceneSearchIndex.Search hitsLimit, picker truncation hint, ctv-field-help, PagedResult, `complete` flag · paths: `web/src/api/paging.ts`, `web/src/screens/RerunCollectionsScreen.tsx`, `web/src/screens/PlaylistsScreen.tsx`, `web/src/screens/FillerPresetsScreen.tsx`, `web/src/screens/MultiCollectionsScreen.tsx`, `docs/spa-conventions.md` §3b · issues: #644'
|
||||
mechanics: '`docs/spa-conventions.md` §3b'
|
||||
mechanics: 'superseded by `spa.library-pickers-resolve-by-search` (ersatztv#651) — Class A (`loadAllPages` for bounded-by-construction lists) survives there unchanged; only the Class B rule is reversed. See `docs/spa-conventions.md` §3b'
|
||||
---
|
||||
|
||||
`fe342a6a` (#644) extracted the `loadAllPages` client-side paging helper and applied it at every
|
||||
@@ -0,0 +1,210 @@
|
||||
---
|
||||
key: api.search-field-values-sources
|
||||
title: '2026-07-26 — Facet-value typeahead, restated: every artist-bearing source is covered, and the JSON-column source is paged by ROW POSITION with no RESIDUAL SQL predicate (#578)'
|
||||
status: active
|
||||
since: '2026-07-26'
|
||||
supersedes: api.search-field-values@2026-07-23
|
||||
superseded-by: none
|
||||
rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort, and its rows are read by a keyset page carrying **NO RESIDUAL predicate** — `SELECT Id, <col> AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL`. The cursor is itself a predicate, but a SEEKABLE one on the ordering key: it positions the scan and never discards a row. A RESIDUAL predicate discards rows the engine already produced, and `LIMIT` truncates only the survivors — so with one present it bounds the OUTPUT rather than the row count. All selectivity is in memory. The guarantee is scoped: **at most 20,000 LOGICAL rows returned/materialized and at most 10 round trips (11 for `artist`)** — NOT bounded physical work and NOT bounded bytes, because MySQL traverses deleted-but-unpurged index records and the `TEXT`/`longtext` payload width is unrestricted. The walk pages 2,000 rows at a time, stopping on the first of enough distinct matches, a short page, or the ceiling.'
|
||||
signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, LIMIT bounds output not work, seekable cursor vs residual predicate, cursor is a predicate too, MySQL purge lag traverses deleted index records, TEXT overflow pages, logical rows not physical work, keyspace is not rows, page by row position, density-independent paging, deleted rows leave Id gaps, ListValuedBatchRows, ListValuedMaxRowsRead, bounded best-effort facet values, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669'
|
||||
mechanics: '`GetSearchFieldValuesHandler` (`GetSource`, `GetSongListValuedColumn`, `GetSongListValuedValues`, `ListValuedSql`, `ParseElements`, `FilterSortTake`); `SearchFieldValuesQueryShapeTests.List_Valued_Page_Query_Has_No_Predicate_Beyond_The_Keyset_Cursor`; api-conventions.md; spa-conventions.md §12'
|
||||
---
|
||||
|
||||
Supersedes `api.search-field-values` (#434). That record did not merely carry a stale implementation
|
||||
detail — it recorded a **call**: free-text music-video/song artist credits were "a known,
|
||||
intentionally-uncovered gap" and `album_artist` was unsupported. #578 reverses that call, so this is
|
||||
a supersession, not a line-edit. Everything #434 settled that still holds is restated here rather
|
||||
than left in the archive: enum fields ship their values inline on `GET /api/v1/search/fields` and
|
||||
need no lookup; text fields need a live one; the source is the database and never the search index;
|
||||
`content_rating` splits its compound `"PG-13/TV-14"` strings in memory; there is no result cache.
|
||||
|
||||
## The three `artist` sources are three different problems
|
||||
|
||||
`LuceneSearchIndex` writes the `artist` field from three places, and only two are ordinary columns:
|
||||
|
||||
- `ArtistMetadata.Title` — a plain column. Already worked.
|
||||
- `MusicVideoArtist.Name` — also a real entity table (`MusicVideoMetadata.HasMany(m => m.Artists)`),
|
||||
so the free-text music-video credits are directly `SELECT DISTINCT`-able. It just joins the
|
||||
existing server-side pipeline as a `Concat`, emitted as one bounded `UNION ALL` +
|
||||
`LOWER(...) LIKE ... LIMIT` on both providers.
|
||||
- `SongMetadata.Artists` (and, for `album_artist`, `AlbumArtists`) — an `IList<string>` EF 9 maps as
|
||||
a **primitive collection**: no `HasConversion` anywhere, one JSON array per row in a single
|
||||
`TEXT`/`longtext` column, with no server-side projection at all. Verified against both providers:
|
||||
SQLite reports *"Translating this query requires the SQL APPLY operation, which is not supported on
|
||||
SQLite"*, Pomelo MySQL 9.0.0 reports *"Primitive collections support has not been enabled"*. Both
|
||||
failures are pinned by a test, so a provider upgrade that fixes them surfaces as a red rather than
|
||||
leaving a workaround in place forever.
|
||||
|
||||
## The SQL predicate is gone, and that is the point
|
||||
|
||||
Three revisions tried to narrow the rows in SQL before filtering them in memory. All three were
|
||||
wrong, in three different ways, and the fourth was wrong too — the history is worth more than the
|
||||
code, so it is written out below under "four wrong quantities". The conclusion is short: **there is
|
||||
no `WHERE` clause beyond the keyset cursor.** No `LIKE`, no `LOWER`, not even `IS NOT NULL`.
|
||||
|
||||
That deletes an entire family of bugs along with the predicate. Gone with it: the JSON-escape
|
||||
reasoning (`Édith` is stored `\u00C9dith`, and SQL `LOWER()` folds the escape *text* rather than the
|
||||
codepoint it denotes, so a `q=é` pattern of `\u00e9` never matched `\u00C9`); the "narrow only on the
|
||||
leading verbatim-ASCII run" rule and the exhaustive Unicode sweep that proved it sound; the
|
||||
`ESCAPE '/'` portability workaround; and the whole may-over-match-never-under-match invariant, which
|
||||
turned out to be conditional on something that was not true. In memory a string is just a string:
|
||||
`element.StartsWith(query, StringComparison.OrdinalIgnoreCase)`.
|
||||
|
||||
Worth keeping one number from that history, because it is the reason the first bug survived review:
|
||||
the JSON-encoded pattern failed on **three of nine** pinned cases, not all nine — those where the
|
||||
query's casing differed from the stored casing, so the two escape texts diverged. When the casings
|
||||
agreed it worked. A bug that fires on some inputs and not others reads as "works" during a spot
|
||||
check.
|
||||
|
||||
## Ordinal everywhere, because the culture is caller-controlled
|
||||
|
||||
`UseRequestLocalization` honours `Accept-Language`, so a caller can select `tr-TR` and turn `q=I`
|
||||
into `ı`. The old chain used `ToLower()` plus the default *linguistic* `StartsWith(string)`, making
|
||||
the same library answer differently per caller. Comparison is now `OrdinalIgnoreCase` and ordering
|
||||
`StringComparer.Ordinal` throughout the in-memory stages, including the shared `FilterSortTake` that
|
||||
`state`, `video_dynamic_range` and `content_rating` also use. That is a deliberate change to shared
|
||||
behaviour, and **not a cosmetic one**: ordering happens before `Take(limit)`, so changing the
|
||||
comparer can change *which* values survive, not merely their order. With `"Zulu"` and `"apple"`, an
|
||||
empty `q` and `limit=1`, linguistic ordering yields `"apple"` and ordinal yields `"Zulu"`. An earlier
|
||||
version of this record claimed the response sets were unchanged; that was false.
|
||||
|
||||
**Scope this claim carefully — it is not endpoint-wide.** A field sourced by a plain EF query runs
|
||||
the database's `LOWER`, `DISTINCT`, `ORDER BY` and `LIMIT` *before* any ordinal code executes, so the
|
||||
database has already decided which values survive. Store a genre `"Éclair"` on SQLite and ask for
|
||||
`genre?q=é`: SQLite's ASCII-only `LOWER()` drops it before the ordinal in-memory filter ever runs, and
|
||||
a case-insensitive collation's `DISTINCT` can likewise collapse values ordinal dedup would have kept.
|
||||
The endpoint description and this record's `rule:` therefore say "the final filter, dedup and
|
||||
ordering", not "matching is ordinal". That gap is now CLOSED by `api.search-field-values-unicode-fold`
|
||||
(**ersatztv#668**) — not by the client-side filtering guessed at here, but by a registered Unicode-correct
|
||||
SQL fold on a second, additive query taken only for non-ASCII queries on SQLite. The scoped wording above
|
||||
still stands as written: it describes what the EF stage itself does, which is unchanged.
|
||||
|
||||
## Ordering is best-effort, and the code says so
|
||||
|
||||
Merging sources does **not** yield the exact first `limit` of the union. Each source truncates using
|
||||
its own ordering — the EF source by the database collation, the list source by primary key — and
|
||||
neither is the ordinal ordering the merge applies. The pair that actually demonstrates it is `"Zulu"`
|
||||
and `"apple"`: ordinal puts every ASCII uppercase letter before every lowercase one, so the merge
|
||||
ranks `"Zulu"` first, while the case-insensitive database ordering ranks `"apple"` first — at
|
||||
`limit=1` the response is `["apple"]`, not the ordinally-first `"Zulu"`. Below the truncation points
|
||||
— the normal typeahead case — the result is exact. An earlier comment claimed exactness the code does
|
||||
not have; do not restore it. (An earlier version of this record used `"Zulu"`/`"Éclair"` as the
|
||||
example, where both orderings pick `"Zulu"` — it demonstrated nothing.)
|
||||
|
||||
## What the bound bounds — four attempts, four wrong quantities
|
||||
|
||||
Read this before "optimizing" the query. Every one of these looked obviously correct when written,
|
||||
and each was caught only by someone constructing the adversarial case rather than reading the code.
|
||||
|
||||
| # | Bounded | Why it wasn't a bound |
|
||||
|---|---|---|
|
||||
| 1–2 | the **result** — fixed `LIMIT 1000` on pre-filtered rows | the pre-filter was deliberately allowed to over-match, so a widened pattern (any non-ASCII or JSON-escaped prefix collapses it to `%"%`) filled the budget with rows that could not match. 1,000 `"zzz"` songs, `"éclair"` at row 1,001, `q=é` → `[]` |
|
||||
| 3 | **candidates returned** — keyset paging + `LIMIT` | a query matching nothing must evaluate every eligible row before it can return an empty page, so the first empty page ended the walk having counted **zero** against the ceiling. Rows returned bounded, rows inspected unbounded |
|
||||
| 4 | **keyspace width** — closed `Id` range per page | keyspace is not rows. Delete 20,000 historical rows, put one song at `Id` 20001, `q=que` → `[]`. **One row in the table, zero rows inspected.** Capacity fell linearly with deletion ratio and no ratio was safe: one placed gap hides the next match |
|
||||
| 5 | **logical rows returned** — keyset page by row position, cursor only, **no residual predicate** | — (physical work still unbounded; see below) |
|
||||
|
||||
The through-line: **`LIMIT` truncates what survives a RESIDUAL predicate.** The distinction is not
|
||||
"predicate vs no predicate" — attempt 5's query still has `Id > @AfterId`. It is:
|
||||
|
||||
- a **seekable predicate on the ordering key** (the cursor) positions the scan and never discards a
|
||||
row, so `LIMIT n` yields `n` rows;
|
||||
- a **residual predicate** (`LIKE`, `LOWER`, `IS NOT NULL`) throws away rows the engine already
|
||||
produced, so `LIMIT` bounds the survivors and says nothing about how many were produced.
|
||||
|
||||
Attempts 3 and 4 both kept selectivity in SQL and tried to add accounting around it. Attempt 5 drops
|
||||
the residual predicate and keeps only the cursor, so the accounting becomes trivial:
|
||||
|
||||
```sql
|
||||
SELECT Id, Artists AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch
|
||||
```
|
||||
|
||||
`ListValuedBatchRows = 2000`, `ListValuedMaxRowsRead = 20000`. The walk stops on the first of: enough
|
||||
distinct matches for `limit`, a short page (with no residual predicate that can only mean exhaustion
|
||||
— it can never mean "this stretch matched nothing", which is exactly why the residual predicate had
|
||||
to go), or the ceiling. Round trips: **at most 10** for `album_artist`, **at most 11** for `artist`,
|
||||
which also runs one EF query for its entity/music-video half.
|
||||
`SearchFieldValuesQueryShapeTests` pins the SQL string exactly and asserts the absence of `LIKE`,
|
||||
`LOWER` and `IS NOT NULL`, so a reviewer reintroducing "just a cheap filter" fails a test instead of
|
||||
silently unbounding the walk.
|
||||
|
||||
### Exactly what is bounded — and what is NOT
|
||||
|
||||
State this precisely, because an earlier version of this record claimed more and the overclaim is
|
||||
more dangerous than the code ever was. What holds:
|
||||
|
||||
- **at most `ListValuedMaxRowsRead` logical rows returned and materialized per request**, and
|
||||
- **at most 10 (or 11) round trips.**
|
||||
|
||||
That is the whole guarantee. It is what makes the walk terminate and what caps the number of rows and
|
||||
round trips. **Explicitly retracted**, having been asserted here in earlier revisions:
|
||||
|
||||
- ~~"`LIMIT n` reads exactly `n` index entries"~~ — **false on MySQL.** Deleted clustered-index
|
||||
records survive until purge runs, and a range scan still traverses them. Hold an old InnoDB
|
||||
snapshot open, delete a million early `SongMetadata` rows, and query from a newer snapshot with
|
||||
purge blocked: returning 2,000 *visible* rows can touch far more index records. **Deletion history
|
||||
therefore still affects physical work** — the very thing attempt 4's failure was supposed to have
|
||||
made irrelevant. Attempt 5 fixes the *logical* dependence on `Id` distribution; it does not make
|
||||
physical work independent of deletion history.
|
||||
- ~~bounded physical work / bounded I/O~~ — row width is unbounded. `Artists`/`AlbumArtists` are
|
||||
unrestricted `TEXT`/`longtext`, and both SQLite and InnoDB spill large payloads to overflow pages,
|
||||
so a row count implies neither a byte count nor a page-read count.
|
||||
- ~~"caps what this process holds in memory"~~ — the same overclaim one level down, and it survived
|
||||
the first retraction. A row count bounds neither bytes buffered nor set size: payload width is
|
||||
unrestricted, and one JSON array can contain arbitrarily many strings, every one of which may enter
|
||||
the in-memory distinct set.
|
||||
|
||||
Nor can the query-shape test carry more than it does: it pins the SQL **string**. It cannot pin an
|
||||
execution plan, MVCC visibility work, or payload I/O — and on MySQL, using the index to satisfy
|
||||
`ORDER BY` is an optimizer choice, not a SQL semantic.
|
||||
|
||||
### The cost, measured
|
||||
|
||||
No server-side narrowing means rows are transferred that will be discarded. **This is ONE data point
|
||||
on ONE library, not a general figure** — see the row-width caveat above: these numbers hold for a
|
||||
library whose artist credits average ~20 bytes of JSON, and a library with long credit lists would
|
||||
transfer proportionally more for the same row count. Measured on a seeded 20,000-song library
|
||||
(in-memory SQLite, so the wall times are a floor, not a production figure):
|
||||
|
||||
| case | rows read | round trips | payload | wall |
|
||||
|---|---|---|---|---|
|
||||
| worst case — no match, full walk | 20,000 | 10 | **391.9 KiB** (avg 20.1 B/row) | 119 ms SQL / ~40 ms warm end-to-end |
|
||||
| empty `q` (fills `limit` on page 1) | 2,000 | 1 | ~39 KiB | ~60 ms |
|
||||
| dense prefix (`rad`) | 2,000 | 1 | ~39 KiB | ~38 ms |
|
||||
| non-ASCII prefix (`beyoncé`) | 2,000 | 1 | ~39 KiB | ~39 ms |
|
||||
|
||||
Judged acceptable **for this shape of library**: the worst case is a debounced typeahead keystroke
|
||||
that matches nothing, at ~392 KiB and tens of milliseconds against a local SQLite file. Dense queries
|
||||
— including the empty `q` the combobox opens with — stop on the first page. Re-measure rather than
|
||||
extrapolate if credit lists are long or the provider is MySQL over a network. **If it ever becomes
|
||||
unacceptable, do not reintroduce selectivity;** that is the trap this record exists to document. Go
|
||||
to #669.
|
||||
|
||||
### Accepted losses
|
||||
|
||||
A match past row 20,000 is not found — 20,000 filler rows then `"éclair"` at 20,001 returns `[]`, and
|
||||
a test pins exactly that rather than pretending otherwise. That is the documented bounded-best-effort
|
||||
contract, and unlike attempts 1–4 it now depends only on row count, not on prefix shape, deletion
|
||||
history or `Id` distribution.
|
||||
|
||||
**Follow-up: a normalized `SongArtist` join table** (the shape `MusicVideoArtist` already has) makes
|
||||
the predicate seekable, so there is nothing left to bound and nothing to transfer. Cost: a
|
||||
dual-provider schema migration plus data backfill, changes to every scanner write path populating
|
||||
`SongMetadata.Artists`, changes to the Lucene indexer, and two representations of the same fact free
|
||||
to drift. Tracked as **ersatztv#669**; rejected for #578 on blast radius, not on merit.
|
||||
|
||||
## `album_artist` 404 → 200 is additive
|
||||
|
||||
Nothing consumes the 404 as a signal: the SPA's `getSearchFieldValues` (`web/src/api/search.ts`)
|
||||
treats any non-200 as "no suggestions, fall back to a free-text input", which it will now do less
|
||||
often. Per `api.versioning-v1`, widening which fields return values adds capability without removing
|
||||
any, so no `/api/v2`.
|
||||
|
||||
## Known limitation inherited, not introduced
|
||||
|
||||
**RESOLVED — see `api.search-field-values-unicode-fold` (ersatztv#668, 2026-07-27).** As written for
|
||||
#578 this said: the **EF-sourced** fields (`genre`, `studio`, `artist`'s entity half, …) still
|
||||
prefix-match through SQL `LOWER()`, which on SQLite is ASCII-only, so a stored `Édith` was unreachable
|
||||
for those fields. That predated #578 and was unchanged by it. It is now fixed — and NOT by the
|
||||
client-side filtering this section anticipated, which would have reintroduced the very scan #578 bounded.
|
||||
The surrounding scoped-ordinal wording is still load-bearing and must not be "tidied" into a broader
|
||||
claim: the EF stage's own behaviour is unchanged, and the defect was SQLite-only and one-sided.
|
||||
@@ -0,0 +1,72 @@
|
||||
---
|
||||
key: api.search-field-values-unicode-fold
|
||||
title: '2026-07-27 — Facet-value typeahead reaches accented values: a registered Unicode fold on the SQLite non-ASCII branch, not a bounded walk (#668)'
|
||||
status: active
|
||||
since: '2026-07-27'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The EF-sourced facet fields (`genre`, `show_genre`, `studio`, `director`, `writer`, `actor`, `tag`, `network`, `collection`, `video_codec`, `album`, and `artist`''s entity half) reach stored values whose prefix carries an uppercase non-ASCII character, on BOTH providers, with no row budget and no accepted loss. The defect was SQLite-only and ONE-SIDED: SQLite''s `LOWER()` folds ASCII only (`lower(''Édith'')` is `''Édith''` unchanged), so the predicate UNDER-matched, which no later stage can repair. MySQL was already correct — its `LOWER()` is Unicode-aware, so `LOWER(''Édith'')` really is `''édith''` and the existing predicate reaches the row unaided. The fix is a SECOND, ADDITIVE query taken only when `isSqlite && q contains a non-ASCII character`: raw Dapper SQL `SELECT DISTINCT <col> AS Value FROM <table> WHERE [<discriminator> AND] etv_upper(<col>) LIKE @Pattern ESCAPE ''\'' ORDER BY <col> LIMIT @Limit`, where `etv_upper` is a `SqliteConnection.CreateFunction` scalar implementing `ToUpperInvariant`. Every other case — all-ASCII `q`, and MySQL for all `q` — runs today''s EF query BYTE-IDENTICALLY. Keeping selectivity in SQL here is NOT the refuted family from `api.search-field-values-sources`: those four attempts bounded a walk around a predicate that could not be made correct over JSON escape text, whereas this is a correct fold on a plain column in an ordinary `LIMIT`ed query. It narrows that record''s "Known limitation inherited, not introduced" clause; everything else it settles still holds.'
|
||||
signals: 'accented facet values missing, Édith not suggested, SQLite LOWER is ASCII only, etv_upper, CreateFunction custom scalar, ToUpperInvariant fold, OrdinalIgnoreCase is not invariant-upper, U+017F long s upper-folds to S, U+212A Kelvin sign, utf8mb4_0900_ai_ci accent insensitive, MySQL LOWER is unicode aware, over-match harmless under-match not, ESCAPE clause raw SQL LIKE wildcards, EF null semantics ExternalTypeId, RegisterUnicodeCaseFunctions provider static, non-sargable LOWER LIKE full table scan · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV.Infrastructure.Sqlite/Data/SqliteUnicodeFunctions.cs`, `ErsatzTV.Infrastructure/Data/TvContext.cs`, `ErsatzTV/Startup.cs`, `ErsatzTV.Scanner/Program.cs` · issues: #668, #578, #434, #669'
|
||||
mechanics: '`GetSearchFieldValuesHandler` (`ContainsNonAscii`, `IsSqlite`, `EscapeLikePrefix`, `UnicodeFoldSql`, `GetUnicodeFoldSources`, `GetUnicodeFoldedValues`, `UpperFunction`); `SqliteUnicodeFunctions.Register`; `TvContext.RegisterUnicodeCaseFunctions`; `GetSearchFieldValuesHandlerTests.Unicode_Fold_Agrees_With_The_Ordinal_Filter`; `SearchFieldValuesQueryShapeTests.Unicode_Fold_Function_Name_Matches_The_Registration`; `ProviderStaticsWiringTests`'
|
||||
---
|
||||
|
||||
Narrows `api.search-field-values-sources` (#578), which deferred this gap; the rest of #578 stands.
|
||||
|
||||
## The defect was one-sided, and the issue described it wrongly
|
||||
|
||||
The handler lowercases `q` with `ToLowerInvariant` **before** SQL, so both casings produce one pattern.
|
||||
A stored **lowercase** accented value was therefore always reachable from either casing; only one whose
|
||||
prefix carries an **uppercase** non-ASCII character was lost. ersatztv#668's body claimed `q=É` failed
|
||||
against a stored `édith`; false, and a test pins the passing case beside the fixed one.
|
||||
|
||||
## MySQL was never broken, for a reason worth recording
|
||||
|
||||
Verified on a live MySQL 8.4: `LOWER('Édith')` is `édith`, so the existing predicate reaches the row.
|
||||
**Measure the query the CODE runs, not one you type.** With a LITERAL pattern `LOWER(name) LIKE 'é%'`
|
||||
also matches `Edith` (the column is accent-insensitive `utf8mb4_0900_ai_ci`), and an earlier revision of
|
||||
this record concluded from exactly that probe that MySQL over-matches and the ordinal filter corrects it.
|
||||
It does not: through EF the driver binds the pattern with a BINARY collation, so the executed comparison
|
||||
is accent-SENSITIVE and returns `Édith` alone — a driver-contingent fact, not a law. MySQL's correctness
|
||||
rests on Unicode-aware `LOWER()`, not on the collation.
|
||||
|
||||
## Why a fold, and not the #578 walk
|
||||
|
||||
Reusing #578's shape — drop SQL selectivity, keyset-walk, filter in memory — answers the wrong question.
|
||||
That walk is best-effort at 20,000 rows; `Genre` and `Actor` carry one row per media item, so a large
|
||||
library exceeds the budget and `Édith` stays unreachable — the bug restated. #578 accepts that contract
|
||||
for `SongMetadata.Artists` because server-side projection is **impossible** there; these are plain
|
||||
columns, where it is merely inconvenient.
|
||||
|
||||
The cost objection to a managed per-row fold is weak: `LOWER(v) LIKE` is non-sargable and **no index on
|
||||
any of these `Name` columns exists** (every index is on the foreign key), so this swaps a native per-row
|
||||
call for a managed one on a scan that already happens — and only on the non-ASCII branch.
|
||||
|
||||
## The correctness property is containment, not equality
|
||||
|
||||
The SQL stage may over-match freely; it must never under-match. `ToUpperInvariant` satisfies that
|
||||
because **`OrdinalIgnoreCase` equality is a strict subset of invariant-uppercase equality**.
|
||||
|
||||
Do not restate this as "`OrdinalIgnoreCase` IS invariant-uppercase-then-ordinal". It is not, and the gap
|
||||
is measurable: `char.ToUpperInvariant('ſ')` (U+017F) is `'S'`, yet
|
||||
`"ſweet".StartsWith("S", OrdinalIgnoreCase)` is **false**. The fold returns that row and the filter drops
|
||||
it — the harmless direction. An earlier draft justified the fold by claiming the opposite;
|
||||
`Fold_LongS_IsNotOrdinalEqualToS` pins the truth.
|
||||
|
||||
That same fact makes the all-ASCII fast path sound: no non-ASCII codepoint is `OrdinalIgnoreCase`-equal to printable ASCII (#578's sweep found 0), so an ASCII query only ever ordinal-matches an ASCII prefix.
|
||||
|
||||
## Three traps, each guarded by a test and explained at its call site
|
||||
|
||||
Raw SQL gets none of EF's LIKE escaping (`EscapeLikePrefix`, backslash first, explicit `ESCAPE`).
|
||||
Discriminators must mirror EF's NULL semantics — `t.ExternalTypeId != X` INCLUDES a NULL-typed row,
|
||||
where plain SQL `<>` drops it. Registration is per-connection and lives at the call site, not in a
|
||||
`DbConnectionInterceptor`: Dapper opens a closed connection itself and a direct ADO open raises no EF
|
||||
interceptor, so that seam would miss exactly this query.
|
||||
|
||||
## Residuals, stated rather than glossed
|
||||
|
||||
**Crowding**: a SQL `LIMIT` can fill with rows the ordinal filter then discards, under-DELIVERING the
|
||||
count (never a wrong value). Not reachable on MySQL under the CURRENT driver behaviour above (a ci-collated
|
||||
pattern would restore it); the SQLite fold has it when limit-many values are upper-equal but ordinal-unequal (
|
||||
`ſ`/`K`/`İ` class), so "no accepted loss" means no unreachable VALUE, not a guaranteed count. An
|
||||
over-fetch was rejected (it perturbs the pinned `"apple"`/`"Zulu"` examples). **Ordering stays
|
||||
best-effort** per #578.
|
||||
@@ -0,0 +1,71 @@
|
||||
---
|
||||
key: api.selection-projection-include-chain
|
||||
title: '2026-07-28 — A tagged-union selection is projected through one shared include chain, and its flattening switch never falls through to null (#671)'
|
||||
status: active
|
||||
since: '2026-07-28'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Every handler that projects an aggregate carrying a tagged-union selection loads it through ONE shared `<Aggregate>QueryExtensions` include chain — `RerunCollectionQueryExtensions.IncludeSelectionDetails()`, joining the existing `ProgramScheduleItemQueryExtensions.IncludeScheduleItemDetails()` — called by the paged-list handler and the by-id handler alike, so the two cannot drift. The media-item flattening switch is likewise ONE shared helper, `MediaCollections.Mapper.ProjectMediaItemToViewModel`, covering all ten selectable media types including `RemoteStream`, whose named projection is `MediaItems.Mapper.ProjectToNamedViewModel` (it cannot be an overload of `ProjectToViewModel(RemoteStream)`, which already exists returning the unrelated `RemoteStreamViewModel`; C# will not overload on return type). That switch NEVER ends in `_ => null`: a null MediaItem is the legitimate not-a-media-item case, while an unrecognized non-null subtype keeps its id and takes a conspicuous `[unsupported media type: X]` name. Fail-soft is deliberate — throwing would fail an entire paged GET over one unreadable row. Finally, every metadata navigation inside `MediaItems.Mapper` is read through `Optional(...).Flatten()` and degrades to the `"???"` placeholder, because those projections are reached from handlers whose include chains differ and a bare `x.Season.Show.ShowMetadata` is a latent 500 on some other caller GET.'
|
||||
signals: 'rerun collection null selection, selectedId null for every row, list badge renders Collection with no name, detail GET 500 on Episode, detail GET 500 on MusicVideo, RemoteStream dropped by the mapper, underscore arrow null fallthrough, AsNoTracking suppresses navigation fixup, eager load missing on paged list, Include after Skip Take, EpisodeTitle NullReferenceException, MusicVideoTitle bare Artist deref, ShowTitle bare Show deref, id only as available as the name, editor silently clears stored selection, ArgumentNullException value cannot be null parameter values, string.Join on null sequence, SongMetadata Artists is null, nullable primitive collection not a navigation, untagged song fallback metadata, playout guide 500 on a song, song artist prefix bare dash, chaptered song renders ErsatzTV.Core.Domain.Song, GetDisplayTitle interpolates the entity not the title · paths: `ErsatzTV.Application/MediaCollections/RerunCollectionQueryExtensions.cs`, `ErsatzTV.Application/MediaCollections/Mapper.cs`, `ErsatzTV.Application/MediaItems/Mapper.cs`, `ErsatzTV.Application/MediaCollections/Queries/GetPagedRerunCollectionsHandler.cs`, `ErsatzTV.Application/MediaCollections/Queries/GetRerunCollectionByIdHandler.cs`, `docs/api-conventions.md` §2a · issues: #671, #651, #229'
|
||||
mechanics: '`RerunCollectionQueryExtensions.IncludeSelectionDetails`; `Mapper.ProjectMediaItemToViewModel`; `MediaItems.Mapper.ProjectToNamedViewModel`; `SelectionSeedData` (`SupportedSelectionTypes`, `ExpectedName`, `SeedSelection`, `ApplySelection`); `RerunCollectionQueryHandlerTests` (`GetById_Should_Resolve_The_Selection`, `GetPaged_Should_Resolve_The_Selection`, `Supported_Selection_Types_Should_Be_The_Full_Documented_Set`, `GetById_Should_Tolerate_Song_Artists`); `GetPlaylistItemsHandlerTests`; `Playouts.Mapper.GetDisplayTitle` + `PlayoutMapperDisplayTitleTests`; `RerunCollectionRequestMapping.IsSupportedSelectionType`'
|
||||
---
|
||||
|
||||
Applies the `#229` shared-include-chain remedy to the READ path — that record framed it as a
|
||||
write-path concern; this is its mirror image, where the GET itself under-loaded.
|
||||
|
||||
## The coupling that hid the bug
|
||||
|
||||
The id and the display name are read off the SAME navigation, so the id is only ever as available as
|
||||
the name — the API never knows WHICH item is selected but not what it is called. Hence the symptom
|
||||
looked like a naming problem (an unlabelled badge) when the real harm is one level down: the selected
|
||||
id is null too, and an editor that round-trips it clears the stored selection. #651's client-side
|
||||
merge-instead-of-replace guard made this survivable and stays, but patched a server defect from the
|
||||
client. The rule: never let the id and the name share a single point of failure — hence the
|
||||
`_ => null` ban, where an unrecognized subtype surrenders its NAME, never its ID. Fail-soft, not a
|
||||
throw, which would fail a whole paged GET over one bad row. (`ProgramSchedules.Mapper`'s switch does
|
||||
throw, correctly — it dispatches on the ITEM type, an internal closed set.)
|
||||
|
||||
## Scope deliberately not widened
|
||||
|
||||
Nine further media-item switches (`ProgramSchedules.Mapper` ×4, `Scheduling.Mapper` ×5) handle only
|
||||
Show/Season/Artist — not the same oversight, since those call sites genuinely restrict selection to
|
||||
those three and load a matching chain. Only RerunCollection and PlaylistItem span the full set, so
|
||||
exactly those two were merged. A THIRD consumer, `ReplacePlaylistItemsHandler`, projects items whose
|
||||
navigations are never loaded — inert only because the controller discards the result and re-queries.
|
||||
|
||||
**Widening a shared switch incurs a debt in every caller loading for it**, discharged by a TEST, not
|
||||
by inspection — inspection is the method that produced this bug. `GetPlaylistItemsHandler` had no
|
||||
handler-level test at all (its controller tests stub the mediator), so it gained the same 13-type
|
||||
matrix via the shared `SelectionSeedData`.
|
||||
|
||||
## `Artists` is a nullable PRIMITIVE COLLECTION, and the sweep must follow the field
|
||||
|
||||
`SongMetadata.Artists` is a nullable EF primitive collection — a JSON array in one column, **not a
|
||||
navigation** — left unassigned by `FallbackMetadataProvider` when a song's tags fail to read, and
|
||||
`string.Join` throws `ArgumentNullException`, not `NullReferenceException`. So a "null navigation"
|
||||
audit misses it and so does a grep for `NullReferenceException`. The guard is
|
||||
`Optional(sm.Artists).Flatten()`, empty filtered too so an artist-less song loses its bare `" - "`.
|
||||
|
||||
Two corrections, because a wrong explanation outlives a wrong line. It was **not** introduced here:
|
||||
`GetPlaylistItemsHandler` already included `SongMetadata` on `origin/main` and already routed `Song`,
|
||||
so `GET /api/v1/playlists/{id}/items` was ALREADY a live 500 — this branch only made the same throw
|
||||
reachable on a second path. And fixing the rerun site alone left the mirror standing: `Playouts/Mapper`
|
||||
had the identical unguarded join on a path that also eager-loads `SongMetadata`, likewise live, swept
|
||||
here. `LibraryBrowseItemMapper` already wrote `Artists ?? []`, so the codebase knew. Filed separately:
|
||||
`SongVideoGenerator` dereferences `Artists.Count`/`.Contains` on the playback path. Sweep by FIELD.
|
||||
|
||||
Adjacent, same review, fixed here: that Song arm interpolated the `case Song s` ENTITY into its
|
||||
chapter branch, rendering a chaptered song as the literal `ErsatzTV.Core.Domain.Song (Chapter 3)`.
|
||||
|
||||
## Verification worth repeating
|
||||
|
||||
Every mechanism was removed in turn and quoted red before restoring it: stripping the list include
|
||||
chain failed all 13 types on "lost its selected id"; the original four-type by-id chain failed exactly
|
||||
the six the issue named; reverting the bare dereferences reproduced `NullReferenceException` for
|
||||
Episode and MusicVideo; reverting either `Artists` guard reproduced `ArgumentNullException`; and
|
||||
reverting the chapter fix rendered the type name. A green test proves little until shown to fail.
|
||||
|
||||
The per-type assertion pins the WHOLE expected string, not merely "is not a placeholder", because the
|
||||
looser form cannot see a missing NESTED leg: drop Episode → Season → Show and the projection still
|
||||
reads `s00e04 - Selected episode`, placeholder-free, and passes. Relatedly an absent Season renders
|
||||
`s??`, never `s00`, which means Specials and would fabricate plausible-looking real data.
|
||||
@@ -0,0 +1,89 @@
|
||||
---
|
||||
key: ci.actions-credential-scoping
|
||||
title: '2026-08-05 — CI''s registry credential is a scoped PAT, not the admin password, because Gitea cannot separate status-write from repo-write (#697)'
|
||||
status: active
|
||||
since: '2026-08-05'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Any credential reachable from an Actions job is scoped to what that job needs. The container-registry secret `REGISTRY_PASSWORD` is a personal access token scoped `write:package` + `read:repository` — never an account PASSWORD. This matters because Gitea has NO `status` token scope: `POST /repos/{o}/{r}/statuses/{sha}` is gated by `reqRepoWriter(unit.TypeCode)`, so ANY credential that can write the repository can forge `review-verdict/h10`, the required context that is supposed to make merge-consent derived rather than assertable. Package-write IS a separate scope, so the registry credential can be made status-incapable at no cost: `scripts/ci-detect-already-validated.sh` only GETs. Do NOT add a `permissions:` key to constrain the injected `GITEA_TOKEN` on the assumption that it binds — below Gitea 1.26.0 it is silently a NO-OP, which is worse than absent because it reads in review as a constraint. That version precondition NO LONGER HOLDS: this instance was upgraded 1.25.4 -> 1.27.1 on 2026-08-05. What has NOT changed is that the consequence is unverified — whether `permissions:` is honored here, and what this instance''s default Actions token permission is, were both left UNPROBED (there is still no API surface: `/api/v1/settings/actions` 404s at 1.27.1). Probe before relying on it; do not read the upgrade alone as the constraint now working. Scoping is necessary and not sufficient: it bounds what a job may DO, never whether attacker YAML runs at all, so a self-referencing trigger needs its own filter (`ci-image.yml`, tracked in #744 — deliberately NOT bundled here, because editing that file re-points `ci-image-pin` at the editing commit and reddens a blocking job). This record closes ONE route. It does not close the class, and four later sections say exactly what survives — read them before citing this record as a mitigation.'
|
||||
signals: 'admin password in CI secrets, registry credential scope, ETV_STATUS_AUTH can write statuses, forge review-verdict/h10, head-resolved workflow holds credentials, Gitea token scopes, no status scope, write:package vs write:repository, permissions key no-op, GITEA_TOKEN default read/write, Restricted default token permissions, orphan secret, deploy key in secret store, toolchain image overwrite, prod floating tag write · paths: `.gitea/workflows/docker-build.yml`, `.gitea/workflows/ci-image.yml`, `.gitea/workflows/renovate.yml`, `scripts/ci-detect-already-validated.sh` · issues: #697, #672, #698, #742, #743, #420, server-management#714'
|
||||
mechanics: 'PAT `ci-registry-scoped-697`, scopes `write:package,read:repository`, stored as repo Actions secret `REGISTRY_PASSWORD`; `REGISTRY_USER` remains `timothy`. Verified 2026-08-05 on Gitea 1.25.4: registry push of a probe tag SUCCEEDED; `GET /commits/{sha}/status` 200; `POST /statuses/{sha}` REFUSED HTTP 403 `token does not have at least one of required scope(s), required=[write:repository], token scope=write:package,read:repository`. Probe artifacts deleted, confirmed 404. NOT measured with this token: the `container:` pull, the buildcache write and the base-image pull. Those rest on Gitea''s scope model (write implies read per category, read at tag `v1.25.4`) — INFERRED. Note WHICH run proves which: only the `container:` pull is exercised by a PR. `cache-to`/`cache-from` and the base-image pull are confined to the `build` job, which carries `if: github.event_name != ''pull_request''`, so they are first exercised on the post-merge push to `main` — AFTER the merge gate has passed. A wrong inference there reddens main, not the PR.'
|
||||
---
|
||||
|
||||
**What was wrong.** `REGISTRY_USER`/`REGISTRY_PASSWORD` were the **admin account's** basic auth, and
|
||||
`docker-build.yml` triggers on `pull_request` — head-resolved — so a PR's own code got instance-admin
|
||||
credentials. Basic auth carries no scope: the secret pushing an image administers every repo on the
|
||||
instance.
|
||||
|
||||
**Why the credential and not only the triggers.** Patching triggers enumerates *instances* of "a
|
||||
ref-resolved workflow obtains status-capable credentials", and adding a new workflow file is itself a
|
||||
route, so that enumeration never completes. But it is not either/or: `ci-image.yml`'s unfiltered
|
||||
`push:` is path-scoped to itself, so any branch push runs attacker YAML on a docker-capable runner
|
||||
with no PR. Scoping bounds what a job may DO; only a filter bounds whether it RUNS. That filter is
|
||||
**#744**, not this record: editing `ci-image.yml` re-points `ci-image-pin`'s `expected` at the editing
|
||||
commit and staleness-fails a **blocking** job. That is a toll, not a wall — the documented two-step
|
||||
(publish `:<short sha>`, then bump all five pins) clears it — but a rebase rewrites the sha and charges
|
||||
it again, so it lands alone (`land-toolchain-image-change-separately`).
|
||||
|
||||
**What the scoped token still reaches — not "just a registry credential".** `write:package` over owner
|
||||
`timothy` writes `ersatztv:prod` (the floating tag prod's `jazz-media` stack follows) and
|
||||
`ersatztv-ci:<sha>` (the toolchain image *executing* five `container:` jobs). A sha-named tag is not an
|
||||
immutable artifact (no container tag immutability in Gitea 1.25 — INFERRED), so overwriting the pinned
|
||||
tag is code execution inside CI, chaining back into the routes below. This is the deployment supply
|
||||
chain for prod and CI itself.
|
||||
|
||||
**Admin ownership is a real residual.** The PAT is minted under `timothy`, a site admin. The 403 proves
|
||||
the scope gate binds the *status* endpoint ahead of any admin bypass; it does NOT establish that for
|
||||
*package* endpoints, where Gitea resolves permission by owner and an admin passes object-level checks,
|
||||
so the token's package reach is plausibly wider than this repo. A non-admin bot account would close
|
||||
this, but is not free: packages live in a user namespace only its owner and admins can write. Both
|
||||
halves INFERRED, neither probed.
|
||||
|
||||
**Provenance, corrected.** `review-verdict.yml` leaves an existing `h10` alone only when it is
|
||||
positively identifiable as human — non-null `.creator.login` plus a `Review-verdict:` description
|
||||
(`release.verdict-status-check`). A user credential posts with a real creator and is INHERITED; an
|
||||
Actions job posts `creator: null` and is re-derived. **That asymmetry is not protection.** Re-derivation
|
||||
fires only on `opened|reopened|synchronize|ready_for_review|edited`, and posting a status is none of
|
||||
them, so a POST timed after the last event stands until the attacker merges. The gain here is that PR
|
||||
code can no longer escalate to instance admin — NOT that the durable forgery route is closed.
|
||||
|
||||
**The boundary is everything reachable from a job, not the secret store.** The store is a useful lower
|
||||
bound — auditing it rather than the workflow set is what found `RENOVATE_TOKEN` and
|
||||
`SERVERMGMT_DEPLOY_KEY` below, since any PR-added workflow can reference any secret. But
|
||||
`GITEA_TOKEN` is injected and never in the store; nor is the credential
|
||||
`actions/checkout` persists into `.git/config` (`docker-build.yml` omits `persist-credentials: false`);
|
||||
and jobs reach the runner's docker daemon.
|
||||
|
||||
**Measured vs inferred.** Measured here: the `v1.25.4` scope enum (`access_token_scope.go`) has no
|
||||
`status` entry; the `reqRepoWriter` gate (`routers/api/v1/api.go`); the probes in `mechanics`. Read from
|
||||
docs, NOT verified (2026-08-05): `permissions:` landed in 1.26.0 (Gitea PR #36173); no `app.ini` lever
|
||||
at any version; Gitea rejects GitHub's `statuses`/`checks` scopes.
|
||||
|
||||
**Version caveat — this record's measurements are pinned to 1.25.4, the instance is now 1.27.1.**
|
||||
The instance was upgraded mid-session on 2026-08-05 (#743). Everything above measured on 1.25.4 is
|
||||
therefore a *dated* claim, not a current one: the scope enum, the `reqRepoWriter` gate and the 403
|
||||
probe were all taken pre-upgrade and have NOT been re-run. They are recorded honestly as of their
|
||||
date and are the best evidence available, but do not cite them as current behaviour without
|
||||
re-probing. Re-verification of the 1.25.4-pinned claims across the CI docs is tracked separately.
|
||||
|
||||
**Surviving routes — this record is not a mitigation for any of them.** `RENOVATE_TOKEN` is a
|
||||
`write:repository` bot PAT in the same store, posting with a real creator, and cannot be scoped down
|
||||
because Renovate needs repo write (#742). The injected `GITEA_TOKEN` is write-capable in every job;
|
||||
only Gitea >=1.26 with the Actions default set to **Restricted** binds it (server-management#714) —
|
||||
the version half of that condition is now satisfied (1.27.1) but the *default* half is unverified, so
|
||||
treat this route as still open until probed. A
|
||||
collaborator's own token always can. `docker-build.yml` publishes `:prod` from a `v*` tag push and a tag
|
||||
may point at ANY commit — a prod image with no PR, review or status (tag protections are empty).
|
||||
**And none of it was necessary: direct pushes to `main` were server-side permitted, so the gate was
|
||||
bypassable with no forgery at all (#743).** That route is now closed — `main` carries
|
||||
`enable_push: false` (`release.main-direct-push-disabled`), which removes `main` as a destination for
|
||||
every write-only credential in this list, including the injected `GITEA_TOKEN` and `RENOVATE_TOKEN`.
|
||||
It does not remove them as *forgery* routes on the PR path, and it does not bind an admin credential,
|
||||
which can PATCH the protection off first. Correction to this record's earlier wording: a push
|
||||
*whitelist* would NOT have closed more of the class than the upgrade — measured 2026-08-05, a
|
||||
whitelist naming `timothy` still admitted the push, and every credential here acts as `timothy`.
|
||||
Treat this list as "at least these", never exhaustive. `SERVERMGMT_DEPLOY_KEY` remains in the
|
||||
store though its `bump-prod-compose` job went in `1b5efd7b9`, and its key on `timothy/server-management`
|
||||
is `read_only: false` — write access to the repo holding prod's GitOps stack definitions. Left in place
|
||||
by explicit decision 2026-08-05; recorded so it is accepted, not forgotten. Severity throughout: push
|
||||
access required, so a compromised contributor or subverted automated session, never an anonymous one.
|
||||
@@ -0,0 +1,94 @@
|
||||
---
|
||||
key: ci.exemption-provenance
|
||||
title: '2026-07-29 — the `review-verdict/h10` exemption path binds the base ref, constrains the bot exemption by CONTENT, and re-derives any success it cannot attribute to a human (#698)'
|
||||
status: active
|
||||
since: '2026-07-29'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The three inputs the exemption decision rests on must each be bound to something the judged PR cannot mutate. (1) BASE — `scripts/pr-changed-files.sh` takes the expected base BRANCH as a REQUIRED 5th argument and re-reads it before and after paging, because `/pulls/{n}/files` diffs against the PR''s live base and retargeting moves the answer without moving the head sha; the workflow passes `github.event.pull_request.base.ref` from the `pull_request_target` payload, which a retarget cannot rewrite. (2) BOT EXEMPTION — an author match is necessary but never sufficient: `pull_request.user.login` is the PR''s immutable CREATOR while its head is not, so the exemption additionally requires EVERY changed path to be a dependency manifest (`Directory.Packages.props` or `.config/dotnet-tools.json`, and ONLY those — the npm manifests are excluded because `package.json` `scripts` are executed by CI). (3) INHERITED SUCCESS — the never-overwrite short-circuit fires only for a status POSITIVELY identified as a human verdict for THIS base, meaning a non-null `.creator.login` AND a `Review-verdict:` description AND, when that description records a base (`(base: …)`, `release.verdict-status-check`), a base matching the PR''s — tested by requiring the description to END with the exact literal `(base: <base>)` and to contain exactly ONE such marker, never by extracting a value (see below); a present-but-different base is rejected, an absent one is not, since verdicts predating that convention carry none; every other shape, including any unrecognised one, is re-derived rather than trusted. The bot and docs-only exemptions are evaluated as INDEPENDENT predicates and the decision made afterwards, never as an `elif` chain. `edited` is in the workflow''s `types:` so a retarget reclassifies — which gives DETECTION, not atomicity: status writes are not serialized, so a stale run can still post over a fresher one. That residual is now FENCED rather than merely tracked — the job refuses to write at all if the PR''s timeline retarget COUNT moved while it was classifying (`ci.verdict-write-retarget-fence`, #706) — leaving only the sub-round-trip window that no API without compare-and-set can close. The PROTECTED path list additionally covers `CLAUDE.md` and `AGENTS.md` (#751) — they are not prose but the documents DEFINING the completion protocol, the merge-consent convention and the H10 rule, so protecting `.claude/` while the file specifying what it enforces stayed docs-only-exempt was the same self-exemption one directory over; driving the real classify body with a lone `CLAUDE.md` change produced an exemption `success`. `README.md` is deliberately not listed. It also covers `.codex/` (#711), which mirrors `.claude/hooks/` byte for byte including the merge-consent hook — latent while that directory is untracked, live the moment it is tracked; the list stays ENUMERATIVE rather than derived, because a derived rule would have to be evaluated against the very file list being classified. Reading the CURRENT status for input (3) must tolerate `statuses: null`: `GET /commits/{sha}/status` serialises a nil slice as `null`, not `[]`, on a head with no statuses yet, and an `array`-only gate made `read_existing_verdict` `exit 1` and post nothing at all (#751, `ci.workflow-run-body-no-expressions`) — `null` is accepted only when `total_count` is 0, so a body that merely lost its array is still refused. Path predicates are evaluated by COUNTING with `grep -c`, never `| grep -q` (SIGPIPE inversion) and never a here-string (temp-space failure) — see `ci.grep-q-pipefail-inversion`.'
|
||||
signals: 'forged review-verdict exemption, retarget race against the docs-only classifier, PR base changed mid-run, hijacked Renovate branch, bot exemption on a code change, machine-written success inherited as a verdict, status creator null vs user, never overwrite a human verdict, exemption chain skips docs-only for bots, why is my Renovate PR asking for a verdict, base ref binding on pr-changed-files.sh · paths: `.gitea/workflows/review-verdict.yml`, `scripts/pr-changed-files.sh`, `.claude/hooks/pretooluse-merge-consent.sh`, `scripts/tests/test_pr_changed_files.py` · issues: #698, #697, #672, #663, #649, #632'
|
||||
mechanics: '`scripts/pr-changed-files.sh <owner> <repo> <pr> <expected-head-sha> <expected-base-ref>` (5 args; a 4-arg call exits 2); workflow env `BASE_REF: ${{ github.event.pull_request.base.ref }}`; `BOT_MANIFESTS` anchored allow-list; short-circuit requires `.creator.login` non-null AND description matching `^Review-verdict:`; `types: [opened, reopened, synchronize, ready_for_review, edited]`'
|
||||
---
|
||||
|
||||
`ci.gate-trigger-base-resolved` stopped a PR supplying the gate's own *definition*. This closes the
|
||||
layer below: the exemption path still **decided from mutable or unattributed PR state**, and a
|
||||
machine-written `success` was never revalidated. Three routes, one root cause, one fix.
|
||||
|
||||
**Route 1 was reproduced, not theorised** (probe PR #703). A head `H` and scratch base `S` chosen so
|
||||
`H` vs `S` is docs-only; opened `H → main` so the trusted base definition ran; retargeted to `S`
|
||||
mid-flight. The job enumerated against the moved base, read docs-only, and posted
|
||||
`review-verdict/h10=success — "Exempt: docs-only change"`. Retargeted back to `main`: **nothing
|
||||
reclassified** (`created_at == updated_at`), leaving a PR into `main` whose diff carried a C# file
|
||||
behind a green required check. Closed unmerged, branches deleted, no forged `h10` left anywhere.
|
||||
**Why a base BINDING and not a pinned diff.** Diffing two immutable shas would close it outright;
|
||||
Gitea 1.25.4 cannot serve that — measured: `compare/{base}...{head}` returns no `files`, and a
|
||||
`--depth=1` fetch of the two shas has no merge base, so three-dot is impossible and two-dot
|
||||
over-reports everything `main` gained since the branch point. So the base is read before the first page
|
||||
and after the last, and **the gap is stated plainly**: a retarget opening *and* closing strictly
|
||||
between the files call and the re-read stays invisible from inside the enumeration.
|
||||
|
||||
**An earlier draft claimed `edited` made that residual non-durable. It does not, and cross-family
|
||||
review was right to call it a Blocker.** `edited` gives DETECTION, not atomicity or ordering: runs are
|
||||
not serialized, so the stale run can post `success` AFTER the reclassifying run posts `pending`, and an
|
||||
already-scheduled merge can fire in the green window between them. The `main → scratch → main` ABA
|
||||
transition is therefore NARROWED and observable, not closed. Tracked as an explicit residual rather
|
||||
than described as fixed. `edited` and re-derivation remain one fix — `edited` alone re-runs and exits
|
||||
on the existing `success`; re-derivation alone never gets a second run — but together they are
|
||||
mitigation, not a guarantee.
|
||||
|
||||
**Resolved 2026-08-03 (#706), and worth recording that the guarantee finally came from somewhere else
|
||||
entirely.** The missing piece was never ordering: `ci.verdict-write-retarget-fence` leaves the runs as
|
||||
unserialized as they ever were and instead makes a run that was overtaken decline to write, keyed on
|
||||
the timeline's monotonic retarget COUNT — the one signal the `main → scratch → main` ABA cannot make
|
||||
look unchanged. Measurement is what redirected it: `pull_request_target` runs were confirmed to
|
||||
overlap live (probe PR #722, the older run finishing 20s after the newer one started), and a
|
||||
non-cancelling concurrency group — the fix this record's residual implied and #706 proposed — was
|
||||
measured doing nothing at all. The paragraph above stands as written; only its last sentence is
|
||||
overtaken, and the sub-round-trip window it describes survives, because Gitea's status API has no
|
||||
compare-and-set.
|
||||
|
||||
**Route 2 — a bot ACCOUNT does not attribute the CODE.** `pull_request.user.login` is the PR's
|
||||
immutable *creator*; its head is not. Push application code onto an open Renovate branch and the PR is
|
||||
still "authored by renovate", touches no protected path, and was exempted. Checking the *pusher* fixes
|
||||
nothing — a git author is self-asserted text. So the exemption is constrained by what a bump can
|
||||
legitimately *be*: across all 11 Renovate PRs this repo has had, the paths touched were
|
||||
`Directory.Packages.props` (10) and `.config/dotnet-tools.json` (1) — and ONLY those. An earlier draft
|
||||
also exempted `web/package.json`/`web/package-lock.json` "so a first SPA bump cannot deadlock"; review
|
||||
called that a Blocker and was right. `renovate.json` enables only nuget/github-actions/dockerfile, so
|
||||
npm is unmanaged here and the entry bought nothing, while `package.json` `scripts` are EXECUTED by CI
|
||||
(`npm ci`, `npm run build`) — widening an exemption onto a code-execution path for no benefit. `*.csproj` is excluded — under Central Package Management
|
||||
versions live in `Directory.Packages.props`, so a Renovate `.csproj` edit is anomalous by
|
||||
construction. Cost stated: such a PR is not blocked, it needs a real verdict. The two exemptions are
|
||||
evaluated as INDEPENDENT predicates: written as an `elif` chain, a Renovate PR touching only `docs/`
|
||||
entered the bot branch, failed the manifest test, and never reached the docs-only branch.
|
||||
|
||||
**A counterfactual, not an incident.** Renovate PR #20 touched a `.csproj` and two C# files but has no
|
||||
`h10` status: it merged 2026-06-27, the gate landed 2026-07-25. The point is what identity-only *would*
|
||||
have done. An earlier draft claimed it HAD been exempted — wrong, and the correction is kept because
|
||||
"was silently exempted" and "would have been" are different claims.
|
||||
|
||||
**Route 3 — provenance, and the direction of the test.** The short-circuit exited on any `success`, so
|
||||
an exemption this job wrote was indistinguishable from a human verdict; obtained once, a forgery was
|
||||
accepted on every later run. It could not simply be deleted — it exists so `pending` cannot un-approve
|
||||
a reviewed head. Measured on the **combined** endpoint: a status POSTed with a user
|
||||
credential carries `.creator.login`, one POSTed by an Actions job carries `"creator": null`. The test is
|
||||
written in the **positive** direction — short-circuit only on something identified as human — because
|
||||
spelled the other way ("skip if it looks machine-written") any unrecognised shape falls through to
|
||||
*trusted*. Both halves are required, so if Gitea later populates `creator` for Actions the description
|
||||
test still fails: the guard degrades toward re-deriving, never toward trusting.
|
||||
|
||||
**What this does NOT close.** Anyone who can POST statuses directly can write both a creator and a
|
||||
`Review-verdict:` description and impersonate a verdict; branch protection binds the *context*, not its
|
||||
issuer. A provenance check, not an authentication one — that was `#697`'s registry credential, fixed by
|
||||
scoping it off `write:repository` (`ci.actions-credential-scoping`); `GITEA_TOKEN`, `RENOVATE_TOKEN`,
|
||||
and a collaborator's own token still can. Severity as `#672`: requires push access, so the threat model
|
||||
is a compromised contributor.
|
||||
|
||||
**Verification honesty.** Route 1 was reproduced live; the "and now it fails" half cannot be shown from
|
||||
a PR, because `pull_request_target` resolves this definition from `main` — the self-test gap
|
||||
`ci.gate-trigger-base-resolved` records. Pre-merge evidence is that reproduction plus the
|
||||
executed-behaviour tests in `scripts/tests/test_pr_changed_files.py`, each verified by mutation; the
|
||||
live re-check happens on `main` right after merge.
|
||||
|
||||
**Separate defect found reviewing this change:** `ci.grep-q-pipefail-inversion` — a pre-existing
|
||||
SIGPIPE inversion that let a large PR skip the `PROTECTED` guard entirely. Fixed in the same PR.
|
||||
@@ -0,0 +1,75 @@
|
||||
---
|
||||
key: ci.gate-trigger-base-resolved
|
||||
title: '2026-07-28 — `review-verdict.yml` triggers on `pull_request_target` scoped to `branches: [main]`, so the PR under judgment cannot supply the gate''s own definition (#672)'
|
||||
status: active
|
||||
since: '2026-07-28'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The workflow that writes the branch-protection-required `review-verdict/h10` status triggers on `pull_request_target` with `branches: [main]`, never on plain `pull_request`. Gitea resolves a `pull_request` workflow DEFINITION from the PR''s own head commit, so under that trigger a PR editing `.gitea/workflows/review-verdict.yml` ran its own rewritten copy and could post `h10=success` for itself; `pull_request_target` resolves the definition from the base instead. The `branches: [main]` filter is part of the rule, not a refinement of it: base resolution only relocates the rewrite from the head to the base, so without the filter a PR opened into an attacker-pushed base branch runs that branch''s gate. `pull_request_target` is safe HERE only because this job never checks out or executes head-supplied code — it checks out `base.sha` and runs only that tree''s scripts (`ci.shared-pr-file-enumeration`); reintroducing a head checkout under this trigger would be worse than the bug it fixed. This closes the rewrite route through THIS workflow and does NOT close the class: Gitea injects a write-capable `GITEA_TOKEN` into EVERY job, so any ref-resolved workflow — and a collaborator''s own API token, since branch protection binds the context and not its issuer — can still forge `review-verdict/h10`. The credential half is now RESOLVED in `ci.actions-credential-scoping` (#697): CI''s registry secret was the ADMIN account''s basic auth and is now a PAT that cannot post a status, which removes the ADMIN escalation and that credential''s route (a user credential''s forgery carries a real `creator` and is inherited as a human verdict; an Actions job''s carries `creator: null` and is re-derived — but do NOT read that asymmetry as protection: re-derivation fires only on the trigger''s `types`, and posting a status is not one of them, so a POST timed after the last PR event simply stands). It does not remove EVERY route: `RENOVATE_TOKEN` is a `write:repository` bot PAT in the same secret store, reachable by any PR-added workflow. The injected token stays write-capable until Gitea >=1.26 with a Restricted default (server-management#714), and a collaborator''s own token remains unfixable; the exemption path has its own separate defects in #698.'
|
||||
signals: 'workflow definition resolved from head, PR rewrites the gate that judges it, self-approve a required status check, pull_request_target vs pull_request, gate trigger branches filter, attacker-supplied base branch, how to test a change to review-verdict.yml, workflow not exercised by its own PR, gate edit goes live only on merge, required_approvals 0 does not bind an author, forged commit status inherited by sha · paths: `.gitea/workflows/review-verdict.yml`, `scripts/tests/test_pr_changed_files.py` · issues: #672, #663, #649, #622'
|
||||
mechanics: '`on: pull_request_target: {branches: [main], types: [opened, reopened, synchronize, ready_for_review, edited]}` (`edited` added by `ci.exemption-provenance` so a retarget reclassifies); asserted by `test_the_workflow_trigger_is_pull_request_TARGET_scoped_to_main` in `scripts/tests/test_pr_changed_files.py`; the job''s own context is renamed to `... (pull_request_target)` and must stay OUT of branch protection''s required list'
|
||||
---
|
||||
|
||||
`ci.shared-pr-file-enumeration` had this job check out the PR's **base** ref so the PR cannot supply
|
||||
the *scripts* that judge it — real but partial, as that record said: it does not bind the job
|
||||
**definition**. This closes that half.
|
||||
|
||||
**What was actually wrong.** Gitea, like GitHub, resolves a `pull_request` workflow definition from
|
||||
the PR's own head, so a PR editing `review-verdict.yml` ran its own rewritten copy — which could
|
||||
delete the base checkout or skip straight to posting `review-verdict/h10=success` for its head sha.
|
||||
Two things that look preventive were not: `PROTECTED` is defined by the same rewritten file, and
|
||||
branch protection requires the *context*, not an author, while carrying `required_approvals: 0`.
|
||||
|
||||
**Measured, not inferred.** The premise is a claim about someone else's software, so it was settled
|
||||
on this instance (Gitea 1.25.4) with **four** scratch PRs, not by analogy to GitHub: `pull_request` ran
|
||||
the head's rewrite and never wrote the real `h10`; `pull_request_target` ignored the identical rewrite
|
||||
and the base definition posted `h10=pending` on `opened` and `synchronize` alike, secrets available;
|
||||
`branches: [main]` produced no run at all from a non-`main` base; and the fourth — the negative one
|
||||
establishing the residual below — is counted because omitting it turns an honest partial into an
|
||||
overclaim. Probes posted only probe-named contexts, never a forged `h10`. Full results in #699.
|
||||
|
||||
**Why `branches: [main]` is load-bearing rather than tidy.** The *base branch* supplies the
|
||||
definition, and anyone who can push a branch can make it a base — so dropping the filter trades a
|
||||
head-supplied gate for a base-supplied one and closes nothing. Worse than lateral: a commit status is
|
||||
repo-global per sha (`#663`), so a `success` forged against a scratch base is **inherited** by a later
|
||||
genuine PR into `main` with the same head.
|
||||
|
||||
**Why `pull_request_target` is not the footgun it usually is.** Its standard danger is executing
|
||||
untrusted head code with a privileged token; this job executes none, checking out `base.sha` with
|
||||
`persist-credentials: false` and running only that tree's scripts. Trigger and checkout are one
|
||||
decision — under this trigger a head checkout would be strictly worse than #672 was.
|
||||
|
||||
**Options not taken.** `required_approvals: 1`, the cheapest mechanical fix, is unusable here: Gitea
|
||||
forbids approving your own PR and this is effectively a single-maintainer repo, so it deadlocks every
|
||||
PR instead of gating the dangerous ones. Verifying the status *author* needs an actor the PR cannot
|
||||
control, and the tampered workflow holds the same `GITEA_TOKEN`.
|
||||
|
||||
**Severity, stated plainly.** Never remotely exploitable — pushing a branch requires write access, so
|
||||
the threat model is a compromised contributor, who has other paths. Fixed because a gate whose
|
||||
authority the judged thing can assert is not a gate, not because an attack was expected.
|
||||
|
||||
**The class is NOT closed, and this record must not be read as claiming otherwise.** This fixed one
|
||||
instance of "a ref-resolved workflow can obtain credentials that POST a commit status", and that
|
||||
inventory is not a short list: Gitea injects `GITEA_TOKEN` into **every** job, defaulting to
|
||||
read/**write**, so head-resolved, `push`-triggered and `workflow_dispatch` workflows alike are routes
|
||||
(1.24+ loads a dispatched definition from the selected branch). A collaborator's own API token is a
|
||||
route with no workflow at all — branch protection binds the *context*, not its issuer. Full inventory
|
||||
in `#697`, whose credential half is resolved in `ci.actions-credential-scoping` — the registry secret
|
||||
no longer carries status-write. That does NOT leave the workflow routes provenance-free: any
|
||||
PR-added workflow can reference `RENOVATE_TOKEN`, a `write:repository` bot PAT in the same store,
|
||||
whose status carries a real creator and IS inherited (`#742`). The exemption path's
|
||||
own defects are `#698`. No in-repository test can establish
|
||||
status-authority isolation: the sibling guard added here catches only plain-text naming of the
|
||||
context.
|
||||
|
||||
**The gate is no longer exercised by its own PR** — base resolution cuts both ways, so an edit here
|
||||
goes live only on merge, repo-wide, untested. Verify one safely per `docs/ci-cd.md` → Review-verdict gate.
|
||||
|
||||
**Residual.** The job's own context is renamed to `... (pull_request_target)`, safe only because it
|
||||
was never one of branch protection's required contexts (the two `docker-build.yml` contexts plus
|
||||
`review-verdict/h10`); adding it would let the workflow satisfy the gate by merely running. **A trap
|
||||
for #697:** those two carry the literal `(pull_request)` suffix, so giving `docker-build.yml` the same
|
||||
treatment renames them and deadlocks merges unless branch protection is edited in the same operation.
|
||||
A non-`main` base now yields no status where it previously got one — fail-closed, removing a `#663`
|
||||
hazard. The `edited` gap this section once recorded as a mere inconvenience ("statusless until its next
|
||||
`synchronize`") was the persistence half of a live forgery; RESOLVED in `ci.exemption-provenance` (#698).
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
key: ci.grep-q-pipefail-inversion
|
||||
title: '2026-07-29 — never feed `grep -q` from a pipe under `set -o pipefail`: SIGPIPE turns a MATCH into a failed pipeline and inverts the guard (#698)'
|
||||
status: active
|
||||
since: '2026-07-29'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'In any script running under `set -o pipefail`, a security or classification predicate of the form `producer | grep -q…` is FORBIDDEN: `grep -q` exits at its first match, the producer then takes SIGPIPE and exits 141 once the data exceeds the pipe buffer (~64K), so `pipefail` reports the pipeline as FAILED even though grep MATCHED — inverting the predicate exactly when the input is large. A here-string (`grep -q… <<< "$data"`) is ALSO forbidden: bash materialises a large here-string via temporary storage, so it fails when temp space is full or unwritable, and inside an `if`/`!` that failure flips the predicate the same way. COUNT instead — `n=$(printf ''%s\n'' "$data" | grep -cE "$re")` — because `grep -c` drains stdin (no early exit, no SIGPIPE) over an ordinary pipe (no temp file). Read grep''s status honestly: exit 1 means a zero count and is a legitimate answer, anything >1 is a real error. Evaluate the counts ONCE at TOP LEVEL, never inline inside an `if`/`elif` condition: inside `$( )` an `exit` leaves only the subshell and `set -e` does not fire, so an error silently reads as "no match". Validate that each result is numeric and fail closed if not. This applies to both the enforced gate `.gitea/workflows/review-verdict.yml` and the advisory hook `.claude/hooks/pretooluse-merge-consent.sh`.'
|
||||
signals: 'grep -q pipefail, exit 141, SIGPIPE in a shell guard, large PR classified docs-only, protected path guard skipped, printf pipe grep -q, classification inverts on big input, pipe buffer 64K shell predicate · paths: `.gitea/workflows/review-verdict.yml`, `.claude/hooks/pretooluse-merge-consent.sh`, `scripts/tests/test_pr_changed_files.py` · issues: #698, #649'
|
||||
mechanics: '`count_matching` / `count_not_matching` helpers in `review-verdict.yml`, DEFINED BEFORE FIRST USE, results precomputed into `n_protected`/`n_not_manifest`/`n_not_docs` at top level and validated numeric; regression tests `test_a_LARGE_pr_*` build 1900+ paths (~171KB) to cross the pipe buffer, `test_the_classify_step_runs_without_SHELL_ERRORS` catches a dead guard, `test_a_protected_path_is_rejected_BY_THE_PROTECTED_BRANCH` asserts the branch rather than the outcome'
|
||||
---
|
||||
|
||||
Found by cross-family adversarial review of the #698 PR, then reproduced directly. It is not a defect
|
||||
of that change — the construct shipped with the ORIGINAL docs-only check and was live on `main`.
|
||||
|
||||
**The mechanism.** `grep -q` stops reading at its first match. If the writer still has data queued, it
|
||||
gets SIGPIPE and exits 141. `pipefail` makes the pipeline's status that of the failing member, so a
|
||||
successful MATCH is reported as a FAILED pipeline. Every guard written as `if ! producer | grep -q…`
|
||||
or `if producer | grep -q…` therefore flips, and only for large inputs — which is why every existing
|
||||
test missed it: they all used a handful of short paths, far below the buffer.
|
||||
|
||||
**Both directions were reproduced**, at 1901 paths / ~171KB, inside the enumerator's 2000-file cap:
|
||||
|
||||
- `A.cs` followed by 1900 docs paths → `docs_only=yes`, pipeline status 141. A C# PR classified
|
||||
docs-only and auto-granted `review-verdict/h10=success`.
|
||||
- `.gitea/workflows/review-verdict.yml` followed by 1900 docs paths → the `PROTECTED` guard MISSES, so
|
||||
a PR editing the gate's own definition skips the protected-path check and falls through to the
|
||||
exemptions.
|
||||
|
||||
The second is the serious one: it reaches a green required status on a PR that rewrites the gate, with
|
||||
**no retarget, no bot account and no forged status** — nothing but a large PR. It is a more direct hole
|
||||
than any of the three routes #698 was filed for.
|
||||
|
||||
**The here-string was the FIRST fix, and it was wrong.** `grep -q… <<< "$data"` does remove the
|
||||
SIGPIPE, and it shipped for one round. But bash materialises a large here-string through temporary
|
||||
storage, so it fails when temp space is full or unwritable — and because these predicates sit inside
|
||||
`if`/`!`, that failure flips them exactly as SIGPIPE did. It did not reproduce on macOS bash 3.2 and
|
||||
did on Linux bash 5.x, which is the environment CI runs; the disagreement is itself the argument for a
|
||||
construct that cannot fail either way. Counting with `grep -c` uses an ordinary pipe and drains stdin,
|
||||
so neither failure mode exists.
|
||||
|
||||
**Two follow-on traps, both found only by re-review.** First, the helpers were defined AFTER the
|
||||
classification chain that called them, so `count_matching` was `command not found` on every run and the
|
||||
`PROTECTED` branch never fired — while three "protected path" tests stayed green, because a protected
|
||||
path is also not a manifest and not docs-only, so the job reached `pending` down another route. Second,
|
||||
`exit 1` inside those helpers only left the command-substitution SUBSHELL, and since the substitution
|
||||
sat in a conditional, `set -e` never fired either. Hence the rule: define before use, evaluate once at
|
||||
top level, validate the result is numeric, and fail closed when it is not.
|
||||
|
||||
**Two testing lessons.** When several branches produce the SAME outcome, asserting the outcome cannot
|
||||
tell you which branch ran — assert the discriminator (here the `Decision:` reason line). And a cheap
|
||||
stderr sweep for `command not found` / `integer expression expected` / `unbound variable` catches a
|
||||
whole family of silently-skipped guards, because each of those makes an `if` condition merely false
|
||||
while the job exits 0 and posts a plausible status.
|
||||
|
||||
**The input-size lesson.** The whole class was invisible because every test used small inputs. A guard whose behaviour depends on a BUFFER THRESHOLD needs a test that crosses
|
||||
the threshold; otherwise the suite is measuring the wrong regime entirely and full coverage of the
|
||||
small regime proves nothing. The regression tests pair each large-input negative with a large-input
|
||||
POSITIVE control, so "large lists now fail closed" (a merge deadlock) cannot masquerade as a fix.
|
||||
@@ -0,0 +1,103 @@
|
||||
---
|
||||
key: ci.jq-version-contract
|
||||
title: '2026-07-26 — jq 1.6 is the FLOOR every shell gate must run on; `scripts/jq-preflight.sh` makes the version observable, and only `script-tests` pins it (#648)'
|
||||
status: active
|
||||
since: '2026-07-26'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Every shell gate that shells out to `jq` is authored to the jq 1.6-compatible subset, because the CI runner ships jq 1.6 while every developer Mac ships 1.8.x. `scripts/jq-preflight.sh` (no args) prints the parsed version and asserts a floor of 1.6 in every gate job''s log; `scripts/jq-preflight.sh --expect 1.6` additionally pins and fails loudly, but ONLY in the `script-tests` job. `review-verdict.yml` never pins — it writes the branch-protection-required `review-verdict/h10` status, so a hard pin there would turn any jq bump into a repo-wide merge deadlock.'
|
||||
signals: 'jq version divergence, jq 1.6 vs 1.8, jq -e exit code on empty input, contains NUL false positive, jq parse-error exit code collision, jq-preflight, script-tests --expect, review-verdict jq floor, merge deadlock from a pinned dependency · paths: `scripts/jq-preflight.sh`, `.gitea/workflows/review-verdict.yml`, `.gitea/workflows/pr-checks.yml`, `docs/ci-cd.md` · issues: #643, #647, #648, #649'
|
||||
mechanics: '`scripts/jq-preflight.sh` (no args) -> floor+observability in every gate job; `scripts/jq-preflight.sh --expect 1.6` -> tripwire, `script-tests` job only; `docs/ci-cd.md` -> "The jq contract"'
|
||||
---
|
||||
|
||||
Three independent jq-version divergences hit inside a single day (#643, #647), all in gates written
|
||||
and tested on a developer Mac (jq 1.8.x) but running on the CI runner (jq 1.6):
|
||||
|
||||
- `jq -e` over EMPTY input exits 4 on jq >= 1.7, but **0** on jq 1.6 — the docs-only pagination guard
|
||||
inferred "transport failure" from that exit status, so on 1.6 a failed page silently passed and the
|
||||
loop walked past unread pages while still reporting `files_complete=yes`.
|
||||
- `contains("\u0000")` — the NUL escape truncates to `""` on jq 1.6, so the containment test is
|
||||
vacuously **true for every string**, not just ones actually containing a NUL. The H10
|
||||
review-verdict classifier that relied on this was entirely inert on the runner.
|
||||
- Parse-error exit code: `jq empty` exits 5 on jq >= 1.7 but **4** on jq 1.6 — the same code jq 1.6
|
||||
uses for "no output produced". A garbage API response and an empty-but-valid one were
|
||||
indistinguishable, and the garbage case was read as "no comments."
|
||||
|
||||
None of these are exotic jq usage — they are constructs anyone would reach for first, and each one
|
||||
was discovered only because a real gate broke, not because anyone thought to test jq 1.6. That is the
|
||||
argument for a *contract*, not three point fixes: the failures share one shape (a shell gate's
|
||||
behavior is a function of an interpreter version nobody was treating as a variable), so patching each
|
||||
construct as it's found does not converge — it just narrows the next surprise.
|
||||
|
||||
**Why 1.6 is the floor and not 1.8.** The runner is the binding constraint, not the author's machine.
|
||||
Baking a pinned jq into `docker/ci/Dockerfile` was the obvious first idea and was rejected because it
|
||||
provably cannot cover the gate that actually broke: `review-verdict.yml` is `runs-on: small` with no
|
||||
toolchain-image pin, and per `ci.small-lane-git-only` the small lane is git-only — it gets the host's
|
||||
jq 1.6 no matter what the toolchain image contains. That was checked against the running binary, not
|
||||
assumed. So the fix has to hold at 1.6, in every gate, regardless of which lane it runs in.
|
||||
|
||||
**Why the pin is asymmetric.** `scripts/jq-preflight.sh` has two modes on purpose:
|
||||
|
||||
- No arguments — print the parsed version and fail only below the 1.6 floor. This is pure
|
||||
observability: the jq version CI actually used is now in the job log, so a future divergence can be
|
||||
diagnosed from the log alone instead of by guessing at the runner image. `review-verdict.yml` runs
|
||||
this mode. It cannot run the pinned mode, because that job's output is the required
|
||||
`review-verdict/h10` status check on `main` — a hard version pin there means the day the runner's jq
|
||||
is upgraded (a base-image bump, a host reimage, anything outside this repo's control), every PR
|
||||
on `main` stops merging until someone notices and re-pins. A required merge gate cannot have a
|
||||
failure mode that is "an upstream package manager did its job."
|
||||
- `--expect 1.6` — pin and fail loudly. Used only by `script-tests`. `scripts/tests/` currently
|
||||
exercises the 1.6 code path only because the runner happens to ship 1.6; if that silently changed,
|
||||
the 1.6 coverage this whole contract depends on would evaporate with no signal. The tripwire forces
|
||||
a human decision — re-pin after re-reading this record, or add a real 1.6 matrix leg — instead of
|
||||
letting the coverage quietly disappear.
|
||||
|
||||
**Be honest about the cost: firing this tripwire DOES block merges.** An earlier draft of this
|
||||
record claimed the pin was safe because `script-tests` is "advisory, not one of the required
|
||||
checks". That reasoning is wrong, and the correction is worth recording because it is easy to make
|
||||
twice. `.claude/hooks/pretooluse-merge-consent.sh` reads the **combined** commit status and denies
|
||||
on anything that is not `success`/`skipped` — see `ci.advisory-red-blocks-the-merge-gate` (#598).
|
||||
`script-tests` is a Gitea Actions job, so its red is a context folded into that combined state.
|
||||
A jq bump therefore reddens `script-tests` and blocks non-docs-only merges until someone re-pins.
|
||||
|
||||
One qualification, so this does not over-correct in the other direction: that combined-status read
|
||||
is guarded by `if [ "$mwcs" != "true" ]`. On the `merge_when_checks_succeed` path the hook does not
|
||||
read the combined status at all and defers to Gitea, which gates on *required* checks only — and
|
||||
`script-tests` is not one. So the blast radius is the hook-mediated merge path, not literally every
|
||||
merge.
|
||||
|
||||
The pin is kept anyway, deliberately: the fix is a one-line edit to the `--expect` value in
|
||||
`pr-checks.yml`, the failure message spells that out, and the alternative — silently losing the
|
||||
only coverage of the version axis that produced three bugs in one day — is worse than a visible
|
||||
stop. What is NOT acceptable is believing it is free. The difference from `review-verdict.yml` is
|
||||
therefore one of *degree and recoverability*, not of "blocks merges vs doesn't": there the check is
|
||||
required per-sha and a jq bump would deadlock merges with no in-repo remedy at all, whereas here a
|
||||
human can unblock the repo in one commit.
|
||||
|
||||
**The parse is strictly fail-closed, and that has an operational edge once it gates merges.**
|
||||
`scripts/jq-preflight.sh` accepts only a FIRST line of the form `jq-<X>.<Y>` or `jq version <X>.<Y>`;
|
||||
anything else — a leading blank line, a wrapper that prints a warning first, a version reported only
|
||||
on stderr — exits 1 rather than guess. That is the right default for a guard whose whole purpose is
|
||||
refusing to certify a version it did not parse, and it was arrived at over four revisions in which
|
||||
every *permissive* variant turned out to be fail-OPEN.
|
||||
|
||||
But when the follow-up wires the floor-only mode into `review-verdict.yml`, that strictness sits in
|
||||
the branch-protection-**required** check. A jq wrapper that starts printing a banner line would then
|
||||
deadlock merges repo-wide — the very failure the pin/floor asymmetry exists to avoid, arriving
|
||||
through the parser instead of the pin. If that ever happens the fix is to widen the accepted forms in
|
||||
`jq-preflight.sh`, **not** to relax the fail-closed behaviour: an unparsed version must never be
|
||||
treated as satisfying the floor.
|
||||
|
||||
**The three constructs to avoid, and their version-stable replacements:**
|
||||
|
||||
- Never infer "empty input" from a jq exit status — check the string in shell before invoking jq.
|
||||
- Never use `contains("\u0000")` (or any raw NUL literal) for a control-character test — use
|
||||
`explode | index(0)`, which does not depend on jq's NUL-escape handling.
|
||||
- Never infer "parse error" from `jq`'s exit code on ambiguous input — `jq empty` is the portable
|
||||
parse-only test, but its exit code collides between "no output" (1.6) and "parse error" (1.6, same
|
||||
code as 1.8's "no output"). Validate the response shape explicitly rather than reading one exit
|
||||
code as a specific failure mode.
|
||||
|
||||
Full narrative of how these were found (inside the #631 `script-tests` rollout) is in
|
||||
`docs/decisions/records/ci/script-tests-job.md`; this record is the durable contract that came out of
|
||||
it, rather than the incident log.
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
key: ci.required-job-step-execution-markers
|
||||
title: '2026-08-10 — every consequential `run:` step in docker-build.yml''s two REQUIRED jobs records that it executed, and a trailing guard fails the job when the set is incomplete (#756)'
|
||||
status: active
|
||||
since: '2026-08-10'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A step the runner declines to interpolate is DROPPED and the job still concludes `success` (`ci.workflow-run-body-no-expressions`). In `review-verdict.yml` that is fail-CLOSED — the required status is absent and the merge is blocked. In `docker-build.yml`''s `test` and `migrations` it is fail-OPEN: those are the other two required contexts on `main`, so the check reports green having done no work. So in those two jobs every `run:` step that is not `continue-on-error: true` calls `"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark <key>` as its FIRST act, and the job''s LAST step calls `ci-step-ran.sh assert --always <keys> --gated <keys>`, which fails the job when an expected key was never recorded. PER STEP, not per job: a marker written by the first step only proves the job started, while the drop that costs something is `Test` or the migration replay. The guard carries NO `if:` — the default `success()` is the wanted condition, because a genuine failure in an early step legitimately skips every later one and an `always()` guard would announce a false "these steps never executed" on every ordinary red build; the invariant that makes the omission safe is that the guard is skipped only when an earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every path to a green job runs the guard. Separately and independently, no `${{` OPENER may appear in any `run:` body of those two jobs OR of `build` — the drop mechanism requires the opener, so banning it makes the class unreachable rather than merely caught, and an UNCLOSED opener triggers the same rewrite as a well-formed pair. Pass values in through the step''s `env:`, which is interpolated per value. The two halves have DIFFERENT scopes on purpose: markers cover the required pair, while the ban also covers `build`, whose `Smoke + IPTV E2E` step runs AFTER the image is pushed, so a drop there publishes a release candidate that was never booted and that `DeployStack jazz-media` then promotes. `functional-e2e` is delimiter-free but deliberately excluded (advisory by declaration), and `api-docs`/`format` keep one `github.base_ref` each and gate nothing that ships. The ban is enforced on the RELEASE PATH itself, not only in review (#767): a `scan` job runs the PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and no image is published. A guard STEP inside `build` was tried first and is wrong — a step cannot protect the job it publishes from, and "my body has no opener so I cannot be dropped" is circular when only the PR-only test enforces that. The pytest in `script-tests` remains, but it is `on: pull_request` and not a required context, so it alone left the tag path unchecked.'
|
||||
signals: 'required check green but no work done, step never ran but job green, Build & test green in seconds, EF migration integrity green without replaying, missing Run Main step marker, Unable to interpolate expression format(, dropped step docker-build, ci-step-ran.sh, marker file, expression delimiter in a required job · paths: `.gitea/workflows/docker-build.yml`, `scripts/ci-step-ran.sh`, `scripts/tests/test_ci_dropped_step_guard.py`, `scripts/tests/test_ci_release_path_scan_job.py` · issues: #756, #751, #684, #767'
|
||||
mechanics: '`scripts/ci-step-ran.sh` owns the marker path so it exists ONCE and the write and the read cannot diverge. It is keyed on `GITHUB_JOB`/`GITHUB_RUN_ID` — REQUIRED, refusing rather than falling back to a reusable name — plus `GITHUB_RUN_ATTEMPT`. All three REFUSE rather than falling back to a reusable name. The third was warn-and-default until its presence was measured: grepping a log for the variable NAME proves nothing, and inferring it from the absence of a stderr warning proves nothing either (stderr capture was itself unestablished), so `assert` was made to print `Marker identity: job=… run=… attempt=… (from the runner)` on STDOUT and the answer was read off run 1916 for both required jobs. That line is retained as standing evidence. Do NOT justify the keying with #751''s "RUNNER_TEMP is /tmp, not a private per-job dir": that was measured on a job with no `container:` and does not transfer — these jobs get a fresh container, which is the primary protection, and the keying is defence in depth. Held by `scripts/tests/test_ci_dropped_step_guard.py`: static (marker set derived from the workflow equals the guard''s expectations, bucket matches each step''s `if:`, guard is last / has no `if:` / is not advisory / has no delimiter) and behavioural (the guard''s real command line executed against markers written by the steps'' real marker lines, dropping each key in turn). The release-path `scan` job (#767) runs the existing PyYAML-based ban test rather than a second implementation, so there is no drift surface; `scripts/tests/test_ci_release_path_scan_job.py` holds the WIRING instead — that `build` needs it, that it carries no job-level `if:` (one excluding the tag push restores the hole, one skipping the job skips `build` too), that no step is advisory, that it actually invokes the ban test, and that its own run bodies are delimiter-free. Its steps carry markers and a trailing assert of their own, verified by the same drop-each-key-in-turn behavioural pattern.'
|
||||
---
|
||||
|
||||
**Why per step, when #756 proposed per job.** A job-start marker answers "did this job begin", which
|
||||
was never in doubt. The fail-open it is supposed to close is a required context reporting success
|
||||
while the work inside it did not happen, and the steps that carry that work are `Test`, `Build` and
|
||||
the two migration replays — all of them well past step one. A guard positioned where it cannot see
|
||||
the case it was built for is the "guard that never executed" failure one level up, and this repo has
|
||||
now shipped that twice in the same file (#751's retarget fence, and #751's own guard).
|
||||
|
||||
**Why a script rather than an inline body, when #751 chose inline.** Two reasons and the second is
|
||||
the load-bearing one. The path literal exists once, so the write and the read cannot drift — #751
|
||||
carries it twice and spends real test effort proving the copies agree, because a divergence reddens
|
||||
every run and then gets deleted as broken. And a one-line `run: scripts/ci-step-ran.sh …` cannot
|
||||
contain an expression delimiter, so the mechanism being guarded against cannot drop the guard. #751's
|
||||
own record names that as the stronger construction and settled for inline only because its
|
||||
measurement showed it was not required there.
|
||||
|
||||
**Why a script is acceptable here and would NOT be in `review-verdict.yml`.** That workflow checks
|
||||
out the PR's BASE precisely so a PR cannot supply the code that judges it. `docker-build.yml` is
|
||||
head-resolved by design — a PR already supplies every test this job runs — so calling a script from
|
||||
the head adds no authority a PR did not already have. This is a correctness gate against silent
|
||||
no-ops, not a security gate against a hostile PR; that job belongs to `review-verdict/h10`. Do not
|
||||
carry this reasoning back into the gate workflow.
|
||||
|
||||
**The premise was re-measured on the BUILD lane, not inherited.** The whole guard rests on the runner
|
||||
still executing a LATER step after dropping an earlier one. #751 established that on the `small`
|
||||
lane; these two jobs run in a `container:` on `ubuntu-latest`, which is a different lane, so assuming
|
||||
it transfers would be the same shape of mistake the guard exists to catch. Measured by scratch PR
|
||||
#765 (Gitea 1.27.1, 2026-08-10), which reintroduced the exact #751 defect — an invalid expression
|
||||
payload inside a shell comment — in the `test` job's `revalidate` step. The step was dropped, the
|
||||
other eleven markers were still recorded — ten of them AFTER the drop, `detect` being the earlier
|
||||
eleventh — and the guard was the ONLY failing step
|
||||
in the job — so without it that run would have concluded `success` having skipped a step. The SAME
|
||||
run supplies the positive control on the same lane: its untouched `migrations` job marked all six
|
||||
steps, reported `All 6 expected step(s) executed`, and concluded `success`.
|
||||
|
||||
A second probe (PR #766, run 1913) settled the one path on which the `if:`-less guard could have been
|
||||
a silent no-op: a FAILING `continue-on-error` step. Had that flipped `success()`, the guard would be
|
||||
skipped on a still-green job. It does not — the advisory step failed, the guard ran anyway, reported
|
||||
`All 12 expected step(s) executed`, and the job stayed `success`. Full log extracts in
|
||||
docs/ci-cd.md.
|
||||
|
||||
**The two halves are deliberately different in kind, and neither is redundant.** The delimiter ban is
|
||||
static and absolute, and it makes the defect class UNREACHABLE in these jobs rather than merely
|
||||
detected — it is the cheaper and more general half, and it is enforceable today only because both
|
||||
jobs were already delimiter-free (measured 2026-08-10: `test` 0, `migrations` 0), and `build` was
|
||||
brought in by moving its two payloads to `env:` — leaving `api-docs` and `format` with one
|
||||
`github.base_ref` each, in detect steps that gate nothing that ships. The runtime markers catch a step
|
||||
that fails to run for any OTHER reason, including reasons not yet met. Keeping only the static half
|
||||
would be trusting that this is the only way a step can vanish, which is exactly the assumption #751
|
||||
falsified about shell comments.
|
||||
|
||||
**What this does not claim.** The guard proves a step STARTED, never that it did its work correctly
|
||||
— that is what the step's own exit status is for. It does not cover `uses:` steps, which are not
|
||||
`run:` bodies and cannot be dropped this way.
|
||||
|
||||
An earlier draft dismissed the non-required jobs as "a smaller cost (no required context lies)", and
|
||||
cold review showed that was false for the one that matters. `build`'s only delimiter-bearing body was
|
||||
`Smoke + IPTV E2E`, which runs AFTER `Build and push`: on a `v*` tag the candidate image is already
|
||||
published, and that step is the only thing that boots it. A drop there ships an unsmoked release
|
||||
candidate under a green tick, and prod promotion pulls exactly that image. It was also the cheap case
|
||||
— both payloads were plain values, so moving them into `env:` cost nothing and let `build` join the
|
||||
ban. The claim not to repeat is the draft's dichotomy ("give up interpolation or move into
|
||||
`scripts/`"); the `env:` escape hatch this record prescribes was the answer all along. What genuinely
|
||||
remains uncovered is `api-docs` and `format`, whose one `github.base_ref` each sits in a detect step
|
||||
that gates nothing that ships, and `functional-e2e`, which is advisory by declaration.
|
||||
|
||||
**The `build` ban is now fail-closed on the release path (ersatztv#767 — this was the open
|
||||
residual).** It used to be enforced only by `script-tests`, which is `on: pull_request` and is not a
|
||||
required context, so nothing re-checked it when a release was actually cut: a delimiter that reached
|
||||
`main` would still drop `Smoke` on the tag build and report green. A `scan` job now runs the
|
||||
PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and
|
||||
no image is published.
|
||||
|
||||
**Two designs were tried, and the first one's failures are the reusable part.** The first put a
|
||||
bespoke stdlib scanner in `build` itself as an unconditional step before `Build and push`. Two
|
||||
independent reviews rejected it on two counts, both easy to re-invent:
|
||||
|
||||
- **A guard step cannot protect the job it lives in.** `build` publishes, so a guard step there is
|
||||
fail-OPEN if the runner drops it. The defence offered — "the guard's own body has no opener, so it
|
||||
cannot be dropped" — is circular, because the only thing enforcing that property was the same
|
||||
PR-only, non-required test being backstopped. A `needs:` edge is not circular: a red job skips its
|
||||
dependents by construction.
|
||||
- **A hand-written parser was strictly weaker than the check it backstopped.** It hand-parsed YAML to
|
||||
avoid provisioning PyYAML on `build`'s bare runner, and review found ~10 false NEGATIVES in one
|
||||
round (flow mappings, a quoted `"run":` key, aliases, multiline quoted scalars). For a security
|
||||
gate only false negatives matter, so this was worse than useless — it looked like enforcement. Do
|
||||
not re-attempt a bespoke scanner to save provisioning a dependency; run the real test.
|
||||
|
||||
**Why this needs no third marker bucket.** The deferral assumed the answer had to be markers on
|
||||
`build`, requiring a bucket that models `Smoke`'s publish-ref `if:`. It does not: the delimiter class
|
||||
is a *static* property of the workflow text, so a job that reads the text catches it without
|
||||
modelling any `if:`. The marker buckets are unchanged. Per-step markers on `build` remain a genuine
|
||||
smaller residual — they would catch a drop caused by something other than a delimiter.
|
||||
|
||||
**What this does not claim.** That no step can ever fail to run for another reason. The `scan` job's
|
||||
own steps carry markers and a trailing assert, which moves the terminal assumption rather than
|
||||
removing it: to fail open you must now drop the pytest step AND the assert step, not either alone.
|
||||
|
||||
**Measured, not assumed.** See the closing record on ersatztv#767 for the run ids of the poisoned and
|
||||
control dispatches. The arrangement: a `workflow_dispatch` on a scratch branch whose `Smoke` body
|
||||
carries a deliberate delimiter must redden `scan` and leave `build` skipped, and the same dispatch
|
||||
without the poison must pass. Note that "no image was published" is NOT part of the evidence — on a
|
||||
scratch ref `Build and push` has `push: false` regardless, so that conjunct could not have come out
|
||||
the other way; the discriminating observation is `scan` red and `build` skipped. Do NOT repeat the
|
||||
cost estimate an earlier draft gave ("would require pushing a real `v*` tag").
|
||||
@@ -59,17 +59,12 @@ both reds were real:
|
||||
|
||||
The second one is the argument for this record in miniature. It sat in the gate that decides whether
|
||||
a PR skips the Done-when checks, it was covered by an existing test, and that test could not catch it
|
||||
on a developer Mac (jq 1.8) — only in CI, where the suite had never run. **Standing rules it leaves behind, all three the same shape — never infer a
|
||||
CONDITION from a jq exit status or a version-dependent builtin:**
|
||||
- never infer "empty input" from a jq exit status; check the string (`#643`);
|
||||
- never infer "parse error" from a jq exit status — `jq empty` is the portable test, because
|
||||
jq >= 1.7 exits 5 where 1.6 exits 4, and 4 is also "no output" (`#647`);
|
||||
- never use `contains("\u0000")` — on jq 1.6 the escape truncates to `""` and it matches every
|
||||
string (`#647`). `explode | index(0)` is version-stable.
|
||||
|
||||
The runner ships **jq 1.6**; a developer Mac ships 1.8.x. Three divergences were found in one day,
|
||||
so the durable fix is to pin or preflight the version rather than keep patching constructs —
|
||||
tracked on `#647`.
|
||||
on a developer Mac (jq 1.8) — only in CI, where the suite had never run. Two further divergences of
|
||||
the same shape (a `contains("\u0000")` false positive and a colliding parse-error exit code) turned
|
||||
up the same day; the durable contract that came out of all three — the exact constructs to avoid, and
|
||||
why `jq-preflight.sh` pins in `script-tests` but only floors the version in `review-verdict.yml` — is
|
||||
recorded once, in `ci.jq-version-contract` (`docs/decisions/records/ci/jq-version-contract.md`), and
|
||||
is not restated here.
|
||||
|
||||
An independent cross-family review of that fix then found **two further fail-opens in the same
|
||||
enumeration, both reachable with no transport error at all** (#643):
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
---
|
||||
key: ci.shared-pr-file-enumeration
|
||||
title: '2026-07-26 — `scripts/pr-changed-files.sh` is the ONE enumeration of a PR''s changed files; the advisory hook and the enforced gate share mechanism, never policy (#649)'
|
||||
status: active
|
||||
since: '2026-07-26'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A PR''s complete set of changed file paths is computed by exactly one implementation, `scripts/pr-changed-files.sh`, called by both `.claude/hooks/pretooluse-merge-consent.sh` (advisory — a failure falls through to a human prompt) and `.gitea/workflows/review-verdict.yml` (enforced — a failure must fail closed, because a match here posts the branch-protection-required `review-verdict/h10` status with nobody in the loop). The script owns exhaustiveness (pagination, rename/path validation, head-sha binding, base-ref binding — see `ci.exemption-provenance` — and base-TIP binding, #707: the ref answers "did this PR RETARGET", the tip answers "did the base ADVANCE mid-enumeration", and only the second can see `/pulls/{n}/files` recomputing each offset-paged page against a moved base and dropping a path out of an already-consumed range; both ends of the window are bound, and an advance BEFORE the window is deliberately not an error, or ordinary churn on `main` would fail every open PR) and returns exit 0 only for a verified-complete list; it does NOT classify paths — each caller keeps its own docs-only allow-list, and the two allow-lists differ on purpose and stay separate.'
|
||||
signals: 'duplicated PR file enumeration, enforced gate weaker than advisory hook, docs-only allow-list drift, shared mechanism not shared policy, pr-changed-files.sh, checkout base ref not PR head, gate judging its own PR, exhaustiveness bug in a security predicate · paths: `scripts/pr-changed-files.sh`, `.claude/hooks/pretooluse-merge-consent.sh`, `.gitea/workflows/review-verdict.yml` · issues: #643, #648, #649'
|
||||
mechanics: '`scripts/pr-changed-files.sh <owner> <repo> <pr> <expected-head-sha> <expected-base-ref>` -> stdout newline-delimited paths, exit 0 only if complete and bound to BOTH the given sha and the given base branch; the 5th argument is REQUIRED and a 4-arg call exits 2 (`ci.exemption-provenance`); callers: `.claude/hooks/pretooluse-merge-consent.sh`, `.gitea/workflows/review-verdict.yml`'
|
||||
---
|
||||
|
||||
Before #649, the PR changed-file enumeration existed as two independent implementations. That would
|
||||
be an ordinary duplication smell anywhere else; here it was actively dangerous, because the two
|
||||
copies had unequal *consequence*. The advisory hook's failure mode is a human permission prompt — a
|
||||
missed guard there just means a person gets asked instead of an automatic decision. The enforced
|
||||
workflow's failure mode is a `success` write to `review-verdict/h10`, the one status branch
|
||||
protection actually requires — a missed guard there merges an unreviewed PR with nobody asked at all.
|
||||
|
||||
**The drift that motivated this.** Four rounds of #643 hardening landed entirely on the copy with the
|
||||
*lower* stakes. The hook accumulated CR/LF rejection, `..` rejection, a closed `.status` allow-list,
|
||||
`previous_filename` validation on every row (not just `renamed`), termination only on a validated
|
||||
empty page, and head-sha binding — while the enforced workflow kept the original, weaker logic. Its
|
||||
fail-closed behavior on a garbage API response was incidental (an empty `n` erroring a bash
|
||||
conditional to false), not a designed property. The gate with real authority was strictly weaker than
|
||||
the gate with none, which is the wrong way around by construction, not by anyone's mistake in a
|
||||
single review — nothing in the original layout forced the two to move together.
|
||||
|
||||
**Why the fix is "one script, two callers" rather than "copy the hardening across."** Copying keeps
|
||||
the two-implementation shape; the next hardening round would only need to happen twice again, and
|
||||
there is no mechanism that would surface a second drift before it mattered. Extracting
|
||||
`scripts/pr-changed-files.sh` makes the enumeration a single artifact with a single test suite
|
||||
(`scripts/tests/test_pr_changed_files.py`), so a future guard is added once and both callers get it
|
||||
atomically.
|
||||
|
||||
**Mechanism, not policy — the two allow-lists stay separate on purpose.** The extracted script
|
||||
answers exactly one question: "what is the complete set of paths this PR touches, at one head, or can
|
||||
we not tell?" It does not decide whether that set makes the PR docs-only. Each caller keeps its own
|
||||
classification:
|
||||
|
||||
- The hook's docs-only pattern also lets `.claude/`, `.gitea/`, `.husky/` through, which is safe there
|
||||
*only* because a non-match falls through to a human prompt rather than an auto-grant.
|
||||
- The workflow's is narrower, because there a match posts a green status with nobody in the loop, and
|
||||
both docs-only and Renovate exemptions are void when the PR touches `.claude/`, `.gitea/`,
|
||||
`.husky/`, `scripts/` or `docker/ci/` — the gate must not be able to exempt itself from review by
|
||||
editing itself.
|
||||
|
||||
Merging the two allow-lists would have quietly widened the enforced exemption to match the advisory
|
||||
one, turning a difference that exists for a reason into an accident of refactoring. Sharing the
|
||||
enumeration closes the drift that actually caused harm without touching the part that was correctly
|
||||
different.
|
||||
|
||||
**What the shared script owns.** Six guards, all now exercised by one test suite instead of a subset
|
||||
in each caller:
|
||||
|
||||
- CR/LF rejection and `..` rejection on every path.
|
||||
- A closed `.status` allow-list — `added`/`deleted`/`changed`/`modified`/`renamed`/`copied`, not an
|
||||
open denylist. Note `changed` and `deleted` are the values live Gitea 1.25.4 actually emits;
|
||||
`modified` is accepted alongside `changed` because a closed list built from the wrong vocabulary
|
||||
would gate every genuine docs-only PR. GitHub's `removed` is deliberately **not** in the list — an
|
||||
earlier draft of this record said it was, which would have sent a maintainer looking for a value
|
||||
the code rejects.
|
||||
- `previous_filename` validated on **every** row the extraction consumes, not only rows whose
|
||||
`.status` is `renamed` — a `modified`/`copied` row can still carry it, and an earlier fix that
|
||||
validated only the `renamed` case was found incomplete for exactly this reason (see
|
||||
`ci.script-tests-job` for the review trail).
|
||||
- Termination only on a validated **empty** page — Gitea's paging can return fewer rows than
|
||||
requested well before the real end of the list, so "short page" is not a valid termination signal.
|
||||
- Head-sha binding: the head is re-read after enumeration, and the caller must refuse to trust the
|
||||
list if it moved mid-enumeration, since paging is several round-trips and a force-push between them
|
||||
would otherwise yield a list belonging to no single commit. **This detects ONE-WAY movement only.**
|
||||
An A→B→A force-push round trip restores the expected sha, so the binding holds while the pages came
|
||||
from two different states — see #664. Closing that needs a commit-pinned files endpoint (Gitea has
|
||||
none) or a local diff, not a tighter check here; the guarantee is stated narrowly rather than left
|
||||
to read as complete.
|
||||
|
||||
**Base-ref checkout — binds the SCRIPTS to the base, not the workflow itself.** `review-verdict.yml`
|
||||
checks out the PR's BASE ref (`ref: ${{ github.event.pull_request.base.sha }}`,
|
||||
`persist-credentials: false`), never the head, so the *scripts the job executes* — above all
|
||||
`scripts/pr-changed-files.sh` — come from the already-reviewed base rather than from the PR under
|
||||
judgment. The checkout and its `ref` are the security-relevant parts: a bare `run:` calling the
|
||||
script would not have been sufficient, since the script would then have come from wherever the
|
||||
runner happened to be.
|
||||
|
||||
**It does NOT mean a PR cannot rewrite the gate that judges it (#672).** Gitea resolves a
|
||||
`pull_request` workflow *definition* from the PR's own head, so a PR editing `review-verdict.yml`
|
||||
runs its own rewritten copy — which can delete this checkout, or simply post
|
||||
`review-verdict/h10=success` for its head sha and stop. Branch protection does not close that: it
|
||||
requires the *context*, not an author, and carries `required_approvals: 0`. An earlier revision of
|
||||
this paragraph said the workflow "cannot be rewritten by that same PR to weaken its own judgment",
|
||||
which is true of the scripts and false of the workflow — and stated in the one sentence a reader
|
||||
resolving this record from the catalog is most likely to stop at.
|
||||
|
||||
**That half is now closed, elsewhere — see `ci.gate-trigger-base-resolved` (#672).** The workflow
|
||||
triggers on `pull_request_target` scoped to `branches: [main]`, so Gitea resolves its definition from
|
||||
the base rather than the head. The paragraph above is kept in the past tense rather than deleted
|
||||
because it names the distinction this record turns on: the base-ref checkout binds the *scripts*, and
|
||||
only the trigger binds the *definition*. Note the dependency runs the other way too — that checkout is
|
||||
what makes `pull_request_target` safe to use at all here, since this job never executes head-supplied
|
||||
code.
|
||||
|
||||
An earlier revision of this record stated the requirement in the future tense, because the wiring
|
||||
was staged over two PRs: the workflow runs the BASE version of the gate, and until the shared script
|
||||
existed on `main` a wired workflow would have exited 127 on its own PR and blocked the merge gate
|
||||
through the combined status. That staging is complete. Both halves are asserted by
|
||||
`scripts/tests/test_pr_changed_files.py`, which parses the workflow YAML rather than substring-
|
||||
matching it — `head.sha` for `base.sha` is a nine-character diff, and a text-level check would still
|
||||
pass if a second checkout step took the head afterwards and won.
|
||||
|
||||
**What is deliberately NOT claimed.** The `PROTECTED`
|
||||
path list remains the guard that stops a bot-authored PR from editing the gate and exempting itself,
|
||||
and mutation testing was what established that `PROTECTED` is load-bearing only on the BOT path —
|
||||
it and `DOCS_ONLY` are disjoint patterns, so on the docs-only path that clause can never fire. A
|
||||
test written against a docs-only-plus-protected file list passed with the clause deleted.
|
||||
|
||||
**A commit status is repo-global, which this record does not fix either.** `review-verdict/h10` is
|
||||
attached to a sha in the repository, not to a pull request, so a success earned on one PR is
|
||||
inherited by any other PR with the same head — including one opened against a different base after
|
||||
the first is closed (#663). That is the same property that makes #622's per-sha binding work, read
|
||||
from the other end. Out of scope here; noted so the enumeration's guarantees are not mistaken for a
|
||||
guarantee about *which PR* a verdict belongs to.
|
||||
|
||||
**Severity, stated honestly.** Every enumeration defect found in this area (#643) downgraded a
|
||||
mechanical deny/ask to a human prompt on the hook side; none produced a silent self-merge on their
|
||||
own. It is still a real weakening worth fixing — the whole point of #649 is that the same class of
|
||||
bug on the *enforced* copy would not have been merely a downgrade.
|
||||
@@ -0,0 +1,100 @@
|
||||
---
|
||||
key: ci.verdict-write-retarget-fence
|
||||
title: '2026-08-03 — the review-verdict job fences its write on the PR timeline''s retarget COUNT, and verifies the exemption write afterwards (#706)'
|
||||
status: active
|
||||
since: '2026-08-03'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The `review-verdict/h10` job counts `change_target_branch` events on the PR''s issue timeline at run start and again immediately before its POST, and writes NOTHING if the count moved. The COUNT is the key because the branch NAME is ABA-vulnerable — `main -> S -> main` reads `main` at both ends, which is how #698 route 1 obtained a forged exemption — while the event count is monotonic and cannot alias. Abstaining is a handoff, not a stall, and that is the property the design rests on: every retarget fires `edited`, which is in this workflow''s `types:`, so the event that makes a run abstain has already queued a successor whose window opens after it; the induction terminates when retargeting stops and the last run writes the final answer. `updated_at` was REJECTED as the key because it also moves for comments and labels, which fire none of this workflow''s `types:` — a run could abstain with no successor coming, which is a real stall. The count is trusted only when paging reached a validated EMPTY page; an untrusted count (unreadable page, non-array body, non-numeric length, page cap hit) blocks the exemption `success` ONLY and still lets `pending` through, because `pending` cannot turn an unreviewed head green while withholding it would strand ordinary PRs for no safety gain. SEPARATELY, and for the human-verdict race the fence does nothing about: after posting an exemption `success` the job re-reads `/statuses/{sha}` and, if a human `Review-verdict:` row appeared with an id ABOVE a high-water mark taken just before the POST, overwrites its own status with `pending` and logs an error. The repair is `pending`, NEVER a copy of the human''s state, since re-posting their `failure` under the machine credential would attribute a human verdict to the job; its description is a SENTINEL that the classification refuses to grant an exemption over AND re-writes verbatim on every later run, so the block is a FIXED POINT rather than decaying — writing the generic `pending` description there instead erases the marker and the exemption simply returns one event later. The mark is captured BEFORE the last-moment re-read, not merely before the POST — a later mark leaves a multi-round-trip blind gap in which a verdict is neither seen by the re-read nor repaired afterwards. The id comparison is load-bearing: a mere presence test would fire forever on a base-mismatched verdict that `read_existing_verdict` deliberately declines to honour, deadlocking that PR''s exemption permanently. Finally, a run whose last-moment re-read finds a sentinel it did not see at its FIRST read ABSTAINS instead of posting: that can only mean an overlapping run repaired a raced verdict mid-flight, and this run''s `success` — frozen at classification time, with the human row below its own mark, so neither the fence nor the post-write check would catch it — would otherwise bury the rejection. That is the one path in this design that failed toward SUCCESS rather than `pending`. The post-write check counts TWO row shapes above the mark, not one — a human `Review-verdict:` row AND a machine sentinel — because with two overlapping runs the human row can sit BELOW the second run''s mark while the first masks it and only then writes the sentinel, leaving the second to post its own `success` on top; counting the sentinel converges both runs on the fixed point instead.'
|
||||
signals: 'stale review-verdict run overwrites a fresher one, retarget ABA against the docs-only classifier, concurrency group does not serialize pull_request_target, gitea auto-cancel push vs pull_request_target, forged exemption restored after reclassification, human BLOCKED silently turned green, post-write status verification, change_target_branch timeline count, why does my PR post no verdict status after a retarget · paths: `.gitea/workflows/review-verdict.yml`, `scripts/tests/test_pr_changed_files.py` · issues: #706, #698, #672, #663, #622'
|
||||
mechanics: '`count_retargets()` pages `GET /repos/{repo}/issues/{pr}/timeline?limit=50&page=N` (cap 20) setting `rt_count`/`rt_ok`, trusted only on a validated empty page, which is a page of EITHER `null` (what this endpoint really returns past the end) or `[]` — an `array`-only type gate read the real terminator as unreadable and withheld every exemption (#751); `retargets_before`/`retargets_before_ok` captured before enumeration, re-counted immediately before the POST; `max_id_before` from `GET /repos/{repo}/statuses/{sha}` (a BARE ARRAY, unlike the combined `/commits/{sha}/status` object); repair POST is `pending`; tests `test_a_RETARGET_DURING_the_run_posts_NOTHING`, `test_a_PR_retargeted_BEFORE_the_run_but_QUIET_during_it_is_STILL_exempt`, `test_an_UNTRUSTED_retarget_count_withholds_the_EXEMPTION`, `test_an_UNTRUSTED_retarget_count_STILL_LETS_PENDING_THROUGH`, `test_a_human_verdict_landing_AFTER_the_POST_is_repaired_to_pending`, `test_a_PRE_EXISTING_human_row_does_NOT_trigger_a_repair`'
|
||||
---
|
||||
|
||||
`ci.exemption-provenance` closed three routes into the exemption path and left one residual it named:
|
||||
status writes are not serialized, so a stale run can post over a fresher one. This record resolves it,
|
||||
**narrowing** that record rather than superseding it.
|
||||
|
||||
## Measured, not reasoned (Gitea 1.25.4, 2026-08-03)
|
||||
|
||||
- **`pull_request_target` runs for one PR overlap, older finishing last.** Probe PR #722: run 7520
|
||||
(`opened`) completed at 18:30:42, twenty seconds *after* run 7521 (`synchronize`) began. Race 1's
|
||||
mechanism, observed rather than argued.
|
||||
- **A non-cancelling concurrency group — #706's own proposal — does nothing.** With it active, runs
|
||||
7528/7529 still overlapped; 7528 ended 36s after 7529 started. Refuted, not declined.
|
||||
- **The control that saved it.** A first probe *with* a group showed cancellations, which looked like
|
||||
confirmation. The identical workflow with **no `concurrency:` key at all** cancelled the same way:
|
||||
Gitea auto-cancels superseded **`push`** runs by itself, and that does not extend to
|
||||
`pull_request_target`. Without the control, a no-op would have shipped as a solution.
|
||||
- **`cancel-in-progress: true` is deliberately untried** — cancellation is precisely what this
|
||||
workflow's header refuses, since a cancelled run leaves an exempt PR statusless with nothing to
|
||||
re-trigger it.
|
||||
|
||||
## Why the count, and why abstaining is safe
|
||||
|
||||
The timeline records each retarget as a `change_target_branch` event. Verified on the route-1
|
||||
reproduction PR #703 (exactly two: `main → probe698/base-S` and back) against PR #717 as a
|
||||
zero-control. The branch *name* aliases under `main → S → main`; the count cannot.
|
||||
|
||||
The standing objection to refuse-on-motion is that it strands the PR — fatal for `updated_at`,
|
||||
harmless here, and not by degree: a retarget **always** fires `edited`, so the abstaining run is
|
||||
guaranteed a successor. It defers rather than declines.
|
||||
|
||||
## What cold review caught (both easy to reintroduce)
|
||||
|
||||
**The mark must be taken BEFORE the last-moment re-read, not merely before the POST.** "As late as
|
||||
possible" is the safer-sounding instinct and is the opposite: a verdict landing between the re-read
|
||||
and a late mark is invisible to the re-read (already done) *and* excluded from the post-write check
|
||||
(id below a mark taken afterwards) — a gap spanning the whole retarget re-count, while the change
|
||||
claimed one round-trip. Early costs nothing, since `id > mark` hides pre-existing rows either way.
|
||||
Pinned structurally, as an order not an output: with the mark late the job still posts and still
|
||||
repairs in every scenario a stub can pose, and only the arithmetic silently changes.
|
||||
|
||||
**The repair must be a FIXED POINT or it merely decays more slowly.** A repaired status is a machine
|
||||
`pending`, indistinguishable to the next run — which re-derived it and posted `success` again. The
|
||||
description is now a sentinel no exemption is granted over *and* is re-written verbatim by every later
|
||||
run: the first attempt refused the exemption but wrote the GENERIC pending text, erasing its own
|
||||
marker, so the exemption returned two events later instead of one. Only a re-posted verdict clears it.
|
||||
|
||||
## What is NOT closed
|
||||
|
||||
1. A retarget between the final timeline read and the POST. Gitea's status API has no conditional
|
||||
write, so without compare-and-set this cannot reach zero. The magnitude changed: a *permanent*
|
||||
forged green became a *transient* one of about one round-trip, and that retarget still fires
|
||||
`edited`, so a later run re-derives it.
|
||||
2. The repair is itself a read-then-write and can be raced; it fails toward `pending`. A transport
|
||||
failure on its POST is retried once then fails the job loudly. A human re-posting a BASE-MISMATCHED
|
||||
verdict after a repair does bury the sentinel — that needs a user credential, so it is #697's.
|
||||
3. **No vocabulary tripwire.** If an upgrade renames `change_target_branch` or drops it, both counts
|
||||
read `0`, compare equal, are "trusted", and the protection evaporates silently. Accepted (an
|
||||
analogue of the jq `--expect` pin needs a live fixture PR), recorded so the silence is chosen.
|
||||
4. **A timeline over the 20-page cap can never be exempted** — `rt_ok` stays `no` on every run, so only
|
||||
a human verdict clears it and comment-flooding becomes a fail-closed denial of exemption.
|
||||
Negligible at 1000 events; the log says so rather than promising a later run will fix it.
|
||||
|
||||
**CORRECTION, 2026-08-06 (ersatztv#751).** Residual 4 above described as a narrow edge case what was
|
||||
in fact the universal behaviour: `rt_ok` stayed `no` on **every** pull request, not only over-cap ones,
|
||||
so the fence withheld **every** exemption `success` from the day it shipped. A page past the end of
|
||||
this endpoint is the JSON value `null`, not `[]` (measured at Gitea 1.27.1 on PR #752; the same
|
||||
instance returns `[]` for an empty `/issues/{n}/comments`, so it is not consistent between endpoints).
|
||||
`count_retargets` gated on `type == "array"` and therefore read the real terminator as unreadable,
|
||||
never reaching the validated empty page it required. Renovate and docs-only PRs got no status at all.
|
||||
|
||||
Two reasons it read as deliberate rather than broken, both worth carrying forward:
|
||||
|
||||
- **It never ran.** This fence shipped in 8f6d4f443 — the same commit whose prose comment stopped the
|
||||
classify step from executing at all (`ci.workflow-run-body-no-expressions`). Merging a guard and
|
||||
first executing it are different events, and only the second tells you anything.
|
||||
- **The double asserted the wrong shape while claiming to be measured.** The stub's comment read "Real
|
||||
shapes, measured on this instance and deliberately mirrored" and it printed `[]` past the end. So the
|
||||
`array`-only gate was never exercised by the suite either. Correcting the double and restoring the
|
||||
old gate reddens most of the fence suite — 18 tests when first measured, 21 once three more
|
||||
fence-dependent tests existed. The invariant, not the number, is that every one of them had been
|
||||
green for the wrong reason. A fidelity claim in a test double is an assertion, and it decays like any
|
||||
other.
|
||||
|
||||
The type is now read as a value (`case` over `jq -r 'type'`) rather than through `jq -e`, whose
|
||||
exit-status semantics already bit this workflow at jq 1.6 (`ci.jq-version-contract`), and both `null`
|
||||
and `[]` terminate the walk. `test_the_fence_TRUSTS_the_count_and_POSTS_when_the_timeline_terminates`
|
||||
is parameterised over both shapes and asserts the POSTED STATUS rather than the log line — on the real
|
||||
probe run the log said `Decision: state=success` and the job still posted nothing, so the decision and
|
||||
the write are separate events and only the write is what a merge reads.
|
||||
@@ -0,0 +1,105 @@
|
||||
---
|
||||
key: ci.workflow-run-body-no-expressions
|
||||
title: '2026-08-06 — an expression delimiter anywhere in a `run:` body, INCLUDING in a comment, silently drops the step and reports the job green (#751)'
|
||||
status: active
|
||||
since: '2026-08-06'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A `run:` body is not shell when the runner reads it: the runner scans the whole scalar for the expression opener and, on finding one, rewrites the ENTIRE body into a single `format(...)` call. That rewrite is all-or-nothing, so a payload that does not evaluate fails the interpolation of the whole scalar — and the runner then DROPS THE STEP AND CONCLUDES THE JOB `success`. A shell comment is therefore NOT inert. In `.gitea/workflows/review-verdict.yml` no expression delimiter may appear in ANY `run:` body, in code or in prose, because a dropped step there is a dead merge gate rather than a failed build; pass values in through the step''s `env:` block, which is interpolated per value so a bad payload cannot take the body with it, and describe an expression in prose by NAMING it (`a github.event.pull_request.number expression`) rather than quoting the delimiters. Repo-wide the rule is weaker and its reach must be stated precisely rather than generously: every expression payload in every workflow field must have a HEAD TOKEN naming a context or function the runner can resolve. That catches the defect above and a nonexistent context; it does NOT catch a syntactically invalid payload whose tokens are all known (`${{ github.ref == }}`), a renamed output (every token after the first is skipped), or an unclosed opener — those need an expression parser, and the guard is kept permissive on purpose because a red here blocks every merge through the combined status. In `review-verdict.yml` specifically, any step whose non-execution is consequential is paired with a start-marker guard that FAILS the job when the marker is absent, and that guard''s own body must be expression-free — a guard the guarded mechanism can silently delete is worse than none. That pairing now also covers `docker-build.yml`''s `test` and `migrations` jobs, where a dropped step is fail-OPEN (the required check goes green having done no work) rather than fail-closed as it is here — see `ci.required-job-step-execution-markers`, which adds per-STEP markers there and extends this file''s delimiter ban to those two jobs. It is still not a repo-wide property, but the remaining exceptions are narrower than this record originally said: `build` was brought into the ban too (its `Smoke + IPTV E2E` runs AFTER the image is pushed, so a drop there ships an unsmoked release candidate — its two payloads moved to `env:`, so the ban was free), leaving only `api-docs` and `format`, whose one `github.base_ref` each sits in a detect step that gates nothing that ships.'
|
||||
signals: 'Unable to interpolate expression format(, step never ran but job green, missing Run Main step marker, review-verdict/h10 absent after a green run, docs-only PR unmergeable, Renovate PR unmergeable, exemption stopped working, expression in a shell comment, workflow comment changed behaviour · paths: `.gitea/workflows/review-verdict.yml`, `scripts/tests/test_pr_changed_files.py` · issues: #751, #706, #748'
|
||||
mechanics: '`RAN_MARKER` written at the top of the classify step and asserted by the `Assert the classifier actually executed` step (`if: always()`, expression-free body, `exit 1` on a missing marker); static guards `test_the_verdict_workflow_has_NO_expression_delimiter_in_any_run_body` (raw scalar, absolute, gate file only) and `test_every_workflow_expression_names_a_REAL_context_or_function` (repo-wide, allow-list of contexts/functions) plus `test_a_dropped_classify_step_FAILS_the_job_instead_of_going_green` (pins marker path agreement and guard ordering across ONE yaml parse)'
|
||||
---
|
||||
|
||||
**How it happened, which is the part that generalises.** The #706 note explaining why a concurrency
|
||||
group does not work in `review-verdict.yml` quoted a `concurrency:` snippet containing a PR-number
|
||||
expression *as an illustration*, inside a shell comment. `pr number` is not a valid expression. From
|
||||
8f6d4f443 (2026-08-03) to 2026-08-06 the classify step therefore never ran, `review-verdict/h10` was
|
||||
posted by nothing but a human hand, and both exemption classes silently stopped working — while every
|
||||
run reported success. The prose documenting a fix disabled the fix.
|
||||
|
||||
**Why nothing caught it.** Every pre-existing workflow-shape test in
|
||||
`scripts/tests/test_pr_changed_files.py` reads `_code_lines()`, which strips comment lines. That is
|
||||
correct for what it was for — its own docstring notes that prose legitimately discusses
|
||||
`pulls/N/files`, and a raw scan would redden the repo over a piece of writing — but it encodes the
|
||||
assumption this bug falsifies: that a comment in a workflow cannot change behaviour. Inside a `run:`
|
||||
scalar it can. The strict test added here reads the RAW scalar for exactly that reason and must never
|
||||
adopt `_code_lines`.
|
||||
|
||||
**The silent green is the defect; the delimiter was only the trigger.** An absent required status
|
||||
reads as "not reviewed yet" on an ordinary PR, which is indistinguishable from the correct pending
|
||||
state — so a normal PR looked normal while the gate was dead. The visible cost landed on the two
|
||||
classes with no human in the loop: PR #739 (docs-only) merged 2026-08-05 with ZERO commit statuses on
|
||||
its head, and got in only because admin force-merge was still enabled. #743 removed that escape the
|
||||
next day, so by the time this was found the workaround that had been absorbing the bug was gone and
|
||||
the next docs-only or Renovate-manifest PR would have been permanently stuck. The two Renovate PRs in
|
||||
the window escaped by timing alone, merging minutes before the bad commit landed.
|
||||
|
||||
**Scope of the strict rule, and why it is not repo-wide.** As of 2026-08-06, `docker-build.yml`,
|
||||
`ci-image.yml` and `pr-checks.yml` interpolated into `run:` bodies legitimately (7 occurrences then;
|
||||
#756 removed `build`'s two, leaving 5 today — see below). A repo-wide ban would be
|
||||
false and would be deleted the first time it got in someone's way. `review-verdict.yml` earns the
|
||||
absolute rule on two counts: it writes the branch-protection-required status, and its `run:` bodies
|
||||
are ~700 lines of dense prose — the only place the delimiter has ever appeared by accident.
|
||||
|
||||
The first of those two counts turned out to apply elsewhere as well, and #756 acted on it: the
|
||||
absolute ban now also covers `docker-build.yml`'s `test` and `migrations` jobs, which write the other
|
||||
two required contexts and were delimiter-free already, so the rule cost nothing to impose there. The
|
||||
remaining 2 occurrences inside `docker-build.yml` — `api-docs` and `format`, one `github.base_ref`
|
||||
each — sit in jobs that gate nothing that ships (5 repo-wide, counting `ci-image.yml` and the two
|
||||
`pr-checks.yml` gates). `build` is banned too, and NOT because it is required (it is not): its
|
||||
`Smoke + IPTV E2E` step runs after the image is pushed, so a drop there publishes a release candidate
|
||||
that was never booted. Read this paragraph as scoping the rule to steps whose non-execution is
|
||||
CONSEQUENTIAL — required contexts and the release path — rather than to this one file.
|
||||
|
||||
**The probe found a SECOND, independent reason the gate posted nothing**, and it is why fixing the
|
||||
interpolation alone would not have restored the exemptions: a page past the end of
|
||||
`/issues/{n}/timeline` is JSON `null`, not `[]`, so the retarget fence never trusted its count for ANY
|
||||
PR and withheld every exemption `success`. Corrected in `ci.verdict-write-retarget-fence`, whose stated
|
||||
residual had described that universal behaviour as a narrow over-cap edge case. Both defects shipped in
|
||||
the same commit, which is the general lesson: a guard that has never executed has told you nothing, and
|
||||
merging it is not executing it.
|
||||
|
||||
**A THIRD instance of the same server behaviour was found by cold review of this fix**, and it is
|
||||
the reason to distrust "I fixed the two I could see". `GET /commits/{sha}/status` also returns
|
||||
`statuses: null` — not `[]` — for a head with no statuses yet (measured on PR #739's head 5fa672e2:
|
||||
`{"state":"pending","total_count":0,"statuses":null}`). `read_existing_verdict` gated on
|
||||
`.statuses | type == "array"` and took its `exit 1` path, posting nothing: fail-closed, but the same
|
||||
user-visible outcome again. Its double printed `{"statuses": []}` at all three no-verdict sites, so
|
||||
that branch was unreachable in the suite; correcting the double and restoring the old gate turns 40+
|
||||
tests red. `scripts/pr-changed-files.sh` was swept too and is unaffected — `pulls/{n}/files` returns
|
||||
`[]`. The generalisable rule is that a nil Go slice serialises to `null`, so EVERY list-shaped field
|
||||
on this API is suspect, and a per-endpoint measurement is the only way to know.
|
||||
|
||||
**Restoring the exemptions restores a hole that had been dead**, and this is worth saying rather than
|
||||
presenting the change as pure repair. `DOCS_ONLY` matched `CLAUDE.md` and `AGENTS.md`, the documents
|
||||
that define the completion protocol and the H10 rule itself — so those were auto-exemptible while
|
||||
`.claude/` was protected, which is the same self-exemption the workflow header rules out, one
|
||||
directory over. Reachable only because exemptions work again, hence fixed here (both added to
|
||||
`PROTECTED`; see `ci.exemption-provenance`). For the same reason, #706's known residual — the
|
||||
sub-round-trip ABA window, "narrowed and observable, not closed" — comes back with the working fence:
|
||||
while `rt_ok` was never `yes`, route 1 was closed by accident.
|
||||
|
||||
**Three guards were proposed or written for the same hole and the first two were no-ops** — the hole
|
||||
being that concluding "no verdict exists" is what licenses posting over one. `total_count` is per-PAGE
|
||||
here (`?limit=1` on a 6-context head gives `len=1, total_count=1`), so length-vs-total is equal by
|
||||
construction; and "refuse on a full page at `limit=100`" was DEAD CODE, because the instance caps
|
||||
`limit` at `MAX_RESPONSE_ITEMS`, measured at 50 — a cap this repo already documented in three places
|
||||
before the guard was written against 100. The working version asks the server: read page 2 when the row
|
||||
is absent from page 1, and refuse if it carries anything. Cap-independent, so no reconfiguration
|
||||
re-breaks it. Second, `jq -r` renders the number `0` and the string `"0"` identically, so the zero
|
||||
check requires the JSON type as well. Neither was a live failure — both are the difference between a
|
||||
guard that holds because the input happens to be well-formed and one that holds because it checks.
|
||||
|
||||
**The tests written to close a review finding then needed closing themselves**, which is the honest
|
||||
shape of work on this file. The behavioural guard test first extracted the two marker lines by text and
|
||||
ran them alone — which passes even if the write is moved into a function nobody calls. It now executes
|
||||
the classify body's real PREFIX down to the write, reproducing the production control flow instead of a
|
||||
reconstruction of it. The anti-vacuity check first hand-counted `run:` keys with a regex, which
|
||||
false-redded legal spellings (`- run: |`, a single-line `run: echo ok`) and could count a `run: |`
|
||||
inside a heredoc; hand-parsing YAML to validate a YAML parse is the wrong shape, so it now asserts on
|
||||
content — the walk reached at least three bodies and one over 5000 characters.
|
||||
|
||||
**Verified by mutation, not by a green suite.** All six mutations produce a red and the restored tree
|
||||
is green: reintroducing the exact defect (caught by both the strict and the general test), deleting
|
||||
the guard step, deleting only the marker write, weakening `if: always()`, turning the guard's
|
||||
`exit 1` into `exit 0`, and putting a delimiter in the guard's own body.
|
||||
@@ -5,8 +5,8 @@ status: active
|
||||
since: '2026-07-26'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The corpus''s size signal is a per-record prose ceiling (`decisions_validate.py --record-ceiling`, default 60, chosen at a natural gap in the distribution), reported as a NON-BLOCKING `::warning::` naming each record over it. The aggregate prose total is still printed every run but carries NO threshold — it is a `::notice::` trend only — because a total over a monotonically growing corpus can only ratchet, and the generated catalog (`docs/decisions/README.md`) is no longer counted at all since it gains one row per record and cannot be consolidated away. Being listed by the ceiling is an invitation to check for REDUNDANCY, never an instruction to cut: a long record that is all distinct findings is a legitimate decline, and should be recorded as one.'
|
||||
signals: 'aggregate active-corpus budget, schedule a consolidation warning, permanently red ratchet, per-record ceiling, 60-line prose ceiling, generated catalog counted in budget, corpus consolidation has no owner, size is not redundancy · paths: `scripts/decisions_validate.py`, `scripts/tests/test_decisions_validate.py`, `docs/ci-cd.md` · issues: #620, #610, #603, #542, #520'
|
||||
rule: 'The corpus''s size signal is a per-record prose ceiling (`decisions_validate.py --record-ceiling`, default 60, chosen at a natural gap in the distribution), reported as a NON-BLOCKING `::warning::` naming each record over it. The aggregate prose total is still printed every run but carries NO threshold — it is a `::notice::` trend only — because a total over a monotonically growing corpus can only ratchet, and the generated catalog (`docs/decisions/README.md`) is no longer counted at all since it gains one row per record and cannot be consolidated away. Being listed by the ceiling is an invitation to check for REDUNDANCY, never an instruction to cut: a long record that is all distinct findings is a legitimate decline, and should be recorded as one. The ceiling''s CALIBRATION is guarded in two pieces of different robustness (#688): the blocking test asserts only the coarse, non-ratcheting property that the ceiling flags a MEANINGFUL MINORITY of records (`0.02 <= fraction_over <= 0.25`), while the fine claim — that it sits between p90 and p95 — is REPORTED by `main()` as a `::notice::` and never asserted against the live corpus. A ceiling drifting out of date is the passage of corpus growth, not a defect in the commit under test, so it gets `stale_records`'' treatment rather than a red in the blocking `script-tests` job.'
|
||||
signals: 'aggregate active-corpus budget, schedule a consolidation warning, permanently red ratchet, per-record ceiling, 60-line prose ceiling, generated catalog counted in budget, corpus consolidation has no owner, size is not redundancy, ceiling calibration reddens script-tests, adding a record fails CI on length, p90 sits on the ceiling, tail-boundary drift notice · paths: `scripts/decisions_validate.py`, `scripts/tests/test_decisions_validate.py`, `docs/ci-cd.md` · issues: #688, #620, #610, #603, #542, #520'
|
||||
mechanics: '`scripts/decisions_validate.py` -> `oversized_records` / `_budget_total`; `docs/ci-cd.md` -> "`decisions-guard` job"'
|
||||
---
|
||||
|
||||
@@ -46,19 +46,49 @@ meant to prevent exactly that could not see it: `max(under) <= 60 < min(over)` i
|
||||
construction** of the two lists it builds, and passes on a distribution with no gap at all. A
|
||||
rationale-guarding test that cannot fail is worse than none, because it launders the claim.
|
||||
|
||||
It is replaced by `test_real_corpus_ceiling_sits_at_the_TAIL_BOUNDARY_of_the_distribution`, which
|
||||
states the property directly and scale-free: **the ceiling sits between the 90th and 95th percentile
|
||||
of record lengths** — that is what "marks the start of the tail" means — and reads the value from
|
||||
`RECORD_CEILING_DEFAULT` so test and CLI cannot drift.
|
||||
It is replaced by `test_real_corpus_ceiling_flags_a_nonempty_proper_minority`, which asserts that the
|
||||
ceiling flags a meaningful minority of records (`0.02 <= fraction_over <= 0.25`) and reads the value
|
||||
from `RECORD_CEILING_DEFAULT`, so test and CLI cannot drift.
|
||||
|
||||
Getting there took four versions, and the failures are the useful part:
|
||||
Getting there took five versions, and the failures are the useful part:
|
||||
|
||||
| | assertion | why it failed |
|
||||
|---|---|---|
|
||||
| v1 | `max(under) <= 60 < min(over)` | true **by construction** of those two lists |
|
||||
| v2 | a minimum gap WIDTH | a ceiling of 200 also sits in a wide gap — it passed |
|
||||
| v3 | 2-12% fraction band + "clear air" above | **hostage to an unrelated record**: one ordinary 62-line addition reddened it with the ceiling correctly placed, and the only remedy was to RAISE the ceiling — this very treadmill, as a hard failure in what #631 makes a blocking job. The fraction band had the same coupling more slowly (12 more long records breached it), and `0 <= headroom` was vacuous. |
|
||||
| v4 | `p90 <= ceiling <= p95` | percentiles move WITH the corpus, so routine growth cannot ratchet it; it fires only when the ceiling genuinely stops marking the tail |
|
||||
| v4 | `p90 <= ceiling <= p95` | percentiles move with the corpus, but an order statistic over a SPARSE distribution is a STEP function. The lengths climb to the ceiling and then jump straight to 81 with NOTHING in between (measured; the multiplicities move with every record added, the gap is the point), so ONE record can move p90 by twenty-one lines (that is today's gap; the #672 event moved it less and still reddened CI). It reddened the blocking job twice live (#672, #706), and both times the only in-scope remedy was to trim the new record to fit the constant — the v3 ratchet, pointed at record authors |
|
||||
| v5 | coarse `0.02 <= fraction_over <= 0.25` asserted; fine `p90 <= ceiling <= p95` REPORTED | splits the claim by robustness instead of hunting for a better single assertion (#688) |
|
||||
|
||||
**v5 is not a fifth attempt at the same shape — it stops trying.** Four versions failed because they
|
||||
all asserted, in the blocking job, a property of a corpus the commit under test does not control.
|
||||
The fine claim is genuinely useful and genuinely fragile, so it is now measured on every run and
|
||||
printed as a `::notice::` — the same treatment `stale_records` gets, and for the same stated reason:
|
||||
a constant going out of date is the passage of time, not a defect in this change. What stays
|
||||
blocking is only what no SINGLE ordinary addition can break — each record moves a fraction by at
|
||||
most 1/N, so from **18/183** over the ceiling it takes **38** consecutive over-ceiling additions to
|
||||
BREACH the 25% cap (37 lands exactly on 0.25, which still passes), against **one** record to break v4.
|
||||
|
||||
**The floor is a fraction, not `> 0`, and review is why.** The first draft of v5 asserted only
|
||||
`0 < fraction_over < 1/3`, which measured against the real corpus accepted **every ceiling from 39
|
||||
to 229** — including the ceiling of 200 the draft itself offered as the case it catches, because a
|
||||
single 230-line record keeps the count nonzero. A bound that a deliberately absurd value satisfies
|
||||
is not a guard. At a 2% floor and a 25% cap the accepted range is **43..180** (measured, contiguous):
|
||||
a ceiling of 200 flags 0.5% of records and is rejected, a ceiling of 20 flags 60% and is rejected,
|
||||
and today's 9.8% sits about 5x ABOVE the floor and 38 over-ceiling additions below the cap.
|
||||
|
||||
**Three arms, and the tightest is CONSOLIDATION** — stated because it is the easy one to forget.
|
||||
Breaching the cap takes 38 over-ceiling additions; diluting below the floor takes 718 short ones;
|
||||
but taking **15** of today's 18 over-ceiling records out of the over-set also drops below it — trimming them to <=60 leaves 3/183 = 1.64%, archiving them leaves 3/168 = 1.79%, since archiving moves the denominator too. That
|
||||
is a real tension with `test_oversized_records_can_go_green`, and it is accepted rather than papered
|
||||
over: at 3/183 the constant genuinely IS mis-calibrated, so the red is the signal working. A
|
||||
consolidation PR big enough to reach it should re-derive the ceiling in the same change.
|
||||
|
||||
The honest cost, stated rather than buried: **nothing now forces a re-derivation.** The ceiling can
|
||||
drift while only a notice complains. That is accepted on the same reasoning this record already
|
||||
applies to its two "keep listed" consolidation candidates — the warning names it on every run, which
|
||||
tracks it better than a red that gets trimmed around, and a red an author can only clear by editing
|
||||
an unrelated constant is not enforcement, it is a toll.
|
||||
|
||||
Two rules came out of that sequence, and they outlive this metric:
|
||||
**a guard test must depend only on the thing it guards**, and
|
||||
@@ -68,7 +98,7 @@ is restated as a self-referential fact** for the same reason: the warning report
|
||||
numbers on every run, and a number frozen in prose is one edit away from being a lie.
|
||||
|
||||
**Size is a proxy for the thing we actually care about, and the proxy is demonstrably wrong.** Of
|
||||
the records over the ceiling, the largest by ~1.6x —
|
||||
the records over the ceiling, the longest —
|
||||
`scan.libraryfolder-unique-identity`, 230 lines — is a dozen-odd **distinct** hard-won traps (MySQL
|
||||
`utf8mb4_bin` PAD SPACE, create-the-composite-index-before-dropping-its-predecessor, clearing the
|
||||
connection pool per MySQL fixture, lazy hash healing that must never abort a scan…). Shortening it
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
---
|
||||
key: docs.frontmatter-pyyaml-crosscheck
|
||||
title: '2026-08-04 — `decisions_validate.py` cross-checks its dependency-free frontmatter parse against PyYAML whenever PyYAML is importable (#674)'
|
||||
status: active
|
||||
since: '2026-08-04'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: '`decisions_validate.py` runs `pyyaml_frontmatter_faults()` over every record-wing file: it loads the frontmatter with PyYAML and reports an ERROR when PyYAML rejects the document OR when any key''s value differs from what the dependency-free `dl._read_frontmatter` read. PyYAML is the WRITER of these files (`migrate_decisions_split.render_record` emits them with `yaml.safe_dump`), so on any disagreement PyYAML is authoritative and the defect is in the FILE, not in either parser. The check is strictly additive: when PyYAML is not importable it is SKIPPED and `main()` says so with a `::notice::`, never silently — the read path stays dependency-free because `decisions-guard`, the Husky hooks and contributor machines install nothing. The comparison has exactly ONE implementation, called by both the validator and `test_frontmatter_reader_matches_pyyaml_on_every_real_record`, so the suite and the tool cannot drift on what "matches PyYAML" means.'
|
||||
signals: 'validator reports OK on a broken record, bare apostrophe in single-quoted frontmatter, unquoted hash truncates a value, hand-rolled frontmatter parser, PyYAML rejects the file but decisions-validate passes, dependency-free read path, frontmatter cross-check skipped · paths: `scripts/decisions_validate.py`, `scripts/decisions_lib.py`, `scripts/tests/test_decisions_validate.py`, `scripts/tests/test_decisions_lib.py` · issues: #674, #578, #651, #621'
|
||||
mechanics: '`scripts/decisions_validate.py` -> `pyyaml_frontmatter_faults` / `_frontmatter_block`, wired into `main()` alongside `record_wing_faults`'
|
||||
---
|
||||
|
||||
The validator read ordinary English prose in a `rule:` field and reported **OK** on a file PyYAML
|
||||
refuses to parse. It was hit **twice in one session by two independent agents** on unrelated
|
||||
branches (#578, #651), which is what makes it a guard rather than a note: it is not an exotic edge
|
||||
case, it is what happens when anyone writes `SQLite's LOWER()` into a single-quoted scalar.
|
||||
|
||||
**Why the hand parser exists, and why it stays.** `dl._read_frontmatter` is deliberately
|
||||
dependency-free — it runs in `decisions-guard`, in the Husky hooks, and on every contributor
|
||||
machine, none of which install anything. Requiring PyYAML there once made the validator crash with
|
||||
`ModuleNotFoundError` on the very records the split had just written. So the fix could not be
|
||||
"import yaml in the reader". It is a second, optional opinion layered on top.
|
||||
|
||||
**The two known hazards fail DIFFERENTLY, and that shaped the fix.**
|
||||
|
||||
| input | dependency-free reader | PyYAML |
|
||||
|---|---|---|
|
||||
| `rule: 'SQLite's LOWER()'` | `SQLite's LOWER()` | **`ParserError`** — the bare apostrophe closes the scalar early |
|
||||
| `rule: use --flag #2` | `use --flag #2` | `use --flag` — ` #` starts a comment, **silently truncating** |
|
||||
|
||||
A `try/except` would have caught only the first row. The second produces no exception at all: a
|
||||
valid record whose `rule` has quietly lost its tail — the `parse-to-WRONG` case
|
||||
`docs.record-wing-parse-guard` explicitly names as the gap its structural check cannot see. So the
|
||||
cross-check compares the parsed **result** key by key, and reports a rejection and a mismatch as two
|
||||
distinct faults with different remedies. The `except` around the load is deliberately broad, not
|
||||
`yaml.YAMLError`: PyYAML's timestamp constructor raises a bare `ValueError` on an impossible date
|
||||
(`stale-after: 2026-06-31`), and an additive check must never be the reason the validator can't run.
|
||||
|
||||
**That is also what makes it general.** #674 asked for a fix that catches the *next* character class
|
||||
rather than enumerating hazards one at a time. Comparing against the writer's own library is that:
|
||||
any construct where the two parsers disagree surfaces as a diff, with nobody having to predict it.
|
||||
|
||||
**Direction is the load-bearing part.** PyYAML is not a second opinion of equal standing — it WROTE
|
||||
these files, so when the two disagree the on-disk bytes mean what PyYAML says, the record is corrupt
|
||||
and the permissive reader is the one hiding it. That is what turns an ambiguous "parsers differ"
|
||||
report into an actionable "this record is silently wrong".
|
||||
|
||||
**A skip is announced, not silent.** When PyYAML is absent the check does not run, which is correct
|
||||
on the dependency-free path — but `main()` prints a `::notice::` saying so. A check that reports
|
||||
success while doing nothing is the defect this corpus keeps re-learning (#603's `stale-after` that
|
||||
never fired, #609's marker that exempted everything while printing OK), and adding a quiet skip
|
||||
while fixing a quiet pass would have reintroduced it one level up.
|
||||
|
||||
**One implementation, two callers.** The comparison already existed — in the test suite only, which
|
||||
is exactly why the validator could disagree with `scripts/tests` about the same file. Rather than
|
||||
leaving a second copy, `test_frontmatter_reader_matches_pyyaml_on_every_real_record` now delegates
|
||||
to `pyyaml_frontmatter_faults`, so the tool and the suite cannot drift on the definition.
|
||||
|
||||
**What this buys, stated precisely, because it is less than it looks.** In CI `decisions-guard`
|
||||
installs nothing, so the validator there always takes the skip path, by design; and `script-tests`
|
||||
already went red on both hazards before this change, and an advisory red still blocks the merge gate
|
||||
(#598). So **no broken record has reached `main` and the CI delta here is close to zero** — though
|
||||
procedurally, not structurally: branch protection on `main` requires exactly three contexts (`Build
|
||||
& test (.NET)`, `EF migration integrity`, `review-verdict/h10`), and NEITHER `script-tests` nor
|
||||
`decisions-guard` is among them. What this fixes is the case #674 described: the LOCAL loop, where
|
||||
the validator is the tool an agent reaches for directly and it printed OK on a corrupt file — plus
|
||||
the tool/suite disagreement, now impossible.
|
||||
|
||||
**Limits, and the positive control.** It does not catch a mis-parse both parsers agree on — strong,
|
||||
not total, the same qualification `record_wing_faults` carries. The suite pins that
|
||||
`record_wing_faults` ALONE still reports both hazard files as clean; without that, the cross-check
|
||||
could be deleted and the tests would stay green while the guard vanished.
|
||||
@@ -58,7 +58,8 @@ mechanics: '`SetRealtimeInput` readrate-burst option; `FFmpegKnownOption.HasOpti
|
||||
session" flag through `FFmpegState`; that complexity was not judged worth a bounded peak.
|
||||
- **Still images are excluded.** Their video input is paced by the realtime *filter* and takes no
|
||||
readrate at all, so a burst would only run the audio input ahead of the video for songs and offline
|
||||
filler, with no cold-start gain to show for it.
|
||||
filler, with no cold-start gain to show for it. (`-readrate_catchup` mirrors this exclusion for the
|
||||
same reason — `ffmpeg.readrate-catchup-sparse-streams`, #726.)
|
||||
- **Non-HLS realtime outputs (`TransportStream`, HLS-Direct) burst too**, since
|
||||
`FFmpegPlaybackSettingsCalculator` makes them unconditionally realtime. That is untested by the
|
||||
benchmark, which was segmenter-only; it is kept because the same first-read throttle delays those
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
---
|
||||
key: ffmpeg.readrate-catchup-sparse-streams
|
||||
title: 2026-08-04 — a realtime input gets `-readrate_catchup`, because `-readrate` paces off its furthest-behind stream (#726)
|
||||
status: active
|
||||
since: '2026-08-04'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'a realtime video/audio input also gets `-readrate_catchup` (6.0) when the binary supports it — but NOT a still-image input (mirroring the #350 exclusion) and NOT a concat input, which keep at most bare `-readrate` (a still image''s video input takes none at all). Reason: `-readrate` paces the whole input off its furthest-behind stream, so a sparse stream sharing that input (an embedded PGS/DVD bitmap subtitle feeding the overlay) otherwise pins output at ~0.53x realtime. Catchup is a ceiling that applies only WHILE an input is behind, never a target, so it does not let a caught-up input race ahead.'
|
||||
signals: 'readrate, readrate_catchup, sparse stream, bitmap subtitle, PGS, DVD subtitle, dvdsub, pgssub, overlay burn-in, Live TV buffering/stalling, "Resumed reading at pts N with rate R after a lag of Ns" · paths: `PipelineBuilderBase.SetRealtimeInput`, `ReadrateInputOption`, `FFmpegKnownOption` · issues: #726, #350, #529'
|
||||
mechanics: '`PipelineBuilderBase.CatchupReadRate` (6.0); `ReadrateInputOption` catchup arg; `FFmpegKnownOption.ReadrateCatchup` capability gate'
|
||||
---
|
||||
|
||||
- **`-readrate` throttles an input, not a stream, and it paces off whichever stream is furthest
|
||||
behind.** An embedded bitmap subtitle is read through the *same* `-i` as the video —
|
||||
`SubtitleInputFile` carries the video's path and `ComplexFilter` resolves it to a stream specifier
|
||||
on that input, and `CommandGenerator` never emits a second `-i` for it. Being sparse, the subtitle
|
||||
stream falls further behind every second and drags the video down with it. FFmpeg says so itself at
|
||||
`-loglevel warning`: `[sist#0:3/dvd_subtitle] Resumed reading at pts 10.400 with rate 6.000 after a
|
||||
lag of 0.922s`, repeating with the lag growing 0.9→3.8 s while `pts` stays pinned (no new packet).
|
||||
- **Measured on prod (QSV, `-threads 1`, `dvd_subtitle`→overlay), 45 s steady-state window after a
|
||||
6 s settle:**
|
||||
|
||||
| variant | throughput |
|
||||
|---|---|
|
||||
| `-readrate 1.05` (baseline) | **0.533x** (×3 runs) |
|
||||
| `+ -readrate_catchup 2.0` | 0.711x |
|
||||
| `+ -readrate_catchup 6.0` | **1.067x** (×2 runs) |
|
||||
| `+ -readrate_catchup 20.0` | 1.067x |
|
||||
| no subtitle overlay (control) | 1.067x |
|
||||
|
||||
A live client consumes at 1.0x, so 0.53x drains its buffer until it stalls — the reported symptom.
|
||||
- **`20.0` measuring the same as `6.0` is why 6.0 was chosen** — above the catch-up point the value
|
||||
is not a throughput dial, so there is nothing to buy by going higher. It is **not** evidence about
|
||||
allocation: that is a steady-state throughput number, not a count of frames in flight.
|
||||
- **Why this does not reopen `ffmpeg.qsv-extra-hw-frames-floor` (#529).** Not because catchup is
|
||||
brief (a permanently GPU-bound channel lags forever, so 6x is a standing licence), and **not**
|
||||
because read rate is allocation-irrelevant — #529 measured that it is not (at `extra_hw_frames=0`,
|
||||
`1.05` without a burst exits 0 while `1.05`+burst hits ENOMEM). Read rate changes how fast frames
|
||||
enter the graph, not how deep its queues are, and #529 showed that only bites when the pool has
|
||||
**no headroom**. The 64-frame floor now guarantees headroom, so the load-bearing measurement is
|
||||
row 5 of that truth table — **no `-readrate` at all with 64 frames → 14 segments, exit 0** — and a
|
||||
6x ceiling is strictly less aggressive than no throttle. Reinforcing it, `-readrate_initial_burst 8`
|
||||
has read *flat out* at the start of every playout item since #350, so an unbounded read here is not
|
||||
new. A 240 s QSV soak (64 frames, 60 segment boundaries) adds 1.043x sustained with **zero**
|
||||
`Cannot allocate memory` — but it stayed largely caught-up, so it corroborates rather than proves;
|
||||
the argument above is what carries the decision.
|
||||
- **Not QSV-specific:** reproduces on libx264 too (0.533x → 1.067x), as expected for an input-pacing
|
||||
option upstream of any encoder or filter choice.
|
||||
- **Raising the base `-readrate` is not an alternative, and was measured:** 2.0→0.62x, 3.0→0.80x,
|
||||
4.0→0.80x, 6.0→0.89x. It asymptotes *below* realtime, because the rate ceiling was never the
|
||||
binding constraint. Recorded so it is not re-proposed.
|
||||
- **Catchup does NOT subsume the #350 burst; they fix orthogonal metrics.** Measured
|
||||
time-to-first-segment: `-readrate` alone 3.71 s, `+burst` **0.72 s**, `+catchup` alone **3.65 s**,
|
||||
both 0.67 s. Catchup buys nothing at cold start (no accumulated lag at t=0 to recover) and the
|
||||
burst buys nothing for throughput (the 0.533x baseline already had it), so removing the burst on
|
||||
the theory that catchup replaces it would regress tune-in ~5x.
|
||||
- **Applied to realtime video/audio inputs generally, not only subtitle pipelines** — it is inert
|
||||
unless an input is behind, and any sparse stream can trigger this, so gating it on "has a bitmap
|
||||
subtitle" would fix the site instead of the boundary. Two deliberate exclusions, both test-pinned:
|
||||
`ConcatInputFile` (reads already-written segments at a flat 1.0, nothing sparse to lag on) and
|
||||
**still images**, mirroring #350 — their video input takes no readrate at all, so catchup would
|
||||
reach only the separate audio input and break the pacing symmetry #350 declined to break. An
|
||||
image-based subtitle always rides the *video* path, so that shape cannot starve this way anyway.
|
||||
- **Capability-gated via `FFmpegKnownOption.HasOption`**, the same fail-safe posture as
|
||||
`-readrate_initial_burst`: detection parses `ffmpeg -h long`, so a binary without the option
|
||||
silently keeps today's behavior instead of failing to start.
|
||||
|
||||
**Accepted residual:** the affected population is items carrying an embedded bitmap subtitle matching
|
||||
the channel's subtitle mode — 3,182 of 24,646 media versions (12.9%) on prod. It is a property of the
|
||||
*item*, not the channel, which is why the stall presented as random: a channel plays one episode fine
|
||||
and stalls on the next.
|
||||
@@ -0,0 +1,104 @@
|
||||
---
|
||||
key: mcp.tool-schema-openapi-parity
|
||||
title: '2026-08-06 — every MCP tool declares exactly its endpoint''s OpenAPI request-body fields and query parameters, asserted in CI (#754, #757)'
|
||||
status: active
|
||||
since: '2026-08-06'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Every POST/PUT/PATCH tool in `ToolCatalog` declares exactly the request-body properties its endpoint accepts, each with a matching type, and EVERY tool (read and write) declares exactly its endpoint''s query parameters, both asserted against the generated `ErsatzTV/wwwroot/openapi/v1.json` (linked into `ErsatzTV.Mcp.Tests`) by `Every_Write_Tool_Should_Declare_Exactly_Its_OpenApi_Request_Body_Fields` and `Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters`. A field the endpoint accepts but the tool omits is a DEFECT, not a deferral: on the full-replace tools (channel update, schedule update, custom-order) the omission is silently applied as a clear. The write tools are NOT uniformly full-replace — add-collection-items is additive, and several leave an omitted field unchanged — so each tool description states its own semantics. An omitted query parameter is UNREACHABLE, not merely undocumented, because `ToolArgumentValidator` rejects undeclared arguments.'
|
||||
signals: 'MCP tool schema drift, full-replace write, silently dropped field, graphicsElementIds, padToNearestMinute, additionalProperties false · paths: `ErsatzTV.Mcp/ToolCatalog.cs`, `ErsatzTV.Mcp.Tests/ToolCatalogTests.cs`, `docs/mcp.md` · issues: #754, #757, #58, #616'
|
||||
mechanics: '`ErsatzTV.Mcp.Tests/ToolCatalogTests.cs`; `ErsatzTV.Mcp.Tests.csproj` links `openapi/v1.json`'
|
||||
---
|
||||
|
||||
`ToolCatalog.ChannelFields()` declared 27 of `UpdateChannelRequest`'s 28 properties. The missing one
|
||||
was `graphicsElementIds`, which attaches channel-level graphics elements including the built-in On
|
||||
Now/Next overlay (`graphics.channel-level-attachment`).
|
||||
|
||||
The cost was not "one field you cannot set". `PUT /api/v1/channels/{id}` is a **full replace**, and
|
||||
the tool's own description instructs the caller to *"send the full desired state"* — which the schema
|
||||
could not express. An agent that faithfully GET-edit-PUT a channel detached every attached graphics
|
||||
element, with a `200` and no error. Nothing surfaced until the overlay stopped rendering at the next
|
||||
transition, hours later. That is the `optional-parameter-on-shared-primitive-is-opt-out` shape: the
|
||||
omission is invisible at the call site and only observable as missing pixels.
|
||||
|
||||
Fixing the one field would have left the mechanism intact, and the mechanism had already produced a
|
||||
second instance: `ScheduleFlags()` omitted `padToNearestMinute`, which both `CreateScheduleRequest`
|
||||
and `UpdateScheduleRequest` carry and `UpdateProgramScheduleHandler` writes unconditionally — so
|
||||
`ersatztv_update_schedule` silently cleared a configured pad the same way. Nothing tied a tool's
|
||||
declared arguments to the contract it wraps, so the next added DTO property would have drifted too.
|
||||
|
||||
So the guard is the decision, and it is asserted against the **generated OpenAPI document** rather
|
||||
than the DTO types: `v1.json` is the actual wire contract, it is already regenerated by
|
||||
`scripts/update-openapi.sh` as part of the API checklist, and asserting against it keeps
|
||||
`ErsatzTV.Mcp.Tests` free of a project reference to the whole ASP.NET host. The test derives each
|
||||
tool's body set exactly as `ErsatzTvApiClient` does — declared arguments minus path parameters, minus
|
||||
query parameters, minus the reserved `ifMatch` header — so the guard cannot disagree with the routing
|
||||
it guards.
|
||||
|
||||
Three anti-vacuity properties are deliberate, per the repo's standing "a test that filters on the
|
||||
property it asserts cannot see what is missing" rule:
|
||||
|
||||
- The **covered write-tool set is pinned by name**, not merely filtered. A tool that stops being a
|
||||
write verb, or a new one that is added, changes this list rather than silently leaving the loop.
|
||||
- A **missing or unrecognised spec is a failure**, never an empty comparison: an absent `v1.json`
|
||||
fails with the path it looked in, and a request body that is not a plain `$ref` (an `allOf`,
|
||||
`oneOf`, or inline schema), or a property whose type is a union this guard has not been taught,
|
||||
fails asking to be taught the shape instead of comparing against `{}`.
|
||||
- **Names are compared with types**, not alone. A name-only guard is the same defect one level down:
|
||||
the tool would advertise `string` for an `int?`, the agent would send `"30"`, and the API would
|
||||
reject it — green test, broken tool. The generator's `["null", T]` nullable form and its `$ref`
|
||||
(enum → `string`, model → `object`) are normalized onto the catalog's vocabulary, arrays down to
|
||||
their element type.
|
||||
|
||||
All were verified by mutation rather than assumed: dropping `graphicsElementIds`, dropping
|
||||
`padToNearestMinute`, retyping either field, drifting an array's element type, and removing the
|
||||
copied spec each turn the suite red, and each failure names the field or path at fault.
|
||||
|
||||
**Query parameters are guarded the same way, across every tool (#757).** A second test compares each
|
||||
tool's routed `QueryParameters` against the spec's `parameters[in=query]` for its path and verb, reads
|
||||
included — the drift that existed when this was written was entirely on reads. An omitted parameter
|
||||
there is worse than an undeclared body field: `additionalProperties:false` means the caller cannot
|
||||
pass it *at all*, so the capability is unreachable rather than merely undocumented (`ersatztv_list_playouts`
|
||||
had lost its channel-name `query` filter and `ersatztv_get_playout_items` its `showFiller`; #616 was
|
||||
the same shape with paging). That test **accumulates** its mismatches and asserts once, so a run
|
||||
reports the whole drift set — failing on the first would invite fixing one tool at a time, which is
|
||||
how the twin in this very issue stayed hidden.
|
||||
|
||||
It also **composes with** the older `Every_Query_Parameter_Should_Be_A_Declared_Property`, and the pair
|
||||
is the clearest illustration in this repo of why "a test that filters on the property it asserts cannot
|
||||
see what is missing" is a rule. That older test filters `Where(t => t.QueryParameters is { Count: > 0 })`
|
||||
— so a tool that lost its query parameters entirely escaped it, which is exactly how `list_playouts` and
|
||||
`get_playout_items` hid. The new test has no filter and reports them as *unreachable*; the old one then
|
||||
checks that a routed parameter is also a declared argument. Neither subsumes the other, and the inner
|
||||
duplicate of the old check was deliberately removed from the new test rather than kept as a second copy.
|
||||
|
||||
**Scope, stated so it is not mistaken for more.** Request bodies are compared for POST/PUT/PATCH only.
|
||||
DELETE is uncovered because `ErsatzTvApiClient` builds a body for POST/PUT/PATCH only, so a body
|
||||
argument on a DELETE tool would be silently dropped; no tool has one today. Header arguments (`ifMatch`)
|
||||
and per-parameter *descriptions* are not compared either — `api.paging-zero-based` is pinned by its own
|
||||
test.
|
||||
|
||||
The type comparison is **lossy by design, at the catalog's ceiling**: the catalog's vocabulary is
|
||||
`{string, integer, number, boolean, object, array<T>}`, so every object component collapses to `object`
|
||||
and every enum to `string`. Swapping one model or enum for another is therefore invisible here
|
||||
(verified by repointing `logo` at a structurally unrelated model — the suite stays green), as is
|
||||
`format` (`int32` vs `int64`). That is the right ceiling rather than a gap to close: comparing deeper
|
||||
than the catalog can express would assert a distinction no tool schema carries, and an opaque object
|
||||
like `logo` is copied through from a GET verbatim, so nested drift cannot cause the silent-clear this
|
||||
record exists to prevent. `integer` vs `number` IS distinguished. The `>1` non-null type-union
|
||||
assertion is a fail-loud guard for a shape this generator does not currently emit, so it is deliberate
|
||||
but **unexercised**.
|
||||
|
||||
The guard is also a **two-job conjunction**, not self-contained: it compares against a checked-in
|
||||
`v1.json`, so it is only as fresh as the regeneration. What keeps it honest is the `api-docs` CI job,
|
||||
whose `^ErsatzTV/Controllers/Api/` path filter covers the directory every request DTO lives in — a
|
||||
new DTO property cannot leave `v1.json` stale without that job going red. That holds for a DTO's OWN
|
||||
properties and no further: a NESTED model such as `ArtworkContentTypeModel` lives in
|
||||
`ErsatzTV.Application/Artworks/`, outside that filter, so changing it can leave `v1.json` stale without
|
||||
the job firing. Pre-existing, and harmless to this guard only because nested shape is not compared.
|
||||
|
||||
`graphicsElementIds` is declared on the **update tool only**, not in the shared `ChannelFields()`:
|
||||
`CreateChannelRequest` has no such property, and the tool schemas are `additionalProperties:false`,
|
||||
so sharing it would make every create call send an unknown property. `padToNearestMinute` is on both
|
||||
schedule requests, so it does belong in the shared `ScheduleFlags()`. The parity test is what makes
|
||||
that per-field placement checkable rather than a matter of care.
|
||||
@@ -5,8 +5,8 @@ status: active
|
||||
since: '2026-07-21'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: Apply the `in-progress` label before starting an issue, and still read its dependency notes before touching shared surfaces — a claim prevents duplicate pickup, not overlapping code changes.
|
||||
signals: 'parallel sessions · in-progress label · claim race · dependency notes · shared surfaces · lore pruning · paths: `docs/handoffs/chicorytv-issue-queue.md` · issues: #542'
|
||||
rule: 'Before starting an issue, check for an existing claim four ways — open PRs referencing it, remote branches naming it, recent comments (a claim can precede the label), and a fresh `git fetch origin main` — then claim with the `in-progress` label plus a comment. A claim prevents duplicate PICKUP, not duplicate WORK. Re-fetch `origin/main` before every push, not only at branch time.'
|
||||
signals: 'parallel sessions · in-progress label · claim race · duplicate implementation · stale base · branch reverts merged work · dependency notes · shared surfaces · lore pruning · paths: `docs/handoffs/chicorytv-issue-queue.md` · issues: #542, #649, #666'
|
||||
mechanics: '`in-progress` label on the Gitea issue. The tiny read→claim race window is accepted; the later claimant backs off. Runner topology: two runners (ci-runner VM 127 + bumblebee-runner), 4 slots total.'
|
||||
---
|
||||
|
||||
@@ -17,3 +17,32 @@ after #231", "coordinate with #215").
|
||||
When editing the standing lore/handoff doc, prune covered and stale bullets rather than appending — it
|
||||
is not append-only, and git keeps the history. `git pull --rebase` before committing it, since it is
|
||||
the single most contended file across parallel sessions.
|
||||
|
||||
## The label is not the check (ersatztv#649, 2026-07-26)
|
||||
|
||||
#649 was implemented **twice, in parallel, to completion**. One session had labelled it `in-progress`
|
||||
and was three commits and four review rounds deep when a reviewer noticed `origin/main` had moved ten
|
||||
commits: the other session had already merged the same work as PR #666. The duplicate branch was
|
||||
discarded — pushing it would have reverted #666 *and* #667, showing the merged work as deletions
|
||||
because its diff was computed against a stale base.
|
||||
|
||||
Two distinct failures, both now covered by the kickoff's step 3:
|
||||
|
||||
1. **The claim was made, and was insufficient.** The other session was presumably already underway
|
||||
when the label went on. A label answers "has anyone announced this?", not "is anyone doing this?"
|
||||
The cheap proxies for the second question are an open PR whose body says `fixes #N`, a remote
|
||||
branch with the number in it, and a claiming *comment* that predates the label — which is exactly
|
||||
the `CLAIM?` flag `scripts/select-queue.sh` already raises and deliberately does not resolve.
|
||||
|
||||
2. **The base went stale and nothing re-checked it.** `origin/main` was read once, at branch time,
|
||||
and not again across many hours. The tell is a `git diff origin/main` that shows deletions you did
|
||||
not make. Re-fetch before every push; rebase (never merge main in) when it has moved.
|
||||
|
||||
Neither session did anything wrong at the moment of claiming. The lesson is that the *duration* of a
|
||||
session is the risk: the longer a branch lives, the more the "I checked at the start" evidence decays.
|
||||
|
||||
Worth noting what worked: the duplicate effort was not wasted. The merged implementation was better in
|
||||
one respect (it exports `ETV_GITEA_URL` as well as `GITEA_BASE_URL`, because `pr-changed-files.sh`
|
||||
reads the former at higher precedence), and the discarded branch's test coverage was salvaged onto the
|
||||
merged code as an additive tests-only PR. When you discover a collision, diff the two implementations
|
||||
before throwing yours away — the loser usually contains something the winner lacks.
|
||||
|
||||
@@ -5,8 +5,8 @@ status: active
|
||||
since: '2026-07-12'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: A blocking `format` CI job runs `dotnet format --verify-no-changes` scoped only to the PR's changed `.cs` files (never the legacy BOM backlog), and a PR branch must be kept current by rebasing on `origin/main` (never merging main in), enforced by `.husky/pre-push` → `prepush-rebase-check.sh`.
|
||||
signals: 'format-as-you-touch, rebase not merge, BOM backlog · paths: `.husky/pre-push`, `.claude/hooks/prepush-rebase-check.sh` · issues: #311 (H11), #309, #310, #269, #312'
|
||||
rule: 'A blocking `format` CI job runs `dotnet format --verify-no-changes` scoped only to the PR''s changed `.cs` files (never the legacy BOM backlog), and a PR branch must be kept current by rebasing on `origin/main` (never merging main in), enforced by `.husky/pre-push` → `prepush-rebase-check.sh`. H11 has ONE always-on carve-out, #719 — a push in which EVERY ref is under `refs/tags/` skips the freshness check, because a tag push cannot revert merged work, which is the failure mode H11 exists to prevent, and the release cut tags from a branch that is behind `origin/main` (observed on the v26.13.0 cut, #719). A push mixing branch and tag refs is still blocked, and so is a push with zero parsed ref lines (the exemption requires at least one, so empty stdin cannot vacuously disable H11).'
|
||||
signals: 'format-as-you-touch, rebase not merge, BOM backlog, tag-only push exemption, H11 blocks release cut, refs/tags pre-push, vacuous-truth guard · paths: `.husky/pre-push`, `.claude/hooks/prepush-rebase-check.sh`, `scripts/tests/test_prepush_rebase_check_tag_exemption.py` · issues: #311 (H11), #719, #309, #310, #269, #312'
|
||||
mechanics: '`docs/contributing.md` §7; `.claude/hooks/prepush-rebase-check.sh`; `npm run check:api`'
|
||||
---
|
||||
|
||||
@@ -36,5 +36,22 @@ git hook has no "ask"); deliberate escape `ETV_SKIP_REBASE_CHECK=1`. This supers
|
||||
guidance to "merge main into your PR branch." (After a rebase that conflicts in *generated* artifacts —
|
||||
v1.json/v1.d.ts/endpoint-index — regenerate, don't hand-resolve; `npm run check:api` guards.)
|
||||
|
||||
**2a. The tag-only carve-out (#719).** H11 fired on the release cut: tagging a commit on `main` from
|
||||
a branch that is behind `origin/main` tripped the freshness check, and the rebase advice it printed
|
||||
did not even apply — no branch was being pushed. Observed while cutting `v26.13.0` (#719); note
|
||||
`docs/ci-cd.md` → "Cutting a release" documents the tag step itself, not the release-notes-PR flow
|
||||
that leaves the branch behind, so the frequency is attested by #719 rather than by that doc. The hook now reads git's pre-push ref lines (`<local ref> <local sha>
|
||||
<remote ref> <remote sha>`) and exits 0 when every parsed line's *remote* ref is under `refs/tags/`.
|
||||
Two details are load-bearing and easy to regress:
|
||||
- `.husky/pre-push` consumes stdin into `$_prepush_refs` before any guard runs, so it must **forward**
|
||||
those lines (`printf '%s\n' "$_prepush_refs" | …`). Without that the check receives EOF and the
|
||||
exemption is dead code that silently never fires. The unit tests drive the hook directly and would
|
||||
still pass, so this wiring is not covered by them.
|
||||
- The exemption requires **at least one** parsed ref line. "All refs are tags" is vacuously true for
|
||||
zero lines, which would disable H11 for every push; with no lines the hook falls through to the
|
||||
normal freshness check. `scripts/tests/test_prepush_rebase_check_tag_exemption.py` pins both the
|
||||
negative control (branch push from a behind branch still blocked), the mixed branch+tag case, and
|
||||
the two zero-line cases.
|
||||
|
||||
Rationale, as with the whole hook program: make the process rule a derivation/hook, not prose to
|
||||
remember (#303 methodology review). Tracked: #311; sibling #312 (H12 issue-qualification audit).
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
---
|
||||
key: release.main-direct-push-disabled
|
||||
title: '2026-08-05 — `main` refuses direct pushes (`enable_push: false`), because a push whitelist would have been a no-op here (#743)'
|
||||
status: active
|
||||
since: '2026-08-05'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Branch protection on `main` carries `enable_push: false` AND `block_admin_merge_override: true`. Both halves are required and neither is sufficient. `enable_push: false` removes the direct-push path, leaving the PR merge path — the only path on which Gitea evaluates `status_check_contexts`, and therefore the only path on which `review-verdict/h10` is consulted at all. `block_admin_merge_override: true` then closes the force-merge bypass on that remaining path: with it false (the default), `CanBypassBranchProtection` returns true for a repo admin, so `POST /pulls/{n}/merge` with `force_merge: true` merges a PR whose `h10` is missing or red — one API call, no forgery, no PATCH. Do NOT "soften" the push half to a push WHITELIST: measured here, a whitelist naming `timothy` still admits the push, and `timothy` is the identity every agent session, PAT and injected `GITEA_TOKEN` already acts as, so the whitelist form closes nothing while reading in review as a control. Same reasoning is why the admin-override half is needed: an admin-shaped control that exempts the only admin exempts everybody. What remains open: a credential that can PATCH branch protection off can still undo either half — an accepted residual, not a closed route. Tag pushes are unaffected (`tag_protections` governs those separately), so the release cut still works.'
|
||||
signals: 'direct push to main, push whitelist, enable_push false, branch protection bypass, review-verdict/h10 bypassable without forging, merge consent derived not asserted, pre-receive hook declined, Not allowed to push to protected branch, protected branch, tag_protections, release tag push, GITEA_TOKEN repo write, RENOVATE_TOKEN, site admin bypass, PR-only flow · paths: `docs/ci-cd.md` · issues: #743, #697, #698, #622, #672, #706, #742, server-management#714'
|
||||
mechanics: 'Gitea 1.27.1. `PATCH /api/v1/repos/timothy/ersatztv/branch_protections/main` with `{"enable_push": false, "block_admin_merge_override": true}`; whitelist fields left off (`enable_push_whitelist: false`, empty arrays), `enable_force_push: false`, `enable_merge_whitelist: false`, `required_approvals: 0`. MEASURED 2026-08-05 against a throwaway `probe-743-*` rule rather than against `main`: with `enable_push: false` a push by `timothy` (site admin) was REFUSED — `pre-receive hook declined`, `Not allowed to push to protected branch`; after PATCHing the same rule to `enable_push: true` + `enable_push_whitelist: true` + `push_whitelist_usernames: ["timothy"]` the identical push SUCCEEDED. Separately probed on a second throwaway rule: a contents-API write (`PUT /repos/{o}/{r}/contents/{path}` with `branch` set to the protected branch) was REFUSED HTTP 403 `user cannot commit to repo [user: timothy]` — so the web-editor/API file-write surface does not bypass it either. Then on `main` itself: `git push origin HEAD:main` REFUSED, and a tag-only push SUCCEEDED from the same worktree. `GET .../tag_protections` returns `[]`; repo is `fork: false`, `mirror: false`. NOT measured, source-attested only (Gitea 1.27 `CanBypassBranchProtection`, `services/pull/check.go`, `routers/private/hook_pre_receive.go`): that `block_admin_merge_override: false` would have let an admin `force_merge` past the required contexts — the field was set to true rather than probed, since probing it means merging an unreviewed PR. All probe artifacts (two rules, two branches, one tag) deleted and confirmed gone; `origin/main` head unchanged at `08e95f9ec` throughout.'
|
||||
---
|
||||
|
||||
**Why a whitelist was the wrong shape.** #743 proposed "a push whitelist on `main` (or disable direct
|
||||
push entirely)" as if the two were interchangeable. They are not, and which one is right depends on a
|
||||
fact about *this* instance: the only accounts with repository write are `timothy` (a site admin) and
|
||||
`renovate`. Every credential in the threat model — an agent session, a collaborator PAT, the
|
||||
`GITEA_TOKEN` Gitea injects into every Actions job — authenticates as one of those two, and
|
||||
overwhelmingly as `timothy`. A whitelist admitting `timothy` therefore admits precisely the identity
|
||||
the control is supposed to constrain. It would have ticked the issue's box while changing nothing.
|
||||
This was measured, not reasoned: the same push was refused under `enable_push: false` and accepted
|
||||
under a whitelist naming `timothy`.
|
||||
|
||||
**Disabling push alone was NOT enough, and the reason is the same argument twice.** The first draft of
|
||||
this record disabled direct push and concluded that `review-verdict/h10` was therefore load-bearing.
|
||||
An independent review caught that this repeated on the merge path exactly the mistake it had just
|
||||
diagnosed on the push path. The push argument was: a whitelist naming `timothy` fails because
|
||||
`timothy` is the identity every credential already holds. The merge path had the identical shape —
|
||||
`block_admin_merge_override` defaulted to `false`, so `CanBypassBranchProtection` returned true for a
|
||||
repo admin and `POST /pulls/{n}/merge` with `force_merge: true` merged straight past a missing or red
|
||||
`h10`. One API call, cheaper than the push route it replaced. **An admin-shaped control that exempts
|
||||
the only admin exempts everybody.** Both fields are now set; treat them as one control, and never
|
||||
cite `enable_push: false` alone as the reason the gate holds.
|
||||
|
||||
**What this actually closes, and what it does not.** It closes the *write-only* credential routes,
|
||||
which is most of #743's own "who can do it" list: the injected `GITEA_TOKEN` (repo write, not admin),
|
||||
`RENOVATE_TOKEN`, and any non-admin collaborator PAT. Those can no longer reach `main` at all, by any
|
||||
path that skips the gate.
|
||||
|
||||
It does **not** close the admin route. `timothy` is a site admin, so a credential holding that
|
||||
identity can `PATCH` either field off, act, and restore it — the exact sequence used to *prove* the
|
||||
push semantics above. Closing that requires agent sessions to run as a scoped non-admin credential,
|
||||
which is a different change with its own cost (packages live in a user namespace; see the "Admin
|
||||
ownership is a real residual" section of `ci.actions-credential-scoping`). Recorded as an accepted
|
||||
residual rather than fixed here, so it is not mistaken for covered. The severity bound from #697 and
|
||||
#743 is unchanged throughout: push access is required, so this is a compromised contributor or a
|
||||
subverted automated session, never an anonymous attacker.
|
||||
|
||||
**Which write surfaces were enumerated.** `git push` (measured, refused), the contents API and by
|
||||
extension the web editor / upload path (measured on a probe branch, refused HTTP 403 — they share the
|
||||
`CanUserPush` predicate, which has no admin special-case and no `unprotected_file_patterns` carve-out
|
||||
since that field is empty), apply-patch / revert / cherry-pick (source-attested, same predicate),
|
||||
force push (`enable_force_push: false`), default-branch deletion (separately refused), and fork-sync /
|
||||
mirror (not applicable: `fork: false`, `mirror: false`). Merge remains the one intended path.
|
||||
|
||||
**Why the release cut does not deadlock.** #743 flagged that the tag path had to keep working, and
|
||||
#719 documents H11 blocking a tag-only push on every release cut. Branch protection is scoped to
|
||||
`refs/heads/main`; tags are governed by an entirely separate mechanism, and `tag_protections` on this
|
||||
repo is empty, so tag pushes are unrestricted by anything except ordinary write permission. Demonstrated
|
||||
rather than assumed: from one worktree, the branch push to `main` was refused and a tag push succeeded.
|
||||
Do not conflate the two mechanisms — disabling branch push says nothing about tags, and a future
|
||||
tag-protection rule would not inherit from this one.
|
||||
|
||||
**The `docker-build.yml` `persist-credentials` question (#743's fourth box), decided and deferred.**
|
||||
Its six `actions/checkout` steps omit `persist-credentials: false`, so a head-resolved job keeps a
|
||||
write-capable credential in `.git/config`. It *should* be set — but not blind, and not in this PR,
|
||||
because two steps run `git fetch --no-tags --depth=100 origin "$base_ref" || true` and feed the result
|
||||
into the changed-file skip logic. That `|| true` means a credential regression does not fail the job;
|
||||
it silently yields an empty changed-file set, and the skip logic then reads "nothing changed". The repo
|
||||
is public, so anonymous fetch is *expected* to cover it — expected is not measured, and the failure
|
||||
mode is silent, which is the shape that has burned this repo before. The correct order is: drop the
|
||||
`|| true` masking so a fetch failure is loud, then set `persist-credentials: false` and confirm both
|
||||
jobs still compute a non-empty changed set on a PR that genuinely changes files.
|
||||
|
||||
**Why this is not redundant with the Husky pre-push hooks.** `.husky/pre-push` guards (H6 done-when,
|
||||
H11 rebase, H13 clean worktree) are client-side and deliberately fail-open — a git hook cannot prompt.
|
||||
They are not installed in CI, not present in a fresh clone until `husky` runs, and `--no-verify`
|
||||
bypasses them, which the worktree workflow uses routinely. They are good friction against mistakes and
|
||||
were never a control against a credential. This record is the server-side half; the hooks remain useful
|
||||
and unchanged.
|
||||
@@ -5,7 +5,7 @@ status: active
|
||||
since: '2026-07-25'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The H10 review verdict is written as a `review-verdict/h10` Gitea **commit status** on the exact reviewed sha by `scripts/post-review-verdict.sh`, and that context is a REQUIRED status check on `main`. Because a status belongs to one sha, a later commit cannot inherit it, so Gitea''s own `merge_when_checks_succeed` refuses to merge a head no one reviewed. The PreToolUse hook additionally refuses to SCHEDULE an auto-merge unless that status is already green on head. A `pull_request` workflow auto-passes the two exempt classes (Renovate-authored, docs-only) unless the PR touches a protected path (`.claude/`, `.gitea/`, `.husky/`, `scripts/`, `docker/ci/`). This extends — does not supersede — `release.review-verdict-gate` (#303 H10), whose comment convention remains the human-readable artifact and the hook''s condition (c).'
|
||||
rule: 'The H10 review verdict is written as a `review-verdict/h10` Gitea **commit status** on the exact reviewed sha by `scripts/post-review-verdict.sh`, and that context is a REQUIRED status check on `main`. Because a status belongs to one sha, a later commit cannot inherit it, so Gitea''s own `merge_when_checks_succeed` refuses to merge a head no one reviewed. The PreToolUse hook additionally refuses to SCHEDULE an auto-merge unless that status is already green on head. A `pull_request_target` workflow auto-passes the two exempt classes (Renovate-authored, docs-only) unless the PR touches a protected path (`.claude/`, `.codex/`, `.gitea/`, `.husky/`, `scripts/`, `docker/ci/`). This extends — does not supersede — `release.review-verdict-gate` (#303 H10), whose comment convention remains the human-readable artifact and the hook''s condition (c).'
|
||||
signals: 'merge_when_checks_succeed freezes consent, auto-merge merges an unreviewed head, verdict bound to sha, review-verdict/h10 required check, post-review-verdict.sh, Renovate platformAutomerge exemption · paths: `scripts/post-review-verdict.sh`, `.gitea/workflows/review-verdict.yml`, `.claude/hooks/pretooluse-merge-consent.sh` · issues: #622, #303 (H6/H10), #242, #619'
|
||||
mechanics: '`scripts/tests/test_post_review_verdict.py` (incl. a TOCTOU head-moved case and cross-checks against the hook''s own condition-(c) regexes); branch protection `status_check_contexts` on `main`'
|
||||
---
|
||||
@@ -142,18 +142,68 @@ described as one:
|
||||
`GET /commits/{sha}/status`, which returns latest-per-context, and the workflow refuses to post
|
||||
anything at all when that read fails or is unparseable, rather than treating it as "no verdict yet".
|
||||
3. **Changing a PR's base does not change its head sha**, so a verdict status keeps applying to a diff
|
||||
that has materially changed. Not currently handled; low exposure here because base changes are rare
|
||||
and manual.
|
||||
4. **A PR that edits `review-verdict.yml` is judged by its own edited copy.** Gitea runs
|
||||
`pull_request` workflows from the PR **head**, not the base — confirmed on the very PR that
|
||||
introduced this workflow (#630): `review-verdict.yml` does not exist on `main`, yet its job ran
|
||||
and posted a status. So `PROTECTED` is a guardrail against *accidental* self-exemption, **not** a
|
||||
tamper-proof control: a PR that rewrote the workflow would be classified by the rewritten rules.
|
||||
Acceptable for a two-account repo (`timothy`, `renovate`) where the threat is a careless change
|
||||
rather than a hostile one; it would not be for an untrusted-contributor repo, which would need
|
||||
the classification moved somewhere the PR cannot edit (a base-branch-pinned workflow, or
|
||||
server-side policy).
|
||||
that has materially changed. **Detected, not prevented** (#632): `post-review-verdict.sh` records
|
||||
the base branch in the status description as a trailing `(base: <ref>)`, and the merge-consent hook
|
||||
reads it back and denies when it no longer matches the PR's live `base.ref`. That covers the hook
|
||||
path only — a commit status carries no base of its own, so the server-side required check cannot
|
||||
see this, and a merge driven through the Gitea UI or API is unaffected. Accepted: base changes are
|
||||
rare, manual, and this is a two-account repo.
|
||||
|
||||
The same head-execution behaviour is what makes the rollout self-hosting in the good case: #630's
|
||||
Two details are load-bearing and each was chosen against a plausible alternative:
|
||||
|
||||
- **The comparator is `base.ref`, not `base.sha`.** `base.sha` tracks the base branch's *tip*,
|
||||
which moves whenever anything merges to `main`; comparing it would invalidate every open verdict
|
||||
on every unrelated merge — a rare-event guard turned into a permanent merge deadlock. A base
|
||||
branch that merely *advances* is deliberately out of scope: rebasing onto it moves the head sha,
|
||||
which the per-sha binding already covers.
|
||||
- **The field goes in the status description, not the verdict comment.** The comment body is parsed
|
||||
by `scripts/check-review-verdict.sh`, whose grammar had three false-opens in its history (#629);
|
||||
nothing parses the description, so this adds a field without reopening that surface.
|
||||
|
||||
Verdicts posted before #632 carry no `(base: …)` and get **no opinion** rather than a deny — the
|
||||
alternative would block every in-flight PR the day it lands, and the window closes on its own since
|
||||
verdicts are per-head and short-lived. **"Could not check" is a third outcome**, deliberately not
|
||||
folded into that one: an unreadable status response or a PR with no resolvable `.base.ref` falls
|
||||
through to a human `ask`. The first draft collapsed them, so a transient Gitea hiccup skipped the
|
||||
comparison in silence and a later successful read could still emit "merge gate: satisfied" for a
|
||||
check that never ran.
|
||||
|
||||
**Docs-only PRs exit before this check**, because the docs-only carve-out short-circuits the whole
|
||||
gate earlier in the hook. That carve-out does not auto-grant — it passes through to an ordinary
|
||||
permission prompt — so the exposure is a missing warning on a merge a human is already confirming,
|
||||
not a silent merge. Worth knowing before reading "the hook denies on a retarget" as unconditional.
|
||||
4. **A PR that edits `review-verdict.yml` WAS judged by its own edited copy — closed in #672, see
|
||||
`ci.gate-trigger-base-resolved`.** Gitea runs `pull_request` workflows from the PR **head**, not
|
||||
the base — confirmed on the very PR that introduced this workflow (#630): `review-verdict.yml`
|
||||
does not exist on `main`, yet its job ran and posted a status. `PROTECTED` was therefore a
|
||||
guardrail against *accidental* self-exemption, **not** a tamper-proof control: a PR that rewrote
|
||||
the workflow would be classified by the rewritten rules. The workflow now triggers on
|
||||
`pull_request_target` scoped to `branches: [main]`, so its definition is taken from the base.
|
||||
|
||||
**Only this file's instance is closed, not the class.** Any head-resolved workflow holding
|
||||
credentials that can POST a commit status can still forge `review-verdict/h10`;
|
||||
`docker-build.yml` demonstrably could, and must stay head-resolved because it builds the PR's own
|
||||
code — so #697 scoped its credential instead (`ci.actions-credential-scoping`), leaving AT LEAST
|
||||
these: the injected `GITEA_TOKEN` (posts with `creator: null`), `RENOVATE_TOKEN` (a
|
||||
`write:repository` bot PAT in the same secret store, so it posts with a real creator and IS
|
||||
inherited, #742), a collaborator's own token, and the `v*` tag push — which matters less for
|
||||
forging this status than for what else it does: `docker-build.yml` publishes `:prod` from a tagged
|
||||
ref, and a tag may point at any commit, so it ships a prod image with no PR, review or status.
|
||||
None of which used to be even required — direct pushes to `main` were server-side permitted, so
|
||||
the gate could be skipped without forging anything (#743). **That route is now closed**
|
||||
(`release.main-direct-push-disabled`): `main` carries `enable_push: false` *and*
|
||||
`block_admin_merge_override: true`, so every change reaches `main` through the PR merge path,
|
||||
which is the only path on which these required contexts are evaluated. What survives is the
|
||||
forgery list above — those routes post a status rather than skip it, so they are still real —
|
||||
**plus one skip route that is not forgery at all**: a credential that can `PATCH` branch
|
||||
protection can turn either field off, act, and restore it. `timothy` is a site admin, so every
|
||||
session holds that capability; it is an accepted residual, recorded in
|
||||
`release.main-direct-push-disabled` and `ci.actions-credential-scoping`, not a closed route. So
|
||||
the "careless change rather than a hostile one" posture below still
|
||||
describes the repo accurately — it is simply no longer *this* workflow that is the weakest link.
|
||||
An untrusted-contributor repo would still need the classification moved somewhere no PR can
|
||||
reach (server-side policy), not merely a base-pinned definition.
|
||||
|
||||
The same head-execution behaviour was what made the rollout self-hosting in the good case: #630's
|
||||
own run correctly identified it as touching `.claude/` and `scripts/`, refused both exemptions,
|
||||
and posted `review-verdict/h10=pending` with an actionable description.
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
---
|
||||
key: spa.library-pickers-resolve-by-search
|
||||
title: '2026-07-26 — a media-library picker resolves by SEARCH, never by a window over the type; `loadAllPages` stays for bounded-by-construction lists (#651)'
|
||||
status: active
|
||||
since: '2026-07-26'
|
||||
supersedes: spa.list-completeness-vs-bounded-pickers@2026-07-26
|
||||
superseded-by: none
|
||||
rule: 'A picker over a media-library table (Episode/Song/Image/Movie/MusicVideo/TelevisionShow/TelevisionSeason/Artist/OtherVideo/RemoteStream) resolves its options by SEARCH — a debounced `SearchPicker` calling `searchLibraryPickerOptions`, which issues at most ONE `getLibraryBrowseItems` request per settled query, bounded to `LIBRARY_PICKER_RESULTS` (25) rows — CLAMPED inside the helper, not merely defaulted — and gated on `LIBRARY_PICKER_MIN_QUERY` (2) characters. It list-loads NOTHING on mount or on a type switch, so there is no truncation to surface and no truncation hint. The typed text is COMPILED (`titleContainsQuery` → `title:*<escaped>*`), never forwarded raw. The current selection renders from the OWNING RECORD, not from the result set (`selectedName` on a rerun collection / playlist item; a single by-id detail read — `getShow`/`getSeason`/`getArtist` — for a filler preset, which stores only the id), and an edit draft is INITIALIZED ONCE from the detail read — never seeded from the list row, never reconciled against a late response — with the form withheld until it lands, the editor failing CLOSED when the response carries no USABLE concurrency token — absent, empty and whitespace-only ETags are ONE case, normalized in one place, so a PUT without `If-Match` is unreachable, and a deadline plus a route back so a hung request cannot strand it. An id NEVER travels without its namespace: search results are cached against `(source, query)` and list-backed options carry the type they were loaded for, so no id from one type can be offered under another; and every id entering editor state — search result, list-backed option, or a selection restored from a detail read — passes ONE shared `isSelectionId` (int32) predicate at that boundary, an unbindable id being treated as ABSENT rather than coerced. Conflicts are detected at SAVE time via `If-Match` -> 412 -> Reload, and Reload simply drops the draft back to null and re-runs the same initialize-once load, so the form is unmounted while the replacement is in flight; an asynchronously-resolved name is keyed to the id it was resolved for and never overwrites a label naming a different id. The typeahead implements the full ARIA combobox keyboard contract, because it replaces a natively keyboard-operable `<select>`. The other half of the superseded record is UNCHANGED: bounded-by-construction admin lists (collections, multi-collections, smart collections, playlists) still page to completeness via `loadAllPages` and still report `complete`/`hint: incomplete`. Server-side caps are not raised — this is a web-only change.'
|
||||
signals: 'library picker typeahead, SearchPicker, searchLibraryPickerOptions, titleContainsQuery, LIBRARY_PICKER_RESULTS, LIBRARY_PICKER_MIN_QUERY, LIBRARY_PICKER_LUCENE_SPECIALS, compile typed text not raw Lucene, Lucene && || escaping, picker truncation hint removed, loadAllPages Class A, LuceneSearchIndex.Search hitsLimit, useIsMountedRef, aria-activedescendant combobox keyboard, initialize-once draft not hydrate-merge, no list-row seeding, fail closed on a missing or blank ETag, usable concurrency token, cross-type id, id never travels without its namespace, results keyed on (source query), failed search not cached as empty, selection id int32 boundary predicate, isSelectionId, npm run typecheck not tsc --noEmit, stale result set not committable by keyboard OR pointer · paths: `web/src/api/libraryBrowse.ts`, `web/src/schedules/pickers.tsx`, `web/src/hooks.ts`, `web/src/screens/RerunCollectionsScreen.tsx`, `web/src/screens/PlaylistsScreen.tsx`, `web/src/screens/FillerPresetsScreen.tsx`, `web/src/api/paging.ts`, `docs/spa-conventions.md` §3b · issues: #651, #644, #578, #440'
|
||||
mechanics: '`docs/spa-conventions.md` §3b'
|
||||
---
|
||||
|
||||
#644 fixed a silent truncation: three `getLibraryBrowseItems` pickers asked for an over-cap
|
||||
`pageSize` and got the server's `MaxPageSize` back with no indication. Its follow-up review
|
||||
(`spa.list-completeness-vs-bounded-pickers`) correctly refused to "fix" that by paging to
|
||||
completeness — ~200 serial requests against a 20,000-row table, each more expensive than the last
|
||||
(`LuceneSearchIndex.Search` computes `hitsLimit = skip + limit`), ending in a `<select>` with
|
||||
20,000 `<option>` nodes — and settled on one bounded page plus a visible `Showing the first 100 of
|
||||
5000 — use search to narrow.` hint. That removed the *silence*. It did not remove the
|
||||
*unusability*: a 100-row window over Episode or Song is not a picker, it is an arbitrary alphabetical
|
||||
prefix, and the hint pointed at a search box that did not exist. The record said so itself, deferring
|
||||
"a full typeahead/search-driven picker over the media library" to a follow-up issue. This is that
|
||||
issue.
|
||||
|
||||
**The fix is to stop windowing and start searching.** `getLibraryBrowseItems` already took a `query`
|
||||
param (`CollectionsScreen`/`SmartCollectionDialog` were already using it). The three pickers —
|
||||
`RerunCollectionsScreen`, `PlaylistsScreen`, `FillerPresetsScreen` — now render the existing
|
||||
`SearchPicker` for their media-library types instead of a `<Select>`, backed by one shared
|
||||
`searchLibraryPickerOptions(mediaType, text)`. Selecting a media-library type now issues **zero**
|
||||
requests; a settled query issues **one**, for at most 25 rows. Both bounds are properties of the
|
||||
helper, not of a caller's discipline, and are pinned by request-count assertions against a
|
||||
20,000-row fixture rather than by inspection.
|
||||
|
||||
**Typed text is compiled, never forwarded.** The rule from #440's Auto-Tune add-source typeahead now
|
||||
binds here too, and its helper is shared rather than re-implemented: `titleContainsQuery` moved out
|
||||
of `AutoTuneScreen` into `web/src/api/libraryBrowse.ts`. The search index's default field does not
|
||||
match bare title words — `Alpha` finds nothing for "Show Alpha" — so forwarding the literal text the
|
||||
way an explicit query box does would look broken in a *name* picker. Every Lucene special (and
|
||||
whitespace) is escaped so the boundary stars are the only live wildcards, the same shape
|
||||
`builder/rules/compile.ts` emits for `contains`.
|
||||
|
||||
**The already-selected item is preserved by rendering it from the record, not the result set.** This
|
||||
is the failure mode that would make a search picker *worse* than the windowed one: a user opening an
|
||||
existing record must see what it points at, before typing anything and after any search that doesn't
|
||||
happen to include it. `SearchPicker` already renders `selectedName` independently of `results`, and
|
||||
rerun collections and playlist items already carry that name on their own DTOs. `FillerPresetFullResponseModel`
|
||||
does **not** — it stores only the id — so the edit path resolves the name through a single by-id
|
||||
detail read (`/api/v1/shows|seasons|artists/{id}`), which is *stricter* than the behaviour it
|
||||
replaces: the old picker could only name a selection that happened to fall inside the first 100
|
||||
browse rows, and rendered a bare `#9999` otherwise. A failed resolution degrades to `#id`; it never
|
||||
clears the id.
|
||||
|
||||
A cold cross-family review found that "renders from the record" was not by itself enough, because
|
||||
the *record* can arrive without its selection. `RerunCollectionsController.ProjectToResponseModel`
|
||||
derives both `selectedId` and `selectedName` from the same eager-loaded navigation, and
|
||||
`GetRerunCollectionByIdHandler` loads media metadata only for Show/Season/Artist/Movie while
|
||||
`MediaCollections/Mapper` maps RemoteStream through `_ => null` — so opening a RemoteStream rerun
|
||||
collection returned HTTP 200 with a null selection and the edit-load refresh *cleared a stored id*,
|
||||
leaving Save permanently disabled. The rule is therefore stated as a prohibition on the client:
|
||||
**no code path may clear a stored id it merely failed to name.** Every affected type (RemoteStream,
|
||||
Episode, MusicVideo, Song, OtherVideo, Image) is covered by its own test. The underlying read-model
|
||||
gaps are server-side, tracked as **#671**; this branch is web-only and the client guard stays after
|
||||
that lands.
|
||||
|
||||
**The obvious form of that fix is worse than the bug, and this is the part worth remembering.** The
|
||||
first attempt coalesced the two fields independently — `refreshed.selectedId ?? current.selectedId`
|
||||
and `refreshed.selectedName || current.selectedName`. But an id and its display name are ONE value:
|
||||
against a `Song` response (id resolves, name does not), a user selecting a different song while the
|
||||
refresh was in flight got the *new* name paired with the *stored* id. The chip read "New Song" and
|
||||
Save wrote 42 — the user's choice discarded with no error and no visual cue, where the original
|
||||
defect at least cleared the field visibly. A second cold review caught it. The trade is: a visible
|
||||
failure is strictly better than a silent one, so a "smarter coalesce" is the wrong shape of fix.
|
||||
|
||||
**Round 5 deleted all of it.** What follows is kept because the reasoning is the point, but the
|
||||
mechanism it describes no longer exists: rounds 2-4 built and rebuilt a layer that reconciled a late
|
||||
detail response against a draft the user was already editing, and that layer produced a HIGH finding
|
||||
every single round — three of them cross-user lost updates. The final one was unfixable in kind: the
|
||||
merge had no immutable baseline, so it could not distinguish "the user changed this" from "the server
|
||||
changed this", giving both a missed conflict and a false one (the false one leaving `etagRef` null,
|
||||
turning the next save into a silent force-write). The fix was to **remove the race rather than
|
||||
referee it**: initialize the draft exactly once from the detail GET, withhold the form until it
|
||||
lands, and detect conflicts at save time through the `If-Match` -> 412 -> Reload path that already
|
||||
existed. `touchedRef`, `hydrateDraft`, `hydrateIdentity`, `identityConflicts`, `replaceDraft` and
|
||||
`replacePending` are all gone.
|
||||
|
||||
Two facts made that safe rather than lossy. First, the list row could never have helped: its handler
|
||||
applies **zero** `.Include()`s where the detail handler applies **fourteen**, and both project
|
||||
through the same mapper, so the list response is a strict subset — the id it was being seeded with
|
||||
is null in production for every row (#671). Every round-1 "preserve the id from the list" guarantee
|
||||
was therefore protecting a value that only existed in test fixtures. Second, the sibling screens
|
||||
(`FillerPresetsScreen`, `PlaylistsScreen`) already worked this way; `RerunCollectionsScreen` was the
|
||||
outlier, which is why nearly every finding in rounds 3-5 traced to it.
|
||||
|
||||
The historical reasoning, retained because the *classes* still bind anywhere a draft is reconciled:
|
||||
|
||||
What replaced it is atomicity plus a race rule — and a third review round showed the first attempt
|
||||
at *that* had made the same mistake one level up: it enumerated the instance (id/name) instead of
|
||||
covering the class. **A picker selection is one value spread across three fields**: `collectionType`
|
||||
says which table an id indexes, `selectedId` picks the row, `selectedName` labels it. Splitting type
|
||||
from id is the identical bug to splitting id from name — the editor displayed and would have saved
|
||||
a Collection id as a RemoteStream id, when the record's type changed server-side mid-load. So the
|
||||
whole `{collectionType, selectedId, selectedName}` unit resolves together: either half touched by
|
||||
the user pins all of it; a differing type takes the response's unit whole (null selection included,
|
||||
since an id from the old type's space cannot be carried across); and only once both sides agree on
|
||||
the type does the id/name rule apply.
|
||||
|
||||
Hydration also **loses every race against the user**: a `touchedRef` records which fields have been
|
||||
edited, through a single `edit()` funnel so "touched" cannot drift from "changed".
|
||||
|
||||
**Refresh and replace are different policies and must be different functions.** The same review
|
||||
found a *cross-user lost update*, the worst defect in the series: the conflict "Reload" — which
|
||||
exists to discard local edits — ran through the refresh path with a touched-set reset. Because the
|
||||
reloaded record reports `selectedId: null` under the #671 gap, the keep-ours fallback restored the
|
||||
user's **dirty** selection, the fresh ETag was installed, and the next Save silently overwrote the
|
||||
collaborator's change with edits the user had explicitly asked to throw away. `replaceDraft` was made a separate function with the mode carried on the load — machinery that
|
||||
round 5 then deleted outright along with the rest of the reconciliation layer. Every interleaving is tested by holding the detail response open, acting as the user, then
|
||||
releasing it.
|
||||
|
||||
Symmetrically, a name resolved asynchronously is **keyed to the id it was resolved for** and refuses
|
||||
to overwrite a label that already names a different id — otherwise a slow edit-load read landing
|
||||
after the user picked something else labels the new selection with the old item's title while the
|
||||
saved id says otherwise. Keying the render alone stops the mislabelling but still throws away the
|
||||
newer, correct label, so both halves are needed.
|
||||
|
||||
**Scope: Lucene-backed types only.** `GetLibraryBrowseItemsHandler` applies `query` two different
|
||||
ways — as a Lucene clause for media items, and as a plain SQL `LIKE` on `Name` for the
|
||||
collection-family types (Collection / SmartCollection / MultiCollection / RerunCollection /
|
||||
Playlist). A compiled `title:*x*` sent at the latter would be LIKE-matched literally and match
|
||||
nothing. So `FillerPresetsScreen` marks only its media-item types `searchable`; its
|
||||
collection-family types keep the bounded single-page load and the truncation hint, and the Class A
|
||||
`loadAllPages` paths in the other two screens are untouched. The `api.search-allitems-paging`
|
||||
precedent holds: the client bounds itself, the server cap is not raised.
|
||||
|
||||
**The generalisation that took four rounds: an id never travels without its namespace.** Rounds 2
|
||||
and 3 made *hydration* treat `{collectionType, selectedId, selectedName}` as one value. Round 4 found
|
||||
the same defect in three more places, because the fix had been applied to the one structure that was
|
||||
named rather than to every structure that carries an id. A typeahead cached its results against the
|
||||
query TEXT, so switching the search source with the same text made the re-query guard *suppress* the
|
||||
new request and leave the previous namespace's hit clickable under the new label. List-backed
|
||||
`<select>` options were normalised to `{id, name}`, dropping the type, so on a slow connection the
|
||||
previous type's rows stayed selectable while the replacement loaded — on both screens. The rule that
|
||||
covers all of them: **every result, option and cached result set carries its source, and identity is
|
||||
compared as `(type, id)`.** `SearchPicker` now takes a required `source` prop (required, not
|
||||
defaulted — a default would silently opt every caller out), and `pickerFor` tags list-backed options
|
||||
with the type that produced them.
|
||||
|
||||
**Two cross-user lost updates make a category, not two incidents.** Round 3's was conflict-Reload
|
||||
running through the refresh policy. Round 4's was subtler: a touched identity pinned against a
|
||||
server-side type change is *correct*, but adopting the response's newest ETag alongside it authorized
|
||||
a Save that silently overwrote the collaborator's change with no 412. The category is **never install
|
||||
a save-authorizing ETag over a local edit the server contradicts** — such a collision is a conflict to
|
||||
surface, not a state to reconcile. Relatedly, the Reload path now renders the editor inert while the
|
||||
replacement is in flight, since the dialog closes immediately and an edit typed in that window was
|
||||
silently erased.
|
||||
|
||||
**Cache provenance must distinguish failure from emptiness — without licensing a retry storm.** The
|
||||
round-3 re-query guard cached a failed search as an authoritative empty result, so a transient 500
|
||||
became a permanent "No matches" that no amount of reopening could retry. Recording `ok` fixed that
|
||||
but created the opposite defect: declining the cached failure re-ran the effect and scheduled a
|
||||
fresh request every debounce. The two concerns are now separate — `ok` says whether the held answer
|
||||
is authoritative, and an `attemptRef` suppresses automatic retries until an explicit user action
|
||||
re-arms one. The picker also races `search` against a deadline (a caller-supplied promise carries no
|
||||
abort signal) and treats a non-array resolution as a failure, since `client.ts` turns a malformed
|
||||
2xx body into `undefined` rather than rejecting.
|
||||
|
||||
**Replacing a native control means owing its keyboard behaviour.** A `<select>` is fully
|
||||
keyboard-operable, so an input-plus-listbox that only responds to Tab and click is a regression
|
||||
introduced by this change rather than a pre-existing gap. `SearchPicker` implements the ARIA
|
||||
combobox pattern: `role="combobox"` with `aria-expanded`/`aria-controls`/`aria-autocomplete`,
|
||||
Arrow/Home/End moving a virtual cursor exposed through `aria-activedescendant`, Enter committing,
|
||||
Escape dismissing, and options as non-tab-stops. Two defects specific to an *asynchronous* combobox
|
||||
also had to be closed: a stale result set was committable (highlight Alpha for "Al", retype "Be",
|
||||
press Enter before the debounce — Enter selected Alpha), so the highlight now drops on input change
|
||||
and the guard lives in the single `choose()` sink rather than on each call site (gating Enter while
|
||||
leaving `onClick` open was the same defect in another modality, found a round later); and Escape
|
||||
closed the popup while focus stayed in the input, where `onFocus` can never re-arm it, so the picker
|
||||
was dead until the user blurred and refocused — typing and ArrowDown now both reopen it, without
|
||||
re-querying results that are already current, since the duplicate response would reset the cursor
|
||||
and leave Enter doing nothing.
|
||||
|
||||
Folded in from #578 (same components): the rule-builder facet typeahead arms on **focus** rather
|
||||
than on mount, so an N-row rule tree no longer fires N unrequested `search/fields/*/values`
|
||||
requests; and both it and `SearchPicker` now pair their `seqRef` stale-response guard with a shared
|
||||
`useIsMountedRef` (`web/src/hooks.ts`) so a fetch resolving after unmount is dropped. Proving that
|
||||
guard needs two tests, because React 19 no longer warns on a setState-after-unmount and an unmounted
|
||||
tree renders nothing either way: a unit test of the hook (including a StrictMode double-invoke for
|
||||
the re-arm) plus an integration test that mocks the hook module and asserts `SearchPicker` actually
|
||||
read `current` and saw `false`. #578's remaining item — extending the `artist` facet source beyond
|
||||
entity artists — is a `GetSearchFieldValuesHandler` change, out of scope for a web-only fix, and is
|
||||
being done on its own branch.
|
||||
|
||||
*(Over the 60-line prose ceiling: checked for redundancy against
|
||||
`spa.list-completeness-vs-bounded-pickers` in `archive/` and declined to cut. The length is six
|
||||
distinct findings — the search bound, the compile rule, selection preservation, the
|
||||
clear-what-you-cannot-name prohibition, the async-name keying, and the keyboard contract — most of
|
||||
which came from review rounds, and each of which names a specific way the obvious implementation is
|
||||
wrong. The recurring error across four review rounds was always the same: patching the named
|
||||
instance instead of covering its class — which is why the record states the rules as classes. Summarising any of them back out would lose the counter-example that makes it actionable.)*
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
key: testing.enumerating-guard-identity-not-position
|
||||
title: '2026-07-27 — an enumerating allow-list guard keys its registry on IDENTITY, never on a source position (#650, #651)'
|
||||
status: active
|
||||
since: '2026-07-27'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A guard that cross-checks a hand-reviewed registry against call sites discovered across the whole repo must key each entry on properties INTRINSIC to the site — file, kind, and the value source text — and never on its absolute line or column. A registry keyed on position is a function of every other file in the repo, so a branch that never touches the guard can invalidate it; and because each PR is green against its own base, that failure is structurally invisible pre-merge and lands on `main` after review and after the merge gate. Dropping the position keeps every mutation the guard exists for — a NEW site, a REMOVED site and a CHANGED value each still fail, since each changes the identity multiset — and costs exactly ONE case, which must be stated rather than implied: a SAME-IDENTITY SUBSTITUTION within one file (delete a registered site, add a different unreviewed one with the same kind and value token, net-zero count) now passes. A REPORTED failure still prints the discovered line:column, because identity and diagnostics need not share a format. Comparison stays a MULTISET count rather than set membership, so two sites in one file sharing an identity must be discovered exactly that many times and a third occurrence still fails. A SCANNER test that asserts real AST positions against FIXED inline fixtures is the opposite case and keeps its line/column identity — it has no churn, because its input does not move.'
|
||||
signals: 'pageSize call-site guard · enumerating allow-list · registry went stale · UNREGISTERED and STALE report · line churn · line drift · semantic merge conflict · guard born red · registry stale on arrival · cancelled run hid a red · registry keyed on line:column · same-identity substitution residual gap · deviation classification · a registry must not launder a defect into a compliant label · multiset count not set membership · `pageSizeSiteId` vs `registryId` · scanner positions vs registry identity · paths: `web/src/api/pageSizeCallSites.guard.test.ts`, `web/src/api/pageSizeScan.ts`, `web/src/api/pageSizeScan.test.ts` · issues: #684, #650, #651, #676, #644'
|
||||
mechanics: 'Registry identity is `${file}:${kind}:${value}` (`registryId` in the guard); the scanner keeps `pageSizeSiteId` (`${line}:${column}:${kind}:${value}`) for `pageSizeScan.test.ts`.'
|
||||
---
|
||||
|
||||
The #650 guard enumerates every `pageSize` call site in the SPA and cross-checks it against a
|
||||
hand-reviewed registry in both directions. That part worked. Its follow-up review (F5/M-6) then made
|
||||
each entry's identity the site's absolute `line:column`, to distinguish two `pageSize` properties on
|
||||
one line. Sound about disambiguation, wrong about the cost.
|
||||
|
||||
**The guard was born red, and the sequence is the argument.** #651 moved `AutoTuneScreen.tsx` up ten
|
||||
lines and `FillerPresetsScreen.tsx` down seventy-two, and merged *before* the guard's own PR (#675).
|
||||
The registry, authored against a pre-#651 base, was stale the instant it landed. **A cancelled CI run
|
||||
is what let it through**: the guard's own merge run was cancelled, so nothing reported the red, which
|
||||
first surfaced on the next push (#676's merge — which touches no `web/src` file and is not the
|
||||
cause). `ci.cancelled-is-not-a-verdict`, paying out.
|
||||
|
||||
That is ONE ordering accident, not a recurring pattern; the honest count, because the argument needs
|
||||
no inflation. **The exposure is general anyway, because CI cannot see it coming.** Each PR is green
|
||||
against its own base, so the breakage exists only in the merge result and surfaces on `main` after
|
||||
review and after the merge gate.
|
||||
|
||||
**Identity should be what makes the site the thing being guarded.** A site's file, `kind` and value
|
||||
source text determine whether it is reviewed; where it sits in the file does not. The multiset
|
||||
comparison is what preserves what F5/M-6 was actually protecting and must not relax to set
|
||||
membership: `TrashScreen.tsx`'s two `PAGE_SIZE` requests must be discovered exactly twice, so a third
|
||||
occurrence still fails. The converse case is `pageSizeScan.test.ts`, which correctly KEEPS positions
|
||||
— verifying real AST positions is its subject and its fixtures cannot drift — which is why this
|
||||
introduced a separate `registryId` rather than changing `pageSizeSiteId` underneath it.
|
||||
|
||||
**What it costs, stated rather than implied.** A same-identity substitution inside one file now
|
||||
passes: delete a registered site, add a different unreviewed one with the same kind and value token,
|
||||
net-zero count. Narrow, and the old identity caught it only incidentally — it fired on every position
|
||||
change, so a reviewer conditioned to re-pin line numbers would likely have waved it through. Accepted
|
||||
knowingly and named in both places, because "costs no coverage" is the kind of claim that outlives
|
||||
whoever made it, and a guard described as exhaustive stops being re-examined.
|
||||
|
||||
**Diagnostics are not the identity.** Dropping position from the comparison key is the fix; dropping
|
||||
it from the failure *message* was collateral damage. The discovered direction prints `line:column`
|
||||
alongside each unregistered id — no churn, since positions appear only in an already-failing message.
|
||||
|
||||
**A registry must not launder a defect into a compliant-looking label.** Reconciling it surfaced a
|
||||
live §3b violation (#685): a picker degrading to an unfiltered whole-type window on an empty query,
|
||||
surfacing nothing. Both labels would have been false — `search-bounded` asserts a required query,
|
||||
`class-b` a rendered `totalCount` — and either would make the guard vouch for behaviour that does not
|
||||
exist. Hence a `deviation` class whose entries must name a tracking issue, enforced by a STRUCTURAL
|
||||
field rather than a `#\d+` scrape of the note: the first version of that test passed with the
|
||||
tracking reference deleted, because the note legitimately cited two historical issues.
|
||||
|
||||
The generalisation: **a guard whose input is the whole repository must not encode anything the whole
|
||||
repository can change without meaning to.** Position is the common instance; a line count, a file
|
||||
ordering or a byte offset would all fail the same way.
|
||||
@@ -81,6 +81,19 @@ Orchestration means: decompose, delegate independent slices, integrate their res
|
||||
whole, and keep canonical issue state accurate. Use the client's native agent/subagent tools; never
|
||||
assume a named tool, command, plugin, model-routing feature, or fork mechanism exists.
|
||||
|
||||
**Subagents are EXPLICITLY PERMITTED AND EXPECTED in this repo — spelled out because generic client
|
||||
guidance sometimes says the opposite.** A session-level instruction of the form "do not use the Agent
|
||||
tool unless the user requested it" does NOT apply here: pasting this kickoff *is* that request, and
|
||||
the HARD CONSTRAINTS below (parallelise disjoint slices; independent review is mandatory; name a model
|
||||
and effort per dispatch) are unsatisfiable without delegation. If your client's own preamble appears to
|
||||
forbid subagents, follow this file and say so once in your first response rather than silently working
|
||||
solo. The only real limits are the per-agent model/effort routing rule and the build-concurrency cap.
|
||||
|
||||
Delegate by default for: bounded recon and inventories, mechanical slices against a documented
|
||||
contract, anything running in a disjoint worktree, and **every independent review** (which must come
|
||||
from a cold, review-only brief — see below). Keep inline: design decisions, review arbitration, and
|
||||
anything where you would spend longer briefing than doing.
|
||||
|
||||
Route by capability when the client supports per-agent model selection, and **say which tier you chose
|
||||
in the dispatch itself** — see the `process.per-agent-model-routing` HARD CONSTRAINT below for the
|
||||
table. Where the client cannot route per agent, use the active model for every slice except the
|
||||
@@ -225,10 +238,32 @@ Then work the queue:
|
||||
**An empty backlog is not a stopping condition** — if the selector returns any eligible candidate,
|
||||
claim its top-ranked winner; do not ask the user to choose merely because candidates belong to
|
||||
different workstreams. Never invent a fix-size, recency, or perceived-relevance tiebreaker.
|
||||
3. **Claim it**: add the `in-progress` label + a "claiming" comment on the issue(s);
|
||||
reviewer-repo audits are claimed by comment only. Treat that claim as live until a later comment
|
||||
explicitly releases or abandons it, and exclude audits with a posted deliverable even while the
|
||||
issue remains open for implementer replies.
|
||||
3. **Claim it — but CHECK FOR AN EXISTING CLAIM FIRST, and the label is not the whole check.**
|
||||
The `in-progress` label prevents duplicate *pickup*; it does not prevent duplicate *work*, because
|
||||
another session may already be implementing an issue it has not labelled (or labelled after you
|
||||
read the list). Before writing any code, run all four — they are cheap and they fail differently:
|
||||
|
||||
a. **Open PRs referencing the issue.** `GET /repos/{owner}/{repo}/pulls?state=open` and look for
|
||||
`fixes #N` / `refs #N` in the body, or the number in the branch name. This is the check that
|
||||
would have caught ersatztv#649 being implemented twice.
|
||||
b. **Remote branches naming the issue.** `git ls-remote --heads origin '*<N>*'` — a branch usually
|
||||
exists before the PR does.
|
||||
c. **Recent comments on the issue**, not just its labels — a "claiming" comment from another
|
||||
session may predate the label, which is exactly what `CLAIM?` from `select-queue.sh` flags.
|
||||
d. **`git fetch origin main`**, so you are reading current state rather than your session's
|
||||
opening snapshot.
|
||||
|
||||
If any of those hit, do not start: report it to the user and take the next candidate. If none do,
|
||||
claim with the `in-progress` label **and** a "claiming" comment (reviewer-repo audits are claimed
|
||||
by comment only). Treat a claim as live until a later comment explicitly releases or abandons it,
|
||||
and exclude audits with a posted deliverable even while the issue remains open for implementer
|
||||
replies.
|
||||
|
||||
**Re-fetch `origin/main` before every push, not only at branch time.** A long session can run for
|
||||
hours across several review rounds; `main` moves underneath it. A branch cut from a stale base
|
||||
whose diff is computed against that stale base will silently show *other people's merged work as
|
||||
deletions*, and pushing it reverts them. Rebase (never merge main in) and re-run the local gate
|
||||
whenever the fetch shows movement. → `process.parallel-session-claim`
|
||||
4. **Scan for bundle-able siblings** (always, right after claiming — not optional). Check all three
|
||||
bundle axes from "Bundles" above: the claimed issue's **milestone**, its **cross-references /
|
||||
backlinks**, and its **shared label(s)** (list the other open issues under each of its labels).
|
||||
@@ -322,8 +357,11 @@ HARD CONSTRAINTS:
|
||||
task-specific delta. → `docs.convention-docs-session-start`
|
||||
- Run `scripts/select-queue.sh` for queue selection; trust its deps/tiering/ordering and resolve only its
|
||||
`CLAIM?`/`UMBRELLA?` flags. → `startup.parallel-orientation`
|
||||
- Claim with `in-progress` before working — but a claim prevents duplicate *pickup*, not overlapping code
|
||||
changes. → `process.parallel-session-claim`
|
||||
- Claim with `in-progress` before working — but **check for an existing claim first** (open PRs
|
||||
referencing the issue, remote branches naming it, comments predating the label, a fresh
|
||||
`git fetch`), because a label prevents duplicate *pickup*, not duplicate *work*: #649 was
|
||||
implemented twice to completion. And re-fetch `origin/main` before every push — a branch on a stale
|
||||
base reverts whatever merged meanwhile. → `process.parallel-session-claim`
|
||||
|
||||
**Building and reviewing**
|
||||
- The PR routine is a fixed sequence; for API changes build the app project FIRST, then
|
||||
|
||||
@@ -76,7 +76,7 @@ Only after Phase 1 sign-off. Per slice (start with #2a Channels):
|
||||
- **One branch = one PR.** PR runs `test` + `migrations` (both **required** to merge). Merge to `main` runs `test`+`migrations`+`build`+smoke/E2E. Verify green before closing each sub-issue.
|
||||
- Migrations only if the model changes — `scripts/add-migration.sh <Name>` does **both** providers.
|
||||
- Adversarial self-review of the diff before closing (see memory: adversarial-self-review-at-milestones). Then Task Completion Protocol / `/done <sub-issue>`.
|
||||
- CI poll: `curl -u timothy:ded89Lm4 …/api/v1/repos/timothy/ersatztv/actions/tasks` (jobs by name), or the runs API.
|
||||
- CI poll: `curl -u "$ETV_GITEA_BASICAUTH" …/api/v1/repos/timothy/ersatztv/actions/tasks` (jobs by name), or the runs API.
|
||||
|
||||
## Repo state at handoff
|
||||
- `main` is green; #1 closed (config/topology, not code — see `docs/m3u-xmltv.md`). #5 triaged as Jellyfin/infra (→ server-management).
|
||||
|
||||
+41
-3
@@ -186,12 +186,50 @@ Re-adding an already-present item is an **idempotent no-op** (no duplicate rows,
|
||||
referenced id does not exist the whole batch is rejected (`422`). So the flow is: search → add ids →
|
||||
re-run to confirm idempotence.
|
||||
|
||||
### Full-replace writes drop what you omit (`mcp.tool-schema-openapi-parity`)
|
||||
|
||||
**Check each tool's own description — the write tools are not uniform, and one is not uniform with
|
||||
itself.** Three are full replaces, where a field you leave out is not "left unchanged" but written as
|
||||
empty: `ersatztv_update_channel`, `ersatztv_update_schedule`, `ersatztv_update_collection_custom_order`.
|
||||
|
||||
`ersatztv_update_playout` is **mixed, and this is the easy one to get wrong**: `scheduleFile` is
|
||||
leave-unchanged, but `dailyRebuildTime` is always applied — `UpdatePlayoutHandler` sets it to `null`
|
||||
unconditionally before re-applying a supplied value, so calling this tool to set `scheduleFile` while
|
||||
omitting `dailyRebuildTime` **silently clears the daily reset**. Send both, or neither.
|
||||
|
||||
The rest are additive or leave-unchanged and say so: `ersatztv_add_collection_items` is an idempotent
|
||||
add (it does **not** replace membership), `ersatztv_update_collection` leaves an omitted
|
||||
`useCustomPlaybackOrder` alone, and `ersatztv_enable_jellyfin_library_sync` leaves an absent row
|
||||
untouched.
|
||||
|
||||
For the full-replace ones, the GET → edit one field → PUT flow is only safe if the tool can express
|
||||
the whole state, and `ersatztv_update_channel` could not — it omitted `graphicsElementIds`, so that flow
|
||||
silently detached every graphics element (including the On Now/Next overlay) with a `200` and no
|
||||
error, visible only as missing pixels at the next transition. `ersatztv_update_schedule` cleared
|
||||
`padToNearestMinute` the same way (ersatztv#754).
|
||||
|
||||
Both are fixed, and the class is now guarded by two tests in `ToolCatalogTests`, comparing against the
|
||||
generated `ErsatzTV/wwwroot/openapi/v1.json`:
|
||||
|
||||
- Every POST/PUT/PATCH tool declares **exactly** the request-body fields its endpoint accepts, each
|
||||
with a matching type. A new property on a request DTO fails until the catalog declares it.
|
||||
- Every tool — read **and** write — declares **exactly** its endpoint's query parameters. An omitted
|
||||
one is not merely undocumented but *unreachable*, since `ToolArgumentValidator` rejects undeclared
|
||||
arguments; that is how #616 hard-capped two paged tools at the first page, and how
|
||||
`ersatztv_list_playouts` (`query`) and `ersatztv_get_playout_items` (`showFiller`) lost their
|
||||
filters until ersatztv#757.
|
||||
|
||||
When adding a write tool, regenerate the spec (`./scripts/update-openapi.sh`) and add the tool to the
|
||||
pinned list in the body test.
|
||||
|
||||
## Deferred
|
||||
|
||||
Channel create/update (`ersatztv_create_channel` / `ersatztv_update_channel`) wrap a 28-field DTO with
|
||||
nine enum fields. Only `name`/`number`/`ffmpegProfileId` are required; the rest have server-side
|
||||
Channel create/update (`ersatztv_create_channel` / `ersatztv_update_channel`) wrap a large DTO with
|
||||
nine enum fields — 27 body fields on create, and 28 on update, which additionally carries
|
||||
`graphicsElementIds`. Only `name`/`number`/`ffmpegProfileId` are required; the rest have server-side
|
||||
defaults, and the enum fields take the enum **name** (the API validates them). Discover an existing
|
||||
channel's shape and current enum values with `ersatztv_get_channel` before creating/updating.
|
||||
channel's shape and current enum values with `ersatztv_get_channel` before creating/updating — and
|
||||
copy its `graphicsElementIds` through unless you mean to detach them.
|
||||
|
||||
Deliberately **not** exposed in this cautious first write pass:
|
||||
|
||||
|
||||
+238
-37
@@ -121,7 +121,7 @@ Convention — when a screen keeps stale results visible during a refetch:
|
||||
current (compare against a ref that always holds the committed value — `SearchScreen` reuses
|
||||
`lastQueryRef`) and **discard** otherwise. Checking only `activeRef` (mounted) is insufficient.
|
||||
|
||||
## 3b. Paged list endpoints clamp server-side — page to completeness ONLY for bounded lists, never a media-library picker
|
||||
## 3b. Paged list endpoints clamp server-side — page to completeness ONLY for bounded lists; a media-library picker searches instead
|
||||
|
||||
Every paged `/api/v1` list endpoint (rerun-collections, multi-collections, library/browse, search,
|
||||
trakt-lists, …) clamps `pageSize` to its own controller's `MaxPageSize` (100, as of #644) regardless
|
||||
@@ -131,14 +131,14 @@ UI to notice the gap. This was issue #644 (following on from #634, which fixed t
|
||||
`SchedulesScreen`'s rerun-collections picker load).
|
||||
|
||||
**Two classes of call site, treated differently** (decision record:
|
||||
`docs/decisions/records/spa/list-completeness-vs-bounded-pickers.md`, `spa.list-completeness-vs-bounded-pickers`
|
||||
— a #644 follow-up review found the original blanket "use `loadAllPages` for any picker" guidance
|
||||
below was itself the defect for one class of caller):
|
||||
`docs/decisions/records/spa/library-pickers-resolve-by-search.md`,
|
||||
`spa.library-pickers-resolve-by-search`, superseding `spa.list-completeness-vs-bounded-pickers` —
|
||||
the #644 follow-up got Class A right and Class B only half right):
|
||||
|
||||
- **Bounded-by-construction lists** (rerun collections, multi-collections, playlists — admin-created,
|
||||
hundreds of rows at most): genuinely need the complete list, and completeness is cheap. Use the
|
||||
shared `loadAllPages` helper (`web/src/api/paging.ts`, re-exported via `web/src/api/index.ts`)
|
||||
instead of an inflated `pageSize`:
|
||||
- **Class A — bounded-by-construction lists** (collections, multi-collections, smart collections,
|
||||
playlists — admin-created, hundreds of rows at most): genuinely need the complete list, and
|
||||
completeness is cheap. Use the shared `loadAllPages` helper (`web/src/api/paging.ts`, re-exported
|
||||
via `web/src/api/index.ts`) instead of an inflated `pageSize`:
|
||||
|
||||
```ts
|
||||
const { items, complete } = await loadAllPages(getMultiCollections); // pages against totalCount, cap defaults to 100
|
||||
@@ -150,41 +150,238 @@ below was itself the defect for one class of caller):
|
||||
a caller that needs the full list must not treat a resolved promise as proof the list is whole (a
|
||||
partial result is otherwise silently indistinguishable from a complete one — the same defect class
|
||||
as #644 itself, since `GetLibraryBrowseItemsHandler.HydrateMediaItems` can legitimately drop stale
|
||||
Lucene hits and produce a short/empty page in normal operation). Pass an `AbortSignal` (4th arg)
|
||||
from the caller's effect cleanup so a superseded load stops issuing further page requests instead
|
||||
of hammering the server for a result nobody will see. **Do not raise the server-side cap to work
|
||||
around this** — the `api.search-allitems-paging` precedent is that the client pages and the server
|
||||
stays bounded; that's a backend decision, out of scope for a screen fix.
|
||||
Lucene hits and produce a short/empty page in normal operation). Render `complete: false` as its own
|
||||
copy ("List may be incomplete — retry to reload"), never through search-narrowing text — a
|
||||
`loadPickerOptions` result that can come from either class carries a `hint: 'incomplete' | 'none'`
|
||||
discriminator, not a boolean shared with an unrelated condition (#644 follow-up round-3 review F1).
|
||||
Pass an `AbortSignal` (4th arg) from the caller's effect cleanup so a superseded load stops issuing
|
||||
further page requests, and gate any `console.warn` on `!signal?.aborted` — a superseded or
|
||||
user-aborted load returns `complete: false` too, and that's expected, not a defect. **Do not raise
|
||||
the server-side cap to work around any of this** — the `api.search-allitems-paging` precedent is
|
||||
that the client bounds itself and the server stays bounded.
|
||||
|
||||
- **Media-library pickers** (`getLibraryBrowseItems` backing a `<select>` for Episode / Song / Image /
|
||||
Movie / MusicVideo / etc. — the largest tables in an install, tens of thousands of rows possible):
|
||||
must **NOT** use `loadAllPages`. Paging to completeness here means on the order of 200 serial
|
||||
requests for a 20k-row library — each more expensive than the last, since
|
||||
`LuceneSearchIndex.Search` computes `hitsLimit = skip + limit` — to populate a native `<select>`
|
||||
with thousands of `<option>` nodes. That is worse than the truncation bug it would "fix". Instead,
|
||||
fetch **one bounded page** directly (`pageSize` at the cap) and make the truncation **visible**
|
||||
rather than silent — e.g. a `ctv-field-help` hint next to the picker: `Showing the first 100 of
|
||||
5000 — use search to narrow.` (wire the response's real `totalCount`). See
|
||||
`RerunCollectionsScreen.tsx`/`PlaylistsScreen.tsx`/`FillerPresetsScreen.tsx`'s `loadPickerOptions`
|
||||
for the pattern. A full typeahead/search-driven picker is a separate, larger feature — out of scope
|
||||
for this fix.
|
||||
- **Class B — media-library pickers** (Episode / Song / Image / Movie / MusicVideo / TelevisionShow /
|
||||
TelevisionSeason / Artist / OtherVideo / RemoteStream — the largest tables in an install, tens of
|
||||
thousands of rows possible): **resolve by search, do not window the type at all** (#651). Neither
|
||||
`loadAllPages` (~200 serial requests for a 20k-row library, each more expensive than the last since
|
||||
`LuceneSearchIndex.Search` computes `hitsLimit = skip + limit`) nor a single bounded page (an
|
||||
arbitrary alphabetical prefix, unusable as a picker even once the truncation is made visible) is
|
||||
acceptable. Render the shared `SearchPicker` (`web/src/schedules/pickers.tsx`) over
|
||||
`searchLibraryPickerOptions` (`web/src/api/libraryBrowse.ts`):
|
||||
|
||||
**A Class B truncation and a Class A `complete: false` are different conditions — don't collapse
|
||||
them into one boolean** (#644 follow-up round-3 review F1): a `loadPickerOptions` result that can
|
||||
come from either a Class A (`loadAllPages`) or Class B (single bounded page) source should carry a
|
||||
`hint: 'incomplete' | 'none' | 'truncated'` discriminator, not a `truncated: boolean` reused for
|
||||
both. `'truncated'` (Class B, an expected cap) keeps the "Showing the first N of M — use search to
|
||||
narrow" copy; `'incomplete'` (Class A, `loadAllPages`'s `complete: false`) renders different copy
|
||||
("List may be incomplete — retry to reload") — rendering both through the search-narrowing text
|
||||
produces a self-contradictory "Showing the first 47 of 47" when a Class A load doesn't converge.
|
||||
Also gate any `console.warn` on a Class A `complete: false` with `!signal?.aborted` — a superseded
|
||||
or user-aborted load returns `complete: false` too, and that's expected, not a defect.
|
||||
```ts
|
||||
const searchLibrary = useCallback( // memoize: SearchPicker lists `search` in its effect deps
|
||||
(q: string) => searchLibraryPickerOptions('Episode', q),
|
||||
[]
|
||||
);
|
||||
```
|
||||
|
||||
The helper owns both bounds: at most ONE `getLibraryBrowseItems` request per settled query, at most
|
||||
`LIBRARY_PICKER_RESULTS` (25) rows, and no request at all below `LIBRARY_PICKER_MIN_QUERY` (2)
|
||||
characters. Selecting a media-library type must issue **zero** requests. The per-kind
|
||||
`LIBRARY_PICKER_RESULTS` cap is the only truncation this class has — there is no whole-type window
|
||||
left to hint at, so the old `Showing the first 100 of 5000 — use search to narrow.` copy is gone
|
||||
from these pickers along with the window it described. Surfacing the per-kind cap is *permitted*
|
||||
wherever it is reachable, and *required* only where bulk selection makes the count actionable — see
|
||||
the `AddItemsDialog` sub-bullet below, which sums the cap across kinds and renders a `Showing N of
|
||||
M matches` hint for exactly that reason. Prove the bound with a **request-count assertion against a
|
||||
large (20k-row) fixture**, not by inspection.
|
||||
|
||||
- **`SearchPicker` is the single-select SHAPE, not the rule itself.** A MULTI-select picker
|
||||
(`CollectionsScreen`'s `AddItemsDialog` — checkbox rows, many items added at once, fanned out
|
||||
over several kinds) cannot render `SearchPicker` and must not be forced to. It satisfies this
|
||||
section by taking the same *constraints* the helper enforces for single-select — the gate on
|
||||
`LIBRARY_PICKER_MIN_QUERY`, `titleContainsQuery` the typed text, `LIBRARY_PICKER_RESULTS` per
|
||||
kind — via `searchLibraryBrowseItems` (`web/src/api/libraryBrowse.ts`), a sibling of
|
||||
`searchLibraryPickerOptions` that returns full `LibraryBrowseItem` rows plus `totalCount`
|
||||
instead of `{id, name}`, so the bound lives in the helper rather than the caller (#685 review
|
||||
finding 2). There is no post-fetch `slice`, but the per-kind cap can still truncate the real
|
||||
match count — this is a bulk multi-select add, where "add the 40 matching episodes" is a
|
||||
first-class use, so `AddItemsDialog` sums each kind's `totalCount` and renders a `Showing N of
|
||||
M matches` hint once it exceeds the rendered rows (finding 4 — an earlier revision of this
|
||||
bullet called the truncation nothing left to hint at). **The gate's home is the shared HELPER,
|
||||
not the screen — however single-sink the screen's own function looks.** #685 got this wrong
|
||||
twice in a row, and the second time is the instructive one: the check sat inside `runSearch`,
|
||||
which genuinely IS the one sink both entry paths route through, so it read as correct. It was
|
||||
still a duplicate of the helper's gate, and the two masked each other: as of `4be3f247d` —
|
||||
which had no unit tests on the helper — deleting EITHER copy left the whole suite green, so the
|
||||
min-query boundary test pinned nothing. Removing the screen's copy is what made the helper's
|
||||
gate load-bearing. **The invariant, not the count: every gate must have at least one test that
|
||||
reddens when that gate ALONE is removed.** A guard you cannot redden is not a guard, and "it's
|
||||
the single sink" is not evidence that it is the only one. **Outstanding on this screen**: `AddItemsDialog` still lacks the monotonic `seqRef`
|
||||
stale-response guard and `useIsMountedRef()` — the same class of guard "Debounced typeaheads"
|
||||
below mandates there, applied to a debounced-while-typing fetch; `AddItemsDialog` is an explicit
|
||||
Search-button submission, not a typeahead, so that mandate doesn't reach it directly, but the
|
||||
same race (a superseded search settling after a newer one) can still occur here — tracked in
|
||||
**ersatztv#740**, not yet fixed here.
|
||||
|
||||
- **Compile typed text; never forward raw Lucene.** Send `titleContainsQuery(text)` →
|
||||
`title:*<escaped>*`. The index's default field does not match bare title words (`Alpha` finds
|
||||
nothing for "Show Alpha" — see `e2e-local.md`), so a raw forward looks broken in a *name* picker.
|
||||
Reuse the helper; do not re-implement the escaping (same rule as the #440 Auto-Tune typeahead,
|
||||
same shape `builder/rules/compile.ts` emits for `contains`). The escaped set includes `&` and
|
||||
`|`, because Lucene's boolean operators are `&&`/`||` and a title like `Rock & Roll` otherwise
|
||||
compiles to something Lucene parses as syntax. **Drive the escaping test from the exported
|
||||
character set** (`LIBRARY_PICKER_LUCENE_SPECIALS`), one character per case — a test carrying its
|
||||
own hand-copied "every special" sample cannot see what is missing from that sample.
|
||||
- **The bound belongs to the helper, not the caller.** `searchLibraryPickerOptions` *clamps*
|
||||
`pageSize` to `LIBRARY_PICKER_RESULTS`; a documented bound a caller can exceed by passing a
|
||||
bigger number is not a bound.
|
||||
- **Render the current selection from the owning record, not from the result set.** An item already
|
||||
selected but outside the current results must still display — losing it on edit is data loss, not
|
||||
a cosmetic defect. Rerun collections and playlist items carry `selectedName` on their own DTOs;
|
||||
`FillerPresetFullResponseModel` stores only an id, so its edit path resolves the name with a
|
||||
single by-id detail read (`getShow`/`getSeason`/`getArtist`) and degrades to `#id` on failure —
|
||||
never to a cleared field.
|
||||
- **A read model that derives an id and its name from the same eager-loaded navigation reports
|
||||
*no selection at all* when that navigation isn't loaded** — a successful 200 indistinguishable
|
||||
from "the user cleared it". (`RerunCollectionsController.ProjectToResponseModel` does exactly
|
||||
this, and `MediaCollections/Mapper` maps RemoteStream through `_ => null`; tracked as **#671**.)
|
||||
An earlier revision of this section required a client-side guard that preserved the id across
|
||||
such a response. **That guard is gone and must not be rebuilt** — it only ever preserved a value
|
||||
seeded from the list row, which is itself null in production for every row, and the reconciliation
|
||||
it required is what the initialize-once rule below replaced. The correct handling is to show the
|
||||
server's answer honestly: no selection, Save disabled, and the validation badge saying why.
|
||||
- **An id NEVER travels without its namespace — in results, in options, in cached result sets.**
|
||||
A media/collection id only means anything inside the type that produced it, so any structure
|
||||
holding ids must hold the type too, and identity is compared as `(type, id)`. Three places this
|
||||
bites, all the same bug:
|
||||
1. A typeahead's cached results must be keyed on `(source, query)`, not the query text. Same
|
||||
query, different source ⇒ the results are not *stale*, they are *wrong*: hide them and
|
||||
re-query. Keying on text alone lets a re-query guard **suppress** the new source's request
|
||||
and leave the old namespace's hit clickable under the new label.
|
||||
2. List-backed `<select>` options must carry the type they were loaded for and be dropped the
|
||||
moment the active type differs — otherwise the previous type's rows stay selectable during
|
||||
the replacement load on a slow connection.
|
||||
3. A local edit whose type contradicts the server's is a **conflict**, not something to
|
||||
reconcile (below).
|
||||
- **Initialize an edit draft ONCE, from the detail read — never reconcile a late response against
|
||||
an open form.** This supersedes an earlier prescription here for merging a refresh into a draft
|
||||
field-by-field/atomically with touched-field tracking. That reconciliation layer produced a HIGH
|
||||
finding in three consecutive review rounds of #651, including three cross-user lost updates, and
|
||||
the last of them (a merge with no immutable baseline, so it could not tell a local edit from a
|
||||
server change) is unfixable without adding a third-way baseline — more machinery on the surface
|
||||
that was generating the bugs. Instead:
|
||||
- `draft` starts as `null` for an existing record and the form does not render until the detail
|
||||
GET lands. There is then no draft for a late response to reconcile against, and no window in
|
||||
which the user can edit something about to be replaced.
|
||||
- **Do not seed from the list row.** It is not authoritative: for rerun collections the list
|
||||
handler applies zero `.Include()`s while the detail handler applies fourteen, and both project
|
||||
through the same mapper, so the list response is a strict SUBSET of the detail one (#671). A
|
||||
seed can only add a race, never information. Verify that claim for your endpoint before
|
||||
relying on it.
|
||||
- **Fail CLOSED on a missing concurrency token.** Writing the ETag in the same callback that
|
||||
sets the draft is *not* the same as "a draft implies an ETag" — the response can simply omit
|
||||
the header, and then the PUT carries no `If-Match` and silently force-writes. No token ⇒ no
|
||||
editable draft (error + Retry/Back). Note this makes your test mocks load-bearing: a detail
|
||||
mock that omits `ETag` was previously exercising the force-write path without saying so, so
|
||||
give every single-record GET mock a real ETag and test the absent case explicitly.
|
||||
- **Bound the load and always offer a way out.** A caller-supplied fetch with no abort signal can
|
||||
hang forever; race it against a deadline, and give the loading view a Back control so a hung
|
||||
request is never a dead end.
|
||||
- **Detect conflicts at save time** via the existing `If-Match` → 412 → Reload path. Reload sets
|
||||
the draft back to `null` and re-runs the same initialize-once load, so "replace" needs no
|
||||
separate policy and the form is unmounted while the replacement is in flight.
|
||||
|
||||
`FillerPresetsScreen` and `PlaylistsScreen` already worked this way; `RerunCollectionsScreen` was
|
||||
the outlier that seeded from its list row, which is where every one of these defects lived.
|
||||
- **A name resolved asynchronously must be keyed to the id it was resolved FOR**, and must refuse
|
||||
to overwrite a label that already names a different id. A slow by-id read landing after the user
|
||||
has picked something else would otherwise label the new selection with the old item's title
|
||||
while the id — and therefore what gets saved — says otherwise. Keep the guard at the *writer*,
|
||||
where it is reachable and testable; a second render-time id comparison is unreachable once
|
||||
every writer sets the label and the id together, and an unreachable guard is an untested one.
|
||||
- **Only Lucene-backed types.** `GetLibraryBrowseItemsHandler` applies `query` as a Lucene clause
|
||||
for media items but as a plain SQL `LIKE` on `Name` for the collection-family types (Collection /
|
||||
SmartCollection / MultiCollection / RerunCollection / Playlist). A compiled `title:*x*` sent at
|
||||
those matches nothing literally. Keep the collection-family pickers on their Class A / bounded
|
||||
single-page loads — `FillerPresetsScreen`'s `COLLECTION_TYPES` marks the search-driven entries
|
||||
with `searchable: true` for exactly this reason.
|
||||
|
||||
**If a screen shows a bounded preview or has real paging UI** (a "load more" button, a page-size
|
||||
selector, a fixed-size typeahead result list), a `pageSize` at or below the cap is correct as-is —
|
||||
`loadAllPages` is only for "I need literally everything, and the list is small by construction"
|
||||
call sites.
|
||||
|
||||
**Debounced typeaheads: arm on focus, and guard on mounted as well as on sequence.** A typeahead that
|
||||
fetches on *mount* multiplies by the number of rows on screen (an N-rule tree fired N unrequested
|
||||
facet lookups before #578); arm the effect on the input's `onFocus` instead. And pair the monotonic
|
||||
`seqRef` stale-response guard with the shared `useIsMountedRef()` (`web/src/hooks.ts`) in every async
|
||||
callback — `seqRef` drops an *older* response, but says nothing about whether the component still
|
||||
exists.
|
||||
|
||||
**A custom picker replacing a native control owes you its keyboard behaviour.** A `<select>` is
|
||||
fully keyboard-operable; swapping in a listbox-and-input is an accessibility *regression* unless it
|
||||
implements the ARIA combobox pattern — `role="combobox"` + `aria-expanded`/`aria-controls`/
|
||||
`aria-autocomplete` on the input, ArrowUp/ArrowDown to move a virtual cursor exposed via
|
||||
`aria-activedescendant`, Enter to commit, Escape to dismiss, options as non-tab-stops
|
||||
(`tabIndex={-1}`) marked with `aria-selected`. Note this changes what `getAllByRole('combobox')`
|
||||
matches in tests: count `<select>` elements when that is what you mean. Two failure modes that only
|
||||
appear once the widget is asynchronous:
|
||||
|
||||
- **Freshness is `(source, query)`, and cached failures are not answers — but they are not licences
|
||||
to retry either.** A `SearchPicker`-style cache must record which source produced the results and
|
||||
whether the attempt *succeeded*. Caching a failure as an authoritative empty result turns a
|
||||
transient 500 into a permanent "No matches" that reopening can never clear. But simply declining
|
||||
the cached failure re-runs the effect and schedules another request every debounce — a **request
|
||||
storm** on a persistent outage. Keep the two apart: a `resultsFor.ok` flag says whether the held
|
||||
answer is authoritative, and a separate *attempted* key (a ref, so writing it doesn't re-render)
|
||||
suppresses automatic retries until an explicit user action — reopen, focus, or edit — re-arms it.
|
||||
- **Put a validity predicate at the BOUNDARY the class crosses, not at the site the bug was found.**
|
||||
An entity-reference id (`selectedId`, `collectionId`, `mediaItemId`, …) is bound by the API as a
|
||||
32-bit integer, so `1.5` or `2147483648` renders and commits fine and then fails on write. Such
|
||||
ids enter editor state through *several* doors — search results, list-backed `<select>` options,
|
||||
and the selection restored from a detail read — so a check added to whichever one surfaced the
|
||||
defect leaves the others open (this is how #651 produced the same finding in two consecutive
|
||||
rounds). Share one predicate (`isSelectionId` / `selectionIdOrNull` in `web/src/api/selectionId.ts`)
|
||||
and apply it on every path — including the ones that don't look like pickers, such as a
|
||||
`playlistGroupId` seeded from the wire into a create dialog. **Treat an unbindable id as ABSENT,
|
||||
never coerce it** — rounding `1.5` to `1` would submit a *different* record — and **clear its
|
||||
label with it**: a row still reading "Blade Runner" over a null id makes two contradictory
|
||||
statements about the same item. Drop rather than render an option that cannot be selected safely.
|
||||
"Surfaces as no selection" is only true if that screen's Save gate actually checks for one — on
|
||||
`PlaylistsScreen` it did not, so this claim was false there for a full round after being written
|
||||
here. **Verify an invariant on every screen it names before writing it down.** Prove it per
|
||||
ingress by asserting zero writes are reachable *after attempting the write*: a write-count
|
||||
assertion on a path that never attempts one is trivially true. And unit-test the predicate's
|
||||
INCLUSIVE endpoints directly — once it is the single point of failure for every ingress, a `>`
|
||||
for `>=` slip passes an entire screen suite.
|
||||
- **A caller-supplied promise needs a deadline, and a 2xx body is not a contract.** `client.ts`
|
||||
turns malformed JSON into `undefined` rather than rejecting, so `setResults(undefined)` throws on
|
||||
the next render. Validate the **elements, not just the container**: `Array.isArray` accepts
|
||||
`[null]`, which then throws on `option.id` during render, and an element with a wrong-typed `id`
|
||||
commits an invalid value through `onSelect`. Treat any malformed payload as a failed attempt (so
|
||||
it stays retryable), not as an empty answer. And a `search` prop carries no abort signal, so race
|
||||
it against a timeout — otherwise a never-settling request leaves the picker spinning with no way
|
||||
back.
|
||||
- **A stale result set must not be committable — by ANY modality.** Between a keystroke and its
|
||||
response, `results` still describe the *previous* query, so highlighting an option, retyping, and
|
||||
pressing Enter commits the old option while the box reads the new text. Drop the highlight on
|
||||
**input change** (not when the next response arrives) and put the guard in the single `choose()`
|
||||
sink rather than on each call site — gating Enter and leaving `onClick` open is the same defect in
|
||||
another modality, and the next path added would be ungated too. Keep the stale list *visible*
|
||||
(hiding it flickers on every keystroke) but genuinely inert: `aria-disabled` plus a dimmed style,
|
||||
not merely a handler that silently no-ops on a normal-looking button.
|
||||
- **Escape must not strand the user.** Closing the popup while focus stays in the input means
|
||||
`onFocus` can never re-arm it, so typing does nothing and the user has to blur and refocus to
|
||||
recover. Typing and ArrowDown must both reopen it — and reopening onto results that are already
|
||||
current must **not** re-query: the duplicate response lands later and resets the cursor the user
|
||||
has since moved, so Enter silently does nothing. Reopening also places the cursor (ARIA APG)
|
||||
rather than swallowing the keypress.
|
||||
|
||||
**The web typecheck gate is `npm run typecheck`, never `npx tsc --noEmit`.** `web/tsconfig.json` is
|
||||
solution-style (`"files": []` + `references`), so a bare `tsc --noEmit` resolves to zero input files
|
||||
and exits 0 **without checking anything** — a green that means "I looked at nothing". CI runs
|
||||
`npm run typecheck` (`tsc -b --pretty false`), which builds the referenced projects and includes the
|
||||
test files. Verified by planting a deliberate type error: `--noEmit` stayed green, `-b` caught it.
|
||||
|
||||
**Testing an is-mounted guard: React 19 does not warn on a setState-after-unmount, and an unmounted
|
||||
tree renders nothing either way** — so no DOM assertion can distinguish "the guard stopped it" from
|
||||
"React discarded it". Prove the *mechanism* (a `useIsMountedRef` unit test, with a StrictMode
|
||||
double-invoke for the re-arm) **and** the *integration* (mock the hook module and assert the
|
||||
component actually read `current` — and saw `false` — when the late response landed). Verify each by
|
||||
removing the mechanism and confirming the test fails.
|
||||
|
||||
## 4. API client modules
|
||||
|
||||
One file per domain in `web/src/api/`, e.g. `logs.ts`, `blocks.ts`, `playouts.ts`. Pattern (see
|
||||
@@ -619,12 +816,16 @@ not wired here.
|
||||
for these two fields — the compiled query is the existing `released_inthelast:"7 day"`-style
|
||||
`CustomMultiFieldQueryParser` macro, so nothing downstream changes. `validation.ts`'s `ruleError`
|
||||
requires the value to parse as a positive integer before it's compiled.
|
||||
- **Facet-value typeahead** (#434, `api.search-field-values`) — the value input for a `text` field
|
||||
- **Facet-value typeahead** (#434/#578, `api.search-field-values-sources`) — the value input for a `text` field
|
||||
(not enum) is a combobox backed by `getSearchFieldValues` (`web/src/api/search.ts` →
|
||||
`GET /api/v1/search/fields/{name}/values?q=&limit=`), debounced on keystroke, prefix-matching the
|
||||
in-progress value against distinct terms already in the index. It always allows free-text entry as a
|
||||
fallback — a 404 (non-text field) or an empty result list (e.g. ElasticSearch backend) degrades to a
|
||||
plain text input rather than blocking the rule.
|
||||
plain text input rather than blocking the rule. Since #578 (`api.search-field-values-sources`)
|
||||
`album_artist` returns values instead of 404ing, and `artist` covers free-text music-video/song credits
|
||||
as well as entity artists; for those two the server's list is **bounded best-effort** on a very large
|
||||
library, so the free-text fallback stays load-bearing — never treat an absent suggestion as an invalid
|
||||
value.
|
||||
- **Single-child-group normalization** (#438) — `normalizeGroup` (`validation.ts`) coerces a group's
|
||||
`match` to `all` whenever it has fewer than two children, recursively. A one-child `any` group is
|
||||
semantically identical to `all` but doesn't round-trip through `compile`→`parse` (the compiled Lucene
|
||||
|
||||
Executable
+247
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env bash
|
||||
# Per-step execution markers for the two REQUIRED docker-build.yml jobs (ersatztv#756).
|
||||
#
|
||||
# WHY THIS EXISTS. A `run:` body the runner declines to interpolate is DROPPED, and the job still
|
||||
# concludes `success` (ersatztv#751, `ci.workflow-run-body-no-expressions`). In
|
||||
# `review-verdict.yml` that is fail-CLOSED — the required `review-verdict/h10` is simply absent and
|
||||
# the merge is blocked. In `docker-build.yml` it is fail-OPEN: `Build & test (.NET)` and
|
||||
# `EF migration integrity (SQLite + MySql)` are the other two required contexts on `main`, so a
|
||||
# dropped step there sends a required check GREEN having done no work. #751 guarded the safe
|
||||
# direction because that is where the live bug was, not because these were checked.
|
||||
#
|
||||
# WHY PER STEP, NOT PER JOB, which is what #756 proposed. A marker written by the job's FIRST step
|
||||
# only proves the job started. The dangerous drop is not step 1 — it is `Test`, or the migration
|
||||
# replay: the job runs everything around them, reports green, and nothing ran that anyone cared
|
||||
# about. A guard that cannot see the fail-open case it was built for is the "guard that never
|
||||
# executed" failure one level up. So every consequential step marks itself and a trailing guard
|
||||
# asserts the whole expected SET.
|
||||
#
|
||||
# THAT GUARD CARRIES NO `if:` — unlike the #751 one, which uses `if: always()` because its job has a
|
||||
# single real step. These jobs have a dozen, and a genuine early failure legitimately skips every
|
||||
# later step, so `always()` would print a false "these steps never executed" on top of every ordinary
|
||||
# red build. The default `success()` is the wanted condition: the guard is skipped only when an
|
||||
# earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every green
|
||||
# path runs the guard.
|
||||
#
|
||||
# WHY A SCRIPT AND NOT AN INLINE BODY, unlike the #751 guard. Two reasons, and the second is the
|
||||
# load-bearing one:
|
||||
#
|
||||
# * The path literal exists ONCE. The #751 guard carries it twice (write + assert) and its tests
|
||||
# spend real effort proving the two copies agree, because a divergence reddens every run and
|
||||
# then gets deleted as broken. Here they cannot diverge.
|
||||
# * A one-line `run: scripts/ci-step-ran.sh …` CANNOT CONTAIN AN EXPRESSION DELIMITER, so the
|
||||
# mechanism this guards against cannot drop the guard itself. #751's own record names this as
|
||||
# the stronger construction ("the body would have had to move into scripts/, where a one-line
|
||||
# run: makes the class unreachable") and settled for inline only because the measurement showed
|
||||
# it was not required there.
|
||||
#
|
||||
# WHY A SCRIPT IS ACCEPTABLE HERE THOUGH IT WOULD NOT BE IN review-verdict.yml. That workflow
|
||||
# checks out the PR's BASE precisely so a PR cannot supply the code that judges it. `docker-build.yml`
|
||||
# is head-resolved by design — a PR already supplies every test this job runs — so calling a script
|
||||
# from the head adds no authority a PR did not already have. This is a CORRECTNESS gate against
|
||||
# silent no-ops, not a security gate against a hostile PR; that job belongs to `review-verdict/h10`.
|
||||
# Do not copy this reasoning back into the gate workflow.
|
||||
#
|
||||
# THE MARKER FILE IS KEYED ON THE RUN, and BE PRECISE ABOUT WHY — the obvious justification is a
|
||||
# #751 measurement that does NOT transfer to these jobs, and saying so is the point. #751 measured
|
||||
# `RUNNER_TEMP` to be `/tmp` and called it "not a private per-job directory"; that was taken on
|
||||
# `review-verdict.yml`, which runs WITHOUT a `container:`. `test` and `migrations` run INSIDE the CI
|
||||
# toolchain image, so their `/tmp` is the job container's own and starts empty. That follows from
|
||||
# `container:`, NOT from a measurement: the build-lane probe confirmed only that `RUNNER_TEMP` is
|
||||
# `/tmp` here (the marker landed at `/tmp/etv-ci-steps-ran-test-1910-1`) — it says nothing about the
|
||||
# directory being private or empty, and an earlier draft of this comment cited it as though it did.
|
||||
# The fresh container is what actually rules out a stale marker here; the keying is defence in depth.
|
||||
#
|
||||
# It is kept because container-per-job is a property of how the lane is configured today, not a
|
||||
# guarantee, and a STALE marker is the one failure that makes this guard PASS on a run whose step was
|
||||
# dropped — a silent success, i.e. the exact thing being removed. Cheap insurance against a lane
|
||||
# change nobody would think to re-check this against.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
ci-step-ran.sh mark <key>
|
||||
Record that this step began executing. Call it as the step's FIRST act, before
|
||||
anything in the body can fail.
|
||||
|
||||
ci-step-ran.sh assert --always <key>... [--gated <key>...]
|
||||
Fail unless every expected key was marked. --always keys are always required.
|
||||
--gated keys are required only when the job's skip gates did NOT fire, read from
|
||||
ETV_DOCS_ONLY / ETV_REVALIDATE_SKIP so this mirrors the steps' own `if:`.
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
# NO SILENT FALLBACK FOR THE RUN IDENTITY — found by cold review. The first version defaulted to
|
||||
# `nojob`/`norunid`/`1`, and those are REUSABLE: with `GITHUB_RUN_ID` unset, every run on the host
|
||||
# would share ONE marker file, so a leftover from any earlier run would satisfy the guard on a run
|
||||
# whose step was dropped. A silent PASS — the exact failure the keying exists to remove, reintroduced
|
||||
# by the code meant to implement it.
|
||||
#
|
||||
# THE TWO HALVES ARE TREATED DIFFERENTLY, ON EVIDENCE, because the blast radii differ and this is a
|
||||
# REQUIRED check — a wrong refusal deadlocks every merge, so strictness is not free:
|
||||
#
|
||||
# * `GITHUB_JOB` and `GITHUB_RUN_ID` are MEASURED present on this runner (#756's build-lane probe
|
||||
# wrote `/tmp/etv-ci-steps-ran-test-1910-1`; `test` is the job id and 1910 is the real API run
|
||||
# id). Absence would mean the runner changed under us, so refusing is safe AND correct.
|
||||
# * `GITHUB_RUN_ATTEMPT` is measured present TOO, as of ersatztv#756's own PR run — but note how,
|
||||
# because the first two attempts to settle it were both bad. Grepping a job log for the variable
|
||||
# NAME proves nothing (logs do not dump the environment). Inferring it from the ABSENCE of this
|
||||
# script's "not set" warning proves nothing either, because that warning goes to stderr and
|
||||
# whether step stderr reaches a job log here was itself never established. So the script was made
|
||||
# to REPORT its resolved identity on stdout, where capture is not in question, and the answer was
|
||||
# then simply read off run 1916: `Marker identity: job=test run=1916 attempt=1 (from the runner)`
|
||||
# and the same for `migrations`. Both required jobs, on the lane that matters.
|
||||
#
|
||||
# That measurement is what promoted it from warn-and-default to REQUIRED, which is why the residual
|
||||
# this comment used to describe — a rerun inheriting attempt 1's markers — no longer exists. If a
|
||||
# future runner stops exporting any of the three, every job reddens with a message naming the
|
||||
# variable; that is loud, instantly diagnosable, and the correct direction for a required check.
|
||||
marker_path() {
|
||||
local missing=""
|
||||
[ -n "${GITHUB_JOB:-}" ] || missing="$missing GITHUB_JOB"
|
||||
[ -n "${GITHUB_RUN_ID:-}" ] || missing="$missing GITHUB_RUN_ID"
|
||||
[ -n "${GITHUB_RUN_ATTEMPT:-}" ] || missing="$missing GITHUB_RUN_ATTEMPT"
|
||||
if [ -n "$missing" ]; then
|
||||
# NOTHING IS PRINTED TO STDOUT HERE, and that is load-bearing rather than style: this
|
||||
# function's stdout IS its return value (it is always called inside `$( )`), so a notice
|
||||
# printed here is captured INTO the path. An earlier revision did exactly that and both
|
||||
# sub-commands then failed on a nonexistent directory. Caught by
|
||||
# test_a_degraded_run_IDENTITY_*, which is why that test asserts on the exit status and on
|
||||
# the absence of any marker file rather than only on the message.
|
||||
echo "::error::ci-step-ran.sh cannot identify this run —${missing} not set. The marker path would fall back to a name other runs also use, and a stale marker would make the dropped-step guard PASS on a run whose step never executed (ersatztv#756). Refusing rather than degrading to a reusable name." >&2
|
||||
exit 3
|
||||
fi
|
||||
printf '%s/etv-ci-steps-ran-%s-%s-%s' \
|
||||
"${RUNNER_TEMP:-${GITHUB_WORKSPACE:-/tmp}}" \
|
||||
"$GITHUB_JOB" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT"
|
||||
}
|
||||
|
||||
cmd_mark() {
|
||||
[ "$#" -eq 1 ] && [ -n "$1" ] || usage
|
||||
# Appended, never truncated: every step in the job shares one file, and a `>` here would erase
|
||||
# its predecessors and make the guard red on every run.
|
||||
#
|
||||
# A failure to write is NOT swallowed. The step is running under `bash -e`, so a non-zero here
|
||||
# fails the step and reddens the job — which is the same direction the guard would take a moment
|
||||
# later, but with a message pointing at the real cause instead of at a missing marker.
|
||||
local target
|
||||
# NOT `>> "$(marker_path)"`: the refusal above `exit`s a SUBSHELL there, and bash discards a
|
||||
# command substitution's exit status when it is only part of a redirection — the write would go
|
||||
# to an empty path and the error would read as a redirection failure rather than the real cause.
|
||||
target="$(marker_path)" || exit $?
|
||||
printf '%s\n' "$1" >> "$target"
|
||||
}
|
||||
|
||||
cmd_assert() {
|
||||
local -a always=() gated=()
|
||||
local bucket=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--always) bucket=always ;;
|
||||
--gated) bucket=gated ;;
|
||||
-*) usage ;;
|
||||
*)
|
||||
case "$bucket" in
|
||||
always) always+=("$1") ;;
|
||||
gated) gated+=("$1") ;;
|
||||
*) usage ;;
|
||||
esac ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
# ANTI-VACUITY, at runtime rather than only in the test suite. An `assert` called with no
|
||||
# expectations passes unconditionally and reports "every expected step executed" — a guard that
|
||||
# proves nothing while looking like it proved everything. Refuse instead.
|
||||
if [ "${#always[@]}" -eq 0 ] && [ "${#gated[@]}" -eq 0 ]; then
|
||||
echo "::error::ci-step-ran.sh assert was called with no expected keys, so it would pass unconditionally. This is a workflow bug, not a build failure." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# The skip gates, mirroring the `if:` every gated step carries:
|
||||
# steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
# Anything other than the exact string `true` means the step was expected to run — including the
|
||||
# EMPTY string, which is what these read as when the detect step itself was dropped. That
|
||||
# direction is deliberate: a dropped detect step must widen what is required, never narrow it.
|
||||
local skipped=no
|
||||
if [ "${ETV_DOCS_ONLY:-}" = "true" ] || [ "${ETV_REVALIDATE_SKIP:-}" = "true" ]; then
|
||||
skipped=yes
|
||||
fi
|
||||
|
||||
local marker attempt_used
|
||||
# `|| exit $?` because `set -e` does NOT fire on a failing command substitution in an assignment;
|
||||
# without it a degraded identity would leave `marker` empty and every key would read as missing —
|
||||
# fail-closed by luck, with a misleading message.
|
||||
marker="$(marker_path)" || exit $?
|
||||
# Read the attempt back OFF THE RESOLVED PATH rather than from the environment. It reports what
|
||||
# the path was actually keyed on, so a future change to how the path is built cannot silently
|
||||
# disagree with the line that documents it.
|
||||
attempt_used="${marker##*-}"
|
||||
# `${arr[@]+"${arr[@]}"}` rather than a bare `"${arr[@]}"`: under `set -u` bash 3.2 (the system
|
||||
# bash on the Macs this suite also runs on) treats expanding an EMPTY array as an unbound
|
||||
# variable and aborts. The CI image ships bash 5, where it is fine — which is exactly the kind of
|
||||
# difference that makes a guard pass locally and die on the runner, or the reverse.
|
||||
local -a expected=(${always[@]+"${always[@]}"})
|
||||
if [ "$skipped" = no ]; then
|
||||
expected+=(${gated[@]+"${gated[@]}"})
|
||||
else
|
||||
echo "Skip gate fired (docs_only='${ETV_DOCS_ONLY:-}', already_validated='${ETV_REVALIDATE_SKIP:-}') — the gated steps were not expected to run."
|
||||
fi
|
||||
|
||||
# RE-CHECKED AFTER GATING, not only on argv — found by cold review, which reproduced it:
|
||||
# `ETV_DOCS_ONLY=true … assert --always --gated foo` printed "All 0 expected step(s) executed"
|
||||
# and exited 0. The argv check above cannot see that, because the set is emptied by the gate, not
|
||||
# by the caller. Unreachable with today's argv (both jobs pass `--always detect revalidate`), but
|
||||
# it directly contradicted the comment above it, and a guard that reports proving everything
|
||||
# while proving nothing is the failure this whole file exists to remove.
|
||||
if [ "${#expected[@]}" -eq 0 ]; then
|
||||
echo "::error::ci-step-ran.sh assert ended up with NO expected keys after the skip gate, so it would pass unconditionally. This is a workflow bug, not a build failure." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
local -a missing=()
|
||||
local key
|
||||
for key in "${expected[@]}"; do
|
||||
# `grep -qxF` over a FILE, never a pipeline: `grep -q` exits at its first match and would
|
||||
# SIGPIPE a producer, which under `set -o pipefail` inverts the result for large inputs
|
||||
# (ersatztv#698). Reading the file directly has no producer to kill. `-x` so a key cannot be
|
||||
# satisfied by another key that contains it, `-F` so a key is never read as a pattern.
|
||||
if ! grep -qxF "$key" "$marker" 2>/dev/null; then
|
||||
missing+=("$key")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${#missing[@]}" -gt 0 ]; then
|
||||
echo "::error::These steps of job '${GITHUB_JOB:-?}' never executed: ${missing[*]}. The runner DROPPED them (an interpolation failure over a run: body does this and still reports the job GREEN — ersatztv#751/#756) or their \`if:\` no longer matches the guard's expectations. This job is a REQUIRED check, so a green here would mean a required context passed having done no work. Failing the job so it is visible."
|
||||
if [ -f "$marker" ]; then
|
||||
echo "Marker file ${marker} recorded:"
|
||||
sed 's/^/ /' "$marker"
|
||||
else
|
||||
echo "There is no marker file at ${marker} at all — not one step of this job executed."
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
# The resolved identity, on stdout, every run. This is what turns "is GITHUB_RUN_ATTEMPT
|
||||
# exported here?" from an inference into something a reader just looks up — and it is why the
|
||||
# variable is still WARN-and-default rather than REFUSE: `GITHUB_JOB` and `GITHUB_RUN_ID` have
|
||||
# positive evidence (the probe's marker filename), this one does not yet, and refusing on an
|
||||
# unestablished variable would redden a REQUIRED check. Promote it once a run has printed
|
||||
# `attempt=<n> (from the runner)`.
|
||||
# Kept after the promotion, though all three components are now required and the line can no
|
||||
# longer report anything but the runner's own values. It is the standing evidence: this is the
|
||||
# line that settled whether GITHUB_RUN_ATTEMPT is exported, and it is what a future reader checks
|
||||
# first if the keying is ever doubted again.
|
||||
echo "Marker identity: job=${GITHUB_JOB} run=${GITHUB_RUN_ID} attempt=${attempt_used} (from the runner)"
|
||||
echo "All ${#expected[@]} expected step(s) executed: ${expected[*]}"
|
||||
}
|
||||
|
||||
[ "$#" -ge 1 ] || usage
|
||||
sub="$1"
|
||||
shift
|
||||
case "$sub" in
|
||||
mark) cmd_mark "$@" ;;
|
||||
assert) cmd_assert "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -17,6 +17,7 @@ import subprocess
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
import scripts.decisions_lib as dl # noqa: E402 (run with PYTHONPATH=. or as module)
|
||||
|
||||
@@ -52,6 +53,18 @@ _STALE_AFTER_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
# guards the calibration claim asserts against the SAME value the CLI uses and the two cannot drift.
|
||||
RECORD_CEILING_DEFAULT = 60
|
||||
|
||||
# The coarse, non-ratcheting calibration bound (#688): the ceiling must flag a MEANINGFUL MINORITY
|
||||
# of records. Below the floor it is parked among the outliers and names almost nobody; above the cap
|
||||
# it is cutting into the bulk rather than marking a tail. See `ceiling_calibration` for why the fine
|
||||
# percentile claim is reported instead of asserted.
|
||||
#
|
||||
# The floor is NOT "at least one record" — that was the first draft and it was nearly unfalsifiable:
|
||||
# measured on the live corpus it accepted every ceiling from 39 to 229, including the ceiling of 200
|
||||
# this module's own docstring offered as the case it catches (one 230-line record keeps the count
|
||||
# nonzero). A 2% floor rejects 200/229/230 and still leaves ~5x headroom below today's 9.8%.
|
||||
CEILING_MINORITY_MIN = 0.02
|
||||
CEILING_MINORITY_MAX = 0.25
|
||||
|
||||
|
||||
def _parse_stale_after(value: str | None) -> date | None:
|
||||
"""`stale-after` as a date, or None if absent, empty, or malformed.
|
||||
@@ -541,6 +554,102 @@ def record_wing_faults(records_dir: Path | None = None, archive_dir: Path | None
|
||||
return faults
|
||||
|
||||
|
||||
def _frontmatter_block(text: str) -> str | None:
|
||||
"""The raw YAML between the opening `---` and the next `---`, or None if there isn't one."""
|
||||
if not dl.has_frontmatter(text):
|
||||
return None
|
||||
lines = text.splitlines()
|
||||
end = next((i for i, ln in enumerate(lines[1:], start=1) if ln.rstrip() == "---"), None)
|
||||
if end is None:
|
||||
return None
|
||||
return "\n".join(lines[1:end])
|
||||
|
||||
|
||||
def pyyaml_frontmatter_faults(files) -> tuple[list[str], bool]:
|
||||
"""Faults where PyYAML disagrees with the dependency-free reader. Returns (faults, ran).
|
||||
|
||||
#674: the two known hazards are a bare apostrophe inside a single-quoted scalar
|
||||
(`rule: 'SQLite's LOWER()'`) and an unquoted ` #` (`rule: use --flag #2`). Before this check the
|
||||
validator reported OK on both, because `dl._read_frontmatter` is a hand parser that cannot see
|
||||
either. It was hit TWICE in one session by two independent agents, which is what makes it worth
|
||||
a guard rather than a note.
|
||||
|
||||
The two hazards fail DIFFERENTLY, and catching only the first would have missed half of it:
|
||||
|
||||
* the apostrophe makes PyYAML **reject** the document outright (`ParserError`);
|
||||
* the unquoted ` #` parses fine and **silently truncates** the value — PyYAML reads
|
||||
`use --flag`, the hand parser reads `use --flag #2`. No exception, a wrong value.
|
||||
|
||||
So this compares the parsed RESULT and does not merely try/except the load. That is also why it
|
||||
generalizes past the two known characters, which is the property #674 asked for: any future
|
||||
construct where the writer's library and our reader disagree shows up as a diff, without anyone
|
||||
enumerating it first.
|
||||
|
||||
DIRECTION MATTERS: PyYAML is the WRITER (`migrate_decisions_split.render_record` emits these
|
||||
files with `yaml.safe_dump`), so it is the authority on what the on-disk bytes mean. The hand
|
||||
reader is the permissive one, and a disagreement is a defect in the FILE, not in either parser.
|
||||
|
||||
`ran` is False when PyYAML is not importable. The read path is deliberately dependency-free —
|
||||
`decisions-guard`, the Husky hooks and every contributor machine install nothing — so this check
|
||||
is strictly additive: it must never be the reason the validator cannot run. main() announces the
|
||||
skip rather than passing quietly, because a check that reports success while doing nothing is
|
||||
the exact defect class this corpus keeps re-learning (#603's `stale-after`, #609's marker).
|
||||
"""
|
||||
try:
|
||||
import yaml # pyright: ignore[reportMissingImports]
|
||||
except Exception:
|
||||
return [], False
|
||||
|
||||
faults: list[str] = []
|
||||
for p in files:
|
||||
try:
|
||||
text = p.read_text(encoding="utf-8")
|
||||
except Exception: # noqa: S112 (record_wing_faults already reports the unreadable file by name)
|
||||
continue
|
||||
block = _frontmatter_block(text)
|
||||
if block is None:
|
||||
continue # no/unterminated frontmatter: reported by record_wing_faults
|
||||
try:
|
||||
theirs_raw = yaml.safe_load(block)
|
||||
except Exception as exc:
|
||||
# DELIBERATELY broad. `yaml.YAMLError` alone is too narrow: PyYAML's timestamp
|
||||
# constructor raises a BARE ValueError for a well-shaped but impossible date
|
||||
# (`stale-after: 2026-06-31` -> "day is out of range for month"), which would escape as
|
||||
# a traceback. This check is meant to be strictly additive — it must never be the
|
||||
# reason the validator cannot run, so every failure to load becomes a reported fault.
|
||||
first = str(exc).splitlines()[0] if str(exc).strip() else exc.__class__.__name__
|
||||
faults.append(
|
||||
f"{p}: PyYAML REJECTS this frontmatter, though the dependency-free reader accepted "
|
||||
f"it ({exc.__class__.__name__}: {first}). PyYAML is what WROTE these files, so its "
|
||||
f"verdict is authoritative. Quote the offending value and, inside single quotes, "
|
||||
f"double any literal apostrophe — as `yaml.safe_dump` does. Usual causes: a bare "
|
||||
f"apostrophe inside a single-quoted value, an unquoted `: ` or leading backtick, or "
|
||||
f"an impossible date."
|
||||
)
|
||||
continue
|
||||
if theirs_raw is None:
|
||||
theirs_raw = {}
|
||||
if not isinstance(theirs_raw, dict):
|
||||
faults.append(f"{p}: frontmatter parses as {type(theirs_raw).__name__}, not a mapping.")
|
||||
continue
|
||||
mine = dl._read_frontmatter(block)
|
||||
if mine is None:
|
||||
continue # the reader bailed: reported by record_wing_faults as parse-to-0
|
||||
theirs = {k: ("" if v is None else str(v)) for k, v in theirs_raw.items()}
|
||||
# `key=str`: PyYAML returns TYPED mapping keys, so a stray `1: x` yields an int key while the
|
||||
# hand reader yields "1", and sorting that mixed set raises TypeError — an uncaught traceback
|
||||
# replacing what `_unknown_frontmatter_keys` used to report as an actionable error.
|
||||
for k in sorted(set(mine) | set(theirs), key=str):
|
||||
if mine.get(k) != theirs.get(k):
|
||||
faults.append(
|
||||
f"{p}: frontmatter key {k!r} means different things to the two parsers — "
|
||||
f"reader={mine.get(k)!r} but PyYAML={theirs.get(k)!r}. PyYAML wrote this file, "
|
||||
f"so its reading is the real value and the record is silently corrupt. Common "
|
||||
f"cause: an unquoted ` #`, which YAML treats as a comment and truncates there."
|
||||
)
|
||||
return faults, True
|
||||
|
||||
|
||||
def _unknown_frontmatter_keys(path: Path) -> set[str]:
|
||||
"""Frontmatter keys outside the known schema. Empty on any read/parse failure (reported elsewhere)."""
|
||||
try:
|
||||
@@ -629,7 +738,7 @@ def oversized_records(records, ceiling: int) -> list[tuple[str, int]]:
|
||||
act on instead of asserting that "the corpus" is too big.
|
||||
|
||||
The ceiling sits at a natural gap in the real distribution rather than a round number: at #620
|
||||
the records run 0..59 prose lines (median 26, p90 52) and then jump straight to 83, with
|
||||
the records ran 2..59 prose lines (median 26, p90 52) and then jumped straight to 83, with
|
||||
nothing in between. 60 separates the bulk from the tail without splitting a cluster.
|
||||
|
||||
IMPORTANT — a prompt for judgement, not a target. Length is a PROXY for "grown past what a
|
||||
@@ -642,6 +751,79 @@ def oversized_records(records, ceiling: int) -> list[tuple[str, int]]:
|
||||
return sorted([kv for kv in out if kv[1] > ceiling], key=lambda kv: -kv[1])
|
||||
|
||||
|
||||
class CeilingCalibration(NamedTuple):
|
||||
n: int
|
||||
p90: int
|
||||
p95: int
|
||||
n_over: int
|
||||
fraction_over: float
|
||||
marks_tail: bool # the FINE claim: p90 <= ceiling <= p95
|
||||
flags_minority: bool # the COARSE claim: MINORITY_MIN <= fraction_over <= MINORITY_MAX
|
||||
|
||||
|
||||
def ceiling_calibration(records, ceiling: int) -> CeilingCalibration:
|
||||
"""How well `ceiling` still marks the start of the corpus's tail (#688).
|
||||
|
||||
Two claims of DIFFERENT robustness, deliberately separated, because conflating them is what
|
||||
made the previous guard a ratchet:
|
||||
|
||||
`marks_tail` — `p90 <= ceiling <= p95`. Correct as a definition of "start of the tail", but an
|
||||
order statistic over a SPARSE distribution is a STEP function: the lengths climb to the ceiling
|
||||
and then jump straight to 81 with nothing in between AS MEASURED TODAY (the gap's width moves
|
||||
with the corpus — this is the shape, not a constant), so ONE new record could move p90 by 21
|
||||
lines, and the only remedy the assertion admitted was to raise the ceiling. It is real signal,
|
||||
but it is signal about the CONSTANT drifting, not a defect in the commit under test — the same
|
||||
shape as `stale_records`, and it is reported the same way: a notice, never a failure.
|
||||
|
||||
`flags_minority` — `CEILING_MINORITY_MIN <= fraction_over <= CEILING_MINORITY_MAX`. Deliberately
|
||||
coarse, and what the blocking test asserts. Each added record moves a fraction by at most 1/N, so
|
||||
NO SINGLE ordinary addition can cross it — this is measured headroom, not immunity. From a live
|
||||
18/183 (9.8%), BREACHING the 25% cap takes 38
|
||||
consecutive over-ceiling additions (37 lands exactly on 0.25, which still passes under `<=`),
|
||||
or 718 short ones to dilute below the floor — against ONE record to break `marks_tail`.
|
||||
|
||||
The THIRD arm is the tightest and is stated here because it is the easy one to forget:
|
||||
CONSOLIDATION. Taking 15 of today's 18 over-ceiling records out of the over-set drops below the
|
||||
2% floor — trimming them to <=60 leaves 3/183 = 1.64%, archiving them outright leaves 3/168 =
|
||||
1.79% (the denominator moves too); either way, under the floor. That is a real tension with
|
||||
`test_oversized_records_can_go_green` — the ceiling is allowed to go green — and it is accepted
|
||||
rather than papered over: at 3/183 the
|
||||
constant genuinely IS mis-calibrated and saying so is the signal working. A consolidation PR
|
||||
large enough to hit it should re-derive the ceiling in the same change.
|
||||
|
||||
It still catches genuine mis-calibration in both directions, measured on the real corpus: a
|
||||
ceiling of 20 flags 60% of records (cutting into the bulk, so every author learns to ignore it),
|
||||
and a ceiling of 200 flags 0.5% — one record — which is below the floor and rejected. Note that
|
||||
"flags NOBODY" is the wrong way to state the upper failure: at a ceiling of 200 the count is
|
||||
still nonzero because one 230-line record exists, which is exactly why the floor is a fraction
|
||||
and not `> 0`.
|
||||
|
||||
Why not simply re-derive the constant instead: re-deriving fixes the instance and keeps the
|
||||
mechanism. The v3 fraction band and the v4 percentile containment both failed the same way, one
|
||||
faster than the other, and picking a new number would queue up the fifth version.
|
||||
"""
|
||||
lengths = sorted(record_prose_lines(r) for r in records if r.key)
|
||||
n = len(lengths)
|
||||
if n == 0:
|
||||
return CeilingCalibration(0, 0, 0, 0, 0.0, False, False)
|
||||
|
||||
def pct(q: float) -> int:
|
||||
return lengths[min(int(n * q), n - 1)]
|
||||
|
||||
n_over = sum(1 for v in lengths if v > ceiling)
|
||||
frac = n_over / n
|
||||
p90, p95 = pct(0.90), pct(0.95)
|
||||
return CeilingCalibration(
|
||||
n=n,
|
||||
p90=p90,
|
||||
p95=p95,
|
||||
n_over=n_over,
|
||||
fraction_over=frac,
|
||||
marks_tail=p90 <= ceiling <= p95,
|
||||
flags_minority=CEILING_MINORITY_MIN <= frac <= CEILING_MINORITY_MAX,
|
||||
)
|
||||
|
||||
|
||||
def _catalog_ok() -> bool:
|
||||
try:
|
||||
import scripts.build_decisions_catalog as bc # pyright: ignore[reportMissingImports]
|
||||
@@ -684,6 +866,19 @@ def main(argv=None) -> int:
|
||||
archive_records += dl.parse_file(f)
|
||||
removed, rewritten, demoted = _diff_findings(args.base, args.head) if args.base and args.head else ([], [], [])
|
||||
oversized = oversized_records(records, args.record_ceiling)
|
||||
|
||||
# #674: cross-check the dependency-free reader against the library that WROTE these files.
|
||||
# Strictly additive — absent PyYAML skips the check (and SAYS so) rather than failing the run.
|
||||
yaml_faults, yaml_ran = pyyaml_frontmatter_faults(record_wing_files())
|
||||
if not yaml_ran:
|
||||
print(
|
||||
"::notice::decisions-validate: PyYAML is not importable, so the frontmatter cross-check "
|
||||
"was SKIPPED — every other check ran. This is the expected state on the dependency-free "
|
||||
"read path (decisions-guard, the Husky hooks); CI's script-tests job runs it with PyYAML "
|
||||
"present.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
errs = validate(
|
||||
records,
|
||||
archive_keys=_archive_keys(),
|
||||
@@ -692,7 +887,7 @@ def main(argv=None) -> int:
|
||||
rewritten=rewritten,
|
||||
archive_records=archive_records,
|
||||
demoted=demoted,
|
||||
wing_faults=record_wing_faults(),
|
||||
wing_faults=record_wing_faults() + yaml_faults,
|
||||
)
|
||||
|
||||
# Aggregate: an unthresholded TREND, not a gate (#620). Printed every run so the number stays
|
||||
@@ -723,6 +918,20 @@ def main(argv=None) -> int:
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# Ceiling calibration drift (#688): a NOTICE, never a failure. The ceiling drifting away from
|
||||
# the tail boundary is the passage of corpus growth, not a defect in the commit under test — the
|
||||
# same reasoning `stale_records` is built on. Asserting it in the blocking `script-tests` job
|
||||
# made the next author of a substantial record pay for an unrelated constant going out of date.
|
||||
cal = ceiling_calibration(records, args.record_ceiling)
|
||||
if cal.n and not cal.marks_tail:
|
||||
print(
|
||||
f"::notice::decisions-validate: the {args.record_ceiling}-line ceiling has drifted from "
|
||||
f"the tail boundary of the distribution (p90={cal.p90}, p95={cal.p95}, "
|
||||
f"{cal.n_over}/{cal.n} records over it). Re-derive it when convenient — this is a "
|
||||
f"maintenance signal about the constant, not a problem with this change.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
stale = stale_records(records, date.today())
|
||||
if stale:
|
||||
listed = "; ".join(f"{h} (stale-after {d})" for h, d in stale)
|
||||
|
||||
Executable
+173
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env bash
|
||||
# Make the jq version a job's shell gates run under OBSERVABLE, and any drift LOUD.
|
||||
#
|
||||
# ersatztv#648. Every shell gate in this repo is authored and tested on a developer Mac shipping
|
||||
# jq 1.8.x. The CI runner ships jq 1.6. Nothing pinned or checked that, and until ersatztv#631 the one
|
||||
# thing that could have noticed (scripts/tests/) never ran on the runner. Three independent divergences
|
||||
# surfaced in a single day:
|
||||
#
|
||||
# ersatztv#643 `jq -e` over EMPTY input -> exit 4 on 1.8, exit 0 on 1.6 (a transport failure
|
||||
# passed the docs-only pagination guard)
|
||||
# ersatztv#647 contains("<NUL>") -> false on 1.8, TRUE for every string on 1.6
|
||||
# (the H10 verdict classifier was entirely inert)
|
||||
# ersatztv#647 parse-error exit code -> 5 on 1.8, 4 on 1.6 — same as "no output"
|
||||
# (garbage API response read as "no comments")
|
||||
#
|
||||
# All three are fixed with version-stable constructs, but patching constructs one at a time does not
|
||||
# scale: the failures share one shape — a shell gate's behaviour is a function of its interpreter's
|
||||
# version, and that version was an UNTESTED AXIS. This script makes the axis explicit.
|
||||
#
|
||||
# WHY A FLOOR AND NOT A PIN EVERYWHERE. The obvious fix — bake a pinned jq into the CI toolchain image
|
||||
# (docker/ci/Dockerfile) — provably does NOT cover the gate that actually broke. `.gitea/workflows/
|
||||
# review-verdict.yml` is `runs-on: small`, carries no toolchain-image pin, and per `ci.small-lane-git-only`
|
||||
# the small lane is git-only. It therefore gets the HOST's jq 1.6 no matter what the image contains.
|
||||
# That was checked, not assumed (ersatztv#648's first Done-when box).
|
||||
#
|
||||
# So the contract is the other way round: 1.6 is the FLOOR every gate must work on, and it is the
|
||||
# runner's own jq that provides the 1.6 coverage `scripts/tests/` runs under.
|
||||
#
|
||||
# TWO MODES, deliberately asymmetric:
|
||||
#
|
||||
# (no --expect) Print the version and assert it is >= MIN_VERSION. Used by jobs on the merge
|
||||
# path, including review-verdict.yml. There is NO upper bound here on purpose:
|
||||
# review-verdict.yml writes `review-verdict/h10`, a REQUIRED status check on
|
||||
# `main`, so a hard pin there would turn any jq upgrade on the runner into a
|
||||
# repo-wide merge deadlock. Observability without a deadlock risk.
|
||||
#
|
||||
# --expect X.Y Additionally assert the version is exactly X.Y, and FAIL if not. Used by the
|
||||
# `script-tests` job. This is the tripwire: `scripts/tests/` currently exercises
|
||||
# the 1.6 path only because the runner happens to ship 1.6. If the runner were
|
||||
# upgraded, that coverage would vanish SILENTLY and the whole class of bug above
|
||||
# would go untested again. Going red forces a human to decide — re-pin, or add a
|
||||
# real 1.6 matrix leg — rather than letting the coverage evaporate unnoticed.
|
||||
#
|
||||
# Usage: jq-preflight.sh [--expect <major.minor>]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# The lowest jq every shell gate in this repo must run correctly on. Do not raise this without
|
||||
# confirming the CI runner has actually been upgraded first — the runner, not the dev Mac, is the
|
||||
# binding constraint.
|
||||
MIN_VERSION="1.6"
|
||||
|
||||
expect=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--expect)
|
||||
# `shift 2` with a missing value fails under `set -e` and exits 1 with NOTHING on either
|
||||
# stream — a CI step dying with an empty log is exactly the diagnostic hole this script exists
|
||||
# to remove. Check explicitly instead.
|
||||
if [ "$#" -lt 2 ] || [ -z "${2:-}" ]; then
|
||||
echo "jq-preflight: --expect requires a <major.minor> value" >&2
|
||||
exit 2
|
||||
fi
|
||||
expect="$2"; shift 2 ;;
|
||||
*) echo "jq-preflight: unknown argument '$1'" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "jq-preflight: jq is not on PATH. The shell gates in scripts/ and .gitea/workflows/ shell out to jq; without it they fail as a pile of opaque assertion errors instead of one clear message." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Take jq's EXIT STATUS seriously, and keep stderr OUT of the parse input.
|
||||
#
|
||||
# This was `raw=$(jq --version 2>&1 || true)`, which did neither — and that combination turned the
|
||||
# guard fail-OPEN on the case it most needs to catch. A jq that cannot start (the canonical one is a
|
||||
# glibc mismatch after a base-image change) exits 127 and writes something like
|
||||
# `jq: /lib/x86_64-linux-gnu/libc.so.6: version 'GLIBC_2.34' not found` to stderr. Folded into `raw`,
|
||||
# that string contains `2.34`, which the version pattern happily matched — so the preflight printed
|
||||
# "parsed 2.34", certified the floor, and exited 0 on a jq that cannot run at all. The strip-based
|
||||
# parse this replaced failed CLOSED there, so it was a regression introduced by the fix.
|
||||
# `$?` inside an `if ! cmd; then` block is the NEGATED status (0), not jq's, so capture it explicitly.
|
||||
set +e
|
||||
raw=$(jq --version 2>/dev/null)
|
||||
jq_rc=$?
|
||||
set -e
|
||||
if [ "$jq_rc" -ne 0 ]; then
|
||||
echo "jq-preflight: 'jq --version' failed (exit ${jq_rc}). jq is on PATH but cannot run — a broken build or a missing shared library. Failing closed rather than certifying a version it did not report." >&2
|
||||
exit 1
|
||||
fi
|
||||
# `jq --version` prints e.g. `jq-1.6`, `jq-1.7.1`, or on some builds `jq-1.8.2-dirty`.
|
||||
# Parse with an explicit regex rather than by stripping around the first `-` and `.`.
|
||||
#
|
||||
# The strip approach had a hole that defeated the whole point of this script. It assumed the format
|
||||
# is exactly `jq-X.Y`, so a build printing anything else — `jq version 1.6` (a distro wrapper),
|
||||
# `JQ-1.6`, `jq-1.-6` — left ONE of major/minor empty. The old sanity check was
|
||||
# `case "$major$minor" in *[!a-9]*|"")`, and on `jq version 1.6` that concatenation is "6": non-empty
|
||||
# and all-digits, so the guard PASSED. The floor comparison then ran `[ "" -lt 1 ]`, which exits 2
|
||||
# with "integer expression expected" — and `set -e` exempts a failing command in an `if` condition,
|
||||
# so the whole conditional read false and the script exited 0 having asserted NOTHING, after printing
|
||||
# a plausible-looking "parsed" line.
|
||||
#
|
||||
# That is the silently-untested-axis failure this script was written to eliminate, reproduced inside
|
||||
# the script itself. Require a real `<digits>.<digits>` match, and fail closed when there isn't one.
|
||||
# ANCHORED to the leading `jq` token, not "first digits.digits anywhere in the string".
|
||||
#
|
||||
# An unanchored match takes whatever number comes first, wherever it is. That accepted a leading
|
||||
# warning line or a date prefix as the version — `2026.07.26 jq-1.6` parsed as 2026.07, which sails
|
||||
# over the floor. Anchoring keeps every legitimate form (`jq-1.6`, `jq version 1.6`, `jq-1.7.1`,
|
||||
# `jq-1.6-dirty`, `jq-1.6 (Debian 1.6-2.1)`) and rejects the rest, which then fails closed below.
|
||||
# FIRST LINE ONLY, and bounded everywhere. Both bounds are load-bearing; this is the third round on
|
||||
# this one predicate and each previous version failed for a variant of the same reason.
|
||||
#
|
||||
# * First line only. `[[:space:]]` matches NEWLINES, so an "anchored" pattern still scanned the
|
||||
# whole output: `jq\n2.34: cannot load` matched `jq`, crossed the newline as separator, and
|
||||
# parsed 2.34 — fail-open, the round-2 bug narrowed but not closed. `[[:blank:]]` (space/tab
|
||||
# only) plus a first-line slice confines the match to the line that can actually carry a version.
|
||||
# * Bounded digit runs. This is the round-1 mechanism resurrected. The regex guaranteed the
|
||||
# operands were digits but not that they fit in `test`'s integer range, so a 23-digit major made
|
||||
# `[ "$major" -lt "$min_major" ]` error with "integer expression expected" — and `set -e` exempts
|
||||
# a failing command in an `if` condition, so the conditional read false and THE FLOOR WAS NEVER
|
||||
# ASSERTED, exit 0. Exactly what the empty-string case did in round 1. `{1,9}` keeps every
|
||||
# operand inside a 32-bit integer, so the comparison can no longer error.
|
||||
# * Bounded separator runs, so the pattern cannot be walked across arbitrary filler.
|
||||
first=${raw%%$'\n'*}
|
||||
first=${first%$'\r'}
|
||||
# The separator is one of the two forms real jq actually emits — `jq-1.6` or `jq version 1.6` — not
|
||||
# "any run of dashes and blanks". A permissive class let the pattern be walked across filler:
|
||||
# `jq -- 2.34 (real jq-1.6)` parsed as 2.34, and `jq<TAB><TAB>9.9` as 9.9. A blank separator now
|
||||
# REQUIRES the literal word `version`, which is the only context a real build puts one in.
|
||||
#
|
||||
# The trailing `([^0-9]|$)` is what actually bounds the digit runs. `{1,9}` alone does not: the regex
|
||||
# is unanchored at the end, so `jq-1.99999999999999999999999` simply matched the first 9 digits of
|
||||
# the minor and compared THAT — a mis-parse that passes the floor. Requiring a non-digit (or
|
||||
# end-of-string) after the minor makes an over-long run fail to match at all, so it fails closed.
|
||||
if [[ "$first" =~ ^[[:blank:]]*[Jj][Qq](-v?|[[:blank:]]+version[[:blank:]]+v?)([0-9]{1,9})\.([0-9]{1,9})([^0-9]|$) ]]; then
|
||||
major="${BASH_REMATCH[2]}"
|
||||
minor="${BASH_REMATCH[3]}"
|
||||
else
|
||||
echo "jq-preflight: could not parse a major.minor version out of '${first}'. Refusing to assert a floor against an unparsed version — that would silently pass." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# THIS LINE IS THE POINT of the no-arg mode: the jq version CI actually used is in the job log, so a
|
||||
# future divergence can be diagnosed from the log alone rather than by guessing at the runner image.
|
||||
# `$first`, not `$raw`: a multi-line `--version` would split this across lines, breaking the single
|
||||
# grep-able log line that is the entire point of the no-arg mode.
|
||||
echo "jq-preflight: jq version in use = ${first} (parsed ${major}.${minor}; floor ${MIN_VERSION})"
|
||||
|
||||
min_major=${MIN_VERSION%%.*}
|
||||
min_minor=${MIN_VERSION#*.}
|
||||
if [ "$major" -lt "$min_major" ] || { [ "$major" -eq "$min_major" ] && [ "$minor" -lt "$min_minor" ]; }; then
|
||||
echo "jq-preflight: jq ${major}.${minor} is BELOW the supported floor ${MIN_VERSION}. The gates in scripts/ and .gitea/workflows/ are written against ${MIN_VERSION}+ semantics and will misbehave silently on older builds." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "$expect" ]; then
|
||||
if [ "${major}.${minor}" != "$expect" ]; then
|
||||
echo "jq-preflight: expected jq ${expect}, found ${major}.${minor}." >&2
|
||||
echo "" >&2
|
||||
echo "This is a TRIPWIRE, not a defect in your change (ersatztv#648). scripts/tests/ was pinned to" >&2
|
||||
echo "jq ${expect} because that is what this runner shipped; it now reports ${major}.${minor}. The ${expect}" >&2
|
||||
echo "coverage the suite assumed has therefore just disappeared, silently — and jq 1.7 altered NUL" >&2
|
||||
echo "handling, exit codes, @base64d and number precision, every one of which a gate here depends on." >&2
|
||||
echo "" >&2
|
||||
echo "Decide explicitly, then update the --expect value in .gitea/workflows/pr-checks.yml:" >&2
|
||||
echo " * re-pin to the new version after re-reading docs/ci-cd.md -> 'The jq contract', or" >&2
|
||||
echo " * add a real matrix leg that runs the suite under ${MIN_VERSION} as well." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "jq-preflight: version matches the expected pin (${expect})."
|
||||
fi
|
||||
@@ -92,6 +92,22 @@ pr_url=$(printf '%s' "$prjson" | jq -r '.html_url // ""')
|
||||
[ "$pr_state" = "open" ] || die "PR #$pr is '$pr_state', not open — refusing to post a verdict"
|
||||
short=${sha:0:7}
|
||||
|
||||
# --- Record the BASE BRANCH the verdict was formed against (ersatztv#632). ----------------------
|
||||
# The sha binding closes "the head moved under a fixed verdict". It does not close the mirror case:
|
||||
# RETARGETING a PR's base changes neither the head sha nor the status, yet changes the effective
|
||||
# diff — so a verdict written while the PR targeted `main` still reads green after it is pointed at
|
||||
# a branch with a very different merge-base. Consent outliving what it was granted for, reached from
|
||||
# the other direction.
|
||||
#
|
||||
# The comparator is `base.ref` (the BRANCH NAME), deliberately NOT `base.sha`. `base.sha` tracks the
|
||||
# base branch's tip, which moves every time anything merges to `main` — comparing it would invalidate
|
||||
# every open verdict on every unrelated merge, i.e. a self-inflicted merge deadlock. `base.ref`
|
||||
# changes exactly when someone retargets the PR, which is the event being guarded. A base branch that
|
||||
# merely ADVANCES is out of scope by design: that is ordinary churn, and rebasing onto it changes the
|
||||
# head sha, which the existing per-sha binding already catches.
|
||||
base_ref=$(printf '%s' "$prjson" | jq -r '.base.ref // ""')
|
||||
[ -n "$base_ref" ] || die "PR #$pr has no resolvable base branch (.base.ref) — refusing to post a verdict that cannot record what it was formed against"
|
||||
|
||||
# --- The comment (human-readable artifact + the hook's condition-(c) input). --------------------
|
||||
# The verdict line MUST start the line: the hook anchors its parser to line-start precisely so a
|
||||
# comment that merely QUOTES the template mid-sentence cannot self-approve a merge.
|
||||
@@ -107,14 +123,33 @@ printf 'posted comment: Review-verdict: %s @ %s\n' "$verdict" "$short"
|
||||
# a verdict written for its parent — reintroducing ersatztv#622 at a smaller time scale. We do NOT
|
||||
# retry against the new head: the new commit is genuinely unreviewed, and silently re-targeting the
|
||||
# verdict at it is exactly the failure this script exists to prevent.
|
||||
sha_now=$(api_get "repos/$owner/$repo/pulls/$pr" | jq -r '.head.sha // ""')
|
||||
# Fail CLOSED if the re-read itself fails. This used to be `sha_now=$(api_get ... | jq ...)`, where
|
||||
# `set -e` + `pipefail` aborted the script on a failed GET — implicitly, but before any status was
|
||||
# written. Folding the two reads into one variable with `|| true` would have swallowed that: both
|
||||
# `sha_now` and `base_now` come back empty, both `[ -n … ]` guards become no-ops, and the status is
|
||||
# written having confirmed NOTHING about the head or the base. That is a fail-open regression
|
||||
# introduced by the refactor, so the refusal is now explicit rather than a side effect of `set -e`.
|
||||
prjson_now=$(api_get "repos/$owner/$repo/pulls/$pr") \
|
||||
|| die "could not re-read PR #$pr to confirm the head and base had not moved while posting — no status was written. Re-run once Gitea is reachable."
|
||||
sha_now=$(printf '%s' "$prjson_now" | jq -r '.head.sha // ""')
|
||||
if [ -n "$sha_now" ] && [ "$sha_now" != "$sha" ]; then
|
||||
die "head moved from $short to ${sha_now:0:7} while posting — that commit is UNREVIEWED, so no status was written. Re-review the new head and run this again."
|
||||
fi
|
||||
# The same TOCTOU window applies to the base (ersatztv#632): a retarget between the read above and
|
||||
# the status write below would bind the verdict to a base that is no longer the PR's, and the head
|
||||
# sha check would not notice because retargeting does not move the head.
|
||||
base_now=$(printf '%s' "$prjson_now" | jq -r '.base.ref // ""')
|
||||
if [ -n "$base_now" ] && [ "$base_now" != "$base_ref" ]; then
|
||||
die "base branch changed from '$base_ref' to '$base_now' while posting — the diff you reviewed is not the diff this PR now merges, so no status was written. Re-review against the new base and run this again."
|
||||
fi
|
||||
|
||||
# The base branch goes in the status DESCRIPTION, not in the comment. The comment body is parsed by
|
||||
# `scripts/check-review-verdict.sh`, whose grammar had three false-opens in its history; nothing
|
||||
# parses the description today, so this adds a field without reopening that surface. The hook reads
|
||||
# it back and compares (ersatztv#632).
|
||||
status_payload=$(jq -n \
|
||||
--arg s "$state" --arg c "$STATUS_CONTEXT" --arg u "$pr_url" \
|
||||
--arg d "Review-verdict: $verdict @ $short" \
|
||||
--arg d "Review-verdict: $verdict @ $short (base: $base_ref)" \
|
||||
'{state:$s, context:$c, description:$d, target_url:$u}')
|
||||
api_post "repos/$owner/$repo/statuses/$sha" "$status_payload" >/dev/null \
|
||||
|| die "failed to post the '$STATUS_CONTEXT' commit status on $short"
|
||||
|
||||
Executable
+278
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env bash
|
||||
# Exhaustively enumerate a PR's changed file paths, or fail closed.
|
||||
#
|
||||
# ersatztv#649. This is the ONE implementation of the security-critical half of the merge gate.
|
||||
# It exists because the same logic was written twice — once in `.claude/hooks/pretooluse-merge-consent.sh`
|
||||
# (advisory: a failure produces a human prompt) and once in `.gitea/workflows/review-verdict.yml`
|
||||
# (ENFORCED: it writes the branch-protection-required `review-verdict/h10` status). The advisory copy
|
||||
# accumulated four rounds of hardening (ersatztv#643) that the enforced copy never received, leaving the
|
||||
# copy with real authority strictly weaker than the copy without. Two copies of a security predicate
|
||||
# drift; one cannot.
|
||||
#
|
||||
# SCOPE — mechanism, not policy. This script answers exactly one question: "what is the complete set of
|
||||
# paths this PR touches, at one head, or can we not tell?" It deliberately does NOT classify the PR.
|
||||
# The two callers' allow-lists differ ON PURPOSE and must stay separate:
|
||||
# * the hook's docs-only pattern also lets .claude/ .gitea/ .husky/ through, which is safe there only
|
||||
# because it falls through to a HUMAN PROMPT;
|
||||
# * the workflow's is narrower, because there a match posts a green status with nobody in the loop.
|
||||
# Sharing the enumeration fixes the drift; sharing the classification would erase an intended difference.
|
||||
#
|
||||
# CONTRACT
|
||||
# Usage: pr-changed-files.sh <owner> <repo> <pr> <expected-head-sha> <expected-base-ref>
|
||||
# stdout: newline-delimited paths, BOTH sides of every rename, no blank lines. May be empty.
|
||||
# exit 0 the enumeration is COMPLETE and bound to <expected-head-sha> AND <expected-base-ref>.
|
||||
# stdout is authoritative.
|
||||
# exit 1 the enumeration could NOT be completed or verified. stdout is meaningless — the caller
|
||||
# MUST fail closed (withhold any exemption). A diagnostic goes to stderr.
|
||||
# exit 2 usage error.
|
||||
# Callers must treat any non-zero exit as "no exemption". Never read stdout without checking the status.
|
||||
#
|
||||
# WHY THE BASE REF IS AN ARGUMENT, AND WHY IT IS NOT OPTIONAL (ersatztv#698 route 1).
|
||||
# `/pulls/{n}/files` computes the diff against the PR's **live** base, which is mutable. Retargeting a
|
||||
# PR changes the enumerated file set without moving the head sha, so head-binding alone does not bind
|
||||
# the ANSWER — only the commit it is nominally about. Reproduced live on this instance: a PR opened
|
||||
# into `main` and retargeted mid-run to a scratch base enumerated as docs-only and was granted
|
||||
# `review-verdict/h10=success`, while its diff against `main` carried a C# file (probe PR #703).
|
||||
#
|
||||
# REQUIRED rather than optional on purpose. An optional binding on a shared security primitive is an
|
||||
# opt-out, and the caller that forgets it is precisely the caller that needed it — silently. Five
|
||||
# arguments or exit 2.
|
||||
#
|
||||
# This NARROWS the window, it does not erase it. The base is re-read after the paging round trips
|
||||
# alongside the head, so a retarget that is still in effect at that point fails closed; a retarget
|
||||
# that opens and closes strictly between the files call and the re-read is not observable from here.
|
||||
# Pinning the diff to two shas would close it, and Gitea 1.25.4 cannot serve that: `compare/{base}...
|
||||
# {head}` returns `total_commits`/`commits` and NO `files`, and a `--depth=1` fetch of the two shas
|
||||
# has no merge base, so a three-dot diff is impossible while a two-dot one over-reports every commit
|
||||
# `main` gained since the branch point (both measured, #698). The remainder is covered one level up
|
||||
# instead, by the workflow reclassifying on `edited` rather than trusting a machine-written success.
|
||||
#
|
||||
# AUTH/TRANSPORT is caller-supplied via env, because the two callers authenticate differently:
|
||||
# ETV_GITEA_TOKEN | GITEA_TOKEN -> `Authorization: token`
|
||||
# ETV_GITEA_BASICAUTH -> curl -u user:pass
|
||||
# ETV_GITEA_URL | GITEA_BASE_URL -> API base; defaults to the homelab Gitea. A value ending in
|
||||
# /api/v1 is used as-is, otherwise /api/v1 is appended.
|
||||
#
|
||||
# jq COMPATIBILITY (ersatztv#648). This runs on the CI runner, which ships **jq 1.6**, while it is
|
||||
# authored on Macs shipping 1.8.x. It is therefore written to the 1.6-compatible subset:
|
||||
# * never rely on `jq -e`'s exit status over EMPTY input — 1.6 exits 0 where >=1.7 exits 4, which is
|
||||
# precisely the fail-open that ersatztv#647 found live in the enforced gate. Emptiness is always
|
||||
# checked explicitly in shell FIRST.
|
||||
# * never use `contains()` for substring tests — on 1.6 `contains("<NUL>")` is true for every string.
|
||||
# * never distinguish a parse error from "no output" by exit code — 1.6 returns 4 for both.
|
||||
# See docs/ci-cd.md -> "The jq contract".
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$#" -ne 5 ]; then
|
||||
echo "usage: pr-changed-files.sh <owner> <repo> <pr> <expected-head-sha> <expected-base-ref>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
owner=$1
|
||||
repo=$2
|
||||
pr=$3
|
||||
expected_sha=$4
|
||||
expected_base=$5
|
||||
|
||||
if [ -z "$owner" ] || [ -z "$repo" ] || [ -z "$pr" ] || [ -z "$expected_sha" ] || [ -z "$expected_base" ]; then
|
||||
echo "pr-changed-files: empty owner/repo/pr/sha/base argument" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
base_url="${ETV_GITEA_URL:-${GITEA_BASE_URL:-http://192.168.1.95:3000}}"
|
||||
case "$base_url" in
|
||||
*/api/v1) : ;;
|
||||
*/) base_url="${base_url}api/v1" ;;
|
||||
*) base_url="${base_url}/api/v1" ;;
|
||||
esac
|
||||
|
||||
# Empty output on ANY failure, so every caller path treats a transport error the same way. The
|
||||
# emptiness is then rejected explicitly below — never inferred from a jq exit code.
|
||||
gq() {
|
||||
local path="$1"
|
||||
if [ -n "${ETV_GITEA_TOKEN:-}" ]; then
|
||||
curl -sf -H "Authorization: token $ETV_GITEA_TOKEN" "$base_url/$path" 2>/dev/null || true
|
||||
elif [ -n "${GITEA_TOKEN:-}" ]; then
|
||||
curl -sf -H "Authorization: token $GITEA_TOKEN" "$base_url/$path" 2>/dev/null || true
|
||||
elif [ -n "${ETV_GITEA_BASICAUTH:-}" ]; then
|
||||
curl -sf -u "$ETV_GITEA_BASICAUTH" "$base_url/$path" 2>/dev/null || true
|
||||
else
|
||||
printf ''
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -z "${ETV_GITEA_TOKEN:-}" ] && [ -z "${GITEA_TOKEN:-}" ] && [ -z "${ETV_GITEA_BASICAUTH:-}" ]; then
|
||||
echo "pr-changed-files: no Gitea credentials in env — cannot enumerate, failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Bind the BASE before the first page is requested (ersatztv#698 route 1). Checking only afterwards
|
||||
# would leave the common case — a PR retargeted before the enumeration even starts — indistinguishable
|
||||
# from an honest one, because every page would agree with every other page while all of them described
|
||||
# a diff against the wrong base. Both ends are checked; neither alone is sufficient.
|
||||
prjson_before=$(gq "repos/$owner/$repo/pulls/$pr")
|
||||
if [ -z "${prjson_before//[[:space:]]/}" ]; then
|
||||
echo "pr-changed-files: could not read PR #$pr to bind the base ref before enumerating — failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
base_before=$(printf '%s' "$prjson_before" | jq -r '.base.ref // ""' 2>/dev/null || true)
|
||||
if [ -z "$base_before" ] || [ "$base_before" != "$expected_base" ]; then
|
||||
echo "pr-changed-files: PR #$pr targets '${base_before:-<unreadable>}', not the expected '$expected_base' — the diff would be computed against a different base, failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Also capture the base's TIP at this same read (ersatztv#707). This costs no extra round trip —
|
||||
# `prjson_before` is already fetched above for the `.base.ref` check. It answers a DIFFERENT
|
||||
# question than that check does, and the two are not interchangeable:
|
||||
# * `.base.ref` (above) answers "did this PR RETARGET to a different branch" — comparing branch
|
||||
# NAMES is deliberate there (ersatztv#698 route 1 / ersatztv#632), because comparing tip shas
|
||||
# for that purpose would self-deadlock: `main` advancing on every unrelated merge would fail
|
||||
# every open enumeration even though the PR still targets the same branch it always did.
|
||||
# * `.base.sha` (here) answers "did `$expected_base` ADVANCE while THIS enumeration was running."
|
||||
# `/pulls/{n}/files` diffs against the base's LIVE tip and is offset-paged over several round
|
||||
# trips; if `main` gains a commit mid-enumeration, Gitea recomputes each subsequent page against
|
||||
# the new tip independently, so rows can drop out of the result entirely (a file `main` no longer
|
||||
# differs on) while later rows shift into offset ranges already consumed on the old tip. The
|
||||
# result reads as a complete, ordinary list — `.base.ref` never changed, `.head.sha` never
|
||||
# changed, page count and termination all look normal — while silently omitting a page's worth of
|
||||
# changed paths, including possibly the only code file in the diff. This is a narrower, additional
|
||||
# check layered on top of the ref check, not a replacement for it.
|
||||
base_sha_before=$(printf '%s' "$prjson_before" | jq -r '.base.sha // ""' 2>/dev/null || true)
|
||||
if [ -z "$base_sha_before" ]; then
|
||||
echo "pr-changed-files: could not read PR #$pr's base tip sha before enumerating — failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PAGE_SIZE=50
|
||||
MAX_PAGES=40 # 2000 files; beyond this we refuse rather than guess
|
||||
|
||||
files=""
|
||||
page=1
|
||||
complete=no
|
||||
|
||||
while [ "$page" -le "$MAX_PAGES" ]; do
|
||||
raw=$(gq "repos/$owner/$repo/pulls/$pr/files?limit=${PAGE_SIZE}&page=${page}")
|
||||
|
||||
# An EMPTY body is rejected in SHELL, before jq sees it. `jq -e` over empty input exits 4 on
|
||||
# jq >= 1.7 but 0 on jq 1.6, and the runner ships 1.6 — leaving this to jq's exit status is the
|
||||
# exact fail-open ersatztv#647 found in the enforced copy. A transport failure must never
|
||||
# masquerade as a legitimate short final page.
|
||||
if [ -z "${raw//[[:space:]]/}" ]; then
|
||||
echo "pr-changed-files: empty/unreadable response for page ${page}" >&2
|
||||
complete=no; break
|
||||
fi
|
||||
|
||||
# VALIDATE EVERY FIELD THE EXTRACTION BELOW CONSUMES, on EVERY row.
|
||||
#
|
||||
# * Top-level type alone is not enough: `[{}]` is a well-formed array whose rows carry no
|
||||
# `filename`, so it contributes no paths, looks like a short page, and would complete the
|
||||
# enumeration from a PARTIAL list — the same failure one level down. It also rejects arrays of
|
||||
# scalars, which would otherwise make the extraction fail under `set -e`.
|
||||
# * CR/LF in a path is rejected outright. `chunk` flattens paths into newline-delimited text, so a
|
||||
# filename containing a newline splits into TWO lines matched against the allow-list separately:
|
||||
# "safe.md\ndocs/Program.cs" yields `safe.md` and `docs/Program.cs`, both of which pass, while the
|
||||
# real single path ends in `.cs`. Git permits newlines in filenames, so this is reachable and was
|
||||
# reproduced against the hook.
|
||||
# * `previous_filename` is validated on EVERY row, not only `renamed` ones, because `chunk` emits it
|
||||
# for every row regardless of `.status`. Validating it only where it is semantically "supposed to"
|
||||
# appear left a hole one predicate wide: a `status: "modified"` row carrying a newline in
|
||||
# `previous_filename` was reproducibly exempted. The validation domain must match the CONSUMPTION
|
||||
# domain.
|
||||
# * `..` is rejected because the callers' allow-lists anchor `^docs/`, so `docs/../ErsatzTV/Program.cs`
|
||||
# matches one. Git will not produce such a path; this guard's job is to fail closed on unexpected
|
||||
# 2xx shapes rather than assume a well-behaved peer.
|
||||
# * `.status` is checked against a CLOSED set. Be precise about what this does and does not do:
|
||||
# the extraction below emits `(.previous_filename // empty)` UNCONDITIONALLY, so a present
|
||||
# `previous_filename` is never dropped on account of `.status`. What the closed set actually buys
|
||||
# is rejecting rows whose vocabulary we do not recognise — where a source path may be absent, or
|
||||
# carried in some other field we are not reading. Without it, `"Renamed"` with a capital R, or an
|
||||
# absent status, silently takes the `else true` branch of the clause below and skips the
|
||||
# "renamed rows MUST carry previous_filename" requirement entirely. (An earlier version of this
|
||||
# comment claimed the source path would be "dropped", which is not the mechanism; a maintainer
|
||||
# who tested that claim would find it false and might conclude the check is redundant.)
|
||||
# `modified` is accepted alongside `changed` deliberately: live Gitea 1.25.4 emits `changed`, but a
|
||||
# closed allow-list built from the wrong vocabulary is a worse failure than the hole it closes — it
|
||||
# would gate every genuine docs-only PR on any version that spells it differently. The property is
|
||||
# "reject values we do not recognise", not "enumerate one version exactly".
|
||||
if ! printf '%s' "$raw" \
|
||||
| jq -e 'def ok: type == "string" and length > 0
|
||||
and (test("[\\r\\n]") | not)
|
||||
and (split("/") | index("..") | not);
|
||||
type == "array" and all(.[];
|
||||
(.filename | ok)
|
||||
and (.previous_filename == null or (.previous_filename | ok))
|
||||
and ((.status // "") as $s | ($s | type) == "string"
|
||||
and (["added","deleted","changed","modified","renamed","copied"] | index($s)) != null)
|
||||
and (if .status == "renamed"
|
||||
then (.previous_filename | type == "string" and length > 0)
|
||||
else true end))' \
|
||||
>/dev/null 2>&1; then
|
||||
echo "pr-changed-files: page ${page} failed row validation" >&2
|
||||
complete=no; break
|
||||
fi
|
||||
|
||||
# BOTH sides of a rename: Gitea reports a `git mv` as ONE row whose `filename` is the DESTINATION,
|
||||
# with the source only in `previous_filename`. Reading `filename` alone lets a PR move a protected
|
||||
# file INTO docs/ and pass as docs-only (verified live: `.gitea/workflows/renovate.yml` ->
|
||||
# `docs/innocuous-note.md` showed no protected path). One renamed row is therefore ONE row but TWO
|
||||
# paths, which is why the two counts below are computed differently.
|
||||
n=$(printf '%s' "$raw" | jq -r 'length')
|
||||
chunk=$(printf '%s' "$raw" | jq -r '.[] | (.filename // empty), (.previous_filename // empty)')
|
||||
[ -n "$chunk" ] && files=$(printf '%s\n%s' "$files" "$chunk")
|
||||
|
||||
# Terminate ONLY on an explicitly validated EMPTY page — never on a merely SHORT one.
|
||||
# "Fewer than 50 rows means last page" assumes the server's page size is the 50 we asked for, but
|
||||
# Gitea caps `limit` at the server-wide MAX_RESPONSE_ITEMS (default 50, configurable) and is free to
|
||||
# return fewer. A 30-row page followed by a page of code would complete the enumeration over a
|
||||
# PARTIAL list — the same fail-open, reached without any transport error. Costs one extra request;
|
||||
# the MAX_PAGES cap still fails closed.
|
||||
if [ "$n" -eq 0 ]; then complete=yes; break; fi
|
||||
page=$((page + 1))
|
||||
done
|
||||
|
||||
if [ "$complete" != yes ]; then
|
||||
echo "pr-changed-files: enumeration incomplete (stopped at page ${page}) — failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Bind the enumeration to ONE head. Paging is several round-trips; a force-push between them means
|
||||
# page 1 came from head A and page 2 from head B, so the assembled list belongs to no single commit —
|
||||
# B's code page can be skipped entirely while B's docs page reads as a clean short tail. Re-read the
|
||||
# head and refuse if it moved.
|
||||
prjson=$(gq "repos/$owner/$repo/pulls/$pr")
|
||||
if [ -z "${prjson//[[:space:]]/}" ]; then
|
||||
echo "pr-changed-files: could not re-read PR head to bind the enumeration — failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
sha_after=$(printf '%s' "$prjson" | jq -r '.head.sha // ""' 2>/dev/null || true)
|
||||
if [ -z "$sha_after" ] || [ "$sha_after" != "$expected_sha" ]; then
|
||||
echo "pr-changed-files: head moved during enumeration (${expected_sha:0:7} -> ${sha_after:0:7}) — failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The same round-trip window applies to the BASE, and the head check cannot see it: retargeting moves
|
||||
# the diff without moving the head sha (ersatztv#698 route 1). Comparing `.base.ref` — the branch NAME,
|
||||
# never its tip — is deliberate and matches `post-review-verdict.sh` (ersatztv#632): a base that merely
|
||||
# ADVANCES is ordinary churn, while comparing tips would fail every enumeration on every unrelated
|
||||
# merge to `main`.
|
||||
base_after=$(printf '%s' "$prjson" | jq -r '.base.ref // ""' 2>/dev/null || true)
|
||||
if [ -z "$base_after" ] || [ "$base_after" != "$expected_base" ]; then
|
||||
echo "pr-changed-files: base moved during enumeration ('$expected_base' -> '${base_after:-<unreadable>}') — the enumerated diff is against a base this PR no longer targets, failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Same window, the tip-advance question this time (ersatztv#707; see the comment at
|
||||
# `base_sha_before` above for why this is a DIFFERENT check from `.base.ref`, not a duplicate of
|
||||
# it). `prjson` is already fetched above to bind the head sha, so this is the same re-read, not a
|
||||
# new round trip. `$expected_base`'s branch name can be unchanged across the whole enumeration
|
||||
# while its TIP moved partway through — the exact #707 window: no retarget, no head movement,
|
||||
# nothing the ref check or the head-sha check can see, yet later pages were diffed against a base
|
||||
# earlier pages never saw.
|
||||
base_sha_after=$(printf '%s' "$prjson" | jq -r '.base.sha // ""' 2>/dev/null || true)
|
||||
if [ -z "$base_sha_after" ] || [ "$base_sha_after" != "$base_sha_before" ]; then
|
||||
echo "pr-changed-files: base '$expected_base' advanced during enumeration (${base_sha_before:0:7} -> ${base_sha_after:0:7}) — later pages may have been diffed against a base earlier pages were not, failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "$files" | grep -v '^$' || true
|
||||
exit 0
|
||||
@@ -0,0 +1,723 @@
|
||||
"""The dropped-step guard on docker-build.yml's two REQUIRED jobs (ersatztv#756).
|
||||
|
||||
WHAT THIS IS PROTECTING. A `run:` body the runner declines to interpolate is DROPPED, and the job
|
||||
still concludes `success` (ersatztv#751, `ci.workflow-run-body-no-expressions`). #751 fixed that in
|
||||
`review-verdict.yml`, where the consequence is fail-CLOSED — `review-verdict/h10` is absent and the
|
||||
merge is blocked. It left the two places where the same drop is fail-OPEN: `Build & test (.NET)` and
|
||||
`EF migration integrity (SQLite + MySql)` are the other two required contexts on `main`, so a dropped
|
||||
step there sends a required check green having done no work.
|
||||
|
||||
THE TESTS COME IN THREE KINDS AND NONE SUBSTITUTES FOR ANOTHER, which is the lesson #751 paid for:
|
||||
|
||||
* STATIC — the marker set and the guard's expectations agree, and the guard is positioned so it
|
||||
can actually run. Cheap, and the only kind that catches a NEW step added without a marker.
|
||||
* BEHAVIOURAL — the guard's real command line is EXECUTED against markers written by the steps'
|
||||
real marker lines, both extracted from the parsed workflow. A structural test cannot prove an
|
||||
exit code, and `exit 1` in a body is satisfiable by dead code.
|
||||
* A LIVE PROBE — that the runner still executes a LATER step after dropping an earlier one, on the
|
||||
BUILD lane rather than the `small` lane #751 measured. That is the premise the whole guard rests
|
||||
on and no test here can establish it; it is recorded in docs/ci-cd.md and on the issue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "docker-build.yml"
|
||||
SCRIPT = REPO_ROOT / "scripts" / "ci-step-ran.sh"
|
||||
|
||||
# The jobs whose contexts branch protection REQUIRES on `main`. Read live on 2026-08-10:
|
||||
# Build ErsatzTV Image / Build & test (.NET) (pull_request)
|
||||
# Build ErsatzTV Image / EF migration integrity (SQLite + MySql) (pull_request)
|
||||
# review-verdict/h10
|
||||
# The third is guarded by test_pr_changed_files.py; these two are this file's subject. `build`,
|
||||
# `api-docs` and `format` are deliberately NOT here — they are not required, and all three
|
||||
# legitimately interpolate into a `run:` body, so extending the absolute rule to them would be false.
|
||||
# Per-step markers apply to the two REQUIRED contexts, where a dropped step is fail-OPEN.
|
||||
MARKED_JOBS = ("test", "migrations")
|
||||
|
||||
# The delimiter ban is WIDER than the marker set, and the extra job is not an afterthought.
|
||||
# `build`'s "Smoke + IPTV E2E" step runs AFTER `Build and push`, so on a `v*` tag the image is
|
||||
# already in the registry as the release candidate and this step is what decides whether it was ever
|
||||
# booted. A drop there publishes an unsmoked candidate and goes green, and `DeployStack jazz-media`
|
||||
# promotes exactly that image — not a "smaller cost than a required context", which is what an
|
||||
# earlier draft of the decision record claimed. Its two payloads moved into the step's `env:`, which
|
||||
# is the free half of the escape hatch, so the ban costs nothing there.
|
||||
#
|
||||
# `functional-e2e` is deliberately NOT here even though it is delimiter-free today: it is advisory by
|
||||
# declaration (not a required check, not a `needs:` of `build`), so the rule stays "ban where a drop
|
||||
# is consequential" rather than "ban wherever it happens to be free right now".
|
||||
# `api-docs` and `format` keep one delimiter each, both `github.base_ref` in a detect step, and gate
|
||||
# nothing that ships.
|
||||
DELIMITER_BAN_JOBS = ("test", "migrations", "build")
|
||||
|
||||
# THE RAW OPENER, not a closed `${{ … }}` pair — found by cold review. The runner's rewrite is
|
||||
# triggered by the OPENER; a closed-pair regex therefore misses `# ${{` with no closer, which would
|
||||
# sail through an "absolute" ban and still drop the step. Nothing in these jobs may contain the
|
||||
# opener at all, so matching it directly is both simpler and strictly stronger. `_EXPR` is kept for
|
||||
# reporting the payload of a well-formed one in the failure message.
|
||||
_OPENER = re.compile(r"\$\{\{")
|
||||
_EXPR = re.compile(r"\$\{\{(.*?)\}\}", re.S)
|
||||
_MARK = re.compile(r'ci-step-ran\.sh"?\s+mark\s+(\S+)')
|
||||
|
||||
|
||||
# ONE parse, shared. `yaml.safe_load` per call returns a fresh object graph, so an identity test
|
||||
# across two helpers (`steps[-1] is guard`) would compare structurally-equal but distinct dicts and
|
||||
# fail — or, worse in the other direction, an `is not` filter would exclude nothing and a step would
|
||||
# match as its own guard. That is not hypothetical: test_pr_changed_files.py records exactly this
|
||||
# going wrong in the #751 guard test, where the assertions then ran against the wrong step.
|
||||
_DOC = yaml.safe_load(WORKFLOW.read_text())
|
||||
|
||||
|
||||
def _doc():
|
||||
return _DOC
|
||||
|
||||
|
||||
def _steps(job: str):
|
||||
return _doc()["jobs"][job]["steps"]
|
||||
|
||||
|
||||
def _run_steps(job: str):
|
||||
return [s for s in _steps(job) if s.get("run")]
|
||||
|
||||
|
||||
def _guard(job: str):
|
||||
"""The trailing assert step. Located by CONTENT, never by index.
|
||||
|
||||
Locating it as `steps[-1]` here and then asserting it is last elsewhere would be circular — the
|
||||
position test would hold by construction. This finds the step that invokes the assert
|
||||
sub-command, and `test_the_guard_is_the_LAST_step` independently checks where it sits.
|
||||
"""
|
||||
hits = [s for s in _run_steps(job) if "ci-step-ran.sh assert" in s["run"]]
|
||||
assert len(hits) == 1, f"job '{job}' has {len(hits)} assert steps, expected exactly 1"
|
||||
return hits[0]
|
||||
|
||||
|
||||
def _marked(job: str):
|
||||
"""[(step, key)] for every step that records its own execution, in declaration order."""
|
||||
out = []
|
||||
for s in _run_steps(job):
|
||||
m = _MARK.search(s["run"])
|
||||
if m:
|
||||
out.append((s, m.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
def _guard_buckets(job: str):
|
||||
"""(always_keys, gated_keys) as the guard's own argv spells them."""
|
||||
argv = _guard(job)["run"].split()
|
||||
assert "--always" in argv and "--gated" in argv, argv
|
||||
a, g = argv.index("--always"), argv.index("--gated")
|
||||
return argv[a + 1:g], argv[g + 1:]
|
||||
|
||||
|
||||
# Mirrors the `if:` every gated step in these jobs carries. Compared as a normalised string rather
|
||||
# than by parsing the expression: what matters is that a step's gating and the guard's bucketing are
|
||||
# the SAME condition, and any rewrite of one that is not mirrored in the other should be loud.
|
||||
SKIP_GATE = ("steps.detect.outputs.docs_only!='true'&&steps.revalidate.outputs.skip!='true'")
|
||||
|
||||
|
||||
def _is_gated(step) -> bool:
|
||||
return re.sub(r"\s+", "", str(step.get("if", ""))) == SKIP_GATE
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# STATIC
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", DELIMITER_BAN_JOBS)
|
||||
def test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body(job):
|
||||
"""The absolute rule from `review-verdict.yml`, extended to the two required build jobs.
|
||||
|
||||
This is the cheaper and more general half of #756: the drop mechanism REQUIRES an opener in the
|
||||
scalar, so a job with none is immune by construction and the runtime markers are a backstop
|
||||
rather than the only line of defence.
|
||||
|
||||
The scope is the three jobs in `DELIMITER_BAN_JOBS` — see the comment there for why `build` is in
|
||||
and `functional-e2e` is not. Do NOT restate this docstring as "scoped to the required pair":
|
||||
round 2 moved `build`'s two payloads into `env:` and brought it into the ban, and this docstring
|
||||
sits directly above the decorator that parametrises over the wider set.
|
||||
|
||||
The escape hatch when a value really is needed is the step's `env:` block, which is interpolated
|
||||
PER VALUE, so a payload that does not evaluate cannot take the body with it.
|
||||
|
||||
The `run:` SCALAR AS PARSED, comments and all. A shell comment inside a `run:` body is NOT inert
|
||||
— that is the whole #751 defect — so this must never filter comments out. Ordinary YAML comments
|
||||
outside a `run:` body ARE inert and are not read here.
|
||||
"""
|
||||
offenders = []
|
||||
for s in _run_steps(job):
|
||||
for m in _OPENER.finditer(s["run"]):
|
||||
closed = _EXPR.match(s["run"], m.start())
|
||||
payload = closed.group(1).strip() if closed else "<unclosed opener>"
|
||||
offenders.append(f"{s.get('name', '?')}: {payload!r}")
|
||||
assert not offenders, (
|
||||
f"job '{job}' of docker-build.yml has an expression delimiter inside a run: body — "
|
||||
f"{offenders}. A dropped step in this job is CONSEQUENTIAL — `test`/`migrations` write "
|
||||
"REQUIRED status contexts, and `build` publishes the release candidate before its smoke step "
|
||||
"runs. Even in a comment a delimiter is unsafe: the runner rewrites the WHOLE body into a "
|
||||
"format(...) call, and if the payload does not parse it DROPS THE STEP and reports the job "
|
||||
"green — so the check passes having done no work (ersatztv#751/#756). Pass the value in "
|
||||
"through the step's `env:` "
|
||||
"block instead; to describe an expression in prose, name it rather than quoting the "
|
||||
"delimiters."
|
||||
)
|
||||
# ANTI-VACUITY. A walk that reached no bodies, or only the trivial ones, would make the
|
||||
# assertion above green while proving nothing. Counted against the job's own step list read
|
||||
# here, so a helper that silently stopped yielding steps is caught rather than rewarded.
|
||||
declared = sum(1 for s in _steps(job) if isinstance(s, dict) and s.get("run"))
|
||||
assert len(_run_steps(job)) == declared >= 3, (
|
||||
f"the walk reached {len(_run_steps(job))} run: bodies but job '{job}' declares {declared}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_every_consequential_run_step_marks_itself_as_its_FIRST_act(job):
|
||||
"""The completeness half — and the only test that catches a NEWLY ADDED step with no marker.
|
||||
|
||||
A guard that checks a fixed list can go quietly incomplete: someone adds a `Test SPA (part 2)`
|
||||
step, it is never marked, the guard never expects it, and a drop of exactly that step is
|
||||
invisible again. So the expectation is DERIVED from the workflow rather than written down twice.
|
||||
|
||||
EXEMPT: steps carrying `continue-on-error: true`. Those are advisory by construction (the
|
||||
peak-anon sampler, the coverage summary) — the workflow already declares that their failure must
|
||||
not redden the job, so their non-execution cannot be a fail-open either. Making them mandatory
|
||||
would be asserting the opposite of what `continue-on-error` means.
|
||||
|
||||
FIRST ACT, not merely present. A marker written at the END of a body records completion, not
|
||||
execution — and this repo has legitimate early-exit paths. More importantly a marker further down
|
||||
can be skipped by an early `exit 0` while the step did nothing, which is the fail-open again one
|
||||
line lower. `set -euo pipefail` is allowed to precede it: it cannot fail, and it is what makes
|
||||
the rest of the body honest.
|
||||
"""
|
||||
missing, late = [], []
|
||||
for s in _run_steps(job):
|
||||
if s.get("continue-on-error") is True or "ci-step-ran.sh assert" in s["run"]:
|
||||
continue
|
||||
m = _MARK.search(s["run"])
|
||||
if not m:
|
||||
missing.append(s.get("name", "?"))
|
||||
continue
|
||||
# By LINE, not by byte offset. The marker sits mid-line (the command is quoted and
|
||||
# prefixed with $GITHUB_WORKSPACE), so slicing at `m.start()` counts the marker's OWN line
|
||||
# prefix as a preceding command and reddens every correctly-written step.
|
||||
lines = s["run"].splitlines()
|
||||
at = next(i for i, ln in enumerate(lines) if _MARK.search(ln))
|
||||
preceding = [
|
||||
ln.strip() for ln in lines[:at]
|
||||
if ln.strip() and not ln.strip().startswith("#")
|
||||
]
|
||||
if [ln for ln in preceding if not ln.startswith("set -")]:
|
||||
late.append((s.get("name", "?"), preceding))
|
||||
assert not missing, (
|
||||
f"these run: steps of the REQUIRED job '{job}' do not record that they executed: {missing}. "
|
||||
"A step the runner drops concludes success, so without a marker its non-execution takes the "
|
||||
"whole required context green having done no work (ersatztv#756). Add "
|
||||
'`\"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh\" mark <key>` as the step\'s first line and '
|
||||
"the key to the guard step's --always/--gated list."
|
||||
)
|
||||
assert not late, (
|
||||
f"these steps of '{job}' mark themselves only after other commands have run: {late}. The "
|
||||
"marker must be the first act, or a body that exits early records nothing while the guard "
|
||||
"still expects it — or worse, records success for work that did not happen."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_the_guard_expects_EXACTLY_the_set_of_marked_keys_in_the_right_bucket(job):
|
||||
"""Set equality in BOTH directions, plus the bucket, because each failure is silent differently.
|
||||
|
||||
A key marked but not expected → the guard never notices that step being dropped: a fail-open
|
||||
that looks fully guarded. A key expected but not marked → the guard reddens on every single run,
|
||||
which is fail-closed but reads as "this guard is broken" and is how a correct guard gets deleted.
|
||||
|
||||
The BUCKET has to match the step's own `if:`. A gated step listed under `--always` reddens every
|
||||
docs-only and already-validated run — the two paths whose entire purpose is to report green in
|
||||
seconds. An always-run step listed under `--gated` stops being checked the moment either skip
|
||||
gate fires, which is a fail-open on precisely the runs where least else is happening.
|
||||
"""
|
||||
marked = _marked(job)
|
||||
keys = [k for _, k in marked]
|
||||
assert len(keys) == len(set(keys)), (
|
||||
f"job '{job}' reuses a marker key: {[k for k in keys if keys.count(k) > 1]}. Two steps "
|
||||
"sharing a key means either one satisfies the guard for both, so dropping one is invisible."
|
||||
)
|
||||
always, gated = _guard_buckets(job)
|
||||
assert sorted(always + gated) == sorted(keys), (
|
||||
f"job '{job}': the guard expects {sorted(always + gated)} but the steps mark "
|
||||
f"{sorted(keys)}. Keys marked-but-unexpected are unguarded drops; keys "
|
||||
"expected-but-unmarked redden every run."
|
||||
)
|
||||
# AN UNRECOGNISED `if:` IS REJECTED, never silently bucketed — found by both reviewers. The
|
||||
# protocol only knows two conditions: absent (always runs) and exactly the skip gate. A marked
|
||||
# step carrying a third condition (`if: github.event_name == 'push'`, or the `always() && <gate>`
|
||||
# spelling the peak-anon steps already use) would fall through to "always", the suite would go
|
||||
# green, and the guard would then demand a step the runner legitimately skipped — reddening a
|
||||
# REQUIRED context and deadlocking `main`. There is already a near-miss in this file: `Report
|
||||
# peak container memory` carries that third spelling and escapes only because it is
|
||||
# `continue-on-error: true` and therefore exempt from marking.
|
||||
for step, key in marked:
|
||||
cond = re.sub(r"\s+", "", str(step.get("if", "")))
|
||||
assert cond in ("", SKIP_GATE), (
|
||||
f"job '{job}': marked step {step.get('name')!r} has an `if:` the guard protocol does not "
|
||||
f"model ({step.get('if')!r}). Only 'absent' and the exact skip gate are understood; "
|
||||
"anything else would be bucketed as --always and would fail the job on a run where the "
|
||||
"step is legitimately skipped. Extend the protocol deliberately, or leave the step "
|
||||
"unmarked."
|
||||
)
|
||||
want = "gated" if _is_gated(step) else "always"
|
||||
got = "gated" if key in gated else "always"
|
||||
assert want == got, (
|
||||
f"job '{job}': step {step.get('name')!r} is {want} (if: {step.get('if')!r}) but the "
|
||||
f"guard lists its key {key!r} under --{got}."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_the_guard_is_the_LAST_step_carries_no_if_and_is_not_advisory(job):
|
||||
"""Position and condition, which together are what make the guard reachable and quiet.
|
||||
|
||||
LAST, because a guard placed before a marked step reads a marker not yet written and fails on
|
||||
every run.
|
||||
|
||||
NO `if:` — a deliberate departure from the #751 guard's `if: always()`, and the thing most likely
|
||||
to be "corrected" back. That job has one real step, so `always()` costs nothing. These jobs have
|
||||
a dozen, and a genuine failure in an early one SKIPS every later step: an `always()` guard would
|
||||
then report "these steps never executed: typecheck web-test build dotnet-test" on top of every
|
||||
ordinary red build. That is the runner obeying its own gating, not a dropped step, and a guard
|
||||
that cries wolf on every red build gets deleted.
|
||||
|
||||
The default `if:` is `success()`, and the invariant that makes relying on it safe rather than
|
||||
lucky: this step is skipped only when an earlier step FAILED, and that failure already fails the
|
||||
job. So `guard skipped => job red`, and every path to a green job runs the guard. A dropped step
|
||||
is invisible precisely because it concludes `success` — which keeps the job green and therefore
|
||||
reaches here.
|
||||
|
||||
NOT `continue-on-error`, which would let it observe the failure and go green anyway — the whole
|
||||
defect, one attribute over.
|
||||
"""
|
||||
steps = _steps(job)
|
||||
guard = _guard(job)
|
||||
assert steps[-1] is guard, (
|
||||
f"the dropped-step guard is not the last step of '{job}' — it is at index "
|
||||
f"{steps.index(guard)} of {len(steps)}, so any marked step after it would be unguarded and "
|
||||
"the guard would read a marker that has not been written yet."
|
||||
)
|
||||
assert "if" not in guard, (
|
||||
f"the '{job}' guard carries `if: {guard.get('if')!r}`. It must have none: the default "
|
||||
"`success()` is what keeps it silent on ordinary red builds, and `always()` would make it "
|
||||
"announce a false 'these steps never executed' on every failing run. See the comment above "
|
||||
"the step for why this is a deliberate departure from the #751 guard."
|
||||
)
|
||||
assert guard.get("continue-on-error") is not True, (
|
||||
f"the '{job}' guard is continue-on-error, so it detects the dropped step and lets the job go "
|
||||
"green regardless — which is the defect it exists to remove."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_the_guards_OWN_body_cannot_be_dropped_by_the_mechanism_it_guards_against(job):
|
||||
"""A guard the guarded mechanism can silently delete is worse than no guard.
|
||||
|
||||
Its absence is silent too: the job simply goes green with nothing checked, which is
|
||||
indistinguishable from a clean run. #751 states the rule; here it is stronger than there,
|
||||
because the body is a single command with no delimiter possible rather than 20 lines of prose
|
||||
that must be kept clean by hand.
|
||||
|
||||
The gate VALUES arrive through `env:`, which the runner interpolates per value — a bad payload
|
||||
there fails that value, not the body. Both are additionally held to naming a real context by
|
||||
test_every_workflow_expression_names_a_REAL_context_or_function in test_pr_changed_files.py.
|
||||
"""
|
||||
guard = _guard(job)
|
||||
assert not _OPENER.search(guard["run"]), (
|
||||
f"the '{job}' guard's own run body contains an expression delimiter, so the runner can drop "
|
||||
"the guard the same way it drops the steps the guard is watching — and that absence is "
|
||||
"silent as well."
|
||||
)
|
||||
assert guard["run"].strip().startswith("scripts/ci-step-ran.sh assert"), (
|
||||
f"the '{job}' guard is no longer a bare invocation: {guard['run']!r}. Keeping it to one "
|
||||
"command is what makes a delimiter impossible rather than merely absent."
|
||||
)
|
||||
# THE VALUES, not just the names — found by cold review. Asserting the keys alone accepts
|
||||
# `ETV_DOCS_ONLY: ${{ steps.detect.outputs.doc_only }}` (note the typo), which names a real
|
||||
# context so the repo-wide expression check passes it too. The guard would then read an EMPTY
|
||||
# value on a docs-only run, demand the gated steps that were correctly skipped, and redden a
|
||||
# REQUIRED context on every docs-only PR.
|
||||
# THE TWO MAPPINGS MUST BE PRESENT AND CORRECT — but this deliberately does NOT demand that the
|
||||
# `env:` block contain ONLY them. An earlier version compared the whole dict, which false-redded
|
||||
# on adding an unrelated variable (an `LC_ALL`, say) and on the equally-valid `${{x}}` spacing;
|
||||
# a red here blocks every merge through the combined status, so brittleness is a real cost and
|
||||
# not a free strictness win. Whitespace inside the delimiters is normalised for the same reason.
|
||||
env = {k: re.sub(r"\s+", "", str(v)) for k, v in (guard.get("env") or {}).items()}
|
||||
for name, want in (("ETV_DOCS_ONLY", "${{steps.detect.outputs.docs_only}}"),
|
||||
("ETV_REVALIDATE_SKIP", "${{steps.revalidate.outputs.skip}}")):
|
||||
assert env.get(name) == want, (
|
||||
f"the '{job}' guard's env: has {name}={guard.get('env', {}).get(name)!r}, expected the "
|
||||
f"output the gated steps' own `if:` reads ({want}). A typo here is SILENT rather than "
|
||||
"loud: it still names a real context, so the repo-wide expression check passes it, the "
|
||||
"value arrives empty, and the guard then demands steps that were legitimately skipped — "
|
||||
"reddening a REQUIRED context on every docs-only run."
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# BEHAVIOURAL — the guard's real command line, against markers written by the steps' real lines
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mark_line(step) -> str:
|
||||
"""The step's OWN marker line, verbatim from the workflow.
|
||||
|
||||
Extracted rather than rebuilt in Python ON PURPOSE. A test that composed the command itself
|
||||
would keep passing after the workflow and the script drifted apart on the path, the quoting or
|
||||
the sub-command — and that divergence is exactly the failure that makes the guard fail on every
|
||||
run and then get deleted as broken. Running the real line proves the two agree by construction.
|
||||
"""
|
||||
line = next(ln for ln in step["run"].splitlines() if _MARK.search(ln))
|
||||
return line.strip()
|
||||
|
||||
|
||||
# THE GATE VALUES DEFAULT TO `"false"`, WHICH IS WHAT THE RUNNER ACTUALLY SENDS — and getting this
|
||||
# wrong made the whole suite blind. Found by cold review, which demonstrated it: every behavioural
|
||||
# test used to leave these UNSET, so the guard was never once driven at its production values. Change
|
||||
# the gate in `ci-step-ran.sh` from `= "true"` to `-n` — a one-token regression — and all 30 tests
|
||||
# stayed GREEN while the guard, run with the real environment, reported
|
||||
# `Skip gate fired (docs_only='false') … All 2 expected step(s) executed` and exited 0. `Build`,
|
||||
# `Test` and both migration replays would have been unguarded on every ordinary run, with the guard
|
||||
# announcing that it had proved everything.
|
||||
#
|
||||
# THE COMPLETE VALUE SET, and where each comes from — worth spelling out, because the obvious reading
|
||||
# of the evidence is wrong. Both producers document `true|false` and write exactly that
|
||||
# (`scripts/ci-detect-docs-only.sh` -> `docs_only=`, `scripts/ci-detect-already-validated.sh` ->
|
||||
# `skip=`), so an ordinary run sends `false` and a skipping run sends `true`.
|
||||
#
|
||||
# The live log of the probe this change cites (run 1910, job 8064) shows `ETV_DOCS_ONLY: false` and
|
||||
# `ETV_REVALIDATE_SKIP:` EMPTY — but do NOT read that as revalidate's normal output. `revalidate` was
|
||||
# the step the probe deliberately dropped, so it wrote no output at all. The empty string is
|
||||
# therefore not an odd third state: it is the SIGNATURE OF THE VERY FAILURE THIS GUARD EXISTS TO
|
||||
# CATCH, which is exactly why the gate must treat anything that is not `true` as "widen what is
|
||||
# required". `None` (unset) is the same case reached a different way.
|
||||
#
|
||||
# A test double is an assertion about what the real system sends, and the earlier version of this one
|
||||
# was wrong about the only field the guard branches on.
|
||||
GATE_VALUES_IN_THE_WILD = ("false", "", None)
|
||||
|
||||
|
||||
def _env(tmp_path, **extra):
|
||||
env = {
|
||||
"PATH": os.environ["PATH"],
|
||||
"GITHUB_WORKSPACE": str(REPO_ROOT),
|
||||
"RUNNER_TEMP": str(tmp_path),
|
||||
"GITHUB_JOB": "test",
|
||||
"GITHUB_RUN_ID": "424242",
|
||||
"GITHUB_RUN_ATTEMPT": "7",
|
||||
"ETV_DOCS_ONLY": "false",
|
||||
"ETV_REVALIDATE_SKIP": "false",
|
||||
}
|
||||
env.update(extra)
|
||||
return {k: v for k, v in env.items() if v is not None}
|
||||
|
||||
|
||||
def _run(script: str, env):
|
||||
return subprocess.run(["bash", "-c", script], cwd=REPO_ROOT, env=env,
|
||||
capture_output=True, text=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gate", GATE_VALUES_IN_THE_WILD,
|
||||
ids=["gate-false", "gate-empty", "gate-unset"])
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_the_guard_PASSES_when_every_step_marked_itself(job, gate, tmp_path):
|
||||
"""The positive control. Without it, a guard that always failed would satisfy every case below.
|
||||
|
||||
`GITHUB_JOB` is set to the job under test, so this also covers the marker file being keyed per
|
||||
job: if it were not, the two jobs would share a file and one job's markers would answer for the
|
||||
other's dropped steps.
|
||||
"""
|
||||
marks = [_mark_line(s) for s, _ in _marked(job)]
|
||||
guard = _guard(job)["run"]
|
||||
# Parametrised over every NOT-SKIPPING spelling the runner emits — `false` on an ordinary run,
|
||||
# empty when the producing step was dropped, absent if the output is never set. All three must
|
||||
# require the gated steps; a gate that treats any of them as a skip is fail-open on that path.
|
||||
env = _env(tmp_path, GITHUB_JOB=job, ETV_DOCS_ONLY=gate, ETV_REVALIDATE_SKIP=gate)
|
||||
r = _run("\n".join(["set -e", *marks, guard]), env)
|
||||
assert r.returncode == 0, (
|
||||
f"the '{job}' guard rejected a run in which every step marked itself — the steps and the "
|
||||
f"guard disagree, so this would fail on every run.\n{r.stdout}\n{r.stderr}"
|
||||
)
|
||||
assert "All" in r.stdout and "executed" in r.stdout, r.stdout
|
||||
# The other half of the identity contract: with GITHUB_RUN_ATTEMPT set (`_env` sends 7) the line
|
||||
# must report the REAL value and say so. A mis-derivation (`${marker#*-}` rather than `##`) or an
|
||||
# inverted provenance test would otherwise ship silently, and the operator reading this line to
|
||||
# settle the promotion question would read it wrong.
|
||||
assert f"Marker identity: job={job} run=424242 attempt=7 (from the runner)" in r.stdout, (
|
||||
f"the guard misreported its marker identity: {r.stdout!r}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_dropping_ANY_single_step_FAILS_the_guard(job, tmp_path):
|
||||
"""Every marked step, one at a time — not a sample.
|
||||
|
||||
An arbitrary sample gives false negatives here: the interesting drop is `Test` or the migration
|
||||
replay, and a test that only omitted the first step would prove the guard catches the one case
|
||||
that was never fail-open anyway. Dropping each key in turn is the only version that establishes
|
||||
the property the issue asks for.
|
||||
"""
|
||||
marked = _marked(job)
|
||||
guard = _guard(job)["run"]
|
||||
for dropped_step, dropped_key in marked:
|
||||
d = tmp_path / dropped_key
|
||||
d.mkdir()
|
||||
marks = [_mark_line(s) for s, k in marked if k != dropped_key]
|
||||
r = _run("\n".join(["set -e", *marks, guard]), _env(d, GITHUB_JOB=job))
|
||||
assert r.returncode != 0, (
|
||||
f"job '{job}': the guard went GREEN with {dropped_step.get('name')!r} "
|
||||
f"(key {dropped_key!r}) never having executed. That is a REQUIRED context reporting "
|
||||
f"success having skipped that work — the exact fail-open of ersatztv#756.\n{r.stdout}"
|
||||
)
|
||||
assert dropped_key in (r.stdout + r.stderr), (
|
||||
f"the guard failed but did not name the missing step {dropped_key!r}: {r.stdout}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
@pytest.mark.parametrize("gate", ["ETV_DOCS_ONLY", "ETV_REVALIDATE_SKIP"])
|
||||
def test_a_fired_skip_gate_does_not_require_the_gated_steps(gate, job, tmp_path):
|
||||
"""The docs-only and already-validated paths must still report green in seconds.
|
||||
|
||||
They are the reason these jobs are never `if:`-skipped at the JOB level (a skipped required
|
||||
context is a state this repo deliberately does not rely on — ersatztv#416/#418), so a guard that
|
||||
reddened them would make every docs-only PR unmergeable. Which is #751's user-visible symptom
|
||||
arriving from the opposite direction, and worth a test rather than a comment.
|
||||
"""
|
||||
marks = [_mark_line(s) for s, k in _marked(job) if k in _guard_buckets(job)[0]]
|
||||
guard = _guard(job)["run"]
|
||||
r = _run("\n".join(["set -e", *marks, guard]), _env(tmp_path, GITHUB_JOB=job, **{gate: "true"}))
|
||||
assert r.returncode == 0, (
|
||||
f"with {gate}=true the guard still demanded the gated steps, so every docs-only / "
|
||||
f"already-validated run of a REQUIRED job would be red.\n{r.stdout}\n{r.stderr}"
|
||||
)
|
||||
assert "Skip gate fired" in r.stdout, r.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_a_fired_skip_gate_STILL_requires_the_ALWAYS_steps(job, tmp_path):
|
||||
"""The negative control for the test above — otherwise `ETV_DOCS_ONLY=true` would be a blanket
|
||||
off-switch and the previous test would be passing for the wrong reason.
|
||||
|
||||
This is the case that matters most on a docs-only run: the detect steps are the only things that
|
||||
execute, so if their drop were unguarded the skip path would be entirely unchecked.
|
||||
"""
|
||||
guard = _guard(job)["run"]
|
||||
r = _run("\n".join(["set -e", guard]), _env(tmp_path, GITHUB_JOB=job, ETV_DOCS_ONLY="true"))
|
||||
assert r.returncode != 0, (
|
||||
"with ETV_DOCS_ONLY=true and NO steps marked at all, the guard passed — the skip gate is "
|
||||
"acting as a blanket off-switch rather than as a narrowing of what is expected."
|
||||
)
|
||||
assert "detect" in (r.stdout + r.stderr), r.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_an_EMPTY_gate_value_requires_the_gated_steps(job, tmp_path):
|
||||
"""A dropped `detect` step leaves its outputs EMPTY, not 'false'.
|
||||
|
||||
Reading empty as "skipped" would mean the one drop that disables the detect step also disables
|
||||
the guard for everything downstream — the guard switching itself off in response to the very
|
||||
failure it exists to catch. The direction has to be: anything that is not exactly `true` widens
|
||||
what is required.
|
||||
"""
|
||||
guard = _guard(job)["run"]
|
||||
r = _run("\n".join(["set -e", guard]),
|
||||
_env(tmp_path, GITHUB_JOB=job, ETV_DOCS_ONLY="", ETV_REVALIDATE_SKIP=""))
|
||||
assert r.returncode != 0
|
||||
assert _guard_buckets(job)[1][-1] in (r.stdout + r.stderr), (
|
||||
f"empty gate values were read as a skip, so the gated steps went unchecked: {r.stdout}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_STALE_marker_from_another_run_cannot_satisfy_the_guard(tmp_path):
|
||||
"""A marker from another run, attempt or job must never answer for this one.
|
||||
|
||||
Do NOT restate this as "RUNNER_TEMP is /tmp, not a private per-job directory". That is a #751
|
||||
measurement taken on a job with no `container:`, and it does not transfer: these two jobs run
|
||||
inside the CI toolchain image, so their `/tmp` is the container's own. The fresh container is
|
||||
what actually rules out staleness here; the keying is defence in depth against a lane change
|
||||
nobody would think to re-check this against, and that is why it is still worth testing.
|
||||
"""
|
||||
marks = [_mark_line(s) for s, _ in _marked("test")]
|
||||
guard = _guard("test")["run"]
|
||||
# Run 1 marks everything.
|
||||
first = _env(tmp_path, GITHUB_JOB="test", GITHUB_RUN_ID="111", GITHUB_RUN_ATTEMPT="1")
|
||||
assert _run("\n".join(["set -e", *marks]), first).returncode == 0
|
||||
# Run 2 shares RUNNER_TEMP but marks nothing. It must NOT inherit run 1's markers.
|
||||
second = _env(tmp_path, GITHUB_JOB="test", GITHUB_RUN_ID="222", GITHUB_RUN_ATTEMPT="1")
|
||||
r = _run(guard, second)
|
||||
assert r.returncode != 0, (
|
||||
"a marker file left by a DIFFERENT run satisfied the guard, so a run whose steps were all "
|
||||
f"dropped would pass silently.\n{r.stdout}"
|
||||
)
|
||||
# ...and a RETRY of run 1 must not inherit run 1's either.
|
||||
retry = _env(tmp_path, GITHUB_JOB="test", GITHUB_RUN_ID="111", GITHUB_RUN_ATTEMPT="2")
|
||||
assert _run(guard, retry).returncode != 0, (
|
||||
"a re-run inherited the first attempt's markers, so a step dropped only on the retry passes"
|
||||
)
|
||||
# ...nor may the OTHER job in the same run inherit them.
|
||||
sibling = _env(tmp_path, GITHUB_JOB="migrations", GITHUB_RUN_ID="111", GITHUB_RUN_ATTEMPT="1")
|
||||
assert _run(_guard("migrations")["run"], sibling).returncode != 0, (
|
||||
"the two required jobs share one marker file, so one job's markers answer for the other's "
|
||||
"dropped steps"
|
||||
)
|
||||
|
||||
|
||||
def test_assert_with_no_expected_keys_REFUSES_instead_of_passing(tmp_path):
|
||||
"""The script's own anti-vacuity check, exercised rather than trusted.
|
||||
|
||||
`assert` with an empty expectation list would print "All 0 expected step(s) executed" and exit 0
|
||||
— a guard that proves nothing while reporting that it proved everything. That is how a guard
|
||||
ends up shipped and dead, which this repo has now done twice (#751's fence, #751's own guard).
|
||||
"""
|
||||
r = _run(f"{SCRIPT} assert", _env(tmp_path))
|
||||
assert r.returncode == 2, f"expected a usage refusal, got {r.returncode}: {r.stdout} {r.stderr}"
|
||||
assert "no expected keys" in (r.stdout + r.stderr)
|
||||
|
||||
|
||||
def test_mark_APPENDS_so_one_step_does_not_erase_its_predecessors(tmp_path):
|
||||
"""`>` instead of `>>` in the script would leave only the last step's key.
|
||||
|
||||
The guard would then redden on every run — fail-closed, but it would look like the guard is
|
||||
broken rather than like a real drop, and that is the state in which a correct guard gets removed.
|
||||
"""
|
||||
env = _env(tmp_path)
|
||||
assert _run(f"{SCRIPT} mark alpha && {SCRIPT} mark beta", env).returncode == 0
|
||||
r = _run(f"{SCRIPT} assert --always alpha beta", env)
|
||||
assert r.returncode == 0, f"the second mark erased the first: {r.stdout} {r.stderr}"
|
||||
|
||||
|
||||
def test_a_key_is_matched_WHOLE_not_as_a_substring(tmp_path):
|
||||
"""`build` must not be satisfied by `web-build`, and `test` not by `web-test`.
|
||||
|
||||
Both pairs are live key names in the `test` job, so a substring match would mean dropping the
|
||||
real `Build` or `Test` step — the two most consequential steps in the whole workflow — is
|
||||
invisible because an SPA step of a similar name ran.
|
||||
"""
|
||||
env = _env(tmp_path)
|
||||
assert _run(f"{SCRIPT} mark web-build && {SCRIPT} mark web-test", env).returncode == 0
|
||||
r = _run(f"{SCRIPT} assert --always build", env)
|
||||
assert r.returncode != 0, (
|
||||
"the key 'build' was satisfied by a marker for 'web-build' — a dropped `dotnet build` would "
|
||||
"pass unnoticed"
|
||||
)
|
||||
|
||||
|
||||
def test_a_degraded_run_IDENTITY_refuses_rather_than_sharing_a_marker_path(tmp_path):
|
||||
"""`GITHUB_RUN_ID` absent must REFUSE, not fall back to a name every run shares.
|
||||
|
||||
The first version of `marker_path` defaulted to `nojob`/`norunid`/`1`. Those are reusable, so a
|
||||
leftover marker from any earlier run on the host would satisfy the guard on a run whose step was
|
||||
dropped — a silent PASS, which is the precise failure the run-keying exists to remove,
|
||||
reintroduced by the code implementing it. Found by cold review.
|
||||
|
||||
Asserted on BOTH sub-commands: a refusal that only `assert` honoured would let `mark` write to a
|
||||
shared path and leave the two disagreeing about where the file is.
|
||||
"""
|
||||
env = _env(tmp_path)
|
||||
for var in ("GITHUB_RUN_ID", "GITHUB_JOB", "GITHUB_RUN_ATTEMPT"):
|
||||
degraded = {k: v for k, v in env.items() if k != var}
|
||||
for argv in (f"{SCRIPT} mark alpha", f"{SCRIPT} assert --always alpha"):
|
||||
r = _run(argv, degraded)
|
||||
assert r.returncode != 0, (
|
||||
f"with {var} unset, `{argv.split()[-2]}` continued and used a fallback path that "
|
||||
f"other runs also use — a stale marker there passes the guard on a dropped run.\n"
|
||||
f"{r.stdout}{r.stderr}"
|
||||
)
|
||||
assert "cannot identify this run" in (r.stdout + r.stderr), (
|
||||
f"refused, but without naming the cause: {r.stdout!r} {r.stderr!r}")
|
||||
assert not list(tmp_path.iterdir()), (
|
||||
"a degraded-identity `mark` still created a marker file somewhere under RUNNER_TEMP")
|
||||
|
||||
|
||||
def test_the_marker_identity_is_REPORTED_on_stdout_every_run(tmp_path):
|
||||
"""The line that settled `GITHUB_RUN_ATTEMPT`, kept as standing evidence.
|
||||
|
||||
Worth recording HOW that was settled, because the first two attempts were both bad. Grepping a
|
||||
job log for the variable NAME proves nothing (logs do not dump the environment). Inferring it
|
||||
from the ABSENCE of a "not set" warning proves nothing either, because that warning goes to
|
||||
stderr and whether step stderr reaches a job log here was itself never established — the control
|
||||
offered for that was an `::error::` this script writes to STDOUT. So the script was made to
|
||||
REPORT its resolved identity on stdout, where capture is not in question, and the answer was read
|
||||
off ersatztv#756's own PR run: `Marker identity: job=test run=1916 attempt=1 (from the runner)`,
|
||||
and the same for `migrations`. That is what promoted the variable from warn-and-default to
|
||||
required.
|
||||
|
||||
Asserted because cold review demonstrated three mutations of this reporting — deleting the echo,
|
||||
mis-deriving the attempt, inverting the provenance — all surviving a 50-green suite. It is a
|
||||
documented contract (the record's `mechanics:`), and a future reader is told to trust it.
|
||||
"""
|
||||
marks = [_mark_line(s) for s, _ in _marked("test")]
|
||||
r = _run("\n".join(["set -e", *marks, _guard("test")["run"]]),
|
||||
_env(tmp_path, GITHUB_RUN_ID="1916", GITHUB_RUN_ATTEMPT="4"))
|
||||
assert r.returncode == 0, r.stdout + r.stderr
|
||||
assert "Marker identity: job=test run=1916 attempt=4 (from the runner)" in r.stdout, (
|
||||
"the guard did not report the identity its marker path was actually keyed on, so a reader "
|
||||
f"cannot audit the keying from a run log: {r.stdout!r}")
|
||||
|
||||
|
||||
def test_a_skip_gate_that_empties_the_expected_set_REFUSES(tmp_path):
|
||||
"""The anti-vacuity check has to run AFTER the gate, not only on argv. Cold review reproduced
|
||||
this exactly:
|
||||
|
||||
ETV_DOCS_ONLY=true … assert --always --gated foo
|
||||
-> "All 0 expected step(s) executed", exit 0
|
||||
|
||||
The argv check cannot see it, because the set is emptied by the gate rather than by the caller.
|
||||
Unreachable with today's argv, but it contradicted the comment directly above it — and "reports
|
||||
that it proved everything while proving nothing" is the failure this whole file exists to remove.
|
||||
"""
|
||||
r = _run(f"{SCRIPT} assert --always --gated foo", _env(tmp_path, ETV_DOCS_ONLY="true"))
|
||||
assert r.returncode != 0, (
|
||||
f"the guard passed with an empty post-gate expectation set: {r.stdout!r}")
|
||||
assert "no expected keys" in (r.stdout + r.stderr).lower() or "NO expected keys" in r.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("revalidate", ["true", "false", "", None],
|
||||
ids=lambda v: f"reval-{v if v is not None else 'unset'}")
|
||||
@pytest.mark.parametrize("docs_only", ["true", "false", "", None],
|
||||
ids=lambda v: f"docs-{v if v is not None else 'unset'}")
|
||||
def test_the_skip_gate_over_the_WHOLE_value_matrix(docs_only, revalidate, tmp_path):
|
||||
"""Every combination of the two gate values, not just the diagonal — cold review's last finding.
|
||||
|
||||
Round 3 fixed the suite's blindness to the production value `false`, but still only exercised
|
||||
matched pairs and single-`true` cases. `(true, true)` is REACHABLE — a docs-only PR merged to
|
||||
`main` whose tree was already validated sets both — and an exclusive-or regression would pass
|
||||
every other test here while demanding all the gated markers on a run that legitimately skipped
|
||||
those steps. That reddens BOTH required contexts, which is the false-red direction: it deadlocks
|
||||
every merge rather than letting one through.
|
||||
|
||||
The property asserted is the whole contract in one line: with only the `--always` keys marked,
|
||||
the guard passes exactly when the gate says the gated steps were skipped — `true` in EITHER
|
||||
variable, and nothing else. Sixteen cases, so no combination is a special case anyone has to
|
||||
remember.
|
||||
|
||||
On `unset`: the workflow's `env:` block always defines both, emitting EMPTY for an output the
|
||||
producing step never wrote, so unset is not reachable through the workflow. It is covered because
|
||||
the script is also runnable by hand, and because "not exactly true" is the property that must
|
||||
hold for every spelling rather than for an enumerated list.
|
||||
"""
|
||||
job = "test"
|
||||
always, gated = _guard_buckets(job)
|
||||
marks = [_mark_line(s) for s, k in _marked(job) if k in always]
|
||||
r = _run("\n".join(["set -e", *marks, _guard(job)["run"]]),
|
||||
_env(tmp_path, ETV_DOCS_ONLY=docs_only, ETV_REVALIDATE_SKIP=revalidate))
|
||||
should_skip = docs_only == "true" or revalidate == "true"
|
||||
assert (r.returncode == 0) is should_skip, (
|
||||
f"with docs_only={docs_only!r} and revalidate={revalidate!r} the guard "
|
||||
f"{'passed' if r.returncode == 0 else 'failed'}, expected it to "
|
||||
f"{'skip the gated keys' if should_skip else 'require them'}. The gate must treat a value as "
|
||||
"a skip if and only if it is exactly `true` in EITHER variable.\n" + r.stdout + r.stderr)
|
||||
@@ -0,0 +1,243 @@
|
||||
"""The `scan` job — the delimiter ban made fail-CLOSED on the release path (ersatztv#767).
|
||||
|
||||
WHAT THIS IS PROTECTING. #756 brought `build` into the delimiter ban, because a dropped
|
||||
`Smoke + IPTV E2E` publishes a release candidate that was never booted and reports the job green.
|
||||
But the ban was enforced ONLY by `test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body`
|
||||
in `script-tests` — `on: pull_request`, not a required context. Nothing re-checked it on a `v*` tag
|
||||
push, which is exactly when the candidate is published.
|
||||
|
||||
WHY A JOB AND NOT A STEP IN `build`, and why this file is structural. The first cut of #767 put a
|
||||
bespoke stdlib scanner in `build` itself. Two independent reviews killed it on two counts, and both
|
||||
are worth keeping written down because both are easy to re-invent:
|
||||
|
||||
* A guard step inside `build` cannot protect `build`. If the runner drops it, the job carries on
|
||||
and publishes — fail-OPEN. The defence offered was "the guard's own body has no opener, so it
|
||||
cannot be dropped", but the only thing enforcing THAT was the same PR-only test being
|
||||
backstopped. Circular. As a `needs:` of `build`, a red here means `build` never runs at all.
|
||||
* The bespoke scanner hand-parsed YAML (to avoid provisioning PyYAML on `build`'s bare runner) and
|
||||
had ~10 false NEGATIVES within one review round — flow mappings, a quoted `"run":` key, aliases,
|
||||
multiline quoted scalars. It was strictly WEAKER than the check it backstopped, in the only
|
||||
direction that matters. The fix was to delete it and run the real PyYAML-based test, which needs
|
||||
no second definition of "what is a `run:` body" and so has no drift surface.
|
||||
|
||||
So the detection logic is not retested here — it lives in `test_ci_dropped_step_guard.py` and this
|
||||
job runs that file. What this file holds is the WIRING, which is what makes the ban fail-closed:
|
||||
the job exists, `build` depends on it, nothing can skip it, its own steps cannot be silently
|
||||
dropped, and it actually invokes the ban test.
|
||||
|
||||
WHAT THIS DOES NOT CLAIM. That no step can ever fail to run for a reason other than the
|
||||
interpolation drop. This job's own steps carry #756 markers and a trailing assert, so the regress
|
||||
terminates where the sibling guards' does — to fail open you must now drop the pytest step AND the
|
||||
assert step, not either one. The end-to-end behaviour (a poisoned `Smoke` body reddens `scan` and
|
||||
`build` never runs) is a LIVE measurement recorded on the issue, not something a static test here
|
||||
can establish.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "docker-build.yml"
|
||||
SCRIPT = REPO_ROOT / "scripts" / "ci-step-ran.sh"
|
||||
|
||||
_DOC = yaml.safe_load(WORKFLOW.read_text())
|
||||
_OPENER = re.compile(r"\$\{\{")
|
||||
_MARK = re.compile(r'ci-step-ran\.sh"?\s+mark\s+(\S+)')
|
||||
|
||||
JOB = "scan"
|
||||
BAN_TEST_FILE = "scripts/tests/test_ci_dropped_step_guard.py"
|
||||
|
||||
|
||||
def _job():
|
||||
assert JOB in _DOC["jobs"], f"the `{JOB}` job is gone — the release path is unguarded again"
|
||||
return _DOC["jobs"][JOB]
|
||||
|
||||
|
||||
def _steps():
|
||||
return _job()["steps"]
|
||||
|
||||
|
||||
def _run_steps():
|
||||
return [s for s in _steps() if s.get("run")]
|
||||
|
||||
|
||||
def _guard():
|
||||
"""The trailing assert step, located by CONTENT — never by index, so that
|
||||
`test_the_guard_is_the_LAST_step` is not true by construction."""
|
||||
hits = [s for s in _run_steps() if "ci-step-ran.sh assert" in s["run"]]
|
||||
assert len(hits) == 1, f"expected exactly 1 assert step in `{JOB}`, found {len(hits)}"
|
||||
return hits[0]
|
||||
|
||||
|
||||
def _marked():
|
||||
out = []
|
||||
for s in _run_steps():
|
||||
m = _MARK.search(s["run"])
|
||||
if m:
|
||||
out.append((s, m.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# WIRING — the properties that make the ban fail-closed
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_DEPENDS_on_the_scan_job():
|
||||
"""This single edge is the whole fail-closed property.
|
||||
|
||||
Without it the scan is advisory: it could go red while `build` publishes anyway.
|
||||
"""
|
||||
needs = _DOC["jobs"]["build"]["needs"]
|
||||
needs = [needs] if isinstance(needs, str) else needs
|
||||
assert JOB in needs, f"`build` no longer needs `{JOB}` — a red scan would not stop a release"
|
||||
|
||||
|
||||
def test_the_scan_job_has_NO_job_level_if():
|
||||
"""Two failure modes at once, in opposite directions.
|
||||
|
||||
An `if:` that excludes the tag push would leave the release path unguarded — the exact hole
|
||||
#767 closed. An `if:` that skipped it for any other reason would SKIP `build` too (a skipped
|
||||
dependency skips its dependents), breaking every release. Neither is wanted: it always runs.
|
||||
"""
|
||||
job = _job()
|
||||
assert "if" not in job, f"`{JOB}` must carry no job-level `if:`, found {job.get('if')!r}"
|
||||
|
||||
|
||||
def test_the_scan_job_actually_invokes_the_ban_test():
|
||||
"""Otherwise the job is an expensive no-op that reports green.
|
||||
|
||||
Asserted against the file path the ban test really lives in, so renaming that file without
|
||||
updating the workflow is a red here rather than a silently unguarded release path.
|
||||
"""
|
||||
assert (REPO_ROOT / BAN_TEST_FILE).is_file()
|
||||
assert any(BAN_TEST_FILE in s["run"] for s in _run_steps()), (
|
||||
f"no step in `{JOB}` runs {BAN_TEST_FILE}"
|
||||
)
|
||||
|
||||
|
||||
def test_no_step_in_the_scan_job_is_advisory():
|
||||
"""`continue-on-error: true` would make the whole gate a no-op while every other test here
|
||||
stayed green — it is the cheapest way to accidentally disarm this."""
|
||||
offenders = [s.get("name") for s in _steps() if s.get("continue-on-error")]
|
||||
assert not offenders, f"advisory step(s) in `{JOB}`: {offenders}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"step_name",
|
||||
[s.get("name", "?") for s in yaml.safe_load(WORKFLOW.read_text())["jobs"][JOB]["steps"]
|
||||
if s.get("run")],
|
||||
)
|
||||
def test_every_run_body_in_the_scan_job_is_delimiter_free(step_name):
|
||||
"""The guard must not be vulnerable to the defect it guards against.
|
||||
|
||||
Not a proof that it always runs — a construction argument about ONE mechanism, the same axiom
|
||||
the sibling guards rest on. It is asserted per step so a failure names which step regressed.
|
||||
"""
|
||||
step = next(s for s in _run_steps() if s.get("name", "?") == step_name)
|
||||
assert not _OPENER.search(step["run"]), (
|
||||
f"step {step_name!r} of `{JOB}` contains an expression delimiter; the runner would rewrite "
|
||||
"the whole body and DROP the step while reporting success (ersatztv#751). Pass values "
|
||||
"through `env:`, which is interpolated per value."
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE JOB'S OWN DROPPED-STEP GUARD
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_every_consequential_step_marks_itself():
|
||||
"""Every `run:` step except the guard records that it executed."""
|
||||
marked = {s.get("name") for s, _ in _marked()}
|
||||
expected = {s.get("name") for s in _run_steps() if s is not _guard()}
|
||||
assert marked == expected, f"unmarked step(s) in `{JOB}`: {expected - marked}"
|
||||
|
||||
|
||||
def test_the_guard_expectations_match_the_markers_exactly():
|
||||
"""The set the guard waits for IS the set the steps write — derived from the workflow, not
|
||||
restated here, so adding a step without a marker is a red."""
|
||||
argv = _guard()["run"].split()
|
||||
assert "--always" in argv, argv
|
||||
always = argv[argv.index("--always") + 1:]
|
||||
assert "--gated" not in argv, "every step in this job is unconditional; there is nothing to gate"
|
||||
assert sorted(always) == sorted(k for _, k in _marked())
|
||||
|
||||
|
||||
def test_the_guard_is_the_LAST_step():
|
||||
assert _steps()[-1] is _guard(), "the assert must run after the steps it checks"
|
||||
|
||||
|
||||
def test_the_guard_has_no_if():
|
||||
"""Same reasoning as the sibling guards: the default `success()` is wanted, because a genuine
|
||||
early failure legitimately skips later steps and already fails the job."""
|
||||
assert "if" not in _guard()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# BEHAVIOURAL — the guard's REAL command line, against the steps' REAL marker lines
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mark_line(step) -> str:
|
||||
"""The step's own marker line, verbatim from the workflow — never rebuilt in Python, so a
|
||||
drift between the workflow and the script cannot hide behind a test that composed its own."""
|
||||
return next(ln for ln in step["run"].splitlines() if _MARK.search(ln)).strip()
|
||||
|
||||
|
||||
def _env(tmp_path, **extra):
|
||||
env = {
|
||||
"PATH": os.environ["PATH"],
|
||||
"GITHUB_WORKSPACE": str(REPO_ROOT),
|
||||
"RUNNER_TEMP": str(tmp_path),
|
||||
"GITHUB_JOB": JOB,
|
||||
"GITHUB_RUN_ID": "424242",
|
||||
"GITHUB_RUN_ATTEMPT": "7",
|
||||
}
|
||||
env.update(extra)
|
||||
return {k: v for k, v in env.items() if v is not None}
|
||||
|
||||
|
||||
def _run(script: str, env):
|
||||
return subprocess.run(["bash", "-c", script], cwd=REPO_ROOT, env=env,
|
||||
capture_output=True, text=True)
|
||||
|
||||
|
||||
def test_the_guard_PASSES_when_every_step_ran(tmp_path):
|
||||
env = _env(tmp_path)
|
||||
for step, _ in _marked():
|
||||
assert _run(_mark_line(step), env).returncode == 0
|
||||
res = _run(_guard()["run"], env)
|
||||
assert res.returncode == 0, res.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dropped", [k for _, k in _marked()])
|
||||
def test_the_guard_FAILS_when_a_step_was_dropped(tmp_path, dropped):
|
||||
"""The positive control. Drop each key in turn — the guard must go red and NAME it.
|
||||
|
||||
A guard only ever exercised on the happy path is indistinguishable from one that passes
|
||||
unconditionally, which is the failure this whole mechanism exists to remove.
|
||||
"""
|
||||
env = _env(tmp_path)
|
||||
for step, key in _marked():
|
||||
if key != dropped:
|
||||
assert _run(_mark_line(step), env).returncode == 0
|
||||
res = _run(_guard()["run"], env)
|
||||
assert res.returncode != 0, f"guard passed despite '{dropped}' never running: {res.stdout}"
|
||||
# BOTH streams: the script's `::error::` lands on stdout here while other diagnostics go to
|
||||
# stderr, and a test that picked the wrong one would assert on an empty string and pass for the
|
||||
# wrong reason on any message change.
|
||||
assert dropped in (res.stdout + res.stderr), (res.stdout, res.stderr)
|
||||
|
||||
|
||||
def test_the_guard_REFUSES_to_pass_with_no_expectations(tmp_path):
|
||||
"""`assert` with an empty expectation set would report success having checked nothing."""
|
||||
res = _run(f"{SCRIPT} assert --always", _env(tmp_path))
|
||||
assert res.returncode != 0
|
||||
@@ -71,23 +71,20 @@ def test_frontmatter_reader_matches_pyyaml_on_every_real_record():
|
||||
made the validator crash with ModuleNotFoundError once the corpus was migrated.) A hand parser
|
||||
is only safe if it provably matches the library that WROTE the files, so this compares the two
|
||||
across every record rather than on a sample.
|
||||
|
||||
Since #674 the comparison itself lives in `decisions_validate.pyyaml_frontmatter_faults`, which
|
||||
the VALIDATOR now runs too — before that it existed only here, so `decisions_validate.py`
|
||||
happily reported OK on a record PyYAML rejects. This test delegates to that one implementation
|
||||
rather than keeping a second copy of the comparison, so the suite and the validator cannot
|
||||
drift apart and agree on what "matches PyYAML" means.
|
||||
"""
|
||||
yaml = pytest.importorskip("yaml")
|
||||
pytest.importorskip("yaml")
|
||||
import scripts.decisions_validate as dv
|
||||
|
||||
files = [p for p in dl.RECORDS_DIR.rglob("*.md")] + [p for p in dl.ARCHIVE_DIR.rglob("*.md")]
|
||||
files = [f for f in files if dl.has_frontmatter(f.read_text(encoding="utf-8"))]
|
||||
assert len(files) > 100, f"only {len(files)} frontmatter files found — test would be near-vacuous"
|
||||
|
||||
diffs = []
|
||||
for f in files:
|
||||
lines = f.read_text(encoding="utf-8").splitlines()
|
||||
end = next(i for i, ln in enumerate(lines[1:], start=1) if ln.rstrip() == "---")
|
||||
block = "\n".join(lines[1:end])
|
||||
mine = dl._read_frontmatter(block)
|
||||
theirs = yaml.safe_load(block) or {}
|
||||
theirs = {k: ("" if v is None else str(v)) for k, v in theirs.items()}
|
||||
if mine != theirs:
|
||||
for k in set(mine or {}) | set(theirs):
|
||||
if (mine or {}).get(k) != theirs.get(k):
|
||||
diffs.append(f"{f.name}:{k}\n mine ={(mine or {}).get(k)!r}\n pyyaml={theirs.get(k)!r}")
|
||||
assert not diffs, f"{len(diffs)} field(s) differ from PyYAML:\n" + "\n".join(diffs[:5])
|
||||
faults, ran = dv.pyyaml_frontmatter_faults(files)
|
||||
assert ran, "PyYAML is importable here, so the comparison must have actually run"
|
||||
assert not faults, f"{len(faults)} frontmatter fault(s) vs PyYAML:\n" + "\n".join(faults[:5])
|
||||
|
||||
@@ -1002,39 +1002,423 @@ def test_budget_total_excludes_the_generated_catalog(tmp_path, monkeypatch):
|
||||
assert total < 100, f"the 500-line generated catalog leaked into the total ({total})"
|
||||
|
||||
|
||||
def test_real_corpus_ceiling_sits_at_the_TAIL_BOUNDARY_of_the_distribution():
|
||||
"""Guards the calibration claim. This is the FOURTH version; the failures are the lesson.
|
||||
def test_real_corpus_ceiling_flags_a_nonempty_proper_minority():
|
||||
"""Guards the calibration claim. This is the FIFTH version; the failures are the lesson.
|
||||
|
||||
v1 `max(under) <= 60 < min(over)` — true by construction of those two lists.
|
||||
v2 a minimum gap WIDTH — but a ceiling of 200 also sits in a wide gap, so it passed.
|
||||
v3 a 2-12% fraction band plus "clear air" measured against `min(over)` — the nearest
|
||||
record ABOVE the ceiling. That made the test a hostage to an unrelated record: one
|
||||
ordinary 62-line addition reddened it with the ceiling correctly placed, and the only
|
||||
remedy the assertion admitted was to RAISE the ceiling. That is the ratchet this whole
|
||||
change abolishes, reinstated as a hard failure in what #631 makes a blocking CI job.
|
||||
The fraction band had the same coupling more slowly (12 more long records breached it),
|
||||
and `0 <= headroom` was vacuous — `max(under)` is by construction <= ceiling.
|
||||
remedy the assertion admitted was to RAISE the ceiling.
|
||||
v4 `p90 <= ceiling <= p95`. Scale-free and correct AS A DEFINITION, but an order statistic
|
||||
over a SPARSE distribution is a STEP function. The lengths climb to the ceiling and then
|
||||
jump STRAIGHT to 81 with nothing between, so ONE new record can move p90 by 21 lines and
|
||||
reddened the BLOCKING `script-tests` job for whoever happened to write it. It reproduced
|
||||
twice live (#672, #706) and both times the only in-scope remedy was to trim the new
|
||||
record to fit the constant — the ratchet pointed at record authors, which is precisely
|
||||
what the v3 note says this whole design abolishes.
|
||||
|
||||
v4 states the property directly and scale-free: **the ceiling marks the start of the tail**,
|
||||
i.e. it sits between the 90th and 95th percentile of record lengths. Percentiles move WITH the
|
||||
corpus, so routine growth cannot ratchet this; it fires only when the ceiling genuinely stops
|
||||
marking the tail boundary, which is exactly when it should be re-derived.
|
||||
v5 SPLITS the claim by robustness instead of hunting for a better single assertion:
|
||||
|
||||
* the COARSE property — the ceiling flags a meaningful minority — is asserted HERE,
|
||||
blocking. One record moves a fraction by at most 1/N, so no SINGLE ordinary addition can
|
||||
cross it — measured headroom, not immunity (38 over-ceiling additions, 718 short ones, or
|
||||
consolidating 15 of the 18 offenders would each reach a bound).
|
||||
* the FINE property — `p90 <= ceiling <= p95` — is now REPORTED by `main()` as a notice.
|
||||
It is real signal about the CONSTANT drifting out of date, which is the passage of corpus
|
||||
growth rather than a defect in the commit under test. That is the same reasoning
|
||||
`stale_records` is built on, and it gets the same treatment.
|
||||
|
||||
Note what did NOT change: the ceiling is still 60, and the fine claim is still measured on
|
||||
every run. v5 moves where each claim is enforced, it does not stop making them.
|
||||
"""
|
||||
recs = [r for r in dl.all_active_records() if r.key]
|
||||
assert len(recs) > 100, f"corpus looks empty ({len(recs)}) — this check would be vacuous"
|
||||
# A low floor on purpose: this guards against a VACUOUS scan, not against corpus shrinkage.
|
||||
# At >100 it would red after ~83 legitimate retirements even with the ceiling still calibrated.
|
||||
assert len(recs) > 20, f"corpus looks empty ({len(recs)}) — this check would be vacuous"
|
||||
|
||||
ceiling = dv.RECORD_CEILING_DEFAULT # the value the CLI actually uses; cannot drift from here
|
||||
lengths = sorted(dv.record_prose_lines(r) for r in recs)
|
||||
p90 = lengths[int(len(lengths) * 0.90)]
|
||||
p95 = lengths[int(len(lengths) * 0.95)]
|
||||
ceiling = dv.RECORD_CEILING_DEFAULT # the value the CLI actually uses; cannot drift from here
|
||||
cal = dv.ceiling_calibration(recs, ceiling)
|
||||
|
||||
assert p90 <= ceiling <= p95, (
|
||||
f"the ceiling ({ceiling}) no longer marks the tail boundary: p90={p90}, p95={p95}. "
|
||||
f"Below p90 it cuts into the bulk and every author will learn to ignore it; above p95 it is "
|
||||
f"parked among the outliers and signals nothing. Re-derive it from the distribution."
|
||||
assert cal.flags_minority, (
|
||||
f"the ceiling ({ceiling}) no longer flags a nonempty proper minority of records: "
|
||||
f"{cal.n_over}/{cal.n} = {cal.fraction_over:.1%} are over it. At 0% it names nobody and "
|
||||
f"signals nothing; above {dv.CEILING_MINORITY_MAX:.0%} it is cutting into the bulk of the "
|
||||
f"corpus rather than marking its tail. Re-derive it from the distribution."
|
||||
)
|
||||
|
||||
|
||||
def test_ceiling_calibration_detects_drift_in_BOTH_directions():
|
||||
"""The fine claim is asserted here, on a distribution the test OWNS.
|
||||
|
||||
This is the point of the v5 split: the property is still pinned, but against synthetic data
|
||||
instead of the live corpus, so it cannot be reddened by someone else's record landing.
|
||||
"""
|
||||
# 100 records: 95 of 20 lines, 5 of 200. Index 90 lands in the short block and index 95 in the
|
||||
# long one, so p90 == 20 and p95 == 200 — a wide, unambiguous tail boundary to aim at.
|
||||
recs = [_rec_body(f"a.s{i}", 20) for i in range(95)] + [_rec_body(f"a.l{i}", 200) for i in range(5)]
|
||||
assert [dv.ceiling_calibration(recs, 60).p90, dv.ceiling_calibration(recs, 60).p95] == [20, 200]
|
||||
|
||||
assert dv.ceiling_calibration(recs, 60).marks_tail, "60 sits between p90=20 and p95=200"
|
||||
assert not dv.ceiling_calibration(recs, 10).marks_tail, "below p90 it cuts into the bulk"
|
||||
assert not dv.ceiling_calibration(recs, 999).marks_tail, "above p95 it is parked among outliers"
|
||||
|
||||
# BOTH ends of `marks_tail` are inclusive. Review found the upper one unpinned — `ceiling <= p95`
|
||||
# mutated to `<` survived the whole suite. It is notice-only rather than blocking, but an
|
||||
# unpinned boundary is how a documented claim quietly stops being true.
|
||||
assert dv.ceiling_calibration(recs, 20).marks_tail, "p90 itself must satisfy the lower bound"
|
||||
assert dv.ceiling_calibration(recs, 200).marks_tail, "p95 itself must satisfy the upper bound"
|
||||
assert not dv.ceiling_calibration(recs, 201).marks_tail, "one line above p95 must not"
|
||||
|
||||
# and the coarse property separates the same two failure modes
|
||||
assert not dv.ceiling_calibration(recs, 999).flags_minority, "a ceiling nobody is over signals nothing"
|
||||
assert not dv.ceiling_calibration(recs, 10).flags_minority, "100% over the ceiling is not a tail"
|
||||
assert dv.ceiling_calibration(recs, 60).flags_minority
|
||||
|
||||
|
||||
def test_the_coarse_bound_REJECTS_a_badly_placed_ceiling():
|
||||
"""The blocking property must have teeth.
|
||||
|
||||
Review's strongest finding on the first draft: a floor of `fraction_over > 0` was nearly
|
||||
unfalsifiable — measured on the live corpus it accepted every ceiling from 39 to 229, including
|
||||
the ceiling of 200 the docstring itself offered as the case it catches, because one 230-line
|
||||
record keeps the count nonzero. A FRACTION floor is what restores the teeth.
|
||||
|
||||
The rejections are pinned on a SYNTHETIC distribution: asserting that a specific absurd ceiling
|
||||
stays rejected by the live corpus is itself growth-coupled (three new 200+ line records flip the
|
||||
200 arm). Only the acceptance of today's ceiling is checked against live data.
|
||||
"""
|
||||
# The TEETH are demonstrated on an owned distribution, for the reason in
|
||||
# `test_v4_would_have_reddened_where_v5_holds`: an assertion that a specific absurd ceiling is
|
||||
# rejected by the LIVE corpus is itself growth-coupled (review found that three new 200+ line
|
||||
# records would flip the 200 arm). 100 records of 30 lines and one of 230 — an outlier-only
|
||||
# tail, which is precisely the shape a badly-placed ceiling fails to distinguish.
|
||||
synthetic = [_rec_body(f"a.s{i}", 30) for i in range(100)] + [_rec_body("a.outlier", 230)]
|
||||
|
||||
for bad in (200, 229, 230):
|
||||
cal = dv.ceiling_calibration(synthetic, bad)
|
||||
assert not cal.flags_minority, (
|
||||
f"a ceiling of {bad} flags only {cal.n_over}/{cal.n} records and must be rejected, got {cal}"
|
||||
)
|
||||
assert not dv.ceiling_calibration(synthetic, 10).flags_minority, "a ceiling of 10 cuts into the bulk"
|
||||
|
||||
# The only claim made against the LIVE corpus is the robust one: today's ceiling is accepted.
|
||||
# Reaching a bound takes 38 consecutive over-ceiling additions, 718 short ones by dilution, or
|
||||
# consolidating 15 of the 18 offenders — the tightest arm, and the one worth remembering.
|
||||
recs = [r for r in dl.all_active_records() if r.key]
|
||||
assert len(recs) > 20, "corpus looks empty — this check would be vacuous"
|
||||
assert dv.ceiling_calibration(recs, dv.RECORD_CEILING_DEFAULT).flags_minority
|
||||
|
||||
|
||||
def test_ceiling_calibration_is_empty_safe():
|
||||
"""A vacuous corpus must report both claims FALSE, never a passing default."""
|
||||
cal = dv.ceiling_calibration([], 60)
|
||||
assert cal.n == 0 and not cal.marks_tail and not cal.flags_minority
|
||||
|
||||
|
||||
def test_the_minority_band_BOUNDARIES_are_exactly_where_documented():
|
||||
"""Pins both constants AND both inclusivities, which review found entirely unmutated.
|
||||
|
||||
Mutating `0.02 -> 0.03`, `0.25 -> 0.30`, or either `<=` to `<` passed all eight calibration
|
||||
tests. These are not free parameters — they ARE the documented CI-red thresholds, so a silent
|
||||
shift changes them (a strict cap reds after 37 long additions instead of 38; a strict floor
|
||||
after 717 short ones instead of 718), quietly falsifying the numbers in `docs.corpus-size-signal`
|
||||
and `docs/ci-cd.md`.
|
||||
|
||||
100-record fixtures make the fraction exact and readable: k over the ceiling IS k%. Both
|
||||
`2/100` and `25/100` are exactly representable and compare equal to the module constants, so
|
||||
these are true boundary cases rather than near-misses.
|
||||
"""
|
||||
|
||||
def corpus(n_over: int, total: int = 100):
|
||||
return [_rec_body(f"a.o{i}", 61) for i in range(n_over)] + [
|
||||
_rec_body(f"b.u{i}", 10) for i in range(total - n_over)
|
||||
]
|
||||
|
||||
# The bounds are INCLUSIVE — exactly on either edge still passes.
|
||||
assert dv.ceiling_calibration(corpus(2), 60).flags_minority, "the 2% floor must be inclusive"
|
||||
assert dv.ceiling_calibration(corpus(25), 60).flags_minority, "the 25% cap must be inclusive"
|
||||
|
||||
# ...and one record beyond either edge does not.
|
||||
assert not dv.ceiling_calibration(corpus(1), 60).flags_minority, "1% is below the floor"
|
||||
assert not dv.ceiling_calibration(corpus(26), 60).flags_minority, "26% is above the cap"
|
||||
|
||||
# The constants themselves, so a change has to be deliberate and visible in the diff.
|
||||
assert (dv.CEILING_MINORITY_MIN, dv.CEILING_MINORITY_MAX) == (0.02, 0.25)
|
||||
|
||||
|
||||
# `test_adding_ordinary_records_cannot_RED_the_blocking_property` used to live here. It appended two
|
||||
# long synthetic records to the LIVE corpus and asserted `flags_minority` on the result — which
|
||||
# crosses the 25% cap TWO records before the production bound does (56/221 vs 54/219), making the
|
||||
# test named "cannot RED the blocking property" a tighter tripwire than the property it guarded.
|
||||
# That is the #688 defect in miniature, and the fourth instance found in this change.
|
||||
#
|
||||
# Deleted rather than tuned, because both of its jobs are covered without touching live data:
|
||||
# `test_v4_would_have_reddened_where_v5_holds` demonstrates the v4/v5 contrast on an owned
|
||||
# distribution, and `test_real_corpus_ceiling_flags_a_nonempty_proper_minority` is the deliberate
|
||||
# live guard — at the production threshold rather than two records inside it.
|
||||
|
||||
|
||||
def test_ceiling_calibration_IGNORES_keyless_records_and_counts_the_rest():
|
||||
"""`n` and the `if r.key` filter, both of which review found unpinned.
|
||||
|
||||
`main()` passes the UNFILTERED record list, so the filter is load-bearing in production while
|
||||
every live-corpus test hands this function a pre-filtered list — the oracle and production's
|
||||
input agreed only by accident. The corpus really does carry keyless entries (the generated
|
||||
"Records formerly in this file" scaffolding, one of them 106 lines), and counting them would
|
||||
drag p90/p95 around with content that is not a record.
|
||||
|
||||
`n` itself lost its only pin when the over-tight live test was deleted: a mutation returning
|
||||
`n=1` passed everything, which would print a wrong denominator in the drift notice.
|
||||
|
||||
The oracle is DYNAMIC and runs at two distinct cardinalities on purpose. The first attempt
|
||||
asserted `n == 10` against a ten-record fixture, and review killed it: a mutation returning a
|
||||
constant 10 for every input satisfied it while changing the live denominator from 183 to 10 —
|
||||
preserving the exact production defect the test claims to close. A single hardcoded count
|
||||
cannot distinguish "counts the input" from "returns this number".
|
||||
"""
|
||||
for size in (7, 13):
|
||||
recs = [_rec_body(f"a.s{i}", 10) for i in range(size)]
|
||||
assert dv.ceiling_calibration(recs, 60).n == size, f"n must count the {size} keyed records given"
|
||||
|
||||
recs = [_rec_body(f"a.s{i}", 10) for i in range(9)] + [_rec_body("b.long", 500)]
|
||||
keyless = _rec(key=None, heading="Records formerly in this file", body="\n".join("x" for _ in range(500)))
|
||||
assert dv.ceiling_calibration(recs + [keyless], 60) == dv.ceiling_calibration(recs, 60)
|
||||
|
||||
|
||||
def test_ceiling_calibration_counts_over_the_ceiling_EXCLUSIVELY():
|
||||
"""`n_over` is recomputed inside `ceiling_calibration`, so its boundary needs its own pin.
|
||||
|
||||
`oversized_records` has an exclusivity test; this counter does not share its code. Flipping
|
||||
`>` to `>=` here would silently shift the fraction by the number of records sitting exactly ON
|
||||
the ceiling (3 in the live corpus), and the mutation survived the whole suite.
|
||||
"""
|
||||
recs = [_rec_body("a.under", 59), _rec_body("b.exact", 60), _rec_body("c.over", 61)]
|
||||
assert dv.ceiling_calibration(recs, 60).n_over == 1
|
||||
|
||||
|
||||
def test_ceiling_calibration_uses_the_95th_percentile_not_a_higher_one():
|
||||
"""Pins p95's quantile. The synthetic 95/5 fixture cannot tell 0.95 from 0.99, so a mutation
|
||||
widening the upper quantile survived the whole suite."""
|
||||
# 100 records: indices 0..89 = 10, 90..94 = 50, 95..98 = 90, 99 = 900.
|
||||
recs = (
|
||||
[_rec_body(f"a.s{i}", 10) for i in range(90)]
|
||||
+ [_rec_body(f"b.m{i}", 50) for i in range(5)]
|
||||
+ [_rec_body(f"c.h{i}", 90) for i in range(4)]
|
||||
+ [_rec_body("d.max", 900)]
|
||||
)
|
||||
cal = dv.ceiling_calibration(recs, 60)
|
||||
assert (cal.p90, cal.p95) == (50, 90), f"p95 must read index 95, not a higher quantile: {cal}"
|
||||
|
||||
|
||||
def test_v4_would_have_reddened_where_v5_holds():
|
||||
"""The v4-vs-v5 contrast, on a distribution the test OWNS rather than the live corpus.
|
||||
|
||||
THIRD TIME for this defect class in one change, which is why the fix is to remove the coupling
|
||||
rather than patch the instance. Round 1 of review caught it in the drift test; round 2 caught it
|
||||
here, in what looked like a safe `if before.marks_tail:` guard — the GUARD was conditional but
|
||||
the CONCLUSION was still an assertion about live order statistics, and appending 16 ordinary
|
||||
30-line records (nothing long, nothing unusual) makes `after.marks_tail` true again and fires it:
|
||||
|
||||
extra= 0 before(marks=True) after(marks=False) -> reds: False
|
||||
extra=16 before(marks=True) after(marks=True) -> reds: True
|
||||
|
||||
Nothing about this demonstration needs the real corpus. The synthetic base reproduces the shape
|
||||
that matters — a sparse gap immediately above the ceiling, which is what #688 measured on
|
||||
`main` (nothing at all between 60 and 81) — so two over-ceiling additions advance p90 off the
|
||||
ceiling and break v4, while v5 is untouched.
|
||||
"""
|
||||
base = (
|
||||
[_rec_body(f"a.s{i}", 30) for i in range(90)] # the bulk
|
||||
+ [_rec_body("a.edge", 60)] # sits exactly ON the ceiling, as main does today
|
||||
+ [_rec_body(f"a.l{i}", 112) for i in range(9)] # the tail, across a sparse gap
|
||||
)
|
||||
before = dv.ceiling_calibration(base, 60)
|
||||
assert (before.p90, before.p95) == (60, 112), before
|
||||
assert before.marks_tail and before.flags_minority, before
|
||||
|
||||
after = dv.ceiling_calibration(base + [_rec_body("new.a", 107), _rec_body("new.b", 107)], 60)
|
||||
|
||||
assert not after.marks_tail, f"v4 must break on these additions, or the contrast is empty: {after}"
|
||||
assert after.flags_minority, f"v5 must survive what broke v4: {after}"
|
||||
|
||||
|
||||
# --- #674: the validator cross-checks its own parse against PyYAML ------------------------------
|
||||
|
||||
|
||||
_HAZARDS = {
|
||||
# PyYAML REJECTS: the bare apostrophe closes the single-quoted scalar early.
|
||||
"apostrophe": "rule: 'SQLite's LOWER() folds ASCII only'",
|
||||
# PyYAML ACCEPTS but reads a DIFFERENT value: ` #` starts a comment, truncating the rule.
|
||||
"unquoted-hash": "rule: use --flag #2 for this",
|
||||
}
|
||||
|
||||
|
||||
def _wing_with(tmp_path: Path, frontmatter_line: str) -> tuple[Path, Path]:
|
||||
"""A record wing containing one file whose frontmatter carries `frontmatter_line`."""
|
||||
records = tmp_path / "records" / "ci"
|
||||
records.mkdir(parents=True)
|
||||
(records / "a.md").write_text(
|
||||
"---\n"
|
||||
"key: ci.a\n"
|
||||
"title: 'T'\n"
|
||||
"status: active\n"
|
||||
"since: '2026-01-01'\n"
|
||||
"supersedes: none\n"
|
||||
"superseded-by: none\n"
|
||||
f"{frontmatter_line}\n"
|
||||
"signals: 's'\n"
|
||||
"---\n\nprose.\n"
|
||||
)
|
||||
archive = tmp_path / "archive"
|
||||
archive.mkdir(parents=True)
|
||||
return records, archive
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hazard", sorted(_HAZARDS))
|
||||
def test_pyyaml_crosscheck_catches_frontmatter_the_hand_reader_accepts(tmp_path, hazard):
|
||||
"""#674, both shapes. Hit twice in one session by two independent agents (#578, #651)."""
|
||||
pytest.importorskip("yaml")
|
||||
records, _ = _wing_with(tmp_path, _HAZARDS[hazard])
|
||||
|
||||
faults, ran = dv.pyyaml_frontmatter_faults(sorted(records.rglob("*.md")))
|
||||
assert ran, "PyYAML is installed here, so the cross-check must have run"
|
||||
assert faults, f"the {hazard} hazard slipped through the cross-check"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hazard", sorted(_HAZARDS))
|
||||
def test_the_hand_reader_really_IS_blind_to_these(tmp_path, hazard):
|
||||
"""The positive control: pin the MECHANISM, so this suite cannot pass for the wrong reason.
|
||||
|
||||
If `record_wing_faults` ever started catching these on its own, the cross-check above could be
|
||||
deleted and the tests would stay green while the guard vanished. Asserting that the pre-#674
|
||||
machinery reports these files as CLEAN is what makes the cross-check's red meaningful — and it
|
||||
is the exact state #674 was filed about: `decisions_validate.py` printed OK on input the
|
||||
writer's own library rejects.
|
||||
"""
|
||||
records, archive = _wing_with(tmp_path, _HAZARDS[hazard])
|
||||
assert dv.record_wing_faults(records, archive) == [], (
|
||||
"the structural guard now catches this by itself — re-derive whether the PyYAML "
|
||||
"cross-check is still the thing closing this gap"
|
||||
)
|
||||
|
||||
|
||||
def test_crosscheck_REPORTS_an_impossible_date_instead_of_crashing(tmp_path):
|
||||
"""PyYAML raises a bare `ValueError`, not a `YAMLError`, for a well-shaped impossible date.
|
||||
|
||||
`stale-after: 2026-06-31` (June has 30 days) escaped an `except yaml.YAMLError` as a traceback,
|
||||
killing the validator on any machine with PyYAML — including the Husky pre-commit hook. A check
|
||||
documented as "strictly additive, must never be the reason the validator cannot run" must
|
||||
REPORT this, so the except is deliberately broad.
|
||||
"""
|
||||
pytest.importorskip("yaml")
|
||||
records, _ = _wing_with(tmp_path, "stale-after: 2026-06-31")
|
||||
|
||||
faults, ran = dv.pyyaml_frontmatter_faults(sorted(records.rglob("*.md")))
|
||||
assert ran
|
||||
assert faults and "REJECTS" in faults[0], faults
|
||||
assert "ValueError" in faults[0], f"the fault must name the real exception: {faults[0]}"
|
||||
|
||||
|
||||
def test_crosscheck_survives_a_TYPED_mapping_key(tmp_path):
|
||||
"""PyYAML returns typed keys, so a stray `1: x` yields int 1 where the reader yields "1".
|
||||
|
||||
Sorting that mixed set raised `TypeError` — an uncaught traceback replacing what
|
||||
`_unknown_frontmatter_keys` used to report as an actionable error. Removing the `key=str` sort
|
||||
key restores the crash, and without this test every other test here stays green.
|
||||
"""
|
||||
pytest.importorskip("yaml")
|
||||
records, _ = _wing_with(tmp_path, "1: stray")
|
||||
|
||||
faults, ran = dv.pyyaml_frontmatter_faults(sorted(records.rglob("*.md")))
|
||||
assert ran
|
||||
assert faults, "a typed mapping key must be reported, not swallowed"
|
||||
assert any("1" in f for f in faults), faults
|
||||
|
||||
|
||||
def test_pyyaml_crosscheck_is_clean_on_the_REAL_corpus():
|
||||
"""No false positives. A cross-check that flags correct records would be reverted within a day."""
|
||||
pytest.importorskip("yaml")
|
||||
files = dv.record_wing_files()
|
||||
assert len(files) > 100, f"only {len(files)} wing files found — this check would be near-vacuous"
|
||||
|
||||
faults, ran = dv.pyyaml_frontmatter_faults(files)
|
||||
assert ran
|
||||
assert faults == [], "the cross-check disagrees with the live corpus:\n" + "\n".join(faults[:5])
|
||||
|
||||
|
||||
def test_crosscheck_skips_cleanly_when_pyyaml_is_absent(tmp_path, monkeypatch):
|
||||
"""The read path stays dependency-free (#674's second Done-when box).
|
||||
|
||||
`decisions-guard`, the Husky hooks and every contributor machine install nothing, so an absent
|
||||
PyYAML must SKIP the cross-check rather than fault or crash — while every other check runs.
|
||||
"""
|
||||
records, _ = _wing_with(tmp_path, _HAZARDS["apostrophe"])
|
||||
|
||||
import builtins
|
||||
|
||||
real_import = builtins.__import__
|
||||
|
||||
def no_yaml(name, *a, **kw):
|
||||
if name == "yaml":
|
||||
raise ImportError("no yaml here")
|
||||
return real_import(name, *a, **kw)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", no_yaml)
|
||||
|
||||
faults, ran = dv.pyyaml_frontmatter_faults(sorted(records.rglob("*.md")))
|
||||
assert ran is False, "an absent PyYAML must report that the check did not run"
|
||||
assert faults == [], "a skipped check must not manufacture faults"
|
||||
|
||||
|
||||
def test_main_ANNOUNCES_a_skipped_crosscheck(capsys, monkeypatch):
|
||||
"""A skipped check that says nothing is the '#603 stale-after' defect: reports success, does
|
||||
nothing. The skip is legitimate; staying quiet about it is not."""
|
||||
monkeypatch.setattr(dv, "pyyaml_frontmatter_faults", lambda files: ([], False))
|
||||
assert dv.main([]) == 0
|
||||
err = capsys.readouterr().err
|
||||
assert "cross-check" in err and "SKIPPED" in err, err
|
||||
|
||||
|
||||
def test_main_FEEDS_the_crosscheck_the_REAL_wing_files(monkeypatch, capsys):
|
||||
"""Pins the cross-check's INPUT, not just that its output is consumed.
|
||||
|
||||
Mutation testing found this hole: replacing `pyyaml_frontmatter_faults(record_wing_files())`
|
||||
with `pyyaml_frontmatter_faults([])` in main() left the ENTIRE suite green — exit 0, no skip
|
||||
notice, every other test passing. The two wiring tests monkeypatch the function itself, so they
|
||||
prove the return value reaches `errs`; nothing proved the argument was the corpus. That is the
|
||||
'#609 marker that printed OK while doing nothing' defect one level up, which is the exact thing
|
||||
this record indicts — and the sibling of `test_main_actually_CALLS_the_wing_scan`.
|
||||
"""
|
||||
seen: list[list] = []
|
||||
|
||||
def spy(files):
|
||||
seen.append(list(files))
|
||||
return [], True
|
||||
|
||||
monkeypatch.setattr(dv, "pyyaml_frontmatter_faults", spy)
|
||||
dv.main([])
|
||||
|
||||
assert seen, "main() never called the cross-check at all"
|
||||
assert len(seen[0]) > 100, f"main() passed only {len(seen[0])} file(s) — not the real wings"
|
||||
assert set(seen[0]) == set(dv.record_wing_files()), (
|
||||
"main() passed a file list that is not record_wing_files() — the cross-check is not seeing "
|
||||
"the corpus it is supposed to check"
|
||||
)
|
||||
|
||||
|
||||
def test_main_FAILS_when_the_crosscheck_reports_a_fault(capsys, monkeypatch):
|
||||
"""Wiring test: the faults must reach the exit code, not just be computed.
|
||||
|
||||
Without this, `wing_faults=record_wing_faults() + yaml_faults` could drop the second term and
|
||||
every other test here would still pass.
|
||||
"""
|
||||
monkeypatch.setattr(dv, "pyyaml_frontmatter_faults", lambda files: (["x.md: bogus fault"], True))
|
||||
assert dv.main([]) == 1
|
||||
assert "bogus fault" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_retired_budget_flag_says_it_is_ignored(capsys):
|
||||
"""A retired flag must announce itself, not no-op silently.
|
||||
|
||||
@@ -1046,24 +1430,104 @@ def test_retired_budget_flag_says_it_is_ignored(capsys):
|
||||
|
||||
|
||||
def test_no_budget_flag_means_no_retirement_warning(capsys):
|
||||
# Match the retirement notice specifically, not a bare "RETIRED": a legitimate record whose
|
||||
# TITLE contains that word and whose `stale-after` has passed gets printed by the stale notice,
|
||||
# which would red this on an unrelated corpus change.
|
||||
dv.main([])
|
||||
assert "RETIRED" not in capsys.readouterr().err
|
||||
assert "is RETIRED and was IGNORED" not in capsys.readouterr().err
|
||||
|
||||
|
||||
|
||||
def test_main_reports_ceiling_drift_as_a_NOTICE_and_still_exits_0(capsys):
|
||||
"""The fine claim's live wiring (#688): the drift notice must fire, and must NOT turn the run
|
||||
red — the entire point of the v5 split.
|
||||
|
||||
The ceiling is DERIVED as one line above the longest record, so it is off the tail boundary by
|
||||
definition. A hardcoded 999 looked safe and was not: review showed ten valid 1000-line records
|
||||
would put p95 at 1000, making 999 calibrated — so the notice would stop firing and this test
|
||||
would go RED, for a corpus change that is nobody's defect.
|
||||
"""
|
||||
longest = max(dv.record_prose_lines(r) for r in dl.all_active_records() if r.key)
|
||||
assert dv.main(["--record-ceiling", str(longest + 1)]) == 0
|
||||
err = capsys.readouterr().err
|
||||
drift = [ln for ln in err.splitlines() if "drifted from the tail boundary" in ln]
|
||||
assert len(drift) == 1, err
|
||||
assert drift[0].startswith("::notice::"), f"drift must be a notice, not a warning: {drift[0]}"
|
||||
|
||||
|
||||
def test_main_reports_drift_IFF_the_ceiling_is_off_the_tail_boundary(capsys):
|
||||
"""The complement of the test above — asserting the WIRING, not the corpus's current state.
|
||||
|
||||
The obvious way to write this is `dv.main([]); assert "drifted" not in err`, and that is a trap
|
||||
review caught: `main()` emits the notice exactly when `p90 <= 60 <= p95` is false over the LIVE
|
||||
corpus, so such a test fails under precisely the condition #688 exists to stop failing — it
|
||||
would move v4's assertion three functions down and leave it in the same blocking job. Today p90
|
||||
sits exactly ON the ceiling, so ONE new over-ceiling record would have reddened it.
|
||||
|
||||
So the oracle is `ceiling_calibration` itself: whatever the corpus currently looks like, the
|
||||
notice must be present iff the fine claim is false. The 999 case pins that at least one branch
|
||||
is genuinely exercised, so this cannot pass by never firing.
|
||||
"""
|
||||
recs = [r for r in dl.all_active_records() if r.key]
|
||||
# Non-empty is all the derivations below need; a higher floor would itself be a growth tripwire.
|
||||
assert recs, "corpus is empty — the derived ceilings need at least one record"
|
||||
lengths = sorted(dv.record_prose_lines(r) for r in recs)
|
||||
|
||||
# Both ceilings are DERIVED so each branch is guaranteed by construction, not by luck. Review
|
||||
# caught the earlier version relying on the live 60/999 pair: once one 61-line record lands,
|
||||
# BOTH of those drift, and an UNCONDITIONAL notice would have passed the test.
|
||||
# * p90 itself is always calibrated — `p90 <= p90 <= p95` holds for any distribution.
|
||||
# * one line above the longest record is always off the tail — it exceeds p95 by definition.
|
||||
quiet_ceiling = lengths[min(int(len(lengths) * 0.90), len(lengths) - 1)]
|
||||
drift_ceiling = lengths[-1] + 1
|
||||
|
||||
expectations = []
|
||||
for ceiling in (quiet_ceiling, drift_ceiling):
|
||||
expected = not dv.ceiling_calibration(recs, ceiling).marks_tail
|
||||
dv.main(["--record-ceiling", str(ceiling)])
|
||||
err = capsys.readouterr().err
|
||||
assert ("drifted from the tail boundary" in err) is expected, (
|
||||
f"ceiling {ceiling}: expected drift notice={expected}, got the opposite"
|
||||
)
|
||||
expectations.append(expected)
|
||||
|
||||
assert expectations == [False, True], (
|
||||
f"the two derived ceilings must exercise BOTH branches, got {expectations} — otherwise an "
|
||||
f"unconditional notice (or none at all) would pass this test"
|
||||
)
|
||||
|
||||
|
||||
def test_main_actually_REPORTS_the_ceiling_and_the_trend(capsys):
|
||||
"""The new signal's live wiring was untested: `if oversized:` -> `if False:`, or bumping the
|
||||
default ceiling to 999999, left every test green while main() reported nothing. Only the pure
|
||||
function `oversized_records()` was covered — so the replacement signal could silently do
|
||||
nothing, which is the exact defect this change exists to retire."""
|
||||
nothing, which is the exact defect this change exists to retire.
|
||||
|
||||
Stated as an IFF against the live offender list rather than `assert over` (#688): the ceiling
|
||||
is ALLOWED to go green — `test_oversized_records_can_go_green` says so explicitly — so a bare
|
||||
precondition that the corpus still has an offender would red the blocking job the day someone
|
||||
consolidates the last one, punishing exactly the work the warning asks for."""
|
||||
dv.main([])
|
||||
err = capsys.readouterr().err
|
||||
assert "prose lines across" in err, "the aggregate trend notice must always print"
|
||||
assert "exceed the" in err and "prose ceiling" in err, "the per-record ceiling warning must print"
|
||||
|
||||
over = [r.key for r in dl.all_active_records()
|
||||
if r.key and dv.record_prose_lines(r) > dv.RECORD_CEILING_DEFAULT]
|
||||
assert over, "precondition: the live corpus has at least one over-ceiling record"
|
||||
assert any(k in err for k in over), "the warning must NAME the offending records"
|
||||
warned = "exceed the" in err and "prose ceiling" in err
|
||||
assert warned is bool(over), f"ceiling warning printed={warned} but {len(over)} record(s) are over it"
|
||||
if over:
|
||||
assert any(k in err for k in over), "the warning must NAME the offending records"
|
||||
|
||||
# The IFF above is only non-vacuous while an offender exists: once the corpus is legitimately
|
||||
# consolidated to zero, `False is False` passes even if main()'s whole `if oversized:` branch
|
||||
# were deleted. So force the branch with a ceiling nothing can sit under. It is -1, not 0:
|
||||
# an empty record body is validator-valid and `record_prose_lines` returns 0, so a corpus of
|
||||
# empty-bodied records has no offender at 0. Below zero the arm cannot go vacuous at all.
|
||||
dv.main(["--record-ceiling", "-1"])
|
||||
forced = capsys.readouterr().err
|
||||
assert "exceed the" in forced and "prose ceiling" in forced, (
|
||||
"at a ceiling of -1 every record is over it — the warning branch must fire"
|
||||
)
|
||||
|
||||
|
||||
def test_trend_notice_reports_record_prose_and_scaffolding_separately(capsys):
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
"""Tests for `scripts/jq-preflight.sh` — the jq version contract (ersatztv#648).
|
||||
|
||||
The axis this guards. Every shell gate in this repo is authored on a Mac shipping jq 1.8.x; the CI
|
||||
runner ships jq 1.6. Nothing pinned or checked that, and three independent divergences surfaced in a
|
||||
single day — `jq -e` over empty input (exit 4 vs 0), `contains("<NUL>")` (false vs true for every
|
||||
string), and the parse-error exit code (5 vs 4, colliding with "no output"). Each was patched with a
|
||||
version-stable construct, but patching constructs one at a time leaves the AXIS untested.
|
||||
|
||||
These tests shim `jq` on PATH with a fake reporting an arbitrary version, so the preflight's own
|
||||
behaviour is verified by MEASUREMENT rather than by observing a green CI tick — ersatztv#648's third
|
||||
Done-when box. Doing it here rather than by pushing a deliberately-red commit also keeps the proof
|
||||
reproducible: it re-runs on every PR instead of living in one CI run's history.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = REPO_ROOT / "scripts" / "jq-preflight.sh"
|
||||
WORKFLOWS = REPO_ROOT / ".gitea" / "workflows"
|
||||
# Resolved BEFORE PATH is narrowed to the shim dir — the tests strip PATH down to just that
|
||||
# directory, so `bash` could not be found by name from inside them.
|
||||
BASH = shutil.which("bash") or "/bin/bash"
|
||||
|
||||
|
||||
def _shq(s):
|
||||
"""Single-quote a string for /bin/sh."""
|
||||
return "'" + s.replace("'", "'\\''") + "'"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def preflight(tmp_path):
|
||||
bindir = tmp_path / "bin"
|
||||
bindir.mkdir()
|
||||
|
||||
class Handle:
|
||||
def with_jq(self, version_line, stderr="", exit_code=0):
|
||||
"""Install a fake `jq` reporting `version_line` for --version.
|
||||
|
||||
`stderr` and `exit_code` exist because an earlier version of this shim ALWAYS exited 0
|
||||
and never wrote to stderr — so it structurally could not observe the worst failure this
|
||||
script has: a jq that cannot start. The preflight was folding stderr into the parse via
|
||||
`2>&1` and discarding the exit status, so a glibc-mismatch message containing `2.34`
|
||||
parsed as version 2.34 and PASSED the floor. Every case the shim could express was clean,
|
||||
so every test passed.
|
||||
"""
|
||||
shim = bindir / "jq"
|
||||
body = "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then\n"
|
||||
if version_line:
|
||||
body += ' printf "%%s\\n" %s\n' % _shq(version_line)
|
||||
if stderr:
|
||||
body += ' printf "%%s\\n" %s >&2\n' % _shq(stderr)
|
||||
body += " exit %d\nfi\nexit 0\n" % exit_code
|
||||
shim.write_text(body)
|
||||
shim.chmod(0o755)
|
||||
|
||||
def without_jq(self):
|
||||
shim = bindir / "jq"
|
||||
if shim.exists():
|
||||
shim.unlink()
|
||||
|
||||
def run(self, *args):
|
||||
env = dict(os.environ)
|
||||
# PATH contains ONLY the shim dir. An earlier draft appended /usr/bin:/bin "for the
|
||||
# basics" and the missing-jq test passed vacuously against the developer machine's real
|
||||
# /usr/bin/jq — the negative case was never negative. The script needs nothing from PATH
|
||||
# but jq itself (`command -v` is a builtin, and bash is invoked by absolute path), so
|
||||
# there is nothing to keep.
|
||||
env["PATH"] = str(bindir)
|
||||
return subprocess.run([BASH, str(SCRIPT), *args],
|
||||
env=env, capture_output=True, text=True)
|
||||
|
||||
def run_bytes(self, *args):
|
||||
"""Same, but WITHOUT text mode.
|
||||
|
||||
`text=True` enables universal-newlines translation, which rewrites `\\r` to `\\n` in the
|
||||
captured output — so any assertion about a stray carriage return is unfalsifiable through
|
||||
`run()`. That is not hypothetical: the CR test passed identically with the strip removed
|
||||
until this was noticed, while the mutant demonstrably emits
|
||||
`... = jq-1.6\\r (parsed 1.6; ...)` at the byte level.
|
||||
"""
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = str(bindir)
|
||||
return subprocess.run([BASH, str(SCRIPT), *args],
|
||||
env=env, capture_output=True)
|
||||
|
||||
return Handle()
|
||||
|
||||
|
||||
def test_the_version_is_printed_so_the_job_log_shows_it(preflight):
|
||||
"""ersatztv#648's second Done-when box: the jq version CI actually uses must be OBSERVABLE."""
|
||||
preflight.with_jq("jq-1.6")
|
||||
r = preflight.run()
|
||||
assert r.returncode == 0, r.stderr
|
||||
assert "jq-1.6" in r.stdout
|
||||
|
||||
|
||||
def test_floor_mode_accepts_the_runner_version(preflight):
|
||||
preflight.with_jq("jq-1.6")
|
||||
assert preflight.run().returncode == 0
|
||||
|
||||
|
||||
def test_floor_mode_accepts_a_newer_jq(preflight):
|
||||
"""No upper bound in floor mode — review-verdict.yml writes the REQUIRED merge check, so a jq
|
||||
bump must never be able to deadlock every merge in the repo."""
|
||||
preflight.with_jq("jq-1.8.2")
|
||||
assert preflight.run().returncode == 0
|
||||
|
||||
|
||||
def test_below_the_floor_is_LOUD(preflight):
|
||||
preflight.with_jq("jq-1.5")
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1
|
||||
assert "below the supported floor" in r.stderr.lower()
|
||||
|
||||
|
||||
def test_missing_jq_is_loud(preflight):
|
||||
preflight.without_jq()
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1
|
||||
assert "not on PATH" in r.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line", ["jq-1.6-dirty", "jq-1.6", "jq-1.6.0"])
|
||||
def test_build_suffixes_still_parse_as_1_6(preflight, version_line):
|
||||
"""A packaging suffix must not fail a perfectly ordinary jq closed — that would be a tripwire
|
||||
firing on noise, which is how tripwires get disabled."""
|
||||
preflight.with_jq(version_line)
|
||||
assert preflight.run("--expect", "1.6").returncode == 0, version_line
|
||||
|
||||
|
||||
def test_expect_mismatch_is_LOUD(preflight):
|
||||
"""THE TRIPWIRE. scripts/tests exercises the jq 1.6 path only because the runner ships 1.6. If
|
||||
the runner were upgraded that coverage would vanish silently, so the pin must go red instead."""
|
||||
preflight.with_jq("jq-1.7.1")
|
||||
r = preflight.run("--expect", "1.6")
|
||||
assert r.returncode == 1
|
||||
assert "expected jq 1.6, found 1.7" in r.stderr
|
||||
|
||||
|
||||
def test_expect_match_passes(preflight):
|
||||
preflight.with_jq("jq-1.6")
|
||||
assert preflight.run("--expect", "1.6").returncode == 0
|
||||
|
||||
|
||||
def test_unknown_argument_is_a_usage_error(preflight):
|
||||
preflight.with_jq("jq-1.6")
|
||||
assert preflight.run("--pin", "1.6").returncode == 2
|
||||
|
||||
|
||||
def test_expect_without_a_value_is_a_usage_error_WITH_output(preflight):
|
||||
"""`shift 2` on a missing value exits 1 under `set -e` with NOTHING on either stream. A CI step
|
||||
that dies with an empty log is the diagnostic hole this script exists to remove."""
|
||||
preflight.with_jq("jq-1.6")
|
||||
r = preflight.run("--expect")
|
||||
assert r.returncode == 2
|
||||
assert "requires a <major.minor> value" in r.stderr
|
||||
|
||||
|
||||
# --- Version parsing: the guard must never assert a floor against an unparsed version ----------
|
||||
#
|
||||
# The original strip-based parse assumed the format is exactly `jq-X.Y`. Anything else left major or
|
||||
# minor EMPTY, and the sanity check concatenated them — so `jq version 1.6` produced "6", which is
|
||||
# non-empty and all-digits, so the check PASSED. The floor comparison then ran `[ "" -lt 1 ]`, which
|
||||
# errors; `set -e` exempts a failing command in an `if` condition, so the conditional read false and
|
||||
# the script exited 0 having asserted NOTHING. That is this script's own stated failure mode,
|
||||
# reproduced inside itself, which is why these cases are pinned rather than left to inspection.
|
||||
|
||||
@pytest.mark.parametrize("version_line", [
|
||||
"jq version 1.6", # some distro wrappers print this form
|
||||
"JQ-1.6",
|
||||
"jq-1.6-dirty",
|
||||
])
|
||||
def test_unusual_but_parseable_version_forms_are_accepted(preflight, version_line):
|
||||
preflight.with_jq(version_line)
|
||||
r = preflight.run()
|
||||
assert r.returncode == 0, f"{version_line!r}: {r.stderr}"
|
||||
assert "parsed 1.6" in r.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line", ["jq-1.-6", "jq-.6", "not-a-version", ""])
|
||||
def test_unparseable_version_fails_CLOSED_rather_than_asserting_nothing(preflight, version_line):
|
||||
preflight.with_jq(version_line)
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1, (
|
||||
f"{version_line!r} exited {r.returncode}: an unparsed version must never reach — or "
|
||||
"silently skip — the floor assertion")
|
||||
assert "could not parse" in r.stderr
|
||||
|
||||
|
||||
def test_a_jq_that_cannot_START_fails_closed(preflight):
|
||||
"""THE case the previous shim could not express, and the guard therefore got wrong.
|
||||
|
||||
A jq broken by a glibc mismatch (the canonical post-base-image-bump failure) exits 127 and writes
|
||||
`... version 'GLIBC_2.34' not found` to STDERR. The preflight was reading `jq --version 2>&1` and
|
||||
discarding the exit status, so that message became the parse input, `2.34` matched, and the floor
|
||||
was certified green on a jq that cannot run at all.
|
||||
"""
|
||||
preflight.with_jq(
|
||||
"", stderr="jq: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found",
|
||||
exit_code=127)
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1
|
||||
assert "cannot run" in r.stderr
|
||||
assert "parsed 2.34" not in r.stdout, "stderr must never be parsed as a version"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line", [
|
||||
"warning: something 3.14", # a noise line carrying a plausible number
|
||||
"2026.07.26 jq-1.6", # a date prefix, which outranks the real version if unanchored
|
||||
"jq-master-v0.0.0-1.6",
|
||||
])
|
||||
def test_a_number_that_is_not_the_VERSION_is_not_accepted_as_one(preflight, version_line):
|
||||
"""Matching the first `<digits>.<digits>` ANYWHERE let a prefix win over the real version.
|
||||
`2026.07.26 jq-1.6` parsed as 2026.07 and sailed over the floor. The pattern is anchored to the
|
||||
leading `jq` token, so these fail closed instead."""
|
||||
preflight.with_jq(version_line)
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1, f"{version_line!r} was accepted as a version"
|
||||
assert "could not parse" in r.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line", [
|
||||
"jq-99999999999999999999999.0",
|
||||
"jq-1.99999999999999999999999",
|
||||
])
|
||||
def test_an_OUT_OF_RANGE_digit_run_fails_closed(preflight, version_line):
|
||||
"""The round-1 fail-open mechanism, resurrected via an over-long number.
|
||||
|
||||
A regex that guarantees *digits* does not guarantee they fit `test`'s integer range. With a
|
||||
23-digit major, `[ "$major" -lt "$min_major" ]` errors with "integer expression expected" — and
|
||||
`set -e` exempts a failing command in an `if` condition, so the conditional read false and THE
|
||||
FLOOR WAS NEVER ASSERTED, exit 0. Identical in shape to the empty-string case that started this.
|
||||
|
||||
Bounding the run with `{1,9}` alone was NOT enough either: the pattern is unanchored at the end,
|
||||
so an over-long minor just matched its first 9 digits and compared that instead — a mis-parse
|
||||
that passes. The trailing non-digit requirement is what actually closes it.
|
||||
"""
|
||||
preflight.with_jq(version_line)
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1, f"{version_line!r} exited 0 — the floor was not asserted"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line", [
|
||||
# Killed by the SEPARATOR restriction (a blank separator must be followed by `version`).
|
||||
"jq\n2.34: cannot load shared library",
|
||||
"jq\n\n\n99.9",
|
||||
"jq -- 2.34 (real jq-1.6)",
|
||||
"jq\t\t9.9",
|
||||
# Killed ONLY by the first-line slice + `[[:blank:]]`. These carry the literal word `version`,
|
||||
# so the separator restriction is satisfied and cannot save us — the newline must be excluded
|
||||
# from the separator class AND the parse confined to line one.
|
||||
#
|
||||
# Without these, a round-5 mutation check found that reverting BOTH of those changes together
|
||||
# (`[[:blank:]]`→`[[:space:]]` and parsing `$raw` instead of `$first`) left the whole suite
|
||||
# GREEN: the four cases above are all killed by the separator alone, so they attributed the fix
|
||||
# to the wrong layer. A test that passes for the wrong reason is how the previous three rounds
|
||||
# each shipped a defect.
|
||||
"jq\nversion\n9.9",
|
||||
"jq\nversion 9.9",
|
||||
"jq \n version \n 9.9",
|
||||
])
|
||||
def test_a_number_AFTER_the_jq_token_is_not_reachable_across_filler(preflight, version_line):
|
||||
"""Two independent layers keep a stray number from being read as the version, and both are
|
||||
pinned here: the separator must be one of the forms real jq emits (`jq-1.6` / `jq version 1.6`),
|
||||
AND the match is confined to the first line with `[[:blank:]]` (which, unlike `[[:space:]]`,
|
||||
does not match a newline). Round 3's 'anchor' had neither and parsed `jq\\n2.34: cannot load`
|
||||
as 2.34."""
|
||||
preflight.with_jq(version_line)
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1, f"{version_line!r} was accepted as a version"
|
||||
|
||||
|
||||
def test_a_CRLF_version_line_parses_and_logs_without_the_carriage_return(preflight):
|
||||
"""The trailing `\\r` strip was unpinned — the commit claimed CRLF was verified, but nothing in
|
||||
the suite contained one. Harmless today (a `\\r` satisfies the trailing non-digit boundary, so
|
||||
the version still parses) but the log line would carry a stray CR."""
|
||||
preflight.with_jq("jq-1.6\r")
|
||||
r = preflight.run_bytes()
|
||||
assert r.returncode == 0, r.stderr
|
||||
assert b"parsed 1.6" in r.stdout
|
||||
# Two separate traps had to be cleared for this assertion to mean anything:
|
||||
# 1. `str.splitlines()` also splits on `\r`, so inspecting the "version in use" line would drop
|
||||
# the stray CR before the assertion could see it;
|
||||
# 2. `subprocess.run(text=True)` translates `\r` to `\n` outright, so even raw-string checks on
|
||||
# `r.stdout` were unfalsifiable.
|
||||
# Both made the test pass identically with the strip removed. Hence `run_bytes()` and a bytes
|
||||
# comparison — verified by mutation, not by reading the code.
|
||||
assert b"\r" not in r.stdout, "the carriage return leaked into the log line"
|
||||
|
||||
|
||||
def test_the_observability_line_stays_on_ONE_line(preflight):
|
||||
"""The no-arg mode exists to put a single grep-able version line in the job log; interpolating a
|
||||
multi-line `--version` would split it."""
|
||||
preflight.with_jq("jq-1.6\ntrailing noise")
|
||||
r = preflight.run()
|
||||
assert r.returncode == 0, r.stderr
|
||||
version_lines = [ln for ln in r.stdout.splitlines() if "version in use" in ln]
|
||||
assert len(version_lines) == 1
|
||||
assert "trailing noise" not in r.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line,expected", [
|
||||
("jq-1.6 (Debian 1.6-2.1)", "1.6"), # distro packaging suffix
|
||||
("jq-1.10", "1.10"), # two-digit minor: must compare numerically, not lexically
|
||||
("jq-1.7.1", "1.7"),
|
||||
("jq-1.6.0", "1.6"),
|
||||
("jq-v1.6", "1.6"),
|
||||
("JQ-1.6", "1.6"),
|
||||
])
|
||||
def test_legitimate_forms_still_parse_to_the_right_version(preflight, version_line, expected):
|
||||
preflight.with_jq(version_line)
|
||||
r = preflight.run()
|
||||
assert r.returncode == 0, f"{version_line!r}: {r.stderr}"
|
||||
assert f"parsed {expected}" in r.stdout
|
||||
|
||||
|
||||
# --- Wiring guards: the preflight is worthless if a caller silently stops running it ------------
|
||||
|
||||
def test_script_tests_pins_the_jq_version():
|
||||
"""The pin is the tripwire, so its presence is asserted rather than merely commented.
|
||||
|
||||
SCOPE NOTE — the symmetric assertion about `review-verdict.yml` (that it runs the FLOOR-only
|
||||
mode and must never pin, because it writes the branch-protection-required `review-verdict/h10`
|
||||
status and a pin would deadlock every merge on a jq bump) lands with the follow-up PR that
|
||||
wires that workflow. It cannot land here: that workflow checks out the BASE ref, and the base
|
||||
is `main`, which does not yet contain `scripts/jq-preflight.sh`.
|
||||
"""
|
||||
pr_checks = (WORKFLOWS / "pr-checks.yml").read_text()
|
||||
assert "jq-preflight.sh --expect" in pr_checks, \
|
||||
"script-tests must pin the jq version — that pin is the tripwire"
|
||||
|
||||
|
||||
def test_review_verdict_never_pins_a_jq_version():
|
||||
"""Whatever else changes, the REQUIRED merge check must never carry a hard version pin.
|
||||
|
||||
Asserted now, before the workflow is wired, so the constraint is already enforced when the
|
||||
follow-up PR adds the floor-only call — rather than being a comment someone can miss.
|
||||
"""
|
||||
review_verdict = (WORKFLOWS / "review-verdict.yml").read_text()
|
||||
assert "jq-preflight.sh --expect" not in review_verdict, \
|
||||
("review-verdict.yml must NOT pin a jq version: it writes the required review-verdict/h10 "
|
||||
"status, so a pin would deadlock every merge on a jq bump (ersatztv#648)")
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Tests for the base-change detection in `.claude/hooks/pretooluse-merge-consent.sh` (#632).
|
||||
|
||||
`review-verdict/h10` is a per-sha commit status, which makes "a new commit inherits an old verdict"
|
||||
impossible by construction (#622). Retargeting a PR's base reaches the same end by the opposite
|
||||
route: the head sha does not move, so the status stays green, while the merge-base — and therefore
|
||||
the effective diff the verdict was formed against — changes underneath it.
|
||||
|
||||
What is asserted here is DETECTION on the hook path only, and the tests are written to keep that
|
||||
claim narrow:
|
||||
|
||||
* a status carries no base field of its own, so the server-side required check cannot see this at
|
||||
all; a merge driven through the Gitea UI or API is unaffected. No test here implies otherwise.
|
||||
* a verdict posted before #632 has no `(base: …)` in its description and must get NO opinion,
|
||||
rather than denying every in-flight PR the day this lands.
|
||||
|
||||
Observable contract: the hook exits 0 with EMPTY stdout when it has no opinion (passthrough to
|
||||
normal permissioning), and emits a JSON `permissionDecision` otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
HOOK = REPO_ROOT / ".claude" / "hooks" / "pretooluse-merge-consent.sh"
|
||||
|
||||
SHA = "a9e3e23abf337980ca4c05854f5b1e210099d08b"
|
||||
|
||||
# The PR is deliberately NOT docs-only: the docs-only exemption short-circuits the whole gate, so a
|
||||
# docs PR would never reach the base check and the tests would pass without exercising it.
|
||||
CURL_SHIM = r'''#!/usr/bin/env python3
|
||||
import json, os, sys, pathlib, urllib.parse
|
||||
|
||||
state = pathlib.Path(os.environ["STUB_DIR"])
|
||||
args = sys.argv[1:]
|
||||
url = [a for a in args if a.startswith("http")][-1]
|
||||
|
||||
if "/pulls/" in url and "/files" in url:
|
||||
q = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||||
page = int(q.get("page", ["1"])[0])
|
||||
if page == 1:
|
||||
print(json.dumps([{"filename": "ErsatzTV/Program.cs", "status": "modified"}]))
|
||||
else:
|
||||
print("[]")
|
||||
sys.exit(0)
|
||||
|
||||
if "/status" in url:
|
||||
desc = (state / "verdict_desc").read_text()
|
||||
if desc == "TRANSPORT-ERROR":
|
||||
sys.exit(22)
|
||||
if desc == "GARBAGE":
|
||||
print('{"message":"internal error"}'); sys.exit(0)
|
||||
if desc == "SCALAR-ROW":
|
||||
print('{"state":"success","statuses":[1]}'); sys.exit(0)
|
||||
if desc == "NONSTRING-DESC":
|
||||
print(json.dumps({"state": "success", "statuses": [
|
||||
{"context": "review-verdict/h10", "status": "success", "description": {"x": 1}}]}))
|
||||
sys.exit(0)
|
||||
rows = [] if desc == "NONE" else [
|
||||
{"context": "review-verdict/h10", "status": "success", "description": desc}]
|
||||
print(json.dumps({"state": "success", "statuses": rows}))
|
||||
sys.exit(0)
|
||||
|
||||
if "/pulls/" in url:
|
||||
body = {"head": {"sha": os.environ["STUB_SHA"]}, "body": "fixes #1"}
|
||||
live = (state / "live_base").read_text().strip()
|
||||
if live != "MISSING":
|
||||
body["base"] = {"ref": live}
|
||||
print(json.dumps(body))
|
||||
sys.exit(0)
|
||||
|
||||
print("{}")
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hook(tmp_path):
|
||||
bindir = tmp_path / "bin"; bindir.mkdir()
|
||||
curl = bindir / "curl"; curl.write_text(CURL_SHIM); curl.chmod(0o755)
|
||||
state = tmp_path / "state"; state.mkdir()
|
||||
(state / "live_base").write_text("main")
|
||||
(state / "verdict_desc").write_text("Review-verdict: MERGEABLE @ a9e3e23 (base: main)")
|
||||
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}"
|
||||
env["STUB_DIR"] = str(state)
|
||||
env["STUB_SHA"] = SHA
|
||||
env["ETV_GITEA_TOKEN"] = "stub"
|
||||
env["ETV_GITEA_URL"] = "http://gitea.example"
|
||||
env.pop("ETV_GITEA_BASICAUTH", None)
|
||||
|
||||
class Handle:
|
||||
def set_live_base(self, ref):
|
||||
(state / "live_base").write_text(ref)
|
||||
|
||||
def set_verdict_description(self, desc):
|
||||
"""'NONE' serves a head with no review-verdict/h10 status at all."""
|
||||
(state / "verdict_desc").write_text(desc)
|
||||
|
||||
def decision(self):
|
||||
payload = {"tool_input": {"method": "merge", "owner": "timothy",
|
||||
"repo": "ersatztv", "pull_number": 42}}
|
||||
r = subprocess.run(["bash", str(HOOK)], input=json.dumps(payload),
|
||||
env=env, capture_output=True, text=True)
|
||||
assert r.returncode == 0, r.stderr
|
||||
if not r.stdout.strip():
|
||||
return None
|
||||
return json.loads(r.stdout)
|
||||
|
||||
def reason(self):
|
||||
d = self.decision()
|
||||
return "" if d is None else json.dumps(d)
|
||||
|
||||
return Handle()
|
||||
|
||||
|
||||
def test_a_retargeted_base_denies_a_verdict_formed_against_the_old_one(hook):
|
||||
hook.set_live_base("release/26.4")
|
||||
reason = hook.reason()
|
||||
assert "deny" in reason, "a verdict formed against a different base was allowed to stand"
|
||||
assert "release/26.4" in reason and "main" in reason, (
|
||||
"the deny must name both bases; a reader cannot act on 'the base changed'")
|
||||
|
||||
|
||||
def test_positive_control_an_unchanged_base_does_not_trigger_the_base_deny(hook):
|
||||
"""Without this, the test above could pass because the hook denies on every path — which it
|
||||
very nearly does, since this PR is non-docs and the rest of the gate is unstubbed."""
|
||||
reason = hook.reason()
|
||||
assert "ersatztv#632" not in reason, (
|
||||
"the base check fired on a PR whose base never moved")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("desc", [
|
||||
"Review-verdict: MERGEABLE @ a9e3e23", # posted before #632
|
||||
"NONE", # no verdict status on this head at all
|
||||
])
|
||||
def test_a_verdict_with_no_recorded_base_gets_no_opinion(hook, desc):
|
||||
"""Graceful adoption. Denying here would block every in-flight PR the day this lands, and the
|
||||
window closes on its own: verdicts are per-head and short-lived, so every verdict posted after
|
||||
#632 carries the field.
|
||||
|
||||
Asserting on the word "base" rather than on the issue tag, per cold review: the tag-only check
|
||||
would have passed for a base-specific ask or deny whose wording happened to omit it, which is
|
||||
the failure mode most likely to appear when someone edits these messages.
|
||||
"""
|
||||
hook.set_live_base("release/26.4")
|
||||
hook.set_verdict_description(desc)
|
||||
assert "base" not in hook.reason(), (
|
||||
"a pre-#632 verdict drew a base-related decision for a field it could not have carried")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["SCALAR-ROW", "NONSTRING-DESC"])
|
||||
def test_a_malformed_status_MEMBER_asks_too(hook, failure):
|
||||
"""One level below the previous fix, and it survived it.
|
||||
|
||||
Validating only that `.statuses` is an array left `{"statuses":[1]}` passing the guard, after
|
||||
which `.context` on a number errors and a `|| true` on the extraction turned that error into an
|
||||
empty description — straight back onto the graceful-adoption path, which is precisely the
|
||||
outcome the guard exists to distinguish from. Same swallow-the-error shape as the bug one level
|
||||
up, which is why the validation domain must match the CONSUMPTION domain rather than stopping at
|
||||
the top-level type.
|
||||
"""
|
||||
hook.set_live_base("release/26.4")
|
||||
hook.set_verdict_description(failure)
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason and "base" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["TRANSPORT-ERROR", "GARBAGE"])
|
||||
def test_an_UNREADABLE_status_response_asks_rather_than_skipping_the_check(hook, failure):
|
||||
""""Could not check" is a third outcome, not a quiet synonym for "no base recorded".
|
||||
|
||||
The first draft collapsed the two: an unreadable status response produced an empty
|
||||
`recorded_base`, took the graceful-adoption path, and skipped validation in silence — after
|
||||
which a later successful status read could still auto-grant, emitting "merge gate: satisfied"
|
||||
for a comparison that never happened. A transient Gitea hiccup is not evidence that the base is
|
||||
unchanged.
|
||||
"""
|
||||
hook.set_live_base("release/26.4")
|
||||
hook.set_verdict_description(failure)
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "an unreadable status response silently skipped the base check"
|
||||
assert "base" in reason, "the ask must name what could not be checked"
|
||||
|
||||
|
||||
def test_a_pr_with_no_resolvable_base_asks(hook):
|
||||
"""A null/absent `.base.ref` is also 'could not check', not 'nothing to check'."""
|
||||
hook.set_live_base("MISSING")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason and "base" in reason
|
||||
|
||||
|
||||
def test_the_comparator_is_the_base_REF_not_its_tip_sha():
|
||||
"""The design decision this test exists to freeze. `base.sha` tracks the base branch's TIP,
|
||||
which moves every time anything merges to `main` — comparing that would invalidate every open
|
||||
verdict on every unrelated merge, turning a rare-event guard into a permanent merge deadlock.
|
||||
A base branch that merely ADVANCES must be silent here; rebasing onto it moves the head sha,
|
||||
which the per-sha binding already covers."""
|
||||
assert ".base.ref" in HOOK.read_text(), "the hook must compare the base BRANCH, not its tip sha"
|
||||
assert ".base.sha" not in HOOK.read_text(), (
|
||||
"comparing base.sha deadlocks every open PR whenever main advances")
|
||||
@@ -64,7 +64,27 @@ if "/pulls/" in url:
|
||||
ctr.write_text(str(nread + 1))
|
||||
if alt.exists() and nread >= 1:
|
||||
shas = [alt.read_text().strip()]
|
||||
print(json.dumps({"head": {"sha": shas[0]}, "body": "no linked issue here"}))
|
||||
# `.base.ref` is served because the hook now reads it and threads it to the enumeration as the
|
||||
# required 5th argument (ersatztv#698 route 1). Without it the hook passes an empty base, the
|
||||
# script exits 2, and EVERY docs-only exemption silently stops being granted — which is exactly
|
||||
# how this stub failed when the argument was added: the eight failures were all positive cases.
|
||||
# Fail-closed, so not dangerous, but it would have made the advisory hook prompt on every
|
||||
# docs-only PR.
|
||||
#
|
||||
# `.base.sha` joined it for the same reason one release later (ersatztv#707), and the symptom
|
||||
# repeated almost exactly: NINE failures, every one a positive control, because the enumeration
|
||||
# now binds the base's TIP across the paging window and an absent tip fails closed. Worth stating
|
||||
# as a standing property of this stub rather than a second anecdote — it serves the fields the
|
||||
# SHARED enumeration reads, so every new binding the script learns must be modelled here too, and
|
||||
# the tell is always a wave of positive cases going red at once.
|
||||
base_sha = os.environ.get("STUB_BASE_SHA", "b" * 40)
|
||||
alt_base = state / "base_sha_after.txt"
|
||||
if alt_base.exists() and nread >= 1:
|
||||
base_sha = alt_base.read_text().strip()
|
||||
print(json.dumps({"head": {"sha": shas[0]},
|
||||
"base": {"ref": os.environ.get("STUB_BASE", "main"),
|
||||
"sha": base_sha},
|
||||
"body": "no linked issue here"}))
|
||||
sys.exit(0)
|
||||
|
||||
print("{}")
|
||||
@@ -495,3 +515,36 @@ def test_array_valued_status_does_not_dodge_the_allow_list(hook):
|
||||
def test_object_valued_status_is_also_rejected(hook):
|
||||
hook.set_pages([{"filename": "docs/a.md", "status": {"x": "renamed"}}], [])
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
|
||||
# --- The `grep -q` / pipefail inversion, on the ADVISORY side (ersatztv#698) --------------------
|
||||
#
|
||||
# Round-2 cross-family review noted the enforced gate gained large-input regression tests while the
|
||||
# hook — which carries the SAME predicate — did not. The hook's blast radius is smaller (a missing
|
||||
# prompt, not a green required check), but `ci.shared-pr-file-enumeration` exists precisely because
|
||||
# the copy with LESS authority is the one that quietly keeps a bug. So test both.
|
||||
#
|
||||
# `grep -q` exits at its first match; the producer then takes SIGPIPE (141) once the path list exceeds
|
||||
# the pipe buffer, and under `set -o pipefail` a MATCH is reported as a FAILED pipeline — inverting the
|
||||
# negated docs-only test. ~171KB is needed to cross the threshold; every other test in this file uses a
|
||||
# handful of short paths, which is exactly why the class was invisible here.
|
||||
|
||||
def _many_docs(n=1900):
|
||||
return [f"docs/{'d' * 40}-{i:040d}.md" for i in range(n)]
|
||||
|
||||
|
||||
def test_a_LARGE_pr_containing_a_code_file_is_NOT_exempt(hook):
|
||||
"""The code file goes FIRST so the guard matches immediately and the producer is left with the
|
||||
bulk of ~171KB still to write."""
|
||||
hook.set_pages(_rows(["A.cs", *_many_docs()]))
|
||||
assert hook.exempted() is False, (
|
||||
"a large PR containing A.cs was granted the docs-only exemption — the predicate inverted")
|
||||
|
||||
|
||||
def test_positive_control_a_LARGE_genuinely_docs_only_pr_IS_still_exempt(hook):
|
||||
"""Guards the opposite failure: if large lists merely errored, the test above would pass while the
|
||||
hook prompted on every big docs PR. Without this, 'fixed' and 'broken' are indistinguishable."""
|
||||
hook.set_pages(_rows(_many_docs()))
|
||||
assert hook.exempted() is True, (
|
||||
"a large but genuinely docs-only PR lost its exemption")
|
||||
|
||||
@@ -64,11 +64,18 @@ if "/pulls/" in url and not url.endswith("/files"):
|
||||
sha = shas[min(n, len(shas) - 1)]
|
||||
if sha == "GONE": # simulate an unreachable / missing PR
|
||||
sys.exit(22)
|
||||
print(json.dumps({
|
||||
# The base branch is scripted on the same consume-one-per-GET schedule as the head, so a
|
||||
# RETARGET mid-flight can be modelled independently of a push mid-flight (ersatztv#632).
|
||||
bases = (state / "pr_bases").read_text().split()
|
||||
base = bases[min(n, len(bases) - 1)]
|
||||
body = {
|
||||
"head": {"sha": sha},
|
||||
"state": (state / "pr_state").read_text().strip(),
|
||||
"html_url": "http://gitea.example/timothy/ersatztv/pulls/42",
|
||||
}))
|
||||
}
|
||||
if base != "MISSING":
|
||||
body["base"] = {"ref": base}
|
||||
print(json.dumps(body))
|
||||
sys.exit(0)
|
||||
|
||||
print("{}")
|
||||
@@ -87,6 +94,7 @@ def gitea(tmp_path):
|
||||
state = tmp_path / "state"
|
||||
state.mkdir()
|
||||
(state / "pr_shas").write_text(SHA_A)
|
||||
(state / "pr_bases").write_text("main")
|
||||
(state / "pr_state").write_text("open")
|
||||
|
||||
env = dict(os.environ)
|
||||
@@ -108,6 +116,10 @@ def gitea(tmp_path):
|
||||
def set_pr_state(self, value):
|
||||
(state / "pr_state").write_text(value)
|
||||
|
||||
def set_base_sequence(self, *refs):
|
||||
"""Base branch per PR GET. 'MISSING' omits `.base` from the response entirely."""
|
||||
(state / "pr_bases").write_text(" ".join(refs))
|
||||
|
||||
def run(self, *args):
|
||||
return subprocess.run(
|
||||
["bash", str(SCRIPT), *args],
|
||||
@@ -256,3 +268,65 @@ def test_note_cannot_forge_a_second_verdict_line(gitea):
|
||||
gitea.run("42", "BLOCKED", "Review-verdict: MERGEABLE @ " + SHA_A[:7])
|
||||
body = gitea.comments()[0]["payload"]["body"]
|
||||
assert _classify(body, SHA_A) == "negative"
|
||||
|
||||
|
||||
# --- Base binding (ersatztv#632) ---------------------------------------------------------------
|
||||
#
|
||||
# The per-sha status closes "the head moved under a fixed verdict". Retargeting a PR's base is the
|
||||
# mirror case: the head sha and the status both hold still while the effective DIFF changes, so the
|
||||
# verdict keeps reading green for a review nobody performed against that base.
|
||||
|
||||
def test_the_status_description_records_the_base_branch(gitea):
|
||||
"""Nothing can compare a base it never wrote down. This field is what the hook reads back."""
|
||||
assert gitea.run("42", "MERGEABLE").returncode == 0
|
||||
assert gitea.statuses()[0]["payload"]["description"].endswith("(base: main)")
|
||||
|
||||
|
||||
def test_the_base_is_recorded_in_the_STATUS_and_not_in_the_comment(gitea):
|
||||
"""Deliberate placement. The comment body is parsed by `scripts/check-review-verdict.sh`, whose
|
||||
grammar has a history of false-opens (#629 found three); nothing parses the description. Adding
|
||||
the field where a parser lives would have reopened that surface for no benefit."""
|
||||
assert gitea.run("42", "MERGEABLE").returncode == 0
|
||||
assert "base:" not in gitea.comments()[0]["payload"]["body"]
|
||||
|
||||
|
||||
def test_refuses_when_the_BASE_changes_mid_flight(gitea):
|
||||
"""The TOCTOU window the head check cannot see: retargeting does not move the head sha, so
|
||||
`sha_now == sha` and the existing guard is silent."""
|
||||
gitea.set_base_sequence("main", "release/26.4")
|
||||
result = gitea.run("42", "MERGEABLE")
|
||||
assert result.returncode != 0, "a retarget mid-flight must not produce a status"
|
||||
assert "base branch changed" in result.stderr
|
||||
assert gitea.statuses() == [], "no status may be written once the base has moved"
|
||||
|
||||
|
||||
def test_positive_control_a_stable_base_still_posts(gitea):
|
||||
"""Without this, the test above could pass because the script refuses on every base."""
|
||||
gitea.set_base_sequence("main", "main")
|
||||
assert gitea.run("42", "MERGEABLE").returncode == 0
|
||||
assert len(gitea.statuses()) == 1
|
||||
|
||||
|
||||
def test_refuses_when_the_pr_has_no_resolvable_base(gitea):
|
||||
"""A verdict that cannot record what it was formed against is not a verdict this gate can
|
||||
later re-check, so it fails closed rather than posting an unbindable success."""
|
||||
gitea.set_base_sequence("MISSING")
|
||||
result = gitea.run("42", "MERGEABLE")
|
||||
assert result.returncode != 0
|
||||
assert gitea.statuses() == []
|
||||
|
||||
|
||||
def test_a_failed_HEAD_RECHECK_writes_no_status(gitea):
|
||||
"""Fail-closed on the re-read itself, not just on a moved head.
|
||||
|
||||
This guard was previously implicit: `sha_now=$(api_get ... | jq ...)` aborted under `set -e` +
|
||||
`pipefail` when the GET failed. Nothing asserted it, so folding the head and base re-reads into
|
||||
one `$(... || true)` variable silently converted it to fail-OPEN — both guards see an empty
|
||||
string, both no-op, and the status is written having confirmed nothing. Asserted now so the
|
||||
behaviour is a contract rather than a side effect of a shell option.
|
||||
"""
|
||||
gitea.set_head_sequence(SHA_A, "GONE")
|
||||
result = gitea.run("42", "MERGEABLE")
|
||||
assert result.returncode != 0
|
||||
assert gitea.statuses() == [], (
|
||||
"a status was written even though the head/base re-read failed — nothing was confirmed")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,186 @@
|
||||
"""Tests for the tag-only-push exemption in `.claude/hooks/prepush-rebase-check.sh` (ersatztv#719).
|
||||
|
||||
H11 refuses to push a branch that is behind `origin/main`, to force a rebase instead of a merge.
|
||||
But the release cut tags a commit on `main` from a branch that is behind `origin/main`, so H11
|
||||
blocked every release -- and its "rebase first" advice did not even apply, because no branch was
|
||||
being pushed. (Observed while cutting v26.13.0; see #719. `docs/ci-cd.md` -> "Cutting a release"
|
||||
documents the tag step itself, not the release-notes-PR flow that puts the branch behind.) A tag
|
||||
push cannot revert anyone's merged work (the failure mode H11 exists to prevent), so the fix skips
|
||||
the freshness check when EVERY ref being pushed is under `refs/tags/`.
|
||||
|
||||
These tests use real local git repositories (a bare "origin" plus a work tree pushed one commit
|
||||
behind it) rather than stubbing `git`, because the hook's decision hinges on genuine
|
||||
`git fetch` / `merge-base` / `rev-list` behavior against an origin that has moved.
|
||||
|
||||
`test_zero_ref_lines_does_not_exempt` is the load-bearing negative case from the issue: "all pushed
|
||||
refs are tags" is vacuously true over zero ref lines, so a naive implementation would disable H11
|
||||
entirely whenever stdin is empty (hook run manually, or a caller that forgot to forward it). The fix
|
||||
must require at least one parsed ref line before granting the exemption.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
HOOK = REPO_ROOT / ".claude" / "hooks" / "prepush-rebase-check.sh"
|
||||
|
||||
DUMMY_SHA_A = "a" * 40
|
||||
DUMMY_SHA_B = "b" * 40
|
||||
|
||||
|
||||
def _git(args, cwd):
|
||||
r = subprocess.run(["git", *args], cwd=str(cwd), capture_output=True, text=True)
|
||||
assert r.returncode == 0, f"git {' '.join(args)} failed: {r.stderr}"
|
||||
return r.stdout
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def behind_repo(tmp_path):
|
||||
"""A work tree whose local `main` is exactly one commit behind `origin/main`."""
|
||||
origin = tmp_path / "origin.git"
|
||||
_git(["init", "--bare", "-q", str(origin)], cwd=tmp_path)
|
||||
|
||||
work = tmp_path / "work"
|
||||
_git(["init", "-q", "-b", "main", str(work)], cwd=tmp_path)
|
||||
_git(["config", "user.email", "test@example.com"], cwd=work)
|
||||
_git(["config", "user.name", "Test"], cwd=work)
|
||||
(work / "f.txt").write_text("one\n")
|
||||
_git(["add", "f.txt"], cwd=work)
|
||||
_git(["commit", "-q", "-m", "initial"], cwd=work)
|
||||
_git(["remote", "add", "origin", str(origin)], cwd=work)
|
||||
_git(["push", "-q", "-u", "origin", "main"], cwd=work)
|
||||
# The bare repo's HEAD symref still points at the (nonexistent) default branch until something
|
||||
# sets it explicitly; without this, `git clone` below checks out an unborn HEAD and "main" never
|
||||
# exists as a local branch in `advancer`.
|
||||
_git(["symbolic-ref", "HEAD", "refs/heads/main"], cwd=origin)
|
||||
|
||||
# Advance origin/main independently, via a second clone, so `work`'s local `main` falls behind.
|
||||
advancer = tmp_path / "advancer"
|
||||
_git(["clone", "-q", str(origin), str(advancer)], cwd=tmp_path)
|
||||
_git(["config", "user.email", "test@example.com"], cwd=advancer)
|
||||
_git(["config", "user.name", "Test"], cwd=advancer)
|
||||
(advancer / "f.txt").write_text("two\n")
|
||||
_git(["add", "f.txt"], cwd=advancer)
|
||||
_git(["commit", "-q", "-m", "advance"], cwd=advancer)
|
||||
_git(["push", "-q", "origin", "main"], cwd=advancer)
|
||||
|
||||
return work
|
||||
|
||||
|
||||
def _run_hook(cwd, stdin_text):
|
||||
env = dict(os.environ)
|
||||
for k in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"):
|
||||
env.pop(k, None)
|
||||
return subprocess.run(
|
||||
["bash", str(HOOK)],
|
||||
cwd=str(cwd),
|
||||
input=stdin_text,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def test_tag_only_push_from_a_behind_branch_is_allowed(behind_repo):
|
||||
"""The fix: a tag-only push must not be blocked by H11 even though the branch is behind."""
|
||||
stdin = f"refs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}\n"
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 0, f"tag-only push was blocked: {r.stdout}{r.stderr}"
|
||||
|
||||
|
||||
def test_tag_only_push_ignores_blank_lines(behind_repo):
|
||||
stdin = f"\nrefs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}\n\n"
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 0, f"tag-only push (with blank lines) was blocked: {r.stdout}{r.stderr}"
|
||||
|
||||
|
||||
def test_negative_control_branch_push_from_behind_is_still_blocked(behind_repo):
|
||||
"""Required by #719: the fix must not weaken H11 for ordinary branch pushes."""
|
||||
stdin = f"refs/heads/feature {DUMMY_SHA_A} refs/heads/feature {DUMMY_SHA_B}\n"
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 1, "a branch push from a behind branch was allowed"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
def test_mixed_branch_and_tag_push_is_still_blocked(behind_repo):
|
||||
stdin = (
|
||||
f"refs/heads/feature {DUMMY_SHA_A} refs/heads/feature {DUMMY_SHA_B}\n"
|
||||
f"refs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}\n"
|
||||
)
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 1, "a mixed branch+tag push was allowed through the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
def test_zero_ref_lines_does_not_exempt(behind_repo):
|
||||
"""Vacuous-truth guard: 'all refs are tags' is trivially true over zero lines. Empty stdin
|
||||
(hook run manually, or a caller that forgot to forward the ref lines) must fall through to the
|
||||
existing behind-origin/main check, not silently disable H11."""
|
||||
r = _run_hook(behind_repo, "")
|
||||
assert r.returncode == 1, "empty stdin vacuously granted the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
def test_zero_ref_lines_of_only_blank_lines_does_not_exempt(behind_repo):
|
||||
r = _run_hook(behind_repo, "\n\n\n")
|
||||
assert r.returncode == 1, "stdin of only blank lines vacuously granted the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
# --- final line with NO trailing newline -------------------------------------------------------
|
||||
# `read` returns non-zero on an unterminated final line, so a bare `while read` silently DROPS it.
|
||||
# Both directions matter and they fail differently, which is why each is pinned:
|
||||
# - tag-only, unterminated -> the line is dropped, no refs are seen, and H11 blocks the release
|
||||
# tag push again, i.e. #719 quietly returns.
|
||||
# - mixed, unterminated -> the BRANCH line is dropped, leaving only tag refs, and the
|
||||
# exemption is granted for a push that includes a branch. That is the dangerous direction.
|
||||
# Git always newline-terminates its ref lines and `.husky/pre-push` re-adds one via `printf '%s\n'`,
|
||||
# so this is reachable only on a hand-piped run — but the guard is cheap and the failure is silent.
|
||||
|
||||
|
||||
def test_unterminated_final_line_tag_only_is_still_exempt(behind_repo):
|
||||
stdin = f"refs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}" # no trailing \n
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 0, f"unterminated tag-only line was dropped, reinstating #719: {r.stdout}"
|
||||
|
||||
|
||||
def test_tty_stdin_does_not_hang_and_does_not_exempt(behind_repo):
|
||||
"""The hook gained a stdin reader in #719; before that it read nothing, and its own docs call
|
||||
'run by hand' a supported case. Without the `[ -t 0 ] ||` guard an interactive run blocks
|
||||
forever waiting on the terminal. A pty gives it a real TTY on fd 0; the `timeout` turns a
|
||||
regression into a clean failure instead of a hung CI job."""
|
||||
primary, secondary = os.openpty()
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["bash", str(HOOK)],
|
||||
cwd=str(behind_repo),
|
||||
stdin=secondary,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
pytest.fail("hook hung on TTY stdin — the `[ -t 0 ] ||` guard is missing or ineffective")
|
||||
finally:
|
||||
os.close(primary)
|
||||
os.close(secondary)
|
||||
# A TTY yields no ref lines, so this is the zero-line fall-through: H11 still applies.
|
||||
assert r.returncode == 1, "TTY stdin vacuously granted the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
def test_unterminated_final_branch_line_is_not_swallowed_into_the_exemption(behind_repo):
|
||||
"""The dangerous direction: if the unterminated BRANCH line is dropped, only tag refs remain
|
||||
and a branch push wins the tag exemption."""
|
||||
stdin = (
|
||||
f"refs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}\n"
|
||||
f"refs/heads/feature {DUMMY_SHA_A} refs/heads/feature {DUMMY_SHA_B}" # no trailing \n
|
||||
)
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 1, "an unterminated branch ref was swallowed into the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
@@ -11,6 +11,15 @@ REPO_ROOT="$(pwd)"
|
||||
# dotnet-getdocument against ErsatzTV.dll + ErsatzTV.deps.json, which don't exist in a clean
|
||||
# tree (e.g. the CI api-docs job, which only restores). Without the build the target fails with
|
||||
# "The specified deps.json … does not exist" (exit 129). Build first, then generate.
|
||||
#
|
||||
# LOCAL-DEV SHARP EDGE: if the project is ALREADY built and nothing changed, MSBuild skips the
|
||||
# document-generation work but still runs RenameOpenApiFiles (AfterTargets), whose Move then fails
|
||||
# with MSB3680 "ErsatzTV.json does not exist" — because nothing produced it. The script correctly
|
||||
# exits non-zero, but a caller that pipes this (`./scripts/update-openapi.sh | tail`) sees the
|
||||
# PIPELINE's status, i.e. tail's 0, and reads a no-op as success — leaving stale artifacts to fail
|
||||
# the blocking api-docs CI job. Before verifying artifacts are current, `touch` a file the project
|
||||
# compiles (or check this script's own exit status, unpiped). CI is unaffected: it restores into a
|
||||
# clean tree, so the generation never skips.
|
||||
(cd ErsatzTV && dotnet build && dotnet build -t:GenerateOpenApiDocuments) || exit
|
||||
|
||||
cd "$REPO_ROOT" || exit
|
||||
|
||||
@@ -17,6 +17,7 @@ export * from './imageFolders';
|
||||
export * from './languages';
|
||||
export * from './libraries';
|
||||
export * from './libraryBrowse';
|
||||
export * from './selectionId';
|
||||
export * from './logs';
|
||||
export * from './maintenance';
|
||||
export * from './mediaDetail';
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { getLibraryBrowseItems } from './libraryBrowse';
|
||||
import {
|
||||
getLibraryBrowseItems,
|
||||
searchLibraryBrowseItems,
|
||||
searchLibraryPickerOptions,
|
||||
titleContainsQuery,
|
||||
LIBRARY_PICKER_LUCENE_SPECIALS,
|
||||
LIBRARY_PICKER_RESULTS
|
||||
} from './libraryBrowse';
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
@@ -56,3 +63,141 @@ describe('getLibraryBrowseItems', () => {
|
||||
expect(browseUrl(fetchMock).searchParams.has('parentId')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('titleContainsQuery (#651 — compile typed text, never forward raw Lucene)', () => {
|
||||
it('wraps the escaped text in boundary wildcards on the title field', () => {
|
||||
expect(titleContainsQuery('Show Alpha')).toBe('title:*Show\\ Alpha*');
|
||||
});
|
||||
|
||||
// The previous version of this test hand-copied a sample string and claimed to cover "every
|
||||
// Lucene special" — it silently omitted `&` and `|`, and a completeness test that carries its own
|
||||
// list of what to check cannot see what is missing from that list (#651 F2). Drive the assertion
|
||||
// from the exported character set instead, one character at a time, so adding a character to the
|
||||
// set without escaping it fails here.
|
||||
it.each(LIBRARY_PICKER_LUCENE_SPECIALS.split(''))('escapes the Lucene special %j', (char) => {
|
||||
expect(titleContainsQuery(`a${char}b`)).toBe(`title:*a\\${char}b*`);
|
||||
});
|
||||
|
||||
it.each([' ', '\t', '\n'])('escapes whitespace %j so it cannot split the term', (char) => {
|
||||
expect(titleContainsQuery(`a${char}b`)).toBe(`title:*a\\${char}b*`);
|
||||
});
|
||||
|
||||
it('leaves every character that is NOT special untouched', () => {
|
||||
const plain = 'abcXYZ019_,.\'@#$%';
|
||||
for (const char of plain) {
|
||||
expect(LIBRARY_PICKER_LUCENE_SPECIALS).not.toContain(char);
|
||||
}
|
||||
expect(titleContainsQuery(plain)).toBe(`title:*${plain}*`);
|
||||
});
|
||||
|
||||
it('neutralises the && and || BOOLEAN operators, not just single characters (#651 F2)', () => {
|
||||
// The regression: `Rock && Roll` used to compile with `&&` live, so Lucene parsed it as boolean
|
||||
// syntax (or rejected the query) and an exactly-matching title returned nothing.
|
||||
expect(titleContainsQuery('Rock && Roll')).toBe('title:*Rock\\ \\&\\&\\ Roll*');
|
||||
expect(titleContainsQuery('A || B')).toBe('title:*A\\ \\|\\|\\ B*');
|
||||
expect(titleContainsQuery('Rock & Roll')).toBe('title:*Rock\\ \\&\\ Roll*');
|
||||
});
|
||||
|
||||
it('leaves a plain single word alone apart from the boundary stars', () => {
|
||||
expect(titleContainsQuery('Alpha')).toBe('title:*Alpha*');
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchLibraryPickerOptions (#651)', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('issues ONE bounded request with the compiled query and maps to {id, name}', async () => {
|
||||
const fetchMock = vi.spyOn(window, 'fetch').mockResolvedValue(
|
||||
jsonResponse({
|
||||
page: [
|
||||
{ id: 1, mediaItemId: 7, mediaType: 'Movie', title: 'Show Alpha' },
|
||||
{ id: 2, mediaItemId: null, mediaType: 'Movie', title: null }
|
||||
],
|
||||
totalCount: 20000
|
||||
})
|
||||
);
|
||||
|
||||
const options = await searchLibraryPickerOptions('Movie', ' Show Alpha ');
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const url = browseUrl(fetchMock);
|
||||
expect(url.searchParams.get('query')).toBe('title:*Show\\ Alpha*');
|
||||
expect(url.searchParams.get('mediaType')).toBe('Movie');
|
||||
expect(url.searchParams.get('pageNum')).toBe('0');
|
||||
expect(url.searchParams.get('pageSize')).toBe(String(LIBRARY_PICKER_RESULTS));
|
||||
// `mediaItemId` wins when present; `id` is the fallback, and a missing title degrades to `#id`.
|
||||
expect(options).toEqual([
|
||||
{ id: 7, name: 'Show Alpha' },
|
||||
{ id: 2, name: '#2' }
|
||||
]);
|
||||
});
|
||||
|
||||
it('#651 F4: CLAMPS an oversized pageSize rather than forwarding it', async () => {
|
||||
const fetchMock = vi.spyOn(window, 'fetch').mockResolvedValue(jsonResponse({ page: [], totalCount: 20000 }));
|
||||
|
||||
await searchLibraryPickerOptions('Episode', 'Alpha', 5000);
|
||||
|
||||
// The 25-row bound is a property of the helper, not of caller discipline.
|
||||
expect(browseUrl(fetchMock).searchParams.get('pageSize')).toBe(String(LIBRARY_PICKER_RESULTS));
|
||||
});
|
||||
|
||||
it('issues NO request for a query below the minimum length', async () => {
|
||||
const fetchMock = vi.spyOn(window, 'fetch').mockResolvedValue(jsonResponse({ page: [], totalCount: 0 }));
|
||||
|
||||
expect(await searchLibraryPickerOptions('Episode', 'a')).toEqual([]);
|
||||
expect(await searchLibraryPickerOptions('Episode', ' ')).toEqual([]);
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchLibraryBrowseItems (#685 — AddItemsDialog sibling of searchLibraryPickerOptions)', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
// The reviewer proved this helper was dead code to the suite: deleting its clamp, or deleting
|
||||
// its gate, both left the whole suite green. These three tests mirror the ones above for
|
||||
// searchLibraryPickerOptions so the same bound is pinned for the sibling helper.
|
||||
|
||||
it('#651 F4: CLAMPS an oversized pageSize rather than forwarding it', async () => {
|
||||
const fetchMock = vi.spyOn(window, 'fetch').mockResolvedValue(jsonResponse({ page: [], totalCount: 20000 }));
|
||||
|
||||
await searchLibraryBrowseItems('Episode', 'Alpha', 5000);
|
||||
|
||||
expect(browseUrl(fetchMock).searchParams.get('pageSize')).toBe(String(LIBRARY_PICKER_RESULTS));
|
||||
});
|
||||
|
||||
it('issues NO request for a query below the minimum length, and resolves an empty result', async () => {
|
||||
const fetchMock = vi.spyOn(window, 'fetch').mockResolvedValue(jsonResponse({ page: [], totalCount: 0 }));
|
||||
|
||||
expect(await searchLibraryBrowseItems('Episode', 'a')).toEqual({ items: [], totalCount: 0 });
|
||||
expect(await searchLibraryBrowseItems('Episode', ' ')).toEqual({ items: [], totalCount: 0 });
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('compiles/escapes the trimmed query, and returns the FULL row (mediaType present) plus totalCount', async () => {
|
||||
const fetchMock = vi.spyOn(window, 'fetch').mockResolvedValue(
|
||||
jsonResponse({
|
||||
page: [{ id: 1, mediaItemId: 7, mediaType: 'Movie', title: 'Show Alpha' }],
|
||||
totalCount: 42
|
||||
})
|
||||
);
|
||||
|
||||
const result = await searchLibraryBrowseItems('Movie', ' Show Alpha ');
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const url = browseUrl(fetchMock);
|
||||
expect(url.searchParams.get('query')).toBe(titleContainsQuery('Show Alpha'));
|
||||
expect(url.searchParams.get('mediaType')).toBe('Movie');
|
||||
expect(url.searchParams.get('pageNum')).toBe('0');
|
||||
expect(url.searchParams.get('pageSize')).toBe(String(LIBRARY_PICKER_RESULTS));
|
||||
// The reason this helper exists rather than reusing searchLibraryPickerOptions: the full row
|
||||
// (mediaType included), not the {id, name} shape.
|
||||
expect(result).toEqual({
|
||||
items: [{ id: 1, mediaItemId: 7, mediaType: 'Movie', title: 'Show Alpha' }],
|
||||
totalCount: 42
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -47,6 +47,105 @@ export function getLibraryBrowseItems(params: GetLibraryBrowseItemsParams = {}):
|
||||
return request<PagedLibraryBrowseItems>(`/api/v1/library/browse${queryString ? `?${queryString}` : ''}`);
|
||||
}
|
||||
|
||||
// A library picker compiles typed text; it never forwards raw Lucene (#440, #651). The search
|
||||
// index's default field does NOT match bare title words (`Alpha` finds nothing for "Show Alpha" —
|
||||
// docs/e2e-local.md), so forwarding the user's literal text the way the explicit query box does
|
||||
// would look broken in a *name* picker. Escape every Lucene special (and whitespace) so the
|
||||
// boundary stars are the only live wildcards — the same shape `builder/rules/compile.ts` emits for
|
||||
// its `contains` operator.
|
||||
//
|
||||
// The exhaustive set of characters Lucene's QueryParser treats as syntax. `&` and `|` are in it
|
||||
// because the boolean operators are `&&`/`||`: escaping each character individually neutralises the
|
||||
// pair. Leaving them live (as this helper's original AutoTuneScreen-local version did) meant a
|
||||
// title like `Rock && Roll` compiled to a query Lucene parsed as boolean syntax — or rejected — so
|
||||
// an exactly-matching title returned nothing (#651 F2). `LIBRARY_PICKER_LUCENE_SPECIALS` is
|
||||
// exported so the test asserts against the character list itself rather than a hand-copied sample
|
||||
// that cannot see its own omissions.
|
||||
export const LIBRARY_PICKER_LUCENE_SPECIALS = '+-&|!(){}[]^"~*?:\\/';
|
||||
const LUCENE_WILD_SPECIAL = /([\s+\-&|!(){}[\]^"~*?:\\/])/g;
|
||||
|
||||
export function titleContainsQuery(text: string): string {
|
||||
return `title:*${text.replace(LUCENE_WILD_SPECIAL, '\\$1')}*`;
|
||||
}
|
||||
|
||||
export interface LibraryPickerOption {
|
||||
id: number;
|
||||
name: string;
|
||||
}
|
||||
|
||||
// How many matches a search-driven library picker offers, and the shortest query worth issuing.
|
||||
// Both are hard bounds: such a picker NEVER loads more than one page of this size, whatever the
|
||||
// media type's row count (#651 — decision key `spa.library-pickers-resolve-by-search`).
|
||||
export const LIBRARY_PICKER_RESULTS = 25;
|
||||
export const LIBRARY_PICKER_MIN_QUERY = 2;
|
||||
|
||||
// Resolve picker options for one media-library type by SEARCH rather than by loading a window of
|
||||
// the whole type. Exactly one bounded request per (debounced) query; a too-short query issues none
|
||||
// at all.
|
||||
//
|
||||
// `pageSize` is CLAMPED to `LIBRARY_PICKER_RESULTS`, not merely defaulted to it (#651 F4): the
|
||||
// bound is documented as a property of this helper, so it must not be defeatable by a caller
|
||||
// passing a larger number.
|
||||
export function searchLibraryPickerOptions(
|
||||
mediaType: LibraryBrowseMediaType,
|
||||
text: string,
|
||||
pageSize: number = LIBRARY_PICKER_RESULTS
|
||||
): Promise<LibraryPickerOption[]> {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length < LIBRARY_PICKER_MIN_QUERY) {
|
||||
return Promise.resolve([]);
|
||||
}
|
||||
|
||||
const boundedPageSize = Math.max(1, Math.min(Math.floor(pageSize), LIBRARY_PICKER_RESULTS));
|
||||
|
||||
return getLibraryBrowseItems({
|
||||
mediaType,
|
||||
pageNum: 0,
|
||||
pageSize: boundedPageSize,
|
||||
query: titleContainsQuery(trimmed)
|
||||
}).then((result) =>
|
||||
(result.page ?? []).map((item) => {
|
||||
const id = item.mediaItemId ?? item.id;
|
||||
return { id, name: item.title ?? `#${id}` };
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export interface LibraryBrowseSearchResult {
|
||||
items: LibraryBrowseItem[];
|
||||
totalCount: number;
|
||||
}
|
||||
|
||||
// Like `searchLibraryPickerOptions` above — same min-query gate, same clamp, same compiled query —
|
||||
// but for a MULTI-select caller (`CollectionsScreen`'s `AddItemsDialog`) that needs the full
|
||||
// `LibraryBrowseItem` row (mediaType + id, for `toAddItemsRequest`) rather than the `{id, name}`
|
||||
// shape a single-select `SearchPicker` renders, plus the response's `totalCount` so the caller can
|
||||
// surface how much of a match was actually returned. The gate/clamp/compile live HERE, not at the
|
||||
// call site, so no caller can accidentally skip them (§3b — "the bound belongs to the helper, not
|
||||
// the caller").
|
||||
export function searchLibraryBrowseItems(
|
||||
mediaType: LibraryBrowseMediaType,
|
||||
text: string,
|
||||
pageSize: number = LIBRARY_PICKER_RESULTS
|
||||
): Promise<LibraryBrowseSearchResult> {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length < LIBRARY_PICKER_MIN_QUERY) {
|
||||
return Promise.resolve({ items: [], totalCount: 0 });
|
||||
}
|
||||
|
||||
const boundedPageSize = Math.max(1, Math.min(Math.floor(pageSize), LIBRARY_PICKER_RESULTS));
|
||||
|
||||
return getLibraryBrowseItems({
|
||||
mediaType,
|
||||
pageNum: 0,
|
||||
pageSize: boundedPageSize,
|
||||
query: titleContainsQuery(trimmed)
|
||||
}).then((result) => ({
|
||||
items: result.page ?? [],
|
||||
totalCount: result.totalCount ?? 0
|
||||
}));
|
||||
}
|
||||
|
||||
export function messageFromLibraryBrowseError(error: unknown, fallback = 'Unable to load library items'): string {
|
||||
if (error instanceof ApiError) {
|
||||
return error.detail ?? error.message;
|
||||
|
||||
@@ -0,0 +1,596 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { scanPageSizeSites } from './pageSizeScan';
|
||||
|
||||
/**
|
||||
* #650 guard: an ENUMERATING allow-list over every `pageSize` call site in the SPA.
|
||||
*
|
||||
* #644 fixed every call site that requested an OVER-cap `pageSize` (e.g. `pageSize: 1000`) to
|
||||
* "get everything in one call" — a pattern that silently truncates to the server's `MaxPageSize`
|
||||
* (100 today) with no error and no truncation indicator. #644's own completeness check (box 4)
|
||||
* was a manual grep for an inflated `pageSize`, which is why it could not see #650: two call
|
||||
* sites requesting EXACTLY the cap (100) truncate exactly as much as an over-cap request, they
|
||||
* just don't match a "pageSize above the cap" pattern.
|
||||
*
|
||||
* So this guard does NOT pattern-match on the pageSize VALUE (that repeats the #644 mistake for
|
||||
* the next magic number). It enumerates every `pageSize` property inside a real object-literal
|
||||
* expression via `scanPageSizeSites` (the TypeScript compiler API — see `pageSizeScan.ts`'s doc
|
||||
* comment for why a hand-rolled text/regex scan was replaced) and cross-checks the discovered set
|
||||
* against a hand-reviewed registry below, in BOTH directions:
|
||||
* - a NEWLY discovered, unregistered site fails (a new call site was added without a documented
|
||||
* classification — the exact way #650 could recur invisibly);
|
||||
* - a REGISTERED site no longer discovered fails (the registry has gone stale — e.g. a site was
|
||||
* removed or refactored to no longer pass a `pageSize` property, and the registry should
|
||||
* shrink to match, not silently claim coverage of code that no longer exists).
|
||||
* Both directions are computed and reported in a SINGLE combined failure message (not two
|
||||
* sequential `expect` calls) — an early throw would otherwise hide the second direction's result
|
||||
* in the same run, understating what actually needs fixing.
|
||||
*
|
||||
* **Identity is `(file, kind, value)` — deliberately NOT line/column.** The original guard keyed
|
||||
* each site on its absolute `line:column` (#650 follow-up F5/M-6). That made the registry a
|
||||
* function of every OTHER file's line count, so a branch that never touches this guard can still
|
||||
* invalidate it. This guard was BORN RED, and the sequence is the whole argument (#684): #651
|
||||
* moved `AutoTuneScreen.tsx` up ten lines and `FillerPresetsScreen.tsx` down seventy-two, and
|
||||
* merged to `main` BEFORE this guard's own PR (#675) did — so the registry, authored against a
|
||||
* pre-#651 base, was stale the instant it landed. Its own merge run was CANCELLED, so nothing
|
||||
* reported it, and the red first surfaced on the NEXT push (#676's merge, which touches no
|
||||
* `web/src` file at all and is in no way the cause).
|
||||
*
|
||||
* That is one ordering accident, not a recurring two-merge pattern — but the exposure is the
|
||||
* general case, because it is structurally invisible pre-merge: every PR is green against its own
|
||||
* base, so the breakage exists only in the merge result and lands after review and after the merge
|
||||
* gate.
|
||||
*
|
||||
* A NEW call site, a REMOVED one, and a CHANGED `pageSize` value each still fail, because each
|
||||
* changes the `(file, kind, value)` multiset. What no longer fails is MOVING an unchanged site
|
||||
* within its own file — no truncation risk, and exactly the churn being removed.
|
||||
*
|
||||
* **The one real coverage case this costs, stated rather than implied** (#684 review M2): a
|
||||
* SAME-IDENTITY SUBSTITUTION inside one file — delete a registered site and add a different,
|
||||
* unreviewed one with the same `kind` and the same value TOKEN, keeping the count equal. Verified
|
||||
* to pass: deleting `TrashScreen.tsx`'s load-more `pageSize: PAGE_SIZE` and adding a whole-library
|
||||
* `getLibraryBrowseItems({ mediaType: 'Movie', pageSize: PAGE_SIZE })` is green. It is narrow (same
|
||||
* file, same kind, same token, net-zero count), and the old identity caught it only incidentally —
|
||||
* it fired on every position change, so a reviewer conditioned to re-pin line numbers would likely
|
||||
* have waved it through anyway. Accepted knowingly; do not describe this guard as exhaustive.
|
||||
*
|
||||
* **Comparison stays a MULTISET count, not set membership** (#650 follow-up M-6, preserved): two
|
||||
* sites in one file sharing an identifier (`TrashScreen.tsx`'s two `PAGE_SIZE` requests,
|
||||
* `paging.ts`'s two `loadAllPages` fetches) register as two entries and must be discovered twice.
|
||||
* So adding a third occurrence, or an accidental duplicate registry entry, is still caught rather
|
||||
* than one occurrence silently covering the others.
|
||||
*
|
||||
* The SCANNER's own positional identity (`pageSizeSiteId`, line:column) is unchanged and still
|
||||
* asserted by `pageSizeScan.test.ts` — verifying the compiler-API scan reports real AST positions
|
||||
* is that test's actual subject, and it runs against fixed inline fixtures, so it has no churn.
|
||||
*
|
||||
* `scanPageSizeSites` itself is verified against inline fixture source strings covering every
|
||||
* input class a text-level scanner previously got wrong (comment-in-string, template
|
||||
* interpolation, ternary, `??`, JSX container, same-line duplicates, parameter/nested
|
||||
* destructuring, a type literal, a string containing the text `pageSize: 100`) in
|
||||
* `pageSizeScan.test.ts` — that test does not depend on the real repo, so it protects the SCANNER
|
||||
* itself, not just today's snapshot of call sites.
|
||||
*
|
||||
* Each registry entry classifies the site per `docs/spa-conventions.md` §3b /
|
||||
* `spa.library-pickers-resolve-by-search` (#651). NOTE the record path: that key SUPERSEDED
|
||||
* `spa.list-completeness-vs-bounded-pickers`, whose record has since moved to
|
||||
* `docs/decisions/archive/spa/` — resolve it through `docs/decisions/README.md` by key, never by
|
||||
* the path a comment happens to name (the breadcrumb rule).
|
||||
* - 'class-a' — bounded-by-construction list, paged to completeness via `loadAllPages`
|
||||
* (or, for the two sites INSIDE `loadAllPages` itself, its
|
||||
* implementation), with a `complete`/`incomplete` flag surfaced (never
|
||||
* silently partial).
|
||||
* - 'search-bounded' — resolves by SEARCH and windows nothing: the typed query is the narrowing
|
||||
* mechanism, and the row bound is a property of the CODE rather than of a
|
||||
* caller's discipline (a clamp inside the shared helper for #651's
|
||||
* library picker; a fixed small constant at the inline preview sites).
|
||||
* Nothing is list-loaded, so a truncation hint is not REQUIRED here — but,
|
||||
* unlike 'class-b', it is not the defining evidence either: a search-bounded
|
||||
* site MAY surface its own per-kind cap (e.g. summed across kinds) once that
|
||||
* count is actionable, without that hint reclassifying it as 'class-b'. What
|
||||
* distinguishes the two classes is the SHAPE of the bound (one clamped
|
||||
* search request per settled query vs. one bounded page of a list), not
|
||||
* whether a hint is rendered. Applies only where the query is genuinely
|
||||
* required: a site that degrades to an unfiltered browse when the query is
|
||||
* empty is NOT search-bounded (see 'deviation').
|
||||
* - 'class-b' — one bounded page at (or under) the cap, with the real truncation
|
||||
* (`totalCount` vs items shown) surfaced to the user. Post-#651 this no
|
||||
* longer covers media-library pickers (those are 'search-bounded').
|
||||
* **The RENDER is the entry requirement, not the intent** — that is the
|
||||
* operative rule, and the only one to apply to a new site. Today's
|
||||
* entries happen to take four shapes: a list bounded by its PARENT
|
||||
* (`ChannelBuilder`, seasons of one show); the collection-family types
|
||||
* §3b excludes from search (`FillerPresetsScreen`), which keep the
|
||||
* bounded page and its hint; a preview over an already-bounded set
|
||||
* (`AutoTuneScreen`'s channel members); and a preview over an UNBOUNDED
|
||||
* user-authored query that surfaces its match count
|
||||
* (`SmartCollectionDialog`). That list is illustrative and NOT
|
||||
* exhaustive: a site qualifies by rendering a real `totalCount`-backed
|
||||
* hint, not by resembling one of these four. (#684 review: an earlier
|
||||
* revision of this comment called it "the whole list" while the registry
|
||||
* below already held a fourth — the same false-exhaustiveness defect this
|
||||
* PR exists to remove.)
|
||||
* - 'paged-ui' — real paging UI (a page/"load more" control, or a user-adjustable
|
||||
* page-size selector, keyed to a genuine `totalCount`), so a `pageSize`
|
||||
* at or below the cap is correct as-is.
|
||||
* - 'deviation' — a KNOWN, TRACKED violation of §3b that this registry refuses to launder
|
||||
* into a compliant-looking label. A registry exists to state what is
|
||||
* true; recording a defect as 'class-b' or 'search-bounded' would make
|
||||
* the guard assert a hint or a query gate that demonstrably does not
|
||||
* exist, and the next reader would trust it. Every such entry MUST carry
|
||||
* its tracking issue in the structural `issue` field — enforced below,
|
||||
* and deliberately NOT a `#\d+` scrape of the note, which passed with the
|
||||
* reference deleted because notes legitimately cite historical issues —
|
||||
* and flips to a real class only when the behaviour is fixed.
|
||||
*
|
||||
* **Known residual gap:** object SPREAD (`getFoo({ ...opts })` where `opts` was built elsewhere
|
||||
* with an at-cap `pageSize`) and a `pageSize` passed as a bare POSITIONAL argument rather than an
|
||||
* object-literal property (`api/search.ts`'s `getAllSearchItemIds(query, pageNum, pageSize)`, the
|
||||
* api.search-allitems-paging precedent) are NOT resolvable by this scan — there is no `pageSize`
|
||||
* token inside an object-literal expression to find. A third, pre-existing gap (#684 review L2): a
|
||||
* `pageSize` whose value is a FORWARDED EXPRESSION rather than a literal or shorthand — e.g.
|
||||
* `api/collections.ts`'s `pageSize: String(pageSize)` — is a real object-literal property that
|
||||
* `scanPageSizeSites` still drops. Written down here, not silently absent: a call site introduced
|
||||
* through any of the three paths needs a human re-grep if that shape becomes common.
|
||||
*/
|
||||
|
||||
interface RegistryEntry {
|
||||
/** Path relative to `src/`, e.g. `api/paging.ts`. */
|
||||
file: string;
|
||||
kind: 'literal' | 'shorthand';
|
||||
/** The `pageSize` value's source text — an identifier (`PAGE_SIZE`) or a numeric literal. */
|
||||
value: string;
|
||||
classification: 'class-a' | 'search-bounded' | 'class-b' | 'paged-ui' | 'deviation';
|
||||
/**
|
||||
* The Gitea issue tracking a 'deviation' — REQUIRED for that class and meaningless otherwise.
|
||||
* A dedicated field rather than a `#\d+` scrape of `note` (#684): notes legitimately cite
|
||||
* historical issues, so the regex passed even with the tracking reference deleted — a test
|
||||
* satisfiable by text that has nothing to do with what it claims to check.
|
||||
*/
|
||||
issue?: number;
|
||||
note: string;
|
||||
}
|
||||
|
||||
// Keep in file order, then in the order the sites appear within the file, so a diff against the
|
||||
// discovered set is easy to read. Two entries sharing a `(file, kind, value)` identity are
|
||||
// deliberate and load-bearing: the multiset comparison requires that site to be discovered exactly
|
||||
// twice (see the identity note above).
|
||||
const REGISTRY: RegistryEntry[] = [
|
||||
{
|
||||
file: 'api/libraryBrowse.ts',
|
||||
kind: 'literal',
|
||||
value: 'boundedPageSize',
|
||||
classification: 'search-bounded',
|
||||
note:
|
||||
"searchLibraryPickerOptions — the #651 shared media-library picker that REPLACED the bounded " +
|
||||
'windows previously registered for PlaylistsScreen and RerunCollectionsScreen (both now ' +
|
||||
'correctly absent). The bound is a clamp, not a default: Math.min(pageSize, ' +
|
||||
'LIBRARY_PICKER_RESULTS) inside the helper, so a caller cannot widen it.'
|
||||
},
|
||||
{
|
||||
file: 'api/libraryBrowse.ts',
|
||||
kind: 'literal',
|
||||
value: 'boundedPageSize',
|
||||
classification: 'search-bounded',
|
||||
note:
|
||||
"searchLibraryBrowseItems — the #685 review fix that moved AddItemsDialog.runSearch's " +
|
||||
'pageSize call site out of screens/CollectionsScreen.tsx and into this shared helper (same ' +
|
||||
'gate/clamp/compile as searchLibraryPickerOptions above), so a multi-select caller needing ' +
|
||||
'full LibraryBrowseItem rows plus totalCount cannot skip the bound either. A deliberate ' +
|
||||
'second occurrence of the same (file, kind, value) identity — the multiset comparison ' +
|
||||
'requires it be discovered twice. No request is issued below LIBRARY_PICKER_MIN_QUERY (a ' +
|
||||
'blank form submit and a kind-chip click below the gate both resolve every kind to ' +
|
||||
'{items: [], totalCount: 0} via the HELPER\'s own gate — CollectionsScreen no longer keeps a ' +
|
||||
'second copy of this check; the #685 second review proved the two masked each other), the ' +
|
||||
'typed text is compiled via titleContainsQuery rather than forwarded raw, and each kind is ' +
|
||||
'bounded to one request per settled query at LIBRARY_PICKER_RESULTS rows. Unlike the #685 ' +
|
||||
'first fix, this is NOT "nothing left to hint at": the per-kind cap can still truncate the ' +
|
||||
"real match count below what totalCount reports, and AddItemsDialog now sums each kind's " +
|
||||
"totalCount and renders a 'Showing N of M' hint when it exceeds the rendered rows. The only " +
|
||||
'remaining client-side filter in AddItemsDialog (ADDABLE_TYPES.has(item.mediaType)) is inert, ' +
|
||||
'not a silent drop, and this is now enforced by the type system rather than by convention on ' +
|
||||
'BOTH ingress paths into the searched kinds — MediaKindFilter (the explicit-chip path) and ' +
|
||||
'DEFAULT_SEARCH_KINDS (the `all` fan-out) are each derived from ADDABLE_TYPE_LIST via ' +
|
||||
'`(typeof ADDABLE_TYPE_LIST)[number]`, so adding a non-addable kind to either is a compile ' +
|
||||
'error. Enforcing only the first was the #685 round-3 review finding: the hint sums ' +
|
||||
'PRE-filter totalCounts against POST-filter rows, so one unenforced ingress is enough to ' +
|
||||
'overstate it with every row of that kind dropped.'
|
||||
},
|
||||
{
|
||||
file: 'api/paging.ts',
|
||||
kind: 'shorthand',
|
||||
value: 'pageSize',
|
||||
classification: 'class-a',
|
||||
note:
|
||||
"loadAllPages's own first-page fetch. This IS the Class A completeness helper every other " +
|
||||
'bounded list uses — not a defect, the fix itself.'
|
||||
},
|
||||
{
|
||||
file: 'api/paging.ts',
|
||||
kind: 'shorthand',
|
||||
value: 'pageSize',
|
||||
classification: 'class-a',
|
||||
note: "loadAllPages's subsequent-page fetch inside the completeness loop; same helper as the entry above."
|
||||
},
|
||||
{
|
||||
file: 'builder/ChannelBuilder.tsx',
|
||||
kind: 'literal',
|
||||
value: '100',
|
||||
classification: 'class-b',
|
||||
note:
|
||||
'SeasonsDialog: TelevisionSeason browse scoped to one show (parentId), so it is bounded by ' +
|
||||
'its PARENT rather than being a picker over the whole type — which is why #651 left it as a ' +
|
||||
"single bounded page. #650 found the response's totalCount went unread; it is now surfaced " +
|
||||
"as a 'Showing the first N of M seasons' hint if a show somehow exceeds the cap."
|
||||
},
|
||||
{
|
||||
file: 'builder/libraryBrowse.ts',
|
||||
kind: 'shorthand',
|
||||
value: 'pageSize',
|
||||
classification: 'paged-ui',
|
||||
note:
|
||||
"loadCollections's per-kind fan-out (#650 fix): forwards a real pageNum/pageSize from the " +
|
||||
"caller and sums each kind's real totalCount, so the builder's Load more button (canLoadMore) " +
|
||||
'is meaningful — this is the paged-ui replacement for the original truncating implementation.'
|
||||
},
|
||||
{
|
||||
file: 'builder/libraryBrowse.ts',
|
||||
kind: 'shorthand',
|
||||
value: 'pageSize',
|
||||
classification: 'paged-ui',
|
||||
note: "loadLibraryItems's per-kind fan-out — same real pageNum/pageSize/totalCount pattern as loadCollections above."
|
||||
},
|
||||
{
|
||||
file: 'builder/SmartCollectionDialog.tsx',
|
||||
kind: 'literal',
|
||||
value: '24',
|
||||
classification: 'class-b',
|
||||
note:
|
||||
'Inline smart-query preview while authoring a query. It DOES surface the real truncation — ' +
|
||||
"the response's totalCount is rendered as a `{count} matches` badge above a 12-row slice of " +
|
||||
'the 24 fetched — which is precisely what class-b requires, so it is not search-bounded ' +
|
||||
'despite being query-driven (#684 review M1: it was the counter-example to a claim that no ' +
|
||||
'such site renders a hint).'
|
||||
},
|
||||
{
|
||||
file: 'screens/AutoTuneScreen.tsx',
|
||||
kind: 'literal',
|
||||
value: 'MEMBER_PREVIEW_SIZE',
|
||||
classification: 'class-b',
|
||||
note:
|
||||
"Channel-member preview: a real bounded window over the members, which is why it DOES render " +
|
||||
"'showing first N' once totalCount exceeds the preview size."
|
||||
},
|
||||
{
|
||||
file: 'screens/AutoTuneScreen.tsx',
|
||||
kind: 'literal',
|
||||
value: 'ADD_SOURCE_RESULTS',
|
||||
classification: 'search-bounded',
|
||||
note:
|
||||
'Tiny (8-row) debounced add-source search typeahead — the #440 picker whose compile-the-typed-' +
|
||||
'text rule #651 generalised. Nothing is windowed: a query narrows, and no hint is owed.'
|
||||
},
|
||||
{
|
||||
file: 'screens/BlockPlayoutTroubleshootingScreen.tsx',
|
||||
kind: 'shorthand',
|
||||
value: 'pageSize',
|
||||
classification: 'paged-ui',
|
||||
note:
|
||||
'Playout block history: forwards a user-adjustable `pageSize` state (persisted, backed by a ' +
|
||||
'page-size <Select>) to a real pager keyed off the response totalCount.'
|
||||
},
|
||||
{
|
||||
file: 'screens/FillerPresetsScreen.tsx',
|
||||
kind: 'literal',
|
||||
value: 'LIBRARY_BROWSE_PAGE_CAP',
|
||||
classification: 'class-b',
|
||||
note:
|
||||
'The COLLECTION-FAMILY fallback (Collection / SmartCollection / MultiCollection / ' +
|
||||
'RerunCollection / Playlist), which spa-conventions §3b explicitly excludes from search ' +
|
||||
'because GetLibraryBrowseItemsHandler LIKE-matches `query` for those types and would match a ' +
|
||||
"compiled `title:*x*` literally. Keeps the bounded page AND its truncation hint; this " +
|
||||
"screen's media-item types went to searchLibraryPickerOptions in #651."
|
||||
},
|
||||
{
|
||||
file: 'screens/LogsScreen.tsx',
|
||||
kind: 'shorthand',
|
||||
value: 'pageSize',
|
||||
classification: 'paged-ui',
|
||||
note:
|
||||
'Log listing: forwards a user-adjustable `pageSize` state (persisted, backed by a page-size ' +
|
||||
'<Select>) to a real pager keyed off the response totalCount.'
|
||||
},
|
||||
{
|
||||
file: 'screens/MediaBrowseScreen.tsx',
|
||||
kind: 'literal',
|
||||
value: 'PAGE_SIZE',
|
||||
classification: 'paged-ui',
|
||||
note: 'Library browse grid has a real page-number pager driven off the real totalCount.'
|
||||
},
|
||||
{
|
||||
file: 'screens/MediaDetailScreen.tsx',
|
||||
kind: 'literal',
|
||||
value: 'CHILD_PAGE_SIZE',
|
||||
classification: 'paged-ui',
|
||||
note: 'Season/episode child list has a real page-number pager driven off the real totalCount.'
|
||||
},
|
||||
{
|
||||
file: 'screens/SearchScreen.tsx',
|
||||
kind: 'literal',
|
||||
value: 'PAGE_SIZE',
|
||||
classification: 'paged-ui',
|
||||
note: 'Per-group search results; hasMore gated on totalCount > items.length with a load-more.'
|
||||
},
|
||||
{
|
||||
file: 'screens/TrashScreen.tsx',
|
||||
kind: 'literal',
|
||||
value: 'PAGE_SIZE',
|
||||
classification: 'paged-ui',
|
||||
note: 'Per-group trash listing; "See all N" load-more gated on totalCount > items.length.'
|
||||
},
|
||||
{
|
||||
file: 'screens/TrashScreen.tsx',
|
||||
kind: 'literal',
|
||||
value: 'PAGE_SIZE',
|
||||
classification: 'paged-ui',
|
||||
note:
|
||||
'The load-more request handler for the SAME per-group trash listing as the entry above — a ' +
|
||||
'deliberate second occurrence of one identity, which the multiset comparison requires to be ' +
|
||||
'discovered exactly twice.'
|
||||
}
|
||||
];
|
||||
|
||||
// Enumerates every source file under `src/` via Vite's `import.meta.glob` — eagerly, as raw text
|
||||
// (`query: '?raw', import: 'default'`) — INSTEAD OF Node's `fs`/`path`/`url` (#650 follow-up).
|
||||
// This is the only file under `src` that ever needed real filesystem access, and `@types/node`
|
||||
// isn't wired into `tsconfig.app.json`'s project (deliberately: it covers production browser code
|
||||
// too, and a file-local `/// <reference types="node" />` was tried and reverted — under `tsc -b`'s
|
||||
// single-program compilation it leaked Node's ambient `setTimeout` into the whole app project,
|
||||
// breaking three unrelated `window.setTimeout` mocks that expect the DOM signature). `import.meta
|
||||
// .glob` needs neither `node:fs` nor a tsconfig change: it's resolved by Vite at transform time,
|
||||
// natively available in the browser/app project, and is the idiomatic Vite/vitest way to enumerate
|
||||
// source files. Keys are POSIX paths from the project root, e.g. `/src/api/pageSizeScan.ts`.
|
||||
const rawSourceModules = import.meta.glob('/src/**/*.{ts,tsx,mts,cts}', {
|
||||
query: '?raw',
|
||||
import: 'default',
|
||||
eager: true
|
||||
}) as Record<string, string>;
|
||||
|
||||
function basename(path: string): string {
|
||||
const idx = path.lastIndexOf('/');
|
||||
return idx === -1 ? path : path.slice(idx + 1);
|
||||
}
|
||||
|
||||
// Extracted from `listSourceFiles`'s inline condition so it's independently testable (#650
|
||||
// follow-up round 4): a plant that adds a real `.mts` FILE and observes the guard notice it
|
||||
// proves the behavior exists today, but pins nothing — revert the glob back to `.ts`/`.tsx` and
|
||||
// both the real-source guard AND `pageSizeScan.test.ts`'s `.mts`/`.cts` PARSING tests stay green,
|
||||
// because this repo has no committed `.mts`/`.cts` source and `scanPageSizeSites` parses any
|
||||
// non-`.tsx` filename as plain TS regardless of extension. Testing this predicate directly, by
|
||||
// filename, is what actually regression-pins the file-discovery fix rather than depending on the
|
||||
// repo happening to contain (or not contain) a matching file. This predicate is still what the
|
||||
// glob's results are filtered THROUGH below (`listSourceFiles`) — the extension SET moved into the
|
||||
// glob literal, but discovery still runs every matched file through this same named, tested
|
||||
// function, not a second copy of the logic.
|
||||
export function isScannableSourceFileName(name: string): boolean {
|
||||
// `.mts`/`.cts` are legal TS extensions `tsconfig.app.json`'s `include` covers alongside
|
||||
// `.ts`/`.tsx` — none exist in this repo today, but the glob must not silently skip one if it
|
||||
// ever does (#650 follow-up round 3 MEDIUM finding).
|
||||
return (
|
||||
/\.(ts|tsx|mts|cts)$/.test(name) && !/\.test\.(tsx?|mts|cts)$/.test(name) && !name.endsWith('.guard.test.ts')
|
||||
);
|
||||
}
|
||||
|
||||
interface ScannableSource {
|
||||
/** Path relative to `src/`, e.g. `api/pageSizeScan.ts` — matches the REGISTRY's `file` field. */
|
||||
file: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
function listSourceFiles(): ScannableSource[] {
|
||||
const out: ScannableSource[] = [];
|
||||
for (const [key, text] of Object.entries(rawSourceModules)) {
|
||||
if (key.includes('/generated/')) {
|
||||
continue;
|
||||
}
|
||||
if (!isScannableSourceFileName(basename(key))) {
|
||||
continue;
|
||||
}
|
||||
out.push({ file: key.replace(/^\/src\//, ''), text });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface DiscoveredSite {
|
||||
file: string;
|
||||
line: number;
|
||||
column: number;
|
||||
kind: 'literal' | 'shorthand';
|
||||
value: string;
|
||||
}
|
||||
|
||||
function discoverPageSizeCallSites(): DiscoveredSite[] {
|
||||
const sites: DiscoveredSite[] = [];
|
||||
|
||||
for (const { file, text } of listSourceFiles()) {
|
||||
for (const site of scanPageSizeSites(text, file)) {
|
||||
sites.push({ file, ...site });
|
||||
}
|
||||
}
|
||||
|
||||
return sites;
|
||||
}
|
||||
|
||||
// The REGISTRY's identity: `(file, kind, value)`, with no source position — see the identity note
|
||||
// in this file's header for why the line/column were dropped. This is deliberately NOT
|
||||
// `pageSizeSiteId` (which keys on line:column and remains the SCANNER's identity, asserted over
|
||||
// fixed fixtures in `pageSizeScan.test.ts`); the two answer different questions, so they are
|
||||
// allowed to differ, and a registry entry has no position to supply anyway.
|
||||
function registryId(site: { file: string; kind: string; value: string }): string {
|
||||
return `${site.file}:${site.kind}:${site.value}`;
|
||||
}
|
||||
|
||||
// Identity and REPORT deliberately have different formats (#684 review M3). The comparison key
|
||||
// carries no position — that is the whole fix — but a bare `TrashScreen.tsx:literal:PAGE_SIZE` is
|
||||
// useless to whoever has to go find it in a file holding two such sites. So the UNREGISTERED
|
||||
// direction, which describes DISCOVERED sites and therefore does have real positions, prints them.
|
||||
// This reintroduces no churn: positions appear only in a failure message, never in a comparison.
|
||||
function describeDiscovered(sites: DiscoveredSite[], ids: string[]): string[] {
|
||||
const positions = new Map<string, string[]>();
|
||||
for (const site of sites) {
|
||||
const id = registryId(site);
|
||||
const at = positions.get(id) ?? [];
|
||||
at.push(`${site.line}:${site.column}`);
|
||||
positions.set(id, at);
|
||||
}
|
||||
return ids.map((id) => {
|
||||
const at = positions.get(id);
|
||||
// Every position sharing this identity, not just the excess one: a positionless key genuinely
|
||||
// cannot tell which occurrence is new, so the candidate set IS the honest answer. Labelled so a
|
||||
// reader does not take all of them as unregistered (#684 review L-a).
|
||||
return at && at.length > 0 ? `${id} (identity seen at: ${at.join(', ')})` : id;
|
||||
});
|
||||
}
|
||||
|
||||
// Multiset (count per identity) comparison, not plain array `.includes` membership (#650
|
||||
// follow-up M-6) — so a registry that accidentally lists the same identity twice, or a future
|
||||
// scanner change that could (in principle) emit a duplicate, is still caught rather than one
|
||||
// occurrence silently covering both.
|
||||
function toCounts(ids: string[]): Map<string, number> {
|
||||
const counts = new Map<string, number>();
|
||||
for (const id of ids) {
|
||||
counts.set(id, (counts.get(id) ?? 0) + 1);
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
// Returns entries present in `left` more times than in `right`, expanded per the excess count —
|
||||
// e.g. a `left` id appearing 3 times against 1 in `right` yields that id listed twice.
|
||||
function multisetExcess(left: Map<string, number>, right: Map<string, number>): string[] {
|
||||
const excess: string[] = [];
|
||||
for (const [id, count] of left) {
|
||||
const remaining = count - (right.get(id) ?? 0);
|
||||
for (let i = 0; i < remaining; i++) {
|
||||
excess.push(id);
|
||||
}
|
||||
}
|
||||
return excess.sort();
|
||||
}
|
||||
|
||||
// These 4 tests are BASELINE assertions about the guard's steady-state behavior against the
|
||||
// current repo snapshot — they all pass equally on the clean `b90f8a3b` commit (before this
|
||||
// round's scanner rewrite), so none of them individually PROVE this round's fixes. What actually
|
||||
// regression-pins the scanner's fixes is `pageSizeScan.test.ts` (synthetic fixtures per input
|
||||
// class, verified against the prior scanner where the review asked for it) — these 4 just confirm
|
||||
// the guard, wired to whichever scanner it currently uses, still holds over real source.
|
||||
describe('pageSize call-site guard (#650)', () => {
|
||||
it.each([
|
||||
['screens/TraktListsScreen.ts', true],
|
||||
['builder/ChannelBuilder.tsx', true],
|
||||
['builder/libraryBrowse.mts', true],
|
||||
['api/pageSizeScan.cts', true],
|
||||
['screens/TraktListsScreen.test.ts', false],
|
||||
['builder/ChannelBuilder.test.tsx', false],
|
||||
['builder/libraryBrowse.test.mts', false],
|
||||
['api/pageSizeScan.test.cts', false],
|
||||
['api/pageSizeCallSites.guard.test.ts', false],
|
||||
['api/generated/v1.ts', true], // the predicate itself is filename-only; the 'generated' DIRECTORY exclusion lives in listSourceFiles, tested separately below.
|
||||
['components.js', false],
|
||||
['data.json', false],
|
||||
['README.md', false],
|
||||
['noextension', false]
|
||||
])(
|
||||
'isScannableSourceFileName(%s) === %s — the file-discovery predicate itself, independent of ' +
|
||||
'whether the repo happens to contain a matching file (#650 follow-up round 4)',
|
||||
(name, expected) => {
|
||||
// A prior verification planted a REAL .mts file and observed the guard notice it — that
|
||||
// proved the .mts/.cts fix works today, but pinned nothing: reverting the glob back to
|
||||
// `.ts`/`.tsx` leaves both the real-source guard AND pageSizeScan.test.ts's .mts/.cts
|
||||
// PARSING tests green, since this repo has no committed .mts/.cts source and the scanner
|
||||
// parses any non-.tsx filename as plain TS regardless of extension. Asserting on the
|
||||
// predicate BY FILENAME, with no filesystem involved, is what actually regression-pins it.
|
||||
expect(isScannableSourceFileName(name)).toBe(expected);
|
||||
}
|
||||
);
|
||||
|
||||
it('scans a healthy number of source files (anti-vacuity: a broken glob must not pass on zero input)', () => {
|
||||
const files = listSourceFiles();
|
||||
expect(files.length).toBeGreaterThan(50);
|
||||
});
|
||||
|
||||
it('discovers a healthy number of pageSize call sites (anti-vacuity: a broken scan must not pass on zero matches)', () => {
|
||||
const sites = discoverPageSizeCallSites();
|
||||
expect(sites.length).toBeGreaterThan(10);
|
||||
});
|
||||
|
||||
it('matches the discovered pageSize call sites EXACTLY against the reviewed registry (not a non-empty check)', () => {
|
||||
const discovered = discoverPageSizeCallSites();
|
||||
const discoveredCounts = toCounts(discovered.map(registryId));
|
||||
const registeredCounts = toCounts(REGISTRY.map(registryId));
|
||||
|
||||
const unregistered = multisetExcess(discoveredCounts, registeredCounts);
|
||||
const stale = multisetExcess(registeredCounts, discoveredCounts);
|
||||
|
||||
// Both directions are folded into ONE assertion so a failure always shows the complete
|
||||
// picture in a single run (#650 follow-up, line-churn concern) — two sequential `expect`
|
||||
// calls would throw on the first failing direction and never evaluate/report the second.
|
||||
if (unregistered.length > 0 || stale.length > 0) {
|
||||
const report = [
|
||||
`UNREGISTERED (${unregistered.length}) — discovered pageSize call site(s) missing from the REGISTRY above:`,
|
||||
...describeDiscovered(discovered, unregistered).map((line) => ` + ${line}`),
|
||||
`STALE (${stale.length}) — REGISTRY entries no longer found as a real pageSize call site:`,
|
||||
...stale.map((id) => ` - ${id}`)
|
||||
].join('\n');
|
||||
throw new Error(report);
|
||||
}
|
||||
});
|
||||
|
||||
// A 'deviation' entry is the registry admitting a live defect rather than laundering it into a
|
||||
// compliant-looking label (#684 review H2). That is only honest if the defect is TRACKED — an
|
||||
// untracked deviation is just a defect with better manners — so the issue reference is enforced
|
||||
// here rather than left to a reviewer noticing its absence.
|
||||
it("every 'deviation' entry names the issue tracking it", () => {
|
||||
const deviations = REGISTRY.filter((entry) => entry.classification === 'deviation');
|
||||
|
||||
// No live 'deviation' entries as of #685 (the last one — CollectionsScreen.tsx's raw-50 window
|
||||
// — was fixed and reclassified 'search-bounded' above). The anti-vacuity
|
||||
// `expect(deviations.length).toBeGreaterThan(0)` this comment used to enforce is deleted
|
||||
// DELIBERATELY here, per its own instruction, rather than left to silently pass over an empty
|
||||
// list — re-add it the day a new 'deviation' entry is registered.
|
||||
|
||||
for (const entry of deviations) {
|
||||
expect(entry.issue, `${entry.file}:${entry.value} is a deviation but names no tracking issue`).toEqual(
|
||||
expect.any(Number)
|
||||
);
|
||||
expect(entry.issue!).toBeGreaterThan(0);
|
||||
}
|
||||
|
||||
// The converse, so the field cannot drift into decoration: only a deviation carries one. With
|
||||
// `deviations` currently empty, the loop above evaluates nothing — so this single bidirectional
|
||||
// assertion is what actually gives the test teeth today: it fails the moment any non-deviation
|
||||
// entry picks up an `issue` field, or a 'deviation' entry is added without one (#685 review
|
||||
// finding 7).
|
||||
expect(REGISTRY.filter((entry) => entry.issue !== undefined)).toEqual(deviations);
|
||||
});
|
||||
|
||||
// Pins the report format, not the comparison key (#684 review M3): dropping the position from
|
||||
// IDENTITY is the fix, dropping it from the failure MESSAGE was collateral damage — it left
|
||||
// `TrashScreen.tsx:literal:PAGE_SIZE` pointing at a file with two such sites.
|
||||
it('reports the discovered line:column for an unregistered site, while comparing without it', () => {
|
||||
const sites = discoverPageSizeCallSites();
|
||||
const target = sites.find((site) => site.file === 'screens/TrashScreen.tsx');
|
||||
expect(target).toBeDefined();
|
||||
|
||||
const described = describeDiscovered(sites, [registryId(target!)]);
|
||||
|
||||
expect(described[0]).toContain(`${target!.line}:${target!.column}`);
|
||||
// ...and the key it was looked up by still carries no position.
|
||||
expect(registryId(target!)).not.toContain(String(target!.line));
|
||||
});
|
||||
|
||||
it('every registry entry documents its class per docs/spa-conventions.md §3b', () => {
|
||||
for (const entry of REGISTRY) {
|
||||
expect(['class-a', 'search-bounded', 'class-b', 'paged-ui', 'deviation']).toContain(entry.classification);
|
||||
expect(entry.note.length).toBeGreaterThan(20);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,247 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { pageSizeSiteId, scanPageSizeSites, type PageSizeSite } from './pageSizeScan';
|
||||
|
||||
/**
|
||||
* Fixture test for `scanPageSizeSites` itself — NOT a scan of the real repo (that's
|
||||
* `pageSizeCallSites.guard.test.ts`). This is what actually protects the SCANNER going forward:
|
||||
* a prior hand-rolled regex/bracket-tracking version passed the guard test against unmodified
|
||||
* source at both #650 commits while still being defeated by every case below, because the guard
|
||||
* only ever exercised today's snapshot of real call sites — it never proved the scanner handles
|
||||
* the INPUT CLASSES that expose a text-level scanner's blind spots. Pinning the exact discovered
|
||||
* set against synthetic source strings closes that gap.
|
||||
*
|
||||
* Not every fixture here is a REGRESSION pin against the prior (round-1, `b90f8a3b`) bracket-
|
||||
* tracking scanner — a round-3 review found that round 1's simple `pageSize:\s*value` regex
|
||||
* already handled a bare URL-string or a bare `??` context correctly on its own (a `//` inside a
|
||||
* string, or the token immediately before `{`, only mattered to round 1's OWN heuristics, not to
|
||||
* a plain regex match). Those two are labelled CONTRACT fixtures below — they pin the documented
|
||||
* behavior going forward, not a fix. The fixtures that genuinely fail against round 1 (verified)
|
||||
* are: the string CONTAINING the literal text `pageSize: 100`, the template-literal
|
||||
* interpolation, the same-line ternary identity/multiplicity, the JSX shorthand container,
|
||||
* parameter destructuring, nested destructuring, and the type-literal declaration — plus the
|
||||
* combined multi-case fixture, which fails round 1 for several of those reasons at once.
|
||||
*/
|
||||
|
||||
function ids(sites: PageSizeSite[]): string[] {
|
||||
return sites.map(pageSizeSiteId);
|
||||
}
|
||||
|
||||
describe('scanPageSizeSites', () => {
|
||||
it('finds a literal pageSize: property in a plain object-literal call argument', () => {
|
||||
const source = `getFoo({ pageSize: 100, query });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:literal:100']);
|
||||
});
|
||||
|
||||
it('finds the ES6 shorthand pageSize property in a plain object-literal call argument', () => {
|
||||
const source = `getFoo({ pageSize, query });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:shorthand:pageSize']);
|
||||
});
|
||||
|
||||
it('is NOT fooled by a "//" inside a string literal — CONTRACT fixture, not a round-1 regression pin (M-3)', () => {
|
||||
// NOTE: round 1's unconditional literal regex (`pageSize:\s*(\d+|identifier)`) already
|
||||
// matched this exact input correctly on its own — a `//` inside a string never confused THAT
|
||||
// narrower pattern. This pins the AST scanner's documented contract going forward; it is the
|
||||
// COMBINED multi-case fixture below (and the M-3-shaped case buried inside it — a literal
|
||||
// `//` immediately preceding a real call site on the SAME conceptual scan) that actually
|
||||
// fails against round 1's comment-stripping step, not this input in isolation.
|
||||
const source = [`const endpoint = 'https://example.test';`, `getFoo({ pageSize: 100 });`, ''].join('\n');
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['2:10:literal:100']);
|
||||
});
|
||||
|
||||
it('does NOT match a string literal that merely CONTAINS the text "pageSize: 100" (L-7)', () => {
|
||||
const source = `const label = "pageSize: 100";\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('finds an object literal passed inside a template-literal interpolation (M-5)', () => {
|
||||
const source = 'const url = `${await getFoo({ pageSize })}`;\n';
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:31:shorthand:pageSize']);
|
||||
});
|
||||
|
||||
it('finds an object literal in each branch of a ternary, even on the SAME line (M-4, M-6)', () => {
|
||||
const source = `return ok ? getA({ pageSize }) : getB({ pageSize });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
// Two distinct occurrences on one line get distinct identities (different columns) — a
|
||||
// single registry entry cannot silently cover both.
|
||||
expect(ids(sites)).toEqual(['1:20:shorthand:pageSize', '1:41:shorthand:pageSize']);
|
||||
expect(sites[0].column).not.toBe(sites[1].column);
|
||||
});
|
||||
|
||||
it('finds an object literal on the right-hand side of ?? — CONTRACT fixture, not a round-1 regression pin (M-4)', () => {
|
||||
// NOTE: like the URL fixture above, round 1's literal-form regex already matched this exact
|
||||
// `pageSize: 50` text correctly on its own — `??` doesn't change what characters precede the
|
||||
// match on the line. This pins the documented contract, not a round-1 regression.
|
||||
const source = `getFoo(options ?? { pageSize: 50 });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:21:literal:50']);
|
||||
});
|
||||
|
||||
it('finds an object literal inside a JSX expression container attribute (M-4)', () => {
|
||||
const source = `const el = <Component options={{ pageSize }} />;\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.tsx');
|
||||
expect(ids(sites)).toEqual(['1:34:shorthand:pageSize']);
|
||||
});
|
||||
|
||||
it('does NOT match a parameter destructuring pattern (L-7)', () => {
|
||||
const source = `function f({ pageSize }: { pageSize: number }) {}\n`;
|
||||
// The destructured PARAMETER `{ pageSize }` is an ObjectBindingPattern, not an
|
||||
// ObjectLiteralExpression — excluded by node kind. Its TYPE annotation `{ pageSize: number }`
|
||||
// is a TypeLiteral (PropertySignature), also excluded by node kind — never an object literal.
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT match a nested destructuring pattern (L-7)', () => {
|
||||
const source = `const { nested: { pageSize } } = input;\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT match a type-literal declaration (L-7)', () => {
|
||||
const source = `type P = { pageSize: 100 };\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT match an interface property declaration', () => {
|
||||
const source = `interface Params {\n pageSize?: number;\n}\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT match a forwarded call expression (a dynamic passthrough, not a fixed value)', () => {
|
||||
const source = `getFoo({ pageSize: String(pageSize) });\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('is not confused by a pageSize reference inside a comment', () => {
|
||||
const source = [`// pageSize: 999 — this is just prose, not code`, `getFoo({ query });`, ''].join('\n');
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('is not confused by a pageSize reference inside a block/JSDoc comment', () => {
|
||||
const source = ['/**', ' * Uses `pageSize` under the hood — see also `{ pageSize: 100 }`.', ' */', 'getFoo({ query });', ''].join(
|
||||
'\n'
|
||||
);
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT match a React dependency array containing pageSize', () => {
|
||||
const source = `useCallback(load, [pageNum, pageSize, sortField]);\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('finds a const-identifier literal value (not just a numeric literal)', () => {
|
||||
const source = `getFoo({ pageSize: LIBRARY_BROWSE_PAGE_CAP });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:literal:LIBRARY_BROWSE_PAGE_CAP']);
|
||||
});
|
||||
|
||||
it('covers every case above together in one multi-line fixture and pins the exact discovered set', () => {
|
||||
const source = [
|
||||
`const endpoint = 'https://example.test';`, // M-3: not a comment
|
||||
`const label = "pageSize: 100";`, // L-7: string contents, not code
|
||||
`// pageSize: 999 in a line comment`, // not code
|
||||
`/** block comment mentioning \`pageSize\` */`, // not code
|
||||
`type P = { pageSize: 100 };`, // L-7: type literal, not a value
|
||||
`interface Q { pageSize?: number; }`, // not a value
|
||||
`function f({ pageSize }: { pageSize: number }) {}`, // L-7: destructuring + its type
|
||||
`const { nested: { pageSize } } = input;`, // L-7: nested destructuring
|
||||
`useCallback(load, [pageNum, pageSize]);`, // dependency array, not an object literal
|
||||
`getFoo({ pageSize: String(pageSize) });`, // forwarded call, not a fixed value
|
||||
`getFoo({ pageSize: 100 });`, // REAL: literal
|
||||
`getBar({ pageSize });`, // REAL: shorthand
|
||||
`getBaz(options ?? { pageSize: 50 });`, // REAL: ?? context (M-4)
|
||||
`const url = \`\${await getQux({ pageSize })}\`;`, // REAL: template interpolation (M-5)
|
||||
`return ok ? getA({ pageSize }) : getB({ pageSize });` // REAL x2: ternary, same line (M-4/M-6)
|
||||
].join('\n');
|
||||
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual([
|
||||
'11:10:literal:100',
|
||||
'12:10:shorthand:pageSize',
|
||||
'13:21:literal:50',
|
||||
'14:31:shorthand:pageSize',
|
||||
'15:20:shorthand:pageSize',
|
||||
'15:41:shorthand:pageSize'
|
||||
]);
|
||||
// Anti-vacuity: the fixture packs in 10 non-matching traps ahead of the 6 real sites — a
|
||||
// scanner that matched everything (or nothing) would fail this count, not just the ids above.
|
||||
expect(sites.length).toBe(6);
|
||||
});
|
||||
|
||||
it('scans .tsx source using the TSX script kind (JSX does not parse under plain .ts rules)', () => {
|
||||
const source = `export function C() {\n return <div data={{ pageSize: 10 }} />;\n}\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.tsx');
|
||||
expect(ids(sites)).toEqual(['2:23:literal:10']);
|
||||
});
|
||||
|
||||
// ---- round-3 MEDIUM finding: transparent TS wrappers around the initializer -----------------
|
||||
|
||||
it('finds a literal wrapped in "as const" (transparent to the runtime value)', () => {
|
||||
const source = `getFoo({ pageSize: 100 as const });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:literal:100']);
|
||||
});
|
||||
|
||||
it('finds a literal wrapped in "satisfies number" (transparent to the runtime value)', () => {
|
||||
const source = `getFoo({ pageSize: 100 satisfies number });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:literal:100']);
|
||||
});
|
||||
|
||||
it('finds a parenthesized literal', () => {
|
||||
const source = `getFoo({ pageSize: (100) });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:literal:100']);
|
||||
});
|
||||
|
||||
it('finds a const identifier through a chain of "as"/"satisfies"/parens wrappers', () => {
|
||||
const source = `getFoo({ pageSize: ((LIBRARY_BROWSE_PAGE_CAP as number) satisfies number) });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:literal:LIBRARY_BROWSE_PAGE_CAP']);
|
||||
});
|
||||
|
||||
it('still rejects a forwarded call expression even when wrapped in "as"', () => {
|
||||
const source = `getFoo({ pageSize: String(pageSize) as string });\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
// ---- round-3 MEDIUM finding: non-Identifier property names -----------------------------------
|
||||
|
||||
it('finds a quoted string property key ("pageSize": 100)', () => {
|
||||
const source = `getFoo({ 'pageSize': 100 });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:literal:100']);
|
||||
});
|
||||
|
||||
it('finds a statically-resolvable computed property key (["pageSize"]: 100)', () => {
|
||||
const source = `getFoo({ ['pageSize']: 100 });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.ts');
|
||||
expect(ids(sites)).toEqual(['1:10:literal:100']);
|
||||
});
|
||||
|
||||
it('does NOT match a computed property key that cannot be resolved statically', () => {
|
||||
const source = `const key = getKey();\ngetFoo({ [key]: 100 });\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT match a quoted string key for a DIFFERENT property name', () => {
|
||||
const source = `getFoo({ 'pageSizeLimit': 100 });\n`;
|
||||
expect(scanPageSizeSites(source, 'fixture.ts')).toEqual([]);
|
||||
});
|
||||
|
||||
// ---- round-3 MEDIUM finding: .mts/.cts are never silently skipped -----------------------------
|
||||
|
||||
it('scans .mts source (parses as plain TS, no JSX grammar)', () => {
|
||||
const source = `export function loadPage() {\n return getFoo({ pageSize: 100 });\n}\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.mts');
|
||||
expect(ids(sites)).toEqual(['2:19:literal:100']);
|
||||
});
|
||||
|
||||
it('scans .cts source (parses as plain TS, no JSX grammar)', () => {
|
||||
const source = `getFoo({ pageSize });\n`;
|
||||
const sites = scanPageSizeSites(source, 'fixture.cts');
|
||||
expect(ids(sites)).toEqual(['1:10:shorthand:pageSize']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import * as ts from 'typescript';
|
||||
|
||||
/**
|
||||
* #650 follow-up: an AST-based scanner for every `pageSize` property that appears inside a real
|
||||
* object LITERAL expression. Extracted into its own module so both the enumerating guard
|
||||
* (`pageSizeCallSites.guard.test.ts`, which scans the real repo) and a fixture test
|
||||
* (`pageSizeScan.test.ts`, which scans synthetic source strings and does NOT touch the repo) can
|
||||
* exercise the exact same scanning logic.
|
||||
*
|
||||
* A prior hand-rolled regex/bracket-tracking version of this scan was replaced after a review
|
||||
* found it defeated by comments-in-strings, template-literal interpolations, ternary/`??`
|
||||
* contexts, JSX containers, and same-line duplicates — each a DIFFERENT input class a text-level
|
||||
* lexer has to special-case one at a time. The TypeScript compiler API sidesteps the whole
|
||||
* category: comments and string/template CONTENTS are trivia/literal text the parser never
|
||||
* revisits as code, and a real object-literal expression (`ObjectLiteralExpression`) is a
|
||||
* structurally different AST node from a type literal (`type X = { pageSize: number }`,
|
||||
* `PropertySignature` inside a `TypeLiteralNode`/`InterfaceDeclaration`) or a destructuring
|
||||
* pattern (`ObjectBindingPattern`, e.g. `function f({ pageSize }) {}` or
|
||||
* `const { pageSize } = x`) — so those are excluded by NODE KIND, not by a preceding-character
|
||||
* heuristic that can be fooled by an unrelated `{`/`(`/`,`.
|
||||
*
|
||||
* A round-3 review found the AST version still had its own — smaller, but real — false
|
||||
* negatives: an initializer wrapped in a transparent TS construct (`pageSize: 100 as const`,
|
||||
* `pageSize: 100 satisfies number`, `pageSize: (100)`) was rejected outright because only a bare
|
||||
* `NumericLiteral`/`Identifier` was checked; a property written as a quoted string key
|
||||
* (`'pageSize': 100`) or a statically-resolvable computed key (`['pageSize']: 100`) was missed
|
||||
* because only an `Identifier` name was checked. `unwrapTransparentExpression` and
|
||||
* `isPageSizePropertyName` close both — see their doc comments below. Genuinely UNRESOLVABLE
|
||||
* cases remain out of reach on purpose and are documented as a residual gap where this scanner is
|
||||
* actually used (`pageSizeCallSites.guard.test.ts`'s module doc comment): object SPREAD
|
||||
* (`getFoo({ ...opts })` built elsewhere) and a `pageSize` passed as a bare POSITIONAL argument
|
||||
* rather than an object-literal property at all.
|
||||
*/
|
||||
|
||||
export interface PageSizeSite {
|
||||
/** 1-based source line of the `pageSize` property (name), matching editor line numbers. */
|
||||
line: number;
|
||||
/** 1-based source column of the `pageSize` property (name). */
|
||||
column: number;
|
||||
kind: 'literal' | 'shorthand';
|
||||
/**
|
||||
* For `kind: 'literal'`: the numeric-literal text or the referenced const identifier's name.
|
||||
* For `kind: 'shorthand'`: always the literal string `'pageSize'` (the shorthand form only ever
|
||||
* forwards whatever `pageSize` binding is in scope — there is no separate "value" to name).
|
||||
*/
|
||||
value: string;
|
||||
}
|
||||
|
||||
function scriptKindFor(fileName: string): ts.ScriptKind {
|
||||
// `.mts`/`.cts` parse as plain TS (no JSX support), same as `.ts` — only `.tsx` needs the JSX
|
||||
// grammar. `tsconfig.app.json`'s `include` covers all of `src`, and `.mts`/`.cts` are legal
|
||||
// TS extensions the guard's file-discovery glob must not silently skip even though none exist
|
||||
// in this repo today (#650 follow-up round 3 MEDIUM finding).
|
||||
return fileName.endsWith('.tsx') ? ts.ScriptKind.TSX : ts.ScriptKind.TS;
|
||||
}
|
||||
|
||||
// Unwraps TS constructs that are transparent to the runtime VALUE but would otherwise hide a
|
||||
// numeric literal / identifier from a naive node-kind check: `expr as T`, `expr satisfies T`,
|
||||
// and `(expr)`. `pageSize: 100 as const` and `pageSize: 100 satisfies number` are both real
|
||||
// fixed-100 call sites; only the TS type-checking wrapper differs (#650 follow-up round 3 MEDIUM).
|
||||
function unwrapTransparentExpression(node: ts.Expression): ts.Expression {
|
||||
let current = node;
|
||||
for (;;) {
|
||||
if (ts.isParenthesizedExpression(current)) {
|
||||
current = current.expression;
|
||||
} else if (ts.isAsExpression(current)) {
|
||||
current = current.expression;
|
||||
} else if (ts.isSatisfiesExpression(current)) {
|
||||
current = current.expression;
|
||||
} else {
|
||||
return current;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A property name is `pageSize` whether written as a plain identifier (`pageSize: 100`), a
|
||||
// quoted string key (`'pageSize': 100`), or a computed key that's STATICALLY a `'pageSize'`
|
||||
// string literal (`['pageSize']: 100`) — all three compile to the identical property, so all
|
||||
// three are real call sites (#650 follow-up round 3 MEDIUM). A computed key that ISN'T a literal
|
||||
// (e.g. `[dynamicKeyVar]: 100`) can't be resolved statically and is correctly left unmatched.
|
||||
function isPageSizePropertyName(name: ts.PropertyName): boolean {
|
||||
if (ts.isIdentifier(name) || ts.isStringLiteral(name)) {
|
||||
return name.text === 'pageSize';
|
||||
}
|
||||
if (ts.isComputedPropertyName(name)) {
|
||||
const expr = unwrapTransparentExpression(name.expression);
|
||||
return ts.isStringLiteral(expr) && expr.text === 'pageSize';
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function scanPageSizeSites(sourceText: string, fileName: string): PageSizeSite[] {
|
||||
const sourceFile = ts.createSourceFile(fileName, sourceText, ts.ScriptTarget.Latest, true, scriptKindFor(fileName));
|
||||
const sites: PageSizeSite[] = [];
|
||||
|
||||
function positionOf(node: ts.Node): { line: number; column: number } {
|
||||
const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile));
|
||||
return { line: line + 1, column: character + 1 };
|
||||
}
|
||||
|
||||
function visit(node: ts.Node): void {
|
||||
if (ts.isObjectLiteralExpression(node)) {
|
||||
for (const property of node.properties) {
|
||||
if (ts.isPropertyAssignment(property) && isPageSizePropertyName(property.name)) {
|
||||
const initializer = unwrapTransparentExpression(property.initializer);
|
||||
// Only a numeric literal or a bare identifier (a const/variable reference) counts as a
|
||||
// fixed value baked into THIS call site. A forwarded expression — `String(pageSize)`, a
|
||||
// ternary, a template, a function call — is a dynamic passthrough of whatever the
|
||||
// caller supplied, not a literal this site chose; it is deliberately not recorded here
|
||||
// (see the module doc comment on `loadAllPages`/positional-argument residual gaps).
|
||||
if (ts.isNumericLiteral(initializer) || ts.isIdentifier(initializer)) {
|
||||
const { line, column } = positionOf(property.name);
|
||||
sites.push({ line, column, kind: 'literal', value: initializer.text });
|
||||
}
|
||||
} else if (ts.isShorthandPropertyAssignment(property) && property.name.text === 'pageSize') {
|
||||
const { line, column } = positionOf(property.name);
|
||||
sites.push({ line, column, kind: 'shorthand', value: 'pageSize' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ts.forEachChild(node, visit);
|
||||
}
|
||||
|
||||
visit(sourceFile);
|
||||
return sites.sort((a, b) => (a.line === b.line ? a.column - b.column : a.line - b.line));
|
||||
}
|
||||
|
||||
export function pageSizeSiteId(site: { line: number; column: number; kind: string; value: string }): string {
|
||||
return `${site.line}:${site.column}:${site.kind}:${site.value}`;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user