diff --git a/.gitea/workflows/docker-build.yml b/.gitea/workflows/docker-build.yml index 6a06566dd..7940ac43b 100644 --- a/.gitea/workflows/docker-build.yml +++ b/.gitea/workflows/docker-build.yml @@ -169,7 +169,12 @@ jobs: # service container above. MySql__ConnectionString maps to config key "MySql:ConnectionString". - name: MySql — model drift + apply all migrations to a fresh DB env: - MySql__ConnectionString: "Server=mysql;Port=3306;Database=ersatztv_migrations;Uid=root;Pwd=ersatztv;" + # DefaultCommandTimeout is raised from MySqlConnector's 30s default: replaying every + # migration to a fresh DB issues DDL commands that can exceed 30s when two migration jobs + # share a runner host (each spins its own mysql:8.4 service) and starve each other. That + # contention produced both "Command Timeout expired" and mid-replay connection drops + # (MySqlEndOfStreamException) — neither is a model problem. See #13 / #236. + MySql__ConnectionString: "Server=mysql;Port=3306;Database=ersatztv_migrations;Uid=root;Pwd=ersatztv;DefaultCommandTimeout=300;" run: | set -euo pipefail export PATH="$PATH:$HOME/.dotnet/tools" @@ -178,8 +183,23 @@ jobs: --context TvContext --startup-project ErsatzTV --project ErsatzTV.Infrastructure.MySql -- --provider MySql echo "::endgroup::" echo "::group::MySql apply all migrations to a fresh DB" - dotnet ef database update --no-build --configuration Release \ - --context TvContext --startup-project ErsatzTV --project ErsatzTV.Infrastructure.MySql -- --provider MySql + # Retry the apply: under concurrent-runner MySQL contention the server can drop the + # connection mid-replay. Each attempt resumes from __EFMigrationsHistory (EF wraps each + # migration in its own transaction, so an interrupted migration rolls back cleanly and the + # retry continues from the last committed one) — so this only papers over infra flakiness, + # never a real migration failure, which fails deterministically on every attempt. + attempt=1 + max=3 + until dotnet ef database update --no-build --configuration Release \ + --context TvContext --startup-project ErsatzTV --project ErsatzTV.Infrastructure.MySql -- --provider MySql; do + if [ "$attempt" -ge "$max" ]; then + echo "MySql apply failed after ${max} attempts" >&2 + exit 1 + fi + echo "MySql apply attempt ${attempt} failed (likely runner MySQL contention); retrying in 15s..." >&2 + attempt=$((attempt + 1)) + sleep 15 + done echo "::endgroup::" build: diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 91d89589b..446794688 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -245,6 +245,20 @@ outside a transaction and warn at startup; they can't be rolled back mid-migrati migrations carefully (this is part of what motivated the apply-to-fresh check before the prod cutover, server-management#481). +**Resilience — the MySql apply is retried (concurrent-runner contention, not a model bug)**: both +runners (`ci-runner` VM 127 + `bumblebee-runner`) serve `ubuntu-latest`, and when two migration jobs +land on the **same host at once** (common when several PRs push together), each spins its own +`mysql:8.4` service container and they starve each other — producing intermittent `Command Timeout +expired` **or** mid-replay `MySqlEndOfStreamException` (dropped connection) on the MySql +apply-to-fresh-DB step. This is pure infra flakiness — `has-pending-model-changes` (the actual model +check) still passes, and the *same commit* passes on a quieter host. The job hardens against it two +ways: the connection string sets `DefaultCommandTimeout=300` (up from MySqlConnector's 30s default), +and the apply is wrapped in a **3× retry** that resumes from `__EFMigrationsHistory` (EF commits each +migration in its own transaction, so an interrupted one rolls back and the retry continues). A *real* +migration failure fails deterministically on every attempt, so the retry never masks it. If a run +still flakes past the retry, re-trigger (Gitea has no rerun API on this version — push, or the run +drains); don't treat a lone MySql-apply red as a code problem without checking the failure mode. + ## Pre-commit hooks (web/) The repo uses **husky** git hooks (installed via `web/`'s **lint-staged** + npm) to catch