test(491): restore per-test databases; keep the pool clear (fixture determinism)
Second CI failure of this fixture, and my previous fix caused it: collapsing to one shared database traded isolation for a wipe that has to succeed, and when it silently did not, the second test seeded onto the first's rows and failed with 'Duplicate entry 1 for key LibraryPath.PRIMARY' in 539ms - too fast to have re-run the migration chain, i.e. MigrateAsync no-opped against an already-current __EFMigrationsHistory. The coordinator's read was right and I verified it rather than assuming: the leak came from never clearing pools, not from names being unique. Harness against a real 8.4 server, unique database name per iteration WITH ClearPoolAsync on that connection string: 0 leaked threads over 30 iterations. So isolation costs nothing and the shared name was solving a problem pool-clearing already solved. Restore a fresh etv491_<guid> database per test, never created out of band (the test's own MigrateAsync(PreviousMigration) creates it, keeping EF the single owner of the schema), dropped in TearDown via the guarded EnsureDeletedAsync and followed by ClearPoolAsync on that exact connection string. The stale-session hazard from the first failure needs the connection string to be REUSED after the drop, which a never-repeated name makes impossible; clearing the pool is the belt to that brace and closes the leak. Verified: 10 consecutive runs 10/10 green, server threads flat at 2 and zero leftover schemas throughout; each test run ALONE twice; both tests with the order REVERSED - order-independence being the evidence this failure would have needed, since contamination is invisible when a test runs first. Negative control re-confirmed after the change: restoring COLLATE utf8mb4_bin still fails with survivors [1,4,5,6,7,9] vs [1,4,5,6,7,9,10], then restored and green again. No Retry anywhere. One shared fixture body across providers; still fail-closed under ETV_REQUIRE_MYSQL_TESTS. Refs #488 #308 fix #491
This commit is contained in:
@@ -48,7 +48,6 @@ public class LibraryFolderDedupeMigrationTests(TestProvider provider)
|
||||
private const string PreviousMigration = "Add_Channel_Origin";
|
||||
|
||||
private const string MySqlConnectionVariable = "ETV_TEST_MYSQL_CONNECTION";
|
||||
private const string MySqlDatabaseName = "etv491_dedupe";
|
||||
private const string MySqlRequiredVariable = "ETV_REQUIRE_MYSQL_TESTS";
|
||||
|
||||
private string _databasePath = null!;
|
||||
@@ -97,11 +96,12 @@ public class LibraryFolderDedupeMigrationTests(TestProvider provider)
|
||||
Assert.Ignore($"{message} Set it to run this locally.");
|
||||
}
|
||||
|
||||
// A database of our own, NOT the one the surrounding CI step migrates. The name is FIXED rather
|
||||
// than per-test: see ResetMySqlDatabase for why that matters. (The mysql service is per CI job, so
|
||||
// a fixed name cannot collide with another job.)
|
||||
// A database of our own, NOT the one the surrounding CI step migrates, and a FRESH one per test:
|
||||
// isolation by construction. A name that has never been used cannot contain another test's rows,
|
||||
// so no wipe has to succeed for the fixture to be correct. It is not created here — the test's own
|
||||
// MigrateAsync(PreviousMigration) creates it, which keeps EF the single owner of the schema.
|
||||
_mySqlConnectionString =
|
||||
new MySqlConnectionStringBuilder(baseConnectionString) { Database = MySqlDatabaseName }
|
||||
new MySqlConnectionStringBuilder(baseConnectionString) { Database = $"etv491_{Guid.NewGuid():N}" }
|
||||
.ConnectionString;
|
||||
|
||||
TvContext.IsSqlite = false;
|
||||
@@ -114,41 +114,42 @@ public class LibraryFolderDedupeMigrationTests(TestProvider provider)
|
||||
ServerVersion serverVersion = ServerVersion.AutoDetect(_mySqlConnectionString);
|
||||
_seedOptions = MySqlOptions(serverVersion);
|
||||
_migrateOptions = MySqlOptions(serverVersion);
|
||||
|
||||
await ResetMySqlDatabase();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drop the database so each test starts from nothing, and clear its connection pool.
|
||||
/// Drop this test's database and clear the connection pool that was keyed to it.
|
||||
/// <para>
|
||||
/// Both halves matter, and the first version of this fixture got both wrong — it created a
|
||||
/// per-test <c>etv491_<guid></c> database out of band and dropped it in
|
||||
/// <see cref="TearDown" />, which failed intermittently in CI with
|
||||
/// <c>Unknown database 'etv491_…'</c> at connect time. Measured against a real 8.4 server:
|
||||
/// Both halves are required, and earlier revisions of this fixture each got one wrong. Measured
|
||||
/// against a real 8.4 server:
|
||||
/// </para>
|
||||
/// <list type="number">
|
||||
/// <item>
|
||||
/// A pooled session survives <c>DROP DATABASE</c> and is handed straight back on the next
|
||||
/// open — the connection looks healthy but its default schema no longer exists, so whether
|
||||
/// a caller sees success or <c>Unknown database</c> depends on whether the pool reuses a
|
||||
/// session or opens a fresh one (a fresh handshake names the dropped schema and fails).
|
||||
/// That is the non-determinism. <c>ClearPool</c> after every drop removes it.
|
||||
/// <b>A pooled session outlives <c>DROP DATABASE</c>.</b> Reopening the dropped database's
|
||||
/// connection string succeeds — MySqlConnector hands back the still-alive session whose
|
||||
/// default schema is gone — so whether a later caller sees success or <c>Unknown database</c>
|
||||
/// depends on whether the pool reuses that session or opens a fresh one (a fresh handshake
|
||||
/// names the dropped schema and fails 1049). <c>ClearPool</c> after the drop removes it.
|
||||
/// Note this hazard needs the connection string to be REUSED after the drop, which a
|
||||
/// never-repeated database name already makes impossible; clearing the pool is the belt to
|
||||
/// that brace, and closes the leak below.
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// A per-test database name means a per-test connection string, and MySqlConnector keys
|
||||
/// pools by connection string — so every test created a NEW pool that was never cleared,
|
||||
/// leaking ~1 server thread per iteration (measured). Left to run, that exhausts
|
||||
/// <c>max_connections</c> (151 by default) outright. A fixed name means exactly one pool.
|
||||
/// <b>An uncleared pool leaks a server connection per test.</b> MySqlConnector keys pools by
|
||||
/// connection string, so a fresh database name means a fresh pool; left uncleared it leaked
|
||||
/// ~1 server thread per iteration and eventually exhausted <c>max_connections</c>.
|
||||
/// <c>ClearPoolAsync</c> on that exact connection string fixes it completely — measured at
|
||||
/// 0 leaked threads over 30 iterations — so per-test isolation costs nothing. A previous
|
||||
/// revision instead collapsed to one shared database to stop the leak; that traded isolation
|
||||
/// for a wipe that has to succeed, and when it silently did not, the second test seeded on
|
||||
/// top of the first's rows and failed with a duplicate primary key.
|
||||
/// </item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// EF owns the drop: <c>EnsureDeletedAsync</c> is guarded (a no-op when the database is absent,
|
||||
/// where a raw <c>DROP DATABASE</c> on an unguarded connection is not) and it goes through the
|
||||
/// same connection string EF will migrate with. Dropping at the START of a test, never at the
|
||||
/// end, means no test ever leaves a live pool pointing at a database it just destroyed.
|
||||
/// unlike a raw <c>DROP DATABASE</c>) and uses the same connection string EF migrated with.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private async Task ResetMySqlDatabase()
|
||||
private async Task DropMySqlDatabase()
|
||||
{
|
||||
await using (TvContext context = MigrateContext())
|
||||
{
|
||||
@@ -159,34 +160,27 @@ public class LibraryFolderDedupeMigrationTests(TestProvider provider)
|
||||
await MySqlConnection.ClearPoolAsync(probe);
|
||||
}
|
||||
|
||||
// Note there is deliberately NO MySql branch here: the next test's SetUp resets that database instead.
|
||||
// Dropping it in TearDown is exactly what made this fixture non-deterministic, because it destroys a
|
||||
// schema the connection pool is still holding live sessions for. See ResetMySqlDatabase.
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
public async Task TearDown()
|
||||
{
|
||||
if (provider is not TestProvider.Sqlite)
|
||||
if (provider is TestProvider.Sqlite)
|
||||
{
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
foreach (string path in new[] { _databasePath, $"{_databasePath}-wal", $"{_databasePath}-shm" })
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
foreach (string path in new[] { _databasePath, $"{_databasePath}-wal", $"{_databasePath}-shm" })
|
||||
if (_mySqlConnectionString is not null)
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[OneTimeTearDown]
|
||||
public async Task OneTimeTearDown()
|
||||
{
|
||||
// Tidy up after the whole fixture, once nothing can open the database again.
|
||||
if (provider is TestProvider.MySql && _mySqlConnectionString is not null)
|
||||
{
|
||||
await ResetMySqlDatabase();
|
||||
await DropMySqlDatabase();
|
||||
_mySqlConnectionString = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user