ersatztv/ErsatzTV.Tests/Integration/SchedulingCollectionColumnNullTests.cs

using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Domain.Scheduling;
using ErsatzTV.Infrastructure;
using ErsatzTV.Infrastructure.Data;
using ErsatzTV.Infrastructure.MySql.Data;
using ErsatzTV.Infrastructure.Sqlite.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using MySqlConnector;
using NUnit.Framework;
using Shouldly;

namespace ErsatzTV.Tests.Integration;

/// <summary>
///     ersatztv#823 — the question ersatztv#701 split out rather than answered: can a runtime null reach
///     one of the SIX collection-valued scalar columns (<c>DaysOfMonth</c>, <c>MonthsOfYear</c>,
///     <c>DaysOfWeek</c> on <see cref="ProgramScheduleAlternate" /> and <see cref="PlayoutTemplate" />)?
///     <para>
///         <b>Measured, not reasoned about</b>, because the reasoning available beforehand pointed the
///         wrong way. The two converters differ on their read side —
///         <c>IntCollectionValueConverter</c> maps null-or-blank to <c>Array.Empty&lt;int&gt;()</c> while
///         <c>EnumCollectionJsonValueConverter</c> would dereference the result of
///         <c>JsonConvert.DeserializeObject</c> — so the expectation was that the two behave differently
///         on a NULL row. They do not: <b>EF does not invoke a value converter for a NULL column at
///         all</b>, so all six materialize as CLR <c>null</c> and the int converter's null-to-empty
///         branch is dead on this path. That is the measurement this fixture pins.
///     </para>
///     <para>
///         It also pins the WRITE half: assigning <c>null</c> to one of the six and calling
///         <c>SaveChanges</c> SUCCEEDS and stores SQL NULL — the converter is skipped on the way out too.
///         State that precisely, because the overclaim is tempting: this is a property of the CODE, not a
///         live caller. NO caller supplies a null today — every production construction of
///         <c>ReplacePlayoutAlternateSchedule</c> / <c>ReplacePlayoutTemplate</c> goes through the HTTP
///         request records, which normalize with <c>?? []</c>. What makes it a latent gun is that
///         <c>ReplacePlayoutAlternateScheduleItemsHandler</c> / <c>ReplacePlayoutTemplateItemsHandler</c>
///         assign the command value straight onto the entity, so nothing in the write path itself refuses.
///     </para>
///     <para>
///         The LEGACY route is narrower than "the columns are nullable", and conflating the two is the easy
///         error: all six are <c>nullable: true</c> on both providers, but five of the six were present at
///         <c>CreateTable</c>, so a NULL there still needs code to write one. EXACTLY ONE case is
///         code-path-free — SQLite's <c>20240113140741_Add_PlayoutTemplate_DaysOfMonth</c> is an
///         <c>AddColumn</c> with <c>nullable: true</c> and NO <c>defaultValue</c>, so <c>PlayoutTemplate</c>
///         rows inserted before it hold NULL, and by construction those rows had no day-of-month
///         restriction. On MySQL <c>PlayoutTemplate</c> arrived whole in
///         <c>20240114034944_Add_BlockScheduling</c>, so there is no code-path-free NULL for any of the six
///         there. This fixture manufactures its NULL with a raw <c>UPDATE</c>, which is a code path — it
///         measures MATERIALIZATION, and the legacy route above is established by reading the migrations.
///     </para>
///     <para>
///         What the selector then does with it is pinned by
///         <c>ErsatzTV.Core.Tests.Scheduling.AlternateScheduleSelectorTests.GetScheduleForDate_NullCollections</c>:
///         unguarded, <c>.Contains</c> throws <see cref="NullReferenceException" />.
///     </para>
///     <para>
///         Runs against BOTH providers from ONE fixture body, because the question is about provider
///         materialization and a SQLite-only answer would not have settled it. MySQL needs a live server,
///         supplied via <c>ETV_TEST_MYSQL_CONNECTION</c>; without it the MySQL fixture <b>ignores</b> — a
///         visible skip, never a silent pass. <c>ETV_REQUIRE_MYSQL_TESTS</c> turns that skip into a hard
///         failure for a runner that is supposed to have one. This mirrors
///         <see cref="LibraryFolderDedupeMigrationTests" />, which established the pattern.
///     </para>
///     <para>
///         <b>Cost, stated.</b> Each test migrates a FRESH database (per-test isolation by construction —
///         ersatztv#491 measured what a shared database and a wipe-that-must-succeed cost). Replaying every
///         migration is the expensive part and it is deliberate: a reachability fixture should stand on the
///         SHIPPED schema, not on one <c>EnsureCreated</c> builds from the current model.
///     </para>
/// </summary>
[TestFixture(TestProvider.Sqlite)]
[TestFixture(TestProvider.MySql)]
[NonParallelizable]
public class SchedulingCollectionColumnNullTests(TestProvider provider)
{
    private const string MySqlConnectionVariable = "ETV_TEST_MYSQL_CONNECTION";
    private const string MySqlRequiredVariable = "ETV_REQUIRE_MYSQL_TESTS";

    private string _databasePath = null!;
    private string? _mySqlConnectionString;
    private DbContextOptions<TvContext> _options = null!;

    private string _originalLastInsertedRowId = null!;
    private string _originalCollation = null!;
    private bool _originalIsSqlite;
    private Func<DbUpdateException, bool> _originalUniqueViolation = null!;

    /// <summary>
    ///     <c>TvContext</c>'s provider statics are process-wide, and the MySQL arm sets them to MySQL
    ///     values. Restoring them is NOT belt-and-braces: <see cref="InMemoryTvContext" /> — the harness
    ///     most of this suite uses — resets only three of the five (<c>IsSqlite</c>,
    ///     <c>IsUniqueConstraintViolation</c>, <c>RegisterUnicodeCaseFunctions</c>) and leaves
    ///     <c>LastInsertedRowId</c> and <c>CaseInsensitiveCollation</c> alone. So without this, a MySQL arm
    ///     running before a SQLite test leaves <c>last_insert_id()</c> in place for a SQLite connection,
    ///     which is an order-dependent failure in a fixture that never mentions MySQL.
    ///     <c>[NonParallelizable]</c> serialises execution; it does not restore state.
    /// </summary>
    [SetUp]
    public async Task SetUp()
    {
        _originalLastInsertedRowId = TvContext.LastInsertedRowId;
        _originalCollation = TvContext.CaseInsensitiveCollation;
        _originalIsSqlite = TvContext.IsSqlite;
        _originalUniqueViolation = TvContext.IsUniqueConstraintViolation;

        if (provider is TestProvider.Sqlite)
        {
            TvContext.IsSqlite = true;
            TvContext.LastInsertedRowId = "last_insert_rowid()";
            TvContext.CaseInsensitiveCollation = "NOCASE";
            TvContext.IsUniqueConstraintViolation = SqliteErrorClassifier.IsUniqueConstraintViolation;

            _databasePath = Path.Combine(Path.GetTempPath(), $"etv823-{Guid.NewGuid():N}.sqlite3");
            _options = new DbContextOptionsBuilder<TvContext>()
                .UseSqlite(
                    $"Data Source={_databasePath};Foreign Keys=False",
                    o => o.MigrationsAssembly("ErsatzTV.Infrastructure.Sqlite"))
                .Options;

            await using TvContext sqlite = Create(_options);
            await sqlite.Database.MigrateAsync();
            return;
        }

        string? baseConnectionString = Environment.GetEnvironmentVariable(MySqlConnectionVariable);
        if (string.IsNullOrWhiteSpace(baseConnectionString))
        {
            string message =
                $"{MySqlConnectionVariable} is not set, so the MySql half of the #823 measurement cannot "
                + "run. Whether a NULL column materializes as CLR null is a provider question, so a "
                + "SQLite-only answer does not settle it.";

            if (IsTrue(Environment.GetEnvironmentVariable(MySqlRequiredVariable)))
            {
                Assert.Fail($"{message} {MySqlRequiredVariable} is set, so this is a failure, not a skip.");
            }

            Assert.Ignore($"{message} Set it to run this locally.");
        }

        // A database of our own, and a FRESH one per test: isolation by construction, per #491's finding
        // that a shared name trades isolation for a wipe that has to succeed.
        _mySqlConnectionString =
            new MySqlConnectionStringBuilder(baseConnectionString) { Database = $"etv823_{Guid.NewGuid():N}" }
                .ConnectionString;

        TvContext.IsSqlite = false;
        TvContext.LastInsertedRowId = "last_insert_id()";
        TvContext.CaseInsensitiveCollation = "utf8mb4_general_ci";
        TvContext.IsUniqueConstraintViolation = MySqlErrorClassifier.IsUniqueConstraintViolation;

        ServerVersion serverVersion = ServerVersion.AutoDetect(_mySqlConnectionString);
        _options = new DbContextOptionsBuilder<TvContext>()
            .UseMySql(
                _mySqlConnectionString,
                serverVersion,
                o => o.MigrationsAssembly("ErsatzTV.Infrastructure.MySql"))
            .Options;

        // NUnit does not run [TearDown] when [SetUp] throws, and by this point the database exists — the
        // migration itself created it. Without this, a migration that fails part way strands a database
        // and its connection pool on a SHARED server, once per attempt.
        try
        {
            await using TvContext mysql = Create(_options);
            await mysql.Database.MigrateAsync();
        }
        catch
        {
            await DropMySqlDatabase();
            _mySqlConnectionString = null;
            throw;
        }
    }

    private async Task DropMySqlDatabase()
    {
        try
        {
            await using (TvContext context = Create(_options))
            {
                await context.Database.EnsureDeletedAsync();
            }

            await using var probe = new MySqlConnection(_mySqlConnectionString);
            await MySqlConnection.ClearPoolAsync(probe);
        }
        catch (Exception ex)
        {
            // Never mask the original failure with a cleanup failure, and never fail a PASSING test in
            // teardown because the server went away -- but say so, because a silent cleanup failure is how
            // a leak becomes invisible.
            await TestContext.Out.WriteLineAsync(
                $"WARNING: could not drop the MySql test database {_mySqlConnectionString}: {ex.Message}");
        }
    }

    [TearDown]
    public async Task TearDown()
    {
        TvContext.LastInsertedRowId = _originalLastInsertedRowId;
        TvContext.CaseInsensitiveCollation = _originalCollation;
        TvContext.IsSqlite = _originalIsSqlite;
        TvContext.IsUniqueConstraintViolation = _originalUniqueViolation;

        if (provider is TestProvider.Sqlite)
        {
            Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
            foreach (string path in new[] { _databasePath, $"{_databasePath}-wal", $"{_databasePath}-shm" })
            {
                if (File.Exists(path))
                {
                    File.Delete(path);
                }
            }

            return;
        }

        if (_mySqlConnectionString is not null)
        {
            await DropMySqlDatabase();
            _mySqlConnectionString = null;
        }
    }

    /// <summary>
    ///     A row whose three columns are NULL materializes as CLR null on every one of them — including
    ///     the two <c>IntCollectionValueConverter</c> columns, whose converter would have produced
    ///     <c>Array.Empty&lt;int&gt;()</c> had it been invoked.
    /// </summary>
    [Test]
    public async Task A_Null_Column_Materializes_As_Clr_Null_On_All_Six()
    {
        await using (TvContext context = Create(_options))
        {
            await DisableForeignKeys(context);

            context.ProgramScheduleAlternates.Add(NewAlternate(playoutId: 1));
            context.PlayoutTemplates.Add(NewTemplate(playoutId: 1));
            await context.SaveChangesAsync();

            // Out of band, so the row is byte-identical to a legacy one. This UPDATE is itself a code
            // path and does NOT demonstrate the legacy route -- see the class docstring for which single
            // (column, provider) case is genuinely code-path-free. What is being measured here is what EF
            // MATERIALIZES from such a row, which is the same regardless of how the NULL got there.
            await context.Database.ExecuteSqlRawAsync(
                "UPDATE ProgramScheduleAlternate SET DaysOfWeek = NULL, DaysOfMonth = NULL, MonthsOfYear = NULL");
            await context.Database.ExecuteSqlRawAsync(
                "UPDATE PlayoutTemplate SET DaysOfWeek = NULL, DaysOfMonth = NULL, MonthsOfYear = NULL");
        }

        // ANTI-VACUITY: a read that found no row would leave every ShouldBeNull below trivially true,
        // so both reads are Single and would throw on an empty table.
        await using (TvContext context = Create(_options))
        {
            ProgramScheduleAlternate alternate =
                await context.ProgramScheduleAlternates.AsNoTracking().SingleAsync();

            alternate.DaysOfWeek.ShouldBeNull(
                "EnumCollectionJsonValueConverter must not be invoked for a NULL column");
            alternate.DaysOfMonth.ShouldBeNull(
                "IntCollectionValueConverter's null-to-empty branch must not run — EF skips the "
                + "converter for a NULL column, so this is null rather than an empty array");
            alternate.MonthsOfYear.ShouldBeNull();

            PlayoutTemplate template = await context.PlayoutTemplates.AsNoTracking().SingleAsync();

            template.DaysOfWeek.ShouldBeNull();
            template.DaysOfMonth.ShouldBeNull();
            template.MonthsOfYear.ShouldBeNull();
        }
    }

    /// <summary>
    ///     The write half. A caller that hands one of the six a null — which the two Replace* handlers do
    ///     verbatim from their command, and only the HTTP request records guard against — gets a stored
    ///     SQL NULL and no error at all.
    /// </summary>
    [Test]
    public async Task Assigning_Null_Persists_Sql_Null_Rather_Than_Throwing()
    {
        await using (TvContext context = Create(_options))
        {
            await DisableForeignKeys(context);

            ProgramScheduleAlternate alternate = NewAlternate(playoutId: 2);
            alternate.DaysOfWeek = null!;
            alternate.DaysOfMonth = null!;
            alternate.MonthsOfYear = null!;

            context.ProgramScheduleAlternates.Add(alternate);

            // Not Should.NotThrow: the point is that this is the SHIPPED behaviour of the write path, so
            // the assertion is that the round-trip below finds NULL, not merely that nothing blew up.
            await context.SaveChangesAsync();
        }

        await using (TvContext context = Create(_options))
        {
            ProgramScheduleAlternate reloaded =
                await context.ProgramScheduleAlternates.AsNoTracking().SingleAsync();

            reloaded.DaysOfWeek.ShouldBeNull();
            reloaded.DaysOfMonth.ShouldBeNull();
            reloaded.MonthsOfYear.ShouldBeNull();
        }

        // ...and it really is SQL NULL in the column, not an empty string the converter round-trips.
        // ExecuteScalar hands back CLR null both for a NULL column and for NO SUCH ROW, so the row is
        // counted first — otherwise a fixture that silently deleted its row would report the same thing.
        await using (TvContext context = Create(_options))
        {
            (await ScalarAsync(context, "SELECT COUNT(*) FROM ProgramScheduleAlternate"))
                .ShouldNotBeNull();
            Convert.ToInt32(await ScalarAsync(context, "SELECT COUNT(*) FROM ProgramScheduleAlternate"))
                .ShouldBe(1, "the row is gone, so a null read below would prove nothing");

            (await ScalarAsync(context, "SELECT DaysOfWeek FROM ProgramScheduleAlternate")).ShouldBeNull();
            (await ScalarAsync(context, "SELECT DaysOfMonth FROM ProgramScheduleAlternate")).ShouldBeNull();
            (await ScalarAsync(context, "SELECT MonthsOfYear FROM ProgramScheduleAlternate")).ShouldBeNull();
        }
    }

    private static ProgramScheduleAlternate NewAlternate(int playoutId) =>
        new()
        {
            PlayoutId = playoutId,
            ProgramScheduleId = 1,
            Index = 0,
            DaysOfWeek = [],
            DaysOfMonth = [],
            MonthsOfYear = [],
            StartMonth = 1,
            StartDay = 1,
            EndMonth = 12,
            EndDay = 31
        };

    private static PlayoutTemplate NewTemplate(int playoutId) =>
        new()
        {
            PlayoutId = playoutId,
            TemplateId = 1,
            Index = 0,
            DaysOfWeek = [],
            DaysOfMonth = [],
            MonthsOfYear = [],
            StartMonth = 1,
            StartDay = 1,
            EndMonth = 12,
            EndDay = 31
        };

    /// <summary>
    ///     The rows here are deliberately partial graphs (a <c>PlayoutId</c> pointing at no Playout), so
    ///     foreign keys are off. SQLite takes it as a connection-string keyword; MySQL's
    ///     <c>foreign_key_checks</c> is a SESSION variable, so it is set on the context's own connection
    ///     and lives as long as that context does.
    /// </summary>
    private async Task DisableForeignKeys(TvContext context)
    {
        if (provider is TestProvider.MySql)
        {
            await context.Database.OpenConnectionAsync();
            await context.Database.ExecuteSqlRawAsync("SET SESSION foreign_key_checks = 0");
        }
    }

    private static async Task<object?> ScalarAsync(TvContext context, string sql)
    {
        await context.Database.OpenConnectionAsync();
        await using System.Data.Common.DbCommand command = context.Database.GetDbConnection().CreateCommand();
        command.CommandText = sql;
        object? value = await command.ExecuteScalarAsync();
        return value is DBNull ? null : value;
    }

    private static bool IsTrue(string? value) =>
        !string.IsNullOrWhiteSpace(value)
        && (value == "1" || value.Equals("true", StringComparison.OrdinalIgnoreCase));

    private static TvContext Create(DbContextOptions<TvContext> options) =>
        new(
            options,
            NullLoggerFactory.Instance,
            new SlowQueryInterceptor(NullLogger<SlowQueryInterceptor>.Instance));
}