ersatztv/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs

using System.Text;
using System.Text.Json;
using Dapper;
using ErsatzTV.Core.Api.Search;
using ErsatzTV.Core.Domain;
using ErsatzTV.Infrastructure.Data;
using Microsoft.EntityFrameworkCore;

namespace ErsatzTV.Application.Search.Queries;

public class GetSearchFieldValuesHandler(IDbContextFactory<TvContext> dbContextFactory)
    : IRequestHandler<GetSearchFieldValues, Option<SearchFieldValuesResponseModel>>
{
    private const int DefaultLimit = 50;
    private const int MaxLimit = 50;

    /// <summary>
    ///     Rows read per round trip when walking the list-valued (JSON-array) columns on
    ///     <c>SongMetadata</c>, and the ceiling on rows read per request.
    ///     <para>
    ///         These count ACTUAL ROWS. A fixed <c>LIMIT</c> budget bounded the RESULT, and
    ///         the pre-filter (allowed to over-match) starved it with rows that could not match. Keyset paging
    ///         with a <c>LIMIT</c> bounded CANDIDATES RETURNED — but a query matching nothing must evaluate
    ///         every eligible row before it can return an empty page, so rows inspected stayed unbounded. A
    ///         closed <c>Id</c> range bounded KEYSPACE WIDTH — but keyspace is not rows: delete 20,000
    ///         historical rows, put one song at <c>Id</c> 20001, and the walk burns its whole allowance on empty
    ///         ranges and inspects nothing.
    ///     </para>
    ///     <para>
    ///         What makes this one hold is that <b>the query has no RESIDUAL predicate</b> — nothing that can
    ///         discard a row the engine already produced. The only condition is the cursor
    ///         <c>Id &gt; @AfterId</c>, which is a seek on the <c>ORDER BY</c> key itself, not a filter. So the
    ///         page returns exactly <see cref="ListValuedBatchRows" /> rows whenever that many logical rows
    ///         remain, independent of how sparse the matches are or where the <c>Id</c> gaps fall.
    ///     </para>
    ///     <para>
    ///         <b>Be precise about what is bounded: LOGICAL ROWS RETURNED AND MATERIALIZED, and the number of
    ///         round trips. Not physical work, and not bytes.</b> Two things break the stronger reading:
    ///         <list type="bullet">
    ///             <item>
    ///                 MySQL purge lag. Deleted clustered-index records survive until purge runs, and a range
    ///                 scan still traverses them, so returning 2,000 VISIBLE rows can touch far more index
    ///                 records. Deletion history therefore still affects physical work — the very thing the
    ///                 keyspace attempt was trying to make irrelevant.
    ///             </item>
    ///             <item>
    ///                 Row width is unbounded. These columns are <c>TEXT</c>/<c>longtext</c>, which both SQLite
    ///                 and InnoDB spill to overflow pages, so a row count implies neither a byte count nor a
    ///                 page-read count.
    ///             </item>
    ///         </list>
    ///         The logical-row bound is still worth having — it is what makes the walk terminate and what caps
    ///         the number of rows and round trips — but do not restate it as bounded I/O, and do not restate it
    ///         as bounded MEMORY either: payload width is unrestricted and a single JSON array can hold
    ///         arbitrarily many strings, every one of which may enter the in-memory set.
    ///     </para>
    ///     <para>
    ///         The trade is real and deliberate: no server-side narrowing, so a query with few matches transfers
    ///         rows it will discard, up to <see cref="ListValuedMaxRowsRead" />. A query with enough matches
    ///         stops as soon as it has <c>limit</c> distinct ones, so the dense cases — including an empty
    ///         <c>q</c> — finish on the first page. See <c>api.search-field-values-sources</c> for the measured
    ///         cost and for why reintroducing a <c>LIKE</c> is not an option.
    ///     </para>
    /// </summary>
    internal const int ListValuedBatchRows = 2000;

    /// <inheritdoc cref="ListValuedBatchRows" />
    internal const int ListValuedMaxRowsRead = 20000;


    public async Task<Option<SearchFieldValuesResponseModel>> Handle(
        GetSearchFieldValues request,
        CancellationToken cancellationToken)
    {
        SearchFieldResponseModel field = SearchFieldCatalog.Fields
            .FirstOrDefault(f => f.Name == request.Name);

        if (field is null || field.Type != "text")
        {
            return Option<SearchFieldValuesResponseModel>.None;
        }

        int limit = request.Limit <= 0 ? DefaultLimit : Math.Clamp(request.Limit, 1, MaxLimit);
        string query = request.Query ?? string.Empty;

        // Invariant, not current-culture: UseRequestLocalization honours Accept-Language, so a caller can select
        // tr-TR and turn `q=I` into `ı` — which then matches nothing a Turkish-dotless-i-free library contains.
        // This feeds the EF-translated filter, which has no StringComparison overload EF can translate.
        string qLower = query.ToLowerInvariant();

        // in-memory special cases (no DB query needed)
        switch (request.Name)
        {
            case "state":
                return new SearchFieldValuesResponseModel(
                    FilterSortTake(Enum.GetNames<MediaItemState>(), query, limit));
            case "video_dynamic_range":
                return new SearchFieldValuesResponseModel(
                    FilterSortTake(["hdr", "sdr"], query, limit));
        }

        await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);

        if (request.Name == "content_rating")
        {
            return new SearchFieldValuesResponseModel(
                await GetContentRatingValues(dbContext, query, limit, cancellationToken));
        }

        IQueryable<string> source = GetSource(dbContext, request.Name);
        string listColumn = GetSongListValuedColumn(request.Name);
        if (source is null && listColumn is null)
        {
            return Option<SearchFieldValuesResponseModel>.None;
        }

        var values = new List<string>();

        if (source is not null)
        {
            values.AddRange(
                await source
                    .Where(v => v != null && v.ToLower().StartsWith(qLower))
                    .Distinct()
                    .OrderBy(v => v)
                    .Take(limit)
                    .ToListAsync(cancellationToken));
        }

        // ersatztv#668. The query above prefix-matches through SQL LOWER(), and SQLite's LOWER() folds ASCII
        // ONLY -- lower('Édith') is 'Édith' unchanged -- so it cannot reach a stored value whose prefix
        // carries an uppercase non-ASCII character, from ANY query. It UNDER-matches, and an under-match is
        // unrecoverable downstream: no later stage can reintroduce a row SQL never returned. So for the only
        // queries that can be affected (those containing a non-ASCII character) run a second, Unicode-correct
        // pass and merge it in. This is ADDITIVE on purpose -- the SQL pass above still contributes, so a
        // value already reachable today cannot stop being reachable.
        //
        // MySQL needs none of this: its LOWER() is Unicode-aware, so LOWER('Édith') really is 'édith' and the
        // existing predicate reaches the row unaided. Measured on 8.4 -- and note the executed path does NOT
        // over-match, even though the column collation (utf8mb4_0900_ai_ci) is accent-insensitive: the driver
        // binds the LIKE pattern with a BINARY collation, so the comparison is accent-sensitive in practice.
        // A hand-typed probe using a LITERAL pattern DOES over-match; that is a different query from the one
        // this code runs, and mistaking the two gives a false read on whether this predicate over-matches.
        if (source is not null && ContainsNonAscii(query) && IsSqlite(dbContext))
        {
            values.AddRange(
                await GetUnicodeFoldedValues(dbContext, request.Name, query, limit, cancellationToken));
        }

        if (listColumn is not null)
        {
            values.AddRange(await GetSongListValuedValues(dbContext, listColumn, query, limit, cancellationToken));
        }

        // ORDERING IS BEST-EFFORT, NOT EXACT. Each source truncates using its own ordering — the EF source by the
        // database collation (SQLite's NOCASE/BINARY is ASCII-only), the list source by primary key — and neither
        // is the ordinal ordering applied here. So when a source actually truncates, a value it dropped may have
        // outranked one that survived: with "Zulu" and "apple" and limit=1 the database keeps "apple" (its
        // ordering is case-insensitive) while ordinal ranks "Zulu" first, so the merge never sees "Zulu".
        // Below the truncation points (the normal typeahead case) the result is exact.
        return new SearchFieldValuesResponseModel(FilterSortTake(values.Distinct(StringComparer.Ordinal), query, limit));
    }

    internal static IQueryable<string> GetSource(TvContext dbContext, string name) => name switch
    {
        "genre" or "show_genre" => dbContext.Set<Genre>().Select(g => g.Name),
        "studio" => dbContext.Set<Studio>().Select(s => s.Name),
        "director" => dbContext.Set<Director>().Select(d => d.Name),
        "writer" => dbContext.Set<Writer>().Select(w => w.Name),
        "actor" => dbContext.Actors.Select(a => a.Name),
        // Mirrors what LuceneSearchIndex writes to the `artist` field: the music video's linked artist entity
        // (ArtistMetadata.Title) plus its free-text credits (MusicVideoArtist rows). The third contributor —
        // SongMetadata.Artists — is a JSON-array column and is handled by GetSongListValuedValues instead.
        "artist" => dbContext.ArtistMetadata.Select(m => m.Title)
            .Concat(dbContext.Set<MusicVideoArtist>().Select(a => a.Name)),
        "tag" => dbContext.Set<Tag>()
            .Where(t => t.ExternalTypeId != Tag.NfoCountryTypeId && t.ExternalTypeId != Tag.PlexNetworkTypeId)
            .Select(t => t.Name),
        "network" => dbContext.Set<Tag>()
            .Where(t => t.ExternalTypeId == Tag.PlexNetworkTypeId)
            .Select(t => t.Name),
        "collection" => dbContext.Collections.Select(c => c.Name),
        "video_codec" => dbContext.MediaStreams
            .Where(s => s.MediaStreamKind == MediaStreamKind.Video && s.Codec != null)
            .Select(s => s.Codec),
        "album" => dbContext.MusicVideoMetadata
            .Where(m => m.Album != null)
            .Select(m => m.Album)
            .Concat(dbContext.SongMetadata.Where(m => m.Album != null).Select(m => m.Album)),
        _ => null
    };

    /// <summary>
    ///     SQL name of the invariant-uppercase fold registered by <c>SqliteUnicodeFunctions</c>. Duplicated
    ///     rather than referenced because Application must not depend on a provider assembly; a test asserts
    ///     the two constants are equal so they cannot drift.
    /// </summary>
    internal const string UpperFunction = "etv_upper";

    /// <summary>
    ///     True when the value contains any character outside US-ASCII, which is exactly when SQLite's
    ///     ASCII-only <c>LOWER()</c> can under-match. Evaluated on the RAW query, never the lowercased copy:
    ///     the trigger must not be coupled to the fold.
    /// </summary>
    internal static bool ContainsNonAscii(string value)
    {
        foreach (char c in value)
        {
            if (c > 0x7F)
            {
                return true;
            }
        }

        return false;
    }

    // Derived per-context rather than read from the TvContext.IsSqlite static on purpose. Nothing MECHANICALLY
    // stops that read -- ProviderStaticsWiringTests only parses the two composition roots for ASSIGNMENTS, not
    // readers -- but that test's scanner exemption for IsSqlite is justified in prose as "read only by
    // DbInitializer + DatabaseMigratorService, both host-only", and reading it here would make that reason
    // false while the test stayed green. Do not "simplify" this to IsSqlite.
    private static bool IsSqlite(TvContext dbContext) =>
        (dbContext.Database.ProviderName ?? string.Empty).Contains("Sqlite", StringComparison.OrdinalIgnoreCase);

    /// <summary>
    ///     Escapes the LIKE metacharacters in a user-supplied prefix and appends the trailing wildcard. The
    ///     backslash MUST be escaped first, or the escapes added for <c>%</c>/<c>_</c> would themselves be
    ///     re-escaped. Paired with an explicit <c>ESCAPE '\'</c> in <see cref="UnicodeFoldSql" />, since raw
    ///     SQL gets none of the escaping EF does for <c>StartsWith</c>.
    /// </summary>
    internal static string EscapeLikePrefix(string value) =>
        value
            .Replace("\\", "\\\\", StringComparison.Ordinal)
            .Replace("%", "\\%", StringComparison.Ordinal)
            .Replace("_", "\\_", StringComparison.Ordinal) + "%";

    /// <summary>
    ///     One bounded, exact prefix query using the Unicode-correct fold. Unlike the list-valued walk this
    ///     KEEPS its selectivity in SQL — it is a normal indexed-or-not <c>LIMIT</c>ed query exactly like the
    ///     EF one it supplements, not a paged walk, so there is no row budget to blow and no reason to strip
    ///     the discriminator predicates out of it.
    /// </summary>
    internal static string UnicodeFoldSql(string table, string column, string predicate)
    {
        var match = $"{UpperFunction}({column}) LIKE @Pattern ESCAPE '\\'";
        string where = predicate is null ? match : $"({predicate}) AND {match}";
        return $"SELECT DISTINCT {column} AS Value FROM {table} WHERE {where} ORDER BY {column} LIMIT @Limit";
    }

    /// <summary>
    ///     The tables/columns behind each EF-sourced field, mirroring <see cref="GetSource" /> 1:1.
    ///     <para>
    ///         The discriminator predicates must mirror EF's NULL semantics, not C#'s reading of the source.
    ///         EF compiles <c>t.ExternalTypeId != Tag.NfoCountryTypeId</c> with null semantics, so a row whose
    ///         <c>ExternalTypeId</c> is NULL IS included; plain SQL <c>&lt;&gt;</c> against NULL yields NULL and
    ///         would silently drop it. Hence the explicit <c>IS NULL</c> arm.
    ///     </para>
    /// </summary>
    private static IReadOnlyList<UnicodeFoldSource> GetUnicodeFoldSources(string name) => name switch
    {
        "genre" or "show_genre" => [new UnicodeFoldSource("Genre", "Name")],
        "studio" => [new UnicodeFoldSource("Studio", "Name")],
        "director" => [new UnicodeFoldSource("Director", "Name")],
        "writer" => [new UnicodeFoldSource("Writer", "Name")],
        "actor" => [new UnicodeFoldSource("Actor", "Name")],
        "artist" =>
        [
            new UnicodeFoldSource("ArtistMetadata", "Title"),
            new UnicodeFoldSource("MusicVideoArtist", "Name")
        ],
        "tag" =>
        [
            new UnicodeFoldSource(
                "Tag",
                "Name",
                "ExternalTypeId IS NULL OR (ExternalTypeId <> @NfoCountryTypeId AND ExternalTypeId <> @PlexNetworkTypeId)",
                new Dictionary<string, object>
                {
                    ["NfoCountryTypeId"] = Tag.NfoCountryTypeId,
                    ["PlexNetworkTypeId"] = Tag.PlexNetworkTypeId
                })
        ],
        "network" =>
        [
            new UnicodeFoldSource(
                "Tag",
                "Name",
                "ExternalTypeId = @PlexNetworkTypeId",
                new Dictionary<string, object> { ["PlexNetworkTypeId"] = Tag.PlexNetworkTypeId })
        ],
        "collection" => [new UnicodeFoldSource("Collection", "Name")],
        "video_codec" =>
        [
            new UnicodeFoldSource(
                "MediaStream",
                "Codec",
                "MediaStreamKind = @VideoStreamKind AND Codec IS NOT NULL",
                new Dictionary<string, object> { ["VideoStreamKind"] = (int)MediaStreamKind.Video })
        ],
        "album" =>
        [
            new UnicodeFoldSource("MusicVideoMetadata", "Album", "Album IS NOT NULL"),
            new UnicodeFoldSource("SongMetadata", "Album", "Album IS NOT NULL")
        ],
        _ => []
    };

    private static async Task<List<string>> GetUnicodeFoldedValues(
        TvContext dbContext,
        string name,
        string query,
        int limit,
        CancellationToken cancellationToken)
    {
        IReadOnlyList<UnicodeFoldSource> sources = GetUnicodeFoldSources(name);
        if (sources.Count == 0)
        {
            return [];
        }

        // CreateFunction is per-connection, so registration happens here, at the one call site that needs
        // the function, rather than through an EF connection interceptor: Dapper opens a closed connection
        // itself and a direct ADO open does not raise EF's interceptors, so an interceptor-based seam would
        // silently miss exactly this query. Opening first makes the registration order-independent.
        await dbContext.Database.OpenConnectionAsync(cancellationToken);
        TvContext.RegisterUnicodeCaseFunctions(dbContext.Connection);

        string pattern = EscapeLikePrefix(query.ToUpperInvariant());
        var values = new List<string>();

        foreach (UnicodeFoldSource source in sources)
        {
            var parameters = new DynamicParameters();
            parameters.Add("Pattern", pattern);
            parameters.Add("Limit", limit);
            if (source.Parameters is not null)
            {
                foreach ((string key, object value) in source.Parameters)
                {
                    parameters.Add(key, value);
                }
            }

            IEnumerable<string> rows = await dbContext.Connection.QueryAsync<string>(
                new CommandDefinition(
                    UnicodeFoldSql(source.Table, source.Column, source.Predicate),
                    parameters,
                    cancellationToken: cancellationToken));

            values.AddRange(rows.Where(v => !string.IsNullOrEmpty(v)));
        }

        return values;
    }

    private sealed record UnicodeFoldSource(
        string Table,
        string Column,
        string Predicate = null,
        IReadOnlyDictionary<string, object> Parameters = null);

    /// <summary>
    ///     Maps a field name onto the <c>SongMetadata</c> column that backs it as an <c>IList&lt;string&gt;</c>.
    ///     The returned value is a compile-time constant from this switch — never caller input — so it is safe
    ///     to interpolate into the SQL in <see cref="ListValuedSql" />.
    /// </summary>
    private static string GetSongListValuedColumn(string name) => name switch
    {
        "artist" => "Artists",
        "album_artist" => "AlbumArtists",
        _ => null
    };

    /// <summary>
    ///     Reads whole values out of a <c>SongMetadata</c> <c>IList&lt;string&gt;</c> column.
    ///     <para>
    ///         EF maps these as primitive collections: one JSON array per row in a single <c>TEXT</c>/
    ///         <c>longtext</c> column. Neither provider can project the elements server-side — SQLite needs
    ///         the SQL <c>APPLY</c> operator it doesn't have, and Pomelo MySQL doesn't implement primitive
    ///         collections at all — so there is no server-side <c>SELECT DISTINCT</c> over the elements.
    ///     </para>
    ///     <para>
    ///         So the rows are walked in primary-key order, keyset-paged by row position, and split +
    ///         exact-filtered in memory. All selectivity is in memory — the query's only condition is the
    ///         cursor, a seek on the ordering key that never discards a row, so its <c>LIMIT</c> bounds the
    ///         LOGICAL ROWS returned. See <see cref="ListValuedBatchRows" /> for the four revisions it took to
    ///         get that right, and for what that bound does and does not cover.
    ///     </para>
    /// </summary>
    private static async Task<List<string>> GetSongListValuedValues(
        TvContext dbContext,
        string column,
        string query,
        int limit,
        CancellationToken cancellationToken)
    {
        string sql = ListValuedSql(column);

        var distinct = new System.Collections.Generic.HashSet<string>(StringComparer.Ordinal);
        var afterId = 0;
        var read = 0;

        while (read < ListValuedMaxRowsRead && distinct.Count < limit)
        {
            int batch = Math.Min(ListValuedBatchRows, ListValuedMaxRowsRead - read);

            List<ListValuedRow> rows = (await dbContext.Connection.QueryAsync<ListValuedRow>(
                new CommandDefinition(
                    sql,
                    new { AfterId = afterId, Batch = batch },
                    cancellationToken: cancellationToken))).AsList();

            if (rows.Count == 0)
            {
                break;
            }

            read += rows.Count;
            afterId = rows[^1].Id;

            foreach (ListValuedRow row in rows)
            {
                foreach (string element in ParseElements(row.Payload))
                {
                    if (element.StartsWith(query, StringComparison.OrdinalIgnoreCase))
                    {
                        distinct.Add(element);
                    }
                }
            }

            if (rows.Count < batch)
            {
                // With no RESIDUAL predicate -- only the cursor, which selects a range rather than discarding
                // rows from it -- a short page can only mean the table is exhausted. It can never mean "this
                // stretch happened to match nothing", which is precisely why the residual predicate had to go.
                // Advancing from the last returned Id is safe for the same reason: nothing was filtered out
                // behind it, so no row can be skipped.
                break;
            }
        }

        return distinct.ToList();
    }

    private static IEnumerable<string> ParseElements(string payload)
    {
        if (string.IsNullOrWhiteSpace(payload))
        {
            return [];
        }

        try
        {
            return (JsonSerializer.Deserialize<string[]>(payload) ?? []).Where(e => !string.IsNullOrEmpty(e));
        }
        catch (JsonException)
        {
            return [];
        }
    }

    /// <summary>
    ///     One keyset page of rows, by ROW POSITION rather than by <c>Id</c> value.
    ///     <para>
    ///         The only condition is the cursor — deliberately <b>no RESIDUAL predicate</b>: no <c>LIKE</c>, no
    ///         <c>LOWER</c>, not even <c>IS NOT NULL</c>. The distinction that matters is not "no predicate"
    ///         (the cursor is one); it is that <c>Id &gt; @AfterId</c> is a <i>seekable predicate on the
    ///         ordering key</i>, which positions the scan and never discards a row, whereas a residual
    ///         predicate throws away rows the engine already produced. <c>LIMIT</c> only truncates what
    ///         survives a residual predicate, so with one present it bounds the output rather than the row
    ///         count — a gap wide enough to scan straight past a nominal row-count bound. With none, <c>LIMIT n</c>
    ///         yields <c>n</c> logical rows. Null payloads are dropped in memory by
    ///         <see cref="ParseElements" />.
    ///     </para>
    ///     <para>
    ///         Note this pins the SQL string only. It cannot pin an execution plan, MVCC visibility work, or
    ///         payload I/O — and on MySQL, using the index to satisfy <c>ORDER BY</c> is an optimizer choice,
    ///         not a semantic guarantee.
    ///     </para>
    /// </summary>
    internal static string ListValuedSql(string column) =>
        $"SELECT Id, {column} AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch";

    private sealed class ListValuedRow
    {
        public int Id { get; init; }

        public string Payload { get; init; }
    }

    private static async Task<List<string>> GetContentRatingValues(
        TvContext dbContext,
        string query,
        int limit,
        CancellationToken cancellationToken)
    {
        List<string> raw = await dbContext.MovieMetadata
            .Where(m => m.ContentRating != null)
            .Select(m => m.ContentRating)
            .Concat(dbContext.ShowMetadata.Where(m => m.ContentRating != null).Select(m => m.ContentRating))
            .Concat(dbContext.OtherVideoMetadata.Where(m => m.ContentRating != null).Select(m => m.ContentRating))
            .Concat(dbContext.RemoteStreamMetadata.Where(m => m.ContentRating != null).Select(m => m.ContentRating))
            .Distinct()
            .ToListAsync(cancellationToken);

        IEnumerable<string> split = raw
            .SelectMany(cr => cr.Split('/'))
            .Select(cr => cr.Trim())
            .Where(cr => !string.IsNullOrEmpty(cr))
            .Distinct();

        return FilterSortTake(split, query, limit);
    }

    /// <summary>
    ///     The one in-memory filter/sort/take every field funnels through. Both the comparison and the ordering
    ///     are ORDINAL on purpose: <c>UseRequestLocalization</c> honours <c>Accept-Language</c>, so the current
    ///     culture is caller-controlled, and <c>ToLower()</c> plus the default (linguistic)
    ///     <c>StartsWith(string)</c> would make the result depend on it — under <c>tr-TR</c>, <c>q=I</c> lowers
    ///     to <c>ı</c> and stops matching <c>Istanbul</c>. Note this is the LAST stage only: a field sourced by
    ///     a plain EF query has already been filtered and truncated by the database collation before it gets
    ///     here, which ordinal semantics downstream cannot undo (ersatztv#668).
    /// </summary>
    private static List<string> FilterSortTake(IEnumerable<string> values, string query, int limit) =>
        values
            .Where(v => v.StartsWith(query, StringComparison.OrdinalIgnoreCase))
            .OrderBy(v => v, StringComparer.Ordinal)
            .Take(limit)
            .ToList();
}