ersatztv/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs

using System.Text;
using System.Text.Json;
using Dapper;
using ErsatzTV.Core.Api.Search;
using ErsatzTV.Core.Domain;
using ErsatzTV.Infrastructure.Data;
using Microsoft.EntityFrameworkCore;

namespace ErsatzTV.Application.Search.Queries;

public class GetSearchFieldValuesHandler(IDbContextFactory<TvContext> dbContextFactory)
    : IRequestHandler<GetSearchFieldValues, Option<SearchFieldValuesResponseModel>>
{
    private const int DefaultLimit = 50;
    private const int MaxLimit = 50;

    /// <summary>
    ///     Row cap for the list-valued (JSON-array) columns on <c>SongMetadata</c>. The SQL pre-filter over the
    ///     raw JSON is a deliberate superset (a matching row is fetched whole, including its non-matching
    ///     elements), so the number of rows it can return has to be bounded independently of <c>limit</c> —
    ///     otherwise an empty <c>q</c> degenerates into materializing the entire table. The cap is applied on
    ///     <c>ORDER BY Id</c>, which is a unique integer primary key on both providers: unlike ordering on the
    ///     JSON column itself, that makes the truncation point genuinely deterministic (MySQL sorts <c>TEXT</c>
    ///     using only the first <c>max_sort_length</c> bytes, so long JSON rows sharing a prefix would tie
    ///     arbitrarily) and lets the engine walk the primary key instead of sorting the whole matching set.
    /// </summary>
    internal const int ListValuedRowCap = 1000;

    /// <summary>
    ///     Escape character for the raw-JSON <c>LIKE</c> pre-filter. Deliberately NOT a backslash: the stored
    ///     JSON is full of backslashes (<c>\u00E9</c>, <c>\u0022</c>) and, worse, <c>ESCAPE '\'</c> is not a
    ///     portable literal — MySQL applies C-style escaping to string literals, so it reads as an unterminated
    ///     string, while SQLite would reject the doubled <c>'\\'</c> form as a two-character escape.
    /// </summary>
    private const char LikeEscapeChar = '/';

    /// <summary>
    ///     The ASCII characters the JSON writer escapes rather than emitting verbatim (<c>&amp;</c> and friends
    ///     become <c>&</c>). A prefix character in this set cannot be matched literally against the stored
    ///     text, so it terminates the narrowing run in <see cref="JsonElementPrefixPattern" />.
    ///     <c>SearchFieldValuesPrefilterSupersetTests</c> pins this set against the encoder itself.
    /// </summary>
    private const string JsonEscapedAscii = "\"&'+<>\\`";

    public async Task<Option<SearchFieldValuesResponseModel>> Handle(
        GetSearchFieldValues request,
        CancellationToken cancellationToken)
    {
        SearchFieldResponseModel field = SearchFieldCatalog.Fields
            .FirstOrDefault(f => f.Name == request.Name);

        if (field is null || field.Type != "text")
        {
            return Option<SearchFieldValuesResponseModel>.None;
        }

        int limit = request.Limit <= 0 ? DefaultLimit : Math.Clamp(request.Limit, 1, MaxLimit);
        string query = request.Query ?? string.Empty;

        // Invariant, not current-culture: UseRequestLocalization honours Accept-Language, so a caller can select
        // tr-TR and turn `q=I` into `ı` — which then matches nothing a Turkish-dotless-i-free library contains.
        // This feeds the EF-translated filter, which has no StringComparison overload EF can translate.
        string qLower = query.ToLowerInvariant();

        // in-memory special cases (no DB query needed)
        switch (request.Name)
        {
            case "state":
                return new SearchFieldValuesResponseModel(
                    FilterSortTake(Enum.GetNames<MediaItemState>(), query, limit));
            case "video_dynamic_range":
                return new SearchFieldValuesResponseModel(
                    FilterSortTake(["hdr", "sdr"], query, limit));
        }

        await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);

        if (request.Name == "content_rating")
        {
            return new SearchFieldValuesResponseModel(
                await GetContentRatingValues(dbContext, query, limit, cancellationToken));
        }

        IQueryable<string> source = GetSource(dbContext, request.Name);
        string listColumn = GetSongListValuedColumn(request.Name);
        if (source is null && listColumn is null)
        {
            return Option<SearchFieldValuesResponseModel>.None;
        }

        var values = new List<string>();

        if (source is not null)
        {
            values.AddRange(
                await source
                    .Where(v => v != null && v.ToLower().StartsWith(qLower))
                    .Distinct()
                    .OrderBy(v => v)
                    .Take(limit)
                    .ToListAsync(cancellationToken));
        }

        if (listColumn is not null)
        {
            values.AddRange(await GetSongListValuedValues(dbContext, listColumn, query, cancellationToken));
        }

        // ORDERING IS BEST-EFFORT, NOT EXACT. Each source truncates using its own ordering — the EF source by the
        // database collation (SQLite's NOCASE/BINARY is ASCII-only), the list source by primary key — and neither
        // is the ordinal ordering applied here. So when a source actually truncates, a value it dropped may have
        // outranked one that survived: with "Zulu" and "Éclair" and limit=1 the EF source keeps "Zulu" and the
        // merge never sees "Éclair". Below the truncation points (the normal typeahead case) the result is exact.
        return new SearchFieldValuesResponseModel(FilterSortTake(values.Distinct(StringComparer.Ordinal), query, limit));
    }

    internal static IQueryable<string> GetSource(TvContext dbContext, string name) => name switch
    {
        "genre" or "show_genre" => dbContext.Set<Genre>().Select(g => g.Name),
        "studio" => dbContext.Set<Studio>().Select(s => s.Name),
        "director" => dbContext.Set<Director>().Select(d => d.Name),
        "writer" => dbContext.Set<Writer>().Select(w => w.Name),
        "actor" => dbContext.Actors.Select(a => a.Name),
        // Mirrors what LuceneSearchIndex writes to the `artist` field: the music video's linked artist entity
        // (ArtistMetadata.Title) plus its free-text credits (MusicVideoArtist rows). The third contributor —
        // SongMetadata.Artists — is a JSON-array column and is handled by GetSongListValuedValues instead.
        "artist" => dbContext.ArtistMetadata.Select(m => m.Title)
            .Concat(dbContext.Set<MusicVideoArtist>().Select(a => a.Name)),
        "tag" => dbContext.Set<Tag>()
            .Where(t => t.ExternalTypeId != Tag.NfoCountryTypeId && t.ExternalTypeId != Tag.PlexNetworkTypeId)
            .Select(t => t.Name),
        "network" => dbContext.Set<Tag>()
            .Where(t => t.ExternalTypeId == Tag.PlexNetworkTypeId)
            .Select(t => t.Name),
        "collection" => dbContext.Collections.Select(c => c.Name),
        "video_codec" => dbContext.MediaStreams
            .Where(s => s.MediaStreamKind == MediaStreamKind.Video && s.Codec != null)
            .Select(s => s.Codec),
        "album" => dbContext.MusicVideoMetadata
            .Where(m => m.Album != null)
            .Select(m => m.Album)
            .Concat(dbContext.SongMetadata.Where(m => m.Album != null).Select(m => m.Album)),
        _ => null
    };

    /// <summary>
    ///     Maps a field name onto the <c>SongMetadata</c> column that backs it as an <c>IList&lt;string&gt;</c>.
    ///     The returned value is a compile-time constant from this switch — never caller input — so it is safe
    ///     to interpolate into the SQL in <see cref="ListValuedSql" />.
    /// </summary>
    private static string GetSongListValuedColumn(string name) => name switch
    {
        "artist" => "Artists",
        "album_artist" => "AlbumArtists",
        _ => null
    };

    /// <summary>
    ///     Reads whole values out of a <c>SongMetadata</c> <c>IList&lt;string&gt;</c> column.
    ///     <para>
    ///         EF maps these as primitive collections: one JSON array per row in a single <c>TEXT</c>/
    ///         <c>longtext</c> column. Neither provider can project the elements server-side — SQLite needs
    ///         the SQL <c>APPLY</c> operator it doesn't have, and Pomelo MySQL doesn't implement primitive
    ///         collections at all — so there is no server-side <c>SELECT DISTINCT</c> over the elements.
    ///     </para>
    ///     <para>
    ///         Instead: pre-filter on the raw JSON text in SQL (a superset — a row is matched, not an element,
    ///         so its other elements come along), cap the rows by primary key, then split and exact-filter in
    ///         memory. Correctness lives in the in-memory filter; the SQL pattern is only an optimization and is
    ///         allowed to over-match but never to under-match. When the library has more matching rows than
    ///         <see cref="ListValuedRowCap" /> the suggestion list for these fields is the lowest-id prefix of
    ///         the matches rather than the complete set.
    ///     </para>
    /// </summary>
    private static async Task<List<string>> GetSongListValuedValues(
        TvContext dbContext,
        string column,
        string query,
        CancellationToken cancellationToken)
    {
        IEnumerable<string> rows = await dbContext.Connection.QueryAsync<string>(
            new CommandDefinition(
                ListValuedSql(column),
                new { Pattern = JsonElementPrefixPattern(query), Cap = ListValuedRowCap },
                cancellationToken: cancellationToken));

        var values = new List<string>();
        foreach (string row in rows)
        {
            if (string.IsNullOrWhiteSpace(row))
            {
                continue;
            }

            string[] elements;
            try
            {
                elements = JsonSerializer.Deserialize<string[]>(row);
            }
            catch (JsonException)
            {
                continue;
            }

            foreach (string element in elements ?? [])
            {
                if (!string.IsNullOrEmpty(element) && element.StartsWith(query, StringComparison.OrdinalIgnoreCase))
                {
                    values.Add(element);
                }
            }
        }

        return values;
    }

    /// <summary>
    ///     The bounded pre-filter query. <c>LOWER(...) LIKE ... ESCAPE</c>, <c>ORDER BY</c> and <c>LIMIT</c> are
    ///     all portable between SQLite and MySQL, and the identifiers are unquoted so neither provider's
    ///     quoting dialect is baked in. The cap rides <c>ORDER BY Id</c> — a unique integer key both providers
    ///     can walk — rather than the JSON column, so the truncation point is well-defined and no full sort of
    ///     the matching set is required.
    /// </summary>
    internal static string ListValuedSql(string column) =>
        $"SELECT {column} FROM SongMetadata WHERE {column} IS NOT NULL " +
        $"AND LOWER({column}) LIKE @Pattern ESCAPE '{LikeEscapeChar}' " +
        "ORDER BY Id LIMIT @Cap";

    /// <summary>
    ///     Builds the raw-JSON <c>LIKE</c> pattern used to narrow the rows fetched for a list-valued column.
    ///     <para>
    ///         <b>This is an optimization with one hard obligation: it must never under-match.</b> Every row
    ///         holding an element that starts with <paramref name="query" /> under
    ///         <see cref="StringComparison.OrdinalIgnoreCase" /> — the comparison
    ///         <see cref="GetSongListValuedValues" /> then applies — has to survive it. Over-matching is free;
    ///         the in-memory filter discards the extras.
    ///     </para>
    ///     <para>
    ///         So the pattern narrows on the <b>leading run of ASCII characters the JSON writer stores
    ///         verbatim</b>, and stops at the first character it cannot reason about — a non-ASCII character, or
    ///         one of <see cref="JsonEscapedAscii" />. A query starting with such a character narrows to the bare
    ///         element-opening anchor <c>%"%</c> and leans entirely on the row cap and the in-memory filter.
    ///     </para>
    ///     <para>
    ///         Why that is sound, and why an earlier "JSON-encode the whole prefix" version was NOT: the stored
    ///         text escapes non-ASCII, so <c>Édith</c> is on disk as <c>Édith</c>. SQL <c>LOWER()</c>
    ///         lowercases the <i>escape text</i>, giving <c>É</c> — it cannot case-fold the codepoint that
    ///         escape denotes, so a <c>q=é</c> pattern of <c>é</c> never matched and every accented artist
    ///         was silently unsuggestable. Restricting the run to verbatim ASCII removes the escape from the
    ///         comparison entirely. And for a run character <c>c</c>, any element character that
    ///         OrdinalIgnoreCase-equals <c>c</c> is itself ASCII — no non-ASCII codepoint in all of Unicode
    ///         OrdinalIgnoreCase-equals a printable ASCII character, which
    ///         <c>SearchFieldValuesPrefilterSupersetTests</c> proves by exhaustive sweep rather than assertion.
    ///         (That is false for <c>InvariantCultureIgnoreCase</c>, which has 190 such codepoints — the choice
    ///         of Ordinal is load-bearing, not stylistic.) ASCII case-folding is exactly what both providers'
    ///         <c>LOWER()</c> implements, and lowercasing the pattern here rather than trusting <c>LIKE</c>'s own
    ///         case rules keeps it correct under a case-sensitive MySQL collation as well as a case-insensitive
    ///         one.
    ///     </para>
    /// </summary>
    internal static string JsonElementPrefixPattern(string query)
    {
        var builder = new StringBuilder("%\"");

        foreach (char c in query ?? string.Empty)
        {
            if (!char.IsAscii(c) || char.IsControl(c) || JsonEscapedAscii.Contains(c))
            {
                // cannot prove this character's stored form, so stop narrowing here
                break;
            }

            char lower = char.ToLowerInvariant(c);
            if (lower is '%' or '_' or LikeEscapeChar)
            {
                builder.Append(LikeEscapeChar);
            }

            builder.Append(lower);
        }

        return builder.Append('%').ToString();
    }

    private static async Task<List<string>> GetContentRatingValues(
        TvContext dbContext,
        string query,
        int limit,
        CancellationToken cancellationToken)
    {
        List<string> raw = await dbContext.MovieMetadata
            .Where(m => m.ContentRating != null)
            .Select(m => m.ContentRating)
            .Concat(dbContext.ShowMetadata.Where(m => m.ContentRating != null).Select(m => m.ContentRating))
            .Concat(dbContext.OtherVideoMetadata.Where(m => m.ContentRating != null).Select(m => m.ContentRating))
            .Concat(dbContext.RemoteStreamMetadata.Where(m => m.ContentRating != null).Select(m => m.ContentRating))
            .Distinct()
            .ToListAsync(cancellationToken);

        IEnumerable<string> split = raw
            .SelectMany(cr => cr.Split('/'))
            .Select(cr => cr.Trim())
            .Where(cr => !string.IsNullOrEmpty(cr))
            .Distinct();

        return FilterSortTake(split, query, limit);
    }

    /// <summary>
    ///     The one in-memory filter/sort/take every field funnels through. Both the comparison and the ordering
    ///     are ORDINAL on purpose: <c>UseRequestLocalization</c> honours <c>Accept-Language</c>, so the current
    ///     culture is caller-controlled, and <c>ToLower()</c> plus the default (linguistic)
    ///     <c>StartsWith(string)</c> would make the result depend on it — under <c>tr-TR</c>, <c>q=I</c> lowers
    ///     to <c>ı</c> and stops matching <c>Istanbul</c>. Ordinal is also what the SQL pre-filter's superset
    ///     guarantee is proved against; see <see cref="JsonElementPrefixPattern" />.
    /// </summary>
    private static List<string> FilterSortTake(IEnumerable<string> values, string query, int limit) =>
        values
            .Where(v => v.StartsWith(query, StringComparison.OrdinalIgnoreCase))
            .OrderBy(v => v, StringComparer.Ordinal)
            .Take(limit)
            .ToList();
}