From cd6f36185cdf09f20545542edc629f69b3f49c58 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sun, 26 Jul 2026 23:23:21 +0200 Subject: [PATCH 1/8] feat(578): artist typeahead covers music-video and song credits; album_artist stops 404ing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GET /api/v1/search/fields/{name}/values` sourced `artist` from `ArtistMetadata.Title` only — entity artists — so the free-text credits that `LuceneSearchIndex` also writes to the `artist` field (`MusicVideoArtist.Name`, `SongMetadata.Artists`) produced no suggestions, and `album_artist` 404'd outright. `MusicVideoArtist` turned out to be a real entity table, so it just joins the existing server-side pipeline as a `Concat` — one bounded `UNION ALL` + `LOWER(...) LIKE ... LIMIT` on both providers. `SongMetadata.Artists`/`AlbumArtists` are the hard case: EF 9 maps them as primitive collections (one JSON array per row in a single column), and neither provider can project the elements server-side — SQLite needs the SQL APPLY operator it lacks, Pomelo MySQL 9.0.0 has no primitive-collection support at all. Both failures are now pinned by a test, so a provider upgrade that fixes them shows up as a red rather than as a stale workaround. For those columns the handler pre-filters on the raw JSON (`LOWER(col) LIKE '%"%' ESCAPE '/'` — a deliberate superset, since it matches a row and not an element), caps the rows at 1000 with an `ORDER BY` that makes the truncation deterministic, then splits and exact-filters in memory. Provider portability is by construction rather than by trusting `LIKE`: the prefix is JSON-encoded before matching (which is how it is stored, and which makes the pattern pure ASCII, so SQLite's ASCII-only `lower()` and MySQL's Unicode-aware `LOWER()` agree) and lowercased in C#, so the match is correct under a case-sensitive MySQL collation as well as a case-insensitive one. The escape character is `/`, never `\`: `ESCAPE '\'` is not a portable SQL literal. An empty `q` stays supported for these fields — the row cap already bounds it, and a non-empty-`q` carve-out would make one group of fields behave differently for the same client code. Docs: new `api.search-field-values-list-columns` record (additive sibling — the base record's rule still reads true; only its body's "intentionally-uncovered gap" claim was stale), plus `api-conventions.md`, `spa-conventions.md` §12 and the regenerated `v1.json`. Decisions-Edit: yes --- .../Queries/GetSearchFieldValuesHandler.cs | 176 +++++++++++++- .../GetSearchFieldValuesHandlerTests.cs | 226 ++++++++++++++++++ .../SearchFieldValuesQueryShapeTests.cs | 107 +++++++++ ErsatzTV/Controllers/Api/SearchController.cs | 3 +- ErsatzTV/wwwroot/openapi/v1.json | 2 +- docs/api-conventions.md | 12 +- docs/decisions.md | 1 + docs/decisions/README.md | 1 + .../api/search-field-values-list-columns.md | 75 ++++++ .../records/api/search-field-values.md | 14 +- docs/spa-conventions.md | 6 +- 11 files changed, 603 insertions(+), 20 deletions(-) create mode 100644 ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs create mode 100644 docs/decisions/records/api/search-field-values-list-columns.md diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs index eb5ca3c8d..456934e37 100644 --- a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs @@ -1,3 +1,6 @@ +using System.Text; +using System.Text.Json; +using Dapper; using ErsatzTV.Core.Api.Search; using ErsatzTV.Core.Domain; using ErsatzTV.Infrastructure.Data; @@ -11,6 +14,24 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF private const int DefaultLimit = 50; private const int MaxLimit = 50; + /// + /// Row cap for the list-valued (JSON-array) columns on SongMetadata. The SQL pre-filter over the + /// raw JSON is a deliberate superset (a matching row is fetched whole, including its non-matching + /// elements), so the number of rows it can return has to be bounded independently of limit — + /// otherwise an empty q degenerates into materializing the entire table. Combined with the + /// ORDER BY in , the truncation point is deterministic rather than + /// whatever the storage engine happened to emit first. + /// + private const int ListValuedRowCap = 1000; + + /// + /// Escape character for the raw-JSON LIKE pre-filter. Deliberately NOT a backslash: the stored + /// JSON is full of backslashes (\u00E9, \u0022) and, worse, ESCAPE '\' is not a + /// portable literal — MySQL applies C-style escaping to string literals, so it reads as an unterminated + /// string, while SQLite would reject the doubled '\\' form as a two-character escape. + /// + private const char LikeEscapeChar = '/'; + public async Task> Handle( GetSearchFieldValues request, CancellationToken cancellationToken) @@ -46,19 +67,34 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF } IQueryable source = GetSource(dbContext, request.Name); - if (source is null) + string listColumn = GetSongListValuedColumn(request.Name); + if (source is null && listColumn is null) { return Option.None; } - List values = await source - .Where(v => v != null && v.ToLower().StartsWith(qLower)) - .Distinct() - .OrderBy(v => v) - .Take(limit) - .ToListAsync(cancellationToken); + var values = new List(); - return new SearchFieldValuesResponseModel(values); + if (source is not null) + { + values.AddRange( + await source + .Where(v => v != null && v.ToLower().StartsWith(qLower)) + .Distinct() + .OrderBy(v => v) + .Take(limit) + .ToListAsync(cancellationToken)); + } + + if (listColumn is not null) + { + values.AddRange( + await GetSongListValuedValues(dbContext, listColumn, request.Query, qLower, cancellationToken)); + } + + // Each contributing source already returned its own prefix-filtered, ascending first-`limit` values, so + // sorting the union and taking `limit` again yields exactly the first `limit` of the combined set. + return new SearchFieldValuesResponseModel(FilterSortTake(values.Distinct(), qLower, limit)); } private static IQueryable GetSource(TvContext dbContext, string name) => name switch @@ -68,8 +104,11 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF "director" => dbContext.Set().Select(d => d.Name), "writer" => dbContext.Set().Select(w => w.Name), "actor" => dbContext.Actors.Select(a => a.Name), - // entity artists only; free-text music-video/song artist credits are not included (known limitation) - "artist" => dbContext.ArtistMetadata.Select(m => m.Title), + // Mirrors what LuceneSearchIndex writes to the `artist` field: the music video's linked artist entity + // (ArtistMetadata.Title) plus its free-text credits (MusicVideoArtist rows). The third contributor — + // SongMetadata.Artists — is a JSON-array column and is handled by GetSongListValuedValues instead. + "artist" => dbContext.ArtistMetadata.Select(m => m.Title) + .Concat(dbContext.Set().Select(a => a.Name)), "tag" => dbContext.Set() .Where(t => t.ExternalTypeId != Tag.NfoCountryTypeId && t.ExternalTypeId != Tag.PlexNetworkTypeId) .Select(t => t.Name), @@ -87,6 +126,123 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF _ => null }; + /// + /// Maps a field name onto the SongMetadata column that backs it as an IList<string>. + /// The returned value is a compile-time constant from this switch — never caller input — so it is safe + /// to interpolate into the SQL in . + /// + private static string GetSongListValuedColumn(string name) => name switch + { + "artist" => "Artists", + "album_artist" => "AlbumArtists", + _ => null + }; + + /// + /// Reads whole values out of a SongMetadata IList<string> column. + /// + /// EF maps these as primitive collections: one JSON array per row in a single TEXT/ + /// longtext column. Neither provider can project the elements server-side — SQLite needs + /// the SQL APPLY operator it doesn't have, and Pomelo MySQL doesn't implement primitive + /// collections at all — so there is no server-side SELECT DISTINCT over the elements. + /// + /// + /// Instead: pre-filter on the raw JSON text in SQL (a superset — a row is matched, not an element, + /// so its other elements come along), cap the rows, then split and exact-filter in memory. When the + /// library has more matching rows than the suggestion list for these + /// fields is a deterministic prefix of the matches rather than the complete set. + /// + /// + private static async Task> GetSongListValuedValues( + TvContext dbContext, + string column, + string query, + string qLower, + CancellationToken cancellationToken) + { + IEnumerable rows = await dbContext.Connection.QueryAsync( + new CommandDefinition( + ListValuedSql(column), + new { Pattern = JsonElementPrefixPattern(query), Cap = ListValuedRowCap }, + cancellationToken: cancellationToken)); + + var values = new List(); + foreach (string row in rows) + { + if (string.IsNullOrWhiteSpace(row)) + { + continue; + } + + string[] elements; + try + { + elements = JsonSerializer.Deserialize(row); + } + catch (JsonException) + { + continue; + } + + foreach (string element in elements ?? []) + { + if (!string.IsNullOrEmpty(element) && element.ToLower().StartsWith(qLower)) + { + values.Add(element); + } + } + } + + return values; + } + + /// + /// The bounded pre-filter query. LOWER(...) LIKE ... ESCAPE, ORDER BY and LIMIT are + /// all portable between SQLite and MySQL, and the identifiers are unquoted so neither provider's + /// quoting dialect is baked in. + /// + private static string ListValuedSql(string column) => + $"SELECT {column} FROM SongMetadata WHERE {column} IS NOT NULL " + + $"AND LOWER({column}) LIKE @Pattern ESCAPE '{LikeEscapeChar}' " + + $"ORDER BY {column} LIMIT @Cap"; + + /// + /// Builds the raw-JSON LIKE pattern that matches a row containing an element starting with + /// . Anchoring on the opening quote (%"prefix%) keeps the superset + /// tight: it matches element starts, not substrings anywhere in the row. + /// + /// The prefix is JSON-encoded first, because that is how it is stored — EF writes the array with the + /// default JavaScriptEncoder, so Beyoncé lands on disk as Beyoncé and a + /// literal quote as ". Encoding also makes the pattern pure ASCII, which is why this + /// match means the same thing on both providers: SQLite's lower() is ASCII-only and MySQL's + /// is Unicode-aware, but on ASCII input they agree. Lowercasing the pattern here (rather than + /// relying on LIKE's own case rules) is what keeps the comparison correct under a + /// case-sensitive MySQL column collation as well as a case-insensitive one; under a + /// case-insensitive collation it simply widens the superset, which the in-memory exact filter + /// then narrows again. + /// + /// + private static string JsonElementPrefixPattern(string query) + { + string encoded = JsonSerializer.Serialize(query ?? string.Empty); + + // strip the quotes JsonSerializer wraps a string in + encoded = encoded.Substring(1, encoded.Length - 2); + + var builder = new StringBuilder("%\""); + foreach (char c in encoded.ToLowerInvariant()) + { + if (c is '%' or '_' or LikeEscapeChar) + { + builder.Append(LikeEscapeChar); + } + + builder.Append(c); + } + + return builder.Append('%').ToString(); + } + private static async Task> GetContentRatingValues( TvContext dbContext, string qLower, diff --git a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs index c3836f4e0..8b5045948 100644 --- a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs +++ b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs @@ -4,6 +4,7 @@ using ErsatzTV.Core.Domain; using ErsatzTV.Infrastructure.Data; using ErsatzTV.Tests.Support; using LanguageExt; +using Microsoft.EntityFrameworkCore; using NUnit.Framework; using Shouldly; @@ -176,6 +177,231 @@ public class GetSearchFieldValuesHandlerTests networkResult.IfSome(r => r.Values.ShouldBe(new List { "HBO" })); } + [Test] + public async Task Artist_Merges_Entity_Artists_Music_Video_Credits_And_Song_Credits() + { + await using (TvContext context = _db.CreateContext()) + { + context.ArtistMetadata.Add(Artist("Alpha Entity")); + + // negative control: an entity artist that must NOT match the "al" prefix + context.ArtistMetadata.Add(Artist("Zeta Entity")); + + context.MusicVideoMetadata.AddRange( + MusicVideo("MV One", "Alpha Credit", "Alpha Shared"), + // "Alpha Shared" appears in two rows, so DISTINCT has something to collapse + MusicVideo("MV Two", "Alpha Shared"), + MusicVideo("MV Three", "Zeta Credit")); + + context.SongMetadata.AddRange( + Song("Song One", ["Alpha Song", "Zeta Song"]), + Song("Song Two", ["Alpha Song"]), + Song("Song Three", ["Zeta Only"])); + + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + Option result = await handler.Handle( + new GetSearchFieldValues("artist", "al", 50), + CancellationToken.None); + + result.IsSome.ShouldBeTrue(); + result.IfSome(r => r.Values.ShouldBe( + new List { "Alpha Credit", "Alpha Entity", "Alpha Shared", "Alpha Song" })); + } + + [Test] + public async Task Artist_Returns_Every_Source_For_Empty_Query() + { + await using (TvContext context = _db.CreateContext()) + { + context.ArtistMetadata.Add(Artist("Entity")); + context.MusicVideoMetadata.Add(MusicVideo("MV", "Credit")); + context.SongMetadata.Add(Song("Song", ["SongArtist"])); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + Option result = await handler.Handle( + new GetSearchFieldValues("artist", string.Empty, 50), + CancellationToken.None); + + result.IsSome.ShouldBeTrue(); + result.IfSome(r => r.Values.ShouldBe(new List { "Credit", "Entity", "SongArtist" })); + } + + [Test] + public async Task Album_Artist_Returns_Song_Album_Artists_Instead_Of_NotFound() + { + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.AddRange( + Song("One", ["Performer"], ["Alpha Album Artist", "Beta Album Artist"]), + // repeated across rows so DISTINCT is exercised + Song("Two", ["Performer"], ["Alpha Album Artist"]), + // negative control: a row whose album artists are absent entirely + Song("Three", ["Performer"], null)); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + Option result = await handler.Handle( + new GetSearchFieldValues("album_artist", string.Empty, 50), + CancellationToken.None); + + result.IsSome.ShouldBeTrue(); + result.IfSome(r => r.Values.ShouldBe(new List { "Alpha Album Artist", "Beta Album Artist" })); + + // the performers on the same rows must not leak into album_artist + result.IfSome(r => r.Values.ShouldNotContain("Performer")); + } + + [Test] + public async Task List_Valued_Fields_Match_Whole_Elements_Not_Substrings_And_Ignore_Neighbours() + { + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.AddRange( + // "Neighbour" rides along in the SQL superset because "Radiohead" matches; it must be + // dropped by the in-memory exact prefix filter. + Song("One", ["Radiohead", "Neighbour"]), + // "The Radio Dept." contains "radio" but does not start with it + Song("Two", ["The Radio Dept."]), + Song("Three", ["Radio Birdman"])); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + Option result = await handler.Handle( + new GetSearchFieldValues("artist", "radio", 50), + CancellationToken.None); + + result.IsSome.ShouldBeTrue(); + result.IfSome(r => r.Values.ShouldBe(new List { "Radio Birdman", "Radiohead" })); + } + + [Test] + public async Task List_Valued_Fields_Survive_Json_Escaping_And_Like_Wildcards() + { + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.AddRange( + // non-ASCII: stored on disk JSON-escaped as é + Song("One", ["Beyoncé"]), + // an embedded quote is stored as " + Song("Two", ["\"Weird Al\" Yankovic"]), + // LIKE wildcards in the query must be treated literally + Song("Three", ["50% Off"]), + Song("Four", ["50 Cent"])); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + (await handler.Handle(new GetSearchFieldValues("artist", "beyoncé", 50), CancellationToken.None)) + .IfSome(r => r.Values.ShouldBe(new List { "Beyoncé" })); + + (await handler.Handle(new GetSearchFieldValues("artist", "\"weird", 50), CancellationToken.None)) + .IfSome(r => r.Values.ShouldBe(new List { "\"Weird Al\" Yankovic" })); + + // "50%" must not behave as the wildcard "50" — "50 Cent" must not come back + (await handler.Handle(new GetSearchFieldValues("artist", "50%", 50), CancellationToken.None)) + .IfSome(r => r.Values.ShouldBe(new List { "50% Off" })); + } + + [Test] + public async Task List_Valued_Source_Is_Bounded_By_A_Row_Cap() + { + // The handler caps the raw-JSON pre-filter at 1000 rows, ordered by the raw column. Seed one row past + // the cap whose JSON sorts last but whose element would sort FIRST in the output: an unbounded + // implementation returns it, a capped one cannot. + const string overCap = "aaa-past-the-row-cap"; + + await using (TvContext context = _db.CreateContext()) + { + for (var i = 0; i < 1000; i++) + { + context.SongMetadata.Add(Song($"Song {i}", [$"b{i:D4}"])); + } + + context.SongMetadata.Add(Song("Over", ["zzz-sorts-last", overCap])); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + Option result = await handler.Handle( + new GetSearchFieldValues("artist", string.Empty, 3), + CancellationToken.None); + + result.IsSome.ShouldBeTrue(); + result.IfSome(r => r.Values.ShouldBe(new List { "b0000", "b0001", "b0002" })); + result.IfSome(r => r.Values.ShouldNotContain(overCap)); + + // positive control: the same row IS reachable once the pre-filter narrows the candidate set below the cap + Option narrowed = await handler.Handle( + new GetSearchFieldValues("artist", "aaa", 50), + CancellationToken.None); + + narrowed.IsSome.ShouldBeTrue(); + narrowed.IfSome(r => r.Values.ShouldBe(new List { overCap })); + } + + [Test] + public async Task List_Valued_Columns_Are_Stored_As_Ascii_Escaped_Json_Arrays() + { + // Pins the on-disk encoding the LIKE pre-filter is built against. If an EF upgrade changes the + // JavaScriptEncoder (e.g. stops escaping non-ASCII), the pattern would silently stop matching — + // this fails loudly instead. It also documents why LOWER() is provider-independent here: the + // stored text is pure ASCII, so SQLite's ASCII-only lower() and MySQL's Unicode-aware LOWER() agree. + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.Add(Song("One", ["Beyoncé", "\"Q\""])); + await context.SaveChangesAsync(); + } + + await using (TvContext context = _db.CreateContext()) + { + List raw = await context.Database + .SqlQueryRaw("SELECT Artists AS Value FROM SongMetadata") + .ToListAsync(); + + raw.ShouldBe(new List { """["Beyonc\u00E9","\u0022Q\u0022"]""" }); + } + } + + private static ArtistMetadata Artist(string title) => new() + { + MetadataKind = MetadataKind.External, + DateAdded = DateTime.UtcNow, + DateUpdated = DateTime.UtcNow, + Title = title + }; + + private static MusicVideoMetadata MusicVideo(string title, params string[] artists) => new() + { + MetadataKind = MetadataKind.External, + DateAdded = DateTime.UtcNow, + DateUpdated = DateTime.UtcNow, + Title = title, + Artists = artists.Map(a => new MusicVideoArtist { Name = a }).ToList() + }; + + private static SongMetadata Song(string title, IList artists, IList albumArtists = null) => new() + { + MetadataKind = MetadataKind.External, + DateAdded = DateTime.UtcNow, + DateUpdated = DateTime.UtcNow, + Title = title, + Artists = artists, + AlbumArtists = albumArtists + }; + [Test] public async Task Dedupes_Repeated_Values() { diff --git a/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs b/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs new file mode 100644 index 000000000..e3c44f535 --- /dev/null +++ b/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs @@ -0,0 +1,107 @@ +using ErsatzTV.Core.Domain; +using ErsatzTV.Infrastructure; +using ErsatzTV.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using Shouldly; + +namespace ErsatzTV.Tests.Application.Search; + +/// +/// Provider-shape guards for the artist / album_artist facet-value sources (#578). +/// +/// runs against in-memory SQLite, so it structurally +/// cannot see a MySQL translation or collation difference. These tests build the same LINQ against the +/// Pomelo MySQL provider and assert the generated SQL — ToQueryString compiles the query without +/// touching a server, so no MySQL instance is needed. +/// +/// +[TestFixture] +[NonParallelizable] +public class SearchFieldValuesQueryShapeTests +{ + private bool _wasSqlite; + + [SetUp] + public void SetUp() => _wasSqlite = TvContext.IsSqlite; + + [TearDown] + public void TearDown() => TvContext.IsSqlite = _wasSqlite; + + [Test] + public void Artist_Entity_Union_Translates_On_Both_Providers_With_Lower_And_A_Row_Limit() + { + foreach ((string provider, Func create) in Providers()) + { + using TvContext context = create(); + + // exactly the queryable half of GetSearchFieldValuesHandler.GetSource("artist") + string sql = context.ArtistMetadata.Select(m => m.Title) + .Concat(context.Set().Select(a => a.Name)) + .Where(v => v != null && v.ToLower().StartsWith("a")) + .Distinct() + .OrderBy(v => v) + .Take(50) + .ToQueryString(); + + // case-insensitivity comes from LOWER() on the column, not from the provider's LIKE collation + sql.ShouldContain("LOWER(", Case.Insensitive, $"{provider}: {sql}"); + sql.ShouldContain("LIKE", Case.Insensitive, $"{provider}: {sql}"); + sql.ShouldContain("MusicVideoArtist", Case.Insensitive, $"{provider}: {sql}"); + // the whole thing is one bounded server-side query, never a client-side scan + sql.ShouldContain("LIMIT", Case.Insensitive, $"{provider}: {sql}"); + } + } + + [Test] + public void Song_List_Columns_Cannot_Be_Projected_Server_Side_On_Either_Provider() + { + // Documents WHY the handler drops to raw SQL for SongMetadata.Artists / .AlbumArtists rather than + // SelectMany-ing them: EF maps them as JSON primitive collections and neither provider can translate + // the projection (SQLite needs APPLY; Pomelo has no primitive-collection support). If a provider + // upgrade ever makes this translate, this test fails and the raw-SQL path can be retired. + foreach ((string provider, Func create) in Providers()) + { + using TvContext context = create(); + + Should.Throw( + () => context.SongMetadata.SelectMany(m => m.Artists).Distinct().Take(50).ToQueryString(), + $"{provider} unexpectedly translated a primitive-collection projection"); + + Should.Throw( + () => context.SongMetadata.SelectMany(m => m.AlbumArtists).Distinct().Take(50).ToQueryString(), + $"{provider} unexpectedly translated a primitive-collection projection"); + } + } + + private static IEnumerable<(string Provider, Func Create)> Providers() => + [ + ("sqlite", Sqlite), + ("mysql", MySql) + ]; + + private static TvContext Sqlite() + { + TvContext.IsSqlite = true; + var builder = new DbContextOptionsBuilder(); + builder.UseSqlite("Data Source=:memory:"); + return Create(builder.Options); + } + + private static TvContext MySql() + { + TvContext.IsSqlite = false; + var builder = new DbContextOptionsBuilder(); + builder.UseMySql( + "Server=localhost;Database=ersatztv_query_shape;User=root;Password=ersatztv;", + new MySqlServerVersion(new Version(8, 0, 36))); + return Create(builder.Options); + } + + private static TvContext Create(DbContextOptions options) => + new( + options, + NullLoggerFactory.Instance, + new SlowQueryInterceptor(NullLogger.Instance)); +} diff --git a/ErsatzTV/Controllers/Api/SearchController.cs b/ErsatzTV/Controllers/Api/SearchController.cs index 1f9c39d86..78e306cb4 100644 --- a/ErsatzTV/Controllers/Api/SearchController.cs +++ b/ErsatzTV/Controllers/Api/SearchController.cs @@ -205,7 +205,8 @@ public class SearchController(IMediator mediator) : ControllerBase "Returns distinct whole values from the database for the given text field, filtered by an " + "optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. " + "404 when the field is unknown, is not a text field, or is a text field with no distinct-value " + - "source.")] + "source. The list-valued music fields (artist, album_artist) are bounded best-effort: on a very " + + "large library they return a deterministic subset of the matches rather than every match.")] [EndpointGroupName("general")] [ProducesResponseType(typeof(SearchFieldValuesResponseModel), StatusCodes.Status200OK)] [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)] diff --git a/ErsatzTV/wwwroot/openapi/v1.json b/ErsatzTV/wwwroot/openapi/v1.json index 383fd5bb5..9d15967d6 100644 --- a/ErsatzTV/wwwroot/openapi/v1.json +++ b/ErsatzTV/wwwroot/openapi/v1.json @@ -18063,7 +18063,7 @@ "Search" ], "summary": "List distinct database values for a text search field", - "description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source.", + "description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source. The list-valued music fields (artist, album_artist) are bounded best-effort: on a very large library they return a deterministic subset of the matches rather than every match.", "operationId": "GetSearchFieldValues", "parameters": [ { diff --git a/docs/api-conventions.md b/docs/api-conventions.md index 260686024..fc2b28e11 100644 --- a/docs/api-conventions.md +++ b/docs/api-conventions.md @@ -444,7 +444,7 @@ standard credential (catalog-read tier — no `[RequiresAuthentication]`): Query params: `q` (optional prefix filter, case-insensitive, default empty) and `limit` (optional, clamped `1..50`, default 50). `{name}` is allow-listed to `SearchFieldCatalog` fields with `type: "text"` AND a distinct-value source in the database — an unknown field, a non-text field (e.g. -an enum), or a text field without a source (`title`, `show_title`, `album_artist`) 404s rather than +an enum), or a text field without a source (`title`, `show_title`) 404s rather than returning an empty list, since enum fields already ship their values inline on `GET /api/v1/search/fields` and never need this endpoint. Returns `SearchFieldValuesResponseModel` (`{ values: string[] }`), sourced from a per-field distinct-values DB query (`IDbContextFactory`), @@ -452,6 +452,16 @@ not the Lucene term dictionary — analyzed text fields store lowercased word to No server-side caching. Powers the visual rule builder's value-input combobox for text fields; see `docs/decisions.md` 2026-07-23 (#434) and `spa-conventions.md` §12. +**Bounded best-effort for list-valued fields (#578)**: `artist` and `album_artist` are backed (wholly +or partly) by `SongMetadata.Artists`/`AlbumArtists`, which EF maps as **primitive collections** — one +JSON array per row in a single column, with no server-side projection on either provider. `album_artist` +therefore no longer 404s, and `artist` now also covers free-text music-video (`MusicVideoArtist`) and +song credits, not only entity artists. For these fields the handler pre-filters on the raw JSON in SQL +and caps the rows it reads, so on a very large library the response is a **deterministic subset** of the +matches rather than every match — say so in the `[EndpointDescription]` of any endpoint that adopts this +shape, and never let an unbounded scan stand in for it. Full rationale, the rejected normalized-side-table +alternative and the dual-provider `LIKE`/`LOWER` portability rules: `api.search-field-values-list-columns`. + **Param + DTO expansion (#293, cap `search/all-items`)**: no new endpoint — `GET /api/v1/search/all-items` gained two **optional** query params (`pageSize` default 500, clamped 1–1000 via the §1 Logs `Math.Clamp` precedent; `pageNum` 0-based, clamped `0..2_000_000` so `pageNum * pageSize` can't overflow `int` to a 500) diff --git a/docs/decisions.md b/docs/decisions.md index 1ebfebcc2..9ea033132 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -215,3 +215,4 @@ another doc or an old issue comment should land here and then follow the link. - 2026-07-25 — Rule-builder group nesting is bounded-arbitrary depth (`MAX_GROUP_DEPTH`), not one level (#436) — [`spa.rulebuilder-nesting`](decisions/records/spa/rulebuilder-nesting.md) - 2026-07-25 — The rationale-edit marker is a git trailer, not a substring anywhere in the commit range (#609) — [`ci.decisions-edit-trailer`](decisions/records/ci/decisions-edit-trailer.md) - 2026-07-25 — UI-E2E: headless Playwright flows in the existing `functional-e2e` job, browser baked into the CI image (#445) — [`ci.ui-e2e-harness`](decisions/records/ci/ui-e2e-harness.md) +- 2026-07-26 — List-valued (JSON primitive collection) facet fields get bounded best-effort values via a raw-JSON LIKE pre-filter, not a normalized side table (#578) — [`api.search-field-values-list-columns`](decisions/records/api/search-field-values-list-columns.md) diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 901b99aaf..ea26ec8c7 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -29,6 +29,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `api.scheduling-hardening` | Create/Replace handlers guard against null/whitespace `name` (`IsNullOrWhiteSpace`, not just `Length`) to prevent NRE-500s, template-item overlap validation compares by index (not record value-equality) to catch exact-duplicate items, and unreachable 404 `ProducesResponseType` attributes on create-only actions are trimmed. | 2026-07-13 | [link](records/api/scheduling-hardening.md) | | `api.search-allitems-paging` | `GET /api/v1/search/all-items` is paginated (capped page size, `Totals` field) to bound DoS exposure; the SPA add-all flow pages to completeness instead of relying on an unbounded response. | 2026-07-18 | [link](records/api/search-allitems-paging.md) | | `api.search-field-values` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for one of a narrow allow-list of catalog fields (not the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a typeahead suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source; case-insensitive prefix-filtered on `q`, `limit` clamped to `[1, 50]` (default 50). | 2026-07-23 | [link](records/api/search-field-values.md) | +| `api.search-field-values-list-columns` | A `text` search field whose values live in an EF **primitive collection** (one JSON array per row in a single column — `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) IS served by `GET /api/v1/search/fields/{name}/values`, not 404'd, but as **bounded best-effort**: SQL pre-filters on the raw JSON with `LOWER(col) LIKE '%"%' ESCAPE '/'` — a deliberate superset, since it matches a row, not an element — capped at a fixed row cap (1000) with an `ORDER BY` on the raw column so the truncation point is deterministic; the rows are then split, exact-prefix-filtered, deduped, sorted and `limit`ed in memory. An empty `q` is supported (it degenerates to `%"%`, still capped), so these fields need no special-case client contract. The prefix is JSON-encoded before matching because that is how it is stored, which also makes the pattern pure ASCII and therefore makes `LOWER()` mean the same thing on SQLite (ASCII-only) and MySQL (Unicode-aware); the pattern is lowercased in C# rather than relying on `LIKE`'s own case rules, so the match is correct under a case-sensitive MySQL collation as well as a case-insensitive one. The escape character is `/`, never `\` — `ESCAPE '\'` is not a portable SQL literal. | 2026-07-26 | [link](records/api/search-field-values-list-columns.md) | | `api.search-paging-cap` | Search stays capped at 100 items per media kind; an overflowing kind's "See all" reuses library-browse paging instead of adding new API surface. | 2026-07-11 | [link](records/api/search-paging-cap.md) | | `api.versioning-v1` | The entire `/api` surface is versioned to `/api/v1` uniformly (no unversioned corner); legacy unversioned callers are rewritten in-pipeline (not redirected) with Deprecation/Link/Sunset headers, and post-freeze `/api/v1` is additive-only — a breaking change requires `/api/v2`. | 2026-07-13 | [link](records/api/versioning-v1.md) | | `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) | diff --git a/docs/decisions/records/api/search-field-values-list-columns.md b/docs/decisions/records/api/search-field-values-list-columns.md new file mode 100644 index 000000000..6bfaa4353 --- /dev/null +++ b/docs/decisions/records/api/search-field-values-list-columns.md @@ -0,0 +1,75 @@ +--- +key: api.search-field-values-list-columns +title: '2026-07-26 — list-valued (JSON primitive collection) fields get bounded best-effort facet values via a raw-JSON LIKE pre-filter, not a normalized side table (#578)' +status: active +since: '2026-07-26' +supersedes: none +superseded-by: none +rule: 'A `text` search field whose values live in an EF **primitive collection** (one JSON array per row in a single column — `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) IS served by `GET /api/v1/search/fields/{name}/values`, not 404''d, but as **bounded best-effort**: SQL pre-filters on the raw JSON with `LOWER(col) LIKE ''%"%'' ESCAPE ''/''` — a deliberate superset, since it matches a row, not an element — capped at a fixed row cap (1000) with an `ORDER BY` on the raw column so the truncation point is deterministic; the rows are then split, exact-prefix-filtered, deduped, sorted and `limit`ed in memory. An empty `q` is supported (it degenerates to `%"%`, still capped), so these fields need no special-case client contract. The prefix is JSON-encoded before matching because that is how it is stored, which also makes the pattern pure ASCII and therefore makes `LOWER()` mean the same thing on SQLite (ASCII-only) and MySQL (Unicode-aware); the pattern is lowercased in C# rather than relying on `LIKE`''s own case rules, so the match is correct under a case-sensitive MySQL collation as well as a case-insensitive one. The escape character is `/`, never `\` — `ESCAPE ''\''` is not a portable SQL literal.' +signals: 'artist typeahead free-text credits, album_artist 404, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist suggestions, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, raw JSON LIKE pre-filter, ListValuedRowCap, bounded best-effort facet values, LIKE ESCAPE portability · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs` · issues: #578, #434' +mechanics: '`GetSearchFieldValuesHandler.GetSongListValuedColumn` / `.GetSongListValuedValues` / `.ListValuedSql` / `.JsonElementPrefixPattern`; `SearchFieldValuesQueryShapeTests` (dual-provider `ToQueryString` guard); `api.search-field-values` (the base endpoint contract this extends)' +--- + +Extends `api.search-field-values` rather than superseding it: that record's rule still reads true — +whole values, from the database, not the Lucene term dictionary, 404 for a field with no +distinct-value source. What changed is only **which** fields have a source. Its body's claim that +music-video/song artist credits are "a known, intentionally-uncovered gap" and that `album_artist` +has no server-side source was corrected in place. + +**The three `artist` sources are not the same problem.** `LuceneSearchIndex` writes the `artist` +field from three places, and only two of them are ordinary columns: + +- `ArtistMetadata.Title` — a plain column on a plain table. +- `MusicVideoArtist.Name` — also a real entity table (`MusicVideoMetadata.HasMany(m => m.Artists)`), + so the free-text music-video credits are directly `SELECT DISTINCT`-able. These two are simply + `Concat`ed into the existing server-side pipeline + (`.Where(v => v.ToLower().StartsWith(qLower)).Distinct().OrderBy(v => v).Take(limit)`), which EF + emits as one bounded `UNION ALL` + `LOWER(...) LIKE ... LIMIT` on both providers. +- `SongMetadata.Artists` (and, for `album_artist`, `SongMetadata.AlbumArtists`) — an + `IList` that EF 9 maps as a **primitive collection**: no `HasConversion` anywhere, one + JSON array per row in a single `TEXT`/`longtext` column. This one has no server-side projection at + all. Verified empirically against both providers: SQLite reports *"Translating this query requires + the SQL APPLY operation, which is not supported on SQLite"*, and Pomelo MySQL 9.0.0 reports + *"Primitive collections support has not been enabled"*. `SearchFieldValuesQueryShapeTests` pins + both failures, so if a provider upgrade ever makes the projection translate, the raw-SQL path can + be retired on a red test rather than on a hunch. + +**Why the LIKE superset and not a normalized `SongArtist` table.** A real join table (the shape +`MusicVideoArtist` already has) would make this exact and indexable forever, but it costs a +dual-provider schema migration plus a data backfill, changes to every scanner write path that +populates `SongMetadata.Artists`, changes to the Lucene indexer, and leaves two representations of +the same fact to drift apart — a large blast radius for a typeahead convenience. Rejected on cost. +The raw-JSON pre-filter is confined to one private method, needs no migration, and is a strictly +additive read path. Also rejected: querying the Lucene term dictionary, which is the thing +`api.search-field-values` exists to avoid (analyzed `TextField`s store lowercased word tokens). + +**The bound is the point, and it is tested behaviourally.** `ListValuedRowCap = 1000` bounds rows +fetched, not values returned, because the pre-filter matches rows whole. The `ORDER BY` on the raw +column is what makes the truncation deterministic — and therefore testable: +`List_Valued_Source_Is_Bounded_By_A_Row_Cap` seeds 1001 rows where the over-cap row's JSON sorts +*last* but its element would sort *first* in the output, so an unbounded implementation returns it +and a capped one cannot. A positive control in the same test narrows `q` until that row is back +under the cap and asserts it reappears, so the test cannot pass by the value being unreachable for +some unrelated reason. + +**`album_artist` 404 → 200 is additive.** Nothing consumes the 404 as a signal: the SPA's +`getSearchFieldValues` (`web/src/api/search.ts`) treats any non-200 as "no suggestions, fall back to +a free-text input", which is exactly what it will now do less often. Per `api.versioning-v1`, +widening which fields return values — and which values a field returns — adds capability without +removing any, so it needs no `/api/v2`. + +**Known imprecision, deliberately accepted.** On a library with more than 1000 rows matching the +pre-filter, the suggestions for `artist`/`album_artist` are a deterministic prefix of the matches +rather than every match. This is stated on the endpoint's `[EndpointDescription]` so it reaches the +OpenAPI document. The alternative — requiring a non-empty `q` for list-valued fields — was rejected: +it would make one group of fields behave differently from every other field for the same client +code, for a case the row cap already handles. + +**Dual-provider verification, and its limit.** The queryable half is pinned on both providers by +`SearchFieldValuesQueryShapeTests`, which compiles the query with `ToQueryString()` — no server +needed. The raw-SQL half cannot be executed against MySQL from the unit-test suite; it is defended +by construction instead (portable `LOWER`/`LIKE`/`ESCAPE`/`ORDER BY`/`LIMIT`, unquoted identifiers, +a non-backslash escape character, and an ASCII-only pattern so collation and `LOWER()` semantics +cannot diverge) plus `List_Valued_Columns_Are_Stored_As_Ascii_Escaped_Json_Arrays`, which pins the +on-disk JSON encoding the pattern is built against so an EF encoder change fails loudly instead of +silently returning nothing. diff --git a/docs/decisions/records/api/search-field-values.md b/docs/decisions/records/api/search-field-values.md index 7314aae14..68262f3ca 100644 --- a/docs/decisions/records/api/search-field-values.md +++ b/docs/decisions/records/api/search-field-values.md @@ -23,8 +23,9 @@ an empty list for a field that will never have values — a 404 tells a caller " **DB-sourced, not the search index.** The handler injects `IDbContextFactory` and resolves an explicit per-field-name `IQueryable` (or, for a few special cases, an in-memory list) rather than querying `ISearchIndex`: `genre`/`show_genre` → `Set()`, `studio` → `Set()`, `director` → -`Set()`, `writer` → `Set()`, `actor` → `Actors`, `artist` → `ArtistMetadata.Title` (entity -artists only — free-text music-video/song artist credits are a known, intentionally-uncovered gap), `tag` → +`Set()`, `writer` → `Set()`, `actor` → `Actors`, `artist` → `ArtistMetadata.Title` +concatenated with `Set().Name` (plus `SongMetadata.Artists` via the list-valued path — +see `api.search-field-values-list-columns`, #578), `tag` → `Set()` excluding `Tag.NfoCountryTypeId`/`Tag.PlexNetworkTypeId` (reapplying the indexer's own exclusions so country/network strings don't leak in as tags), `network` → `Set()` filtered to `Tag.PlexNetworkTypeId`, `collection` → `Collections`, `video_codec` → `MediaStreams` filtered to @@ -36,10 +37,11 @@ of queried: `state` (the fixed 4-value `MediaItemState` enum) and `video_dynamic `MovieMetadata`/`ShowMetadata`/`OtherVideoMetadata`/`RemoteStreamMetadata`, so the handler pulls the distinct raw strings then `Split('/')`s, trims, and dedupes in memory before the same prefix-filter/sort/take — this matches what search actually matches on, rather than surfacing the compound string as one facet value. -**`title`, `show_title`, `album_artist` are explicitly NOT supported** (404, free-text fallback): `title`/ -`show_title` are near-unique free-text fields spanning ~9 metadata tables where a distinct list of every -title isn't a useful facet; `album_artist` backs onto `SongMetadata.AlbumArtists`, a value-converted -`IList` column EF can't translate into a server-side distinct query. +**`title` and `show_title` are explicitly NOT supported** (404, free-text fallback): they are near-unique +free-text fields spanning ~9 metadata tables where a distinct list of every title isn't a useful facet. +`album_artist` was originally 404 here for a different reason — it backs onto `SongMetadata.AlbumArtists`, +which EF can't translate into a server-side distinct query — and since #578 it returns bounded best-effort +values instead; see `api.search-field-values-list-columns`. **Why a thin query, not a cache.** No result cache, no debounce on the server side (the SPA combobox debounces the keystroke) — each per-field query is a bounded, indexed `Distinct`/`Take`; adding a cache diff --git a/docs/spa-conventions.md b/docs/spa-conventions.md index c7c4c016b..9a67c771a 100644 --- a/docs/spa-conventions.md +++ b/docs/spa-conventions.md @@ -624,7 +624,11 @@ not wired here. `GET /api/v1/search/fields/{name}/values?q=&limit=`), debounced on keystroke, prefix-matching the in-progress value against distinct terms already in the index. It always allows free-text entry as a fallback — a 404 (non-text field) or an empty result list (e.g. ElasticSearch backend) degrades to a - plain text input rather than blocking the rule. + plain text input rather than blocking the rule. Since #578 (`api.search-field-values-list-columns`) + `album_artist` returns values instead of 404ing, and `artist` covers free-text music-video/song credits + as well as entity artists; for those two the server's list is **bounded best-effort** on a very large + library, so the free-text fallback stays load-bearing — never treat an absent suggestion as an invalid + value. - **Single-child-group normalization** (#438) — `normalizeGroup` (`validation.ts`) coerces a group's `match` to `all` whenever it has fewer than two children, recursively. A one-child `any` group is semantically identical to `all` but doesn't round-trip through `compile`→`parse` (the compiled Lucene -- 2.47.3 From 1641ca83057e891df5c681abced18c1aa4b50ea2 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sun, 26 Jul 2026 23:52:05 +0200 Subject: [PATCH 2/8] fix(578): the LIKE prefilter under-matched every accented artist; make the superset provable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of 1b78dc9e found the pre-filter's correctness claim was false, and the claim was in the decision record as well as the code. F1 (high). The pattern JSON-encoded the whole query prefix on the reasoning that the stored text escapes non-ASCII, so encoding the prefix the same way would line up. It does not: SQL LOWER() lowercases the *escape text* (`É` -> `é`); it cannot case-fold the codepoint that escape denotes. So `q=é` built `%"é%`, the stored `Édith Piaf` never matched, and the row was discarded before the in-memory filter could accept it. Every accented artist — Beyoncé, Björk, Sigur Rós, Édith Piaf — was silently unsuggestable, which in a music library is the common case. The invariant that was missing, now stated in the code: the SQL pre-filter is an OPTIMIZATION. It may over-match; it must never under-match. Correctness lives in the in-memory filter. So the pattern now narrows only on the leading run of characters the JSON writer stores verbatim and stops at the first character it cannot prove — `q=Beyoncé` still narrows on `beyonc`, `q=é` narrows on nothing and leans on the row cap. Soundness rests on two facts now asserted by exhaustive computation rather than argued: no non-ASCII codepoint in U+0080..U+10FFFF OrdinalIgnoreCase-equals a printable ASCII character (false for InvariantCultureIgnoreCase, which folds ~190 — the choice of Ordinal is load-bearing), and the exact set of ASCII the encoder escapes. F1b. `UseRequestLocalization` honours Accept-Language, so the culture was caller-controlled and `ToLower()` plus the default linguistic `StartsWith(string)` let a header change the answer. Comparison is now OrdinalIgnoreCase and ordering StringComparer.Ordinal throughout — including the shared FilterSortTake that state/video_dynamic_range/content_rating also use. Sets unchanged, order now ordinal rather than culture-dependent. F2. The merge comment asserted an exactness the code does not have: sources truncate by their own ordering (DB collation / primary key), not the merge's, so a dropped value can outrank a survivor. Comment and record now say best-effort, exact only below the truncation points. F3/F4. The cap now rides `ORDER BY Id` rather than the JSON column: MySQL sorts TEXT by only max_sort_length bytes, so the old ordering was not deterministic there, and sorting the whole matching set was avoidable work. What the cap still does NOT bound is the scan — a leading-wildcard LIKE cannot seek an index — so that cost is now documented as accepted, with a normalized `SongArtist` table named as the follow-up candidate rather than left implicit. Every clause above is covered by a test verified to FAIL when that clause is mutated (old pattern builder: 5 red; culture chain: 3 red; cap=3 / cap=limit / ORDER BY json / no cap: red each). F5. Converted to a proper supersession. The old record did not merely hold a stale fact — it recorded song/music-video credits as an "intentionally-uncovered gap" and album_artist as unsupported, and this reverses that call, which `docs.decision-lifecycle` says is never a line-edit. `api.search-field-values` is archived with its original prose restored, and `api.search-field-values-sources` replaces it carrying the whole endpoint contract. --- .../Queries/GetSearchFieldValuesHandler.cs | 143 ++++++++---- .../GetSearchFieldValuesHandlerTests.cs | 216 ++++++++++++++++-- ...SearchFieldValuesPrefilterSupersetTests.cs | 122 ++++++++++ .../SearchFieldValuesQueryShapeTests.cs | 13 +- ErsatzTV/Controllers/Api/SearchController.cs | 5 +- ErsatzTV/wwwroot/openapi/v1.json | 2 +- docs/api-conventions.md | 17 +- docs/decisions.md | 4 +- docs/decisions/README.md | 3 +- .../api/search-field-values.md | 22 +- .../api/search-field-values-list-columns.md | 75 ------ .../api/search-field-values-sources.md | 116 ++++++++++ docs/spa-conventions.md | 4 +- 13 files changed, 569 insertions(+), 173 deletions(-) create mode 100644 ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs rename docs/decisions/{records => archive}/api/search-field-values.md (70%) delete mode 100644 docs/decisions/records/api/search-field-values-list-columns.md create mode 100644 docs/decisions/records/api/search-field-values-sources.md diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs index 456934e37..ac8c6a1aa 100644 --- a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs @@ -18,11 +18,13 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// Row cap for the list-valued (JSON-array) columns on SongMetadata. The SQL pre-filter over the /// raw JSON is a deliberate superset (a matching row is fetched whole, including its non-matching /// elements), so the number of rows it can return has to be bounded independently of limit — - /// otherwise an empty q degenerates into materializing the entire table. Combined with the - /// ORDER BY in , the truncation point is deterministic rather than - /// whatever the storage engine happened to emit first. + /// otherwise an empty q degenerates into materializing the entire table. The cap is applied on + /// ORDER BY Id, which is a unique integer primary key on both providers: unlike ordering on the + /// JSON column itself, that makes the truncation point genuinely deterministic (MySQL sorts TEXT + /// using only the first max_sort_length bytes, so long JSON rows sharing a prefix would tie + /// arbitrarily) and lets the engine walk the primary key instead of sorting the whole matching set. /// - private const int ListValuedRowCap = 1000; + internal const int ListValuedRowCap = 1000; /// /// Escape character for the raw-JSON LIKE pre-filter. Deliberately NOT a backslash: the stored @@ -32,6 +34,14 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// private const char LikeEscapeChar = '/'; + /// + /// The ASCII characters the JSON writer escapes rather than emitting verbatim (& and friends + /// become &). A prefix character in this set cannot be matched literally against the stored + /// text, so it terminates the narrowing run in . + /// SearchFieldValuesPrefilterSupersetTests pins this set against the encoder itself. + /// + private const string JsonEscapedAscii = "\"&'+<>\\`"; + public async Task> Handle( GetSearchFieldValues request, CancellationToken cancellationToken) @@ -45,17 +55,22 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF } int limit = request.Limit <= 0 ? DefaultLimit : Math.Clamp(request.Limit, 1, MaxLimit); - string qLower = (request.Query ?? string.Empty).ToLower(); + string query = request.Query ?? string.Empty; + + // Invariant, not current-culture: UseRequestLocalization honours Accept-Language, so a caller can select + // tr-TR and turn `q=I` into `ı` — which then matches nothing a Turkish-dotless-i-free library contains. + // This feeds the EF-translated filter, which has no StringComparison overload EF can translate. + string qLower = query.ToLowerInvariant(); // in-memory special cases (no DB query needed) switch (request.Name) { case "state": return new SearchFieldValuesResponseModel( - FilterSortTake(Enum.GetNames(), qLower, limit)); + FilterSortTake(Enum.GetNames(), query, limit)); case "video_dynamic_range": return new SearchFieldValuesResponseModel( - FilterSortTake(["hdr", "sdr"], qLower, limit)); + FilterSortTake(["hdr", "sdr"], query, limit)); } await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken); @@ -63,7 +78,7 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF if (request.Name == "content_rating") { return new SearchFieldValuesResponseModel( - await GetContentRatingValues(dbContext, qLower, limit, cancellationToken)); + await GetContentRatingValues(dbContext, query, limit, cancellationToken)); } IQueryable source = GetSource(dbContext, request.Name); @@ -88,16 +103,18 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF if (listColumn is not null) { - values.AddRange( - await GetSongListValuedValues(dbContext, listColumn, request.Query, qLower, cancellationToken)); + values.AddRange(await GetSongListValuedValues(dbContext, listColumn, query, cancellationToken)); } - // Each contributing source already returned its own prefix-filtered, ascending first-`limit` values, so - // sorting the union and taking `limit` again yields exactly the first `limit` of the combined set. - return new SearchFieldValuesResponseModel(FilterSortTake(values.Distinct(), qLower, limit)); + // ORDERING IS BEST-EFFORT, NOT EXACT. Each source truncates using its own ordering — the EF source by the + // database collation (SQLite's NOCASE/BINARY is ASCII-only), the list source by primary key — and neither + // is the ordinal ordering applied here. So when a source actually truncates, a value it dropped may have + // outranked one that survived: with "Zulu" and "Éclair" and limit=1 the EF source keeps "Zulu" and the + // merge never sees "Éclair". Below the truncation points (the normal typeahead case) the result is exact. + return new SearchFieldValuesResponseModel(FilterSortTake(values.Distinct(StringComparer.Ordinal), query, limit)); } - private static IQueryable GetSource(TvContext dbContext, string name) => name switch + internal static IQueryable GetSource(TvContext dbContext, string name) => name switch { "genre" or "show_genre" => dbContext.Set().Select(g => g.Name), "studio" => dbContext.Set().Select(s => s.Name), @@ -148,16 +165,17 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// /// /// Instead: pre-filter on the raw JSON text in SQL (a superset — a row is matched, not an element, - /// so its other elements come along), cap the rows, then split and exact-filter in memory. When the - /// library has more matching rows than the suggestion list for these - /// fields is a deterministic prefix of the matches rather than the complete set. + /// so its other elements come along), cap the rows by primary key, then split and exact-filter in + /// memory. Correctness lives in the in-memory filter; the SQL pattern is only an optimization and is + /// allowed to over-match but never to under-match. When the library has more matching rows than + /// the suggestion list for these fields is the lowest-id prefix of + /// the matches rather than the complete set. /// /// private static async Task> GetSongListValuedValues( TvContext dbContext, string column, string query, - string qLower, CancellationToken cancellationToken) { IEnumerable rows = await dbContext.Connection.QueryAsync( @@ -186,7 +204,7 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF foreach (string element in elements ?? []) { - if (!string.IsNullOrEmpty(element) && element.ToLower().StartsWith(qLower)) + if (!string.IsNullOrEmpty(element) && element.StartsWith(query, StringComparison.OrdinalIgnoreCase)) { values.Add(element); } @@ -199,45 +217,66 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// /// The bounded pre-filter query. LOWER(...) LIKE ... ESCAPE, ORDER BY and LIMIT are /// all portable between SQLite and MySQL, and the identifiers are unquoted so neither provider's - /// quoting dialect is baked in. + /// quoting dialect is baked in. The cap rides ORDER BY Id — a unique integer key both providers + /// can walk — rather than the JSON column, so the truncation point is well-defined and no full sort of + /// the matching set is required. /// - private static string ListValuedSql(string column) => + internal static string ListValuedSql(string column) => $"SELECT {column} FROM SongMetadata WHERE {column} IS NOT NULL " + $"AND LOWER({column}) LIKE @Pattern ESCAPE '{LikeEscapeChar}' " + - $"ORDER BY {column} LIMIT @Cap"; + "ORDER BY Id LIMIT @Cap"; /// - /// Builds the raw-JSON LIKE pattern that matches a row containing an element starting with - /// . Anchoring on the opening quote (%"prefix%) keeps the superset - /// tight: it matches element starts, not substrings anywhere in the row. + /// Builds the raw-JSON LIKE pattern used to narrow the rows fetched for a list-valued column. /// - /// The prefix is JSON-encoded first, because that is how it is stored — EF writes the array with the - /// default JavaScriptEncoder, so Beyoncé lands on disk as Beyoncé and a - /// literal quote as ". Encoding also makes the pattern pure ASCII, which is why this - /// match means the same thing on both providers: SQLite's lower() is ASCII-only and MySQL's - /// is Unicode-aware, but on ASCII input they agree. Lowercasing the pattern here (rather than - /// relying on LIKE's own case rules) is what keeps the comparison correct under a - /// case-sensitive MySQL column collation as well as a case-insensitive one; under a - /// case-insensitive collation it simply widens the superset, which the in-memory exact filter - /// then narrows again. + /// This is an optimization with one hard obligation: it must never under-match. Every row + /// holding an element that starts with under + /// — the comparison + /// then applies — has to survive it. Over-matching is free; + /// the in-memory filter discards the extras. + /// + /// + /// So the pattern narrows on the leading run of ASCII characters the JSON writer stores + /// verbatim, and stops at the first character it cannot reason about — a non-ASCII character, or + /// one of . A query starting with such a character narrows to the bare + /// element-opening anchor %"% and leans entirely on the row cap and the in-memory filter. + /// + /// + /// Why that is sound, and why an earlier "JSON-encode the whole prefix" version was NOT: the stored + /// text escapes non-ASCII, so Édith is on disk as Édith. SQL LOWER() + /// lowercases the escape text, giving É — it cannot case-fold the codepoint that + /// escape denotes, so a q=é pattern of é never matched and every accented artist + /// was silently unsuggestable. Restricting the run to verbatim ASCII removes the escape from the + /// comparison entirely. And for a run character c, any element character that + /// OrdinalIgnoreCase-equals c is itself ASCII — no non-ASCII codepoint in all of Unicode + /// OrdinalIgnoreCase-equals a printable ASCII character, which + /// SearchFieldValuesPrefilterSupersetTests proves by exhaustive sweep rather than assertion. + /// (That is false for InvariantCultureIgnoreCase, which has 190 such codepoints — the choice + /// of Ordinal is load-bearing, not stylistic.) ASCII case-folding is exactly what both providers' + /// LOWER() implements, and lowercasing the pattern here rather than trusting LIKE's own + /// case rules keeps it correct under a case-sensitive MySQL collation as well as a case-insensitive + /// one. /// /// - private static string JsonElementPrefixPattern(string query) + internal static string JsonElementPrefixPattern(string query) { - string encoded = JsonSerializer.Serialize(query ?? string.Empty); - - // strip the quotes JsonSerializer wraps a string in - encoded = encoded.Substring(1, encoded.Length - 2); - var builder = new StringBuilder("%\""); - foreach (char c in encoded.ToLowerInvariant()) + + foreach (char c in query ?? string.Empty) { - if (c is '%' or '_' or LikeEscapeChar) + if (!char.IsAscii(c) || char.IsControl(c) || JsonEscapedAscii.Contains(c)) + { + // cannot prove this character's stored form, so stop narrowing here + break; + } + + char lower = char.ToLowerInvariant(c); + if (lower is '%' or '_' or LikeEscapeChar) { builder.Append(LikeEscapeChar); } - builder.Append(c); + builder.Append(lower); } return builder.Append('%').ToString(); @@ -245,7 +284,7 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF private static async Task> GetContentRatingValues( TvContext dbContext, - string qLower, + string query, int limit, CancellationToken cancellationToken) { @@ -264,13 +303,21 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF .Where(cr => !string.IsNullOrEmpty(cr)) .Distinct(); - return FilterSortTake(split, qLower, limit); + return FilterSortTake(split, query, limit); } - private static List FilterSortTake(IEnumerable values, string qLower, int limit) => + /// + /// The one in-memory filter/sort/take every field funnels through. Both the comparison and the ordering + /// are ORDINAL on purpose: UseRequestLocalization honours Accept-Language, so the current + /// culture is caller-controlled, and ToLower() plus the default (linguistic) + /// StartsWith(string) would make the result depend on it — under tr-TR, q=I lowers + /// to ı and stops matching Istanbul. Ordinal is also what the SQL pre-filter's superset + /// guarantee is proved against; see . + /// + private static List FilterSortTake(IEnumerable values, string query, int limit) => values - .Where(v => v.ToLower().StartsWith(qLower)) - .OrderBy(v => v) + .Where(v => v.StartsWith(query, StringComparison.OrdinalIgnoreCase)) + .OrderBy(v => v, StringComparer.Ordinal) .Take(limit) .ToList(); } diff --git a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs index 8b5045948..e83d2896e 100644 --- a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs +++ b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs @@ -1,3 +1,4 @@ +using System.Globalization; using ErsatzTV.Application.Search.Queries; using ErsatzTV.Core.Api.Search; using ErsatzTV.Core.Domain; @@ -315,50 +316,95 @@ public class GetSearchFieldValuesHandlerTests } [Test] - public async Task List_Valued_Source_Is_Bounded_By_A_Row_Cap() + public async Task List_Valued_Source_Row_Cap_Is_Exactly_1000_Rows() { - // The handler caps the raw-JSON pre-filter at 1000 rows, ordered by the raw column. Seed one row past - // the cap whose JSON sorts last but whose element would sort FIRST in the output: an unbounded - // implementation returns it, a capped one cannot. - const string overCap = "aaa-past-the-row-cap"; + // Pins the cap value in BOTH directions. Asserting only that an over-cap row is dropped would also pass + // with a cap of 3, or with the handler wrongly using `limit` as the row cap; asserting only that an + // under-cap row survives would pass with no cap at all. The marker row's element sorts first in the + // output, so an unbounded implementation cannot help but return it. + const string marker = "aaa-the-marker-row"; - await using (TvContext context = _db.CreateContext()) + // rows are capped by ascending Id, and Id is assigned in insertion order + async Task SeedFillerThenMarker(int fillerRows) { - for (var i = 0; i < 1000; i++) + await using TvContext context = _db.CreateContext(); + for (var i = 0; i < fillerRows; i++) { context.SongMetadata.Add(Song($"Song {i}", [$"b{i:D4}"])); } - context.SongMetadata.Add(Song("Over", ["zzz-sorts-last", overCap])); + context.SongMetadata.Add(Song("Marker", [marker])); await context.SaveChangesAsync(); } var handler = new GetSearchFieldValuesHandler(_db.Factory); - Option result = await handler.Handle( + // 999 filler rows -> the marker is row 1000 -> inside a cap of 1000 + await SeedFillerThenMarker(999); + + Option atCap = await handler.Handle( new GetSearchFieldValues("artist", string.Empty, 3), CancellationToken.None); - result.IsSome.ShouldBeTrue(); - result.IfSome(r => r.Values.ShouldBe(new List { "b0000", "b0001", "b0002" })); - result.IfSome(r => r.Values.ShouldNotContain(overCap)); + atCap.IsSome.ShouldBeTrue(); + atCap.IfSome( + r => r.Values.ShouldBe( + new List { marker, "b0000", "b0001" }, + "the 1000th row must still be read - a cap below 1000 would drop it")); - // positive control: the same row IS reachable once the pre-filter narrows the candidate set below the cap + // one more filler row pushes the marker to row 1001 -> outside a cap of 1000 + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.Add(Song("Extra", ["b9999"])); + await context.SaveChangesAsync(); + } + + await using (TvContext context = _db.CreateContext()) + { + // move the marker to the end so it sits at Id 1002, past the cap + SongMetadata existing = await context.SongMetadata.SingleAsync(m => m.Title == "Marker"); + context.SongMetadata.Remove(existing); + await context.SaveChangesAsync(); + context.SongMetadata.Add(Song("Marker", [marker])); + await context.SaveChangesAsync(); + + (await context.SongMetadata.CountAsync()).ShouldBe(1001); + } + + Option pastCap = await handler.Handle( + new GetSearchFieldValues("artist", string.Empty, 3), + CancellationToken.None); + + pastCap.IsSome.ShouldBeTrue(); + pastCap.IfSome( + r => r.Values.ShouldNotContain( + marker, + "the 1001st row must not be read - an unbounded query would return it first")); + pastCap.IfSome(r => r.Values.ShouldBe(new List { "b0000", "b0001", "b0002" })); + + // positive control: the same row IS reachable once the pre-filter narrows the candidates below the cap, + // so the assertion above cannot be passing because the value is unreachable for an unrelated reason Option narrowed = await handler.Handle( new GetSearchFieldValues("artist", "aaa", 50), CancellationToken.None); narrowed.IsSome.ShouldBeTrue(); - narrowed.IfSome(r => r.Values.ShouldBe(new List { overCap })); + narrowed.IfSome(r => r.Values.ShouldBe(new List { marker })); } [Test] - public async Task List_Valued_Columns_Are_Stored_As_Ascii_Escaped_Json_Arrays() + public async Task Regression_Pin_List_Valued_Columns_Are_Stored_As_Ascii_Escaped_Json_Arrays() { - // Pins the on-disk encoding the LIKE pre-filter is built against. If an EF upgrade changes the - // JavaScriptEncoder (e.g. stops escaping non-ASCII), the pattern would silently stop matching — - // this fails loudly instead. It also documents why LOWER() is provider-independent here: the - // stored text is pure ASCII, so SQLite's ASCII-only lower() and MySQL's Unicode-aware LOWER() agree. + // REGRESSION PIN, not coverage of #578: this asserts pre-existing EF behaviour and passes against the + // code before this change. It is here because the pre-filter's correctness argument depends on the + // stored form, so an encoder change must fail loudly rather than silently under-match. + // + // Specifically it pins that non-ASCII IS escaped — which is exactly why SQL LOWER() canNOT be used to + // case-fold it. LOWER() lowercases the escape TEXT (\u00E9 -> \u00e9); it does not touch the codepoint + // the escape denotes, so the stored `É` never folds to `é`. An earlier version of this change + // claimed the opposite and JSON-encoded the whole query prefix, which silently made every accented + // artist unsuggestable. The pattern builder now narrows only on verbatim ASCII; see + // SearchFieldValuesPrefilterSupersetTests. await using (TvContext context = _db.CreateContext()) { context.SongMetadata.Add(Song("One", ["Beyoncé", "\"Q\""])); @@ -375,6 +421,138 @@ public class GetSearchFieldValuesHandlerTests } } + [Test] + [TestCase("é", "\u00C9dith Piaf")] + [TestCase("\u00C9", "\u00C9dith Piaf")] + [TestCase("\u00E9dith", "\u00C9dith Piaf")] + [TestCase("bj", "Bj\u00F6rk")] + [TestCase("bj\u00F6", "Bj\u00F6rk")] + [TestCase("BJ\u00D6RK", "Bj\u00F6rk")] + [TestCase("beyonc\u00E9", "Beyonc\u00E9")] + [TestCase("sigur r", "Sigur R\u00F3s")] + [TestCase("\u00D6", "\u00D6zdemir")] + public async Task Prefilter_Never_Under_Matches_A_NonAscii_Value(string query, string stored) + { + // The SQL pre-filter narrows on raw JSON, where non-ASCII is stored escaped (\u00C9). Any narrowing + // that includes an escaped character under-matches, because SQL LOWER() folds the escape TEXT and not + // the codepoint — every one of these cases returned EMPTY before the pattern builder was restricted to + // the leading verbatim-ASCII run. Accented artists are the common case in a music library, not an edge. + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.AddRange( + Song("Hit", [stored]), + // negative control: a row that must never come back for any of these queries + Song("Other", ["Nothing Relevant"])); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + Option result = await handler.Handle( + new GetSearchFieldValues("artist", query, 50), + CancellationToken.None); + + result.IsSome.ShouldBeTrue(); + result.IfSome(r => r.Values.ShouldBe(new List { stored })); + } + + [Test] + public async Task Results_Do_Not_Depend_On_The_Request_Culture() + { + // UseRequestLocalization honours Accept-Language, so CurrentCulture is caller-controlled. Under tr-TR + // the old `q.ToLower()` turned "I" into "\u0131" and the default linguistic StartsWith(string) compounded + // it, so the same library answered differently per caller. The contract is ordinal: "I" matches + // "Istanbul" and does NOT match "\u0131pek", in every culture. + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.Add(Song("One", ["Istanbul Orkestrasi", "\u0131pek"])); + context.ArtistMetadata.Add(Artist("Idil Biret")); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + var expected = new List { "Idil Biret", "Istanbul Orkestrasi" }; + + CultureInfo original = CultureInfo.CurrentCulture; + try + { + foreach (string culture in new[] { "en-US", "tr-TR", "az-AZ", "lt-LT" }) + { + CultureInfo.CurrentCulture = new CultureInfo(culture); + + Option result = await handler.Handle( + new GetSearchFieldValues("artist", "I", 50), + CancellationToken.None); + + result.IsSome.ShouldBeTrue(); + result.IfSome(r => r.Values.ShouldBe(expected, $"culture {culture} changed the result")); + } + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + + [Test] + public async Task Ordering_Is_Ordinal_And_Culture_Independent() + { + // The merge sorts ordinally rather than by culture, so the response order does not depend on the caller + // either. Ordinal puts all ASCII uppercase before ASCII lowercase, and non-ASCII last. + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.Add(Song("One", ["Zulu", "apple", "\u00C9clair", "Apple"])); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + var expected = new List { "Apple", "Zulu", "apple", "\u00C9clair" }; + + CultureInfo original = CultureInfo.CurrentCulture; + try + { + foreach (string culture in new[] { "en-US", "sv-SE" }) + { + CultureInfo.CurrentCulture = new CultureInfo(culture); + + Option result = await handler.Handle( + new GetSearchFieldValues("artist", string.Empty, 50), + CancellationToken.None); + + result.IfSome(r => r.Values.ShouldBe(expected, $"culture {culture} changed the order")); + } + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + + [Test] + public async Task Ordering_Is_Best_Effort_When_A_Source_Truncates() + { + // Documents the acknowledged imprecision rather than claiming exactness the code does not have. The EF + // source truncates by the DATABASE collation (SQLite's is ASCII-only), which is not the ordinal ordering + // the merge then applies — so a value the database ranked outside its first `limit` never reaches the + // merge, even if the merge would have ranked it first. + await using (TvContext context = _db.CreateContext()) + { + context.ArtistMetadata.Add(Artist("Zulu")); + context.ArtistMetadata.Add(Artist("\u00C9clair")); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + // with room for both, the ordinal merge ranks "Zulu" (ASCII) before "\u00C9clair" (non-ASCII) + (await handler.Handle(new GetSearchFieldValues("artist", string.Empty, 50), CancellationToken.None)) + .IfSome(r => r.Values.ShouldBe(new List { "Zulu", "\u00C9clair" })); + + // with limit=1 the database picks the survivor by ITS collation, and the merge only sees that one + (await handler.Handle(new GetSearchFieldValues("artist", string.Empty, 1), CancellationToken.None)) + .IfSome(r => r.Values.Count.ShouldBe(1)); + } + private static ArtistMetadata Artist(string title) => new() { MetadataKind = MetadataKind.External, diff --git a/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs b/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs new file mode 100644 index 000000000..f02958c11 --- /dev/null +++ b/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs @@ -0,0 +1,122 @@ +using System.Text.Json; +using ErsatzTV.Application.Search.Queries; +using NUnit.Framework; +using Shouldly; + +namespace ErsatzTV.Tests.Application.Search; + +/// +/// Proof obligations for the raw-JSON LIKE pre-filter that narrows the list-valued facet-value +/// sources (#578). The pre-filter is an optimization layered under an in-memory exact filter, so it is +/// allowed to over-match but must NEVER under-match — an under-match is a silent false negative, invisible +/// to any test that only checks the values it does return. +/// +/// The end-to-end behavioural cases live in GetSearchFieldValuesHandlerTests. These are the two +/// facts the superset argument rests on, asserted directly rather than trusted. +/// +/// +[TestFixture] +public class SearchFieldValuesPrefilterSupersetTests +{ + /// + /// Printable ASCII, the alphabet + /// narrows over (minus whatever the JSON writer escapes, which the sibling test pins). + /// + private static IEnumerable PrintableAscii() + { + for (var c = ' '; c <= '~'; c++) + { + yield return c; + } + } + + [Test] + public void No_NonAscii_Codepoint_OrdinalIgnoreCase_Equals_A_Printable_Ascii_Character() + { + // THIS is why the narrowing run may be restricted to ASCII and still be a superset. The in-memory filter + // compares with OrdinalIgnoreCase, so for a run character `c` the pre-filter must catch every element + // character that OrdinalIgnoreCase-equals `c`. If any non-ASCII codepoint did, that element would be + // stored as a \uXXXX escape, the ASCII pattern would miss the row, and the value would be unsuggestable. + // + // Note this is NOT true of InvariantCultureIgnoreCase, which folds ~190 codepoints (U+00AA -> a, + // U+017F -> s, the modifier letters, ...) onto ASCII letters. The choice of Ordinal is load-bearing. + char[] ascii = PrintableAscii().ToArray(); + var offenders = new List(); + + for (var cp = 0x80; cp <= 0x10FFFF; cp++) + { + if (cp is >= 0xD800 and <= 0xDFFF) + { + continue; + } + + string s = char.ConvertFromUtf32(cp); + foreach (char a in ascii) + { + if (string.Equals(s, a.ToString(), StringComparison.OrdinalIgnoreCase)) + { + offenders.Add($"U+{cp:X4} == '{a}'"); + } + } + } + + offenders.ShouldBeEmpty(); + } + + [Test] + public void The_Json_Writer_Escapes_Exactly_The_Ascii_Characters_The_Pattern_Builder_Refuses_To_Narrow_On() + { + // The narrowing run may only contain characters stored VERBATIM. Anything the writer escapes is stored + // as \uXXXX and must terminate the run. If an encoder change adds a character to this set, the pattern + // builder's exclusion list goes stale and starts under-matching — so pin the set, not a sample. + var escaped = PrintableAscii() + .Where(c => JsonSerializer.Serialize(c.ToString()) != $"\"{c}\"") + .ToArray(); + + escaped.ShouldBe(['"', '&', '\'', '+', '<', '>', '\\', '`']); + + // and the pattern builder must stop at every one of them (empty narrowing run -> bare anchor) + foreach (char c in escaped) + { + GetSearchFieldValuesHandler.JsonElementPrefixPattern($"{c}abc") + .ShouldBe("%\"%", $"expected no narrowing for the JSON-escaped '{c}'"); + } + } + + [Test] + public void Pattern_Narrows_Only_On_The_Leading_Verbatim_Ascii_Run() + { + // plain ASCII: full narrowing, lowercased + GetSearchFieldValuesHandler.JsonElementPrefixPattern("Radio").ShouldBe("%\"radio%"); + + // non-ASCII terminates the run, keeping the ASCII head as the narrowing + GetSearchFieldValuesHandler.JsonElementPrefixPattern("Beyoncé").ShouldBe("%\"beyonc%"); + + // a leading non-ASCII character means no narrowing at all + GetSearchFieldValuesHandler.JsonElementPrefixPattern("édith").ShouldBe("%\"%"); + GetSearchFieldValuesHandler.JsonElementPrefixPattern("Ünen").ShouldBe("%\"%"); + + // empty query: the bare element-opening anchor + GetSearchFieldValuesHandler.JsonElementPrefixPattern(string.Empty).ShouldBe("%\"%"); + GetSearchFieldValuesHandler.JsonElementPrefixPattern(null).ShouldBe("%\"%"); + + // LIKE wildcards inside the run are escaped, not passed through + GetSearchFieldValuesHandler.JsonElementPrefixPattern("50%").ShouldBe("%\"50/%%"); + GetSearchFieldValuesHandler.JsonElementPrefixPattern("a_b").ShouldBe("%\"a/_b%"); + + // the escape character itself is escaped + GetSearchFieldValuesHandler.JsonElementPrefixPattern("AC/DC").ShouldBe("%\"ac//dc%"); + } + + [Test] + public void Row_Cap_Is_Applied_On_The_Primary_Key_Not_The_Json_Column() + { + // MySQL sorts TEXT/longtext using only max_sort_length bytes, so ordering on the JSON column would let + // rows sharing a long prefix tie arbitrarily and the surviving 1000 would be unspecified. Id is a unique + // integer key on both providers. + string sql = GetSearchFieldValuesHandler.ListValuedSql("Artists"); + + sql.ShouldContain("ORDER BY Id LIMIT @Cap"); + sql.ShouldNotContain("ORDER BY Artists"); + } +} diff --git a/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs b/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs index e3c44f535..d43e71d70 100644 --- a/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs +++ b/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs @@ -1,4 +1,4 @@ -using ErsatzTV.Core.Domain; +using ErsatzTV.Application.Search.Queries; using ErsatzTV.Infrastructure; using ErsatzTV.Infrastructure.Data; using Microsoft.EntityFrameworkCore; @@ -36,9 +36,9 @@ public class SearchFieldValuesQueryShapeTests { using TvContext context = create(); - // exactly the queryable half of GetSearchFieldValuesHandler.GetSource("artist") - string sql = context.ArtistMetadata.Select(m => m.Title) - .Concat(context.Set().Select(a => a.Name)) + // calls the handler's own source builder (internal, via InternalsVisibleTo) rather than rebuilding + // the LINQ here — a copy would keep passing after the handler's query changed underneath it + string sql = GetSearchFieldValuesHandler.GetSource(context, "artist") .Where(v => v != null && v.ToLower().StartsWith("a")) .Distinct() .OrderBy(v => v) @@ -55,8 +55,11 @@ public class SearchFieldValuesQueryShapeTests } [Test] - public void Song_List_Columns_Cannot_Be_Projected_Server_Side_On_Either_Provider() + public void Regression_Pin_Song_List_Columns_Cannot_Be_Projected_Server_Side_On_Either_Provider() { + // REGRESSION PIN, not coverage of #578: this asserts pre-existing EF/provider behaviour and passes + // against the code before this change. + // // Documents WHY the handler drops to raw SQL for SongMetadata.Artists / .AlbumArtists rather than // SelectMany-ing them: EF maps them as JSON primitive collections and neither provider can translate // the projection (SQLite needs APPLY; Pomelo has no primitive-collection support). If a provider diff --git a/ErsatzTV/Controllers/Api/SearchController.cs b/ErsatzTV/Controllers/Api/SearchController.cs index 78e306cb4..9f794c634 100644 --- a/ErsatzTV/Controllers/Api/SearchController.cs +++ b/ErsatzTV/Controllers/Api/SearchController.cs @@ -205,8 +205,9 @@ public class SearchController(IMediator mediator) : ControllerBase "Returns distinct whole values from the database for the given text field, filtered by an " + "optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. " + "404 when the field is unknown, is not a text field, or is a text field with no distinct-value " + - "source. The list-valued music fields (artist, album_artist) are bounded best-effort: on a very " + - "large library they return a deterministic subset of the matches rather than every match.")] + "source. Matching, dedup and ordering are ordinal, not culture-dependent. The list-valued music " + + "fields (artist, album_artist) are bounded best-effort: on a very large library they return a " + + "bounded subset of the matches rather than every match.")] [EndpointGroupName("general")] [ProducesResponseType(typeof(SearchFieldValuesResponseModel), StatusCodes.Status200OK)] [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)] diff --git a/ErsatzTV/wwwroot/openapi/v1.json b/ErsatzTV/wwwroot/openapi/v1.json index 9d15967d6..22395ee44 100644 --- a/ErsatzTV/wwwroot/openapi/v1.json +++ b/ErsatzTV/wwwroot/openapi/v1.json @@ -18063,7 +18063,7 @@ "Search" ], "summary": "List distinct database values for a text search field", - "description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source. The list-valued music fields (artist, album_artist) are bounded best-effort: on a very large library they return a deterministic subset of the matches rather than every match.", + "description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source. Matching, dedup and ordering are ordinal, not culture-dependent. The list-valued music fields (artist, album_artist) are bounded best-effort: on a very large library they return a bounded subset of the matches rather than every match.", "operationId": "GetSearchFieldValues", "parameters": [ { diff --git a/docs/api-conventions.md b/docs/api-conventions.md index fc2b28e11..06567872f 100644 --- a/docs/api-conventions.md +++ b/docs/api-conventions.md @@ -456,11 +456,18 @@ No server-side caching. Powers the visual rule builder's value-input combobox fo or partly) by `SongMetadata.Artists`/`AlbumArtists`, which EF maps as **primitive collections** — one JSON array per row in a single column, with no server-side projection on either provider. `album_artist` therefore no longer 404s, and `artist` now also covers free-text music-video (`MusicVideoArtist`) and -song credits, not only entity artists. For these fields the handler pre-filters on the raw JSON in SQL -and caps the rows it reads, so on a very large library the response is a **deterministic subset** of the -matches rather than every match — say so in the `[EndpointDescription]` of any endpoint that adopts this -shape, and never let an unbounded scan stand in for it. Full rationale, the rejected normalized-side-table -alternative and the dual-provider `LIKE`/`LOWER` portability rules: `api.search-field-values-list-columns`. +song credits, not only entity artists. For these fields the handler narrows on the raw JSON in SQL and +caps the rows it reads, so on a very large library the response is a bounded subset of the matches rather +than every match — say so in the `[EndpointDescription]` of any endpoint that adopts this shape. + +Two rules generalize beyond this endpoint. **A SQL pre-filter layered under an in-memory exact filter may +over-match but must NEVER under-match** — an under-match is a silent false negative that no assertion over +the returned values can see, so prove the superset (ours is proved by an exhaustive Unicode sweep, not by +argument) rather than eyeballing it. And **prefix matching, dedup and ordering on any `/api/*` read must be +ordinal, never current-culture**: `UseRequestLocalization` honours `Accept-Language`, so `ToLower()` and the +default linguistic `StartsWith(string)` let a caller change the result by changing a header. Full rationale, +the rejected normalized-side-table alternative, the accepted scan cost and the dual-provider `LIKE`/`LOWER` +rules: `api.search-field-values-sources` (which supersedes `api.search-field-values`). **Param + DTO expansion (#293, cap `search/all-items`)**: no new endpoint — `GET /api/v1/search/all-items` gained two **optional** query params (`pageSize` default 500, clamped 1–1000 via the §1 Logs `Math.Clamp` diff --git a/docs/decisions.md b/docs/decisions.md index 9ea033132..2b33c74b1 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -206,7 +206,7 @@ another doc or an old issue comment should land here and then follow the link. - 2026-07-22 — per-schedule clock-boundary padding is a synthetic content-less Pad over the existing per-episode machinery (#392) — [`sched.clock-padding-schedule-toggle`](decisions/records/sched/clock-padding-schedule-toggle.md) - 2026-07-23 — Channel health = a server-derived `health` object on the channel DTOs, built-timeline detection (#415) — [`api.channel-health-object`](decisions/records/api/channel-health-object.md) - 2026-07-23 — Channel origin is immutable creation-provenance, stamped at insert, not a health signal (#414) — [`channel.origin-marker`](decisions/records/channel/origin-marker.md) -- 2026-07-23 — Facet-value typeahead is a new endpoint, allow-listed to text fields, no caching (#434) — [`api.search-field-values`](decisions/records/api/search-field-values.md) +- 2026-07-23 — Facet-value typeahead is a new endpoint, allow-listed to text fields, no caching (#434) — [`api.search-field-values`](decisions/archive/api/search-field-values.md) (superseded by `api.search-field-values-sources`) - 2026-07-23 — Relative-date rule builder operators are a frontend-only mapping onto existing Lucene macros (#435) — [`rulebuilder.relative-date-macros`](decisions/records/rulebuilder/relative-date-macros.md) - 2026-07-25 — A media-server sweep also refuses when the api client silently dropped items whose projection threw; the ratio threshold is rejected (#484) — [`scan.projection-failure-sweep-guard`](decisions/records/scan/projection-failure-sweep-guard.md) - 2026-07-25 — LibraryFolder identity is enforced by a unique index on `(LibraryPathId, PathHash)`, not an in-process lock (#491) — [`scan.libraryfolder-unique-identity`](decisions/records/scan/libraryfolder-unique-identity.md) @@ -215,4 +215,4 @@ another doc or an old issue comment should land here and then follow the link. - 2026-07-25 — Rule-builder group nesting is bounded-arbitrary depth (`MAX_GROUP_DEPTH`), not one level (#436) — [`spa.rulebuilder-nesting`](decisions/records/spa/rulebuilder-nesting.md) - 2026-07-25 — The rationale-edit marker is a git trailer, not a substring anywhere in the commit range (#609) — [`ci.decisions-edit-trailer`](decisions/records/ci/decisions-edit-trailer.md) - 2026-07-25 — UI-E2E: headless Playwright flows in the existing `functional-e2e` job, browser baked into the CI image (#445) — [`ci.ui-e2e-harness`](decisions/records/ci/ui-e2e-harness.md) -- 2026-07-26 — List-valued (JSON primitive collection) facet fields get bounded best-effort values via a raw-JSON LIKE pre-filter, not a normalized side table (#578) — [`api.search-field-values-list-columns`](decisions/records/api/search-field-values-list-columns.md) +- 2026-07-26 — Facet-value typeahead restated: every artist source covered; list-valued columns via a superset LIKE pre-filter that may over-match but never under-match (#578) — [`api.search-field-values-sources`](decisions/records/api/search-field-values-sources.md) diff --git a/docs/decisions/README.md b/docs/decisions/README.md index ea26ec8c7..72bbd138b 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -28,8 +28,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `api.schedule-item-flat-dto` | Schedule-item GET/POST/PUT use a flat, non-polymorphic `ScheduleItemResponseModel` (every subtype field promoted to a nullable top-level member) instead of the polymorphic Application VM hierarchy, with mutation field names matching `ScheduleItemRequest` 1:1 for a lossless round-trip. | 2026-07-10 | [link](records/api/schedule-item-flat-dto.md) | | `api.scheduling-hardening` | Create/Replace handlers guard against null/whitespace `name` (`IsNullOrWhiteSpace`, not just `Length`) to prevent NRE-500s, template-item overlap validation compares by index (not record value-equality) to catch exact-duplicate items, and unreachable 404 `ProducesResponseType` attributes on create-only actions are trimmed. | 2026-07-13 | [link](records/api/scheduling-hardening.md) | | `api.search-allitems-paging` | `GET /api/v1/search/all-items` is paginated (capped page size, `Totals` field) to bound DoS exposure; the SPA add-all flow pages to completeness instead of relying on an unbounded response. | 2026-07-18 | [link](records/api/search-allitems-paging.md) | -| `api.search-field-values` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for one of a narrow allow-list of catalog fields (not the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a typeahead suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source; case-insensitive prefix-filtered on `q`, `limit` clamped to `[1, 50]` (default 50). | 2026-07-23 | [link](records/api/search-field-values.md) | -| `api.search-field-values-list-columns` | A `text` search field whose values live in an EF **primitive collection** (one JSON array per row in a single column — `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) IS served by `GET /api/v1/search/fields/{name}/values`, not 404'd, but as **bounded best-effort**: SQL pre-filters on the raw JSON with `LOWER(col) LIKE '%"%' ESCAPE '/'` — a deliberate superset, since it matches a row, not an element — capped at a fixed row cap (1000) with an `ORDER BY` on the raw column so the truncation point is deterministic; the rows are then split, exact-prefix-filtered, deduped, sorted and `limit`ed in memory. An empty `q` is supported (it degenerates to `%"%`, still capped), so these fields need no special-case client contract. The prefix is JSON-encoded before matching because that is how it is stored, which also makes the pattern pure ASCII and therefore makes `LOWER()` mean the same thing on SQLite (ASCII-only) and MySQL (Unicode-aware); the pattern is lowercased in C# rather than relying on `LIKE`'s own case rules, so the match is correct under a case-sensitive MySQL collation as well as a case-insensitive one. The escape character is `/`, never `\` — `ESCAPE '\'` is not a portable SQL literal. | 2026-07-26 | [link](records/api/search-field-values-list-columns.md) | +| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). Prefix matching, dedup and ordering are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled. A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&'+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; the rows are capped at 1000 on `ORDER BY Id` (a unique integer key, not the JSON column) and then split, filtered, deduped, sorted and `limit`ed in memory. Ordering across merged sources is best-effort, exact only below the truncation points. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | | `api.search-paging-cap` | Search stays capped at 100 items per media kind; an overflowing kind's "See all" reuses library-browse paging instead of adding new API surface. | 2026-07-11 | [link](records/api/search-paging-cap.md) | | `api.versioning-v1` | The entire `/api` surface is versioned to `/api/v1` uniformly (no unversioned corner); legacy unversioned callers are rewritten in-pipeline (not redirected) with Deprecation/Link/Sunset headers, and post-freeze `/api/v1` is additive-only — a breaking change requires `/api/v2`. | 2026-07-13 | [link](records/api/versioning-v1.md) | | `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) | diff --git a/docs/decisions/records/api/search-field-values.md b/docs/decisions/archive/api/search-field-values.md similarity index 70% rename from docs/decisions/records/api/search-field-values.md rename to docs/decisions/archive/api/search-field-values.md index 68262f3ca..d1578a20f 100644 --- a/docs/decisions/records/api/search-field-values.md +++ b/docs/decisions/archive/api/search-field-values.md @@ -1,13 +1,13 @@ --- key: api.search-field-values title: 2026-07-23 — Facet-value typeahead is a new endpoint, allow-listed to text fields, no caching (#434) -status: active +status: superseded since: '2026-07-23' supersedes: none -superseded-by: none -rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for one of a narrow allow-list of catalog fields (not the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a typeahead suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source; case-insensitive prefix-filtered on `q`, `limit` clamped to `[1, 50]` (default 50).' +superseded-by: api.search-field-values-sources@2026-07-26 +rule: '(superseded) `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for one of a narrow allow-list of catalog fields (not the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a typeahead suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source; case-insensitive prefix-filtered on `q`, `limit` clamped to `[1, 50]` (default 50).' signals: 'facet-value typeahead, rule builder value combobox, distinct field values, GetSearchFieldValues, text field allow-list, DB-sourced distinct values, content_rating split · paths: `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Application/Search/Queries/GetSearchFieldValues.cs`, `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `web/src/api/search.ts` · issues: #434, #176' -mechanics: '`SearchController.GetSearchFieldValues`; `GetSearchFieldValuesHandler`; api-conventions.md; spa-conventions.md §12' +mechanics: superseded by `api.search-field-values-sources` (ersatztv#578), which keeps this endpoint contract and reverses the "no distinct-value source" call for the list-valued music fields --- Enum fields (e.g. `type`, `content_rating` group) already ship their allowed values inline on @@ -23,9 +23,8 @@ an empty list for a field that will never have values — a 404 tells a caller " **DB-sourced, not the search index.** The handler injects `IDbContextFactory` and resolves an explicit per-field-name `IQueryable` (or, for a few special cases, an in-memory list) rather than querying `ISearchIndex`: `genre`/`show_genre` → `Set()`, `studio` → `Set()`, `director` → -`Set()`, `writer` → `Set()`, `actor` → `Actors`, `artist` → `ArtistMetadata.Title` -concatenated with `Set().Name` (plus `SongMetadata.Artists` via the list-valued path — -see `api.search-field-values-list-columns`, #578), `tag` → +`Set()`, `writer` → `Set()`, `actor` → `Actors`, `artist` → `ArtistMetadata.Title` (entity +artists only — free-text music-video/song artist credits are a known, intentionally-uncovered gap), `tag` → `Set()` excluding `Tag.NfoCountryTypeId`/`Tag.PlexNetworkTypeId` (reapplying the indexer's own exclusions so country/network strings don't leak in as tags), `network` → `Set()` filtered to `Tag.PlexNetworkTypeId`, `collection` → `Collections`, `video_codec` → `MediaStreams` filtered to @@ -37,11 +36,10 @@ of queried: `state` (the fixed 4-value `MediaItemState` enum) and `video_dynamic `MovieMetadata`/`ShowMetadata`/`OtherVideoMetadata`/`RemoteStreamMetadata`, so the handler pulls the distinct raw strings then `Split('/')`s, trims, and dedupes in memory before the same prefix-filter/sort/take — this matches what search actually matches on, rather than surfacing the compound string as one facet value. -**`title` and `show_title` are explicitly NOT supported** (404, free-text fallback): they are near-unique -free-text fields spanning ~9 metadata tables where a distinct list of every title isn't a useful facet. -`album_artist` was originally 404 here for a different reason — it backs onto `SongMetadata.AlbumArtists`, -which EF can't translate into a server-side distinct query — and since #578 it returns bounded best-effort -values instead; see `api.search-field-values-list-columns`. +**`title`, `show_title`, `album_artist` are explicitly NOT supported** (404, free-text fallback): `title`/ +`show_title` are near-unique free-text fields spanning ~9 metadata tables where a distinct list of every +title isn't a useful facet; `album_artist` backs onto `SongMetadata.AlbumArtists`, a value-converted +`IList` column EF can't translate into a server-side distinct query. **Why a thin query, not a cache.** No result cache, no debounce on the server side (the SPA combobox debounces the keystroke) — each per-field query is a bounded, indexed `Distinct`/`Take`; adding a cache diff --git a/docs/decisions/records/api/search-field-values-list-columns.md b/docs/decisions/records/api/search-field-values-list-columns.md deleted file mode 100644 index 6bfaa4353..000000000 --- a/docs/decisions/records/api/search-field-values-list-columns.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -key: api.search-field-values-list-columns -title: '2026-07-26 — list-valued (JSON primitive collection) fields get bounded best-effort facet values via a raw-JSON LIKE pre-filter, not a normalized side table (#578)' -status: active -since: '2026-07-26' -supersedes: none -superseded-by: none -rule: 'A `text` search field whose values live in an EF **primitive collection** (one JSON array per row in a single column — `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) IS served by `GET /api/v1/search/fields/{name}/values`, not 404''d, but as **bounded best-effort**: SQL pre-filters on the raw JSON with `LOWER(col) LIKE ''%"%'' ESCAPE ''/''` — a deliberate superset, since it matches a row, not an element — capped at a fixed row cap (1000) with an `ORDER BY` on the raw column so the truncation point is deterministic; the rows are then split, exact-prefix-filtered, deduped, sorted and `limit`ed in memory. An empty `q` is supported (it degenerates to `%"%`, still capped), so these fields need no special-case client contract. The prefix is JSON-encoded before matching because that is how it is stored, which also makes the pattern pure ASCII and therefore makes `LOWER()` mean the same thing on SQLite (ASCII-only) and MySQL (Unicode-aware); the pattern is lowercased in C# rather than relying on `LIKE`''s own case rules, so the match is correct under a case-sensitive MySQL collation as well as a case-insensitive one. The escape character is `/`, never `\` — `ESCAPE ''\''` is not a portable SQL literal.' -signals: 'artist typeahead free-text credits, album_artist 404, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist suggestions, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, raw JSON LIKE pre-filter, ListValuedRowCap, bounded best-effort facet values, LIKE ESCAPE portability · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs` · issues: #578, #434' -mechanics: '`GetSearchFieldValuesHandler.GetSongListValuedColumn` / `.GetSongListValuedValues` / `.ListValuedSql` / `.JsonElementPrefixPattern`; `SearchFieldValuesQueryShapeTests` (dual-provider `ToQueryString` guard); `api.search-field-values` (the base endpoint contract this extends)' ---- - -Extends `api.search-field-values` rather than superseding it: that record's rule still reads true — -whole values, from the database, not the Lucene term dictionary, 404 for a field with no -distinct-value source. What changed is only **which** fields have a source. Its body's claim that -music-video/song artist credits are "a known, intentionally-uncovered gap" and that `album_artist` -has no server-side source was corrected in place. - -**The three `artist` sources are not the same problem.** `LuceneSearchIndex` writes the `artist` -field from three places, and only two of them are ordinary columns: - -- `ArtistMetadata.Title` — a plain column on a plain table. -- `MusicVideoArtist.Name` — also a real entity table (`MusicVideoMetadata.HasMany(m => m.Artists)`), - so the free-text music-video credits are directly `SELECT DISTINCT`-able. These two are simply - `Concat`ed into the existing server-side pipeline - (`.Where(v => v.ToLower().StartsWith(qLower)).Distinct().OrderBy(v => v).Take(limit)`), which EF - emits as one bounded `UNION ALL` + `LOWER(...) LIKE ... LIMIT` on both providers. -- `SongMetadata.Artists` (and, for `album_artist`, `SongMetadata.AlbumArtists`) — an - `IList` that EF 9 maps as a **primitive collection**: no `HasConversion` anywhere, one - JSON array per row in a single `TEXT`/`longtext` column. This one has no server-side projection at - all. Verified empirically against both providers: SQLite reports *"Translating this query requires - the SQL APPLY operation, which is not supported on SQLite"*, and Pomelo MySQL 9.0.0 reports - *"Primitive collections support has not been enabled"*. `SearchFieldValuesQueryShapeTests` pins - both failures, so if a provider upgrade ever makes the projection translate, the raw-SQL path can - be retired on a red test rather than on a hunch. - -**Why the LIKE superset and not a normalized `SongArtist` table.** A real join table (the shape -`MusicVideoArtist` already has) would make this exact and indexable forever, but it costs a -dual-provider schema migration plus a data backfill, changes to every scanner write path that -populates `SongMetadata.Artists`, changes to the Lucene indexer, and leaves two representations of -the same fact to drift apart — a large blast radius for a typeahead convenience. Rejected on cost. -The raw-JSON pre-filter is confined to one private method, needs no migration, and is a strictly -additive read path. Also rejected: querying the Lucene term dictionary, which is the thing -`api.search-field-values` exists to avoid (analyzed `TextField`s store lowercased word tokens). - -**The bound is the point, and it is tested behaviourally.** `ListValuedRowCap = 1000` bounds rows -fetched, not values returned, because the pre-filter matches rows whole. The `ORDER BY` on the raw -column is what makes the truncation deterministic — and therefore testable: -`List_Valued_Source_Is_Bounded_By_A_Row_Cap` seeds 1001 rows where the over-cap row's JSON sorts -*last* but its element would sort *first* in the output, so an unbounded implementation returns it -and a capped one cannot. A positive control in the same test narrows `q` until that row is back -under the cap and asserts it reappears, so the test cannot pass by the value being unreachable for -some unrelated reason. - -**`album_artist` 404 → 200 is additive.** Nothing consumes the 404 as a signal: the SPA's -`getSearchFieldValues` (`web/src/api/search.ts`) treats any non-200 as "no suggestions, fall back to -a free-text input", which is exactly what it will now do less often. Per `api.versioning-v1`, -widening which fields return values — and which values a field returns — adds capability without -removing any, so it needs no `/api/v2`. - -**Known imprecision, deliberately accepted.** On a library with more than 1000 rows matching the -pre-filter, the suggestions for `artist`/`album_artist` are a deterministic prefix of the matches -rather than every match. This is stated on the endpoint's `[EndpointDescription]` so it reaches the -OpenAPI document. The alternative — requiring a non-empty `q` for list-valued fields — was rejected: -it would make one group of fields behave differently from every other field for the same client -code, for a case the row cap already handles. - -**Dual-provider verification, and its limit.** The queryable half is pinned on both providers by -`SearchFieldValuesQueryShapeTests`, which compiles the query with `ToQueryString()` — no server -needed. The raw-SQL half cannot be executed against MySQL from the unit-test suite; it is defended -by construction instead (portable `LOWER`/`LIKE`/`ESCAPE`/`ORDER BY`/`LIMIT`, unquoted identifiers, -a non-backslash escape character, and an ASCII-only pattern so collation and `LOWER()` semantics -cannot diverge) plus `List_Valued_Columns_Are_Stored_As_Ascii_Escaped_Json_Arrays`, which pins the -on-disk JSON encoding the pattern is built against so an EF encoder change fails loudly instead of -silently returning nothing. diff --git a/docs/decisions/records/api/search-field-values-sources.md b/docs/decisions/records/api/search-field-values-sources.md new file mode 100644 index 000000000..f138aab11 --- /dev/null +++ b/docs/decisions/records/api/search-field-values-sources.md @@ -0,0 +1,116 @@ +--- +key: api.search-field-values-sources +title: '2026-07-26 — Facet-value typeahead, restated: every artist-bearing source is covered, list-valued columns via a superset LIKE pre-filter that may over-match but never under-match (#578)' +status: active +since: '2026-07-26' +supersedes: api.search-field-values@2026-07-23 +superseded-by: none +rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). Prefix matching, dedup and ordering are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled. A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&''+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; the rows are capped at 1000 on `ORDER BY Id` (a unique integer key, not the JSON column) and then split, filtered, deduped, sorted and `limit`ed in memory. Ordering across merged sources is best-effort, exact only below the truncation points.' +signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, raw JSON LIKE pre-filter, superset never under-match, accented artist unsuggestable, LOWER cannot fold a \u00XX escape, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, ListValuedRowCap, ORDER BY Id not TEXT, max_sort_length, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176' +mechanics: '`GetSearchFieldValuesHandler` (`GetSource`, `GetSongListValuedColumn`, `GetSongListValuedValues`, `ListValuedSql`, `JsonElementPrefixPattern`, `FilterSortTake`); `SearchFieldValuesPrefilterSupersetTests` (the superset proof); api-conventions.md; spa-conventions.md §12' +--- + +Supersedes `api.search-field-values` (#434). That record did not merely carry a stale implementation +detail — it recorded a **call**: free-text music-video/song artist credits were "a known, +intentionally-uncovered gap" and `album_artist` was unsupported. #578 reverses that call, so this is +a supersession, not a line-edit. Everything #434 settled that still holds is restated here rather +than left in the archive: enum fields ship their values inline on `GET /api/v1/search/fields` and +need no lookup; text fields need a live one; the source is the database and never the search index; +`content_rating` splits its compound `"PG-13/TV-14"` strings in memory; there is no result cache. + +## The three `artist` sources are three different problems + +`LuceneSearchIndex` writes the `artist` field from three places, and only two are ordinary columns: + +- `ArtistMetadata.Title` — a plain column. Already worked. +- `MusicVideoArtist.Name` — also a real entity table (`MusicVideoMetadata.HasMany(m => m.Artists)`), + so the free-text music-video credits are directly `SELECT DISTINCT`-able. It just joins the + existing server-side pipeline as a `Concat`, emitted as one bounded `UNION ALL` + + `LOWER(...) LIKE ... LIMIT` on both providers. +- `SongMetadata.Artists` (and, for `album_artist`, `AlbumArtists`) — an `IList` EF 9 maps as + a **primitive collection**: no `HasConversion` anywhere, one JSON array per row in a single + `TEXT`/`longtext` column, with no server-side projection at all. Verified against both providers: + SQLite reports *"Translating this query requires the SQL APPLY operation, which is not supported on + SQLite"*, Pomelo MySQL 9.0.0 reports *"Primitive collections support has not been enabled"*. Both + failures are pinned by a test, so a provider upgrade that fixes them surfaces as a red rather than + leaving a workaround in place forever. + +## The pre-filter is an optimization; it may over-match, it must never under-match + +This is the load-bearing invariant, and the first implementation got it wrong in a way that no test +of the returned values could see. It JSON-encoded the whole query prefix, reasoning that since the +stored text escapes non-ASCII (`Édith` is on disk as `Édith`), encoding the prefix the same way +would line up. **It does not.** SQL `LOWER()` lowercases the *escape text* — `É` — it cannot +case-fold the codepoint that escape denotes. So `q=é` produced a pattern of `é` that never +matched, the row was discarded before the in-memory filter ever saw it, and **every accented artist +was silently unsuggestable** — in a music library, the common case, not an edge case. + +The fix is to narrow only on ground that can be proved: the leading run of characters the JSON +writer stores **verbatim**, stopping at the first non-ASCII or JSON-escaped character. A query +starting with such a character narrows to the bare element-opening anchor `%"%` and leans on the row +cap and the in-memory filter. `q=Beyoncé` still narrows on `beyonc`; `q=é` narrows on nothing. + +Soundness rests on two facts, both **asserted by exhaustive computation** in +`SearchFieldValuesPrefilterSupersetTests` rather than argued: + +1. **No non-ASCII codepoint in all of Unicode `OrdinalIgnoreCase`-equals a printable ASCII + character** (swept over `U+0080`–`U+10FFFF`). So for a run character `c`, every element character + that can match `c` is itself ASCII, hence stored verbatim, hence foldable by both providers' + `LOWER()`. This is *false* for `InvariantCultureIgnoreCase`, which folds ~190 codepoints + (`U+00AA`→`a`, `U+017F`→`s`, the modifier letters) onto ASCII letters — **the choice of Ordinal is + load-bearing, not stylistic**. +2. **The exact set of printable-ASCII characters the JSON writer escapes** is `"`, `&`, `'`, `+`, + `<`, `>`, `\`, `` ` ``, pinned against the encoder itself so an encoder change fails the build + instead of quietly shrinking the superset. + +## Ordinal everywhere, because the culture is caller-controlled + +`UseRequestLocalization` honours `Accept-Language`, so a caller can select `tr-TR` and turn `q=I` +into `ı`. The old chain used `ToLower()` plus the default *linguistic* `StartsWith(string)`, making +the same library answer differently per caller. Comparison is now `OrdinalIgnoreCase` and ordering +`StringComparer.Ordinal` throughout, including the shared `FilterSortTake` that `state`, +`video_dynamic_range` and `content_rating` also use. That is a deliberate change to shared behaviour: +response *sets* are unchanged, response *order* is now ordinal rather than culture-dependent. + +## Ordering is best-effort, and the code says so + +Merging sources does **not** yield the exact first `limit` of the union. Each source truncates using +its own ordering — the EF source by the database collation (SQLite's is ASCII-only), the list source +by primary key — and neither is the ordinal ordering the merge applies. With `"Zulu"` and `"Éclair"` +and `limit=1`, the EF source keeps `"Zulu"` and the merge never sees `"Éclair"`, which it would have +ranked first. Below the truncation points — the normal typeahead case — the result is exact. An +earlier comment claimed exactness the code does not have; do not restore it. + +## What the row cap does and does not bound + +`ListValuedRowCap = 1000` bounds **rows fetched**, not values returned, because the pre-filter +matches rows whole. The cap rides `ORDER BY Id` — a unique integer primary key both providers can +walk — rather than the JSON column: ordering on `TEXT` would not be deterministic on MySQL, which +sorts using only the first `max_sort_length` (default 1024) bytes, so long rows sharing a prefix +would tie arbitrarily; and it avoids sorting the whole matching set. + +**It does not bound database work, and that is a known cost.** `LOWER(col) LIKE '%…'` has a leading +wildcard, so no index can be sought and every `SongMetadata` row is scanned before the cap applies. +An empty `q` matches nearly every row. Every `artist` request now scans the song table's JSON column +in addition to its previous work, once per debounced keystroke; the endpoint is authenticated by +default but `Api:RequireKeyForReads=false` makes it anonymous. Accepted for now because the scan is +bounded in *memory* and the affected table is one of the smaller ones, and because the alternative is +the migration below. **Follow-up candidate: a normalized `SongArtist` join table** (the shape +`MusicVideoArtist` already has) would make this exact, indexable and seekable — at the cost of a +dual-provider schema migration plus data backfill, changes to every scanner write path populating +`SongMetadata.Artists`, changes to the Lucene indexer, and two representations of the same fact free +to drift. Rejected for #578 on blast radius, not on merit. + +## `album_artist` 404 → 200 is additive + +Nothing consumes the 404 as a signal: the SPA's `getSearchFieldValues` (`web/src/api/search.ts`) +treats any non-200 as "no suggestions, fall back to a free-text input", which it will now do less +often. Per `api.versioning-v1`, widening which fields return values adds capability without removing +any, so no `/api/v2`. + +## Known limitation inherited, not introduced + +The **EF-sourced** fields (`genre`, `studio`, `artist`'s entity half, …) still prefix-match through +SQL `LOWER()`, which on SQLite is ASCII-only — so `q=é` does not match a stored `Édith` for those +fields either. That predates #578 and is unchanged by it; fixing it would mean filtering those tables +client-side. Noted here so the next reader does not mistake the list-valued fix for a global one. diff --git a/docs/spa-conventions.md b/docs/spa-conventions.md index 9a67c771a..827f70f0e 100644 --- a/docs/spa-conventions.md +++ b/docs/spa-conventions.md @@ -619,12 +619,12 @@ not wired here. for these two fields — the compiled query is the existing `released_inthelast:"7 day"`-style `CustomMultiFieldQueryParser` macro, so nothing downstream changes. `validation.ts`'s `ruleError` requires the value to parse as a positive integer before it's compiled. -- **Facet-value typeahead** (#434, `api.search-field-values`) — the value input for a `text` field +- **Facet-value typeahead** (#434/#578, `api.search-field-values-sources`) — the value input for a `text` field (not enum) is a combobox backed by `getSearchFieldValues` (`web/src/api/search.ts` → `GET /api/v1/search/fields/{name}/values?q=&limit=`), debounced on keystroke, prefix-matching the in-progress value against distinct terms already in the index. It always allows free-text entry as a fallback — a 404 (non-text field) or an empty result list (e.g. ElasticSearch backend) degrades to a - plain text input rather than blocking the rule. Since #578 (`api.search-field-values-list-columns`) + plain text input rather than blocking the rule. Since #578 (`api.search-field-values-sources`) `album_artist` returns values instead of 404ing, and `artist` covers free-text music-video/song credits as well as entity artists; for those two the server's list is **bounded best-effort** on a very large library, so the free-text fallback stays load-bearing — never treat an absent suggestion as an invalid -- 2.47.3 From a37847e5098883eb383d558aa4363b6c360ebd4e Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 27 Jul 2026 00:20:49 +0200 Subject: [PATCH 3/8] fix(578): a fixed row budget turns an over-matching prefilter into a false negative MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BLOCKER 1. The invariant a883e5f0 established — "the pre-filter may over-match, it must never under-match" — is only sound while the candidate set is not truncated. It was truncated. A non-ASCII or JSON-escaped prefix collapses the pattern to the bare `%"%` anchor, so every row becomes a candidate, and `ORDER BY Id LIMIT 1000` then spent the whole budget on rows that could not match. Seed 1000 songs by "zzz", put the only "éclair" in row 1001, ask for album_artist?q=é: a883e5f0 returns [], while 1b78dc9e returned "éclair" because its (separately broken) tighter pattern kept the candidate set small. Neither revision was correct — the old one under-matched at the pattern, the new one under-matched at the cap. Widening a predicate under a fixed budget starves it. So the budget is gone. Candidate rows are now walked keyset-paged on Id (`Id > @AfterId … ORDER BY Id LIMIT @Batch`), continuing past non-matching candidates and stopping on the first of: enough distinct exact matches for `limit`, a short page (source exhausted), or a 20,000-candidate-row ceiling in 2,000-row batches. The bound is on effort; it no longer silently decides the result, and the lossy case needs 20,000 rows that already passed the pre-filter before it bites. BLOCKER 2. The endpoint description and the record's rule claimed ordinal matching/dedup/ordering endpoint-wide. False for EF-backed fields: the database runs LOWER/DISTINCT/ORDER BY/LIMIT before any ordinal code, so `genre?q=é` still misses a stored "Éclair" on SQLite. Both are now scoped to the final in-memory stages, and the underlying gap is referenced as #668 rather than described as fixed. #669 (normalized SongArtist table) is referenced as the follow-up for the scan cost. Accuracy corrections to my own claims, all verified by re-running the mutations: - Only THREE of the nine Unicode cases fail 1b78dc9e (é/édith/BJÖRK — where query and stored casing differ, so the escape texts diverge); the other six pass it. The comment said all nine. They stay as continuity coverage, now labelled as such rather than as regression guards. - Ordering_Is_Best_Effort used "Zulu"/"Éclair", where DB and ordinal orderings BOTH pick "Zulu" — it could not demonstrate the divergence it claimed. Now "Zulu"/"apple", which actually diverges: ordinal ranks "Zulu" first, the DB ranks "apple" first, and limit=1 returns ["apple"]. The record sentence was false and is corrected. - The record printed literal "Édith"/"é" where it needed to show the escape TEXT (Édith, é), contradicting the very explanation it was giving. - Corrected the cost claim: the leading wildcard forces scan ACCESS, but each page stops once it has filled @Batch, so a dense query finishes early — it is not necessarily a full table scan. - The Unicode sweep is labelled a PROOF OBLIGATION: it is revision-independent and passes every revision, which is correct for what it is but must not read as regression coverage. One process note: the new record's frontmatter had a lone apostrophe inside a single-quoted YAML scalar ("SQLite's"). decisions_validate.py's hand parser accepted it; scripts/tests caught it. --- .../Queries/GetSearchFieldValuesHandler.cs | 137 +++++++++----- .../GetSearchFieldValuesHandlerTests.cs | 177 ++++++++++++------ ...SearchFieldValuesPrefilterSupersetTests.cs | 36 +++- ErsatzTV/Controllers/Api/SearchController.cs | 8 +- ErsatzTV/wwwroot/openapi/v1.json | 2 +- docs/api-conventions.md | 32 +++- docs/decisions/README.md | 2 +- .../api/search-field-values-sources.md | 90 +++++---- 8 files changed, 328 insertions(+), 156 deletions(-) diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs index ac8c6a1aa..cc5f6f5fa 100644 --- a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs @@ -15,16 +15,28 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF private const int MaxLimit = 50; /// - /// Row cap for the list-valued (JSON-array) columns on SongMetadata. The SQL pre-filter over the - /// raw JSON is a deliberate superset (a matching row is fetched whole, including its non-matching - /// elements), so the number of rows it can return has to be bounded independently of limit — - /// otherwise an empty q degenerates into materializing the entire table. The cap is applied on - /// ORDER BY Id, which is a unique integer primary key on both providers: unlike ordering on the - /// JSON column itself, that makes the truncation point genuinely deterministic (MySQL sorts TEXT - /// using only the first max_sort_length bytes, so long JSON rows sharing a prefix would tie - /// arbitrarily) and lets the engine walk the primary key instead of sorting the whole matching set. + /// Rows fetched per round trip when walking the list-valued (JSON-array) columns on + /// SongMetadata, and the overall ceiling on rows materialized for one request. + /// + /// These bound EFFORT, not the result. An earlier revision used a single fixed row budget, which + /// looked equivalent and was not: the SQL pre-filter is deliberately allowed to over-match, so + /// widening it (which is what a non-ASCII or JSON-escaped prefix does — it collapses to the bare + /// `%"%` anchor) floods the candidate set with rows that cannot match, and a fixed budget then + /// discards real matches sitting behind them. Seed 1000 songs and put the only accented artist in + /// row 1001 and the endpoint returned nothing. Over-matching is only free when the candidate set is + /// not truncated. + /// + /// + /// So the walk is keyset-paged on Id and stops on the first of: enough distinct exact matches + /// to satisfy limit, the source exhausted, or rows + /// examined. Only the last of those is lossy, and it needs that many CANDIDATE rows — rows that + /// already passed the SQL pre-filter — before it bites. + /// /// - internal const int ListValuedRowCap = 1000; + internal const int ListValuedBatchRows = 2000; + + /// + internal const int ListValuedMaxScannedRows = 20000; /// /// Escape character for the raw-JSON LIKE pre-filter. Deliberately NOT a backslash: the stored @@ -103,7 +115,7 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF if (listColumn is not null) { - values.AddRange(await GetSongListValuedValues(dbContext, listColumn, query, cancellationToken)); + values.AddRange(await GetSongListValuedValues(dbContext, listColumn, query, limit, cancellationToken)); } // ORDERING IS BEST-EFFORT, NOT EXACT. Each source truncates using its own ordering — the EF source by the @@ -164,67 +176,102 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// collections at all — so there is no server-side SELECT DISTINCT over the elements. /// /// - /// Instead: pre-filter on the raw JSON text in SQL (a superset — a row is matched, not an element, - /// so its other elements come along), cap the rows by primary key, then split and exact-filter in - /// memory. Correctness lives in the in-memory filter; the SQL pattern is only an optimization and is - /// allowed to over-match but never to under-match. When the library has more matching rows than - /// the suggestion list for these fields is the lowest-id prefix of - /// the matches rather than the complete set. + /// So the rows are walked in primary-key order, keyset-paged, and split + exact-filtered in memory. + /// The SQL LIKE is only a narrowing hint: correctness lives entirely in the in-memory filter, + /// and the walk continues past non-matching candidates rather than letting a row budget decide the + /// answer. See for why a single fixed budget was wrong. /// /// private static async Task> GetSongListValuedValues( TvContext dbContext, string column, string query, + int limit, CancellationToken cancellationToken) { - IEnumerable rows = await dbContext.Connection.QueryAsync( - new CommandDefinition( - ListValuedSql(column), - new { Pattern = JsonElementPrefixPattern(query), Cap = ListValuedRowCap }, - cancellationToken: cancellationToken)); + string sql = ListValuedSql(column); + string pattern = JsonElementPrefixPattern(query); - var values = new List(); - foreach (string row in rows) + var distinct = new System.Collections.Generic.HashSet(StringComparer.Ordinal); + var afterId = 0; + var scanned = 0; + + while (scanned < ListValuedMaxScannedRows && distinct.Count < limit) { - if (string.IsNullOrWhiteSpace(row)) + int batch = Math.Min(ListValuedBatchRows, ListValuedMaxScannedRows - scanned); + + List rows = (await dbContext.Connection.QueryAsync( + new CommandDefinition( + sql, + new { Pattern = pattern, AfterId = afterId, Batch = batch }, + cancellationToken: cancellationToken))).AsList(); + + if (rows.Count == 0) { - continue; + break; } - string[] elements; - try - { - elements = JsonSerializer.Deserialize(row); - } - catch (JsonException) - { - continue; - } + scanned += rows.Count; + afterId = rows[^1].Id; - foreach (string element in elements ?? []) + foreach (ListValuedRow row in rows) { - if (!string.IsNullOrEmpty(element) && element.StartsWith(query, StringComparison.OrdinalIgnoreCase)) + foreach (string element in ParseElements(row.Payload)) { - values.Add(element); + if (element.StartsWith(query, StringComparison.OrdinalIgnoreCase)) + { + distinct.Add(element); + } } } + + if (rows.Count < batch) + { + // short page: the source is exhausted, not merely bounded + break; + } } - return values; + return distinct.ToList(); + } + + private static IEnumerable ParseElements(string payload) + { + if (string.IsNullOrWhiteSpace(payload)) + { + return []; + } + + try + { + return (JsonSerializer.Deserialize(payload) ?? []).Where(e => !string.IsNullOrEmpty(e)); + } + catch (JsonException) + { + return []; + } } /// - /// The bounded pre-filter query. LOWER(...) LIKE ... ESCAPE, ORDER BY and LIMIT are - /// all portable between SQLite and MySQL, and the identifiers are unquoted so neither provider's - /// quoting dialect is baked in. The cap rides ORDER BY Id — a unique integer key both providers - /// can walk — rather than the JSON column, so the truncation point is well-defined and no full sort of - /// the matching set is required. + /// One keyset page of candidate rows. LOWER(...) LIKE ... ESCAPE, ORDER BY and + /// LIMIT are all portable between SQLite and MySQL, and the identifiers are unquoted so neither + /// provider's quoting dialect is baked in. Paging rides Id — a unique integer key both providers + /// can walk — rather than the JSON column: ordering on TEXT would not be deterministic on MySQL, + /// which sorts using only the first max_sort_length bytes, and a keyset predicate on the primary + /// key lets each page resume instead of re-scanning what the previous page already rejected. /// internal static string ListValuedSql(string column) => - $"SELECT {column} FROM SongMetadata WHERE {column} IS NOT NULL " + + $"SELECT Id, {column} AS Payload FROM SongMetadata WHERE {column} IS NOT NULL " + + "AND Id > @AfterId " + $"AND LOWER({column}) LIKE @Pattern ESCAPE '{LikeEscapeChar}' " + - "ORDER BY Id LIMIT @Cap"; + "ORDER BY Id LIMIT @Batch"; + + private sealed class ListValuedRow + { + public int Id { get; init; } + + public string Payload { get; init; } + } /// /// Builds the raw-JSON LIKE pattern used to narrow the rows fetched for a list-valued column. diff --git a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs index e83d2896e..b5c04239f 100644 --- a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs +++ b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs @@ -316,80 +316,77 @@ public class GetSearchFieldValuesHandlerTests } [Test] - public async Task List_Valued_Source_Row_Cap_Is_Exactly_1000_Rows() + public async Task List_Valued_Walk_Is_Bounded_By_A_Scan_Ceiling_Of_Exactly_20000_Candidate_Rows() { - // Pins the cap value in BOTH directions. Asserting only that an over-cap row is dropped would also pass - // with a cap of 3, or with the handler wrongly using `limit` as the row cap; asserting only that an - // under-cap row survives would pass with no cap at all. The marker row's element sorts first in the - // output, so an unbounded implementation cannot help but return it. - const string marker = "aaa-the-marker-row"; + // The bound is on EFFORT, not on the result: the walk pages past non-matching candidates (proved by + // Prefilter_Widening_Must_Not_Starve_The_Candidate_Set_Past_The_Row_Bound) but must still stop somewhere, + // or a bare-anchor query on a huge library becomes an unbounded scan. + // + // Pinned in BOTH directions so the constant itself is nailed down: a needle at candidate row 20000 is + // found, the same needle at row 20001 is not. Asserting only one side would pass with a ceiling of 3, or + // with no ceiling at all. + const string needle = "\u00E9clair-the-needle"; - // rows are capped by ascending Id, and Id is assigned in insertion order - async Task SeedFillerThenMarker(int fillerRows) + // seeded through raw SQL: 20k rows through the change tracker is minutes, this is milliseconds + async Task SeedFiller(TvContext context, int rows) => + await context.Database.ExecuteSqlRawAsync( + $""" + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < {rows}) + INSERT INTO SongMetadata (SongId, MetadataKind, Title, Artists, DateAdded, DateUpdated) + SELECT 0, 0, 'Filler ' || n, '["zzz-filler"]', '2026-01-01', '2026-01-01' FROM seq + """); + + await using (TvContext context = _db.CreateContext()) { - await using TvContext context = _db.CreateContext(); - for (var i = 0; i < fillerRows; i++) - { - context.SongMetadata.Add(Song($"Song {i}", [$"b{i:D4}"])); - } - - context.SongMetadata.Add(Song("Marker", [marker])); + // 19999 filler rows + the needle == candidate row 20000, the last one inside the ceiling + await SeedFiller(context, 19999); + context.SongMetadata.Add(Song("Needle", [needle])); await context.SaveChangesAsync(); + + (await context.SongMetadata.CountAsync()).ShouldBe(20000); } var handler = new GetSearchFieldValuesHandler(_db.Factory); - // 999 filler rows -> the marker is row 1000 -> inside a cap of 1000 - await SeedFillerThenMarker(999); - - Option atCap = await handler.Handle( - new GetSearchFieldValues("artist", string.Empty, 3), + Option atCeiling = await handler.Handle( + new GetSearchFieldValues("artist", "\u00E9", 50), CancellationToken.None); - atCap.IsSome.ShouldBeTrue(); - atCap.IfSome( + atCeiling.IsSome.ShouldBeTrue(); + atCeiling.IfSome( r => r.Values.ShouldBe( - new List { marker, "b0000", "b0001" }, - "the 1000th row must still be read - a cap below 1000 would drop it")); + new List { needle }, + "candidate row 20000 is inside the ceiling and must be reached")); - // one more filler row pushes the marker to row 1001 -> outside a cap of 1000 + // one more filler row pushes the needle to candidate row 20001, past the ceiling await using (TvContext context = _db.CreateContext()) { - context.SongMetadata.Add(Song("Extra", ["b9999"])); - await context.SaveChangesAsync(); - } - - await using (TvContext context = _db.CreateContext()) - { - // move the marker to the end so it sits at Id 1002, past the cap - SongMetadata existing = await context.SongMetadata.SingleAsync(m => m.Title == "Marker"); + SongMetadata existing = await context.SongMetadata.SingleAsync(m => m.Title == "Needle"); context.SongMetadata.Remove(existing); + await SeedFiller(context, 1); await context.SaveChangesAsync(); - context.SongMetadata.Add(Song("Marker", [marker])); + context.SongMetadata.Add(Song("Needle", [needle])); await context.SaveChangesAsync(); - (await context.SongMetadata.CountAsync()).ShouldBe(1001); + (await context.SongMetadata.CountAsync()).ShouldBe(20001); } - Option pastCap = await handler.Handle( - new GetSearchFieldValues("artist", string.Empty, 3), + Option pastCeiling = await handler.Handle( + new GetSearchFieldValues("artist", "\u00E9", 50), CancellationToken.None); - pastCap.IsSome.ShouldBeTrue(); - pastCap.IfSome( - r => r.Values.ShouldNotContain( - marker, - "the 1001st row must not be read - an unbounded query would return it first")); - pastCap.IfSome(r => r.Values.ShouldBe(new List { "b0000", "b0001", "b0002" })); + pastCeiling.IsSome.ShouldBeTrue(); + pastCeiling.IfSome( + r => r.Values.ShouldBeEmpty("candidate row 20001 is past the ceiling and must not be reached")); - // positive control: the same row IS reachable once the pre-filter narrows the candidates below the cap, - // so the assertion above cannot be passing because the value is unreachable for an unrelated reason + // positive control: narrowing the prefix drops the filler rows from the candidate set entirely, so the + // same row is reachable again -- the assertion above is about the ceiling, not an unreachable value Option narrowed = await handler.Handle( - new GetSearchFieldValues("artist", "aaa", 50), + new GetSearchFieldValues("artist", "zzz", 50), CancellationToken.None); narrowed.IsSome.ShouldBeTrue(); - narrowed.IfSome(r => r.Values.ShouldBe(new List { marker })); + narrowed.IfSome(r => r.Values.ShouldBe(new List { "zzz-filler" })); } [Test] @@ -433,10 +430,17 @@ public class GetSearchFieldValuesHandlerTests [TestCase("\u00D6", "\u00D6zdemir")] public async Task Prefilter_Never_Under_Matches_A_NonAscii_Value(string query, string stored) { - // The SQL pre-filter narrows on raw JSON, where non-ASCII is stored escaped (\u00C9). Any narrowing - // that includes an escaped character under-matches, because SQL LOWER() folds the escape TEXT and not - // the codepoint — every one of these cases returned EMPTY before the pattern builder was restricted to - // the leading verbatim-ASCII run. Accented artists are the common case in a music library, not an edge. + // The SQL pre-filter narrows on raw JSON, where non-ASCII is stored escaped (\u00C9). A narrowing run + // that includes an escaped character can under-match, because SQL LOWER() folds the escape TEXT and not + // the codepoint it denotes. + // + // Be precise about what these cases guard: THREE of the nine fail against 1b78dc9e (the JSON-encode-the- + // whole-prefix revision) — ("é", "Édith Piaf"), ("édith", "Édith Piaf") and ("BJÖRK", "Björk") — because + // those are the ones where the query and stored casings differ, so the lowercased escape text (\u00e9) + // and the stored escape text (\u00C9) diverge. The other SIX pass that revision: when the query casing + // already matches the stored casing the escape texts happen to line up. They are kept as continuity + // coverage of the ASCII-run narrowing, NOT as regression guards. Accented artists are the common case in + // a music library, so both groups are worth pinning. await using (TvContext context = _db.CreateContext()) { context.SongMetadata.AddRange( @@ -532,25 +536,80 @@ public class GetSearchFieldValuesHandlerTests public async Task Ordering_Is_Best_Effort_When_A_Source_Truncates() { // Documents the acknowledged imprecision rather than claiming exactness the code does not have. The EF - // source truncates by the DATABASE collation (SQLite's is ASCII-only), which is not the ordinal ordering - // the merge then applies — so a value the database ranked outside its first `limit` never reaches the - // merge, even if the merge would have ranked it first. + // source truncates by the DATABASE collation, which is NOT the ordinal ordering the merge then applies — + // so a value the database ranked outside its first `limit` never reaches the merge, even if the merge + // would have ranked it first. + // + // "Zulu" vs "apple" is the pair that actually diverges: ordinal puts every ASCII uppercase letter before + // every lowercase one, so ordinal ranks "Zulu" first, while a case-insensitive database ordering ranks + // "apple" first. (An earlier version used "Zulu"/"Éclair", where BOTH orderings pick "Zulu" — it could + // not have told the two apart, and the divergence it claimed to show did not exist.) await using (TvContext context = _db.CreateContext()) { context.ArtistMetadata.Add(Artist("Zulu")); - context.ArtistMetadata.Add(Artist("\u00C9clair")); + context.ArtistMetadata.Add(Artist("apple")); await context.SaveChangesAsync(); } var handler = new GetSearchFieldValuesHandler(_db.Factory); - // with room for both, the ordinal merge ranks "Zulu" (ASCII) before "\u00C9clair" (non-ASCII) + // with room for both, the ordinal merge ranks "Zulu" first (await handler.Handle(new GetSearchFieldValues("artist", string.Empty, 50), CancellationToken.None)) - .IfSome(r => r.Values.ShouldBe(new List { "Zulu", "\u00C9clair" })); + .IfSome(r => r.Values.ShouldBe(new List { "Zulu", "apple" })); - // with limit=1 the database picks the survivor by ITS collation, and the merge only sees that one + // with limit=1 the database picks the survivor by ITS ordering, and the merge only ever sees that one (await handler.Handle(new GetSearchFieldValues("artist", string.Empty, 1), CancellationToken.None)) - .IfSome(r => r.Values.Count.ShouldBe(1)); + .IfSome(r => r.Values.ShouldBe(new List { "apple" })); + } + + [Test] + public async Task Prefilter_Widening_Must_Not_Starve_The_Candidate_Set_Past_The_Row_Bound() + { + // BLOCKER-1 REGRESSION (fails a883e5f0). A non-ASCII prefix narrows to the bare anchor `%"%`, so EVERY + // row qualifies. Under a fixed 1000-row budget the 1001st row -- the only exact match -- was discarded + // before the in-memory filter ever saw it, and the endpoint returned []. "Over-matching is free" only + // holds when the candidate set is not truncated; with a fixed cap, widening the predicate STARVES it. + await using (TvContext context = _db.CreateContext()) + { + for (var i = 0; i < 1000; i++) + { + context.SongMetadata.Add(Song($"Filler {i}", ["zzz-filler"], ["zzz-filler-album"])); + } + + context.SongMetadata.Add(Song("Needle", ["\u00E9clair"], ["\u00E9clair"])); + await context.SaveChangesAsync(); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + Option albumArtist = await handler.Handle( + new GetSearchFieldValues("album_artist", "\u00E9", 50), + CancellationToken.None); + + albumArtist.IsSome.ShouldBeTrue(); + albumArtist.IfSome(r => r.Values.ShouldBe(new List { "\u00E9clair" })); + + // same starvation shape on the merged `artist` field + Option artist = await handler.Handle( + new GetSearchFieldValues("artist", "\u00E9", 50), + CancellationToken.None); + + artist.IsSome.ShouldBeTrue(); + artist.IfSome(r => r.Values.ShouldBe(new List { "\u00E9clair" })); + + // ... and for a prefix stopped early by a JSON-escaped character, which also widens to the bare anchor + await using (TvContext context = _db.CreateContext()) + { + context.SongMetadata.Add(Song("Ampersand", ["&Me"])); + await context.SaveChangesAsync(); + } + + Option escapedPrefix = await handler.Handle( + new GetSearchFieldValues("artist", "&M", 50), + CancellationToken.None); + + escapedPrefix.IsSome.ShouldBeTrue(); + escapedPrefix.IfSome(r => r.Values.ShouldBe(new List { "&Me" })); } private static ArtistMetadata Artist(string title) => new() diff --git a/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs b/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs index f02958c11..e0bd95f07 100644 --- a/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs +++ b/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs @@ -30,6 +30,13 @@ public class SearchFieldValuesPrefilterSupersetTests } } + /// + /// PROOF OBLIGATION, not regression coverage. This is revision-independent — it asserts a property of + /// .NET's ordinal case folding and passes against every revision of the handler, including the ones with + /// the under-matching pattern builder. It is here because the superset argument DEPENDS on it: if a + /// future runtime changed this, the ASCII narrowing would start under-matching with nothing else to + /// catch it. + /// [Test] public void No_NonAscii_Codepoint_OrdinalIgnoreCase_Equals_A_Printable_Ascii_Character() { @@ -63,6 +70,10 @@ public class SearchFieldValuesPrefilterSupersetTests offenders.ShouldBeEmpty(); } + /// + /// PROOF OBLIGATION plus one regression assertion. The escaped-set half is revision-independent (it + /// pins the encoder); the pattern-builder half below fails against 1b78dc9e. + /// [Test] public void The_Json_Writer_Escapes_Exactly_The_Ascii_Characters_The_Pattern_Builder_Refuses_To_Narrow_On() { @@ -109,14 +120,29 @@ public class SearchFieldValuesPrefilterSupersetTests } [Test] - public void Row_Cap_Is_Applied_On_The_Primary_Key_Not_The_Json_Column() + public void Candidate_Rows_Are_Keyset_Paged_On_The_Primary_Key_Not_The_Json_Column() { - // MySQL sorts TEXT/longtext using only max_sort_length bytes, so ordering on the JSON column would let - // rows sharing a long prefix tie arbitrarily and the surviving 1000 would be unspecified. Id is a unique - // integer key on both providers. + // Two things at once. ORDER BY rides Id because MySQL sorts TEXT/longtext using only max_sort_length + // bytes, so ordering on the JSON column would let long rows sharing a prefix tie arbitrarily. And the + // `Id > @AfterId` keyset predicate is what makes the walk RESUMABLE — without it the bound would have to + // be a single fixed budget, which is exactly the starvation defect this replaced. string sql = GetSearchFieldValuesHandler.ListValuedSql("Artists"); - sql.ShouldContain("ORDER BY Id LIMIT @Cap"); + sql.ShouldContain("Id > @AfterId"); + sql.ShouldContain("ORDER BY Id LIMIT @Batch"); sql.ShouldNotContain("ORDER BY Artists"); } + + [Test] + public void The_Scan_Ceiling_Is_A_Whole_Number_Of_Batches() + { + // The walk trims its last batch to land exactly on the ceiling, so this is not load-bearing for + // correctness — but a ceiling that is not a multiple of the batch size means the final round trip + // fetches a short page, which the loop would otherwise read as "source exhausted". + (GetSearchFieldValuesHandler.ListValuedMaxScannedRows % GetSearchFieldValuesHandler.ListValuedBatchRows) + .ShouldBe(0); + + GetSearchFieldValuesHandler.ListValuedMaxScannedRows + .ShouldBeGreaterThan(GetSearchFieldValuesHandler.ListValuedBatchRows); + } } diff --git a/ErsatzTV/Controllers/Api/SearchController.cs b/ErsatzTV/Controllers/Api/SearchController.cs index 9f794c634..539141f68 100644 --- a/ErsatzTV/Controllers/Api/SearchController.cs +++ b/ErsatzTV/Controllers/Api/SearchController.cs @@ -205,9 +205,11 @@ public class SearchController(IMediator mediator) : ControllerBase "Returns distinct whole values from the database for the given text field, filtered by an " + "optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. " + "404 when the field is unknown, is not a text field, or is a text field with no distinct-value " + - "source. Matching, dedup and ordering are ordinal, not culture-dependent. The list-valued music " + - "fields (artist, album_artist) are bounded best-effort: on a very large library they return a " + - "bounded subset of the matches rather than every match.")] + "source. The final filter, dedup and ordering applied to the response are ordinal and not " + + "culture-dependent; note that fields sourced by a plain database query are additionally " + + "pre-filtered by the database collation first, which on SQLite is ASCII-only. The list-valued " + + "music fields (artist, album_artist) are bounded best-effort: on a very large library they may " + + "return a bounded subset of the matches rather than every match.")] [EndpointGroupName("general")] [ProducesResponseType(typeof(SearchFieldValuesResponseModel), StatusCodes.Status200OK)] [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)] diff --git a/ErsatzTV/wwwroot/openapi/v1.json b/ErsatzTV/wwwroot/openapi/v1.json index 22395ee44..84cdaaae0 100644 --- a/ErsatzTV/wwwroot/openapi/v1.json +++ b/ErsatzTV/wwwroot/openapi/v1.json @@ -18063,7 +18063,7 @@ "Search" ], "summary": "List distinct database values for a text search field", - "description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source. Matching, dedup and ordering are ordinal, not culture-dependent. The list-valued music fields (artist, album_artist) are bounded best-effort: on a very large library they return a bounded subset of the matches rather than every match.", + "description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source. The final filter, dedup and ordering applied to the response are ordinal and not culture-dependent; note that fields sourced by a plain database query are additionally pre-filtered by the database collation first, which on SQLite is ASCII-only. The list-valued music fields (artist, album_artist) are bounded best-effort: on a very large library they may return a bounded subset of the matches rather than every match.", "operationId": "GetSearchFieldValues", "parameters": [ { diff --git a/docs/api-conventions.md b/docs/api-conventions.md index 06567872f..fb04a9665 100644 --- a/docs/api-conventions.md +++ b/docs/api-conventions.md @@ -457,17 +457,29 @@ or partly) by `SongMetadata.Artists`/`AlbumArtists`, which EF maps as **primitiv JSON array per row in a single column, with no server-side projection on either provider. `album_artist` therefore no longer 404s, and `artist` now also covers free-text music-video (`MusicVideoArtist`) and song credits, not only entity artists. For these fields the handler narrows on the raw JSON in SQL and -caps the rows it reads, so on a very large library the response is a bounded subset of the matches rather -than every match — say so in the `[EndpointDescription]` of any endpoint that adopts this shape. +walks candidate rows keyset-paged on the primary key, so on a very large library the response may be a +bounded subset of the matches — say so in the `[EndpointDescription]` of any endpoint that adopts this shape. -Two rules generalize beyond this endpoint. **A SQL pre-filter layered under an in-memory exact filter may -over-match but must NEVER under-match** — an under-match is a silent false negative that no assertion over -the returned values can see, so prove the superset (ours is proved by an exhaustive Unicode sweep, not by -argument) rather than eyeballing it. And **prefix matching, dedup and ordering on any `/api/*` read must be -ordinal, never current-culture**: `UseRequestLocalization` honours `Accept-Language`, so `ToLower()` and the -default linguistic `StartsWith(string)` let a caller change the result by changing a header. Full rationale, -the rejected normalized-side-table alternative, the accepted scan cost and the dual-provider `LIKE`/`LOWER` -rules: `api.search-field-values-sources` (which supersedes `api.search-field-values`). +Three rules generalize beyond this endpoint. + +1. **A SQL pre-filter layered under an in-memory exact filter may over-match but must NEVER under-match.** + An under-match is a silent false negative that no assertion over the returned values can see, so prove + the superset (ours is proved by an exhaustive Unicode sweep) rather than eyeballing it. +2. **That licence to over-match is void the moment the candidate set is truncated by a fixed row budget.** + Widening the predicate then starves the budget with rows that cannot match and drops real matches sitting + behind them — the same false negative, one stage later. Bound the WORK (page with a keyset cursor until you + have enough matches, the source is exhausted, or a scan ceiling is hit), never the candidate rows with a + single `LIMIT` whose survivors silently become the answer. Both revisions of this endpoint got this wrong + in opposite directions before it was written down. +3. **Prefix matching, dedup and ordering must be ordinal, not current-culture** (`OrdinalIgnoreCase`, + `StringComparer.Ordinal`): `UseRequestLocalization` honours `Accept-Language`, so `ToLower()` and the + default linguistic `StartsWith(string)` let a caller change the result by changing a header. **Scope the + claim to the stage that actually holds it** — a value set that a database `LOWER`/`DISTINCT`/`ORDER BY`/ + `LIMIT` already filtered and truncated is not ordinal no matter what runs after it, and saying otherwise in + an `[EndpointDescription]` publishes a false contract (ersatztv#668). + +Full rationale, the rejected normalized-side-table alternative (ersatztv#669), the accepted scan cost and the +dual-provider `LIKE`/`LOWER` rules: `api.search-field-values-sources` (supersedes `api.search-field-values`). **Param + DTO expansion (#293, cap `search/all-items`)**: no new endpoint — `GET /api/v1/search/all-items` gained two **optional** query params (`pageSize` default 500, clamped 1–1000 via the §1 Logs `Math.Clamp` diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 72bbd138b..af8e4d2d5 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -28,7 +28,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `api.schedule-item-flat-dto` | Schedule-item GET/POST/PUT use a flat, non-polymorphic `ScheduleItemResponseModel` (every subtype field promoted to a nullable top-level member) instead of the polymorphic Application VM hierarchy, with mutation field names matching `ScheduleItemRequest` 1:1 for a lossless round-trip. | 2026-07-10 | [link](records/api/schedule-item-flat-dto.md) | | `api.scheduling-hardening` | Create/Replace handlers guard against null/whitespace `name` (`IsNullOrWhiteSpace`, not just `Length`) to prevent NRE-500s, template-item overlap validation compares by index (not record value-equality) to catch exact-duplicate items, and unreachable 404 `ProducesResponseType` attributes on create-only actions are trimmed. | 2026-07-13 | [link](records/api/scheduling-hardening.md) | | `api.search-allitems-paging` | `GET /api/v1/search/all-items` is paginated (capped page size, `Totals` field) to bound DoS exposure; the SPA add-all flow pages to completeness instead of relying on an unbounded response. | 2026-07-18 | [link](records/api/search-allitems-paging.md) | -| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). Prefix matching, dedup and ordering are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled. A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&'+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; the rows are capped at 1000 on `ORDER BY Id` (a unique integer key, not the JSON column) and then split, filtered, deduped, sorted and `limit`ed in memory. Ordering across merged sources is best-effort, exact only below the truncation points. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | +| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — this is scoped to the in-memory stages on purpose: a field sourced by a plain EF query is pre-filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&'+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; candidate rows are walked keyset-paged on `Id` (a unique integer key, not the JSON column) and split + exact-filtered in memory, stopping on the first of enough distinct matches, source exhausted, or a 20,000-candidate-row scan ceiling — the bound is on EFFORT, never a fixed row budget, because a fixed budget lets a widened pre-filter starve the candidate set and drop real matches. Ordering across merged sources is best-effort, exact only below the truncation points. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | | `api.search-paging-cap` | Search stays capped at 100 items per media kind; an overflowing kind's "See all" reuses library-browse paging instead of adding new API surface. | 2026-07-11 | [link](records/api/search-paging-cap.md) | | `api.versioning-v1` | The entire `/api` surface is versioned to `/api/v1` uniformly (no unversioned corner); legacy unversioned callers are rewritten in-pipeline (not redirected) with Deprecation/Link/Sunset headers, and post-freeze `/api/v1` is additive-only — a breaking change requires `/api/v2`. | 2026-07-13 | [link](records/api/versioning-v1.md) | | `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) | diff --git a/docs/decisions/records/api/search-field-values-sources.md b/docs/decisions/records/api/search-field-values-sources.md index f138aab11..48973e499 100644 --- a/docs/decisions/records/api/search-field-values-sources.md +++ b/docs/decisions/records/api/search-field-values-sources.md @@ -5,8 +5,8 @@ status: active since: '2026-07-26' supersedes: api.search-field-values@2026-07-23 superseded-by: none -rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). Prefix matching, dedup and ordering are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled. A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&''+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; the rows are capped at 1000 on `ORDER BY Id` (a unique integer key, not the JSON column) and then split, filtered, deduped, sorted and `limit`ed in memory. Ordering across merged sources is best-effort, exact only below the truncation points.' -signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, raw JSON LIKE pre-filter, superset never under-match, accented artist unsuggestable, LOWER cannot fold a \u00XX escape, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, ListValuedRowCap, ORDER BY Id not TEXT, max_sort_length, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176' +rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — this is scoped to the in-memory stages on purpose: a field sourced by a plain EF query is pre-filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&''+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; candidate rows are walked keyset-paged on `Id` (a unique integer key, not the JSON column) and split + exact-filtered in memory, stopping on the first of enough distinct matches, source exhausted, or a 20,000-candidate-row scan ceiling — the bound is on EFFORT, never a fixed row budget, because a fixed budget lets a widened pre-filter starve the candidate set and drop real matches. Ordering across merged sources is best-effort, exact only below the truncation points.' +signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, raw JSON LIKE pre-filter, superset never under-match, accented artist unsuggestable, LOWER cannot fold a \u00XX escape, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, ListValuedRowCap, ORDER BY Id not TEXT, max_sort_length, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669' mechanics: '`GetSearchFieldValuesHandler` (`GetSource`, `GetSongListValuedColumn`, `GetSongListValuedValues`, `ListValuedSql`, `JsonElementPrefixPattern`, `FilterSortTake`); `SearchFieldValuesPrefilterSupersetTests` (the superset proof); api-conventions.md; spa-conventions.md §12' --- @@ -39,10 +39,10 @@ need no lookup; text fields need a live one; the source is the database and neve This is the load-bearing invariant, and the first implementation got it wrong in a way that no test of the returned values could see. It JSON-encoded the whole query prefix, reasoning that since the -stored text escapes non-ASCII (`Édith` is on disk as `Édith`), encoding the prefix the same way -would line up. **It does not.** SQL `LOWER()` lowercases the *escape text* — `É` — it cannot -case-fold the codepoint that escape denotes. So `q=é` produced a pattern of `é` that never -matched, the row was discarded before the in-memory filter ever saw it, and **every accented artist +stored text escapes non-ASCII (`Édith` is on disk as `\u00C9dith`), encoding the prefix the same way +would line up. **It does not.** SQL `LOWER()` lowercases the *escape text* — `\u00C9` becomes `\u00c9` — +it cannot case-fold the codepoint that escape denotes, so the stored `\u00C9` never folds to `\u00e9`. +A `q=é` prefix produced the pattern `\u00e9`, which never matched, the row was discarded before the in-memory filter ever saw it, and **every accented artist was silently unsuggestable** — in a music library, the common case, not an edge case. The fix is to narrow only on ground that can be proved: the leading run of characters the JSON @@ -68,38 +68,62 @@ Soundness rests on two facts, both **asserted by exhaustive computation** in `UseRequestLocalization` honours `Accept-Language`, so a caller can select `tr-TR` and turn `q=I` into `ı`. The old chain used `ToLower()` plus the default *linguistic* `StartsWith(string)`, making the same library answer differently per caller. Comparison is now `OrdinalIgnoreCase` and ordering -`StringComparer.Ordinal` throughout, including the shared `FilterSortTake` that `state`, -`video_dynamic_range` and `content_rating` also use. That is a deliberate change to shared behaviour: -response *sets* are unchanged, response *order* is now ordinal rather than culture-dependent. +`StringComparer.Ordinal` throughout the in-memory stages, including the shared `FilterSortTake` that +`state`, `video_dynamic_range` and `content_rating` also use. That is a deliberate change to shared +behaviour: response *sets* are unchanged, response *order* is now ordinal rather than +culture-dependent. + +**Scope this claim carefully — it is not endpoint-wide.** A field sourced by a plain EF query runs +the database's `LOWER`, `DISTINCT`, `ORDER BY` and `LIMIT` *before* any ordinal code executes, so the +database has already decided which values survive. Store a genre `"Éclair"` on SQLite and ask for +`genre?q=é`: SQLite's ASCII-only `LOWER()` drops it before the ordinal in-memory filter ever runs, and +a case-insensitive collation's `DISTINCT` can likewise collapse values ordinal dedup would have kept. +The endpoint description and this record's `rule:` therefore say "the final filter, dedup and +ordering", not "matching is ordinal". Closing that gap means filtering those tables client-side and is +tracked separately as **ersatztv#668** — do not describe it as fixed here. ## Ordering is best-effort, and the code says so Merging sources does **not** yield the exact first `limit` of the union. Each source truncates using -its own ordering — the EF source by the database collation (SQLite's is ASCII-only), the list source -by primary key — and neither is the ordinal ordering the merge applies. With `"Zulu"` and `"Éclair"` -and `limit=1`, the EF source keeps `"Zulu"` and the merge never sees `"Éclair"`, which it would have -ranked first. Below the truncation points — the normal typeahead case — the result is exact. An -earlier comment claimed exactness the code does not have; do not restore it. +its own ordering — the EF source by the database collation, the list source by primary key — and +neither is the ordinal ordering the merge applies. The pair that actually demonstrates it is `"Zulu"` +and `"apple"`: ordinal puts every ASCII uppercase letter before every lowercase one, so the merge +ranks `"Zulu"` first, while the case-insensitive database ordering ranks `"apple"` first — at +`limit=1` the response is `["apple"]`, not the ordinally-first `"Zulu"`. Below the truncation points +— the normal typeahead case — the result is exact. An earlier comment claimed exactness the code does +not have; do not restore it. (An earlier version of this record used `"Zulu"`/`"Éclair"` as the +example, where both orderings pick `"Zulu"` — it demonstrated nothing.) -## What the row cap does and does not bound +## What the bound bounds — and why a fixed row budget was wrong -`ListValuedRowCap = 1000` bounds **rows fetched**, not values returned, because the pre-filter -matches rows whole. The cap rides `ORDER BY Id` — a unique integer primary key both providers can -walk — rather than the JSON column: ordering on `TEXT` would not be deterministic on MySQL, which -sorts using only the first `max_sort_length` (default 1024) bytes, so long rows sharing a prefix -would tie arbitrarily; and it avoids sorting the whole matching set. +The first two revisions used a single fixed row budget (`LIMIT 1000`) and called it "bounded". It was +bounded, and it was also **wrong**, in a way that only shows up in composition with the pre-filter. +The pre-filter is deliberately allowed to over-match; a non-ASCII or JSON-escaped prefix makes it +collapse to the bare `%"%` anchor, so *every* row becomes a candidate. Under a fixed budget the +budget is then spent on rows that cannot match, and real matches behind them are discarded before the +in-memory filter sees them. Seed 1,000 songs by `"zzz…"`, put the only `"éclair"` in row 1,001, ask +for `album_artist?q=é`, and the endpoint returns `[]`. **"Over-matching is free" is only true when the +candidate set is not truncated.** Widening the predicate under a fixed budget starves it — the same +false-negative class the pre-filter fix was addressing, moved one stage later. -**It does not bound database work, and that is a known cost.** `LOWER(col) LIKE '%…'` has a leading -wildcard, so no index can be sought and every `SongMetadata` row is scanned before the cap applies. -An empty `q` matches nearly every row. Every `artist` request now scans the song table's JSON column -in addition to its previous work, once per debounced keystroke; the endpoint is authenticated by -default but `Api:RequireKeyForReads=false` makes it anonymous. Accepted for now because the scan is -bounded in *memory* and the affected table is one of the smaller ones, and because the alternative is -the migration below. **Follow-up candidate: a normalized `SongArtist` join table** (the shape -`MusicVideoArtist` already has) would make this exact, indexable and seekable — at the cost of a -dual-provider schema migration plus data backfill, changes to every scanner write path populating -`SongMetadata.Artists`, changes to the Lucene indexer, and two representations of the same fact free -to drift. Rejected for #578 on blast radius, not on merit. +So the walk is **keyset-paged on `Id`** (`Id > @AfterId … ORDER BY Id LIMIT @Batch`) and continues +past non-matching candidates, stopping on the first of: enough distinct exact matches to satisfy +`limit`, a short page (source exhausted), or `ListValuedMaxScannedRows = 20000` candidate rows +examined, in `ListValuedBatchRows = 2000` batches. Only the ceiling is lossy, and it takes 20,000 rows +that already passed the SQL pre-filter before it bites. Paging on the primary key rather than the JSON +column also keeps the truncation point well-defined on MySQL, which sorts `TEXT` using only the first +`max_sort_length` (default 1024) bytes. + +**The remaining cost is scan ACCESS, not necessarily a full table scan.** `LOWER(col) LIKE '%…'` has a +leading wildcard, so no index can be sought and the engine walks rows — but each page stops as soon as +it has filled `@Batch`, so a query whose candidates are dense finishes early and only a sparse +bare-anchor query walks far. Worst case is bounded at 20,000 candidate rows across at most 10 round +trips. The endpoint is authenticated by default, though `Api:RequireKeyForReads=false` makes it +anonymous. **Follow-up: a normalized `SongArtist` join table** (the shape `MusicVideoArtist` already +has) would make this exact, indexable and seekable, at the cost of a dual-provider schema migration +plus data backfill, changes to every scanner write path populating `SongMetadata.Artists`, changes to +the Lucene indexer, and two representations of the same fact free to drift. Tracked as +**ersatztv#669**; rejected for #578 on blast radius, not on merit. ## `album_artist` 404 → 200 is additive @@ -113,4 +137,6 @@ any, so no `/api/v2`. The **EF-sourced** fields (`genre`, `studio`, `artist`'s entity half, …) still prefix-match through SQL `LOWER()`, which on SQLite is ASCII-only — so `q=é` does not match a stored `Édith` for those fields either. That predates #578 and is unchanged by it; fixing it would mean filtering those tables -client-side. Noted here so the next reader does not mistake the list-valued fix for a global one. +client-side. Tracked as **ersatztv#668**. Noted here so the next reader does not mistake the +list-valued fix for a global one, and so the scoped ordinal wording above is not "tidied" into a +broader claim. -- 2.47.3 From fbc7b2a1dd64adbc88091106ca04af19ab095a37 Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 27 Jul 2026 01:16:28 +0200 Subject: [PATCH 4/8] =?UTF-8?q?fix(578):=20bound=20the=20Id=20KEYSPACE=20?= =?UTF-8?q?=E2=80=94=20LIMIT=20cannot=20bound=20a=20query=20that=20matches?= =?UTF-8?q?=20nothing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BLOCKER. Round 3's ceiling counted LIKE-positive candidates, which is not the quantity that needed bounding. To return an empty page the engine must first evaluate every eligible row, so a no-match query came back with rows.Count == 0, ended the walk having counted zero against the ceiling, and had already inspected the entire table. Round trips and materialized rows were bounded; database work was not. Worse, a dense widened prefix could materialize 20x the candidates and make 10x the round trips of round 1 — a regression dressed as a bound. Third time bounding the wrong quantity: revisions 1-2 bounded the RESULT (a fixed LIMIT budget the widened pattern starved), revision 3 bounded the CANDIDATES, and neither bounds what the database LOOKS AT. Now the Id range is closed on both sides — `Id > @AfterId AND Id <= @AfterId + @Window` — so each round trip is a primary-key range scan of known width. The LIKE still decides what comes back; it no longer decides how much gets looked at. The walk advances by the WINDOW, never by what returned (an empty page means "nothing matched in this stretch", not "exhausted"), and there is deliberately no LIMIT in the SQL — the window caps the row count, and a LIMIT would only restore the illusion that it is doing the bounding. One indexed SELECT MAX(Id) up front stops the walk burning windows on empty keyspace. Result: at most 10 round trips and 20,000 rows inspected for any q, matching or not. New test `List_Valued_Walk_Cannot_Inspect_Past_The_Bound_When_Nothing_Matches` covers the case every earlier bound test missed — they all gave the pre-filter something to return. With a narrow pattern and a needle at row 30,000, revision 3 returns the needle (right answer, unbounded work) and this revision returns []. Paired with a positive control at row 20,000 inside the bound. The ceiling's remaining false negative (20,000 "zzz" rows then "éclair" at 20,001 -> []) is kept and still pinned: it is the documented bounded-best-effort contract at a rarer threshold, not a defect. Stale/false comments, several of them repeat offenders: - The handler still used "Zulu"/"Éclair" as the truncation example, where both orderings pick "Zulu". I had corrected the test and left the comment. Now "Zulu"/"apple" in both. - "leans entirely on the row cap" described a mechanism that no longer exists. - The escape example rendered as literal Édith/É/é rather than the escape TEXT (Édith, É, é) in both the handler and the record, contradicting the explanation it was giving. - "every accented artist was unsuggestable" overstated it: exactly three of the nine pinned cases fail 1b78dc9e (those where query and stored casing differ, so the escape texts diverge); the other six pass. The record now says what the test comment already said. - The record claimed the ordinal switch left response SETS unchanged. False — ordering runs before Take(limit), so with "Zulu"/"apple", empty q and limit=1 the comparer changes which value survives. - The_Scan_Ceiling_Is_A_Whole_Number_Of_Batches was a style preference wearing a correctness costume (a 20,500 ceiling is perfectly safe — the final window clamps). Replaced with the invariant that actually matters: the walk's own arithmetic terminates and lands exactly on the ceiling. --- .../Queries/GetSearchFieldValuesHandler.cs | 117 ++++++++++-------- .../GetSearchFieldValuesHandlerTests.cs | 117 +++++++++++------- ...SearchFieldValuesPrefilterSupersetTests.cs | 48 ++++--- docs/decisions/README.md | 2 +- .../api/search-field-values-sources.md | 94 +++++++++----- 5 files changed, 235 insertions(+), 143 deletions(-) diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs index cc5f6f5fa..6700a44f5 100644 --- a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs @@ -15,28 +15,34 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF private const int MaxLimit = 50; /// - /// Rows fetched per round trip when walking the list-valued (JSON-array) columns on - /// SongMetadata, and the overall ceiling on rows materialized for one request. + /// Width of the Id keyspace window examined per round trip when walking the list-valued + /// (JSON-array) columns on SongMetadata, and the ceiling on total keyspace visited per request. /// - /// These bound EFFORT, not the result. An earlier revision used a single fixed row budget, which - /// looked equivalent and was not: the SQL pre-filter is deliberately allowed to over-match, so - /// widening it (which is what a non-ASCII or JSON-escaped prefix does — it collapses to the bare - /// `%"%` anchor) floods the candidate set with rows that cannot match, and a fixed budget then - /// discards real matches sitting behind them. Seed 1000 songs and put the only accented artist in - /// row 1001 and the endpoint returned nothing. Over-matching is only free when the candidate set is - /// not truncated. + /// What these bound is rows the database LOOKS AT, and getting that right took three tries. + /// A single fixed LIMIT budget was wrong because the pre-filter is deliberately allowed to + /// over-match: a non-ASCII or JSON-escaped prefix collapses the pattern to the bare %"% + /// anchor, every row becomes a candidate, and the budget gets spent on rows that cannot match while + /// real matches sit behind them. Keyset-paging with a LIMIT fixed that but bounded the wrong + /// quantity: LIMIT caps rows RETURNED, and a query with no matches must evaluate every + /// eligible row before it can return an empty page. Rows examined stayed bounded only by table size. /// /// - /// So the walk is keyset-paged on Id and stops on the first of: enough distinct exact matches - /// to satisfy limit, the source exhausted, or rows - /// examined. Only the last of those is lossy, and it needs that many CANDIDATE rows — rows that - /// already passed the SQL pre-filter — before it bites. + /// So the window is on the keyspace — Id > @AfterId AND Id <= @AfterId + @Window + /// — which the primary key can range-scan. The LIKE still decides what comes BACK; it no + /// longer decides how much gets LOOKED AT. Each round trip inspects at most + /// rows and the walk visits at most in total, whether or not + /// anything matches. + /// + /// + /// The walk stops on the first of: enough distinct exact matches to satisfy limit, the + /// keyspace exhausted, or the ceiling. Only the ceiling is lossy — a match beyond it is not found, + /// which is the bounded-best-effort contract the endpoint documents. /// /// - internal const int ListValuedBatchRows = 2000; + internal const int ListValuedWindowRows = 2000; - /// - internal const int ListValuedMaxScannedRows = 20000; + /// + internal const int ListValuedMaxVisitedRows = 20000; /// /// Escape character for the raw-JSON LIKE pre-filter. Deliberately NOT a backslash: the stored @@ -121,8 +127,9 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF // ORDERING IS BEST-EFFORT, NOT EXACT. Each source truncates using its own ordering — the EF source by the // database collation (SQLite's NOCASE/BINARY is ASCII-only), the list source by primary key — and neither // is the ordinal ordering applied here. So when a source actually truncates, a value it dropped may have - // outranked one that survived: with "Zulu" and "Éclair" and limit=1 the EF source keeps "Zulu" and the - // merge never sees "Éclair". Below the truncation points (the normal typeahead case) the result is exact. + // outranked one that survived: with "Zulu" and "apple" and limit=1 the database keeps "apple" (its + // ordering is case-insensitive) while ordinal ranks "Zulu" first, so the merge never sees "Zulu". + // Below the truncation points (the normal typeahead case) the result is exact. return new SearchFieldValuesResponseModel(FilterSortTake(values.Distinct(StringComparer.Ordinal), query, limit)); } @@ -192,27 +199,26 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF string sql = ListValuedSql(column); string pattern = JsonElementPrefixPattern(query); + // one indexed lookup, so an exhausted keyspace ends the walk instead of burning windows on empty range + int maxId = await dbContext.Connection.ExecuteScalarAsync( + new CommandDefinition( + "SELECT COALESCE(MAX(Id), 0) FROM SongMetadata", + cancellationToken: cancellationToken)); + var distinct = new System.Collections.Generic.HashSet(StringComparer.Ordinal); var afterId = 0; - var scanned = 0; + var visited = 0; - while (scanned < ListValuedMaxScannedRows && distinct.Count < limit) + while (afterId < maxId && visited < ListValuedMaxVisitedRows && distinct.Count < limit) { - int batch = Math.Min(ListValuedBatchRows, ListValuedMaxScannedRows - scanned); + int window = Math.Min(ListValuedWindowRows, ListValuedMaxVisitedRows - visited); + int upperId = afterId + window; - List rows = (await dbContext.Connection.QueryAsync( + IEnumerable rows = await dbContext.Connection.QueryAsync( new CommandDefinition( sql, - new { Pattern = pattern, AfterId = afterId, Batch = batch }, - cancellationToken: cancellationToken))).AsList(); - - if (rows.Count == 0) - { - break; - } - - scanned += rows.Count; - afterId = rows[^1].Id; + new { Pattern = pattern, AfterId = afterId, UpperId = upperId }, + cancellationToken: cancellationToken)); foreach (ListValuedRow row in rows) { @@ -225,11 +231,10 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF } } - if (rows.Count < batch) - { - // short page: the source is exhausted, not merely bounded - break; - } + // advance by the WINDOW, never by what came back: an empty page means "nothing matched in this + // stretch of keyspace", not "the source is exhausted" + afterId = upperId; + visited += window; } return distinct.ToList(); @@ -253,18 +258,23 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF } /// - /// One keyset page of candidate rows. LOWER(...) LIKE ... ESCAPE, ORDER BY and - /// LIMIT are all portable between SQLite and MySQL, and the identifiers are unquoted so neither - /// provider's quoting dialect is baked in. Paging rides Id — a unique integer key both providers - /// can walk — rather than the JSON column: ordering on TEXT would not be deterministic on MySQL, - /// which sorts using only the first max_sort_length bytes, and a keyset predicate on the primary - /// key lets each page resume instead of re-scanning what the previous page already rejected. + /// One keyspace window of candidate rows. LOWER(...) LIKE ... ESCAPE and ORDER BY are + /// portable between SQLite and MySQL, and the identifiers are unquoted so neither provider's quoting + /// dialect is baked in. + /// + /// The upper bound is what makes this bounded work rather than a bounded result: with only + /// Id > @AfterId and a LIMIT, a no-match query still forces the engine to evaluate + /// every remaining row before it can return an empty page. Closing the range turns each round trip + /// into a primary-key range scan of known width. There is deliberately no LIMIT — the window + /// already caps the row count, and a LIMIT here would re-introduce the illusion that it is + /// doing the bounding. + /// /// internal static string ListValuedSql(string column) => $"SELECT Id, {column} AS Payload FROM SongMetadata WHERE {column} IS NOT NULL " + - "AND Id > @AfterId " + + "AND Id > @AfterId AND Id <= @UpperId " + $"AND LOWER({column}) LIKE @Pattern ESCAPE '{LikeEscapeChar}' " + - "ORDER BY Id LIMIT @Batch"; + "ORDER BY Id"; private sealed class ListValuedRow { @@ -286,15 +296,20 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// So the pattern narrows on the leading run of ASCII characters the JSON writer stores /// verbatim, and stops at the first character it cannot reason about — a non-ASCII character, or /// one of . A query starting with such a character narrows to the bare - /// element-opening anchor %"% and leans entirely on the row cap and the in-memory filter. + /// element-opening anchor %"%, which matches every non-empty row — so the walk's keyspace + /// bound, not the pattern, is what keeps that case bounded, and the in-memory filter is what keeps + /// it correct. /// /// /// Why that is sound, and why an earlier "JSON-encode the whole prefix" version was NOT: the stored - /// text escapes non-ASCII, so Édith is on disk as Édith. SQL LOWER() - /// lowercases the escape text, giving É — it cannot case-fold the codepoint that - /// escape denotes, so a q=é pattern of é never matched and every accented artist - /// was silently unsuggestable. Restricting the run to verbatim ASCII removes the escape from the - /// comparison entirely. And for a run character c, any element character that + /// text escapes non-ASCII, so Édith is on disk as \u00C9dith. SQL LOWER() + /// lowercases the escape text — \u00C9 becomes \u00c9 — it cannot case-fold the + /// codepoint that escape denotes. So a q=é prefix encoded to the pattern \u00e9, which + /// never matched the stored \u00C9. It failed whenever the query's casing differed from the + /// stored casing (three of the nine cases pinned in GetSearchFieldValuesHandlerTests); when + /// the casings happened to agree the escape texts lined up and it worked, which is exactly why the + /// bug survived review. Restricting the run to verbatim ASCII removes the escape from the comparison + /// entirely. And for a run character c, any element character that /// OrdinalIgnoreCase-equals c is itself ASCII — no non-ASCII codepoint in all of Unicode /// OrdinalIgnoreCase-equals a printable ASCII character, which /// SearchFieldValuesPrefilterSupersetTests proves by exhaustive sweep rather than assertion. diff --git a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs index b5c04239f..f3573cb5a 100644 --- a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs +++ b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs @@ -315,30 +315,28 @@ public class GetSearchFieldValuesHandlerTests .IfSome(r => r.Values.ShouldBe(new List { "50% Off" })); } - [Test] - public async Task List_Valued_Walk_Is_Bounded_By_A_Scan_Ceiling_Of_Exactly_20000_Candidate_Rows() - { - // The bound is on EFFORT, not on the result: the walk pages past non-matching candidates (proved by - // Prefilter_Widening_Must_Not_Starve_The_Candidate_Set_Past_The_Row_Bound) but must still stop somewhere, - // or a bare-anchor query on a huge library becomes an unbounded scan. - // - // Pinned in BOTH directions so the constant itself is nailed down: a needle at candidate row 20000 is - // found, the same needle at row 20001 is not. Asserting only one side would pass with a ceiling of 3, or - // with no ceiling at all. - const string needle = "\u00E9clair-the-needle"; + /// + /// Seeds non-matching songs through raw SQL — 20k rows via the change + /// tracker is minutes, this is milliseconds. + /// + private static Task SeedFiller(TvContext context, int fillerRows) => + context.Database.ExecuteSqlRawAsync( + $""" + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < {fillerRows}) + INSERT INTO SongMetadata (SongId, MetadataKind, Title, Artists, DateAdded, DateUpdated) + SELECT 0, 0, 'Filler ' || n, '["zzz-filler"]', '2026-01-01', '2026-01-01' FROM seq + """); - // seeded through raw SQL: 20k rows through the change tracker is minutes, this is milliseconds - async Task SeedFiller(TvContext context, int rows) => - await context.Database.ExecuteSqlRawAsync( - $""" - WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < {rows}) - INSERT INTO SongMetadata (SongId, MetadataKind, Title, Artists, DateAdded, DateUpdated) - SELECT 0, 0, 'Filler ' || n, '["zzz-filler"]', '2026-01-01', '2026-01-01' FROM seq - """); + [Test] + public async Task List_Valued_Walk_Visits_At_Most_20000_Rows_When_Every_Row_Is_A_Candidate() + { + // The WIDENED case: a non-ASCII prefix collapses the pattern to the bare `%"%` anchor, so every row is + // a candidate. Pinned in both directions so the ceiling itself is nailed down — a needle at row 20000 is + // reached, the same needle at 20001 is not. + const string needle = "\u00E9clair-the-needle"; await using (TvContext context = _db.CreateContext()) { - // 19999 filler rows + the needle == candidate row 20000, the last one inside the ceiling await SeedFiller(context, 19999); context.SongMetadata.Add(Song("Needle", [needle])); await context.SaveChangesAsync(); @@ -348,17 +346,9 @@ public class GetSearchFieldValuesHandlerTests var handler = new GetSearchFieldValuesHandler(_db.Factory); - Option atCeiling = await handler.Handle( - new GetSearchFieldValues("artist", "\u00E9", 50), - CancellationToken.None); + (await handler.Handle(new GetSearchFieldValues("artist", "\u00E9", 50), CancellationToken.None)) + .IfSome(r => r.Values.ShouldBe(new List { needle }, "row 20000 is inside the ceiling")); - atCeiling.IsSome.ShouldBeTrue(); - atCeiling.IfSome( - r => r.Values.ShouldBe( - new List { needle }, - "candidate row 20000 is inside the ceiling and must be reached")); - - // one more filler row pushes the needle to candidate row 20001, past the ceiling await using (TvContext context = _db.CreateContext()) { SongMetadata existing = await context.SongMetadata.SingleAsync(m => m.Title == "Needle"); @@ -371,22 +361,63 @@ public class GetSearchFieldValuesHandlerTests (await context.SongMetadata.CountAsync()).ShouldBe(20001); } - Option pastCeiling = await handler.Handle( - new GetSearchFieldValues("artist", "\u00E9", 50), - CancellationToken.None); + (await handler.Handle(new GetSearchFieldValues("artist", "\u00E9", 50), CancellationToken.None)) + .IfSome( + r => r.Values.ShouldBeEmpty( + "row 20001 is past the ceiling; this false negative is the documented bounded-best-effort " + + "contract, deliberately pinned rather than papered over")); - pastCeiling.IsSome.ShouldBeTrue(); - pastCeiling.IfSome( - r => r.Values.ShouldBeEmpty("candidate row 20001 is past the ceiling and must not be reached")); + // positive control: narrowing the prefix drops the filler from the candidate set, so the ceiling is what + // the assertion above is about, not an unreachable value + (await handler.Handle(new GetSearchFieldValues("artist", "zzz", 50), CancellationToken.None)) + .IfSome(r => r.Values.ShouldBe(new List { "zzz-filler" })); + } - // positive control: narrowing the prefix drops the filler rows from the candidate set entirely, so the - // same row is reachable again -- the assertion above is about the ceiling, not an unreachable value - Option narrowed = await handler.Handle( - new GetSearchFieldValues("artist", "zzz", 50), - CancellationToken.None); + [Test] + public async Task List_Valued_Walk_Cannot_Inspect_Past_The_Bound_When_Nothing_Matches() + { + // THE case every earlier bound test missed: they all gave the SQL pre-filter something to return. + // + // Here the pattern is NARROW (an ASCII prefix), so the LIKE matches nothing until the needle. A LIMIT + // cannot bound this: to return an empty page the engine must evaluate every eligible row first. The + // previous revision therefore let the database walk the entire table and FOUND a needle at row 30000 — + // correct answer, unbounded work. The closed keyspace window stops at 20000 rows inspected, so the + // needle beyond it is not found, and that is the observable difference. + const string needle = "qqq-far-past-the-bound"; - narrowed.IsSome.ShouldBeTrue(); - narrowed.IfSome(r => r.Values.ShouldBe(new List { "zzz-filler" })); + await using (TvContext context = _db.CreateContext()) + { + await SeedFiller(context, 29999); + context.SongMetadata.Add(Song("Needle", [needle])); + await context.SaveChangesAsync(); + + (await context.SongMetadata.CountAsync()).ShouldBe(30000); + } + + var handler = new GetSearchFieldValuesHandler(_db.Factory); + + (await handler.Handle(new GetSearchFieldValues("artist", "qqq", 50), CancellationToken.None)) + .IfSome( + r => r.Values.ShouldBeEmpty( + "a no-match walk must stop at the visit ceiling; reaching row 30000 means the bound is on " + + "rows RETURNED rather than rows INSPECTED")); + + // ... while the identical query against a needle inside the bound does find it, so the assertion above + // is about the ceiling and not about the narrow pattern failing to match at all + await using (InMemoryTvContext smaller = await InMemoryTvContext.CreateAsync()) + { + await using (TvContext context = smaller.CreateContext()) + { + await SeedFiller(context, 19999); + context.SongMetadata.Add(Song("Needle", [needle])); + await context.SaveChangesAsync(); + } + + (await new GetSearchFieldValuesHandler(smaller.Factory).Handle( + new GetSearchFieldValues("artist", "qqq", 50), + CancellationToken.None)) + .IfSome(r => r.Values.ShouldBe(new List { needle }, "row 20000 is inside the ceiling")); + } } [Test] diff --git a/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs b/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs index e0bd95f07..834a0d385 100644 --- a/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs +++ b/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs @@ -120,29 +120,49 @@ public class SearchFieldValuesPrefilterSupersetTests } [Test] - public void Candidate_Rows_Are_Keyset_Paged_On_The_Primary_Key_Not_The_Json_Column() + public void Candidate_Rows_Are_Read_In_A_Closed_Id_Keyspace_Window() { - // Two things at once. ORDER BY rides Id because MySQL sorts TEXT/longtext using only max_sort_length - // bytes, so ordering on the JSON column would let long rows sharing a prefix tie arbitrarily. And the - // `Id > @AfterId` keyset predicate is what makes the walk RESUMABLE — without it the bound would have to - // be a single fixed budget, which is exactly the starvation defect this replaced. string sql = GetSearchFieldValuesHandler.ListValuedSql("Artists"); + // THE upper bound is the whole point. With only `Id > @AfterId` plus a LIMIT, a query that matches + // nothing still forces the engine to evaluate every remaining row before it can return an empty page — + // so rows RETURNED were bounded while rows INSPECTED were not. Closing the range makes each round trip + // a primary-key range scan of known width. sql.ShouldContain("Id > @AfterId"); - sql.ShouldContain("ORDER BY Id LIMIT @Batch"); + sql.ShouldContain("Id <= @UpperId"); + + // ORDER BY rides Id, not the JSON column: MySQL sorts TEXT using only max_sort_length bytes. + sql.ShouldContain("ORDER BY Id"); sql.ShouldNotContain("ORDER BY Artists"); + + // and no LIMIT — the window caps the row count, and a LIMIT here would only restore the illusion that + // it is what does the bounding + sql.ShouldNotContain("LIMIT"); } [Test] - public void The_Scan_Ceiling_Is_A_Whole_Number_Of_Batches() + public void The_Visit_Ceiling_Is_A_Whole_Number_Of_Windows_So_The_Walk_Cannot_Overrun_It() { - // The walk trims its last batch to land exactly on the ceiling, so this is not load-bearing for - // correctness — but a ceiling that is not a multiple of the batch size means the final round trip - // fetches a short page, which the loop would otherwise read as "source exhausted". - (GetSearchFieldValuesHandler.ListValuedMaxScannedRows % GetSearchFieldValuesHandler.ListValuedBatchRows) - .ShouldBe(0); + // The invariant that actually matters is that the ceiling is reachable and never overshot. The walk + // clamps its final window (`Math.Min(WindowRows, MaxVisitedRows - visited)`), so a ceiling that is not a + // multiple of the window size would still be safe — an earlier version of this test rejected that, which + // made it a style preference wearing a correctness costume. + GetSearchFieldValuesHandler.ListValuedWindowRows.ShouldBeGreaterThan(0); + GetSearchFieldValuesHandler.ListValuedMaxVisitedRows + .ShouldBeGreaterThanOrEqualTo(GetSearchFieldValuesHandler.ListValuedWindowRows); - GetSearchFieldValuesHandler.ListValuedMaxScannedRows - .ShouldBeGreaterThan(GetSearchFieldValuesHandler.ListValuedBatchRows); + // simulate the walk's own arithmetic and assert it lands exactly on the ceiling, never past it + var visited = 0; + var windows = 0; + while (visited < GetSearchFieldValuesHandler.ListValuedMaxVisitedRows) + { + visited += Math.Min( + GetSearchFieldValuesHandler.ListValuedWindowRows, + GetSearchFieldValuesHandler.ListValuedMaxVisitedRows - visited); + windows++; + windows.ShouldBeLessThan(1000, "the walk must terminate"); + } + + visited.ShouldBe(GetSearchFieldValuesHandler.ListValuedMaxVisitedRows); } } diff --git a/docs/decisions/README.md b/docs/decisions/README.md index af8e4d2d5..c71122d9e 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -28,7 +28,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `api.schedule-item-flat-dto` | Schedule-item GET/POST/PUT use a flat, non-polymorphic `ScheduleItemResponseModel` (every subtype field promoted to a nullable top-level member) instead of the polymorphic Application VM hierarchy, with mutation field names matching `ScheduleItemRequest` 1:1 for a lossless round-trip. | 2026-07-10 | [link](records/api/schedule-item-flat-dto.md) | | `api.scheduling-hardening` | Create/Replace handlers guard against null/whitespace `name` (`IsNullOrWhiteSpace`, not just `Length`) to prevent NRE-500s, template-item overlap validation compares by index (not record value-equality) to catch exact-duplicate items, and unreachable 404 `ProducesResponseType` attributes on create-only actions are trimmed. | 2026-07-13 | [link](records/api/scheduling-hardening.md) | | `api.search-allitems-paging` | `GET /api/v1/search/all-items` is paginated (capped page size, `Totals` field) to bound DoS exposure; the SPA add-all flow pages to completeness instead of relying on an unbounded response. | 2026-07-18 | [link](records/api/search-allitems-paging.md) | -| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — this is scoped to the in-memory stages on purpose: a field sourced by a plain EF query is pre-filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&'+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; candidate rows are walked keyset-paged on `Id` (a unique integer key, not the JSON column) and split + exact-filtered in memory, stopping on the first of enough distinct matches, source exhausted, or a 20,000-candidate-row scan ceiling — the bound is on EFFORT, never a fixed row budget, because a fixed budget lets a widened pre-filter starve the candidate set and drop real matches. Ordering across merged sources is best-effort, exact only below the truncation points. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | +| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — this is scoped to the in-memory stages on purpose: a field sourced by a plain EF query is pre-filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&'+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; candidate rows are walked in CLOSED `Id` keyspace windows (`Id > @AfterId AND Id <= @AfterId + @Window`, no `LIMIT`) and split + exact-filtered in memory, stopping on the first of enough distinct matches, `MAX(Id)` passed, or a 20,000-row visit ceiling — the bound is on rows the database LOOKS AT, not on rows returned: a fixed `LIMIT` budget lets a widened pre-filter starve the candidate set, and a `LIMIT` alone cannot bound a no-match query, which must evaluate every eligible row before it can return an empty page. Ordering across merged sources is best-effort, exact only below the truncation points. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | | `api.search-paging-cap` | Search stays capped at 100 items per media kind; an overflowing kind's "See all" reuses library-browse paging instead of adding new API surface. | 2026-07-11 | [link](records/api/search-paging-cap.md) | | `api.versioning-v1` | The entire `/api` surface is versioned to `/api/v1` uniformly (no unversioned corner); legacy unversioned callers are rewritten in-pipeline (not redirected) with Deprecation/Link/Sunset headers, and post-freeze `/api/v1` is additive-only — a breaking change requires `/api/v2`. | 2026-07-13 | [link](records/api/versioning-v1.md) | | `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) | diff --git a/docs/decisions/records/api/search-field-values-sources.md b/docs/decisions/records/api/search-field-values-sources.md index 48973e499..23f2109a5 100644 --- a/docs/decisions/records/api/search-field-values-sources.md +++ b/docs/decisions/records/api/search-field-values-sources.md @@ -5,7 +5,7 @@ status: active since: '2026-07-26' supersedes: api.search-field-values@2026-07-23 superseded-by: none -rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — this is scoped to the in-memory stages on purpose: a field sourced by a plain EF query is pre-filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&''+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; candidate rows are walked keyset-paged on `Id` (a unique integer key, not the JSON column) and split + exact-filtered in memory, stopping on the first of enough distinct matches, source exhausted, or a 20,000-candidate-row scan ceiling — the bound is on EFFORT, never a fixed row budget, because a fixed budget lets a widened pre-filter starve the candidate set and drop real matches. Ordering across merged sources is best-effort, exact only below the truncation points.' +rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — this is scoped to the in-memory stages on purpose: a field sourced by a plain EF query is pre-filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&''+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; candidate rows are walked in CLOSED `Id` keyspace windows (`Id > @AfterId AND Id <= @AfterId + @Window`, no `LIMIT`) and split + exact-filtered in memory, stopping on the first of enough distinct matches, `MAX(Id)` passed, or a 20,000-row visit ceiling — the bound is on rows the database LOOKS AT, not on rows returned: a fixed `LIMIT` budget lets a widened pre-filter starve the candidate set, and a `LIMIT` alone cannot bound a no-match query, which must evaluate every eligible row before it can return an empty page. Ordering across merged sources is best-effort, exact only below the truncation points.' signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, raw JSON LIKE pre-filter, superset never under-match, accented artist unsuggestable, LOWER cannot fold a \u00XX escape, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, ListValuedRowCap, ORDER BY Id not TEXT, max_sort_length, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669' mechanics: '`GetSearchFieldValuesHandler` (`GetSource`, `GetSongListValuedColumn`, `GetSongListValuedValues`, `ListValuedSql`, `JsonElementPrefixPattern`, `FilterSortTake`); `SearchFieldValuesPrefilterSupersetTests` (the superset proof); api-conventions.md; spa-conventions.md §12' --- @@ -42,13 +42,20 @@ of the returned values could see. It JSON-encoded the whole query prefix, reason stored text escapes non-ASCII (`Édith` is on disk as `\u00C9dith`), encoding the prefix the same way would line up. **It does not.** SQL `LOWER()` lowercases the *escape text* — `\u00C9` becomes `\u00c9` — it cannot case-fold the codepoint that escape denotes, so the stored `\u00C9` never folds to `\u00e9`. -A `q=é` prefix produced the pattern `\u00e9`, which never matched, the row was discarded before the in-memory filter ever saw it, and **every accented artist -was silently unsuggestable** — in a music library, the common case, not an edge case. +A `q=é` prefix produced the pattern `\u00e9`, which never matched the stored `\u00C9`, so the row was +discarded before the in-memory filter ever saw it. Be precise about the blast radius, because the +imprecision is what let it through review: it failed whenever the query's casing differed from the +stored casing — **three of the nine cases** now pinned in `GetSearchFieldValuesHandlerTests` +(`q=é`/`q=édith` against `"Édith Piaf"`, `q=BJÖRK` against `"Björk"`). When the casings happened to +agree the two escape texts lined up and it worked, which is why six of the nine pass the broken +revision. Accented artists are the common case in a music library, so a bug that fires on some +casings and not others is still a bug worth this much prose. The fix is to narrow only on ground that can be proved: the leading run of characters the JSON writer stores **verbatim**, stopping at the first non-ASCII or JSON-escaped character. A query -starting with such a character narrows to the bare element-opening anchor `%"%` and leans on the row -cap and the in-memory filter. `q=Beyoncé` still narrows on `beyonc`; `q=é` narrows on nothing. +starting with such a character narrows to the bare element-opening anchor `%"%`, so the walk's +keyspace bound — not the pattern — is what keeps that case bounded, and the in-memory filter is what +keeps it correct. `q=Beyoncé` still narrows on `beyonc`; `q=é` narrows on nothing. Soundness rests on two facts, both **asserted by exhaustive computation** in `SearchFieldValuesPrefilterSupersetTests` rather than argued: @@ -70,8 +77,10 @@ into `ı`. The old chain used `ToLower()` plus the default *linguistic* `StartsW the same library answer differently per caller. Comparison is now `OrdinalIgnoreCase` and ordering `StringComparer.Ordinal` throughout the in-memory stages, including the shared `FilterSortTake` that `state`, `video_dynamic_range` and `content_rating` also use. That is a deliberate change to shared -behaviour: response *sets* are unchanged, response *order* is now ordinal rather than -culture-dependent. +behaviour, and **not a cosmetic one**: ordering happens before `Take(limit)`, so changing the +comparer can change *which* values survive, not merely their order. With `"Zulu"` and `"apple"`, an +empty `q` and `limit=1`, linguistic ordering yields `"apple"` and ordinal yields `"Zulu"`. An earlier +version of this record claimed the response sets were unchanged; that was false. **Scope this claim carefully — it is not endpoint-wide.** A field sourced by a plain EF query runs the database's `LOWER`, `DISTINCT`, `ORDER BY` and `LIMIT` *before* any ordinal code executes, so the @@ -94,36 +103,53 @@ ranks `"Zulu"` first, while the case-insensitive database ordering ranks `"apple not have; do not restore it. (An earlier version of this record used `"Zulu"`/`"Éclair"` as the example, where both orderings pick `"Zulu"` — it demonstrated nothing.) -## What the bound bounds — and why a fixed row budget was wrong +## What the bound bounds — three revisions, three different wrong quantities -The first two revisions used a single fixed row budget (`LIMIT 1000`) and called it "bounded". It was -bounded, and it was also **wrong**, in a way that only shows up in composition with the pre-filter. -The pre-filter is deliberately allowed to over-match; a non-ASCII or JSON-escaped prefix makes it -collapse to the bare `%"%` anchor, so *every* row becomes a candidate. Under a fixed budget the -budget is then spent on rows that cannot match, and real matches behind them are discarded before the -in-memory filter sees them. Seed 1,000 songs by `"zzz…"`, put the only `"éclair"` in row 1,001, ask -for `album_artist?q=é`, and the endpoint returns `[]`. **"Over-matching is free" is only true when the -candidate set is not truncated.** Widening the predicate under a fixed budget starves it — the same -false-negative class the pre-filter fix was addressing, moved one stage later. +Worth reading in full, because "it has a LIMIT so it is bounded" was wrong three times in a row and +each time the code looked obviously fine. -So the walk is **keyset-paged on `Id`** (`Id > @AfterId … ORDER BY Id LIMIT @Batch`) and continues -past non-matching candidates, stopping on the first of: enough distinct exact matches to satisfy -`limit`, a short page (source exhausted), or `ListValuedMaxScannedRows = 20000` candidate rows -examined, in `ListValuedBatchRows = 2000` batches. Only the ceiling is lossy, and it takes 20,000 rows -that already passed the SQL pre-filter before it bites. Paging on the primary key rather than the JSON -column also keeps the truncation point well-defined on MySQL, which sorts `TEXT` using only the first -`max_sort_length` (default 1024) bytes. +**Revision 1 and 2 bounded the RESULT.** A single fixed `LIMIT 1000` on candidate rows. Bounded, and +wrong in composition with the pre-filter: the pre-filter is deliberately allowed to over-match, and a +non-ASCII or JSON-escaped prefix collapses it to the bare `%"%` anchor, so *every* row becomes a +candidate. The budget then gets spent on rows that cannot match while real matches sit behind them. +Seed 1,000 songs by `"zzz…"`, put the only `"éclair"` in row 1,001, ask for `album_artist?q=é` — +the endpoint returns `[]`. **"Over-matching is free" is only true while the candidate set is not +truncated.** Widening the predicate under a fixed budget starves it: the same false-negative class the +pre-filter fix was addressing, moved one stage later. -**The remaining cost is scan ACCESS, not necessarily a full table scan.** `LOWER(col) LIKE '%…'` has a -leading wildcard, so no index can be sought and the engine walks rows — but each page stops as soon as -it has filled `@Batch`, so a query whose candidates are dense finishes early and only a sparse -bare-anchor query walks far. Worst case is bounded at 20,000 candidate rows across at most 10 round -trips. The endpoint is authenticated by default, though `Api:RequireKeyForReads=false` makes it -anonymous. **Follow-up: a normalized `SongArtist` join table** (the shape `MusicVideoArtist` already -has) would make this exact, indexable and seekable, at the cost of a dual-provider schema migration -plus data backfill, changes to every scanner write path populating `SongMetadata.Artists`, changes to -the Lucene indexer, and two representations of the same fact free to drift. Tracked as -**ersatztv#669**; rejected for #578 on blast radius, not on merit. +**Revision 3 bounded the CANDIDATES.** Keyset paging (`Id > @AfterId … LIMIT @Batch`), walking past +non-matching rows, with a ceiling on candidate rows materialized. That fixed the starvation, and still +bounded the wrong quantity: `LIMIT` caps rows *returned*, and **a query with no matches must evaluate +every eligible row before it can return an empty page**. An empty first page ended the walk having +counted zero against the ceiling, so rows *inspected* remained bounded only by table size. Round trips +and materialized rows were bounded; database work was not. + +**Revision 4 bounds the KEYSPACE.** Each round trip closes the `Id` range on both sides — +`Id > @AfterId AND Id <= @AfterId + @Window` — which the primary key range-scans. The `LIKE` still +decides what comes *back*; it no longer decides how much gets *looked at*. The walk advances by the +window, never by what returned (an empty page means "nothing matched in this stretch", not "source +exhausted"), and stops on the first of: enough distinct matches for `limit`, `MAX(Id)` passed, or +`ListValuedMaxVisitedRows = 20000` keyspace positions visited in `ListValuedWindowRows = 2000` +windows. There is deliberately **no `LIMIT` in the SQL** — the window already caps the row count, and +a `LIMIT` would only restore the illusion that it is doing the bounding. A single indexed +`SELECT MAX(Id)` up front stops the walk from burning windows on empty keyspace. + +So: at most 10 round trips, at most 20,000 rows inspected, whether or not anything matches, for any +`q`. Ranging on the primary key rather than the JSON column also keeps the truncation point +well-defined on MySQL, which sorts `TEXT` using only the first `max_sort_length` (default 1024) bytes. + +**Two costs are accepted, not solved.** A match beyond the ceiling is not found — 20,000 `"zzz"` rows +followed by `"éclair"` at 20,001 returns `[]`, and a test pins exactly that rather than pretending +otherwise; it is the documented bounded-best-effort contract at a rarer threshold. And on a heavily +fragmented `Id` space (many deletions) a window covers fewer live rows than its width, so the walk +reaches less of the table — conservative in the safe direction, lossy in the same best-effort way. +The endpoint is authenticated by default, though `Api:RequireKeyForReads=false` makes it anonymous. + +**Follow-up: a normalized `SongArtist` join table** (the shape `MusicVideoArtist` already has) removes +all of this — the predicate becomes seekable, so there is nothing to bound. Cost: a dual-provider +schema migration plus data backfill, changes to every scanner write path populating +`SongMetadata.Artists`, changes to the Lucene indexer, and two representations of the same fact free +to drift. Tracked as **ersatztv#669**; rejected for #578 on blast radius, not on merit. ## `album_artist` 404 → 200 is additive -- 2.47.3 From 373956fcee51a31ef3ecfb5291af11411571b14f Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 27 Jul 2026 02:00:49 +0200 Subject: [PATCH 5/8] =?UTF-8?q?fix(578):=20delete=20the=20SQL=20predicate?= =?UTF-8?q?=20=E2=80=94=20LIMIT=20only=20bounds=20work=20when=20there=20is?= =?UTF-8?q?=20nothing=20to=20discard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BLOCKER. Attempt 4 bounded the Id KEYSPACE, and keyspace is not rows. Delete 20,000 historical rows, put one song at Id 20001, query artist?q=que: the walk burned all ten windows on empty ranges and returned [] for a table containing exactly one row. Capacity fell linearly with deletion ratio and no ratio was safe — one placed gap hides the next match. My record called that "heavily fragmented" and the endpoint description said loss happens "on a very large library"; the one-row example disproves both. Option A. The query now carries NO predicate at all — no LIKE, no LOWER, not even IS NOT NULL: SELECT Id, Artists AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch That is the whole fix, and it is the point. LIMIT truncates what survives a predicate, so with any predicate present it bounds the OUTPUT and says nothing about the WORK; the engine may evaluate and discard arbitrarily many rows first. Stripped to a bare primary-key range, LIMIT n reads exactly n index entries and n rows — independent of sparsity, deletion history or where the gaps fall. All selectivity moves into memory. A short page can now only mean exhaustion, which is precisely what it could not mean while a predicate was present. Four attempts, four wrong quantities: the result (a fixed budget the over-matching pre-filter starved), candidates returned (a no-match query must evaluate every eligible row before returning an empty page), keyspace width (above), and finally actual rows. The record carries the table; it is worth more than the code. Deleting the predicate deletes a whole bug family with it: the JSON-escape reasoning, the narrow-only-on-verbatim-ASCII rule, the exhaustive Unicode sweep that proved it sound, the ESCAPE '/' portability workaround, and the may-over-match-never-under-match invariant that turned out to be conditional on something untrue. SearchFieldValuesPrefilterSupersetTests is deleted entirely; the one assertion worth keeping — that the SQL has no predicate — moved to the query-shape suite, which pins the SQL string exactly so "just a cheap filter" fails a test instead of silently unbounding the walk. Measured cost of no server-side narrowing, on a seeded 20,000-song library (in-memory SQLite): worst case (no match, full walk) 20,000 rows / 10 round trips / 391.9 KiB / 119ms SQL, ~40ms warm end-to-end. Empty q, dense and non-ASCII prefixes all stop on page 1 at ~39 KiB and ~40ms. Judged acceptable for a debounced typeahead against a local file. If it ever is not, the answer is #669, not reintroducing selectivity — the record says so explicitly. Also fixed: - Round-trip count was advertised as 10; it is at most 10 for album_artist and 11 for artist, which also runs its EF query. The MAX(Id) probe is gone with the keyspace scheme, so there is no extra scalar call. - The duplicated-formula ceiling test is deleted rather than rewritten. It re-implemented the loop's arithmetic and would have passed through an off-by-one or a stall in the real loop; the dense integration tests carry that coverage. Its MaxVisited >= Window assertion was a style constraint in correctness clothing. - Stale text swept by grepping the mechanism nouns rather than re-reading: candidate/keyspace/ pre-filter/superset/row cap/LIKE/ESCAPE and the removed constant names, across handler, tests, record, api-conventions and the endpoint description. The two surviving "pre-filter" mentions are deliberate history. Test comments that rendered escaped non-ASCII as literal characters (which contradicted the raw-storage assertion in the same file) now show the escape text. New test List_Valued_Walk_Reads_Live_Rows_Regardless_Of_Id_Density reproduces the one-row killer and fails against attempt 4. --- .../Queries/GetSearchFieldValuesHandler.cs | 193 ++++++------------ .../GetSearchFieldValuesHandlerTests.cs | 131 ++++-------- ...SearchFieldValuesPrefilterSupersetTests.cs | 168 --------------- .../SearchFieldValuesQueryShapeTests.cs | 19 ++ ErsatzTV/Controllers/Api/SearchController.cs | 4 +- ErsatzTV/wwwroot/openapi/v1.json | 2 +- docs/api-conventions.md | 41 ++-- docs/decisions/README.md | 2 +- .../api/search-field-values-sources.md | 144 +++++++------ 9 files changed, 222 insertions(+), 482 deletions(-) delete mode 100644 ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs index 6700a44f5..bb43d4548 100644 --- a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs @@ -15,50 +15,38 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF private const int MaxLimit = 50; /// - /// Width of the Id keyspace window examined per round trip when walking the list-valued - /// (JSON-array) columns on SongMetadata, and the ceiling on total keyspace visited per request. + /// Rows read per round trip when walking the list-valued (JSON-array) columns on + /// SongMetadata, and the ceiling on rows read per request. /// - /// What these bound is rows the database LOOKS AT, and getting that right took three tries. - /// A single fixed LIMIT budget was wrong because the pre-filter is deliberately allowed to - /// over-match: a non-ASCII or JSON-escaped prefix collapses the pattern to the bare %"% - /// anchor, every row becomes a candidate, and the budget gets spent on rows that cannot match while - /// real matches sit behind them. Keyset-paging with a LIMIT fixed that but bounded the wrong - /// quantity: LIMIT caps rows RETURNED, and a query with no matches must evaluate every - /// eligible row before it can return an empty page. Rows examined stayed bounded only by table size. + /// These count ACTUAL ROWS, and arriving at that took four tries — each earlier attempt bounded a + /// quantity that sounded like rows and was not. A fixed LIMIT budget bounded the RESULT, and + /// the pre-filter (allowed to over-match) starved it with rows that could not match. Keyset paging + /// with a LIMIT bounded CANDIDATES RETURNED — but a query matching nothing must evaluate + /// every eligible row before it can return an empty page, so rows inspected stayed unbounded. A + /// closed Id range bounded KEYSPACE WIDTH — but keyspace is not rows: delete 20,000 + /// historical rows, put one song at Id 20001, and the walk burns its whole allowance on empty + /// ranges and inspects nothing. /// /// - /// So the window is on the keyspace — Id > @AfterId AND Id <= @AfterId + @Window - /// — which the primary key can range-scan. The LIKE still decides what comes BACK; it no - /// longer decides how much gets LOOKED AT. Each round trip inspects at most - /// rows and the walk visits at most in total, whether or not - /// anything matches. + /// What makes this one hold is that the query has no predicate to evaluate and discard. It is + /// a plain keyset page over the primary key, so LIMIT bounds index entries scanned and rows + /// returned alike, and it returns exactly rows whenever that many + /// remain — independent of how sparse the matches are, how many rows were deleted, or where the + /// gaps fall. Selectivity moves entirely into memory. /// /// - /// The walk stops on the first of: enough distinct exact matches to satisfy limit, the - /// keyspace exhausted, or the ceiling. Only the ceiling is lossy — a match beyond it is not found, - /// which is the bounded-best-effort contract the endpoint documents. + /// The trade is real and deliberate: no server-side narrowing, so a query with few matches transfers + /// rows it will discard, up to . A query with enough matches + /// stops as soon as it has limit distinct ones, so the dense cases — including an empty + /// q — finish on the first page. See api.search-field-values-sources for the measured + /// cost and for why reintroducing a LIKE is not an option. /// /// - internal const int ListValuedWindowRows = 2000; + internal const int ListValuedBatchRows = 2000; - /// - internal const int ListValuedMaxVisitedRows = 20000; + /// + internal const int ListValuedMaxRowsRead = 20000; - /// - /// Escape character for the raw-JSON LIKE pre-filter. Deliberately NOT a backslash: the stored - /// JSON is full of backslashes (\u00E9, \u0022) and, worse, ESCAPE '\' is not a - /// portable literal — MySQL applies C-style escaping to string literals, so it reads as an unterminated - /// string, while SQLite would reject the doubled '\\' form as a two-character escape. - /// - private const char LikeEscapeChar = '/'; - - /// - /// The ASCII characters the JSON writer escapes rather than emitting verbatim (& and friends - /// become &). A prefix character in this set cannot be matched literally against the stored - /// text, so it terminates the narrowing run in . - /// SearchFieldValuesPrefilterSupersetTests pins this set against the encoder itself. - /// - private const string JsonEscapedAscii = "\"&'+<>\\`"; public async Task> Handle( GetSearchFieldValues request, @@ -183,10 +171,10 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// collections at all — so there is no server-side SELECT DISTINCT over the elements. /// /// - /// So the rows are walked in primary-key order, keyset-paged, and split + exact-filtered in memory. - /// The SQL LIKE is only a narrowing hint: correctness lives entirely in the in-memory filter, - /// and the walk continues past non-matching candidates rather than letting a row budget decide the - /// answer. See for why a single fixed budget was wrong. + /// So the rows are walked in primary-key order, keyset-paged by row position, and split + + /// exact-filtered in memory. All selectivity is in memory — the query carries no predicate at all, + /// which is what makes its LIMIT a real bound on work. See + /// for the four revisions it took to get that right. /// /// private static async Task> GetSongListValuedValues( @@ -197,28 +185,28 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF CancellationToken cancellationToken) { string sql = ListValuedSql(column); - string pattern = JsonElementPrefixPattern(query); - - // one indexed lookup, so an exhausted keyspace ends the walk instead of burning windows on empty range - int maxId = await dbContext.Connection.ExecuteScalarAsync( - new CommandDefinition( - "SELECT COALESCE(MAX(Id), 0) FROM SongMetadata", - cancellationToken: cancellationToken)); var distinct = new System.Collections.Generic.HashSet(StringComparer.Ordinal); var afterId = 0; - var visited = 0; + var read = 0; - while (afterId < maxId && visited < ListValuedMaxVisitedRows && distinct.Count < limit) + while (read < ListValuedMaxRowsRead && distinct.Count < limit) { - int window = Math.Min(ListValuedWindowRows, ListValuedMaxVisitedRows - visited); - int upperId = afterId + window; + int batch = Math.Min(ListValuedBatchRows, ListValuedMaxRowsRead - read); - IEnumerable rows = await dbContext.Connection.QueryAsync( + List rows = (await dbContext.Connection.QueryAsync( new CommandDefinition( sql, - new { Pattern = pattern, AfterId = afterId, UpperId = upperId }, - cancellationToken: cancellationToken)); + new { AfterId = afterId, Batch = batch }, + cancellationToken: cancellationToken))).AsList(); + + if (rows.Count == 0) + { + break; + } + + read += rows.Count; + afterId = rows[^1].Id; foreach (ListValuedRow row in rows) { @@ -231,10 +219,14 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF } } - // advance by the WINDOW, never by what came back: an empty page means "nothing matched in this - // stretch of keyspace", not "the source is exhausted" - afterId = upperId; - visited += window; + if (rows.Count < batch) + { + // With no predicate in the query, a short page can only mean the table is exhausted -- it can + // never mean "this stretch happened to match nothing", which is precisely why the predicate had + // to go. Advancing from the last returned Id is likewise safe: nothing was filtered out behind + // it, so no row can be skipped. + break; + } } return distinct.ToList(); @@ -258,23 +250,18 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF } /// - /// One keyspace window of candidate rows. LOWER(...) LIKE ... ESCAPE and ORDER BY are - /// portable between SQLite and MySQL, and the identifiers are unquoted so neither provider's quoting - /// dialect is baked in. + /// One keyset page of rows, by ROW POSITION rather than by Id value. /// - /// The upper bound is what makes this bounded work rather than a bounded result: with only - /// Id > @AfterId and a LIMIT, a no-match query still forces the engine to evaluate - /// every remaining row before it can return an empty page. Closing the range turns each round trip - /// into a primary-key range scan of known width. There is deliberately no LIMIT — the window - /// already caps the row count, and a LIMIT here would re-introduce the illusion that it is - /// doing the bounding. + /// There is deliberately no WHERE clause beyond the keyset cursor — no LIKE, and + /// not even IS NOT NULL. Any predicate is something the engine must evaluate and discard, + /// which is what let every earlier revision scan past its own bound: LIMIT only truncates + /// what survives a predicate, so with one present it bounds the output and not the work. Stripped to + /// a bare primary-key range, LIMIT n reads exactly n index entries and n rows. + /// Null payloads are dropped in memory by . /// /// internal static string ListValuedSql(string column) => - $"SELECT Id, {column} AS Payload FROM SongMetadata WHERE {column} IS NOT NULL " + - "AND Id > @AfterId AND Id <= @UpperId " + - $"AND LOWER({column}) LIKE @Pattern ESCAPE '{LikeEscapeChar}' " + - "ORDER BY Id"; + $"SELECT Id, {column} AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch"; private sealed class ListValuedRow { @@ -283,67 +270,6 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF public string Payload { get; init; } } - /// - /// Builds the raw-JSON LIKE pattern used to narrow the rows fetched for a list-valued column. - /// - /// This is an optimization with one hard obligation: it must never under-match. Every row - /// holding an element that starts with under - /// — the comparison - /// then applies — has to survive it. Over-matching is free; - /// the in-memory filter discards the extras. - /// - /// - /// So the pattern narrows on the leading run of ASCII characters the JSON writer stores - /// verbatim, and stops at the first character it cannot reason about — a non-ASCII character, or - /// one of . A query starting with such a character narrows to the bare - /// element-opening anchor %"%, which matches every non-empty row — so the walk's keyspace - /// bound, not the pattern, is what keeps that case bounded, and the in-memory filter is what keeps - /// it correct. - /// - /// - /// Why that is sound, and why an earlier "JSON-encode the whole prefix" version was NOT: the stored - /// text escapes non-ASCII, so Édith is on disk as \u00C9dith. SQL LOWER() - /// lowercases the escape text — \u00C9 becomes \u00c9 — it cannot case-fold the - /// codepoint that escape denotes. So a q=é prefix encoded to the pattern \u00e9, which - /// never matched the stored \u00C9. It failed whenever the query's casing differed from the - /// stored casing (three of the nine cases pinned in GetSearchFieldValuesHandlerTests); when - /// the casings happened to agree the escape texts lined up and it worked, which is exactly why the - /// bug survived review. Restricting the run to verbatim ASCII removes the escape from the comparison - /// entirely. And for a run character c, any element character that - /// OrdinalIgnoreCase-equals c is itself ASCII — no non-ASCII codepoint in all of Unicode - /// OrdinalIgnoreCase-equals a printable ASCII character, which - /// SearchFieldValuesPrefilterSupersetTests proves by exhaustive sweep rather than assertion. - /// (That is false for InvariantCultureIgnoreCase, which has 190 such codepoints — the choice - /// of Ordinal is load-bearing, not stylistic.) ASCII case-folding is exactly what both providers' - /// LOWER() implements, and lowercasing the pattern here rather than trusting LIKE's own - /// case rules keeps it correct under a case-sensitive MySQL collation as well as a case-insensitive - /// one. - /// - /// - internal static string JsonElementPrefixPattern(string query) - { - var builder = new StringBuilder("%\""); - - foreach (char c in query ?? string.Empty) - { - if (!char.IsAscii(c) || char.IsControl(c) || JsonEscapedAscii.Contains(c)) - { - // cannot prove this character's stored form, so stop narrowing here - break; - } - - char lower = char.ToLowerInvariant(c); - if (lower is '%' or '_' or LikeEscapeChar) - { - builder.Append(LikeEscapeChar); - } - - builder.Append(lower); - } - - return builder.Append('%').ToString(); - } - private static async Task> GetContentRatingValues( TvContext dbContext, string query, @@ -373,8 +299,9 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// are ORDINAL on purpose: UseRequestLocalization honours Accept-Language, so the current /// culture is caller-controlled, and ToLower() plus the default (linguistic) /// StartsWith(string) would make the result depend on it — under tr-TR, q=I lowers - /// to ı and stops matching Istanbul. Ordinal is also what the SQL pre-filter's superset - /// guarantee is proved against; see . + /// to ı and stops matching Istanbul. Note this is the LAST stage only: a field sourced by + /// a plain EF query has already been filtered and truncated by the database collation before it gets + /// here, which ordinal semantics downstream cannot undo (ersatztv#668). /// private static List FilterSortTake(IEnumerable values, string query, int limit) => values diff --git a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs index f3573cb5a..1f4b8c0f1 100644 --- a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs +++ b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs @@ -267,7 +267,7 @@ public class GetSearchFieldValuesHandlerTests await using (TvContext context = _db.CreateContext()) { context.SongMetadata.AddRange( - // "Neighbour" rides along in the SQL superset because "Radiohead" matches; it must be + // "Neighbour" arrives on the same row as "Radiohead" -- rows are read whole -- and must be // dropped by the in-memory exact prefix filter. Song("One", ["Radiohead", "Neighbour"]), // "The Radio Dept." contains "radio" but does not start with it @@ -287,16 +287,16 @@ public class GetSearchFieldValuesHandlerTests } [Test] - public async Task List_Valued_Fields_Survive_Json_Escaping_And_Like_Wildcards() + public async Task List_Valued_Fields_Match_Literally_Including_Json_Escaped_And_Wildcard_Characters() { await using (TvContext context = _db.CreateContext()) { context.SongMetadata.AddRange( - // non-ASCII: stored on disk JSON-escaped as é + // non-ASCII: stored on disk JSON-escaped as \u00E9, and must survive the round trip Song("One", ["Beyoncé"]), - // an embedded quote is stored as " + // an embedded quote is stored as \u0022 Song("Two", ["\"Weird Al\" Yankovic"]), - // LIKE wildcards in the query must be treated literally + // SQL wildcards must be ordinary characters here, matched literally Song("Three", ["50% Off"]), Song("Four", ["50 Cent"])); await context.SaveChangesAsync(); @@ -328,11 +328,11 @@ public class GetSearchFieldValuesHandlerTests """); [Test] - public async Task List_Valued_Walk_Visits_At_Most_20000_Rows_When_Every_Row_Is_A_Candidate() + public async Task List_Valued_Walk_Reads_At_Most_20000_Rows() { - // The WIDENED case: a non-ASCII prefix collapses the pattern to the bare `%"%` anchor, so every row is - // a candidate. Pinned in both directions so the ceiling itself is nailed down — a needle at row 20000 is - // reached, the same needle at 20001 is not. + // Pinned in both directions so the ceiling itself is nailed down: a match in row 20000 is read, the same + // match in row 20001 is not. The query carries no predicate, so "rows read" is literally what LIMIT + // returns -- there is no gap between what the engine looks at and what it hands back. const string needle = "\u00E9clair-the-needle"; await using (TvContext context = _db.CreateContext()) @@ -366,87 +366,47 @@ public class GetSearchFieldValuesHandlerTests r => r.Values.ShouldBeEmpty( "row 20001 is past the ceiling; this false negative is the documented bounded-best-effort " + "contract, deliberately pinned rather than papered over")); - - // positive control: narrowing the prefix drops the filler from the candidate set, so the ceiling is what - // the assertion above is about, not an unreachable value - (await handler.Handle(new GetSearchFieldValues("artist", "zzz", 50), CancellationToken.None)) - .IfSome(r => r.Values.ShouldBe(new List { "zzz-filler" })); } [Test] - public async Task List_Valued_Walk_Cannot_Inspect_Past_The_Bound_When_Nothing_Matches() + public async Task List_Valued_Walk_Reads_Live_Rows_Regardless_Of_Id_Density() { - // THE case every earlier bound test missed: they all gave the SQL pre-filter something to return. + // THE round-4 killer. That revision bounded the Id KEYSPACE, and keyspace is not rows: with 20,000 + // historical rows deleted and one live song at Id 20001, the walk spent its whole allowance on empty + // ranges and returned [] for a table containing exactly one row. Capacity degraded linearly with + // deletion ratio, and no ratio was safe -- one placed gap hid the next match. // - // Here the pattern is NARROW (an ASCII prefix), so the LIKE matches nothing until the needle. A LIMIT - // cannot bound this: to return an empty page the engine must evaluate every eligible row first. The - // previous revision therefore let the database walk the entire table and FOUND a needle at row 30000 — - // correct answer, unbounded work. The closed keyspace window stops at 20000 rows inspected, so the - // needle beyond it is not found, and that is the observable difference. - const string needle = "qqq-far-past-the-bound"; - + // Paging by row position rather than Id value makes density irrelevant: LIMIT @Batch returns @Batch + // ROWS, wherever they sit in the keyspace. await using (TvContext context = _db.CreateContext()) { - await SeedFiller(context, 29999); - context.SongMetadata.Add(Song("Needle", [needle])); + await SeedFiller(context, 20000); + await context.Database.ExecuteSqlRawAsync("DELETE FROM SongMetadata"); + context.SongMetadata.Add(Song("Survivor", ["Queen"])); await context.SaveChangesAsync(); - (await context.SongMetadata.CountAsync()).ShouldBe(30000); + // one live row, sitting past the old keyspace allowance + (await context.SongMetadata.CountAsync()).ShouldBe(1); + (await context.SongMetadata.Select(m => m.Id).SingleAsync()).ShouldBeGreaterThan(20000); } var handler = new GetSearchFieldValuesHandler(_db.Factory); - (await handler.Handle(new GetSearchFieldValues("artist", "qqq", 50), CancellationToken.None)) + (await handler.Handle(new GetSearchFieldValues("artist", "que", 50), CancellationToken.None)) .IfSome( - r => r.Values.ShouldBeEmpty( - "a no-match walk must stop at the visit ceiling; reaching row 30000 means the bound is on " - + "rows RETURNED rather than rows INSPECTED")); + r => r.Values.ShouldBe( + new List { "Queen" }, + "a one-row table must be fully readable no matter where its Id sits")); - // ... while the identical query against a needle inside the bound does find it, so the assertion above - // is about the ceiling and not about the narrow pattern failing to match at all - await using (InMemoryTvContext smaller = await InMemoryTvContext.CreateAsync()) - { - await using (TvContext context = smaller.CreateContext()) - { - await SeedFiller(context, 19999); - context.SongMetadata.Add(Song("Needle", [needle])); - await context.SaveChangesAsync(); - } - - (await new GetSearchFieldValuesHandler(smaller.Factory).Handle( - new GetSearchFieldValues("artist", "qqq", 50), - CancellationToken.None)) - .IfSome(r => r.Values.ShouldBe(new List { needle }, "row 20000 is inside the ceiling")); - } - } - - [Test] - public async Task Regression_Pin_List_Valued_Columns_Are_Stored_As_Ascii_Escaped_Json_Arrays() - { - // REGRESSION PIN, not coverage of #578: this asserts pre-existing EF behaviour and passes against the - // code before this change. It is here because the pre-filter's correctness argument depends on the - // stored form, so an encoder change must fail loudly rather than silently under-match. - // - // Specifically it pins that non-ASCII IS escaped — which is exactly why SQL LOWER() canNOT be used to - // case-fold it. LOWER() lowercases the escape TEXT (\u00E9 -> \u00e9); it does not touch the codepoint - // the escape denotes, so the stored `É` never folds to `é`. An earlier version of this change - // claimed the opposite and JSON-encoded the whole query prefix, which silently made every accented - // artist unsuggestable. The pattern builder now narrows only on verbatim ASCII; see - // SearchFieldValuesPrefilterSupersetTests. + // and a leading gap must not hide a later match either await using (TvContext context = _db.CreateContext()) { - context.SongMetadata.Add(Song("One", ["Beyoncé", "\"Q\""])); + context.SongMetadata.Add(Song("Second", ["Queens of the Stone Age"])); await context.SaveChangesAsync(); } - await using (TvContext context = _db.CreateContext()) - { - List raw = await context.Database - .SqlQueryRaw("SELECT Artists AS Value FROM SongMetadata") - .ToListAsync(); - - raw.ShouldBe(new List { """["Beyonc\u00E9","\u0022Q\u0022"]""" }); - } + (await handler.Handle(new GetSearchFieldValues("artist", "que", 50), CancellationToken.None)) + .IfSome(r => r.Values.ShouldBe(new List { "Queen", "Queens of the Stone Age" })); } [Test] @@ -459,19 +419,18 @@ public class GetSearchFieldValuesHandlerTests [TestCase("beyonc\u00E9", "Beyonc\u00E9")] [TestCase("sigur r", "Sigur R\u00F3s")] [TestCase("\u00D6", "\u00D6zdemir")] - public async Task Prefilter_Never_Under_Matches_A_NonAscii_Value(string query, string stored) + public async Task Matches_NonAscii_Values_In_Any_Casing(string query, string stored) { - // The SQL pre-filter narrows on raw JSON, where non-ASCII is stored escaped (\u00C9). A narrowing run - // that includes an escaped character can under-match, because SQL LOWER() folds the escape TEXT and not - // the codepoint it denotes. + // Accented artists are the common case in a music library, so non-ASCII matching is pinned end to + // end, in both casings of the query. // - // Be precise about what these cases guard: THREE of the nine fail against 1b78dc9e (the JSON-encode-the- - // whole-prefix revision) — ("é", "Édith Piaf"), ("édith", "Édith Piaf") and ("BJÖRK", "Björk") — because - // those are the ones where the query and stored casings differ, so the lowercased escape text (\u00e9) - // and the stored escape text (\u00C9) diverge. The other SIX pass that revision: when the query casing - // already matches the stored casing the escape texts happen to line up. They are kept as continuity - // coverage of the ASCII-run narrowing, NOT as regression guards. Accented artists are the common case in - // a music library, so both groups are worth pinning. + // Historical note, because it is why this suite exists: revision 1b78dc9e narrowed rows in SQL + // with a LIKE built by JSON-encoding the query, which cannot work -- non-ASCII is stored escaped + // (\u00C9) and SQL LOWER() folds the escape TEXT, not the codepoint it denotes. THREE of these nine + // cases fail against that revision (the ones where query and stored casing differ, so \u00e9 and + // \u00C9 diverge); the other six pass it, because when the casings agree the escape texts line up. + // The SQL predicate is gone entirely now -- matching happens in memory, where a string is just a + // string -- so these cases pin current behaviour rather than guard that revision. await using (TvContext context = _db.CreateContext()) { context.SongMetadata.AddRange( @@ -594,12 +553,12 @@ public class GetSearchFieldValuesHandlerTests } [Test] - public async Task Prefilter_Widening_Must_Not_Starve_The_Candidate_Set_Past_The_Row_Bound() + public async Task A_Match_Behind_Many_NonMatching_Rows_Is_Still_Found() { - // BLOCKER-1 REGRESSION (fails a883e5f0). A non-ASCII prefix narrows to the bare anchor `%"%`, so EVERY - // row qualifies. Under a fixed 1000-row budget the 1001st row -- the only exact match -- was discarded - // before the in-memory filter ever saw it, and the endpoint returned []. "Over-matching is free" only - // holds when the candidate set is not truncated; with a fixed cap, widening the predicate STARVES it. + // Fails a883e5f0, which capped rows at a fixed 1000 AFTER a deliberately over-matching SQL pre-filter: + // the 1001st row -- the only exact match -- was discarded before the in-memory filter ever saw it and + // the endpoint returned []. The pre-filter is gone, but the property it broke is permanent: a match must + // not be hidden by however many non-matching rows happen to precede it. await using (TvContext context = _db.CreateContext()) { for (var i = 0; i < 1000; i++) diff --git a/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs b/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs deleted file mode 100644 index 834a0d385..000000000 --- a/ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs +++ /dev/null @@ -1,168 +0,0 @@ -using System.Text.Json; -using ErsatzTV.Application.Search.Queries; -using NUnit.Framework; -using Shouldly; - -namespace ErsatzTV.Tests.Application.Search; - -/// -/// Proof obligations for the raw-JSON LIKE pre-filter that narrows the list-valued facet-value -/// sources (#578). The pre-filter is an optimization layered under an in-memory exact filter, so it is -/// allowed to over-match but must NEVER under-match — an under-match is a silent false negative, invisible -/// to any test that only checks the values it does return. -/// -/// The end-to-end behavioural cases live in GetSearchFieldValuesHandlerTests. These are the two -/// facts the superset argument rests on, asserted directly rather than trusted. -/// -/// -[TestFixture] -public class SearchFieldValuesPrefilterSupersetTests -{ - /// - /// Printable ASCII, the alphabet - /// narrows over (minus whatever the JSON writer escapes, which the sibling test pins). - /// - private static IEnumerable PrintableAscii() - { - for (var c = ' '; c <= '~'; c++) - { - yield return c; - } - } - - /// - /// PROOF OBLIGATION, not regression coverage. This is revision-independent — it asserts a property of - /// .NET's ordinal case folding and passes against every revision of the handler, including the ones with - /// the under-matching pattern builder. It is here because the superset argument DEPENDS on it: if a - /// future runtime changed this, the ASCII narrowing would start under-matching with nothing else to - /// catch it. - /// - [Test] - public void No_NonAscii_Codepoint_OrdinalIgnoreCase_Equals_A_Printable_Ascii_Character() - { - // THIS is why the narrowing run may be restricted to ASCII and still be a superset. The in-memory filter - // compares with OrdinalIgnoreCase, so for a run character `c` the pre-filter must catch every element - // character that OrdinalIgnoreCase-equals `c`. If any non-ASCII codepoint did, that element would be - // stored as a \uXXXX escape, the ASCII pattern would miss the row, and the value would be unsuggestable. - // - // Note this is NOT true of InvariantCultureIgnoreCase, which folds ~190 codepoints (U+00AA -> a, - // U+017F -> s, the modifier letters, ...) onto ASCII letters. The choice of Ordinal is load-bearing. - char[] ascii = PrintableAscii().ToArray(); - var offenders = new List(); - - for (var cp = 0x80; cp <= 0x10FFFF; cp++) - { - if (cp is >= 0xD800 and <= 0xDFFF) - { - continue; - } - - string s = char.ConvertFromUtf32(cp); - foreach (char a in ascii) - { - if (string.Equals(s, a.ToString(), StringComparison.OrdinalIgnoreCase)) - { - offenders.Add($"U+{cp:X4} == '{a}'"); - } - } - } - - offenders.ShouldBeEmpty(); - } - - /// - /// PROOF OBLIGATION plus one regression assertion. The escaped-set half is revision-independent (it - /// pins the encoder); the pattern-builder half below fails against 1b78dc9e. - /// - [Test] - public void The_Json_Writer_Escapes_Exactly_The_Ascii_Characters_The_Pattern_Builder_Refuses_To_Narrow_On() - { - // The narrowing run may only contain characters stored VERBATIM. Anything the writer escapes is stored - // as \uXXXX and must terminate the run. If an encoder change adds a character to this set, the pattern - // builder's exclusion list goes stale and starts under-matching — so pin the set, not a sample. - var escaped = PrintableAscii() - .Where(c => JsonSerializer.Serialize(c.ToString()) != $"\"{c}\"") - .ToArray(); - - escaped.ShouldBe(['"', '&', '\'', '+', '<', '>', '\\', '`']); - - // and the pattern builder must stop at every one of them (empty narrowing run -> bare anchor) - foreach (char c in escaped) - { - GetSearchFieldValuesHandler.JsonElementPrefixPattern($"{c}abc") - .ShouldBe("%\"%", $"expected no narrowing for the JSON-escaped '{c}'"); - } - } - - [Test] - public void Pattern_Narrows_Only_On_The_Leading_Verbatim_Ascii_Run() - { - // plain ASCII: full narrowing, lowercased - GetSearchFieldValuesHandler.JsonElementPrefixPattern("Radio").ShouldBe("%\"radio%"); - - // non-ASCII terminates the run, keeping the ASCII head as the narrowing - GetSearchFieldValuesHandler.JsonElementPrefixPattern("Beyoncé").ShouldBe("%\"beyonc%"); - - // a leading non-ASCII character means no narrowing at all - GetSearchFieldValuesHandler.JsonElementPrefixPattern("édith").ShouldBe("%\"%"); - GetSearchFieldValuesHandler.JsonElementPrefixPattern("Ünen").ShouldBe("%\"%"); - - // empty query: the bare element-opening anchor - GetSearchFieldValuesHandler.JsonElementPrefixPattern(string.Empty).ShouldBe("%\"%"); - GetSearchFieldValuesHandler.JsonElementPrefixPattern(null).ShouldBe("%\"%"); - - // LIKE wildcards inside the run are escaped, not passed through - GetSearchFieldValuesHandler.JsonElementPrefixPattern("50%").ShouldBe("%\"50/%%"); - GetSearchFieldValuesHandler.JsonElementPrefixPattern("a_b").ShouldBe("%\"a/_b%"); - - // the escape character itself is escaped - GetSearchFieldValuesHandler.JsonElementPrefixPattern("AC/DC").ShouldBe("%\"ac//dc%"); - } - - [Test] - public void Candidate_Rows_Are_Read_In_A_Closed_Id_Keyspace_Window() - { - string sql = GetSearchFieldValuesHandler.ListValuedSql("Artists"); - - // THE upper bound is the whole point. With only `Id > @AfterId` plus a LIMIT, a query that matches - // nothing still forces the engine to evaluate every remaining row before it can return an empty page — - // so rows RETURNED were bounded while rows INSPECTED were not. Closing the range makes each round trip - // a primary-key range scan of known width. - sql.ShouldContain("Id > @AfterId"); - sql.ShouldContain("Id <= @UpperId"); - - // ORDER BY rides Id, not the JSON column: MySQL sorts TEXT using only max_sort_length bytes. - sql.ShouldContain("ORDER BY Id"); - sql.ShouldNotContain("ORDER BY Artists"); - - // and no LIMIT — the window caps the row count, and a LIMIT here would only restore the illusion that - // it is what does the bounding - sql.ShouldNotContain("LIMIT"); - } - - [Test] - public void The_Visit_Ceiling_Is_A_Whole_Number_Of_Windows_So_The_Walk_Cannot_Overrun_It() - { - // The invariant that actually matters is that the ceiling is reachable and never overshot. The walk - // clamps its final window (`Math.Min(WindowRows, MaxVisitedRows - visited)`), so a ceiling that is not a - // multiple of the window size would still be safe — an earlier version of this test rejected that, which - // made it a style preference wearing a correctness costume. - GetSearchFieldValuesHandler.ListValuedWindowRows.ShouldBeGreaterThan(0); - GetSearchFieldValuesHandler.ListValuedMaxVisitedRows - .ShouldBeGreaterThanOrEqualTo(GetSearchFieldValuesHandler.ListValuedWindowRows); - - // simulate the walk's own arithmetic and assert it lands exactly on the ceiling, never past it - var visited = 0; - var windows = 0; - while (visited < GetSearchFieldValuesHandler.ListValuedMaxVisitedRows) - { - visited += Math.Min( - GetSearchFieldValuesHandler.ListValuedWindowRows, - GetSearchFieldValuesHandler.ListValuedMaxVisitedRows - visited); - windows++; - windows.ShouldBeLessThan(1000, "the walk must terminate"); - } - - visited.ShouldBe(GetSearchFieldValuesHandler.ListValuedMaxVisitedRows); - } -} diff --git a/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs b/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs index d43e71d70..4417d3f1b 100644 --- a/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs +++ b/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs @@ -78,6 +78,25 @@ public class SearchFieldValuesQueryShapeTests } } + [Test] + public void List_Valued_Page_Query_Has_No_Predicate_Beyond_The_Keyset_Cursor() + { + // This is the whole basis of the row bound, so it is asserted rather than assumed. LIMIT truncates what + // survives a predicate — so with ANY predicate present it bounds the output, not the work, and the + // engine may evaluate and discard arbitrarily many rows first. That is how four successive revisions + // scanned past their own bound. Stripped to a bare primary-key range, LIMIT n reads exactly n index + // entries and n rows, whatever the data looks like. + string sql = GetSearchFieldValuesHandler.ListValuedSql("Artists"); + + sql.ShouldBe( + "SELECT Id, Artists AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch"); + + // named explicitly so a future "optimization" that reintroduces server-side selectivity fails here + sql.ShouldNotContain("LIKE"); + sql.ShouldNotContain("LOWER"); + sql.ShouldNotContain("IS NOT NULL"); + } + private static IEnumerable<(string Provider, Func Create)> Providers() => [ ("sqlite", Sqlite), diff --git a/ErsatzTV/Controllers/Api/SearchController.cs b/ErsatzTV/Controllers/Api/SearchController.cs index 539141f68..2a8683411 100644 --- a/ErsatzTV/Controllers/Api/SearchController.cs +++ b/ErsatzTV/Controllers/Api/SearchController.cs @@ -208,8 +208,8 @@ public class SearchController(IMediator mediator) : ControllerBase "source. The final filter, dedup and ordering applied to the response are ordinal and not " + "culture-dependent; note that fields sourced by a plain database query are additionally " + "pre-filtered by the database collation first, which on SQLite is ASCII-only. The list-valued " + - "music fields (artist, album_artist) are bounded best-effort: on a very large library they may " + - "return a bounded subset of the matches rather than every match.")] + "music fields (artist, album_artist) are bounded best-effort: the server reads a bounded number " + + "of song rows per request, so a library larger than that bound may yield a subset of the matches.")] [EndpointGroupName("general")] [ProducesResponseType(typeof(SearchFieldValuesResponseModel), StatusCodes.Status200OK)] [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)] diff --git a/ErsatzTV/wwwroot/openapi/v1.json b/ErsatzTV/wwwroot/openapi/v1.json index 84cdaaae0..1f18196a3 100644 --- a/ErsatzTV/wwwroot/openapi/v1.json +++ b/ErsatzTV/wwwroot/openapi/v1.json @@ -18063,7 +18063,7 @@ "Search" ], "summary": "List distinct database values for a text search field", - "description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source. The final filter, dedup and ordering applied to the response are ordinal and not culture-dependent; note that fields sourced by a plain database query are additionally pre-filtered by the database collation first, which on SQLite is ASCII-only. The list-valued music fields (artist, album_artist) are bounded best-effort: on a very large library they may return a bounded subset of the matches rather than every match.", + "description": "Returns distinct whole values from the database for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown, is not a text field, or is a text field with no distinct-value source. The final filter, dedup and ordering applied to the response are ordinal and not culture-dependent; note that fields sourced by a plain database query are additionally pre-filtered by the database collation first, which on SQLite is ASCII-only. The list-valued music fields (artist, album_artist) are bounded best-effort: the server reads a bounded number of song rows per request, so a library larger than that bound may yield a subset of the matches.", "operationId": "GetSearchFieldValues", "parameters": [ { diff --git a/docs/api-conventions.md b/docs/api-conventions.md index fb04a9665..9eedb2b56 100644 --- a/docs/api-conventions.md +++ b/docs/api-conventions.md @@ -456,30 +456,35 @@ No server-side caching. Powers the visual rule builder's value-input combobox fo or partly) by `SongMetadata.Artists`/`AlbumArtists`, which EF maps as **primitive collections** — one JSON array per row in a single column, with no server-side projection on either provider. `album_artist` therefore no longer 404s, and `artist` now also covers free-text music-video (`MusicVideoArtist`) and -song credits, not only entity artists. For these fields the handler narrows on the raw JSON in SQL and -walks candidate rows keyset-paged on the primary key, so on a very large library the response may be a -bounded subset of the matches — say so in the `[EndpointDescription]` of any endpoint that adopts this shape. +song credits, not only entity artists. Those rows are read by a keyset page that carries **no predicate +at all** and filtered in memory, bounded at 20,000 rows per request — so on a larger library the +response may be a bounded subset of the matches. Say so in the `[EndpointDescription]` of any endpoint +that adopts this shape. Three rules generalize beyond this endpoint. -1. **A SQL pre-filter layered under an in-memory exact filter may over-match but must NEVER under-match.** - An under-match is a silent false negative that no assertion over the returned values can see, so prove - the superset (ours is proved by an exhaustive Unicode sweep) rather than eyeballing it. -2. **That licence to over-match is void the moment the candidate set is truncated by a fixed row budget.** - Widening the predicate then starves the budget with rows that cannot match and drops real matches sitting - behind them — the same false negative, one stage later. Bound the WORK (page with a keyset cursor until you - have enough matches, the source is exhausted, or a scan ceiling is hit), never the candidate rows with a - single `LIMIT` whose survivors silently become the answer. Both revisions of this endpoint got this wrong - in opposite directions before it was written down. +1. **`LIMIT` bounds the OUTPUT, not the WORK, whenever a predicate is present.** It truncates what + survives the predicate, and the engine may evaluate and discard arbitrarily many rows first — a query + that matches nothing must examine every eligible row before it can return an empty page. If you need + to bound work, remove the predicate and page by row position over the primary key, then filter in + memory. This endpoint got it wrong four times: bounding the result, then candidates returned, then + `Id` keyspace width (keyspace is not rows — one live row at `Id` 20001 behind 20,000 deleted ones + reads nothing), before arriving at "no predicate, so `LIMIT n` reads exactly `n` rows". +2. **A SQL pre-filter under an in-memory exact filter may over-match but must never under-match — and + that licence is void the moment the candidate set is truncated.** Widening the predicate then starves + the budget with rows that cannot match. If you find yourself proving a superset property to keep a + pre-filter honest, consider deleting the pre-filter instead: here it removed a JSON-escaping bug + class, an exhaustive Unicode sweep and an `ESCAPE` portability workaround along with it. 3. **Prefix matching, dedup and ordering must be ordinal, not current-culture** (`OrdinalIgnoreCase`, `StringComparer.Ordinal`): `UseRequestLocalization` honours `Accept-Language`, so `ToLower()` and the - default linguistic `StartsWith(string)` let a caller change the result by changing a header. **Scope the - claim to the stage that actually holds it** — a value set that a database `LOWER`/`DISTINCT`/`ORDER BY`/ - `LIMIT` already filtered and truncated is not ordinal no matter what runs after it, and saying otherwise in - an `[EndpointDescription]` publishes a false contract (ersatztv#668). + default linguistic `StartsWith(string)` let a caller change the result by changing a header. **Scope + the claim to the stage that actually holds it** — a value set that a database `LOWER`/`DISTINCT`/ + `ORDER BY`/`LIMIT` already filtered and truncated is not ordinal no matter what runs after it, and + saying otherwise in an `[EndpointDescription]` publishes a false contract (ersatztv#668). -Full rationale, the rejected normalized-side-table alternative (ersatztv#669), the accepted scan cost and the -dual-provider `LIKE`/`LOWER` rules: `api.search-field-values-sources` (supersedes `api.search-field-values`). +Full rationale, the measured transfer cost, the four-attempts table and the rejected +normalized-side-table alternative (ersatztv#669): `api.search-field-values-sources` (supersedes +`api.search-field-values`). **Param + DTO expansion (#293, cap `search/all-items`)**: no new endpoint — `GET /api/v1/search/all-items` gained two **optional** query params (`pageSize` default 500, clamped 1–1000 via the §1 Logs `Math.Clamp` diff --git a/docs/decisions/README.md b/docs/decisions/README.md index c71122d9e..60a5779fd 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -28,7 +28,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `api.schedule-item-flat-dto` | Schedule-item GET/POST/PUT use a flat, non-polymorphic `ScheduleItemResponseModel` (every subtype field promoted to a nullable top-level member) instead of the polymorphic Application VM hierarchy, with mutation field names matching `ScheduleItemRequest` 1:1 for a lossless round-trip. | 2026-07-10 | [link](records/api/schedule-item-flat-dto.md) | | `api.scheduling-hardening` | Create/Replace handlers guard against null/whitespace `name` (`IsNullOrWhiteSpace`, not just `Length`) to prevent NRE-500s, template-item overlap validation compares by index (not record value-equality) to catch exact-duplicate items, and unreachable 404 `ProducesResponseType` attributes on create-only actions are trimmed. | 2026-07-13 | [link](records/api/scheduling-hardening.md) | | `api.search-allitems-paging` | `GET /api/v1/search/all-items` is paginated (capped page size, `Totals` field) to bound DoS exposure; the SPA add-all flow pages to completeness instead of relying on an unbounded response. | 2026-07-18 | [link](records/api/search-allitems-paging.md) | -| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — this is scoped to the in-memory stages on purpose: a field sourced by a plain EF query is pre-filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&'+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; candidate rows are walked in CLOSED `Id` keyspace windows (`Id > @AfterId AND Id <= @AfterId + @Window`, no `LIMIT`) and split + exact-filtered in memory, stopping on the first of enough distinct matches, `MAX(Id)` passed, or a 20,000-row visit ceiling — the bound is on rows the database LOOKS AT, not on rows returned: a fixed `LIMIT` budget lets a widened pre-filter starve the candidate set, and a `LIMIT` alone cannot bound a no-match query, which must evaluate every eligible row before it can return an empty page. Ordering across merged sources is best-effort, exact only below the truncation points. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | +| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort, and its rows are read by a keyset page carrying **NO predicate whatsoever** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL` — because `LIMIT` only truncates what survives a predicate, so any predicate at all makes it bound the OUTPUT rather than the WORK. All selectivity is in memory. The walk reads at most 20,000 actual rows in 2,000-row pages, density-independent, stopping on the first of enough distinct matches, a short page, or the ceiling; the cost of having no server-side narrowing is a measured 392 KiB of JSON transferred in the worst case (20,000 rows, ~20 B/row). | 2026-07-26 | [link](records/api/search-field-values-sources.md) | | `api.search-paging-cap` | Search stays capped at 100 items per media kind; an overflowing kind's "See all" reuses library-browse paging instead of adding new API surface. | 2026-07-11 | [link](records/api/search-paging-cap.md) | | `api.versioning-v1` | The entire `/api` surface is versioned to `/api/v1` uniformly (no unversioned corner); legacy unversioned callers are rewritten in-pipeline (not redirected) with Deprecation/Link/Sunset headers, and post-freeze `/api/v1` is additive-only — a breaking change requires `/api/v2`. | 2026-07-13 | [link](records/api/versioning-v1.md) | | `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) | diff --git a/docs/decisions/records/api/search-field-values-sources.md b/docs/decisions/records/api/search-field-values-sources.md index 23f2109a5..9e5e4e43e 100644 --- a/docs/decisions/records/api/search-field-values-sources.md +++ b/docs/decisions/records/api/search-field-values-sources.md @@ -1,13 +1,13 @@ --- key: api.search-field-values-sources -title: '2026-07-26 — Facet-value typeahead, restated: every artist-bearing source is covered, list-valued columns via a superset LIKE pre-filter that may over-match but never under-match (#578)' +title: '2026-07-26 — Facet-value typeahead, restated: every artist-bearing source is covered, and the JSON-column source is paged by ROW POSITION with no SQL predicate at all (#578)' status: active since: '2026-07-26' supersedes: api.search-field-values@2026-07-23 superseded-by: none -rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — this is scoped to the in-memory stages on purpose: a field sourced by a plain EF query is pre-filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort: SQL narrows on the raw JSON with a `LIKE` pattern built ONLY from the leading run of characters the JSON writer stores verbatim (printable ASCII minus `"&''+<>\`), stopping at the first character it cannot prove — this pre-filter is an optimization that MAY over-match and MUST NEVER under-match, with correctness living in the in-memory exact filter; candidate rows are walked in CLOSED `Id` keyspace windows (`Id > @AfterId AND Id <= @AfterId + @Window`, no `LIMIT`) and split + exact-filtered in memory, stopping on the first of enough distinct matches, `MAX(Id)` passed, or a 20,000-row visit ceiling — the bound is on rows the database LOOKS AT, not on rows returned: a fixed `LIMIT` budget lets a widened pre-filter starve the candidate set, and a `LIMIT` alone cannot bound a no-match query, which must evaluate every eligible row before it can return an empty page. Ordering across merged sources is best-effort, exact only below the truncation points.' -signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, raw JSON LIKE pre-filter, superset never under-match, accented artist unsuggestable, LOWER cannot fold a \u00XX escape, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, ListValuedRowCap, ORDER BY Id not TEXT, max_sort_length, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesPrefilterSupersetTests.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669' -mechanics: '`GetSearchFieldValuesHandler` (`GetSource`, `GetSongListValuedColumn`, `GetSongListValuedValues`, `ListValuedSql`, `JsonElementPrefixPattern`, `FilterSortTake`); `SearchFieldValuesPrefilterSupersetTests` (the superset proof); api-conventions.md; spa-conventions.md §12' +rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort, and its rows are read by a keyset page carrying **NO predicate whatsoever** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL` — because `LIMIT` only truncates what survives a predicate, so any predicate at all makes it bound the OUTPUT rather than the WORK. All selectivity is in memory. The walk reads at most 20,000 actual rows in 2,000-row pages, density-independent, stopping on the first of enough distinct matches, a short page, or the ceiling; the cost of having no server-side narrowing is a measured 392 KiB of JSON transferred in the worst case (20,000 rows, ~20 B/row).' +signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, LIMIT bounds output not work, predicate defeats LIMIT as a bound, keyspace is not rows, page by row position, density-independent paging, deleted rows leave Id gaps, ListValuedBatchRows, ListValuedMaxRowsRead, bounded best-effort facet values, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669' +mechanics: '`GetSearchFieldValuesHandler` (`GetSource`, `GetSongListValuedColumn`, `GetSongListValuedValues`, `ListValuedSql`, `ParseElements`, `FilterSortTake`); `SearchFieldValuesQueryShapeTests.List_Valued_Page_Query_Has_No_Predicate_Beyond_The_Keyset_Cursor`; api-conventions.md; spa-conventions.md §12' --- Supersedes `api.search-field-values` (#434). That record did not merely carry a stale implementation @@ -35,40 +35,26 @@ need no lookup; text fields need a live one; the source is the database and neve failures are pinned by a test, so a provider upgrade that fixes them surfaces as a red rather than leaving a workaround in place forever. -## The pre-filter is an optimization; it may over-match, it must never under-match +## The SQL predicate is gone, and that is the point -This is the load-bearing invariant, and the first implementation got it wrong in a way that no test -of the returned values could see. It JSON-encoded the whole query prefix, reasoning that since the -stored text escapes non-ASCII (`Édith` is on disk as `\u00C9dith`), encoding the prefix the same way -would line up. **It does not.** SQL `LOWER()` lowercases the *escape text* — `\u00C9` becomes `\u00c9` — -it cannot case-fold the codepoint that escape denotes, so the stored `\u00C9` never folds to `\u00e9`. -A `q=é` prefix produced the pattern `\u00e9`, which never matched the stored `\u00C9`, so the row was -discarded before the in-memory filter ever saw it. Be precise about the blast radius, because the -imprecision is what let it through review: it failed whenever the query's casing differed from the -stored casing — **three of the nine cases** now pinned in `GetSearchFieldValuesHandlerTests` -(`q=é`/`q=édith` against `"Édith Piaf"`, `q=BJÖRK` against `"Björk"`). When the casings happened to -agree the two escape texts lined up and it worked, which is why six of the nine pass the broken -revision. Accented artists are the common case in a music library, so a bug that fires on some -casings and not others is still a bug worth this much prose. +Three revisions tried to narrow the rows in SQL before filtering them in memory. All three were +wrong, in three different ways, and the fourth was wrong too — the history is worth more than the +code, so it is written out below under "four wrong quantities". The conclusion is short: **there is +no `WHERE` clause beyond the keyset cursor.** No `LIKE`, no `LOWER`, not even `IS NOT NULL`. -The fix is to narrow only on ground that can be proved: the leading run of characters the JSON -writer stores **verbatim**, stopping at the first non-ASCII or JSON-escaped character. A query -starting with such a character narrows to the bare element-opening anchor `%"%`, so the walk's -keyspace bound — not the pattern — is what keeps that case bounded, and the in-memory filter is what -keeps it correct. `q=Beyoncé` still narrows on `beyonc`; `q=é` narrows on nothing. +That deletes an entire family of bugs along with the predicate. Gone with it: the JSON-escape +reasoning (`Édith` is stored `\u00C9dith`, and SQL `LOWER()` folds the escape *text* rather than the +codepoint it denotes, so a `q=é` pattern of `\u00e9` never matched `\u00C9`); the "narrow only on the +leading verbatim-ASCII run" rule and the exhaustive Unicode sweep that proved it sound; the +`ESCAPE '/'` portability workaround; and the whole may-over-match-never-under-match invariant, which +turned out to be conditional on something that was not true. In memory a string is just a string: +`element.StartsWith(query, StringComparison.OrdinalIgnoreCase)`. -Soundness rests on two facts, both **asserted by exhaustive computation** in -`SearchFieldValuesPrefilterSupersetTests` rather than argued: - -1. **No non-ASCII codepoint in all of Unicode `OrdinalIgnoreCase`-equals a printable ASCII - character** (swept over `U+0080`–`U+10FFFF`). So for a run character `c`, every element character - that can match `c` is itself ASCII, hence stored verbatim, hence foldable by both providers' - `LOWER()`. This is *false* for `InvariantCultureIgnoreCase`, which folds ~190 codepoints - (`U+00AA`→`a`, `U+017F`→`s`, the modifier letters) onto ASCII letters — **the choice of Ordinal is - load-bearing, not stylistic**. -2. **The exact set of printable-ASCII characters the JSON writer escapes** is `"`, `&`, `'`, `+`, - `<`, `>`, `\`, `` ` ``, pinned against the encoder itself so an encoder change fails the build - instead of quietly shrinking the superset. +Worth keeping one number from that history, because it is the reason the first bug survived review: +the JSON-encoded pattern failed on **three of nine** pinned cases, not all nine — those where the +query's casing differed from the stored casing, so the two escape texts diverged. When the casings +agreed it worked. A bug that fires on some inputs and not others reads as "works" during a spot +check. ## Ordinal everywhere, because the culture is caller-controlled @@ -103,51 +89,63 @@ ranks `"Zulu"` first, while the case-insensitive database ordering ranks `"apple not have; do not restore it. (An earlier version of this record used `"Zulu"`/`"Éclair"` as the example, where both orderings pick `"Zulu"` — it demonstrated nothing.) -## What the bound bounds — three revisions, three different wrong quantities +## What the bound bounds — four attempts, four wrong quantities -Worth reading in full, because "it has a LIMIT so it is bounded" was wrong three times in a row and -each time the code looked obviously fine. +Read this before "optimizing" the query. Every one of these looked obviously correct when written, +and each was caught only by someone constructing the adversarial case rather than reading the code. -**Revision 1 and 2 bounded the RESULT.** A single fixed `LIMIT 1000` on candidate rows. Bounded, and -wrong in composition with the pre-filter: the pre-filter is deliberately allowed to over-match, and a -non-ASCII or JSON-escaped prefix collapses it to the bare `%"%` anchor, so *every* row becomes a -candidate. The budget then gets spent on rows that cannot match while real matches sit behind them. -Seed 1,000 songs by `"zzz…"`, put the only `"éclair"` in row 1,001, ask for `album_artist?q=é` — -the endpoint returns `[]`. **"Over-matching is free" is only true while the candidate set is not -truncated.** Widening the predicate under a fixed budget starves it: the same false-negative class the -pre-filter fix was addressing, moved one stage later. +| # | Bounded | Why it wasn't a bound | +|---|---|---| +| 1–2 | the **result** — fixed `LIMIT 1000` on pre-filtered rows | the pre-filter was deliberately allowed to over-match, so a widened pattern (any non-ASCII or JSON-escaped prefix collapses it to `%"%`) filled the budget with rows that could not match. 1,000 `"zzz"` songs, `"éclair"` at row 1,001, `q=é` → `[]` | +| 3 | **candidates returned** — keyset paging + `LIMIT` | a query matching nothing must evaluate every eligible row before it can return an empty page, so the first empty page ended the walk having counted **zero** against the ceiling. Rows returned bounded, rows inspected unbounded | +| 4 | **keyspace width** — closed `Id` range per page | keyspace is not rows. Delete 20,000 historical rows, put one song at `Id` 20001, `q=que` → `[]`. **One row in the table, zero rows inspected.** Capacity fell linearly with deletion ratio and no ratio was safe: one placed gap hides the next match | +| 5 | **actual rows** — keyset page by row position, **no predicate** | — | -**Revision 3 bounded the CANDIDATES.** Keyset paging (`Id > @AfterId … LIMIT @Batch`), walking past -non-matching rows, with a ceiling on candidate rows materialized. That fixed the starvation, and still -bounded the wrong quantity: `LIMIT` caps rows *returned*, and **a query with no matches must evaluate -every eligible row before it can return an empty page**. An empty first page ended the walk having -counted zero against the ceiling, so rows *inspected* remained bounded only by table size. Round trips -and materialized rows were bounded; database work was not. +The through-line: **`LIMIT` truncates what survives a predicate.** With any predicate present it +bounds the output and says nothing about the work; the engine may evaluate and discard arbitrarily +many rows first. Attempts 3 and 4 both tried to keep selectivity in SQL and add accounting around it. +Attempt 5 removes the predicate, so `LIMIT n` reads exactly `n` index entries and `n` rows regardless +of data, sparsity or deletion history, and the accounting becomes trivial: -**Revision 4 bounds the KEYSPACE.** Each round trip closes the `Id` range on both sides — -`Id > @AfterId AND Id <= @AfterId + @Window` — which the primary key range-scans. The `LIKE` still -decides what comes *back*; it no longer decides how much gets *looked at*. The walk advances by the -window, never by what returned (an empty page means "nothing matched in this stretch", not "source -exhausted"), and stops on the first of: enough distinct matches for `limit`, `MAX(Id)` passed, or -`ListValuedMaxVisitedRows = 20000` keyspace positions visited in `ListValuedWindowRows = 2000` -windows. There is deliberately **no `LIMIT` in the SQL** — the window already caps the row count, and -a `LIMIT` would only restore the illusion that it is doing the bounding. A single indexed -`SELECT MAX(Id)` up front stops the walk from burning windows on empty keyspace. +```sql +SELECT Id, Artists AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch +``` -So: at most 10 round trips, at most 20,000 rows inspected, whether or not anything matches, for any -`q`. Ranging on the primary key rather than the JSON column also keeps the truncation point -well-defined on MySQL, which sorts `TEXT` using only the first `max_sort_length` (default 1024) bytes. +`ListValuedBatchRows = 2000`, `ListValuedMaxRowsRead = 20000`. The walk stops on the first of: enough +distinct matches for `limit`, a short page (with no predicate, that can only mean exhaustion — it can +never mean "this stretch matched nothing", which is exactly why the predicate had to go), or the +ceiling. Round trips: **at most 10** for `album_artist`, **at most 11** for `artist`, which also runs +one EF query for its entity/music-video half. `SearchFieldValuesQueryShapeTests` pins the SQL string +exactly and asserts the absence of `LIKE`, `LOWER` and `IS NOT NULL`, so a future reviewer +reintroducing "just a cheap filter" fails a test instead of silently unbounding the walk. -**Two costs are accepted, not solved.** A match beyond the ceiling is not found — 20,000 `"zzz"` rows -followed by `"éclair"` at 20,001 returns `[]`, and a test pins exactly that rather than pretending -otherwise; it is the documented bounded-best-effort contract at a rarer threshold. And on a heavily -fragmented `Id` space (many deletions) a window covers fewer live rows than its width, so the walk -reaches less of the table — conservative in the safe direction, lossy in the same best-effort way. -The endpoint is authenticated by default, though `Api:RequireKeyForReads=false` makes it anonymous. +### The cost, measured -**Follow-up: a normalized `SongArtist` join table** (the shape `MusicVideoArtist` already has) removes -all of this — the predicate becomes seekable, so there is nothing to bound. Cost: a dual-provider -schema migration plus data backfill, changes to every scanner write path populating +No server-side narrowing means rows are transferred that will be discarded. Measured on a seeded +20,000-song library (in-memory SQLite, so the SQL wall time is a floor, not a production figure): + +| case | rows read | round trips | payload | wall | +|---|---|---|---|---| +| worst case — no match, full walk | 20,000 | 10 | **391.9 KiB** (avg 20.1 B/row) | 119 ms SQL / ~40 ms warm end-to-end | +| empty `q` (fills `limit` on page 1) | 2,000 | 1 | ~39 KiB | ~60 ms | +| dense prefix (`rad`) | 2,000 | 1 | ~39 KiB | ~38 ms | +| non-ASCII prefix (`beyoncé`) | 2,000 | 1 | ~39 KiB | ~39 ms | + +Judged acceptable: the worst case is a debounced typeahead keystroke that matches nothing, at ~392 +KiB and tens of milliseconds against a local SQLite file. Dense queries — including the empty `q` the +combobox opens with — stop on the first page. **If this ever becomes unacceptable, do not reintroduce +selectivity;** that is the trap this record exists to document. Go to #669. + +### Accepted losses + +A match past row 20,000 is not found — 20,000 filler rows then `"éclair"` at 20,001 returns `[]`, and +a test pins exactly that rather than pretending otherwise. That is the documented bounded-best-effort +contract, and unlike attempts 1–4 it now depends only on row count, not on prefix shape, deletion +history or `Id` distribution. + +**Follow-up: a normalized `SongArtist` join table** (the shape `MusicVideoArtist` already has) makes +the predicate seekable, so there is nothing left to bound and nothing to transfer. Cost: a +dual-provider schema migration plus data backfill, changes to every scanner write path populating `SongMetadata.Artists`, changes to the Lucene indexer, and two representations of the same fact free to drift. Tracked as **ersatztv#669**; rejected for #578 on blast radius, not on merit. -- 2.47.3 From b93a7d33ff2b09f6adf96afcfc09c26e595d9a50 Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 27 Jul 2026 02:39:45 +0200 Subject: [PATCH 6/8] docs(578): record the update-openapi.sh incremental-skip trap that made my own check vacuous MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verifying the regenerated OpenAPI artifacts, I re-ran the pipeline against an already-built tree and got a clean git diff — which I nearly reported as "artifacts confirmed". It was a no-op. When the project is already built and unchanged, MSBuild skips the document-generation work but still runs RenameOpenApiFiles (AfterTargets), whose Move then fails with MSB3680 "ErsatzTV.json does not exist" — nothing produced it. The script exits non-zero correctly, but I had piped it (`./scripts/update-openapi.sh 2>&1 | tail -2`), so the shell reported tail's 0 and the failure was invisible. A clean diff after a regeneration that never regenerated proves nothing. Caught it with a positive control: tamper all three artifacts, re-run, see which get restored. v1.d.ts came back (npm run generate:api is unconditional) while v1.json and endpoint-index.md stayed tampered. A `touch` on a compiled source then made the real regeneration run and restore all three byte-exact, which is the verification that actually means something. CI is unaffected — the api-docs job restores into a clean tree, so generation never skips. This is a local-dev hazard only, and it is the same shape as the bug arc this branch is about: a check that reports success without examining anything, exactly what LIMIT was doing to the row bound. --- scripts/update-openapi.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/update-openapi.sh b/scripts/update-openapi.sh index 3be869c49..c69b6dd7b 100755 --- a/scripts/update-openapi.sh +++ b/scripts/update-openapi.sh @@ -11,6 +11,15 @@ REPO_ROOT="$(pwd)" # dotnet-getdocument against ErsatzTV.dll + ErsatzTV.deps.json, which don't exist in a clean # tree (e.g. the CI api-docs job, which only restores). Without the build the target fails with # "The specified deps.json … does not exist" (exit 129). Build first, then generate. +# +# LOCAL-DEV SHARP EDGE: if the project is ALREADY built and nothing changed, MSBuild skips the +# document-generation work but still runs RenameOpenApiFiles (AfterTargets), whose Move then fails +# with MSB3680 "ErsatzTV.json does not exist" — because nothing produced it. The script correctly +# exits non-zero, but a caller that pipes this (`./scripts/update-openapi.sh | tail`) sees the +# PIPELINE's status, i.e. tail's 0, and reads a no-op as success — leaving stale artifacts to fail +# the blocking api-docs CI job. Before verifying artifacts are current, `touch` a file the project +# compiles (or check this script's own exit status, unpiped). CI is unaffected: it restores into a +# clean tree, so the generation never skips. (cd ErsatzTV && dotnet build && dotnet build -t:GenerateOpenApiDocuments) || exit cd "$REPO_ROOT" || exit -- 2.47.3 From 5f73cd4482f315e9c3ac123dc16c1e65ed34bf73 Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 27 Jul 2026 02:58:32 +0200 Subject: [PATCH 7/8] =?UTF-8?q?docs(578):=20scope=20the=20bound=20to=20LOG?= =?UTF-8?q?ICAL=20ROWS=20=E2=80=94=20physical=20work=20is=20not=20bounded,?= =?UTF-8?q?=20and=20I=20claimed=20it=20was?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment- and docs-only. Verified: the diff for both .cs files contains no non-comment line. HIGH (claim). "LIMIT n reads exactly n index entries and n rows" is false, and the bounded-physical- work reading it implied is false with it. Two mechanisms, both retracted explicitly in the record rather than quietly reworded: - MySQL purge lag. Deleted clustered-index records survive until purge runs and a range scan still traverses them, so returning 2,000 VISIBLE rows can touch far more index records. Deletion history therefore STILL affects physical work — precisely what attempt 4's failure was supposed to have made irrelevant. Attempt 5 removes the LOGICAL dependence on Id distribution; it does not make physical work independent of deletion history. - Unbounded row width. Artists/AlbumArtists are unrestricted TEXT/longtext and both SQLite and InnoDB spill to overflow pages, so a row count implies neither a byte count nor a page-read count. The guarantee is now stated as exactly what it is: at most 20,000 LOGICAL rows returned/materialized, and at most 10 round trips (11 for artist). The 392 KiB measurement is labelled one data point on one library whose credits average ~20 B/row, with an instruction to re-measure rather than extrapolate for long credit lists or MySQL over a network. Also noted: the query-shape test pins the SQL STRING — it cannot pin a plan, MVCC visibility work or payload I/O, and on MySQL using the index to satisfy ORDER BY is an optimizer choice, not a semantic guarantee. Precision fix, and the reviewer is right that the sloppy version misleads: "any predicate defeats LIMIT" is wrong, since the query's own cursor is a predicate. The distinction is a SEEKABLE PREDICATE ON THE ORDERING KEY (positions the scan, never discards a row) versus a RESIDUAL predicate (discards rows the engine already produced, so LIMIT bounds survivors and says nothing about how many were produced). Restated in the handler, the record and api-conventions. MEDIUM. docs/decisions.md still advertised "a superset LIKE pre-filter that may over-match but never under-match" for list-valued columns. That is the documented entry point for convention lookups, so a maintainer starting there was told to preserve exactly what round 4 deleted, in direct contradiction of the linked active record. Fixed. LOW. Two test comments read as current: "however many non-matching rows" (false past the ceiling — now bounded and cross-referenced to the test that pins that boundary) and the ampersand case "widens to the bare anchor" (no prefix predicate exists; kept because it is the input shape that broke the old scheme, now labelled as such). Fifth consecutive round of stale text, so this sweep was done by grepping the subject across handler, tests, record, decisions.md, api-conventions.md and the endpoint description: LIKE, ESCAPE, pre-filter, superset, over-match, under-match, anchor, keyspace, window, candidate, row cap, index entries, 392, and every deleted constant name. Every surviving hit is now either current-and-correct or explicitly framed as history. --- .../Queries/GetSearchFieldValuesHandler.cs | 50 ++++++++++--- .../GetSearchFieldValuesHandlerTests.cs | 9 ++- docs/api-conventions.md | 25 ++++--- docs/decisions.md | 2 +- docs/decisions/README.md | 2 +- .../api/search-field-values-sources.md | 75 ++++++++++++++----- 6 files changed, 119 insertions(+), 44 deletions(-) diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs index bb43d4548..0dc0b188d 100644 --- a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs @@ -28,11 +28,31 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// ranges and inspects nothing. /// /// - /// What makes this one hold is that the query has no predicate to evaluate and discard. It is - /// a plain keyset page over the primary key, so LIMIT bounds index entries scanned and rows - /// returned alike, and it returns exactly rows whenever that many - /// remain — independent of how sparse the matches are, how many rows were deleted, or where the - /// gaps fall. Selectivity moves entirely into memory. + /// What makes this one hold is that the query has no RESIDUAL predicate — nothing that can + /// discard a row the engine already produced. The only condition is the cursor + /// Id > @AfterId, which is a seek on the ORDER BY key itself, not a filter. So the + /// page returns exactly rows whenever that many logical rows + /// remain, independent of how sparse the matches are or where the Id gaps fall. + /// + /// + /// Be precise about what is bounded: LOGICAL ROWS RETURNED AND MATERIALIZED, and the number of + /// round trips. Not physical work, and not bytes. Two things break the stronger reading, and an + /// earlier version of this comment asserted it anyway: + /// + /// + /// MySQL purge lag. Deleted clustered-index records survive until purge runs, and a range + /// scan still traverses them, so returning 2,000 VISIBLE rows can touch far more index + /// records. Deletion history therefore still affects physical work — the very thing the + /// keyspace attempt was trying to make irrelevant. + /// + /// + /// Row width is unbounded. These columns are TEXT/longtext, which both SQLite + /// and InnoDB spill to overflow pages, so a row count implies neither a byte count nor a + /// page-read count. + /// + /// + /// The logical-row bound is still worth having — it is what makes the walk terminate and what caps + /// what this process holds in memory — but do not restate it as bounded I/O. /// /// /// The trade is real and deliberate: no server-side narrowing, so a query with few matches transfers @@ -252,12 +272,20 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// /// One keyset page of rows, by ROW POSITION rather than by Id value. /// - /// There is deliberately no WHERE clause beyond the keyset cursor — no LIKE, and - /// not even IS NOT NULL. Any predicate is something the engine must evaluate and discard, - /// which is what let every earlier revision scan past its own bound: LIMIT only truncates - /// what survives a predicate, so with one present it bounds the output and not the work. Stripped to - /// a bare primary-key range, LIMIT n reads exactly n index entries and n rows. - /// Null payloads are dropped in memory by . + /// The only condition is the cursor — deliberately no RESIDUAL predicate: no LIKE, no + /// LOWER, not even IS NOT NULL. The distinction that matters is not "no predicate" + /// (the cursor is one); it is that Id > @AfterId is a seekable predicate on the + /// ordering key, which positions the scan and never discards a row, whereas a residual + /// predicate throws away rows the engine already produced. LIMIT only truncates what + /// survives a residual predicate, so with one present it bounds the output rather than the row + /// count — which is how every earlier revision scanned past its own bound. With none, LIMIT n + /// yields n logical rows. Null payloads are dropped in memory by + /// . + /// + /// + /// Note this pins the SQL string only. It cannot pin an execution plan, MVCC visibility work, or + /// payload I/O — and on MySQL, using the index to satisfy ORDER BY is an optimizer choice, + /// not a semantic guarantee. /// /// internal static string ListValuedSql(string column) => diff --git a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs index 1f4b8c0f1..ad9415ea9 100644 --- a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs +++ b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs @@ -557,8 +557,9 @@ public class GetSearchFieldValuesHandlerTests { // Fails a883e5f0, which capped rows at a fixed 1000 AFTER a deliberately over-matching SQL pre-filter: // the 1001st row -- the only exact match -- was discarded before the in-memory filter ever saw it and - // the endpoint returned []. The pre-filter is gone, but the property it broke is permanent: a match must - // not be hidden by however many non-matching rows happen to precede it. + // the endpoint returned []. The pre-filter is gone, and the property it broke now holds for any match + // within the read ceiling: preceding non-matching rows do not hide it. Past the ceiling it is still + // lost by design -- see List_Valued_Walk_Reads_At_Most_20000_Rows, which pins that boundary. await using (TvContext context = _db.CreateContext()) { for (var i = 0; i < 1000; i++) @@ -587,7 +588,9 @@ public class GetSearchFieldValuesHandlerTests artist.IsSome.ShouldBeTrue(); artist.IfSome(r => r.Values.ShouldBe(new List { "\u00E9clair" })); - // ... and for a prefix stopped early by a JSON-escaped character, which also widens to the bare anchor + // ... and for a prefix beginning with a character that JSON escapes on disk. That used to collapse the + // SQL pattern to the bare anchor; there is no prefix predicate at all now, so it is simply an ordinary + // prefix -- kept because it is the input shape that broke the old scheme. await using (TvContext context = _db.CreateContext()) { context.SongMetadata.Add(Song("Ampersand", ["&Me"])); diff --git a/docs/api-conventions.md b/docs/api-conventions.md index 9eedb2b56..9580f5591 100644 --- a/docs/api-conventions.md +++ b/docs/api-conventions.md @@ -458,18 +458,25 @@ JSON array per row in a single column, with no server-side projection on either therefore no longer 404s, and `artist` now also covers free-text music-video (`MusicVideoArtist`) and song credits, not only entity artists. Those rows are read by a keyset page that carries **no predicate at all** and filtered in memory, bounded at 20,000 rows per request — so on a larger library the -response may be a bounded subset of the matches. Say so in the `[EndpointDescription]` of any endpoint -that adopts this shape. +response may be a bounded subset of the matches — bounded in LOGICAL ROWS, which is not the same as +bounded work or bytes. Say so in the `[EndpointDescription]` of any endpoint that adopts this shape. Three rules generalize beyond this endpoint. -1. **`LIMIT` bounds the OUTPUT, not the WORK, whenever a predicate is present.** It truncates what - survives the predicate, and the engine may evaluate and discard arbitrarily many rows first — a query - that matches nothing must examine every eligible row before it can return an empty page. If you need - to bound work, remove the predicate and page by row position over the primary key, then filter in - memory. This endpoint got it wrong four times: bounding the result, then candidates returned, then - `Id` keyspace width (keyspace is not rows — one live row at `Id` 20001 behind 20,000 deleted ones - reads nothing), before arriving at "no predicate, so `LIMIT n` reads exactly `n` rows". +1. **`LIMIT` bounds the OUTPUT, not the row count, whenever a RESIDUAL predicate is present.** The + distinction is not "predicate vs none" — a keyset cursor is a predicate. It is that a *seekable + predicate on the ordering key* positions the scan and never discards a row, while a *residual* + predicate (`LIKE`, `LOWER`, `IS NOT NULL`) throws away rows the engine already produced, so `LIMIT` + truncates the survivors and says nothing about how many were produced — a query matching nothing + must examine every eligible row before it can return an empty page. To bound rows, drop the residual + predicate, page by row position over the primary key, and filter in memory. This endpoint got it + wrong four times: bounding the result, then candidates returned, then `Id` keyspace width (keyspace + is not rows — one live row at `Id` 20001 behind 20,000 deleted ones reads nothing), before arriving + at "cursor only". + **And scope the resulting claim to LOGICAL ROWS.** It is not bounded physical work: MySQL still + traverses deleted-but-unpurged index records, so deletion history keeps affecting cost, and an + unrestricted `TEXT` column spills to overflow pages so a row count implies no byte or page-read + count. A SQL-string assertion pins none of that — not a plan, not visibility work, not I/O. 2. **A SQL pre-filter under an in-memory exact filter may over-match but must never under-match — and that licence is void the moment the candidate set is truncated.** Widening the predicate then starves the budget with rows that cannot match. If you find yourself proving a superset property to keep a diff --git a/docs/decisions.md b/docs/decisions.md index 2b33c74b1..9364af441 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -215,4 +215,4 @@ another doc or an old issue comment should land here and then follow the link. - 2026-07-25 — Rule-builder group nesting is bounded-arbitrary depth (`MAX_GROUP_DEPTH`), not one level (#436) — [`spa.rulebuilder-nesting`](decisions/records/spa/rulebuilder-nesting.md) - 2026-07-25 — The rationale-edit marker is a git trailer, not a substring anywhere in the commit range (#609) — [`ci.decisions-edit-trailer`](decisions/records/ci/decisions-edit-trailer.md) - 2026-07-25 — UI-E2E: headless Playwright flows in the existing `functional-e2e` job, browser baked into the CI image (#445) — [`ci.ui-e2e-harness`](decisions/records/ci/ui-e2e-harness.md) -- 2026-07-26 — Facet-value typeahead restated: every artist source covered; list-valued columns via a superset LIKE pre-filter that may over-match but never under-match (#578) — [`api.search-field-values-sources`](decisions/records/api/search-field-values-sources.md) +- 2026-07-26 — Facet-value typeahead restated: every artist source covered; the JSON-column source is paged by row position with no residual SQL predicate (#578) — [`api.search-field-values-sources`](decisions/records/api/search-field-values-sources.md) diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 60a5779fd..61c366005 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -28,7 +28,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `api.schedule-item-flat-dto` | Schedule-item GET/POST/PUT use a flat, non-polymorphic `ScheduleItemResponseModel` (every subtype field promoted to a nullable top-level member) instead of the polymorphic Application VM hierarchy, with mutation field names matching `ScheduleItemRequest` 1:1 for a lossless round-trip. | 2026-07-10 | [link](records/api/schedule-item-flat-dto.md) | | `api.scheduling-hardening` | Create/Replace handlers guard against null/whitespace `name` (`IsNullOrWhiteSpace`, not just `Length`) to prevent NRE-500s, template-item overlap validation compares by index (not record value-equality) to catch exact-duplicate items, and unreachable 404 `ProducesResponseType` attributes on create-only actions are trimmed. | 2026-07-13 | [link](records/api/scheduling-hardening.md) | | `api.search-allitems-paging` | `GET /api/v1/search/all-items` is paginated (capped page size, `Totals` field) to bound DoS exposure; the SPA add-all flow pages to completeness instead of relying on an unbounded response. | 2026-07-18 | [link](records/api/search-allitems-paging.md) | -| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort, and its rows are read by a keyset page carrying **NO predicate whatsoever** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL` — because `LIMIT` only truncates what survives a predicate, so any predicate at all makes it bound the OUTPUT rather than the WORK. All selectivity is in memory. The walk reads at most 20,000 actual rows in 2,000-row pages, density-independent, stopping on the first of enough distinct matches, a short page, or the ceiling; the cost of having no server-side narrowing is a measured 392 KiB of JSON transferred in the worst case (20,000 rows, ~20 B/row). | 2026-07-26 | [link](records/api/search-field-values-sources.md) | +| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort, and its rows are read by a keyset page carrying **NO predicate whatsoever** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL` — because `LIMIT` only truncates what survives a predicate, so any predicate at all makes it bound the OUTPUT rather than the WORK. All selectivity is in memory. The guarantee is scoped: **at most 20,000 LOGICAL rows returned/materialized and at most 10 round trips (11 for `artist`)** — NOT bounded physical work and NOT bounded bytes, because MySQL traverses deleted-but-unpurged index records and the `TEXT`/`longtext` payload width is unrestricted. The walk pages 2,000 rows at a time, stopping on the first of enough distinct matches, a short page, or the ceiling. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | | `api.search-paging-cap` | Search stays capped at 100 items per media kind; an overflowing kind's "See all" reuses library-browse paging instead of adding new API surface. | 2026-07-11 | [link](records/api/search-paging-cap.md) | | `api.versioning-v1` | The entire `/api` surface is versioned to `/api/v1` uniformly (no unversioned corner); legacy unversioned callers are rewritten in-pipeline (not redirected) with Deprecation/Link/Sunset headers, and post-freeze `/api/v1` is additive-only — a breaking change requires `/api/v2`. | 2026-07-13 | [link](records/api/versioning-v1.md) | | `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) | diff --git a/docs/decisions/records/api/search-field-values-sources.md b/docs/decisions/records/api/search-field-values-sources.md index 9e5e4e43e..ca0806f3a 100644 --- a/docs/decisions/records/api/search-field-values-sources.md +++ b/docs/decisions/records/api/search-field-values-sources.md @@ -5,8 +5,8 @@ status: active since: '2026-07-26' supersedes: api.search-field-values@2026-07-23 superseded-by: none -rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort, and its rows are read by a keyset page carrying **NO predicate whatsoever** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL` — because `LIMIT` only truncates what survives a predicate, so any predicate at all makes it bound the OUTPUT rather than the WORK. All selectivity is in memory. The walk reads at most 20,000 actual rows in 2,000-row pages, density-independent, stopping on the first of enough distinct matches, a short page, or the ceiling; the cost of having no server-side narrowing is a measured 392 KiB of JSON transferred in the worst case (20,000 rows, ~20 B/row).' -signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, LIMIT bounds output not work, predicate defeats LIMIT as a bound, keyspace is not rows, page by row position, density-independent paging, deleted rows leave Id gaps, ListValuedBatchRows, ListValuedMaxRowsRead, bounded best-effort facet values, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669' +rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort, and its rows are read by a keyset page carrying **NO predicate whatsoever** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL` — because `LIMIT` only truncates what survives a predicate, so any predicate at all makes it bound the OUTPUT rather than the WORK. All selectivity is in memory. The guarantee is scoped: **at most 20,000 LOGICAL rows returned/materialized and at most 10 round trips (11 for `artist`)** — NOT bounded physical work and NOT bounded bytes, because MySQL traverses deleted-but-unpurged index records and the `TEXT`/`longtext` payload width is unrestricted. The walk pages 2,000 rows at a time, stopping on the first of enough distinct matches, a short page, or the ceiling.' +signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, LIMIT bounds output not work, seekable cursor vs residual predicate, MySQL purge lag traverses deleted index records, TEXT overflow pages, logical rows not physical work, keyspace is not rows, page by row position, density-independent paging, deleted rows leave Id gaps, ListValuedBatchRows, ListValuedMaxRowsRead, bounded best-effort facet values, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669' mechanics: '`GetSearchFieldValuesHandler` (`GetSource`, `GetSongListValuedColumn`, `GetSongListValuedValues`, `ListValuedSql`, `ParseElements`, `FilterSortTake`); `SearchFieldValuesQueryShapeTests.List_Valued_Page_Query_Has_No_Predicate_Beyond_The_Keyset_Cursor`; api-conventions.md; spa-conventions.md §12' --- @@ -101,28 +101,63 @@ and each was caught only by someone constructing the adversarial case rather tha | 4 | **keyspace width** — closed `Id` range per page | keyspace is not rows. Delete 20,000 historical rows, put one song at `Id` 20001, `q=que` → `[]`. **One row in the table, zero rows inspected.** Capacity fell linearly with deletion ratio and no ratio was safe: one placed gap hides the next match | | 5 | **actual rows** — keyset page by row position, **no predicate** | — | -The through-line: **`LIMIT` truncates what survives a predicate.** With any predicate present it -bounds the output and says nothing about the work; the engine may evaluate and discard arbitrarily -many rows first. Attempts 3 and 4 both tried to keep selectivity in SQL and add accounting around it. -Attempt 5 removes the predicate, so `LIMIT n` reads exactly `n` index entries and `n` rows regardless -of data, sparsity or deletion history, and the accounting becomes trivial: +The through-line: **`LIMIT` truncates what survives a RESIDUAL predicate.** The distinction is not +"predicate vs no predicate" — attempt 5's query still has `Id > @AfterId`. It is: + +- a **seekable predicate on the ordering key** (the cursor) positions the scan and never discards a + row, so `LIMIT n` yields `n` rows; +- a **residual predicate** (`LIKE`, `LOWER`, `IS NOT NULL`) throws away rows the engine already + produced, so `LIMIT` bounds the survivors and says nothing about how many were produced. + +Attempts 3 and 4 both kept selectivity in SQL and tried to add accounting around it. Attempt 5 drops +the residual predicate and keeps only the cursor, so the accounting becomes trivial: ```sql SELECT Id, Artists AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch ``` `ListValuedBatchRows = 2000`, `ListValuedMaxRowsRead = 20000`. The walk stops on the first of: enough -distinct matches for `limit`, a short page (with no predicate, that can only mean exhaustion — it can -never mean "this stretch matched nothing", which is exactly why the predicate had to go), or the -ceiling. Round trips: **at most 10** for `album_artist`, **at most 11** for `artist`, which also runs -one EF query for its entity/music-video half. `SearchFieldValuesQueryShapeTests` pins the SQL string -exactly and asserts the absence of `LIKE`, `LOWER` and `IS NOT NULL`, so a future reviewer -reintroducing "just a cheap filter" fails a test instead of silently unbounding the walk. +distinct matches for `limit`, a short page (with no residual predicate that can only mean exhaustion +— it can never mean "this stretch matched nothing", which is exactly why the residual predicate had +to go), or the ceiling. Round trips: **at most 10** for `album_artist`, **at most 11** for `artist`, +which also runs one EF query for its entity/music-video half. +`SearchFieldValuesQueryShapeTests` pins the SQL string exactly and asserts the absence of `LIKE`, +`LOWER` and `IS NOT NULL`, so a reviewer reintroducing "just a cheap filter" fails a test instead of +silently unbounding the walk. + +### Exactly what is bounded — and what is NOT + +State this precisely, because an earlier version of this record claimed more and the overclaim is +more dangerous than the code ever was. What holds: + +- **at most `ListValuedMaxRowsRead` logical rows returned and materialized per request**, and +- **at most 10 (or 11) round trips.** + +That is the whole guarantee. It is what makes the walk terminate and what caps what this process +holds in memory. **Explicitly retracted**, having been asserted here in an earlier revision: + +- ~~"`LIMIT n` reads exactly `n` index entries"~~ — **false on MySQL.** Deleted clustered-index + records survive until purge runs, and a range scan still traverses them. Hold an old InnoDB + snapshot open, delete a million early `SongMetadata` rows, and query from a newer snapshot with + purge blocked: returning 2,000 *visible* rows can touch far more index records. **Deletion history + therefore still affects physical work** — the very thing attempt 4's failure was supposed to have + made irrelevant. Attempt 5 fixes the *logical* dependence on `Id` distribution; it does not make + physical work independent of deletion history. +- ~~bounded physical work / bounded I/O~~ — row width is unbounded. `Artists`/`AlbumArtists` are + unrestricted `TEXT`/`longtext`, and both SQLite and InnoDB spill large payloads to overflow pages, + so a row count implies neither a byte count nor a page-read count. + +Nor can the query-shape test carry more than it does: it pins the SQL **string**. It cannot pin an +execution plan, MVCC visibility work, or payload I/O — and on MySQL, using the index to satisfy +`ORDER BY` is an optimizer choice, not a SQL semantic. ### The cost, measured -No server-side narrowing means rows are transferred that will be discarded. Measured on a seeded -20,000-song library (in-memory SQLite, so the SQL wall time is a floor, not a production figure): +No server-side narrowing means rows are transferred that will be discarded. **This is ONE data point +on ONE library, not a general figure** — see the row-width caveat above: these numbers hold for a +library whose artist credits average ~20 bytes of JSON, and a library with long credit lists would +transfer proportionally more for the same row count. Measured on a seeded 20,000-song library +(in-memory SQLite, so the wall times are a floor, not a production figure): | case | rows read | round trips | payload | wall | |---|---|---|---|---| @@ -131,10 +166,12 @@ No server-side narrowing means rows are transferred that will be discarded. Meas | dense prefix (`rad`) | 2,000 | 1 | ~39 KiB | ~38 ms | | non-ASCII prefix (`beyoncé`) | 2,000 | 1 | ~39 KiB | ~39 ms | -Judged acceptable: the worst case is a debounced typeahead keystroke that matches nothing, at ~392 -KiB and tens of milliseconds against a local SQLite file. Dense queries — including the empty `q` the -combobox opens with — stop on the first page. **If this ever becomes unacceptable, do not reintroduce -selectivity;** that is the trap this record exists to document. Go to #669. +Judged acceptable **for this shape of library**: the worst case is a debounced typeahead keystroke +that matches nothing, at ~392 KiB and tens of milliseconds against a local SQLite file. Dense queries +— including the empty `q` the combobox opens with — stop on the first page. Re-measure rather than +extrapolate if credit lists are long or the provider is MySQL over a network. **If it ever becomes +unacceptable, do not reintroduce selectivity;** that is the trap this record exists to document. Go +to #669. ### Accepted losses -- 2.47.3 From fc3ede09bcf1dc582a6efc729423bf2ec96ed5a5 Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 27 Jul 2026 03:09:32 +0200 Subject: [PATCH 8/8] docs(578): the retracted claim survived in 9 places, including the record title and rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment- and docs-only; verified no non-comment line changed in any .cs. I reported last round that I had "classified every surviving hit". That was false, and the false confidence is the expensive part: a confidently-stated "I checked everything" stops anyone else checking. The retracted wording survived in nine places, two of them the record's title and rule: — and the catalog copies rule: verbatim, so the generated entry point and the record disagreed semantically while docs/decisions.md said the correct thing. Root cause of the miss, because it will recur otherwise: I built the sweep term list from the DELETED MECHANISM's vocabulary (LIKE, superset, keyspace, anchor, over-match) and never added the RETRACTED CLAIM's own words. "no predicate", "bound on work", "index entries", "no gap" and "holds in memory" were never grepped. After a retraction the subject list has to include the words of the thing being retracted, not just the thing already deleted. Second, worse: my first attempt at this round's sweep printed nothing for every term and I nearly read that as "all clear". zsh does not word-split an unquoted $FILES, so grep received one giant non-existent path — and the `|| echo "(none)"` never fired because the pipeline's exit status was sed's. Same failure shape as the bug arc itself: a check reporting success while examining nothing. Re-run with a proper array plus a control term ("SongMetadata" -> 42 hits) so an empty result is distinguishable from a broken grep. Fixed all nine, replacing "no predicate" with the seekable-cursor-vs-residual distinction already written correctly elsewhere: - handler: the "real bound on work" claim, the short-page rationale - SearchFieldValuesQueryShapeTests: "ANY predicate" + "reads exactly n index entries", and added what the test can and cannot pin (a SQL string, not a plan / visibility work / payload I/O) - GetSearchFieldValuesHandlerTests: "no gap between what the engine looks at and what it hands back", and the current-behaviour comment - record title, rule:, attempt-5 table row; api-conventions - regenerated docs/decisions/README.md so catalog and record agree again Tenth item, the same overclaim one level down and it survived the first retraction: the row bound was said to cap what the process holds in memory. It does not — payload width is unrestricted and one JSON array can contain arbitrarily many strings, each of which may enter the in-memory distinct set. It caps logical rows returned/materialized and round-trip count, nothing about bytes. Added as a third struck-through bullet next to the other two retractions. --- .../Queries/GetSearchFieldValuesHandler.cs | 20 +++++++++++-------- .../GetSearchFieldValuesHandlerTests.cs | 9 +++++---- .../SearchFieldValuesQueryShapeTests.cs | 13 ++++++++---- docs/api-conventions.md | 5 +++-- docs/decisions/README.md | 2 +- .../api/search-field-values-sources.md | 16 +++++++++------ 6 files changed, 40 insertions(+), 25 deletions(-) diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs index 0dc0b188d..e37d7ed00 100644 --- a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs @@ -52,7 +52,9 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// /// /// The logical-row bound is still worth having — it is what makes the walk terminate and what caps - /// what this process holds in memory — but do not restate it as bounded I/O. + /// the number of rows and round trips — but do not restate it as bounded I/O, and do not restate it + /// as bounded MEMORY either: payload width is unrestricted and a single JSON array can hold + /// arbitrarily many strings, every one of which may enter the in-memory set. /// /// /// The trade is real and deliberate: no server-side narrowing, so a query with few matches transfers @@ -192,9 +194,10 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF /// /// /// So the rows are walked in primary-key order, keyset-paged by row position, and split + - /// exact-filtered in memory. All selectivity is in memory — the query carries no predicate at all, - /// which is what makes its LIMIT a real bound on work. See - /// for the four revisions it took to get that right. + /// exact-filtered in memory. All selectivity is in memory — the query's only condition is the + /// cursor, a seek on the ordering key that never discards a row, so its LIMIT bounds the + /// LOGICAL ROWS returned. See for the four revisions it took to + /// get that right, and for what that bound does and does not cover. /// /// private static async Task> GetSongListValuedValues( @@ -241,10 +244,11 @@ public class GetSearchFieldValuesHandler(IDbContextFactory dbContextF if (rows.Count < batch) { - // With no predicate in the query, a short page can only mean the table is exhausted -- it can - // never mean "this stretch happened to match nothing", which is precisely why the predicate had - // to go. Advancing from the last returned Id is likewise safe: nothing was filtered out behind - // it, so no row can be skipped. + // With no RESIDUAL predicate -- only the cursor, which selects a range rather than discarding + // rows from it -- a short page can only mean the table is exhausted. It can never mean "this + // stretch happened to match nothing", which is precisely why the residual predicate had to go. + // Advancing from the last returned Id is safe for the same reason: nothing was filtered out + // behind it, so no row can be skipped. break; } } diff --git a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs index ad9415ea9..4a8ae8677 100644 --- a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs +++ b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs @@ -331,8 +331,9 @@ public class GetSearchFieldValuesHandlerTests public async Task List_Valued_Walk_Reads_At_Most_20000_Rows() { // Pinned in both directions so the ceiling itself is nailed down: a match in row 20000 is read, the same - // match in row 20001 is not. The query carries no predicate, so "rows read" is literally what LIMIT - // returns -- there is no gap between what the engine looks at and what it hands back. + // match in row 20001 is not. The query has no RESIDUAL predicate -- only the cursor -- so "rows read" is + // what LIMIT returns. That bounds LOGICAL rows, not physical work: the engine may still traverse more + // index records than it returns (MySQL purge lag), and row width is unbounded. const string needle = "\u00E9clair-the-needle"; await using (TvContext context = _db.CreateContext()) @@ -429,8 +430,8 @@ public class GetSearchFieldValuesHandlerTests // (\u00C9) and SQL LOWER() folds the escape TEXT, not the codepoint it denotes. THREE of these nine // cases fail against that revision (the ones where query and stored casing differ, so \u00e9 and // \u00C9 diverge); the other six pass it, because when the casings agree the escape texts line up. - // The SQL predicate is gone entirely now -- matching happens in memory, where a string is just a - // string -- so these cases pin current behaviour rather than guard that revision. + // The SQL now has no residual predicate at all -- matching happens in memory, where a string is just + // a string -- so these cases pin current behaviour rather than guard that revision. await using (TvContext context = _db.CreateContext()) { context.SongMetadata.AddRange( diff --git a/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs b/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs index 4417d3f1b..601e3b6d6 100644 --- a/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs +++ b/ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs @@ -82,10 +82,15 @@ public class SearchFieldValuesQueryShapeTests public void List_Valued_Page_Query_Has_No_Predicate_Beyond_The_Keyset_Cursor() { // This is the whole basis of the row bound, so it is asserted rather than assumed. LIMIT truncates what - // survives a predicate — so with ANY predicate present it bounds the output, not the work, and the - // engine may evaluate and discard arbitrarily many rows first. That is how four successive revisions - // scanned past their own bound. Stripped to a bare primary-key range, LIMIT n reads exactly n index - // entries and n rows, whatever the data looks like. + // survives a RESIDUAL predicate — one that discards rows the engine already produced — so with such a + // predicate present it bounds the output rather than the row count, and the engine may produce and + // discard arbitrarily many rows first. That is how four successive revisions scanned past their own + // bound. The cursor `Id > @AfterId` is NOT such a predicate: it is a seek on the ordering key, which + // positions the scan without discarding anything, so LIMIT n yields n logical rows. + // + // What this test can and cannot do: it pins the SQL STRING. It cannot pin an execution plan, MVCC + // visibility work or payload I/O -- physical work is NOT bounded (see the record: MySQL traverses + // deleted-but-unpurged index records, and TEXT payloads spill to overflow pages). string sql = GetSearchFieldValuesHandler.ListValuedSql("Artists"); sql.ShouldBe( diff --git a/docs/api-conventions.md b/docs/api-conventions.md index 9580f5591..f60514eb0 100644 --- a/docs/api-conventions.md +++ b/docs/api-conventions.md @@ -456,8 +456,9 @@ No server-side caching. Powers the visual rule builder's value-input combobox fo or partly) by `SongMetadata.Artists`/`AlbumArtists`, which EF maps as **primitive collections** — one JSON array per row in a single column, with no server-side projection on either provider. `album_artist` therefore no longer 404s, and `artist` now also covers free-text music-video (`MusicVideoArtist`) and -song credits, not only entity artists. Those rows are read by a keyset page that carries **no predicate -at all** and filtered in memory, bounded at 20,000 rows per request — so on a larger library the +song credits, not only entity artists. Those rows are read by a keyset page whose only condition is the +**cursor** — no residual predicate that could discard a row — and filtered in memory, bounded at 20,000 +logical rows per request; so on a larger library the response may be a bounded subset of the matches — bounded in LOGICAL ROWS, which is not the same as bounded work or bytes. Say so in the `[EndpointDescription]` of any endpoint that adopts this shape. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 61c366005..9f498ca36 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -28,7 +28,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `api.schedule-item-flat-dto` | Schedule-item GET/POST/PUT use a flat, non-polymorphic `ScheduleItemResponseModel` (every subtype field promoted to a nullable top-level member) instead of the polymorphic Application VM hierarchy, with mutation field names matching `ScheduleItemRequest` 1:1 for a lossless round-trip. | 2026-07-10 | [link](records/api/schedule-item-flat-dto.md) | | `api.scheduling-hardening` | Create/Replace handlers guard against null/whitespace `name` (`IsNullOrWhiteSpace`, not just `Length`) to prevent NRE-500s, template-item overlap validation compares by index (not record value-equality) to catch exact-duplicate items, and unreachable 404 `ProducesResponseType` attributes on create-only actions are trimmed. | 2026-07-13 | [link](records/api/scheduling-hardening.md) | | `api.search-allitems-paging` | `GET /api/v1/search/all-items` is paginated (capped page size, `Totals` field) to bound DoS exposure; the SPA add-all flow pages to completeness instead of relying on an unbounded response. | 2026-07-18 | [link](records/api/search-allitems-paging.md) | -| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort, and its rows are read by a keyset page carrying **NO predicate whatsoever** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL` — because `LIMIT` only truncates what survives a predicate, so any predicate at all makes it bound the OUTPUT rather than the WORK. All selectivity is in memory. The guarantee is scoped: **at most 20,000 LOGICAL rows returned/materialized and at most 10 round trips (11 for `artist`)** — NOT bounded physical work and NOT bounded bytes, because MySQL traverses deleted-but-unpurged index records and the `TEXT`/`longtext` payload width is unrestricted. The walk pages 2,000 rows at a time, stopping on the first of enough distinct matches, a short page, or the ceiling. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | +| `api.search-field-values-sources` | `GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite's `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404'd, as bounded best-effort, and its rows are read by a keyset page carrying **NO RESIDUAL predicate** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL`. The cursor is itself a predicate, but a SEEKABLE one on the ordering key: it positions the scan and never discards a row. A RESIDUAL predicate discards rows the engine already produced, and `LIMIT` truncates only the survivors — so with one present it bounds the OUTPUT rather than the row count. All selectivity is in memory. The guarantee is scoped: **at most 20,000 LOGICAL rows returned/materialized and at most 10 round trips (11 for `artist`)** — NOT bounded physical work and NOT bounded bytes, because MySQL traverses deleted-but-unpurged index records and the `TEXT`/`longtext` payload width is unrestricted. The walk pages 2,000 rows at a time, stopping on the first of enough distinct matches, a short page, or the ceiling. | 2026-07-26 | [link](records/api/search-field-values-sources.md) | | `api.search-paging-cap` | Search stays capped at 100 items per media kind; an overflowing kind's "See all" reuses library-browse paging instead of adding new API surface. | 2026-07-11 | [link](records/api/search-paging-cap.md) | | `api.versioning-v1` | The entire `/api` surface is versioned to `/api/v1` uniformly (no unversioned corner); legacy unversioned callers are rewritten in-pipeline (not redirected) with Deprecation/Link/Sunset headers, and post-freeze `/api/v1` is additive-only — a breaking change requires `/api/v2`. | 2026-07-13 | [link](records/api/versioning-v1.md) | | `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) | diff --git a/docs/decisions/records/api/search-field-values-sources.md b/docs/decisions/records/api/search-field-values-sources.md index ca0806f3a..a19693afd 100644 --- a/docs/decisions/records/api/search-field-values-sources.md +++ b/docs/decisions/records/api/search-field-values-sources.md @@ -1,12 +1,12 @@ --- key: api.search-field-values-sources -title: '2026-07-26 — Facet-value typeahead, restated: every artist-bearing source is covered, and the JSON-column source is paged by ROW POSITION with no SQL predicate at all (#578)' +title: '2026-07-26 — Facet-value typeahead, restated: every artist-bearing source is covered, and the JSON-column source is paged by ROW POSITION with no RESIDUAL SQL predicate (#578)' status: active since: '2026-07-26' supersedes: api.search-field-values@2026-07-23 superseded-by: none -rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort, and its rows are read by a keyset page carrying **NO predicate whatsoever** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL` — because `LIMIT` only truncates what survives a predicate, so any predicate at all makes it bound the OUTPUT rather than the WORK. All selectivity is in memory. The guarantee is scoped: **at most 20,000 LOGICAL rows returned/materialized and at most 10 round trips (11 for `artist`)** — NOT bounded physical work and NOT bounded bytes, because MySQL traverses deleted-but-unpurged index records and the `TEXT`/`longtext` payload width is unrestricted. The walk pages 2,000 rows at a time, stopping on the first of enough distinct matches, a short page, or the ceiling.' -signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, LIMIT bounds output not work, seekable cursor vs residual predicate, MySQL purge lag traverses deleted index records, TEXT overflow pages, logical rows not physical work, keyspace is not rows, page by row position, density-independent paging, deleted rows leave Id gaps, ListValuedBatchRows, ListValuedMaxRowsRead, bounded best-effort facet values, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669' +rule: '`GET /api/v1/search/fields/{name}/values?q=&limit=` returns distinct WHOLE values from the database for a narrow allow-list of catalog fields (never the Lucene term dictionary — analyzed `TextField`s store lowercased word tokens, e.g. "Science Fiction" → `science`/`fiction`, useless as a suggestion), 404 for an unknown field, a non-`text` field, or a `text` field with no distinct-value source (`title`, `show_title` only); `limit` clamped to `[1, 50]` (default 50). The FINAL filter, dedup and ordering applied to the response are ORDINAL (`OrdinalIgnoreCase` / `StringComparer.Ordinal`), never current-culture, because `UseRequestLocalization` makes the culture caller-controlled — scoped to the in-memory stages on purpose: a field sourced by a plain EF query is filtered and truncated by the DATABASE collation first (SQLite''s `LOWER()` is ASCII-only), which ordinal semantics downstream cannot undo (ersatztv#668). A field whose values live in an EF **primitive collection** (one JSON array per row in a single column: `SongMetadata.Artists`, `SongMetadata.AlbumArtists`) is served, not 404''d, as bounded best-effort, and its rows are read by a keyset page carrying **NO RESIDUAL predicate** — `SELECT Id, AS Payload FROM SongMetadata WHERE Id > @AfterId ORDER BY Id LIMIT @Batch`, no `LIKE`, no `LOWER`, not even `IS NOT NULL`. The cursor is itself a predicate, but a SEEKABLE one on the ordering key: it positions the scan and never discards a row. A RESIDUAL predicate discards rows the engine already produced, and `LIMIT` truncates only the survivors — so with one present it bounds the OUTPUT rather than the row count. All selectivity is in memory. The guarantee is scoped: **at most 20,000 LOGICAL rows returned/materialized and at most 10 round trips (11 for `artist`)** — NOT bounded physical work and NOT bounded bytes, because MySQL traverses deleted-but-unpurged index records and the `TEXT`/`longtext` payload width is unrestricted. The walk pages 2,000 rows at a time, stopping on the first of enough distinct matches, a short page, or the ceiling.' +signals: 'artist typeahead free-text credits, album_artist 404, artist suggestions missing music videos, SongMetadata.Artists, SongMetadata.AlbumArtists, MusicVideoArtist, EF primitive collection, PrimitiveCollection JSON column, SelectMany requires APPLY on SQLite, Pomelo primitive collections not enabled, LIMIT bounds output not work, seekable cursor vs residual predicate, cursor is a predicate too, MySQL purge lag traverses deleted index records, TEXT overflow pages, logical rows not physical work, keyspace is not rows, page by row position, density-independent paging, deleted rows leave Id gaps, ListValuedBatchRows, ListValuedMaxRowsRead, bounded best-effort facet values, OrdinalIgnoreCase vs InvariantCultureIgnoreCase folding, Accept-Language tr-TR dotless i, content_rating split, text field allow-list · paths: `ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs`, `ErsatzTV/Controllers/Api/SearchController.cs`, `ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs`, `ErsatzTV.Tests/Application/Search/SearchFieldValuesQueryShapeTests.cs`, `web/src/api/search.ts` · issues: #578, #434, #176, #668, #669' mechanics: '`GetSearchFieldValuesHandler` (`GetSource`, `GetSongListValuedColumn`, `GetSongListValuedValues`, `ListValuedSql`, `ParseElements`, `FilterSortTake`); `SearchFieldValuesQueryShapeTests.List_Valued_Page_Query_Has_No_Predicate_Beyond_The_Keyset_Cursor`; api-conventions.md; spa-conventions.md §12' --- @@ -99,7 +99,7 @@ and each was caught only by someone constructing the adversarial case rather tha | 1–2 | the **result** — fixed `LIMIT 1000` on pre-filtered rows | the pre-filter was deliberately allowed to over-match, so a widened pattern (any non-ASCII or JSON-escaped prefix collapses it to `%"%`) filled the budget with rows that could not match. 1,000 `"zzz"` songs, `"éclair"` at row 1,001, `q=é` → `[]` | | 3 | **candidates returned** — keyset paging + `LIMIT` | a query matching nothing must evaluate every eligible row before it can return an empty page, so the first empty page ended the walk having counted **zero** against the ceiling. Rows returned bounded, rows inspected unbounded | | 4 | **keyspace width** — closed `Id` range per page | keyspace is not rows. Delete 20,000 historical rows, put one song at `Id` 20001, `q=que` → `[]`. **One row in the table, zero rows inspected.** Capacity fell linearly with deletion ratio and no ratio was safe: one placed gap hides the next match | -| 5 | **actual rows** — keyset page by row position, **no predicate** | — | +| 5 | **logical rows returned** — keyset page by row position, cursor only, **no residual predicate** | — (physical work still unbounded; see below) | The through-line: **`LIMIT` truncates what survives a RESIDUAL predicate.** The distinction is not "predicate vs no predicate" — attempt 5's query still has `Id > @AfterId`. It is: @@ -133,8 +133,8 @@ more dangerous than the code ever was. What holds: - **at most `ListValuedMaxRowsRead` logical rows returned and materialized per request**, and - **at most 10 (or 11) round trips.** -That is the whole guarantee. It is what makes the walk terminate and what caps what this process -holds in memory. **Explicitly retracted**, having been asserted here in an earlier revision: +That is the whole guarantee. It is what makes the walk terminate and what caps the number of rows and +round trips. **Explicitly retracted**, having been asserted here in earlier revisions: - ~~"`LIMIT n` reads exactly `n` index entries"~~ — **false on MySQL.** Deleted clustered-index records survive until purge runs, and a range scan still traverses them. Hold an old InnoDB @@ -146,6 +146,10 @@ holds in memory. **Explicitly retracted**, having been asserted here in an earli - ~~bounded physical work / bounded I/O~~ — row width is unbounded. `Artists`/`AlbumArtists` are unrestricted `TEXT`/`longtext`, and both SQLite and InnoDB spill large payloads to overflow pages, so a row count implies neither a byte count nor a page-read count. +- ~~"caps what this process holds in memory"~~ — the same overclaim one level down, and it survived + the first retraction. A row count bounds neither bytes buffered nor set size: payload width is + unrestricted, and one JSON array can contain arbitrarily many strings, every one of which may enter + the in-memory distinct set. Nor can the query-shape test carry more than it does: it pins the SQL **string**. It cannot pin an execution plan, MVCC visibility work, or payload I/O — and on MySQL, using the index to satisfy -- 2.47.3