From b4ae0bde1854bbc2fdff6a2c8dbcc7f46c1a81bf Mon Sep 17 00:00:00 2001 From: Timothy Date: Thu, 23 Jul 2026 19:34:49 +0200 Subject: [PATCH] feat(434): distinct-values search endpoint (text fields, Lucene term enumeration) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds GET /api/v1/search/fields/{name}/values?q=&limit= — the backend slice of the visual rule builder's facet-value typeahead (#434). Enumerates distinct Lucene term values for a text field via MultiFields.GetTerms + TermsEnum, filtered by a case-insensitive prefix, limit clamped to [1,50]. 404s when the field is absent from SearchFieldCatalog or is not type "text". ElasticSearchIndex (the optional external backend) throws NotSupportedException for this method — its text fields are analyzed, not keyword-mapped, so a terms aggregation isn't safe to guess at without verifying against a live cluster. Regenerated OpenAPI trio (v1.json, v1.d.ts, endpoint-index.md). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Search/Queries/GetSearchFieldValues.cs | 6 + .../Queries/GetSearchFieldValuesHandler.cs | 28 +++++ .../Search/SearchFieldValuesResponseModel.cs | 8 ++ .../Interfaces/Search/ISearchIndex.cs | 4 +- .../Search/ElasticSearchIndex.cs | 116 ++++++++++-------- .../Search/LuceneSearchIndex.cs | 38 +++++- .../GetSearchFieldValuesHandlerTests.cs | 84 +++++++++++++ ErsatzTV/Controllers/Api/SearchController.cs | 22 ++++ ErsatzTV/wwwroot/openapi/v1.json | 116 ++++++++++++++++++ docs/endpoint-index.md | 3 +- web/src/api/generated/v1.d.ts | 3 + 11 files changed, 370 insertions(+), 58 deletions(-) create mode 100644 ErsatzTV.Application/Search/Queries/GetSearchFieldValues.cs create mode 100644 ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs create mode 100644 ErsatzTV.Core/Api/Search/SearchFieldValuesResponseModel.cs create mode 100644 ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValues.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValues.cs new file mode 100644 index 000000000..92938b177 --- /dev/null +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValues.cs @@ -0,0 +1,6 @@ +using ErsatzTV.Core.Api.Search; + +namespace ErsatzTV.Application.Search.Queries; + +public record GetSearchFieldValues(string Name, string Query, int Limit) + : IRequest>; diff --git a/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs new file mode 100644 index 000000000..75b204ddd --- /dev/null +++ b/ErsatzTV.Application/Search/Queries/GetSearchFieldValuesHandler.cs @@ -0,0 +1,28 @@ +using ErsatzTV.Core.Api.Search; +using ErsatzTV.Core.Interfaces.Search; + +namespace ErsatzTV.Application.Search.Queries; + +public class GetSearchFieldValuesHandler(ISearchIndex searchIndex) + : IRequestHandler> +{ + private const int DefaultLimit = 50; + private const int MaxLimit = 50; + + public async Task> Handle( + GetSearchFieldValues request, + CancellationToken cancellationToken) + { + SearchFieldResponseModel field = SearchFieldCatalog.Fields + .FirstOrDefault(f => f.Name == request.Name); + + if (field is null || field.Type != "text") + { + return Option.None; + } + + int limit = request.Limit <= 0 ? DefaultLimit : Math.Clamp(request.Limit, 1, MaxLimit); + List values = await searchIndex.GetFieldValues(request.Name, request.Query, limit); + return new SearchFieldValuesResponseModel(values); + } +} diff --git a/ErsatzTV.Core/Api/Search/SearchFieldValuesResponseModel.cs b/ErsatzTV.Core/Api/Search/SearchFieldValuesResponseModel.cs new file mode 100644 index 000000000..cc262711f --- /dev/null +++ b/ErsatzTV.Core/Api/Search/SearchFieldValuesResponseModel.cs @@ -0,0 +1,8 @@ +#nullable enable +namespace ErsatzTV.Core.Api.Search; + +/// +/// Distinct term values for a single text field in the search index, used to power the visual rule +/// builder's facet-value typeahead. +/// +public record SearchFieldValuesResponseModel(List Values); diff --git a/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs b/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs index 1dd7e9c41..9b4e36477 100644 --- a/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs +++ b/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs @@ -1,4 +1,4 @@ -using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Domain; using ErsatzTV.Core.Interfaces.Metadata; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Search; @@ -36,6 +36,8 @@ public interface ISearchIndex : IDisposable Task RemoveItems(IEnumerable ids); + Task> GetFieldValues(string field, string query, int limit); + Task Search( string query, string smartCollectionName, diff --git a/ErsatzTV.Infrastructure/Search/ElasticSearchIndex.cs b/ErsatzTV.Infrastructure/Search/ElasticSearchIndex.cs index 657498174..6350882a6 100644 --- a/ErsatzTV.Infrastructure/Search/ElasticSearchIndex.cs +++ b/ErsatzTV.Infrastructure/Search/ElasticSearchIndex.cs @@ -166,6 +166,16 @@ public class ElasticSearchIndex : ISearchIndex return deleteResponse.IsValidResponse; } + public Task> GetFieldValues(string field, string query, int limit) => + // Distinct-value enumeration relies on term-level enumeration of an unanalyzed field (see + // LuceneSearchIndex.GetFieldValues). The Elasticsearch index maps searchable text fields as + // analyzed `Text` (not `Keyword`), so a terms aggregation would fail at query time without + // per-field `.keyword` sub-fields the mapping does not define. Fail loudly rather than + // returning a silently-empty/incorrect result; add ES support if this backend needs the + // rule-builder typeahead. + throw new NotSupportedException( + "Distinct field-value listing is not supported by the Elasticsearch search index backend."); + public Task Search( string query, string smartCollectionName, @@ -440,64 +450,64 @@ public class ElasticSearchIndex : ISearchIndex Season season) { foreach (SeasonMetadata metadata in season.SeasonMetadata.HeadOrNone()) - foreach (ShowMetadata showMetadata in season.Show.ShowMetadata.HeadOrNone()) - { - try + foreach (ShowMetadata showMetadata in season.Show.ShowMetadata.HeadOrNone()) { - var seasonTitle = $"{showMetadata.Title} - S{season.SeasonNumber}"; - string sortTitle = $"{showMetadata.SortTitle}_{season.SeasonNumber:0000}" - .ToLowerInvariant(); - string titleAndYear = $"{showMetadata.Title}_{showMetadata.Year}_{season.SeasonNumber}" - .ToLowerInvariant(); - - var doc = new ElasticSearchItem + try { - Id = season.Id, - Type = LuceneSearchIndex.SeasonType, - Title = seasonTitle, - SortTitle = sortTitle, - LibraryName = season.LibraryPath.Library.Name, - LibraryId = season.LibraryPath.Library.Id, - TitleAndYear = titleAndYear, - TitleAndYearSearch = LuceneSearchIndex.GetTitleAndYearSearch(metadata), - JumpLetter = LuceneSearchIndex.GetJumpLetter(showMetadata), - State = season.State.ToString(), - SeasonNumber = season.SeasonNumber, - ShowTitle = showMetadata.Title, - ShowGenre = showMetadata.Genres.Map(g => g.Name).ToList(), - ShowTag = showMetadata.Tags.Map(t => t.Name).ToList(), - ShowStudio = showMetadata.Studios.Map(s => s.Name).ToList(), - ShowContentRating = GetContentRatings(showMetadata.ContentRating), - Language = GetLanguages( - languageCodeService, - await searchRepository.GetLanguagesForSeason(season)), - LanguageTag = await searchRepository.GetLanguagesForSeason(season), - SubLanguage = GetLanguages( - languageCodeService, - await searchRepository.GetSubLanguagesForSeason(season)), - SubLanguageTag = await searchRepository.GetSubLanguagesForSeason(season), - ContentRating = GetContentRatings(showMetadata.ContentRating), - ReleaseDate = GetReleaseDate(metadata.ReleaseDate), - AddedDate = GetAddedDate(metadata.DateAdded), - TraktList = season.TraktListItems - .Map(t => t.TraktList.TraktId.ToString(CultureInfo.InvariantCulture)).ToList(), - Tag = metadata.Tags.Map(a => a.Name).ToList(), - TagFull = metadata.Tags.Map(t => t.Name).ToList() - }; + var seasonTitle = $"{showMetadata.Title} - S{season.SeasonNumber}"; + string sortTitle = $"{showMetadata.SortTitle}_{season.SeasonNumber:0000}" + .ToLowerInvariant(); + string titleAndYear = $"{showMetadata.Title}_{showMetadata.Year}_{season.SeasonNumber}" + .ToLowerInvariant(); - foreach ((string key, List value) in GetMetadataGuids(metadata)) - { - doc.AdditionalProperties.Add(key, value); + var doc = new ElasticSearchItem + { + Id = season.Id, + Type = LuceneSearchIndex.SeasonType, + Title = seasonTitle, + SortTitle = sortTitle, + LibraryName = season.LibraryPath.Library.Name, + LibraryId = season.LibraryPath.Library.Id, + TitleAndYear = titleAndYear, + TitleAndYearSearch = LuceneSearchIndex.GetTitleAndYearSearch(metadata), + JumpLetter = LuceneSearchIndex.GetJumpLetter(showMetadata), + State = season.State.ToString(), + SeasonNumber = season.SeasonNumber, + ShowTitle = showMetadata.Title, + ShowGenre = showMetadata.Genres.Map(g => g.Name).ToList(), + ShowTag = showMetadata.Tags.Map(t => t.Name).ToList(), + ShowStudio = showMetadata.Studios.Map(s => s.Name).ToList(), + ShowContentRating = GetContentRatings(showMetadata.ContentRating), + Language = GetLanguages( + languageCodeService, + await searchRepository.GetLanguagesForSeason(season)), + LanguageTag = await searchRepository.GetLanguagesForSeason(season), + SubLanguage = GetLanguages( + languageCodeService, + await searchRepository.GetSubLanguagesForSeason(season)), + SubLanguageTag = await searchRepository.GetSubLanguagesForSeason(season), + ContentRating = GetContentRatings(showMetadata.ContentRating), + ReleaseDate = GetReleaseDate(metadata.ReleaseDate), + AddedDate = GetAddedDate(metadata.DateAdded), + TraktList = season.TraktListItems + .Map(t => t.TraktList.TraktId.ToString(CultureInfo.InvariantCulture)).ToList(), + Tag = metadata.Tags.Map(a => a.Name).ToList(), + TagFull = metadata.Tags.Map(t => t.Name).ToList() + }; + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName, ES.Id.From(doc)); + } + catch (Exception ex) + { + metadata.Season = null; + _logger.LogWarning(ex, "Error indexing season with metadata {@Metadata}", metadata); } - - await _client.IndexAsync(doc, IndexName, ES.Id.From(doc)); } - catch (Exception ex) - { - metadata.Season = null; - _logger.LogWarning(ex, "Error indexing season with metadata {@Metadata}", metadata); - } - } } private async Task UpdateArtist( diff --git a/ErsatzTV.Infrastructure/Search/LuceneSearchIndex.cs b/ErsatzTV.Infrastructure/Search/LuceneSearchIndex.cs index 41e9fb856..2ec2cfdcf 100644 --- a/ErsatzTV.Infrastructure/Search/LuceneSearchIndex.cs +++ b/ErsatzTV.Infrastructure/Search/LuceneSearchIndex.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using System.Globalization; using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Interfaces.Metadata; @@ -145,7 +145,7 @@ public sealed class LuceneSearchIndex : ISearchIndex _directory = FSDirectory.Open(FileSystemLayout.SearchIndexFolder); Analyzer analyzer = SearchQueryParser.AnalyzerWrapper(); var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) - { OpenMode = OpenMode.CREATE_OR_APPEND }; + { OpenMode = OpenMode.CREATE_OR_APPEND }; _writer = new IndexWriter(_directory, indexConfig); _initialized = true; } @@ -209,6 +209,38 @@ public sealed class LuceneSearchIndex : ISearchIndex return Task.FromResult(true); } + public Task> GetFieldValues(string field, string query, int limit) + { + var values = new List(); + + if (_writer.MaxDoc == 0) + { + return Task.FromResult(values); + } + + string normalizedQuery = (query ?? string.Empty).Trim(); + + using DirectoryReader reader = _writer.GetReader(true); + Terms terms = MultiFields.GetTerms(reader, field); + if (terms is null) + { + return Task.FromResult(values); + } + + TermsEnum termsEnum = terms.GetEnumerator(); + while (termsEnum.MoveNext() && values.Count < limit) + { + string value = termsEnum.Term.Utf8ToString(); + if (string.IsNullOrEmpty(normalizedQuery) || + value.StartsWith(normalizedQuery, StringComparison.OrdinalIgnoreCase)) + { + values.Add(value); + } + } + + return Task.FromResult(values); + } + // default to title field only public Task Search( string query, @@ -328,7 +360,7 @@ public sealed class LuceneSearchIndex : ISearchIndex using (Analyzer analyzer = SearchQueryParser.AnalyzerWrapper()) { var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) - { OpenMode = OpenMode.CREATE_OR_APPEND }; + { OpenMode = OpenMode.CREATE_OR_APPEND }; using (var w = new IndexWriter(d, indexConfig)) { using (DirectoryReader _ = w.GetReader(true)) diff --git a/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs new file mode 100644 index 000000000..8b2ddb5d9 --- /dev/null +++ b/ErsatzTV.Tests/Application/Search/GetSearchFieldValuesHandlerTests.cs @@ -0,0 +1,84 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ErsatzTV.Application.Search.Queries; +using ErsatzTV.Core.Api.Search; +using ErsatzTV.Core.Interfaces.Search; +using LanguageExt; +using NSubstitute; +using NUnit.Framework; +using Shouldly; + +namespace ErsatzTV.Tests.Application.Search; + +[TestFixture] +public class GetSearchFieldValuesHandlerTests +{ + [Test] + public async Task Returns_Filtered_Values_For_AllowListed_Text_Field() + { + ISearchIndex searchIndex = Substitute.For(); + searchIndex.GetFieldValues("genre", "A", 50) + .Returns(Task.FromResult(new List { "Action", "Adventure" })); + + var handler = new GetSearchFieldValuesHandler(searchIndex); + + Option result = await handler.Handle( + new GetSearchFieldValues("genre", "A", 50), + CancellationToken.None); + + result.IsSome.ShouldBeTrue(); + result.IfSome(r => r.Values.ShouldBe(new List { "Action", "Adventure" })); + + await searchIndex.Received(1).GetFieldValues("genre", "A", 50); + } + + [Test] + public async Task Returns_NotFound_For_Unknown_Field() + { + ISearchIndex searchIndex = Substitute.For(); + var handler = new GetSearchFieldValuesHandler(searchIndex); + + Option result = await handler.Handle( + new GetSearchFieldValues("nope", string.Empty, 50), + CancellationToken.None); + + result.IsNone.ShouldBeTrue(); + await searchIndex.DidNotReceive().GetFieldValues( + Arg.Any(), + Arg.Any(), + Arg.Any()); + } + + [Test] + public async Task Returns_NotFound_For_NonText_Field() + { + ISearchIndex searchIndex = Substitute.For(); + var handler = new GetSearchFieldValuesHandler(searchIndex); + + // "minutes" is a "number" field in SearchFieldCatalog, not "text" + Option result = await handler.Handle( + new GetSearchFieldValues("minutes", string.Empty, 50), + CancellationToken.None); + + result.IsNone.ShouldBeTrue(); + await searchIndex.DidNotReceive().GetFieldValues( + Arg.Any(), + Arg.Any(), + Arg.Any()); + } + + [Test] + public async Task Clamps_Limit_To_Fifty() + { + ISearchIndex searchIndex = Substitute.For(); + searchIndex.GetFieldValues(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new List())); + + var handler = new GetSearchFieldValuesHandler(searchIndex); + + await handler.Handle(new GetSearchFieldValues("genre", string.Empty, 500), CancellationToken.None); + + await searchIndex.Received(1).GetFieldValues("genre", string.Empty, 50); + } +} diff --git a/ErsatzTV/Controllers/Api/SearchController.cs b/ErsatzTV/Controllers/Api/SearchController.cs index 217613005..c191d4b06 100644 --- a/ErsatzTV/Controllers/Api/SearchController.cs +++ b/ErsatzTV/Controllers/Api/SearchController.cs @@ -189,6 +189,28 @@ public class SearchController(IMediator mediator) : ControllerBase public Task> GetSearchFields(CancellationToken cancellationToken) => mediator.Send(new GetSearchFieldCatalog(), cancellationToken); + [HttpGet("/api/v1/search/fields/{name}/values", Name = "GetSearchFieldValues")] + [Tags("Search")] + [EndpointSummary("List distinct term values for a text search field")] + [EndpointDescription( + "Returns distinct term values from the search index for the given text field, filtered by an " + + "optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. " + + "404 when the field is unknown or is not a text field.")] + [EndpointGroupName("general")] + [ProducesResponseType(typeof(SearchFieldValuesResponseModel), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)] + public async Task GetSearchFieldValues( + [FromRoute] string name, + [FromQuery] string q = "", + [FromQuery] int limit = 50, + CancellationToken cancellationToken = default) + { + Option result = await mediator.Send( + new GetSearchFieldValues(name, q ?? string.Empty, limit), + cancellationToken); + return result.ToGetResult(); + } + private static SearchResultAllItemsResponseModel Project(SearchResultAllItemsViewModel vm) => new( vm.MovieIds, diff --git a/ErsatzTV/wwwroot/openapi/v1.json b/ErsatzTV/wwwroot/openapi/v1.json index 2312c6b8b..f749ea132 100644 --- a/ErsatzTV/wwwroot/openapi/v1.json +++ b/ErsatzTV/wwwroot/openapi/v1.json @@ -18035,6 +18035,108 @@ ] } }, + "/api/v1/search/fields/{name}/values": { + "get": { + "tags": [ + "Search" + ], + "summary": "List distinct term values for a text search field", + "description": "Returns distinct term values from the search index for the given text field, filtered by an optional case-insensitive prefix. Powers the visual rule builder's facet-value typeahead. 404 when the field is unknown or is not a text field.", + "operationId": "GetSearchFieldValues", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "q", + "in": "query", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "limit", + "in": "query", + "schema": { + "type": "integer", + "format": "int32", + "default": 50 + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "text/plain": { + "schema": { + "$ref": "#/components/schemas/SearchFieldValuesResponseModel" + } + }, + "application/json": { + "schema": { + "$ref": "#/components/schemas/SearchFieldValuesResponseModel" + } + }, + "text/json": { + "schema": { + "$ref": "#/components/schemas/SearchFieldValuesResponseModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "text/plain": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + }, + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + }, + "text/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "401": { + "description": "API key missing or invalid.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "400": { + "description": "Request validation failed (model binding or FluentValidation).", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationProblemDetails" + } + } + } + } + }, + "security": [ + { } + ] + } + }, "/api/v1/seasons/{id}": { "get": { "tags": [ @@ -30612,6 +30714,20 @@ } } }, + "SearchFieldValuesResponseModel": { + "required": [ + "values" + ], + "type": "object", + "properties": { + "values": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "SearchResultAllItemsResponseModel": { "required": [ "movieIds", diff --git a/docs/endpoint-index.md b/docs/endpoint-index.md index 124c7e57b..3c1cafbc7 100644 --- a/docs/endpoint-index.md +++ b/docs/endpoint-index.md @@ -2,7 +2,7 @@ *Generated by `scripts/generate-endpoint-index.py` from `ErsatzTV/wwwroot/openapi/v1.json`. Do not edit by hand -- regenerated by `scripts/update-openapi.sh`.* -166 endpoints, 249 operations. +167 endpoints, 250 operations. ## Artists @@ -338,6 +338,7 @@ | GET | `/api/v1/search/artists` | SearchArtists | Search artists by name | | GET | `/api/v1/search/collections` | SearchCollections | Search collections by name | | GET | `/api/v1/search/fields` | GetSearchFields | List the filterable fields for the visual rule builder | +| GET | `/api/v1/search/fields/{name}/values` | GetSearchFieldValues | List distinct term values for a text search field | | GET | `/api/v1/search/multi-collections` | SearchMultiCollections | Search multi collections by name | | GET | `/api/v1/search/smart-collections` | SearchSmartCollections | Search smart collections by name | | GET | `/api/v1/search/television-seasons` | SearchTelevisionSeasons | Search television seasons by name | diff --git a/web/src/api/generated/v1.d.ts b/web/src/api/generated/v1.d.ts index 6bf414e15..76bfc5545 100644 --- a/web/src/api/generated/v1.d.ts +++ b/web/src/api/generated/v1.d.ts @@ -1461,6 +1461,9 @@ export interface components { "type": null | string; "group": null | string; "values": null | Array; + }; + "SearchFieldValuesResponseModel": { + "values": Array; }; "SearchResultAllItemsResponseModel": { "movieIds": Array;