timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

better search index thread fix (#143)

Browse Source 
* Revert "fix search index threading (#141)"

This reverts commit 3fb6da0754.

* better search index thread fix
This commit is contained in:
Jason Dove
2021-04-05 16:18:17 -05:00
committed by GitHub
parent 9809c60924
commit 79bfba6428
10 changed files with 67 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs
+7 -1
View File
@@ -4,6 +4,7 @@ using System.Threading;
using System.Threading.Tasks;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Metadata;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Interfaces.Search;
using LanguageExt;
@@ -14,6 +15,7 @@ namespace ErsatzTV.Application.Search.Commands
public class RebuildSearchIndexHandler : MediatR.IRequestHandler<RebuildSearchIndex, Unit>
{
private readonly IConfigElementRepository _configElementRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILogger<RebuildSearchIndexHandler> _logger;
private readonly ISearchIndex _searchIndex;
private readonly ISearchRepository _searchRepository;
@@ -22,18 +24,22 @@ namespace ErsatzTV.Application.Search.Commands
ISearchIndex searchIndex,
ISearchRepository searchRepository,
IConfigElementRepository configElementRepository,
ILocalFileSystem localFileSystem,
ILogger<RebuildSearchIndexHandler> logger)
{
_searchIndex = searchIndex;
_logger = logger;
_searchRepository = searchRepository;
_configElementRepository = configElementRepository;
_localFileSystem = localFileSystem;
}
public async Task<Unit> Handle(RebuildSearchIndex request, CancellationToken cancellationToken)
{
bool indexFolderExists = Directory.Exists(FileSystemLayout.SearchIndexFolder);
await _searchIndex.Initialize(_localFileSystem);
if (!indexFolderExists ||
await _configElementRepository.GetValue<int>(ConfigElementKey.SearchIndexVersion) <
_searchIndex.Version)
@@ -41,7 +47,7 @@ namespace ErsatzTV.Application.Search.Commands
_logger.LogDebug("Migrating search index to version {Version}", _searchIndex.Version);
List<int> itemIds = await _searchRepository.GetItemIdsToIndex();
await _searchIndex.Rebuild(itemIds);
await _searchIndex.Rebuild(_searchRepository, itemIds);
Option<ConfigElement> maybeVersion =
await _configElementRepository.Get(ConfigElementKey.SearchIndexVersion);
ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs
+8 -4
View File
@@ -1,19 +1,23 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Metadata;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Search;
using LanguageExt;
namespace ErsatzTV.Core.Interfaces.Search
{
public interface ISearchIndex
public interface ISearchIndex : IDisposable
{
public int Version { get; }
Task<bool> Initialize();
Task<Unit> Rebuild(List<int> itemIds);
Task<bool> Initialize(ILocalFileSystem localFileSystem);
Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds);
Task<Unit> AddItems(List<MediaItem> items);
Task<Unit> UpdateItems(List<MediaItem> items);
Task<Unit> RemoveItems(List<int> ids);
Task<SearchResult> Search(string query, int skip, int limit, string searchField = "");
void Commit();
}
}
ErsatzTV.Core/Metadata/MovieFolderScanner.cs
+1
View File
@@ -140,6 +140,7 @@ namespace ErsatzTV.Core.Metadata
}
}
_searchIndex.Commit();
return Unit.Default;
}
ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs
+1
View File
@@ -133,6 +133,7 @@ namespace ErsatzTV.Core.Metadata
}
}
_searchIndex.Commit();
return Unit.Default;
}
ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs
+1
View File
@@ -124,6 +124,7 @@ namespace ErsatzTV.Core.Metadata
List<int> ids = await _televisionRepository.DeleteEmptyShows(libraryPath);
await _searchIndex.RemoveItems(ids);
_searchIndex.Commit();
return Unit.Default;
}
ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs
+1
View File
@@ -102,6 +102,7 @@ namespace ErsatzTV.Core.Plex
return Task.CompletedTask;
});
_searchIndex.Commit();
return Unit.Default;
}
ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs
+1
View File
@@ -95,6 +95,7 @@ namespace ErsatzTV.Core.Plex
await _mediator.Publish(new LibraryScanProgress(plexMediaSourceLibrary.Id, 0));
_searchIndex.Commit();
return Unit.Default;
},
error =>
ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs
+6
View File
@@ -45,6 +45,12 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Studios)
.Include(mi => (mi as MusicVideo).MusicVideoMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as MusicVideo).MusicVideoMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as MusicVideo).MusicVideoMetadata)
.ThenInclude(mm => mm.Studios)
.OrderBy(mi => mi.Id)
.SingleOrDefaultAsync(mi => mi.Id == id)
.Map(Optional);
ErsatzTV.Infrastructure/Search/SearchIndex.cs
+39 -56
View File
@@ -23,7 +23,7 @@ using Query = Lucene.Net.Search.Query;
namespace ErsatzTV.Infrastructure.Search
{
public class SearchIndex : ISearchIndex
public sealed class SearchIndex : ISearchIndex
{
private const LuceneVersion AppLuceneVersion = LuceneVersion.LUCENE_48;
@@ -45,65 +45,52 @@ namespace ErsatzTV.Infrastructure.Search
private const string ShowType = "show";
private const string MusicVideoType = "music_video";
private static bool _isRebuilding;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILogger<SearchIndex> _logger;
private readonly ISearchRepository _searchRepository;
private FSDirectory _directory;
private IndexWriter _writer;
public SearchIndex(
ILocalFileSystem localFileSystem,
ISearchRepository searchRepository,
ILogger<SearchIndex> logger)
{
_localFileSystem = localFileSystem;
_searchRepository = searchRepository;
_logger = logger;
}
public SearchIndex(ILogger<SearchIndex> logger) => _logger = logger;
public int Version => 2;
public Task<bool> Initialize()
public Task<bool> Initialize(ILocalFileSystem localFileSystem)
{
_localFileSystem.EnsureFolderExists(FileSystemLayout.SearchIndexFolder);
localFileSystem.EnsureFolderExists(FileSystemLayout.SearchIndexFolder);
_directory = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
var analyzer = new StandardAnalyzer(AppLuceneVersion);
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer)
{ OpenMode = OpenMode.CREATE_OR_APPEND };
_writer = new IndexWriter(_directory, indexConfig);
return Task.FromResult(true);
}
public async Task<Unit> Rebuild(List<int> itemIds)
public async Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds)
{
_isRebuilding = true;
await Initialize();
using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
var analyzer = new StandardAnalyzer(AppLuceneVersion);
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.CREATE };
using var writer = new IndexWriter(dir, indexConfig);
foreach (int id in itemIds)
{
Option<MediaItem> maybeMediaItem = await _searchRepository.GetItemToIndex(id);
Option<MediaItem> maybeMediaItem = await searchRepository.GetItemToIndex(id);
if (maybeMediaItem.IsSome)
{
MediaItem mediaItem = maybeMediaItem.ValueUnsafe();
switch (mediaItem)
{
case Movie movie:
UpdateMovie(movie, writer);
UpdateMovie(movie);
break;
case Show show:
UpdateShow(show, writer);
UpdateShow(show);
break;
case MusicVideo musicVideo:
UpdateMusicVideo(musicVideo, writer);
UpdateMusicVideo(musicVideo);
break;
}
}
}
_isRebuilding = false;
_writer.Commit();
return Unit.Default;
}
@@ -111,23 +98,18 @@ namespace ErsatzTV.Infrastructure.Search
public Task<Unit> UpdateItems(List<MediaItem> items)
{
using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
var analyzer = new StandardAnalyzer(AppLuceneVersion);
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.APPEND };
using var writer = new IndexWriter(dir, indexConfig);
foreach (MediaItem item in items)
{
switch (item)
{
case Movie movie:
UpdateMovie(movie, writer);
UpdateMovie(movie);
break;
case Show show:
UpdateShow(show, writer);
UpdateShow(show);
break;
case MusicVideo musicVideo:
UpdateMusicVideo(musicVideo, writer);
UpdateMusicVideo(musicVideo);
break;
}
}
@@ -137,14 +119,9 @@ namespace ErsatzTV.Infrastructure.Search
public Task<Unit> RemoveItems(List<int> ids)
{
using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
var analyzer = new StandardAnalyzer(AppLuceneVersion);
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.APPEND };
using var writer = new IndexWriter(dir, indexConfig);
foreach (int id in ids)
{
writer.DeleteDocuments(new Term(IdField, id.ToString()));
_writer.DeleteDocuments(new Term(IdField, id.ToString()));
}
return Task.FromResult(Unit.Default);
@@ -152,14 +129,12 @@ namespace ErsatzTV.Infrastructure.Search
public Task<SearchResult> Search(string searchQuery, int skip, int limit, string searchField = "")
{
if (_isRebuilding ||
string.IsNullOrWhiteSpace(searchQuery.Replace("*", string.Empty).Replace("?", string.Empty)))
if (string.IsNullOrWhiteSpace(searchQuery.Replace("*", string.Empty).Replace("?", string.Empty)))
{
return new SearchResult(new List<SearchItem>(), 0).AsTask();
}
using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
using var reader = DirectoryReader.Open(dir);
using DirectoryReader reader = _writer.GetReader(true);
var searcher = new IndexSearcher(reader);
int hitsLimit = skip + limit;
using var analyzer = new StandardAnalyzer(AppLuceneVersion);
@@ -182,6 +157,14 @@ namespace ErsatzTV.Infrastructure.Search
return searchResult.AsTask();
}
public void Commit() => _writer.Commit();
public void Dispose()
{
_writer?.Dispose();
_directory?.Dispose();
}
private static Option<SearchPageMap> GetSearchPageMap(
IndexSearcher searcher,
Query query,
@@ -228,7 +211,7 @@ namespace ErsatzTV.Infrastructure.Search
return new SearchPageMap(map);
}
private void UpdateMovie(Movie movie, IndexWriter writer)
private void UpdateMovie(Movie movie)
{
Option<MovieMetadata> maybeMetadata = movie.MovieMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
@@ -277,7 +260,7 @@ namespace ErsatzTV.Infrastructure.Search
doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
_writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
}
catch (Exception ex)
{
@@ -287,7 +270,7 @@ namespace ErsatzTV.Infrastructure.Search
}
}
private void UpdateShow(Show show, IndexWriter writer)
private void UpdateShow(Show show)
{
Option<ShowMetadata> maybeMetadata = show.ShowMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
@@ -336,7 +319,7 @@ namespace ErsatzTV.Infrastructure.Search
doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
_writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
}
catch (Exception ex)
{
@@ -346,7 +329,7 @@ namespace ErsatzTV.Infrastructure.Search
}
}
private void UpdateMusicVideo(MusicVideo musicVideo, IndexWriter writer)
private void UpdateMusicVideo(MusicVideo musicVideo)
{
Option<MusicVideoMetadata> maybeMetadata = musicVideo.MusicVideoMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
@@ -396,7 +379,7 @@ namespace ErsatzTV.Infrastructure.Search
doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, musicVideo.Id.ToString()), doc);
_writer.UpdateDocument(new Term(IdField, musicVideo.Id.ToString()), doc);
}
catch (Exception ex)
{
ErsatzTV/Startup.cs
+1 -1
View File
@@ -185,6 +185,7 @@ namespace ErsatzTV
services.AddSingleton<IPlexSecretStore, PlexSecretStore>();
services.AddSingleton<IPlexTvApiClient, PlexTvApiClient>(); // TODO: does this need to be singleton?
services.AddSingleton<IEntityLocker, EntityLocker>();
services.AddSingleton<ISearchIndex, SearchIndex>();
AddChannel<IBackgroundServiceRequest>(services);
AddChannel<IPlexBackgroundServiceRequest>(services);
@@ -217,7 +218,6 @@ namespace ErsatzTV
services.AddScoped<IPlexMovieLibraryScanner, PlexMovieLibraryScanner>();
services.AddScoped<IPlexTelevisionLibraryScanner, PlexTelevisionLibraryScanner>();
services.AddScoped<IPlexServerApiClient, PlexServerApiClient>();
services.AddScoped<ISearchIndex, SearchIndex>();
services.AddScoped<IRuntimeInfo, RuntimeInfo>();
services.AddScoped<IPlexPathReplacementService, PlexPathReplacementService>();
services.AddScoped<IFFmpegStreamSelector, FFmpegStreamSelector>();