better search index thread fix (#143)

* Revert "fix search index threading (#141)"

This reverts commit 3fb6da0754.

* better search index thread fix

ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs

@@ -4,6 +4,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.Metadata;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Search;
 using LanguageExt;
@@ -14,6 +15,7 @@ namespace ErsatzTV.Application.Search.Commands
     public class RebuildSearchIndexHandler : MediatR.IRequestHandler<RebuildSearchIndex, Unit>
     {
         private readonly IConfigElementRepository _configElementRepository;
+        private readonly ILocalFileSystem _localFileSystem;
         private readonly ILogger<RebuildSearchIndexHandler> _logger;
         private readonly ISearchIndex _searchIndex;
         private readonly ISearchRepository _searchRepository;
@@ -22,18 +24,22 @@ namespace ErsatzTV.Application.Search.Commands
             ISearchIndex searchIndex,
             ISearchRepository searchRepository,
             IConfigElementRepository configElementRepository,
+            ILocalFileSystem localFileSystem,
             ILogger<RebuildSearchIndexHandler> logger)
         {
             _searchIndex = searchIndex;
             _logger = logger;
             _searchRepository = searchRepository;
             _configElementRepository = configElementRepository;
+            _localFileSystem = localFileSystem;
         }
 
         public async Task<Unit> Handle(RebuildSearchIndex request, CancellationToken cancellationToken)
         {
             bool indexFolderExists = Directory.Exists(FileSystemLayout.SearchIndexFolder);
 
+            await _searchIndex.Initialize(_localFileSystem);
+            
             if (!indexFolderExists ||
                 await _configElementRepository.GetValue<int>(ConfigElementKey.SearchIndexVersion) <
                 _searchIndex.Version)
@@ -41,7 +47,7 @@ namespace ErsatzTV.Application.Search.Commands
                 _logger.LogDebug("Migrating search index to version {Version}", _searchIndex.Version);
 
                 List<int> itemIds = await _searchRepository.GetItemIdsToIndex();
-                await _searchIndex.Rebuild(itemIds);
+                await _searchIndex.Rebuild(_searchRepository, itemIds);
 
                 Option<ConfigElement> maybeVersion =
                     await _configElementRepository.Get(ConfigElementKey.SearchIndexVersion);

ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs

@@ -1,19 +1,23 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.Metadata;
+using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Search;
 using LanguageExt;
 
 namespace ErsatzTV.Core.Interfaces.Search
 {
-    public interface ISearchIndex
+    public interface ISearchIndex : IDisposable
     {
         public int Version { get; }
-        Task<bool> Initialize();
+        Task<bool> Initialize(ILocalFileSystem localFileSystem);
-        Task<Unit> Rebuild(List<int> itemIds);
+        Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds);
         Task<Unit> AddItems(List<MediaItem> items);
         Task<Unit> UpdateItems(List<MediaItem> items);
         Task<Unit> RemoveItems(List<int> ids);
         Task<SearchResult> Search(string query, int skip, int limit, string searchField = "");
+        void Commit();
     }
 }

ErsatzTV.Core/Metadata/MovieFolderScanner.cs

@@ -140,6 +140,7 @@ namespace ErsatzTV.Core.Metadata
                 }
             }
 
+            _searchIndex.Commit();
             return Unit.Default;
         }
 

ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs

@@ -133,6 +133,7 @@ namespace ErsatzTV.Core.Metadata
                 }
             }
             
+            _searchIndex.Commit();
             return Unit.Default;
         }
 

ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs

@@ -124,6 +124,7 @@ namespace ErsatzTV.Core.Metadata
             List<int> ids = await _televisionRepository.DeleteEmptyShows(libraryPath);
             await _searchIndex.RemoveItems(ids);
 
+            _searchIndex.Commit();
             return Unit.Default;
         }
 

ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs

@@ -102,6 +102,7 @@ namespace ErsatzTV.Core.Plex
                     return Task.CompletedTask;
                 });
 
+            _searchIndex.Commit();
             return Unit.Default;
         }
 

ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs

@@ -95,6 +95,7 @@ namespace ErsatzTV.Core.Plex
 
                     await _mediator.Publish(new LibraryScanProgress(plexMediaSourceLibrary.Id, 0));
 
+                    _searchIndex.Commit();
                     return Unit.Default;
                 },
                 error =>

ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs

@@ -45,6 +45,12 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
                 .ThenInclude(mm => mm.Tags)
                 .Include(mi => (mi as Show).ShowMetadata)
                 .ThenInclude(mm => mm.Studios)
+                .Include(mi => (mi as MusicVideo).MusicVideoMetadata)
+                .ThenInclude(mm => mm.Genres)
+                .Include(mi => (mi as MusicVideo).MusicVideoMetadata)
+                .ThenInclude(mm => mm.Tags)
+                .Include(mi => (mi as MusicVideo).MusicVideoMetadata)
+                .ThenInclude(mm => mm.Studios)
                 .OrderBy(mi => mi.Id)
                 .SingleOrDefaultAsync(mi => mi.Id == id)
                 .Map(Optional);

ErsatzTV.Infrastructure/Search/SearchIndex.cs

@@ -23,7 +23,7 @@ using Query = Lucene.Net.Search.Query;
 
 namespace ErsatzTV.Infrastructure.Search
 {
-    public class SearchIndex : ISearchIndex
+    public sealed class SearchIndex : ISearchIndex
     {
         private const LuceneVersion AppLuceneVersion = LuceneVersion.LUCENE_48;
 
@@ -45,65 +45,52 @@ namespace ErsatzTV.Infrastructure.Search
         private const string ShowType = "show";
         private const string MusicVideoType = "music_video";
 
-        private static bool _isRebuilding;
-
-        private readonly ILocalFileSystem _localFileSystem;
         private readonly ILogger<SearchIndex> _logger;
 
-        private readonly ISearchRepository _searchRepository;
+        private FSDirectory _directory;
+        private IndexWriter _writer;
 
-        public SearchIndex(
+        public SearchIndex(ILogger<SearchIndex> logger) => _logger = logger;
-            ILocalFileSystem localFileSystem,
-            ISearchRepository searchRepository,
-            ILogger<SearchIndex> logger)
-        {
-            _localFileSystem = localFileSystem;
-            _searchRepository = searchRepository;
-            _logger = logger;
-        }
 
         public int Version => 2;
 
-        public Task<bool> Initialize()
+        public Task<bool> Initialize(ILocalFileSystem localFileSystem)
         {
-            _localFileSystem.EnsureFolderExists(FileSystemLayout.SearchIndexFolder);
+            localFileSystem.EnsureFolderExists(FileSystemLayout.SearchIndexFolder);
+
+            _directory = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
+            var analyzer = new StandardAnalyzer(AppLuceneVersion);
+            var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer)
+                { OpenMode = OpenMode.CREATE_OR_APPEND };
+            _writer = new IndexWriter(_directory, indexConfig);
+
             return Task.FromResult(true);
         }
 
-        public async Task<Unit> Rebuild(List<int> itemIds)
+        public async Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds)
         {
-            _isRebuilding = true;
-
-            await Initialize();
-
-            using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
-            var analyzer = new StandardAnalyzer(AppLuceneVersion);
-            var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.CREATE };
-            using var writer = new IndexWriter(dir, indexConfig);
-
             foreach (int id in itemIds)
             {
-                Option<MediaItem> maybeMediaItem = await _searchRepository.GetItemToIndex(id);
+                Option<MediaItem> maybeMediaItem = await searchRepository.GetItemToIndex(id);
                 if (maybeMediaItem.IsSome)
                 {
                     MediaItem mediaItem = maybeMediaItem.ValueUnsafe();
                     switch (mediaItem)
                     {
                         case Movie movie:
-                            UpdateMovie(movie, writer);
+                            UpdateMovie(movie);
                             break;
                         case Show show:
-                            UpdateShow(show, writer);
+                            UpdateShow(show);
                             break;
                         case MusicVideo musicVideo:
-                            UpdateMusicVideo(musicVideo, writer);
+                            UpdateMusicVideo(musicVideo);
                             break;
                     }
                 }
             }
 
-            _isRebuilding = false;
+            _writer.Commit();
-
             return Unit.Default;
         }
 
@@ -111,23 +98,18 @@ namespace ErsatzTV.Infrastructure.Search
 
         public Task<Unit> UpdateItems(List<MediaItem> items)
         {
-            using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
-            var analyzer = new StandardAnalyzer(AppLuceneVersion);
-            var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.APPEND };
-            using var writer = new IndexWriter(dir, indexConfig);
-
             foreach (MediaItem item in items)
             {
                 switch (item)
                 {
                     case Movie movie:
-                        UpdateMovie(movie, writer);
+                        UpdateMovie(movie);
                         break;
                     case Show show:
-                        UpdateShow(show, writer);
+                        UpdateShow(show);
                         break;
                     case MusicVideo musicVideo:
-                        UpdateMusicVideo(musicVideo, writer);
+                        UpdateMusicVideo(musicVideo);
                         break;
                 }
             }
@@ -137,14 +119,9 @@ namespace ErsatzTV.Infrastructure.Search
 
         public Task<Unit> RemoveItems(List<int> ids)
         {
-            using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
-            var analyzer = new StandardAnalyzer(AppLuceneVersion);
-            var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.APPEND };
-            using var writer = new IndexWriter(dir, indexConfig);
-
             foreach (int id in ids)
             {
-                writer.DeleteDocuments(new Term(IdField, id.ToString()));
+                _writer.DeleteDocuments(new Term(IdField, id.ToString()));
             }
 
             return Task.FromResult(Unit.Default);
@@ -152,14 +129,12 @@ namespace ErsatzTV.Infrastructure.Search
 
         public Task<SearchResult> Search(string searchQuery, int skip, int limit, string searchField = "")
         {
-            if (_isRebuilding ||
+            if (string.IsNullOrWhiteSpace(searchQuery.Replace("*", string.Empty).Replace("?", string.Empty)))
-                string.IsNullOrWhiteSpace(searchQuery.Replace("*", string.Empty).Replace("?", string.Empty)))
             {
                 return new SearchResult(new List<SearchItem>(), 0).AsTask();
             }
 
-            using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder);
+            using DirectoryReader reader = _writer.GetReader(true);
-            using var reader = DirectoryReader.Open(dir);
             var searcher = new IndexSearcher(reader);
             int hitsLimit = skip + limit;
             using var analyzer = new StandardAnalyzer(AppLuceneVersion);
@@ -182,6 +157,14 @@ namespace ErsatzTV.Infrastructure.Search
             return searchResult.AsTask();
         }
 
+        public void Commit() => _writer.Commit();
+
+        public void Dispose()
+        {
+            _writer?.Dispose();
+            _directory?.Dispose();
+        }
+
         private static Option<SearchPageMap> GetSearchPageMap(
             IndexSearcher searcher,
             Query query,
@@ -228,7 +211,7 @@ namespace ErsatzTV.Infrastructure.Search
             return new SearchPageMap(map);
         }
 
-        private void UpdateMovie(Movie movie, IndexWriter writer)
+        private void UpdateMovie(Movie movie)
         {
             Option<MovieMetadata> maybeMetadata = movie.MovieMetadata.HeadOrNone();
             if (maybeMetadata.IsSome)
@@ -277,7 +260,7 @@ namespace ErsatzTV.Infrastructure.Search
                         doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO));
                     }
 
-                    writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
+                    _writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
                 }
                 catch (Exception ex)
                 {
@@ -287,7 +270,7 @@ namespace ErsatzTV.Infrastructure.Search
             }
         }
 
-        private void UpdateShow(Show show, IndexWriter writer)
+        private void UpdateShow(Show show)
         {
             Option<ShowMetadata> maybeMetadata = show.ShowMetadata.HeadOrNone();
             if (maybeMetadata.IsSome)
@@ -336,7 +319,7 @@ namespace ErsatzTV.Infrastructure.Search
                         doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO));
                     }
 
-                    writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
+                    _writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
                 }
                 catch (Exception ex)
                 {
@@ -346,7 +329,7 @@ namespace ErsatzTV.Infrastructure.Search
             }
         }
 
-        private void UpdateMusicVideo(MusicVideo musicVideo, IndexWriter writer)
+        private void UpdateMusicVideo(MusicVideo musicVideo)
         {
             Option<MusicVideoMetadata> maybeMetadata = musicVideo.MusicVideoMetadata.HeadOrNone();
             if (maybeMetadata.IsSome)
@@ -396,7 +379,7 @@ namespace ErsatzTV.Infrastructure.Search
                         doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO));
                     }
 
-                    writer.UpdateDocument(new Term(IdField, musicVideo.Id.ToString()), doc);
+                    _writer.UpdateDocument(new Term(IdField, musicVideo.Id.ToString()), doc);
                 }
                 catch (Exception ex)
                 {

ErsatzTV/Startup.cs

@@ -185,6 +185,7 @@ namespace ErsatzTV
             services.AddSingleton<IPlexSecretStore, PlexSecretStore>();
             services.AddSingleton<IPlexTvApiClient, PlexTvApiClient>(); // TODO: does this need to be singleton?
             services.AddSingleton<IEntityLocker, EntityLocker>();
+            services.AddSingleton<ISearchIndex, SearchIndex>();
             AddChannel<IBackgroundServiceRequest>(services);
             AddChannel<IPlexBackgroundServiceRequest>(services);
 
@@ -217,7 +218,6 @@ namespace ErsatzTV
             services.AddScoped<IPlexMovieLibraryScanner, PlexMovieLibraryScanner>();
             services.AddScoped<IPlexTelevisionLibraryScanner, PlexTelevisionLibraryScanner>();
             services.AddScoped<IPlexServerApiClient, PlexServerApiClient>();
-            services.AddScoped<ISearchIndex, SearchIndex>();
             services.AddScoped<IRuntimeInfo, RuntimeInfo>();
             services.AddScoped<IPlexPathReplacementService, PlexPathReplacementService>();
             services.AddScoped<IFFmpegStreamSelector, FFmpegStreamSelector>();