search overhaul (#87)

* add letter bar with no links

* use lucene for search, add paged search results

* add search index version

* index library_name; rebuild index when folder is missing

* maintain index as local movies change

* fix tests

* maintain index as local shows change

* maintain index as plex movies change

* maintain index as plex shows change

* code cleanup

* add duplicate filter to search

* add links to letter bar

* code cleanup
This commit is contained in:
Jason Dove
2021-03-20 15:49:50 +00:00
committed by GitHub
parent f96efa9b2f
commit 2efcbca2da
69 changed files with 1351 additions and 324 deletions
@@ -84,6 +84,12 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
@"SELECT COUNT(*) FROM MediaItem WHERE LibraryPathId = @LibraryPathId",
new { LibraryPathId = libraryPathId });
public Task<List<int>> GetMediaIdsByLocalPath(int libraryPathId) =>
_dbConnection.QueryAsync<int>(
@"SELECT Id FROM MediaItem WHERE LibraryPathId = @LibraryPathId",
new { LibraryPathId = libraryPathId })
.Map(result => result.ToList());
public async Task DeleteLocalPath(int libraryPathId)
{
await using TvContext context = _dbContextFactory.CreateDbContext();
@@ -112,8 +112,9 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
int? id = await _dbConnection.QuerySingleAsync<int?>(
@"SELECT L.MediaSourceId FROM Library L
INNER JOIN PlexLibrary PL on L.Id = PL.Id
WHERE L.Id = @PlexLibraryId", new { PlexLibraryId = plexLibraryId });
WHERE L.Id = @PlexLibraryId",
new { PlexLibraryId = plexLibraryId });
await using TvContext context = _dbContextFactory.CreateDbContext();
return await context.PlexMediaSources
.Include(p => p.Connections)
@@ -176,18 +177,18 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
public async Task<Unit> DeleteAllPlex()
{
await using TvContext context = _dbContextFactory.CreateDbContext();
List<PlexMediaSource> allMediaSources = await context.PlexMediaSources.ToListAsync();
context.PlexMediaSources.RemoveRange(allMediaSources);
List<PlexLibrary> allPlexLibraries = await context.PlexLibraries.ToListAsync();
context.PlexLibraries.RemoveRange(allPlexLibraries);
await context.SaveChangesAsync();
return Unit.Default;
}
public async Task DisablePlexLibrarySync(List<int> libraryIds)
public async Task<List<int>> DisablePlexLibrarySync(List<int> libraryIds)
{
await _dbConnection.ExecuteAsync(
"UPDATE PlexLibrary SET ShouldSyncItems = 0 WHERE Id IN @ids",
@@ -197,6 +198,14 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
"UPDATE Library SET LastScan = null WHERE Id IN @ids",
new { ids = libraryIds });
List<int> movieIds = await _dbConnection.QueryAsync<int>(
@"SELECT m.Id FROM MediaItem m
INNER JOIN PlexMovie pm ON pm.Id = m.Id
INNER JOIN LibraryPath lp ON lp.Id = m.LibraryPathId
INNER JOIN Library l ON l.Id = lp.LibraryId
WHERE l.Id IN @ids",
new { ids = libraryIds }).Map(result => result.ToList());
await _dbConnection.ExecuteAsync(
@"DELETE FROM MediaItem WHERE Id IN
(SELECT m.Id FROM MediaItem m
@@ -224,6 +233,14 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
WHERE l.Id IN @ids)",
new { ids = libraryIds });
List<int> showIds = await _dbConnection.QueryAsync<int>(
@"SELECT m.Id FROM MediaItem m
INNER JOIN PlexShow ps ON ps.Id = m.Id
INNER JOIN LibraryPath lp ON lp.Id = m.LibraryPathId
INNER JOIN Library l ON l.Id = lp.LibraryId
WHERE l.Id IN @ids",
new { ids = libraryIds }).Map(result => result.ToList());
await _dbConnection.ExecuteAsync(
@"DELETE FROM MediaItem WHERE Id IN
(SELECT m.Id FROM MediaItem m
@@ -232,6 +249,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
INNER JOIN Library l ON l.Id = lp.LibraryId
WHERE l.Id IN @ids)",
new { ids = libraryIds });
return movieIds.Append(showIds).ToList();
}
public Task EnablePlexLibrarySync(IEnumerable<int> libraryIds) =>
@@ -19,9 +19,6 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
_dbConnection = dbConnection;
}
public Task<Unit> RemoveGenre(Genre genre) =>
_dbConnection.ExecuteAsync("DELETE FROM Genre WHERE Id = @GenreId", new { GenreId = genre.Id }).ToUnit();
public async Task<bool> Update(Metadata metadata)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
@@ -53,7 +50,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
return await dbContext.SaveChangesAsync() > 0;
}
public Task<Unit> UpdatePlexStatistics(MediaVersion mediaVersion) =>
public Task<bool> UpdatePlexStatistics(MediaVersion mediaVersion) =>
_dbConnection.ExecuteAsync(
@"UPDATE MediaVersion SET
SampleAspectRatio = @SampleAspectRatio,
@@ -66,7 +63,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
mediaVersion.VideoScanKind,
mediaVersion.DateUpdated,
MediaVersionId = mediaVersion.Id
}).ToUnit();
}).Map(result => result > 0);
public Task<Unit> UpdateArtworkPath(Artwork artwork) =>
_dbConnection.ExecuteAsync(
@@ -111,5 +108,9 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
@"DELETE FROM Artwork WHERE ArtworkKind = @ArtworkKind AND (MovieMetadataId = @Id
OR ShowMetadataId = @Id OR SeasonMetadataId = @Id OR EpisodeMetadataId = @Id)",
new { ArtworkKind = artworkKind, metadata.Id }).ToUnit();
public Task<bool> RemoveGenre(Genre genre) =>
_dbConnection.ExecuteAsync("DELETE FROM Genre WHERE Id = @GenreId", new { GenreId = genre.Id })
.Map(result => result > 0);
}
}
@@ -7,6 +7,7 @@ using Dapper;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Metadata;
using LanguageExt;
using Microsoft.EntityFrameworkCore;
using static LanguageExt.Prelude;
@@ -45,7 +46,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.Map(Optional);
}
public async Task<Either<BaseError, Movie>> GetOrAdd(LibraryPath libraryPath, string path)
public async Task<Either<BaseError, MediaItemScanResult<Movie>>> GetOrAdd(LibraryPath libraryPath, string path)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
Option<Movie> maybeExisting = await dbContext.Movies
@@ -56,17 +57,22 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Tags)
.Include(i => i.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(i => i.MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.OrderBy(i => i.MediaVersions.First().MediaFiles.First().Path)
.SingleOrDefaultAsync(i => i.MediaVersions.First().MediaFiles.First().Path == path);
return await maybeExisting.Match(
mediaItem => Right<BaseError, Movie>(mediaItem).AsTask(),
mediaItem =>
Right<BaseError, MediaItemScanResult<Movie>>(
new MediaItemScanResult<Movie>(mediaItem) { IsAdded = false }).AsTask(),
async () => await AddMovie(dbContext, libraryPath.Id, path));
}
public async Task<Either<BaseError, PlexMovie>> GetOrAdd(PlexLibrary library, PlexMovie item)
public async Task<Either<BaseError, MediaItemScanResult<PlexMovie>>> GetOrAdd(
PlexLibrary library,
PlexMovie item)
{
await using TvContext context = _dbContextFactory.CreateDbContext();
Option<PlexMovie> maybeExisting = await context.PlexMovies
@@ -74,14 +80,20 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Genres)
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Tags)
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(i => i.MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.LibraryPath)
.ThenInclude(lp => lp.Library)
.OrderBy(i => i.Key)
.SingleOrDefaultAsync(i => i.Key == item.Key);
return await maybeExisting.Match(
plexMovie => Right<BaseError, PlexMovie>(plexMovie).AsTask(),
plexMovie =>
Right<BaseError, MediaItemScanResult<PlexMovie>>(
new MediaItemScanResult<PlexMovie>(plexMovie) { IsAdded = true }).AsTask(),
async () => await AddPlexMovie(context, library, item));
}
@@ -104,6 +116,17 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ToListAsync();
}
public async Task<List<MovieMetadata>> GetMoviesForCards(List<int> ids)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
return await dbContext.MovieMetadata
.AsNoTracking()
.Filter(mm => ids.Contains(mm.MovieId))
.Include(mm => mm.Artwork)
.OrderBy(mm => mm.SortTitle)
.ToListAsync();
}
public Task<IEnumerable<string>> FindMoviePaths(LibraryPath libraryPath) =>
_dbConnection.QueryAsync<string>(
@"SELECT MF.Path
@@ -114,17 +137,18 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
WHERE MI.LibraryPathId = @LibraryPathId",
new { LibraryPathId = libraryPath.Id });
public async Task<Unit> DeleteByPath(LibraryPath libraryPath, string path)
public async Task<List<int>> DeleteByPath(LibraryPath libraryPath, string path)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
IEnumerable<int> ids = await _dbConnection.QueryAsync<int>(
@"SELECT M.Id
List<int> ids = await _dbConnection.QueryAsync<int>(
@"SELECT M.Id
FROM Movie M
INNER JOIN MediaItem MI on M.Id = MI.Id
INNER JOIN MediaVersion MV on M.Id = MV.MovieId
INNER JOIN MediaFile MF on MV.Id = MF.MediaVersionId
WHERE MI.LibraryPathId = @LibraryPathId AND MF.Path = @Path",
new { LibraryPathId = libraryPath.Id, Path = path });
new { LibraryPathId = libraryPath.Id, Path = path })
.Map(result => result.ToList());
foreach (int movieId in ids)
{
@@ -132,26 +156,36 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
dbContext.Movies.Remove(movie);
}
await dbContext.SaveChangesAsync();
return Unit.Default;
bool changed = await dbContext.SaveChangesAsync() > 0;
return changed ? ids : new List<int>();
}
public Task<Unit> AddGenre(MovieMetadata metadata, Genre genre) =>
public Task<bool> AddGenre(MovieMetadata metadata, Genre genre) =>
_dbConnection.ExecuteAsync(
"INSERT INTO Genre (Name, MovieMetadataId) VALUES (@Name, @MetadataId)",
new { genre.Name, MetadataId = metadata.Id }).ToUnit();
new { genre.Name, MetadataId = metadata.Id }).Map(result => result > 0);
public Task<Unit> RemoveMissingPlexMovies(PlexLibrary library, List<string> movieKeys) =>
_dbConnection.ExecuteAsync(
public async Task<List<int>> RemoveMissingPlexMovies(PlexLibrary library, List<string> movieKeys)
{
List<int> ids = await _dbConnection.QueryAsync<int>(
@"SELECT m.Id FROM MediaItem m
INNER JOIN PlexMovie pm ON pm.Id = m.Id
INNER JOIN LibraryPath lp ON lp.Id = m.LibraryPathId
WHERE lp.LibraryId = @LibraryId AND pm.Key not in @Keys",
new { LibraryId = library.Id, Keys = movieKeys }).Map(result => result.ToList());
await _dbConnection.ExecuteAsync(
@"DELETE FROM MediaItem WHERE Id IN
(SELECT m.Id FROM MediaItem m
INNER JOIN PlexMovie pm ON pm.Id = m.Id
INNER JOIN LibraryPath lp ON lp.Id = m.LibraryPathId
WHERE lp.LibraryId = @LibraryId AND pm.Key not in @Keys)",
new { LibraryId = library.Id, Keys = movieKeys }).ToUnit();
new { LibraryId = library.Id, Keys = movieKeys });
private static async Task<Either<BaseError, Movie>> AddMovie(
return ids;
}
private static async Task<Either<BaseError, MediaItemScanResult<Movie>>> AddMovie(
TvContext dbContext,
int libraryPathId,
string path)
@@ -175,7 +209,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
await dbContext.Movies.AddAsync(movie);
await dbContext.SaveChangesAsync();
await dbContext.Entry(movie).Reference(m => m.LibraryPath).LoadAsync();
return movie;
await dbContext.Entry(movie.LibraryPath).Reference(lp => lp.Library).LoadAsync();
return new MediaItemScanResult<Movie>(movie) { IsAdded = true };
}
catch (Exception ex)
{
@@ -183,7 +218,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
}
}
private async Task<Either<BaseError, PlexMovie>> AddPlexMovie(
private async Task<Either<BaseError, MediaItemScanResult<PlexMovie>>> AddPlexMovie(
TvContext context,
PlexLibrary library,
PlexMovie item)
@@ -195,7 +230,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
await context.PlexMovies.AddAsync(item);
await context.SaveChangesAsync();
await context.Entry(item).Reference(i => i.LibraryPath).LoadAsync();
return item;
await context.Entry(item.LibraryPath).Reference(lp => lp.Library).LoadAsync();
return new MediaItemScanResult<PlexMovie>(item) { IsAdded = true };
}
catch (Exception ex)
{
@@ -21,6 +21,24 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
_dbConnection = dbConnection;
}
public async Task<List<MediaItem>> GetItemsToIndex()
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
return await dbContext.MediaItems
.AsNoTracking()
.Include(mi => mi.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Tags)
.ToListAsync();
}
public async Task<List<MediaItem>> SearchMediaItemsByTitle(string query)
{
List<int> ids = await _dbConnection.QueryAsync<int>(
@@ -7,6 +7,7 @@ using Dapper;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Metadata;
using LanguageExt;
using Microsoft.EntityFrameworkCore;
using static LanguageExt.Prelude;
@@ -82,6 +83,17 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ToListAsync();
}
public async Task<List<ShowMetadata>> GetShowsForCards(List<int> ids)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
return await dbContext.ShowMetadata
.AsNoTracking()
.Filter(sm => ids.Contains(sm.ShowId))
.Include(sm => sm.Artwork)
.OrderBy(sm => sm.SortTitle)
.ToListAsync();
}
public async Task<List<Season>> GetAllSeasons()
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
@@ -201,6 +213,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ThenInclude(sm => sm.Genres)
.Include(s => s.ShowMetadata)
.ThenInclude(sm => sm.Tags)
.Include(s => s.LibraryPath)
.ThenInclude(lp => lp.Library)
.OrderBy(s => s.Id)
.SingleOrDefaultAsync(s => s.Id == id)
.Map(Optional);
@@ -208,7 +222,10 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
() => Option<Show>.None.AsTask());
}
public async Task<Either<BaseError, Show>> AddShow(int libraryPathId, string showFolder, ShowMetadata metadata)
public async Task<Either<BaseError, MediaItemScanResult<Show>>> AddShow(
int libraryPathId,
string showFolder,
ShowMetadata metadata)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
@@ -226,8 +243,10 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
await dbContext.Shows.AddAsync(show);
await dbContext.SaveChangesAsync();
await dbContext.Entry(show).Reference(s => s.LibraryPath).LoadAsync();
await dbContext.Entry(show.LibraryPath).Reference(lp => lp.Library).LoadAsync();
return show;
return new MediaItemScanResult<Show>(show) { IsAdded = true };
}
catch (Exception ex)
{
@@ -314,19 +333,22 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
return Unit.Default;
}
public async Task<Unit> DeleteEmptyShows(LibraryPath libraryPath)
public async Task<List<int>> DeleteEmptyShows(LibraryPath libraryPath)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
List<Show> shows = await dbContext.Shows
.Filter(s => s.LibraryPathId == libraryPath.Id)
.Filter(s => s.Seasons.Count == 0)
.ToListAsync();
var ids = shows.Map(s => s.Id).ToList();
dbContext.Shows.RemoveRange(shows);
await dbContext.SaveChangesAsync();
return Unit.Default;
return ids;
}
public async Task<Either<BaseError, PlexShow>> GetOrAddPlexShow(PlexLibrary library, PlexShow item)
public async Task<Either<BaseError, MediaItemScanResult<PlexShow>>> GetOrAddPlexShow(
PlexLibrary library,
PlexShow item)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
Option<PlexShow> maybeExisting = await dbContext.PlexShows
@@ -334,12 +356,17 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.Include(i => i.ShowMetadata)
.ThenInclude(mm => mm.Genres)
.Include(i => i.ShowMetadata)
.ThenInclude(mm => mm.Tags)
.Include(i => i.ShowMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(i => i.LibraryPath)
.ThenInclude(lp => lp.Library)
.OrderBy(i => i.Key)
.SingleOrDefaultAsync(i => i.Key == item.Key);
return await maybeExisting.Match(
plexShow => Right<BaseError, PlexShow>(plexShow).AsTask(),
plexShow => Right<BaseError, MediaItemScanResult<PlexShow>>(
new MediaItemScanResult<PlexShow>(plexShow) { IsAdded = true }).AsTask(),
async () => await AddPlexShow(dbContext, library, item));
}
@@ -375,20 +402,6 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
async () => await AddPlexEpisode(dbContext, library, item));
}
public Task<Unit> AddGenre(ShowMetadata metadata, Genre genre) =>
_dbConnection.ExecuteAsync(
"INSERT INTO Genre (Name, SeasonMetadataId) VALUES (@Name, @MetadataId)",
new { genre.Name, MetadataId = metadata.Id }).ToUnit();
public Task<Unit> RemoveMissingPlexShows(PlexLibrary library, List<string> showKeys) =>
_dbConnection.ExecuteAsync(
@"DELETE FROM MediaItem WHERE Id IN
(SELECT m.Id FROM MediaItem m
INNER JOIN PlexShow ps ON ps.Id = m.Id
INNER JOIN LibraryPath lp ON lp.Id = m.LibraryPathId
WHERE lp.LibraryId = @LibraryId AND ps.Key not in @Keys)",
new { LibraryId = library.Id, Keys = showKeys }).ToUnit();
public Task<Unit> RemoveMissingPlexSeasons(string showKey, List<string> seasonKeys) =>
_dbConnection.ExecuteAsync(
@"DELETE FROM MediaItem WHERE Id IN
@@ -414,7 +427,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
episode.EpisodeNumber = episodeNumber;
await _dbConnection.ExecuteAsync(
@"UPDATE Episode SET EpisodeNumber = @EpisodeNumber WHERE Id = @Id",
new { EpisodeNumber = episodeNumber, Id = episode.Id });
new { EpisodeNumber = episodeNumber, episode.Id });
return Unit.Default;
}
@@ -449,6 +462,31 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ToListAsync();
}
public Task<bool> AddGenre(ShowMetadata metadata, Genre genre) =>
_dbConnection.ExecuteAsync(
"INSERT INTO Genre (Name, SeasonMetadataId) VALUES (@Name, @MetadataId)",
new { genre.Name, MetadataId = metadata.Id }).Map(result => result > 0);
public async Task<List<int>> RemoveMissingPlexShows(PlexLibrary library, List<string> showKeys)
{
List<int> ids = await _dbConnection.QueryAsync<int>(
@"SELECT m.Id FROM MediaItem m
INNER JOIN PlexShow ps ON ps.Id = m.Id
INNER JOIN LibraryPath lp ON lp.Id = m.LibraryPathId
WHERE lp.LibraryId = @LibraryId AND ps.Key not in @Keys",
new { LibraryId = library.Id, Keys = showKeys }).Map(result => result.ToList());
await _dbConnection.ExecuteAsync(
@"DELETE FROM MediaItem WHERE Id IN
(SELECT m.Id FROM MediaItem m
INNER JOIN PlexShow ps ON ps.Id = m.Id
INNER JOIN LibraryPath lp ON lp.Id = m.LibraryPathId
WHERE lp.LibraryId = @LibraryId AND ps.Key not in @Keys)",
new { LibraryId = library.Id, Keys = showKeys });
return ids;
}
private static async Task<Either<BaseError, Season>> AddSeason(
TvContext dbContext,
Show show,
@@ -519,7 +557,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
}
}
private static async Task<Either<BaseError, PlexShow>> AddPlexShow(
private static async Task<Either<BaseError, MediaItemScanResult<PlexShow>>> AddPlexShow(
TvContext dbContext,
PlexLibrary library,
PlexShow item)
@@ -531,7 +569,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
await dbContext.PlexShows.AddAsync(item);
await dbContext.SaveChangesAsync();
await dbContext.Entry(item).Reference(i => i.LibraryPath).LoadAsync();
return item;
await dbContext.Entry(item.LibraryPath).Reference(lp => lp.Library).LoadAsync();
return new MediaItemScanResult<PlexShow>(item) { IsAdded = true };
}
catch (Exception ex)
{