timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

fix languages that have multiple codes (#270)

Browse Source
This commit is contained in:
Jason Dove
2021-06-15 06:12:40 -05:00
committed by GitHub
parent c10bc6b184
commit b38ba14c40
21 changed files with 3836 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ErsatzTV.Infrastructure/Data/Configurations/LanguageCodeConfiguration.cs
+11
View File
@@ -0,0 +1,11 @@
using ErsatzTV.Core.Domain;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
namespace ErsatzTV.Infrastructure.Data.Configurations
{
public class LanguageCodeConfiguration : IEntityTypeConfiguration<LanguageCode>
{
public void Configure(EntityTypeBuilder<LanguageCode> builder) => builder.ToTable("LanguageCode");
}
}
ErsatzTV.Infrastructure/Data/DbInitializer.cs
+39
View File
@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
@@ -12,6 +14,43 @@ namespace ErsatzTV.Infrastructure.Data
{
public static async Task<Unit> Initialize(TvContext context, CancellationToken cancellationToken)
{
if (!context.LanguageCodes.Any())
{
var assembly = Assembly.GetEntryAssembly();
if (assembly != null)
{
await using Stream resource =
assembly.GetManifestResourceStream("ErsatzTV.Resources.ISO-639-2_utf-8.txt");
if (resource != null)
{
using var reader = new StreamReader(resource);
while (!reader.EndOfStream)
{
string line = await reader.ReadLineAsync();
if (line != null)
{
string[] split = line.Split("|");
if (split.Length == 5)
{
var languageCode = new LanguageCode
{
ThreeCode1 = split[0],
ThreeCode2 = split[1],
TwoCode = split[2],
EnglishName = split[3],
FrenchName = split[4]
};
await context.LanguageCodes.AddAsync(languageCode, cancellationToken);
}
}
}
}
}
await context.SaveChangesAsync(cancellationToken);
}
if (context.Resolutions.Any())
{
return Unit.Default;
ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs
+7
View File
@@ -5,6 +5,7 @@ using System.Threading.Tasks;
using Dapper;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Infrastructure.Extensions;
using LanguageExt;
using Microsoft.EntityFrameworkCore;
using static LanguageExt.Prelude;
@@ -110,5 +111,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
INNER JOIN Artist A on MV.ArtistId = A.Id
WHERE MediaStreamKind = 2 AND A.Id = @ArtistId",
new { ArtistId = artist.Id }).Map(result => result.ToList());
public async Task<List<string>> GetAllLanguageCodes(List<string> mediaCodes)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
return await dbContext.LanguageCodes.GetAllLanguageCodes(mediaCodes);
}
}
}
ErsatzTV.Infrastructure/Data/TvContext.cs
+1
View File
@@ -65,6 +65,7 @@ namespace ErsatzTV.Infrastructure.Data
public DbSet<PlayoutProgramScheduleAnchor> PlayoutProgramScheduleItemAnchors { get; set; }
public DbSet<FFmpegProfile> FFmpegProfiles { get; set; }
public DbSet<Resolution> Resolutions { get; set; }
public DbSet<LanguageCode> LanguageCodes { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) =>
optionsBuilder.UseLoggerFactory(_loggerFactory);
ErsatzTV.Infrastructure/Extensions/LanguageCodeQueryableExtensions.cs
+78
View File
@@ -0,0 +1,78 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using Microsoft.EntityFrameworkCore;
namespace ErsatzTV.Infrastructure.Extensions
{
public static class LanguageCodeQueryableExtensions
{
public static async Task<List<string>> GetAllLanguageCodes(
this IQueryable<LanguageCode> languageCodes,
string mediaCode)
{
if (string.IsNullOrWhiteSpace(mediaCode))
{
return new List<string>();
}
string code = mediaCode.ToLowerInvariant();
List<LanguageCode> maybeLanguages = await languageCodes
.Filter(lc => lc.ThreeCode1 == code || lc.ThreeCode2 == code)
.ToListAsync();
var result = new HashSet<string>();
foreach (LanguageCode language in maybeLanguages)
{
if (!string.IsNullOrWhiteSpace(language.ThreeCode1))
{
result.Add(language.ThreeCode1);
}
if (!string.IsNullOrWhiteSpace(language.ThreeCode2))
{
result.Add(language.ThreeCode2);
}
}
return result.ToList();
}
public static async Task<List<string>> GetAllLanguageCodes(
this IQueryable<LanguageCode> languageCodes,
List<string> mediaCodes)
{
var validCodes = mediaCodes
.Filter(c => !string.IsNullOrWhiteSpace(c))
.Map(c => c.ToLowerInvariant())
.ToList();
if (validCodes.Count == 0)
{
return new List<string>();
}
List<LanguageCode> maybeLanguages = await languageCodes
.Filter(lc => validCodes.Contains(lc.ThreeCode1) || validCodes.Contains(lc.ThreeCode2))
.ToListAsync();
var result = new HashSet<string>();
foreach (LanguageCode language in maybeLanguages)
{
if (!string.IsNullOrWhiteSpace(language.ThreeCode1))
{
result.Add(language.ThreeCode1);
}
if (!string.IsNullOrWhiteSpace(language.ThreeCode2))
{
result.Add(language.ThreeCode2);
}
}
return result.ToList();
}
}
}
ErsatzTV.Infrastructure/Migrations/20210615014552_Add_LanguageCode.Designer.cs
Generated
+2970
View File
File diff suppressed because it is too large Load Diff
ErsatzTV.Infrastructure/Migrations/20210615014552_Add_LanguageCode.cs
+33
View File
@@ -0,0 +1,33 @@
using Microsoft.EntityFrameworkCore.Migrations;
namespace ErsatzTV.Infrastructure.Migrations
{
public partial class Add_LanguageCode : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "LanguageCode",
columns: table => new
{
Id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
ThreeCode1 = table.Column<string>(type: "TEXT", nullable: true),
ThreeCode2 = table.Column<string>(type: "TEXT", nullable: true),
TwoCode = table.Column<string>(type: "TEXT", nullable: true),
EnglishName = table.Column<string>(type: "TEXT", nullable: true),
FrenchName = table.Column<string>(type: "TEXT", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_LanguageCode", x => x.Id);
});
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "LanguageCode");
}
}
}
ErsatzTV.Infrastructure/Migrations/TvContextModelSnapshot.cs
+26
View File
@@ -583,6 +583,32 @@ namespace ErsatzTV.Infrastructure.Migrations
b.ToTable("JellyfinPathReplacement");
});
modelBuilder.Entity("ErsatzTV.Core.Domain.LanguageCode", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("EnglishName")
.HasColumnType("TEXT");
b.Property<string>("FrenchName")
.HasColumnType("TEXT");
b.Property<string>("ThreeCode1")
.HasColumnType("TEXT");
b.Property<string>("ThreeCode2")
.HasColumnType("TEXT");
b.Property<string>("TwoCode")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("LanguageCode");
});
modelBuilder.Entity("ErsatzTV.Core.Domain.Library", b =>
{
b.Property<int>("Id")
ErsatzTV.Infrastructure/Search/SearchIndex.cs
+69 -82
View File
@@ -72,7 +72,7 @@ namespace ErsatzTV.Infrastructure.Search
_initialized = false;
}
public int Version => 13;
public int Version => 14;
public Task<bool> Initialize(ILocalFileSystem localFileSystem)
{
@@ -91,41 +91,6 @@ namespace ErsatzTV.Infrastructure.Search
return Task.FromResult(_initialized);
}
public async Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds)
{
_writer.DeleteAll();
foreach (int id in itemIds)
{
Option<MediaItem> maybeMediaItem = await searchRepository.GetItemToIndex(id);
if (maybeMediaItem.IsSome)
{
MediaItem mediaItem = maybeMediaItem.ValueUnsafe();
switch (mediaItem)
{
case Movie movie:
UpdateMovie(movie);
break;
case Show show:
await UpdateShow(searchRepository, show);
break;
case Artist artist:
await UpdateArtist(searchRepository, artist);
break;
case MusicVideo musicVideo:
UpdateMusicVideo(musicVideo);
break;
case Episode episode:
UpdateEpisode(episode);
break;
}
}
}
_writer.Commit();
return Unit.Default;
}
public Task<Unit> AddItems(ISearchRepository searchRepository, List<MediaItem> items) =>
UpdateItems(searchRepository, items);
@@ -136,7 +101,7 @@ namespace ErsatzTV.Infrastructure.Search
switch (item)
{
case Movie movie:
UpdateMovie(movie);
await UpdateMovie(searchRepository, movie);
break;
case Show show:
await UpdateShow(searchRepository, show);
@@ -145,10 +110,10 @@ namespace ErsatzTV.Infrastructure.Search
await UpdateArtist(searchRepository, artist);
break;
case MusicVideo musicVideo:
UpdateMusicVideo(musicVideo);
await UpdateMusicVideo(searchRepository, musicVideo);
break;
case Episode episode:
UpdateEpisode(episode);
await UpdateEpisode(searchRepository, episode);
break;
}
}
@@ -217,6 +182,41 @@ namespace ErsatzTV.Infrastructure.Search
_directory?.Dispose();
}
public async Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds)
{
_writer.DeleteAll();
foreach (int id in itemIds)
{
Option<MediaItem> maybeMediaItem = await searchRepository.GetItemToIndex(id);
if (maybeMediaItem.IsSome)
{
MediaItem mediaItem = maybeMediaItem.ValueUnsafe();
switch (mediaItem)
{
case Movie movie:
await UpdateMovie(searchRepository, movie);
break;
case Show show:
await UpdateShow(searchRepository, show);
break;
case Artist artist:
await UpdateArtist(searchRepository, artist);
break;
case MusicVideo musicVideo:
await UpdateMusicVideo(searchRepository, musicVideo);
break;
case Episode episode:
await UpdateEpisode(searchRepository, episode);
break;
}
}
}
_writer.Commit();
return Unit.Default;
}
private static Option<SearchPageMap> GetSearchPageMap(
IndexSearcher searcher,
Query query,
@@ -263,7 +263,7 @@ namespace ErsatzTV.Infrastructure.Search
return new SearchPageMap(map);
}
private void UpdateMovie(Movie movie)
private async Task UpdateMovie(ISearchRepository searchRepository, Movie movie)
{
Option<MovieMetadata> maybeMetadata = movie.MovieMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
@@ -284,7 +284,7 @@ namespace ErsatzTV.Infrastructure.Search
new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
};
AddLanguages(doc, movie.MediaVersions);
await AddLanguages(searchRepository, doc, movie.MediaVersions);
if (!string.IsNullOrWhiteSpace(metadata.ContentRating))
{
@@ -349,25 +349,38 @@ namespace ErsatzTV.Infrastructure.Search
}
}
private void AddLanguages(Document doc, List<MediaVersion> mediaVersions)
private async Task AddLanguages(ISearchRepository searchRepository, Document doc, List<MediaVersion> mediaVersions)
{
Option<MediaVersion> maybeVersion = mediaVersions.HeadOrNone();
if (maybeVersion.IsSome)
{
MediaVersion version = maybeVersion.ValueUnsafe();
foreach (CultureInfo cultureInfo in version.Streams
var mediaCodes = version.Streams
.Filter(ms => ms.MediaStreamKind == MediaStreamKind.Audio)
.Map(ms => ms.Language).Distinct()
.Filter(s => !string.IsNullOrWhiteSpace(s))
.Map(
l => _cultureInfos.Filter(
c => string.Equals(c.ThreeLetterISOLanguageName, l, StringComparison.OrdinalIgnoreCase)))
.Sequence()
.Flatten())
.ToList();
await AddLanguages(searchRepository, doc, mediaCodes);
}
}
private async Task AddLanguages(ISearchRepository searchRepository, Document doc, List<string> mediaCodes)
{
var englishNames = new System.Collections.Generic.HashSet<string>();
foreach (string code in await searchRepository.GetAllLanguageCodes(mediaCodes))
{
Option<CultureInfo> maybeCultureInfo = _cultureInfos.Find(
ci => string.Equals(ci.ThreeLetterISOLanguageName, code, StringComparison.OrdinalIgnoreCase));
foreach (CultureInfo cultureInfo in maybeCultureInfo)
{
doc.Add(new TextField(LanguageField, cultureInfo.EnglishName, Field.Store.NO));
englishNames.Add(cultureInfo.EnglishName);
}
}
foreach (string englishName in englishNames)
{
doc.Add(new TextField(LanguageField, englishName, Field.Store.NO));
}
}
private async Task UpdateShow(ISearchRepository searchRepository, Show show)
@@ -392,20 +405,7 @@ namespace ErsatzTV.Infrastructure.Search
};
List<string> languages = await searchRepository.GetLanguagesForShow(show);
foreach (CultureInfo cultureInfo in languages
.Distinct()
.Filter(s => !string.IsNullOrWhiteSpace(s))
.Map(
l => _cultureInfos.Filter(
c => string.Equals(
c.ThreeLetterISOLanguageName,
l,
StringComparison.OrdinalIgnoreCase)))
.Sequence()
.Flatten())
{
doc.Add(new TextField(LanguageField, cultureInfo.EnglishName, Field.Store.NO));
}
await AddLanguages(searchRepository, doc, languages);
if (!string.IsNullOrWhiteSpace(metadata.ContentRating))
{
@@ -482,20 +482,7 @@ namespace ErsatzTV.Infrastructure.Search
};
List<string> languages = await searchRepository.GetLanguagesForArtist(artist);
foreach (CultureInfo cultureInfo in languages
.Distinct()
.Filter(s => !string.IsNullOrWhiteSpace(s))
.Map(
l => _cultureInfos.Filter(
c => string.Equals(
c.ThreeLetterISOLanguageName,
l,
StringComparison.OrdinalIgnoreCase)))
.Sequence()
.Flatten())
{
doc.Add(new TextField(LanguageField, cultureInfo.EnglishName, Field.Store.NO));
}
await AddLanguages(searchRepository, doc, languages);
foreach (Genre genre in metadata.Genres)
{
@@ -522,7 +509,7 @@ namespace ErsatzTV.Infrastructure.Search
}
}
private void UpdateMusicVideo(MusicVideo musicVideo)
private async Task UpdateMusicVideo(ISearchRepository searchRepository, MusicVideo musicVideo)
{
Option<MusicVideoMetadata> maybeMetadata = musicVideo.MusicVideoMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
@@ -543,7 +530,7 @@ namespace ErsatzTV.Infrastructure.Search
new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
};
AddLanguages(doc, musicVideo.MediaVersions);
await AddLanguages(searchRepository, doc, musicVideo.MediaVersions);
if (metadata.ReleaseDate.HasValue)
{
@@ -584,7 +571,7 @@ namespace ErsatzTV.Infrastructure.Search
}
}
private void UpdateEpisode(Episode episode)
private async Task UpdateEpisode(ISearchRepository searchRepository, Episode episode)
{
foreach (EpisodeMetadata metadata in episode.EpisodeMetadata)
{
@@ -611,7 +598,7 @@ namespace ErsatzTV.Infrastructure.Search
doc.Add(new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO));
doc.Add(new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES));
AddLanguages(doc, episode.MediaVersions);
await AddLanguages(searchRepository, doc, episode.MediaVersions);
if (metadata.ReleaseDate.HasValue)
{