cleanup some exceptions; add health check (#2495)

* handle artwork timeouts so they aren't reported * catch some more cancellation errors * add free space validation on startup * add downgrade health check * update dependencies
2025-10-07 09:50:55 -05:00
parent 6a38c91d54
commit 9016523757
20 changed files with 525 additions and 327 deletions
@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [Unreleased]
 ### Added
 - Add search field to filter blocks table
+- Show full error/exception details in playback troubleshooting logs
+- Add basic free space validation on startup
+  - ETV will now fail to start with less than 128 MB free space in config or transcode folders
+- Add downgrade health check to inform users when they are doing something that WILL impact stability

 ### Fixed
 - Do not allow deleting ffmpeg profiles that are used by channels
@@ -16,7 +20,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

 ### Changed
 - Use table instead of tree view on blocks page
- Show full error/exception details in playback troubleshooting logs

 ## [25.7.0] - 2025-09-14
 ### Added
@@ -46,247 +46,261 @@ public class RefreshChannelDataHandler : IRequestHandler<RefreshChannelData>

    public async Task Handle(RefreshChannelData request, CancellationToken cancellationToken)
    {
-        _logger.LogDebug("Refreshing channel data (XMLTV) for channel {Channel}", request.ChannelNumber);
-
-        _localFileSystem.EnsureFolderExists(FileSystemLayout.ChannelGuideCacheFolder);
-
-        string targetFile = Path.Combine(FileSystemLayout.ChannelGuideCacheFolder, $"{request.ChannelNumber}.xml");
-        await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
-
-        int hiddenCount = await dbContext.Channels
-            .Where(c => c.Number == request.ChannelNumber && c.ShowInEpg == false)
-            .CountAsync(cancellationToken);
-        if (hiddenCount > 0)
+        try
        {
-            File.Delete(targetFile);
-            return;
-        }
+            _logger.LogDebug("Refreshing channel data (XMLTV) for channel {Channel}", request.ChannelNumber);

-        string movieTemplateFileName = GetMovieTemplateFileName();
-        string episodeTemplateFileName = GetEpisodeTemplateFileName();
-        string musicVideoTemplateFileName = GetMusicVideoTemplateFileName();
-        string songTemplateFileName = GetSongTemplateFileName();
-        string otherVideoTemplateFileName = GetOtherVideoTemplateFileName();
-        if (movieTemplateFileName is null || episodeTemplateFileName is null || musicVideoTemplateFileName is null ||
-            songTemplateFileName is null || otherVideoTemplateFileName is null)
-        {
-            return;
-        }
+            _localFileSystem.EnsureFolderExists(FileSystemLayout.ChannelGuideCacheFolder);

-        var minifier = new XmlMinifier(
-            new XmlMinificationSettings
+            string targetFile = Path.Combine(FileSystemLayout.ChannelGuideCacheFolder, $"{request.ChannelNumber}.xml");
+            await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
+
+            int hiddenCount = await dbContext.Channels
+                .Where(c => c.Number == request.ChannelNumber && c.ShowInEpg == false)
+                .CountAsync(cancellationToken);
+            if (hiddenCount > 0)
            {
-                MinifyWhitespace = true,
-                RemoveXmlComments = true,
-                CollapseTagsWithoutContent = true
-            });
-
-        var templateContext = new XmlTemplateContext();
-
-        string movieText = await File.ReadAllTextAsync(movieTemplateFileName, cancellationToken);
-        var movieTemplate = Template.Parse(movieText, movieTemplateFileName);
-
-        string episodeText = await File.ReadAllTextAsync(episodeTemplateFileName, cancellationToken);
-        var episodeTemplate = Template.Parse(episodeText, episodeTemplateFileName);
-
-        string musicVideoText = await File.ReadAllTextAsync(musicVideoTemplateFileName, cancellationToken);
-        var musicVideoTemplate = Template.Parse(musicVideoText, musicVideoTemplateFileName);
-
-        string songText = await File.ReadAllTextAsync(songTemplateFileName, cancellationToken);
-        var songTemplate = Template.Parse(songText, songTemplateFileName);
-
-        string otherVideoText = await File.ReadAllTextAsync(otherVideoTemplateFileName, cancellationToken);
-        var otherVideoTemplate = Template.Parse(otherVideoText, otherVideoTemplateFileName);
-
-        TimeSpan playoutOffset = TimeSpan.Zero;
-        string mirrorChannelNumber = null;
-        Option<Channel> maybeChannel = await dbContext.Channels
-            .AsNoTracking()
-            .Include(c => c.MirrorSourceChannel)
-            .Filter(c => c.PlayoutSource == ChannelPlayoutSource.Mirror && c.MirrorSourceChannelId != null)
-            .SelectOneAsync(c => c.Number == request.ChannelNumber, c => c.Number == request.ChannelNumber, cancellationToken);
-        foreach (Channel channel in maybeChannel)
-        {
-            mirrorChannelNumber = channel.MirrorSourceChannel.Number;
-            playoutOffset = channel.PlayoutOffset ?? TimeSpan.Zero;
-        }
-
-        List<Playout> playouts = await dbContext.Playouts
-            .AsNoTracking()
-            .Filter(pi => pi.Channel.Number == (mirrorChannelNumber ?? request.ChannelNumber))
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Episode).EpisodeMetadata)
-            .ThenInclude(em => em.Guids)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Episode).Season)
-            .ThenInclude(s => s.Show)
-            .ThenInclude(s => s.ShowMetadata)
-            .ThenInclude(sm => sm.Artwork)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Episode).Season)
-            .ThenInclude(s => s.Show)
-            .ThenInclude(s => s.ShowMetadata)
-            .ThenInclude(sm => sm.Genres)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Episode).Season)
-            .ThenInclude(s => s.Show)
-            .ThenInclude(s => s.ShowMetadata)
-            .ThenInclude(sm => sm.Guids)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Movie).MovieMetadata)
-            .ThenInclude(mm => mm.Artwork)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Movie).MovieMetadata)
-            .ThenInclude(mm => mm.Genres)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Movie).MovieMetadata)
-            .ThenInclude(mm => mm.Guids)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
-            .ThenInclude(mm => mm.Artwork)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
-            .ThenInclude(mvm => mvm.Genres)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
-            .ThenInclude(mvm => mvm.Studios)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
-            .ThenInclude(mvm => mvm.Directors)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
-            .ThenInclude(mvm => mvm.Artists)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as MusicVideo).Artist)
-            .ThenInclude(a => a.ArtistMetadata)
-            .ThenInclude(am => am.Genres)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as OtherVideo).OtherVideoMetadata)
-            .ThenInclude(vm => vm.Artwork)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Song).SongMetadata)
-            .ThenInclude(vm => vm.Artwork)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Song).SongMetadata)
-            .ThenInclude(sm => sm.Genres)
-            .Include(p => p.Items)
-            .ThenInclude(i => i.MediaItem)
-            .ThenInclude(i => (i as Song).SongMetadata)
-            .ThenInclude(sm => sm.Studios)
-            .ToListAsync(cancellationToken);
-
-        await using RecyclableMemoryStream ms = _recyclableMemoryStreamManager.GetStream();
-        await using var xml = XmlWriter.Create(
-            ms,
-            new XmlWriterSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment });
-
-        int daysToBuild = await _configElementRepository
-            .GetValue<int>(ConfigElementKey.XmltvDaysToBuild, cancellationToken)
-            .IfNoneAsync(2);
-
-        DateTimeOffset finish = DateTimeOffset.UtcNow.AddDays(daysToBuild);
-
-        foreach (Playout playout in playouts)
-        {
-            switch (playout.ScheduleKind)
-            {
-                case PlayoutScheduleKind.Classic:
-                case PlayoutScheduleKind.Sequential:
-                case PlayoutScheduleKind.Scripted:
-                    var floodSorted = playouts
-                        .Collect(p => p.Items)
-                        .OrderBy(pi => pi.Start)
-                        .Filter(pi => pi.StartOffset <= finish)
-                        .ToList();
-                    foreach (var item in floodSorted)
-                    {
-                        item.Start += playoutOffset;
-                        item.Finish += playoutOffset;
-                    }
-                    await WritePlayoutXml(
-                        request,
-                        floodSorted,
-                        templateContext,
-                        movieTemplate,
-                        episodeTemplate,
-                        musicVideoTemplate,
-                        songTemplate,
-                        otherVideoTemplate,
-                        minifier,
-                        xml,
-                        cancellationToken);
-                    break;
-                case PlayoutScheduleKind.Block:
-                    var blockSorted = playouts
-                        .Collect(p => p.Items)
-                        .OrderBy(pi => pi.Start)
-                        .Filter(pi => pi.StartOffset <= finish)
-                        .ToList();
-                    foreach (var item in blockSorted)
-                    {
-                        item.Start += playoutOffset;
-                        item.Finish += playoutOffset;
-                    }
-                    await WriteBlockPlayoutXml(
-                        request,
-                        blockSorted,
-                        templateContext,
-                        movieTemplate,
-                        episodeTemplate,
-                        musicVideoTemplate,
-                        songTemplate,
-                        otherVideoTemplate,
-                        minifier,
-                        xml,
-                        cancellationToken);
-                    break;
-                case PlayoutScheduleKind.ExternalJson:
-                    var externalJsonSorted = (await CollectExternalJsonItems(playout.ScheduleFile))
-                        .Filter(pi => pi.StartOffset <= finish)
-                        .ToList();
-                    foreach (var item in externalJsonSorted)
-                    {
-                        item.Start += playoutOffset;
-                        item.Finish += playoutOffset;
-                    }
-                    await WritePlayoutXml(
-                        request,
-                        externalJsonSorted,
-                        templateContext,
-                        movieTemplate,
-                        episodeTemplate,
-                        musicVideoTemplate,
-                        songTemplate,
-                        otherVideoTemplate,
-                        minifier,
-                        xml,
-                        cancellationToken);
-                    break;
+                File.Delete(targetFile);
+                return;
            }
+
+            string movieTemplateFileName = GetMovieTemplateFileName();
+            string episodeTemplateFileName = GetEpisodeTemplateFileName();
+            string musicVideoTemplateFileName = GetMusicVideoTemplateFileName();
+            string songTemplateFileName = GetSongTemplateFileName();
+            string otherVideoTemplateFileName = GetOtherVideoTemplateFileName();
+            if (movieTemplateFileName is null || episodeTemplateFileName is null ||
+                musicVideoTemplateFileName is null ||
+                songTemplateFileName is null || otherVideoTemplateFileName is null)
+            {
+                return;
+            }
+
+            var minifier = new XmlMinifier(
+                new XmlMinificationSettings
+                {
+                    MinifyWhitespace = true,
+                    RemoveXmlComments = true,
+                    CollapseTagsWithoutContent = true
+                });
+
+            var templateContext = new XmlTemplateContext();
+
+            string movieText = await File.ReadAllTextAsync(movieTemplateFileName, cancellationToken);
+            var movieTemplate = Template.Parse(movieText, movieTemplateFileName);
+
+            string episodeText = await File.ReadAllTextAsync(episodeTemplateFileName, cancellationToken);
+            var episodeTemplate = Template.Parse(episodeText, episodeTemplateFileName);
+
+            string musicVideoText = await File.ReadAllTextAsync(musicVideoTemplateFileName, cancellationToken);
+            var musicVideoTemplate = Template.Parse(musicVideoText, musicVideoTemplateFileName);
+
+            string songText = await File.ReadAllTextAsync(songTemplateFileName, cancellationToken);
+            var songTemplate = Template.Parse(songText, songTemplateFileName);
+
+            string otherVideoText = await File.ReadAllTextAsync(otherVideoTemplateFileName, cancellationToken);
+            var otherVideoTemplate = Template.Parse(otherVideoText, otherVideoTemplateFileName);
+
+            TimeSpan playoutOffset = TimeSpan.Zero;
+            string mirrorChannelNumber = null;
+            Option<Channel> maybeChannel = await dbContext.Channels
+                .AsNoTracking()
+                .Include(c => c.MirrorSourceChannel)
+                .Filter(c => c.PlayoutSource == ChannelPlayoutSource.Mirror && c.MirrorSourceChannelId != null)
+                .SelectOneAsync(
+                    c => c.Number == request.ChannelNumber,
+                    c => c.Number == request.ChannelNumber,
+                    cancellationToken);
+            foreach (Channel channel in maybeChannel)
+            {
+                mirrorChannelNumber = channel.MirrorSourceChannel.Number;
+                playoutOffset = channel.PlayoutOffset ?? TimeSpan.Zero;
+            }
+
+            List<Playout> playouts = await dbContext.Playouts
+                .AsNoTracking()
+                .Filter(pi => pi.Channel.Number == (mirrorChannelNumber ?? request.ChannelNumber))
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Episode).EpisodeMetadata)
+                .ThenInclude(em => em.Guids)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Episode).Season)
+                .ThenInclude(s => s.Show)
+                .ThenInclude(s => s.ShowMetadata)
+                .ThenInclude(sm => sm.Artwork)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Episode).Season)
+                .ThenInclude(s => s.Show)
+                .ThenInclude(s => s.ShowMetadata)
+                .ThenInclude(sm => sm.Genres)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Episode).Season)
+                .ThenInclude(s => s.Show)
+                .ThenInclude(s => s.ShowMetadata)
+                .ThenInclude(sm => sm.Guids)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Movie).MovieMetadata)
+                .ThenInclude(mm => mm.Artwork)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Movie).MovieMetadata)
+                .ThenInclude(mm => mm.Genres)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Movie).MovieMetadata)
+                .ThenInclude(mm => mm.Guids)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
+                .ThenInclude(mm => mm.Artwork)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
+                .ThenInclude(mvm => mvm.Genres)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
+                .ThenInclude(mvm => mvm.Studios)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
+                .ThenInclude(mvm => mvm.Directors)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as MusicVideo).MusicVideoMetadata)
+                .ThenInclude(mvm => mvm.Artists)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as MusicVideo).Artist)
+                .ThenInclude(a => a.ArtistMetadata)
+                .ThenInclude(am => am.Genres)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as OtherVideo).OtherVideoMetadata)
+                .ThenInclude(vm => vm.Artwork)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Song).SongMetadata)
+                .ThenInclude(vm => vm.Artwork)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Song).SongMetadata)
+                .ThenInclude(sm => sm.Genres)
+                .Include(p => p.Items)
+                .ThenInclude(i => i.MediaItem)
+                .ThenInclude(i => (i as Song).SongMetadata)
+                .ThenInclude(sm => sm.Studios)
+                .ToListAsync(cancellationToken);
+
+            await using RecyclableMemoryStream ms = _recyclableMemoryStreamManager.GetStream();
+            await using var xml = XmlWriter.Create(
+                ms,
+                new XmlWriterSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment });
+
+            int daysToBuild = await _configElementRepository
+                .GetValue<int>(ConfigElementKey.XmltvDaysToBuild, cancellationToken)
+                .IfNoneAsync(2);
+
+            DateTimeOffset finish = DateTimeOffset.UtcNow.AddDays(daysToBuild);
+
+            foreach (Playout playout in playouts)
+            {
+                switch (playout.ScheduleKind)
+                {
+                    case PlayoutScheduleKind.Classic:
+                    case PlayoutScheduleKind.Sequential:
+                    case PlayoutScheduleKind.Scripted:
+                        var floodSorted = playouts
+                            .Collect(p => p.Items)
+                            .OrderBy(pi => pi.Start)
+                            .Filter(pi => pi.StartOffset <= finish)
+                            .ToList();
+                        foreach (var item in floodSorted)
+                        {
+                            item.Start += playoutOffset;
+                            item.Finish += playoutOffset;
+                        }
+
+                        await WritePlayoutXml(
+                            request,
+                            floodSorted,
+                            templateContext,
+                            movieTemplate,
+                            episodeTemplate,
+                            musicVideoTemplate,
+                            songTemplate,
+                            otherVideoTemplate,
+                            minifier,
+                            xml,
+                            cancellationToken);
+                        break;
+                    case PlayoutScheduleKind.Block:
+                        var blockSorted = playouts
+                            .Collect(p => p.Items)
+                            .OrderBy(pi => pi.Start)
+                            .Filter(pi => pi.StartOffset <= finish)
+                            .ToList();
+                        foreach (var item in blockSorted)
+                        {
+                            item.Start += playoutOffset;
+                            item.Finish += playoutOffset;
+                        }
+
+                        await WriteBlockPlayoutXml(
+                            request,
+                            blockSorted,
+                            templateContext,
+                            movieTemplate,
+                            episodeTemplate,
+                            musicVideoTemplate,
+                            songTemplate,
+                            otherVideoTemplate,
+                            minifier,
+                            xml,
+                            cancellationToken);
+                        break;
+                    case PlayoutScheduleKind.ExternalJson:
+                        var externalJsonSorted = (await CollectExternalJsonItems(playout.ScheduleFile))
+                            .Filter(pi => pi.StartOffset <= finish)
+                            .ToList();
+                        foreach (var item in externalJsonSorted)
+                        {
+                            item.Start += playoutOffset;
+                            item.Finish += playoutOffset;
+                        }
+
+                        await WritePlayoutXml(
+                            request,
+                            externalJsonSorted,
+                            templateContext,
+                            movieTemplate,
+                            episodeTemplate,
+                            musicVideoTemplate,
+                            songTemplate,
+                            otherVideoTemplate,
+                            minifier,
+                            xml,
+                            cancellationToken);
+                        break;
+                }
+            }
+
+            await xml.FlushAsync();
+
+            string tempFile = Path.GetTempFileName();
+            await File.WriteAllBytesAsync(tempFile, ms.ToArray(), cancellationToken);
+
+            File.Move(tempFile, targetFile, true);
+        }
+        catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
+        {
+            // do nothing
        }
-
-        await xml.FlushAsync();
-
-        string tempFile = Path.GetTempFileName();
-        await File.WriteAllBytesAsync(tempFile, ms.ToArray(), cancellationToken);
-
-        File.Move(tempFile, targetFile, true);
    }

    private async Task WritePlayoutXml(
@@ -21,7 +21,7 @@ namespace ErsatzTV.Application.Libraries;

 public abstract class CallLibraryScannerHandler<TRequest>
 {
-    private readonly int _batchSize = 100;
+    private const int BatchSize = 100;
    private readonly ChannelWriter<ISearchIndexBackgroundServiceRequest> _channel;
    private readonly IConfigElementRepository _configElementRepository;
    private readonly IDbContextFactory<TvContext> _dbContextFactory;
@@ -55,8 +55,7 @@ public abstract class CallLibraryScannerHandler<TRequest>
            using var forcefulCts = new CancellationTokenSource();

            await using CancellationTokenRegistration link =
-                cancellationToken.Register(() => forcefulCts.CancelAfter(TimeSpan.FromSeconds(10))
-                );
+                cancellationToken.Register(() => forcefulCts.CancelAfter(TimeSpan.FromSeconds(10)));

            CommandResult process = await Cli.Wrap(scanner)
                .WithArguments(arguments)
@@ -72,12 +71,14 @@ public abstract class CallLibraryScannerHandler<TRequest>

            if (_toReindex.Count > 0)
            {
+                // ReSharper disable once PossiblyMistakenUseOfCancellationToken
                await _channel.WriteAsync(new ReindexMediaItems(_toReindex.ToArray()), cancellationToken);
                _toReindex.Clear();
            }

            if (_toRemove.Count > 0)
            {
+                // ReSharper disable once PossiblyMistakenUseOfCancellationToken
                await _channel.WriteAsync(new RemoveMediaItems(_toReindex.ToArray()), cancellationToken);
                _toRemove.Clear();
            }
@@ -139,14 +140,14 @@ public abstract class CallLibraryScannerHandler<TRequest>
                    }

                    _toReindex.AddRange(progressUpdate.ItemsToReindex);
-                    if (_toReindex.Count >= _batchSize)
+                    if (_toReindex.Count >= BatchSize)
                    {
                        await _channel.WriteAsync(new ReindexMediaItems(_toReindex.ToArray()));
                        _toReindex.Clear();
                    }

                    _toRemove.AddRange(progressUpdate.ItemsToRemove);
-                    if (_toRemove.Count >= _batchSize)
+                    if (_toRemove.Count >= BatchSize)
                    {
                        await _channel.WriteAsync(new RemoveMediaItems(_toReindex.ToArray()));
                        _toRemove.Clear();
@@ -177,41 +178,48 @@ public abstract class CallLibraryScannerHandler<TRequest>

    protected async Task<Validation<BaseError, string>> Validate(TRequest request, CancellationToken cancellationToken)
    {
-        int libraryRefreshInterval = await _configElementRepository
-            .GetValue<int>(ConfigElementKey.LibraryRefreshInterval, cancellationToken)
-            .IfNoneAsync(0);
-
-        libraryRefreshInterval = Math.Clamp(libraryRefreshInterval, 0, 999_999);
-
-        await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
-
-        DateTimeOffset lastScan = await GetLastScan(dbContext, request, cancellationToken);
-        if (!ScanIsRequired(lastScan, libraryRefreshInterval, request))
+        try
        {
-            return new ScanIsNotRequired();
-        }
+            int libraryRefreshInterval = await _configElementRepository
+                .GetValue<int>(ConfigElementKey.LibraryRefreshInterval, cancellationToken)
+                .IfNoneAsync(0);

-        string executable = _runtimeInfo.IsOSPlatform(OSPlatform.Windows)
-            ? "ErsatzTV.Scanner.exe"
-            : "ErsatzTV.Scanner";
+            libraryRefreshInterval = Math.Clamp(libraryRefreshInterval, 0, 999_999);

-        string processFileName = Environment.ProcessPath ?? string.Empty;
-        string processExecutable = Path.GetFileNameWithoutExtension(processFileName);
-        string folderName = Path.GetDirectoryName(processFileName);
-        if ("dotnet".Equals(processExecutable, StringComparison.OrdinalIgnoreCase))
-        {
-            folderName = AppContext.BaseDirectory;
-        }
+            await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);

-        if (!string.IsNullOrWhiteSpace(folderName))
-        {
-            string localFileName = Path.Combine(folderName, executable);
-            if (File.Exists(localFileName))
+            DateTimeOffset lastScan = await GetLastScan(dbContext, request, cancellationToken);
+            if (!ScanIsRequired(lastScan, libraryRefreshInterval, request))
            {
-                return localFileName;
+                return new ScanIsNotRequired();
            }
-        }

-        return BaseError.New("Unable to locate ErsatzTV.Scanner executable");
+            string executable = _runtimeInfo.IsOSPlatform(OSPlatform.Windows)
+                ? "ErsatzTV.Scanner.exe"
+                : "ErsatzTV.Scanner";
+
+            string processFileName = Environment.ProcessPath ?? string.Empty;
+            string processExecutable = Path.GetFileNameWithoutExtension(processFileName);
+            string folderName = Path.GetDirectoryName(processFileName);
+            if ("dotnet".Equals(processExecutable, StringComparison.OrdinalIgnoreCase))
+            {
+                folderName = AppContext.BaseDirectory;
+            }
+
+            if (!string.IsNullOrWhiteSpace(folderName))
+            {
+                string localFileName = Path.Combine(folderName, executable);
+                if (File.Exists(localFileName))
+                {
+                    return localFileName;
+                }
+            }
+
+            return BaseError.New("Unable to locate ErsatzTV.Scanner executable");
+        }
+        catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
+        {
+            return BaseError.New("Scan was canceled");
+        }
    }
 }
@@ -22,7 +22,7 @@
        </PackageReference>
        <PackageReference Include="NSubstitute" Version="5.3.0" />
        <PackageReference Include="NUnit" Version="4.4.0" />
-        <PackageReference Include="NUnit3TestAdapter" Version="5.1.0" />
+        <PackageReference Include="NUnit3TestAdapter" Version="5.2.0" />
        <PackageReference Include="Serilog" Version="4.3.0" />
        <PackageReference Include="Serilog.Extensions.Logging" Version="9.0.2" />
        <PackageReference Include="Serilog.Sinks.Debug" Version="3.0.0" />
@@ -0,0 +1,5 @@
+namespace ErsatzTV.Core.Health.Checks;
+
+public interface IDowngradeHealthCheck : IHealthCheck
+{
+}
@@ -0,0 +1,6 @@
+namespace ErsatzTV.Core.Interfaces.Database;
+
+public interface IDatabaseMigrations
+{
+    Task<IReadOnlyList<string>> GetUnknownMigrations();
+}
@@ -12,7 +12,7 @@
        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
        <PackageReference Include="NSubstitute" Version="5.3.0" />
        <PackageReference Include="NUnit" Version="4.4.0" />
-        <PackageReference Include="NUnit3TestAdapter" Version="5.1.0" />
+        <PackageReference Include="NUnit3TestAdapter" Version="5.2.0" />
        <PackageReference Include="Shouldly" Version="4.3.0" />
        <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
        <PackageReference Include="System.Text.Encoding.Extensions" Version="4.3.0" />
@@ -12,7 +12,7 @@
        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
        <PackageReference Include="NSubstitute" Version="5.3.0" />
        <PackageReference Include="NUnit" Version="4.4.0" />
-        <PackageReference Include="NUnit3TestAdapter" Version="5.1.0" />
+        <PackageReference Include="NUnit3TestAdapter" Version="5.2.0" />
        <PackageReference Include="NUnit.Analyzers" Version="4.10.0">
            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
            <PrivateAssets>all</PrivateAssets>
@@ -0,0 +1,43 @@
+using ErsatzTV.Core.Interfaces.Database;
+using ErsatzTV.Infrastructure.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace ErsatzTV.Infrastructure.Database;
+
+public class DatabaseMigrations(IDbContextFactory<TvContext> dbContextFactory, ILogger<DatabaseMigrations> logger)
+    : IDatabaseMigrations
+{
+    private IReadOnlyList<string> _unknownMigrations;
+
+    public async Task<IReadOnlyList<string>> GetUnknownMigrations()
+    {
+        if (_unknownMigrations is not null)
+        {
+            return _unknownMigrations;
+        }
+
+        try
+        {
+            await using TvContext context = await dbContextFactory.CreateDbContextAsync();
+
+            IEnumerable<string> appliedMigrations = await context.Database.GetAppliedMigrationsAsync();
+            IEnumerable<string> definedMigrations = context.Database.GetMigrations();
+
+            _unknownMigrations = appliedMigrations.Except(definedMigrations).ToList().AsReadOnly();
+            if (_unknownMigrations.Any())
+            {
+                logger.LogCritical(
+                    "Downgrade detected! Database has migrations not known to this version of ErsatzTV: {Migrations}",
+                    _unknownMigrations);
+            }
+        }
+        catch (Exception ex)
+        {
+            logger.LogCritical(ex, "Error checking for downgrade-related database migrations");
+            _unknownMigrations = [];
+        }
+
+        return _unknownMigrations;
+    }
+}
@@ -0,0 +1,24 @@
+using ErsatzTV.Core.Health;
+using ErsatzTV.Core.Health.Checks;
+using ErsatzTV.Core.Interfaces.Database;
+
+namespace ErsatzTV.Infrastructure.Health.Checks;
+
+public class DowngradeHealthCheck(IDatabaseMigrations databaseMigrations) : BaseHealthCheck, IDowngradeHealthCheck
+{
+    public override string Title => "ErsatzTV Downgrade";
+
+    public async Task<HealthCheckResult> Check(CancellationToken cancellationToken)
+    {
+        IReadOnlyList<string> unknownMigrations = await databaseMigrations.GetUnknownMigrations();
+        if (unknownMigrations.Any())
+        {
+            return FailResult(
+                "Downgrade detected; THIS IS NOT SUPPORTED AND WILL IMPACT STABILITY",
+                "Downgrade detected",
+                new HealthCheckLink("https://ersatztv.org/docs/installation/#downgrading"));
+        }
+
+        return NotApplicableResult();
+    }
+}
@@ -28,6 +28,7 @@ public class HealthCheckService : IHealthCheckService
        IVaapiDriverHealthCheck vaapiDriverHealthCheck,
        IErrorReportsHealthCheck errorReportsHealthCheck,
        IUnifiedDockerHealthCheck unifiedDockerHealthCheck,
+        IDowngradeHealthCheck downgradeHealthCheck,
        IMemoryCache memoryCache,
        IMediator mediator,
        ILogger<HealthCheckService> logger)
@@ -37,6 +38,7 @@ public class HealthCheckService : IHealthCheckService
        _logger = logger;
        _checks =
        [
+            downgradeHealthCheck,
            macOsConfigFolderHealthCheck,
            unifiedDockerHealthCheck,
            ffmpegVersionHealthCheck,
@@ -13,7 +13,7 @@
        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
        <PackageReference Include="NSubstitute" Version="5.3.0" />
        <PackageReference Include="NUnit" Version="4.4.0" />
-        <PackageReference Include="NUnit3TestAdapter" Version="5.1.0" />
+        <PackageReference Include="NUnit3TestAdapter" Version="5.2.0" />
        <PackageReference Include="NUnit.Analyzers" Version="4.10.0">
            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
            <PrivateAssets>all</PrivateAssets>
@@ -188,22 +188,29 @@ public class ArtworkController : ControllerBase
            Left: _ => new NotFoundResult().AsTask<IActionResult>(),
            Right: async r =>
            {
-                HttpClient client = _httpClientFactory.CreateClient();
-                HttpContext.Response.RegisterForDispose(client);
-                client.DefaultRequestHeaders.Add("X-Plex-Token", r.AuthToken);
+                try
+                {
+                    HttpClient client = _httpClientFactory.CreateClient();
+                    HttpContext.Response.RegisterForDispose(client);
+                    client.DefaultRequestHeaders.Add("X-Plex-Token", r.AuthToken);

-                var fullPath = new Uri(r.Uri, transcodePath);
-                HttpResponseMessage response = await client.GetAsync(
-                    fullPath,
-                    HttpCompletionOption.ResponseHeadersRead,
-                    cancellationToken);
-                HttpContext.Response.RegisterForDispose(response);
+                    var fullPath = new Uri(r.Uri, transcodePath);
+                    HttpResponseMessage response = await client.GetAsync(
+                        fullPath,
+                        HttpCompletionOption.ResponseHeadersRead,
+                        cancellationToken);
+                    HttpContext.Response.RegisterForDispose(response);

-                Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);
+                    Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);

-                return new FileStreamResult(
-                    stream,
-                    response.Content.Headers.ContentType?.MediaType ?? "image/jpeg");
+                    return new FileStreamResult(
+                        stream,
+                        response.Content.Headers.ContentType?.MediaType ?? "image/jpeg");
+                }
+                catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
+                {
+                    return NotFound();
+                }
            });
 #endif
    }
@@ -221,21 +228,28 @@ public class ArtworkController : ControllerBase
            Left: _ => new NotFoundResult().AsTask<IActionResult>(),
            Right: async vm =>
            {
-                HttpClient client = _httpClientFactory.CreateClient();
-                HttpContext.Response.RegisterForDispose(client);
+                try
+                {
+                    HttpClient client = _httpClientFactory.CreateClient();
+                    HttpContext.Response.RegisterForDispose(client);

-                Url fullPath = JellyfinUrl.ForArtwork(vm.Address, path);
-                HttpResponseMessage response = await client.GetAsync(
-                    fullPath,
-                    HttpCompletionOption.ResponseHeadersRead,
-                    cancellationToken);
-                HttpContext.Response.RegisterForDispose(response);
+                    Url fullPath = JellyfinUrl.ForArtwork(vm.Address, path);
+                    HttpResponseMessage response = await client.GetAsync(
+                        fullPath,
+                        HttpCompletionOption.ResponseHeadersRead,
+                        cancellationToken);
+                    HttpContext.Response.RegisterForDispose(response);

-                Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);
+                    Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);

-                return new FileStreamResult(
-                    stream,
-                    response.Content.Headers.ContentType?.MediaType ?? "image/jpeg");
+                    return new FileStreamResult(
+                        stream,
+                        response.Content.Headers.ContentType?.MediaType ?? "image/jpeg");
+                }
+                catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
+                {
+                    return NotFound();
+                }
            });
 #endif
    }
@@ -253,21 +267,28 @@ public class ArtworkController : ControllerBase
            Left: _ => new NotFoundResult().AsTask<IActionResult>(),
            Right: async vm =>
            {
-                HttpClient client = _httpClientFactory.CreateClient();
-                HttpContext.Response.RegisterForDispose(client);
+                try
+                {
+                    HttpClient client = _httpClientFactory.CreateClient();
+                    HttpContext.Response.RegisterForDispose(client);

-                Url fullPath = EmbyUrl.ForArtwork(vm.Address, path);
-                HttpResponseMessage response = await client.GetAsync(
-                    fullPath,
-                    HttpCompletionOption.ResponseHeadersRead,
-                    cancellationToken);
-                HttpContext.Response.RegisterForDispose(response);
+                    Url fullPath = EmbyUrl.ForArtwork(vm.Address, path);
+                    HttpResponseMessage response = await client.GetAsync(
+                        fullPath,
+                        HttpCompletionOption.ResponseHeadersRead,
+                        cancellationToken);
+                    HttpContext.Response.RegisterForDispose(response);

-                Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);
+                    Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);

-                return new FileStreamResult(
-                    stream,
-                    response.Content.Headers.ContentType?.MediaType ?? "image/jpeg");
+                    return new FileStreamResult(
+                        stream,
+                        response.Content.Headers.ContentType?.MediaType ?? "image/jpeg");
+                }
+                catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
+                {
+                    return NotFound();
+                }
            });
 #endif
    }
@@ -88,9 +88,18 @@
                                {
                                    foreach (HealthCheckLink link in context.Link)
                                    {
-                                        <MudLink Href="@link.Link">
-                                            @context.Message
-                                        </MudLink>
+                                        if (link.Link.StartsWith("http"))
+                                        {
+                                            <MudLink Href="@link.Link" Target="_blank">
+                                                @context.Message
+                                            </MudLink>
+                                        }
+                                        else
+                                        {
+                                            <MudLink Href="@link.Link">
+                                                @context.Message
+                                            </MudLink>
+                                        }
                                    }
                                }
                                else
@@ -3,6 +3,7 @@ using System.Globalization;
 using System.Runtime.InteropServices;
 using Destructurama;
 using ErsatzTV.Core;
+using ErsatzTV.Services.Validators;
 using Serilog;
 using Serilog.Events;
 using Serilog.Sinks.SystemConsole.Themes;
@@ -126,7 +127,17 @@ public class Program
        try
        {
            Environment.SetEnvironmentVariable("DOTNET_HOSTBUILDER__RELOADCONFIGONCHANGE", "false");
-            await CreateHostBuilder(args).Build().RunAsync();
+
+            IHost host = CreateHostBuilder(args).Build();
+
+            // run environment validation and exit on failure
+            var validator = host.Services.GetRequiredService<IEnvironmentValidator>();
+            if (!await validator.Validate())
+            {
+                return 1;
+            }
+
+            await host.RunAsync();
            return 0;
        }
        catch (Exception ex)
@@ -11,4 +11,4 @@
      }
    }
  }
-}
+}
@@ -2,22 +2,13 @@

 namespace ErsatzTV.Services.RunOnce;

-public class EndpointValidatorService : BackgroundService
+public class EndpointValidatorService(ILogger<EndpointValidatorService> logger) : BackgroundService
 {
-    private readonly IConfiguration _configuration;
-    private readonly ILogger<EndpointValidatorService> _logger;
-
-    public EndpointValidatorService(IConfiguration configuration, ILogger<EndpointValidatorService> logger)
-    {
-        _configuration = configuration;
-        _logger = logger;
-    }
-
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        await Task.Yield();

-        _logger.LogInformation(
+        logger.LogInformation(
            "Server will listen on streaming port {StreamingPort}, UI port {UiPort} - try UI at {UI}",
            Settings.StreamingPort,
            Settings.UiPort,
@@ -0,0 +1,48 @@
+using ErsatzTV.Core;
+
+namespace ErsatzTV.Services.Validators;
+
+public class EnvironmentValidator(ILogger<EnvironmentValidator> logger)
+    : IEnvironmentValidator
+{
+    private const long OneHundredTwentyEightMegabytes = 128000000;
+
+    public Task<bool> Validate()
+    {
+        long configFreeSpace = long.MaxValue;
+        long transcodeFreeSpace = long.MaxValue;
+
+        try
+        {
+            var configDriveInfo = new DriveInfo(FileSystemLayout.AppDataFolder);
+            configFreeSpace = configDriveInfo.AvailableFreeSpace;
+
+            var transcodeDriveInfo = new DriveInfo(FileSystemLayout.TranscodeFolder);
+            transcodeFreeSpace = transcodeDriveInfo.AvailableFreeSpace;
+        }
+        catch (Exception ex)
+        {
+            logger.LogWarning(ex, "Unable to validate available free space");
+        }
+
+        if (configFreeSpace < OneHundredTwentyEightMegabytes)
+        {
+            logger.LogCritical(
+                "ErsatzTV requires at least 128 MB of free space at {ConfigFolder}",
+                FileSystemLayout.AppDataFolder);
+
+            return Task.FromResult(false);
+        }
+
+        if (transcodeFreeSpace < OneHundredTwentyEightMegabytes)
+        { 
+            logger.LogCritical(
+                "ErsatzTV requires at least 128 MB of free space at {TranscodeFolder}",
+                FileSystemLayout.TranscodeFolder);
+
+            return Task.FromResult(false);
+        }
+
+        return Task.FromResult(true);
+    }
+}
@@ -0,0 +1,6 @@
+namespace ErsatzTV.Services.Validators;
+
+public interface IEnvironmentValidator
+{
+    Task<bool> Validate();
+}
@@ -18,6 +18,7 @@ using ErsatzTV.Core.Graphics;
 using ErsatzTV.Core.Health;
 using ErsatzTV.Core.Health.Checks;
 using ErsatzTV.Core.Images;
+using ErsatzTV.Core.Interfaces.Database;
 using ErsatzTV.Core.Interfaces.Emby;
 using ErsatzTV.Core.Interfaces.FFmpeg;
 using ErsatzTV.Core.Interfaces.GitHub;
@@ -53,6 +54,7 @@ using ErsatzTV.Formatters;
 using ErsatzTV.Infrastructure.Data;
 using ErsatzTV.Infrastructure.Data.Repositories;
 using ErsatzTV.Infrastructure.Data.Repositories.Caching;
+using ErsatzTV.Infrastructure.Database;
 using ErsatzTV.Infrastructure.Emby;
 using ErsatzTV.Infrastructure.FFmpeg;
 using ErsatzTV.Infrastructure.GitHub;
@@ -74,6 +76,7 @@ using ErsatzTV.Infrastructure.Trakt;
 using ErsatzTV.Serialization;
 using ErsatzTV.Services;
 using ErsatzTV.Services.RunOnce;
+using ErsatzTV.Services.Validators;
 using FluentValidation;
 using FluentValidation.AspNetCore;
 using Ganss.Xss;
@@ -673,6 +676,9 @@ public class Startup

    private static void CustomServices(IServiceCollection services)
    {
+        services.AddSingleton<IEnvironmentValidator, EnvironmentValidator>();
+
+        services.AddSingleton<IDatabaseMigrations, DatabaseMigrations>();
        services.AddSingleton<IPlexSecretStore, PlexSecretStore>();
        services.AddSingleton<IPlexTvApiClient, PlexTvApiClient>(); // TODO: does this need to be singleton?
        services.AddSingleton<ITraktApiClient, TraktApiClient>();
@@ -725,6 +731,7 @@ public class Startup
        services.AddScoped<IVaapiDriverHealthCheck, VaapiDriverHealthCheck>();
        services.AddScoped<IErrorReportsHealthCheck, ErrorReportsHealthCheck>();
        services.AddScoped<IUnifiedDockerHealthCheck, UnifiedDockerHealthCheck>();
+        services.AddScoped<IDowngradeHealthCheck, DowngradeHealthCheck>();
        services.AddScoped<IHealthCheckService, HealthCheckService>();

        services.AddScoped<IChannelRepository, ChannelRepository>();