fix block playout history regression (#2343)

* minor tweaks

* fix block change detection bug

* history cleanup cleanup

ErsatzTV.Core.Tests/Scheduling/BlockScheduling/BlockPlayoutChangeDetectionTests.cs

@@ -72,7 +72,9 @@ public static class BlockPlayoutChangeDetectionTests
             collectionEtags = collectionEtags.Add(collectionKey1, JsonConvert.SerializeObject(collectionKey1));
             collectionEtags = collectionEtags.Add(collectionKey2, JsonConvert.SerializeObject(collectionKey2));
 
+            var buildStart = GetLocalDate(2024, 1, 17);
             Tuple<List<EffectiveBlock>, List<PlayoutItem>> result = BlockPlayoutChangeDetection.FindUpdatedItems(
+                buildStart,
                 playoutItems,
                 itemBlockKeys,
                 effectiveBlocks,
@@ -85,6 +87,65 @@ public static class BlockPlayoutChangeDetectionTests
             result.Item2.Count.ShouldBe(0);
         }
 
+        [Test]
+        public void Should_Not_Remove_Items_From_Outside_The_Scheduling_Window()
+        {
+            // This test demonstrates a bug where playout items from the past are removed
+            // because their generating block is not in the current scheduling window.
+
+            DateTimeOffset dateUpdated = DateTimeOffset.Now;
+            List<Block> blocks = Blocks(dateUpdated);
+            var template1 = new Template { Id = 1, DateUpdated = dateUpdated.UtcDateTime };
+            var playoutTemplate1 = new PlayoutTemplate { Id = 10, DateUpdated = dateUpdated.UtcDateTime };
+
+            var template2 = new Template { Id = 2, DateUpdated = dateUpdated.UtcDateTime };
+            var playoutTemplate2 = new PlayoutTemplate { Id = 20, DateUpdated = dateUpdated.UtcDateTime };
+
+            Block block1 = blocks[0]; // Yesterday's block
+            Block block2 = blocks[1]; // Today's block
+
+            var blockKey1 = new BlockKey(block1, template1, playoutTemplate1);
+            var blockKey2 = new BlockKey(block2, template2, playoutTemplate2);
+
+            var collectionKey1 = CollectionKey.ForBlockItem(block1.Items.Head());
+            var collectionKey2 = CollectionKey.ForBlockItem(block2.Items.Head());
+
+            // Playout item from yesterday
+            PlayoutItem playoutItem1 = PlayoutItem(blockKey1, collectionKey1, GetLocalDate(2024, 1, 17).AddHours(9));
+
+            List<PlayoutItem> playoutItems = [playoutItem1];
+            Dictionary<PlayoutItem, BlockKey> itemBlockKeys =
+                new()
+                {
+                    { playoutItem1, blockKey1 }
+                };
+
+            // Effective blocks for today - does not include yesterday's block
+            List<EffectiveBlock> effectiveBlocks =
+            [
+                new(block2, blockKey2, GetLocalDate(2024, 1, 18).AddHours(9), 1)
+            ];
+
+            Map<CollectionKey, string> collectionEtags = LanguageExt.Map<CollectionKey, string>.Empty;
+            collectionEtags = collectionEtags.Add(collectionKey1, JsonConvert.SerializeObject(collectionKey1));
+            collectionEtags = collectionEtags.Add(collectionKey2, JsonConvert.SerializeObject(collectionKey2));
+
+            var buildStart = GetLocalDate(2024, 1, 18);
+            Tuple<List<EffectiveBlock>, List<PlayoutItem>> result = BlockPlayoutChangeDetection.FindUpdatedItems(
+                buildStart,
+                playoutItems,
+                itemBlockKeys,
+                effectiveBlocks,
+                collectionEtags);
+
+            // should schedule today's block
+            result.Item1.Count.ShouldBe(1);
+            result.Item1.Head().Block.ShouldBe(block2);
+
+            // should NOT remove yesterday's item
+            result.Item2.Count.ShouldBe(0);
+        }
+
         private static List<Block> Blocks(DateTimeOffset dateUpdated)
         {
             List<Block> blocks =

ErsatzTV.Core/Scheduling/BlockScheduling/BlockPlayoutBuilder.cs

@@ -77,12 +77,13 @@ public class BlockPlayoutBuilder(
             result.ItemsToRemove.Add(item.Id);
         }
 
-        // // remove old items
+        // remove old items
-        // // importantly, this should not remove their history
+        // importantly, this should not remove their history
-        // result = result with { RemoveBefore = start };
+        result = result with { RemoveBefore = start };
 
         (List<EffectiveBlock> updatedEffectiveBlocks, List<PlayoutItem> playoutItemsToRemove) =
             BlockPlayoutChangeDetection.FindUpdatedItems(
+                start,
                 referenceData.ExistingItems,
                 itemBlockKeys,
                 blocksToSchedule,
@@ -231,6 +232,7 @@ public class BlockPlayoutBuilder(
                         PlaybackOrder = blockItem.PlaybackOrder,
                         Index = enumerator.State.Index,
                         When = currentTime.UtcDateTime,
+                        Finish = playoutItem.FinishOffset.UtcDateTime,
                         Key = historyKey,
                         Details = HistoryDetails.ForMediaItem(mediaItem)
                     };
@@ -333,30 +335,15 @@ public class BlockPlayoutBuilder(
         PlayoutBuildResult result)
     {
         IEnumerable<PlayoutHistory> allItemsToDelete = referenceData.PlayoutHistory
-            .Append(result.AddedHistory)
+            .GroupBy(h => h.Key)
-            .GroupBy(h => (h.BlockId, h.Key))
             .SelectMany(group => group
-                .Filter(h => h.When < start.UtcDateTime)
+                .Filter(h => h.Finish < start.UtcDateTime)
-                .OrderByDescending(h => h.When)
+                .OrderByDescending(h => h.Finish)
                 .Tail());
 
-        var addedToRemove = new System.Collections.Generic.HashSet<PlayoutHistory>();
-
         foreach (PlayoutHistory delete in allItemsToDelete)
         {
-            if (delete.Id > 0)
+            result.HistoryToRemove.Add(delete.Id);
-            {
-                result.HistoryToRemove.Add(delete.Id);
-            }
-            else
-            {
-                addedToRemove.Add(delete);
-            }
-        }
-
-        if (addedToRemove.Count > 0)
-        {
-            result.AddedHistory.RemoveAll(addedToRemove.Contains);
         }
 
         return result;

ErsatzTV.Core/Scheduling/BlockScheduling/BlockPlayoutChangeDetection.cs

@@ -24,6 +24,7 @@ internal static class BlockPlayoutChangeDetection
     }
 
     public static Tuple<List<EffectiveBlock>, List<PlayoutItem>> FindUpdatedItems(
+        DateTimeOffset buildStart,
         List<PlayoutItem> playoutItems,
         Dictionary<PlayoutItem, BlockKey> itemBlockKeys,
         List<EffectiveBlock> blocksToSchedule,
@@ -124,8 +125,8 @@ internal static class BlockPlayoutChangeDetection
             Log.Logger.Debug("Earliest block id: {Id} => {Value}", blockId, value);
         }
 
-        // find affected playout items
+        // find affected playout items (importantly, from within the effective window)
-        foreach (PlayoutItem item in playoutItems)
+        foreach (PlayoutItem item in playoutItems.Where(pi => pi.StartOffset >= buildStart))
         {
             if (!itemBlockKeys.TryGetValue(item, out BlockKey blockKey))
             {
@@ -144,7 +145,8 @@ internal static class BlockPlayoutChangeDetection
             }
         }
 
-        return Tuple(updatedBlocks.ToList(), playoutItems.Filter(i => updatedItemIds.Contains(i.Id)).ToList());
+        var itemsToRemove = playoutItems.Filter(i => updatedItemIds.Contains(i.Id)).ToList();
+        return Tuple(updatedBlocks.ToList(), itemsToRemove);
     }
 
     public static void RemoveItemAndHistory(

ErsatzTV.Core/Scheduling/BlockScheduling/BlockPlayoutFillerBuilder.cs

@@ -168,6 +168,7 @@ public class BlockPlayoutFillerBuilder(
                             PlaybackOrder = PlaybackOrder.Shuffle,
                             Index = enumerator.State.Index,
                             When = current.UtcDateTime,
+                            Finish = filler.FinishOffset.UtcDateTime,
                             Key = historyKey,
                             Details = HistoryDetails.ForMediaItem(mediaItem)
                         };

ErsatzTV.Infrastructure.MySql/Migrations/20250825155751_Populate_PlayoutHistory_Finish.cs

@@ -0,0 +1,31 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ErsatzTV.Infrastructure.MySql.Migrations
+{
+    /// <inheritdoc />
+    public partial class Populate_PlayoutHistory_Finish : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.Sql(
+                @"UPDATE PlayoutHistory
+                    SET Finish =
+                            (SELECT PlayoutItem.Finish
+                             FROM PlayoutItem
+                             WHERE PlayoutItem.PlayoutId = PlayoutHistory.PlayoutId
+                               AND PlayoutItem.Start = PlayoutHistory.`When`)
+                    WHERE EXISTS (SELECT 1
+                                  FROM PlayoutItem
+                                  WHERE PlayoutItem.PlayoutId = PlayoutHistory.PlayoutId
+                                    AND PlayoutItem.Start = PlayoutHistory.`When`);");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+        }
+    }
+}

ErsatzTV.Infrastructure.Sqlite/Migrations/20250825155536_Populate_PlayoutHistory_Finish.cs

@@ -0,0 +1,31 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ErsatzTV.Infrastructure.Sqlite.Migrations
+{
+    /// <inheritdoc />
+    public partial class Populate_PlayoutHistory_Finish : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.Sql(
+                @"UPDATE PlayoutHistory
+                    SET Finish =
+                            (SELECT PlayoutItem.Finish
+                             FROM PlayoutItem
+                             WHERE PlayoutItem.PlayoutId = PlayoutHistory.PlayoutId
+                               AND PlayoutItem.Start = PlayoutHistory.`When`)
+                    WHERE EXISTS (SELECT 1
+                                  FROM PlayoutItem
+                                  WHERE PlayoutItem.PlayoutId = PlayoutHistory.PlayoutId
+                                    AND PlayoutItem.Start = PlayoutHistory.`When`);");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+        }
+    }
+}

ErsatzTV/Controllers/Api/ChannelController.cs

@@ -32,13 +32,13 @@ public class ChannelController(ChannelWriter<IBackgroundServiceRequest> workerCh
 
     // for debugging by fast-forwarding a playout
     // [HttpPost("/api/channels/{channelNumber}/playout/continue")]
-    // public async Task<IActionResult> ContinuePlayout(string channelNumber)
+    // public async Task<IActionResult> ContinuePlayout(string channelNumber, [FromQuery] int days = 1)
     // {
     //     Option<int> maybePlayoutId = await mediator.Send(new GetPlayoutIdByChannelNumber(channelNumber));
     //     foreach (int playoutId in maybePlayoutId)
     //     {
     //         DateTimeOffset start = DateTimeOffset.Now;
-    //         for (int i = 0; i < 24; i++)
+    //         for (int i = 0; i < 24 * days; i++)
     //         {
     //             await workerChannel.WriteAsync(new BuildPlayout(playoutId, PlayoutBuildMode.Continue, start));
     //             start += TimeSpan.FromHours(1);