feat: implement missing job status verification and handling

- Added `IsJobMissing` functionality to `IJobService.cs` interface to verify if a job is missing. - Incorporated handling for `VideoStatus.Missing` in `FC2Service.cs`, `TwitcastingService.cs` & `TwitchService.cs` to warn about possible server malfunction during previous recording and revert video status to `WaitingToRecord`. - Added methods to `ACIService.cs` & `KubernetesService.cs` to check if a job is missing. - Improved readability and safety of `GetResourceByKeywordAsync` function in `ACIService.cs`. - Included checking for missing job status and updating video status accordingly in `RecordService.cs`. Signed-off-by: 陳鈞 <jim60105@gmail.com>
Recorder-moe · May 23, 2024 · fdab544 · fdab544
1 parent 0aaab8a
commit fdab544
Show file tree

Hide file tree

Showing 7 changed files with 105 additions and 7 deletions.
diff --git a/Interfaces/IJobService.cs b/Interfaces/IJobService.cs
@@ -4,6 +4,8 @@ namespace LivestreamRecorderService.Interfaces;
 
 public interface IJobService
 {
+    Task<bool> IsJobMissing(Video video, CancellationToken cancellation);
+    Task<bool> IsJobMissing(string keyword, CancellationToken cancellation);
     Task<bool> IsJobFailedAsync(Video video, CancellationToken cancellation = default);
     Task<bool> IsJobFailedAsync(string keyword, CancellationToken cancellation = default);
     Task<bool> IsJobSucceededAsync(Video video, CancellationToken cancellation = default);

diff --git a/ScopedServices/PlatformService/FC2Service.cs b/ScopedServices/PlatformService/FC2Service.cs
@@ -84,6 +84,13 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo
                     case VideoStatus.Skipped:
                         logger.LogTrace("{videoId} is rejected for recording.", video.id);
                         return;
+                    case VideoStatus.Missing:
+                        logger.LogWarning(
+                            "{videoId} has been marked missing. It is possible that a server malfunction occurred during the previous recording. Changed its state back to Recording.",
+                            video.id);
+
+                        video.Status = VideoStatus.WaitingToRecord;
+                        break;
                     case VideoStatus.Archived:
                     case VideoStatus.PermanentArchived:
                         logger.LogWarning(
@@ -98,8 +105,8 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo
                     case VideoStatus.Pending:
                     case VideoStatus.WaitingToDownload:
                     case VideoStatus.Downloading:
+                    //case VideoStatus.Uploading:
                     case VideoStatus.Expired:
-                    case VideoStatus.Missing:
                     case VideoStatus.Error:
                     case VideoStatus.Exist:
                     case VideoStatus.Edited:

diff --git a/ScopedServices/PlatformService/TwitcastingService.cs b/ScopedServices/PlatformService/TwitcastingService.cs
@@ -71,6 +71,13 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo
                 case VideoStatus.Skipped:
                     logger.LogTrace("{videoId} is rejected for recording.", video.id);
                     return;
+                case VideoStatus.Missing:
+                    logger.LogWarning(
+                        "{videoId} has been marked missing. It is possible that a server malfunction occurred during the previous recording. Changed its state back to Recording.",
+                        video.id);
+
+                    video.Status = VideoStatus.WaitingToRecord;
+                    break;
                 case VideoStatus.Archived:
                 case VideoStatus.PermanentArchived:
                     logger.LogWarning(
@@ -79,7 +86,20 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo
 
                     video.Status = VideoStatus.WaitingToRecord;
                     break;
+
+                case VideoStatus.Unknown:
+                case VideoStatus.Scheduled:
+                case VideoStatus.Pending:
+                case VideoStatus.WaitingToDownload:
+                case VideoStatus.Downloading:
+                //case VideoStatus.Uploading:
+                case VideoStatus.Expired:
+                case VideoStatus.Error:
+                case VideoStatus.Exist:
+                case VideoStatus.Edited:
+                case VideoStatus.Deleted:
                 default:
+                    // All cases should be handled
                     logger.LogWarning("{videoId} is in {status}.", video.id, Enum.GetName(typeof(VideoStatus), video.Status));
                     return;
             }

diff --git a/ScopedServices/PlatformService/TwitchService.cs b/ScopedServices/PlatformService/TwitchService.cs
@@ -82,6 +82,33 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo
                     case VideoStatus.Skipped:
                         logger.LogTrace("{videoId} is rejected for recording.", video.id);
                         return;
+                    case VideoStatus.Missing:
+                        logger.LogWarning(
+                            "{videoId} has been marked missing. It is possible that a server malfunction occurred during the previous recording. Changed its state back to Recording.",
+                            video.id);
+
+                        video.Status = VideoStatus.WaitingToRecord;
+                        break;
+                    case VideoStatus.Archived:
+                    case VideoStatus.PermanentArchived:
+                        logger.LogWarning(
+                            "{videoId} has already been archived. It is possible that an internet disconnect occurred during the process. Changed its state back to Recording.",
+                            video.id);
+
+                        video.Status = VideoStatus.WaitingToRecord;
+                        break;
+
+                    case VideoStatus.Unknown:
+                    case VideoStatus.Scheduled:
+                    case VideoStatus.Pending:
+                    case VideoStatus.WaitingToDownload:
+                    case VideoStatus.Downloading:
+                    //case VideoStatus.Uploading:
+                    case VideoStatus.Expired:
+                    case VideoStatus.Error:
+                    case VideoStatus.Exist:
+                    case VideoStatus.Edited:
+                    case VideoStatus.Deleted:
                     default:
                         logger.LogWarning("{videoId} is in {status}, skip.", video.id, Enum.GetName(typeof(VideoStatus), video.Status));
                         return;

diff --git a/SingletonServices/ACIService.cs b/SingletonServices/ACIService.cs
@@ -21,6 +21,16 @@ public class AciService(ILogger<AciService> logger,
     private const string FallbackRegistry = "recordermoe/";
     private readonly string _resourceGroupName = options.Value.ContainerInstance!.ResourceGroupName;
 
+    public Task<bool> IsJobMissing(Video video, CancellationToken cancellation)
+    {
+        return IsJobMissing(NameHelper.CleanUpInstanceName(video.id), cancellation);
+    }
+
+    public async Task<bool> IsJobMissing(string keyword, CancellationToken cancellation)
+    {
+        return null == (await GetResourceByKeywordAsync(keyword, cancellation));
+    }
+
     public Task<bool> IsJobSucceededAsync(Video video, CancellationToken cancellation = default)
     {
         return IsJobSucceededAsync(NameHelper.CleanUpInstanceName(video.id), cancellation);
@@ -29,7 +39,9 @@ public Task<bool> IsJobSucceededAsync(Video video, CancellationToken cancellatio
     public async Task<bool> IsJobSucceededAsync(string keyword, CancellationToken cancellation = default)
     {
         ContainerGroupResource? resource = await GetResourceByKeywordAsync(keyword, cancellation);
-        return null != resource && resource.HasData && resource.Data.InstanceView.State == "Succeeded";
+        return null != resource
+               && resource.HasData
+               && resource.Data.InstanceView.State == "Succeeded";
     }
 
     public Task<bool> IsJobFailedAsync(Video video, CancellationToken cancellation = default)
@@ -40,7 +52,9 @@ public Task<bool> IsJobFailedAsync(Video video, CancellationToken cancellation =
     public async Task<bool> IsJobFailedAsync(string keyword, CancellationToken cancellation)
     {
         ContainerGroupResource? resource = await GetResourceByKeywordAsync(keyword, cancellation);
-        return null == resource || !resource.HasData || resource.Data.InstanceView.State == "Failed";
+        return null != resource
+               && (!resource.HasData
+                   || resource.Data.InstanceView.State == "Failed");
     }
 
     /// <summary>
@@ -166,9 +180,12 @@ await armDeploymentCollection.CreateOrUpdateAsync(
             resourceGroupResource.GetContainerGroups()
                                  .FirstOrDefault(p => p.Id.Name.Contains(NameHelper.CleanUpInstanceName(keyword)));
 
-        return null == containerGroupResourceTemp
-            ? null
-            : (await resourceGroupResource.GetContainerGroupAsync(containerGroupResourceTemp.Id.Name, cancellation)).Value;
+        if (null == containerGroupResourceTemp) return null;
+
+        Response<ContainerGroupResource> response =
+            (await resourceGroupResource.GetContainerGroupAsync(containerGroupResourceTemp.Id.Name, cancellation));
+
+        return response.HasValue ? response.Value : null;
     }
 
     private async Task<ResourceGroupResource> GetResourceGroupAsync(CancellationToken cancellation = default)

diff --git a/SingletonServices/KubernetesService.cs b/SingletonServices/KubernetesService.cs
@@ -19,6 +19,16 @@ public class KubernetesService(
 
     private readonly string _kubernetesNamespace = options.Value.Namespace ?? "recordermoe";
 
+    public Task<bool> IsJobMissing(Video video, CancellationToken cancellation)
+    {
+        return IsJobMissing(NameHelper.CleanUpInstanceName(video.id), cancellation);
+    }
+
+    public async Task<bool> IsJobMissing(string keyword, CancellationToken cancellation)
+    {
+        return (await GetJobsByKeywordAsync(keyword, cancellation)).Count == 0;
+    }
+
     public Task<bool> IsJobSucceededAsync(Video video, CancellationToken cancellation = default)
     {
         return IsJobSucceededAsync(NameHelper.CleanUpInstanceName(video.id), cancellation);

diff --git a/SingletonServices/RecordService.cs b/SingletonServices/RecordService.cs
@@ -66,7 +66,22 @@ public async Task HandledFailedJobsAsync(VideoService videoService, Cancellation
             _logger.LogTrace("No videos recording/downloading");
 
         foreach (Video video in videos)
-            if (await _jobService.IsJobFailedAsync(video, stoppingToken))
+            if (await _jobService.IsJobMissing(video, stoppingToken))
+                switch (video.Source)
+                {
+                    case "Youtube":
+                        await videoService.UpdateVideoStatusAsync(video, VideoStatus.Pending);
+                        _logger.LogWarning("{videoId} is missing. Set status to {status}", video.id, video.Status);
+                        break;
+                    default:
+                        await videoService.UpdateVideoStatusAsync(video, VideoStatus.Missing);
+                        await videoService.UpdateVideoNoteAsync(video,
+                                                                "This video archive is missing. If you would like to provide it, please contact admin.");
+
+                        _logger.LogWarning("{videoId} is missing.", video.id);
+                        break;
+                }
+            else if (await _jobService.IsJobFailedAsync(video, stoppingToken))
                 switch (video.Source)
                 {
                     case "Youtube":