check status of stuck jobs before starting new runner

Author: elasticroentgen

Database/DbContext.cs

@@ -130,7 +130,8 @@ public enum JobState
     Unknown = 0,
     Queued = 1,
     InProgress = 2,
-    Completed = 3
+    Completed = 3,
+    Vanished = 4
 }
 public class RunnerLifecycle
 {

GitHubApi.cs

@@ -156,4 +156,66 @@ public static async Task<bool> RemoveRunnerFromRepo(string repoName, string orgG
         return response.IsSuccessStatusCode;
 
     }
-}
+
+    public static async Task<GitHubJob> GetJobInfo(long stuckJobGithubJobId,string repoName, string orgGitHubToken)
+    {
+        using HttpClient client = new();
+        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
+        client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
+        client.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse($"Bearer {orgGitHubToken}");
+        client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("hetzner-autoscale", "1"));
+
+        HttpResponseMessage response = await client.GetAsync(
+            $"https://api.github.com/orgs/{repoName}/actions/jobs/{stuckJobGithubJobId}");
+        if(response.IsSuccessStatusCode)
+        {
+            string content = await response.Content.ReadAsStringAsync();
+            GitHubJob responseObject = JsonSerializer.Deserialize<GitHubJob>(content);
+            
+            return responseObject;
+        }
+        Log.Warning($"Unable to get GH job info for {repoName}/{stuckJobGithubJobId}: [{response.StatusCode}] {response.ReasonPhrase}");
+
+        return null;
+    }
+}
+
+public class GitHubJob
+{
+
+    public int Id { get; set; }
+    public int RunId { get; set; }
+    public string RunUrl { get; set; }
+    public int RunAttempt { get; set; }
+    public string NodeId { get; set; }
+    public string HeadSha { get; set; }
+    public string Url { get; set; }
+    public string HtmlUrl { get; set; }
+    public string Status { get; set; }
+    public string Conclusion { get; set; }
+    public DateTime CreatedAt { get; set; }
+    public DateTime StartedAt { get; set; }
+    public DateTime? CompletedAt { get; set; } // Nullable
+    public string Name { get; set; }
+    public List<GitHubJobStep> Steps { get; set; }
+    public string CheckRunUrl { get; set; }
+    public List<string> Labels { get; set; }
+    public int? RunnerId { get; set; } // Nullable
+    public string RunnerName { get; set; } // Nullable
+    public int? RunnerGroupId { get; set; } // Nullable
+    public string RunnerGroupName { get; set; } // Nullable
+    public string WorkflowName { get; set; } // Nullable
+    public string HeadBranch { get; set; } // Nullable
+
+}
+
+public class GitHubJobStep
+    {
+        public string Status { get; set; }
+        public string Conclusion { get; set; }
+        public string Name { get; set; }
+        public int Number { get; set; }
+        public DateTime? StartedAt { get; set; } // Nullable
+        public DateTime? CompletedAt { get; set; } // Nullable
+    }
+

PoolManager.cs

@@ -277,7 +277,24 @@ private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfi
             int replacementsInQueue =  _queues.CreateTasks.Count(x => x.IsStuckReplacement);
             if (replacementsInQueue > 25)
             {
-                _logger.LogWarning($"Creating queue already has {replacementsInQueue} stuck jobs replacements. No adding more strain");
+                _logger.LogWarning($"Creating queue already has {replacementsInQueue} stuck jobs replacements. Not adding more strain.");
+                continue;
+            }
+            
+            // check job on github
+            GitHubJob ghJob =  await GitHubApi.GetJobInfo(stuckJob.GithubJobId, owner.Name, owner.GitHubToken);
+            if (ghJob == null || ghJob.Status != "queued")
+            {
+                _logger.LogWarning($"job info for {stuckJob.JobId} not found or job not queued anymore.");
+
+                if (stuckJob.QueueTime + TimeSpan.FromHours(2) > DateTime.UtcNow)
+                {
+                    _logger.LogWarning($"Marking stuck job {stuckJob.GithubJobId} vanished as it's no longer in the GitHub queued state for more than 2h.");
+                    stuckJob.State = JobState.Vanished;
+                    stuckJob.CompleteTime = DateTime.UtcNow;
+                    await db.SaveChangesAsync();
+                }
+                
                 continue;
             }