limit stuck job creation and make private networking optional

Author: elasticroentgen

CloudController.cs

@@ -64,7 +64,7 @@ private async Task<string> GenerateName()
         return name;
     }
 
-    public async Task<Machine> CreateNewRunner(string arch, string size, string runnerToken, string targetName, bool isCustom = false, string profileName = "default")
+    public async Task<Machine> CreateNewRunner(string arch, string size, string runnerToken, string targetName, bool isCustom = false, string profileName = "default", bool usePrivateNetworks = false)
     {
 
         // Select VM size for job - All AMD 
@@ -164,7 +164,9 @@ public async Task<Machine> CreateNewRunner(string arch, string size, string runn
             }
             try
             {
-                newSrv = await _client.Server.Create(dataCenters[ct], imageId.Value, name, srvType.Value, userData: cloudInitcontent, sshKeysIds: srvKeys, privateNetoworksIds: networks.Select(x => x.Id).ToList());
+                var privateNetworks = usePrivateNetworks ? networks.Select(x => x.Id).ToList() : new List<long>();
+                
+                newSrv = await _client.Server.Create(dataCenters[ct], imageId.Value, name, srvType.Value, userData: cloudInitcontent, sshKeysIds: srvKeys, privateNetoworksIds: privateNetworks);
                 success = true;
             }
             catch (Exception ex)

Database/DbContext.cs

@@ -108,6 +108,7 @@ public DateTime LastStateTime
     }
 
     public bool StuckJobReplacement { get; set; } = false;
+    public bool UsePrivateNetwork { get; set; }
 }
 
 public enum RunnerStatus

Migrations/20241030082800_add-private-net-flag.Designer.cs

@@ -0,0 +1,29 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace GithubActionsOrchestrator.Migrations
+{
+    /// <inheritdoc />
+    public partial class addprivatenetflag : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<bool>(
+                name: "UsePrivateNetwork",
+                table: "Runners",
+                type: "boolean",
+                nullable: false,
+                defaultValue: false);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "UsePrivateNetwork",
+                table: "Runners");
+        }
+    }
+}

Migrations/20241030082800_add-private-net-flag.cs

@@ -118,6 +118,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.Property<bool>("StuckJobReplacement")
                         .HasColumnType("boolean");
 
+                    b.Property<bool>("UsePrivateNetwork")
+                        .HasColumnType("boolean");
+
                     b.HasKey("RunnerId");
 
                     b.HasIndex("JobId")

Migrations/ActionsRunnerContextModelSnapshot.cs

@@ -9,4 +9,5 @@ public record CreateRunnerTask
     public TargetType TargetType { get; set; }
     public string RepoName { get; set; }
     public int RunnerDbId { get; set; }
+    public bool IsStuckReplacement { get; set; }
 }

Models/CreateRunnerTask.cs

@@ -266,7 +266,14 @@ private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfi
                 _logger.LogError($"Unable to get owner for stuck job. {stuckJob.JobId}");
                 continue;
             }
-            
+            int replacementsInQueue =  _queues.CreateTasks.Count(x => x.IsStuckReplacement);
+
+            if (replacementsInQueue > 25)
+            {
+                _logger.LogWarning($"Creating queue already has {replacementsInQueue} stuck jobs replacements. No adding more strain");
+                continue;
+            }
+
             string runnerToken = owner.Target switch
             {
                 TargetType.Repository => await GitHubApi.GetRunnerTokenForRepo(owner.GitHubToken, owner.Name),
@@ -300,15 +307,15 @@ private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfi
             };
             await db.Runners.AddAsync(newRunner);
             await db.SaveChangesAsync();
-            
-            _queues.CreateTasks.Enqueue(new CreateRunnerTask
-            {
-                RunnerToken = runnerToken,
-                RepoName = stuckJob.Repository,
-                TargetType = owner.Target,
-                RunnerDbId = newRunner.RunnerId,
-                
-            }); 
+           
+                _queues.CreateTasks.Enqueue(new CreateRunnerTask
+                {
+                    RunnerToken = runnerToken,
+                    RepoName = stuckJob.Repository,
+                    TargetType = owner.Target,
+                    RunnerDbId = newRunner.RunnerId,
+                    IsStuckReplacement = true
+                });
         } 
     }
 
@@ -543,7 +550,7 @@ private async Task<bool> CreateRunner(CreateRunnerTask rt)
                 TargetType.Organization => runner.Owner,
                 _ => throw new ArgumentOutOfRangeException()
             };
-            Machine newRunner = await _cc.CreateNewRunner(runner.Arch, runner.Size, rt.RunnerToken, targetName, runner.IsCustom, runner.Profile);
+            Machine newRunner = await _cc.CreateNewRunner(runner.Arch, runner.Size, rt.RunnerToken, targetName, runner.IsCustom, runner.Profile, runner.UsePrivateNetwork);
             _logger.LogInformation($"New Runner {newRunner.Name} [{runner.Size} on {runner.Arch}] entering pool for {targetName}.");
             MachineCreatedCount.Labels(runner.Owner, runner.Size).Inc();
 

PoolManager.cs

@@ -340,7 +340,9 @@ private static async Task<IResult> RunnerStateReportHandler(HttpRequest request,
                         Arch = runner.Arch,
                         IPv4 = string.Empty,
                         IsCustom = runner.IsCustom,
-                        Owner = runner.Owner
+                        Owner = runner.Owner,
+                        UsePrivateNetwork = runner.UsePrivateNetwork
+                        
 
                     };
                     await db.Runners.AddAsync(newRunner);
@@ -630,7 +632,8 @@ private static async Task JobQueued(ILogger<Program> logger, string repoName, Li
             Arch = arch,
             IPv4 = string.Empty,
             IsCustom = isCustom,
-            Owner = owner
+            Owner = owner,
+            UsePrivateNetwork = labels.Contains("use-cache-server")
 
         };
         await db.Runners.AddAsync(newRunner);