Fail runner when test run is expired

Author: ChrisBr

ruby/lib/ci/queue/redis/base.rb

@@ -5,6 +5,7 @@ module Redis
       class Base
         include Common
 
+        TEN_MINUTES = 60 * 10
         CONNECTION_ERRORS = [
           ::Redis::BaseConnectionError,
           ::SocketError, # https://github.com/redis/redis-rb/pull/631
@@ -20,6 +21,15 @@ def exhausted?
           queue_initialized? && size == 0
         end
 
+        def expired?
+          if (created_at = redis.get(key('master-created-at')))
+            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < Time.now.to_f
+          else
+            # if there is no created at set anymore we assume queue is expired
+            true
+          end
+        end
+
         def size
           redis.multi do |transaction|
             transaction.llen(key('queue'))

ruby/lib/ci/queue/redis/worker.rb

@@ -202,10 +202,12 @@ def push(tests)
               transaction.lpush(key('queue'), tests) unless tests.empty?
               transaction.set(key('total'), @total)
               transaction.set(key('master-status'), 'ready')
+              transaction.set(key('master-created-at'), Time.now.to_f)
 
               transaction.expire(key('queue'), config.redis_ttl)
               transaction.expire(key('total'), config.redis_ttl)
               transaction.expire(key('master-status'), config.redis_ttl)
+              transaction.expire(key('master-created-at'), config.redis_ttl)
             end
           end
           register

ruby/lib/minitest/queue/runner.rb

@@ -47,7 +47,10 @@ def retry_command
       end
 
       def run_command
-        if queue.retrying?
+        if queue.retrying? || retry?
+          if queue.expired?
+            abort! "The test run is too old and can't be retried"
+          end
           reset_counters
           retry_queue = queue.retry_queue
           if retry_queue.exhausted?
@@ -546,6 +549,11 @@ def abort!(message)
         puts red(message)
         exit! 1 # exit! is required to avoid minitest at_exit callback
       end
+
+      def retry?
+        ENV["BUILDKITE_RETRY_COUNT"].to_i > 0 ||
+          ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"
+      end
     end
   end
 end

ruby/test/integration/minitest_redis_test.rb

@@ -217,6 +217,50 @@ def test_retry_success
       assert_equal 'All tests were ran already', output
     end
 
+    def test_retry_fails_when_test_run_is_expired
+      out, err = capture_subprocess_io do
+        system(
+          @exe, 'run',
+          '--queue', @redis_url,
+          '--seed', 'foobar',
+          '--build', '1',
+          '--worker', '1',
+          '--timeout', '1',
+          '--max-requeues', '1',
+          '--requeue-tolerance', '1',
+          '-Itest',
+          'test/passing_test.rb',
+          chdir: 'test/fixtures/',
+        )
+      end
+      assert_empty err
+      output = normalize(out.lines.last.strip)
+      assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output
+
+      one_day = 60 * 60 * 24
+      key = ['build', "1", "master-created-at"].join(':')
+      @redis.set(key, Time.now - one_day)
+
+      out, err = capture_subprocess_io do
+        system(
+          @exe, 'run',
+          '--queue', @redis_url,
+          '--seed', 'foobar',
+          '--build', '1',
+          '--worker', '1',
+          '--timeout', '1',
+          '--max-requeues', '1',
+          '--requeue-tolerance', '1',
+          '-Itest',
+          'test/passing_test.rb',
+          chdir: 'test/fixtures/',
+        )
+      end
+      assert_empty err
+      output = normalize(out.lines.last.strip)
+      assert_equal "The test run is too old and can't be retried", output
+    end
+
     def test_retry_report
       # Run first worker, failing all tests
       out, err = capture_subprocess_io do