tools/lab_bisect: Add retry logic for server connection errors (#75)

When Batfish server fails to start or connection errors occur during
testing, retry once with a clean server restart before skipping the
commit. This prevents transient server issues from being treated as
test failures during bisection.

- Add retry_count parameter to test_commit() method
- Retry server startup failures once before giving up
- Detect connection errors in test output and retry once
- Skip commits (return None) after retry failure instead of treating
as test failures (return False)
- Add _is_connection_error() helper to detect various connection
patterns

---

**Stack**:
- #76
- #75 ⬅


⚠️ *Part of a stack created by [spr](https://github.com/ejoffe/spr). Do
not merge manually using the UI - doing so may have unexpected results.*

Author: dhalperi

tools/lab_bisect.py

@@ -96,6 +96,20 @@ def finalize_log(self, first_bad_commit: Optional[str] = None):
         self.logger.info(f"Bisection complete. Full log saved to {self.log_file}")
         return self.log_file
 
+    def _is_connection_error(self, test_output: str) -> bool:
+        """Check if test output indicates a connection error to Batfish server."""
+        connection_error_indicators = [
+            "Connection refused",
+            "ConnectionRefusedError",
+            "Failed to establish a new connection",
+            "NewConnectionError",
+            "Max retries exceeded",
+            "HTTPConnectionPool(host='localhost', port=9996)",
+        ]
+        return any(
+            indicator in test_output for indicator in connection_error_indicators
+        )
+
     def cleanup_containers(self) -> bool:
         """Remove containers directory to clear cached files."""
         try:
@@ -263,9 +277,9 @@ def get_commit_from_date(self, date: str) -> Optional[str]:
             self.logger.error(f"Error getting commit for date {date}: {e}")
             return None
 
-    def test_commit(self, commit: str) -> Optional[bool]:
+    def test_commit(self, commit: str, retry_count: int = 0) -> Optional[bool]:
         """Test a specific commit. Returns True if good, False if bad, None if error."""
-        self.logger.info(f"Testing commit: {commit}")
+        self.logger.info(f"Testing commit: {commit} (attempt {retry_count + 1})")
         error_msg = ""
         test_output = ""
 
@@ -288,12 +302,32 @@ def test_commit(self, commit: str) -> Optional[bool]:
             # Start server
             if not self.start_batfish_server():
                 error_msg = "Failed to start Batfish server"
-                self.log_commit_test(commit, None, test_output, error_msg)
-                return None
+                # Retry once for server startup failures
+                if retry_count == 0:
+                    self.logger.warning(
+                        f"Batfish server failed to start, retrying once..."
+                    )
+                    return self.test_commit(commit, retry_count + 1)
+                else:
+                    self.log_commit_test(commit, None, test_output, error_msg)
+                    return None
 
             # Test the lab
             result, test_output = self.test_lab()
 
+            # Check if the test failed due to connection errors (indicating server issues)
+            if not result and self._is_connection_error(test_output):
+                # Stop server and retry once for connection errors
+                self.stop_batfish_server()
+                if retry_count == 0:
+                    self.logger.warning(f"Connection error detected, retrying once...")
+                    return self.test_commit(commit, retry_count + 1)
+                else:
+                    # After retry, treat as skip (return None) rather than failure (return False)
+                    error_msg = "Connection error persisted after retry - skipping"
+                    self.log_commit_test(commit, None, test_output, error_msg)
+                    return None
+
             # Stop server
             self.stop_batfish_server()