Address review feedback: revert CI/conftest changes, improve profile script

- Revert CI workflow to original --durations=20 (no timeout)
- Remove global timeout from pyproject.toml
- Revert conftest.py verify_cache_state scope to function
- Update profile_tests.sh: accept CLI args (-m, -d, -t, -o) with defaults

Author: Abhishek

.github/workflows/test.yml

@@ -153,7 +153,7 @@ jobs:
           marks="not production_server"
         fi
 
-        pytest -n 4 --durations=0 --timeout=600 --dist load -sv $codecov -o log_cli=true -m "$marks"
+        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
 
     - name: Run tests on Ubuntu Production
       if: matrix.os == 'ubuntu-latest'
@@ -171,14 +171,14 @@ jobs:
           marks="production_server"
         fi
 
-        pytest -n 4 --durations=0 --timeout=600 --dist load -sv $codecov -o log_cli=true -m "$marks"
+        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
 
     - name: Run tests on Windows
       if: matrix.os == 'windows-latest'
       env:
         OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
       run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
-        pytest -n 4 --durations=0 --timeout=600 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server"
+        pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server"
 
     - name: Upload coverage
       if: matrix.code-cov && always()

pyproject.toml

@@ -130,7 +130,6 @@ log_level="DEBUG"
 testpaths = ["tests"]
 minversion = "7.0"
 xfail_strict = true
-timeout = 600
 filterwarnings=[
     "ignore:the matrix subclass:PendingDeprecationWarning"
 ]

scripts/profile_tests.sh

@@ -1,27 +1,51 @@
 #!/bin/bash
 # Profile test durations to diagnose slow tests (Issue #1633)
-# Usage: ./scripts/profile_tests.sh [marker_filter]
+#
+# Usage: ./scripts/profile_tests.sh [options]
+#
+# Options:
+#   -m MARKER    Pytest marker filter (default: "not production_server and not test_server")
+#   -d DURATION  Number of slowest durations to show, 0 for all (default: 20)
+#   -t TIMEOUT   Per-test timeout in seconds (default: 300)
+#   -o OUTPUT    Output file path for the report (default: test_durations_report.txt)
 #
 # Examples:
-#   ./scripts/profile_tests.sh                               # non-server tests
-#   ./scripts/profile_tests.sh "production_server"            # production server tests only
-#   ./scripts/profile_tests.sh "sklearn"                      # sklearn tests only
+#   ./scripts/profile_tests.sh
+#   ./scripts/profile_tests.sh -m "production_server" -d 0 -t 600
+#   ./scripts/profile_tests.sh -m "sklearn" -o sklearn_report.txt
 
 set -euo pipefail
 
-MARKER_FILTER="${1:-not production_server and not test_server}"
+# Default values
+MARKER_FILTER="not production_server and not test_server"
+DURATIONS=20
+TIMEOUT=300
+OUTPUT_FILE="test_durations_report.txt"
+
+# Parse command line arguments
+while getopts "m:d:t:o:" opt; do
+  case $opt in
+    m) MARKER_FILTER="$OPTARG" ;;
+    d) DURATIONS="$OPTARG" ;;
+    t) TIMEOUT="$OPTARG" ;;
+    o) OUTPUT_FILE="$OPTARG" ;;
+    *) echo "Usage: $0 [-m marker] [-d durations] [-t timeout] [-o output_file]" && exit 1 ;;
+  esac
+done
 
 echo "=== OpenML Test Duration Profiler ==="
 echo "Marker filter: $MARKER_FILTER"
-echo "Timeout per test: 300s"
+echo "Durations to show: $DURATIONS"
+echo "Timeout per test: ${TIMEOUT}s"
+echo "Output file: $OUTPUT_FILE"
 echo ""
 
 pytest \
-  --durations=0 \
-  --timeout=300 \
+  --durations="$DURATIONS" \
+  --timeout="$TIMEOUT" \
   -q \
   -m "$MARKER_FILTER" \
-  2>&1 | tee test_durations_report.txt
+  2>&1 | tee "$OUTPUT_FILE"
 
 echo ""
-echo "=== Report saved to test_durations_report.txt ==="
+echo "=== Report saved to $OUTPUT_FILE ==="

tests/conftest.py

@@ -276,7 +276,7 @@ def test_apikey_v2() -> str:
     return openml.config.get_test_servers()[APIVersion.V2]["apikey"]
 
 
-@pytest.fixture(autouse=True, scope="module")
+@pytest.fixture(autouse=True, scope="function")
 def verify_cache_state(test_files_directory) -> Iterator[None]:
     assert_static_test_cache_correct(test_files_directory)
     yield