Various packages upgrades (Django 5.2) (#2200)

* Fix gather to work with Python 3.12

* packages upgrade with uv lock --upgrade

* bump gunicorn to 23; new major version of flake8 so new fixes

* bump jinja, requests, pillow

* bump uvicorn (includes h11)

* bump python to 3.10.19 for the compute worker; bump websockets to 16 for instance and compute worker (which also fixes an issue on the compute worker with python 3.10)

* Upgrade package.json packages (still some left, like Riot which is a major rewrite)

* remove six from pyproject and code

* flake8 fixes

* bump boto3, botocore, s3transfer

* bump urllib3

* bump python to 3.13 on compute worker

* bump pyyaml, ipdb, twisted, flex, markdown, flex, pygments, whitenoise, bleach, oyaml, aiofiles, bpython, argh, watchdog

* bump celery to 5.6.2 for main instance

* bump python to 3.13.11 on instance

* flake8 fixes

* bump python to 3.14 for playwright pytests

* merge pytest.ini into pyproject.toml; bump pytest to 9.0.2

* bump various django related packages; bump pytest to 9.0.2 and merge pytest config from setup.cfg into pyproject.toml

* bump django_filter and fix code to make it work

* remove coreapi and legacy-cgi, bump setuptools

* remove duplicate build in docker-compose.yml and assign better names to containers

* bump caddy version and remove useless line in circle-ci config file

* bump django to 4.2.29 and various other packages with uv lock --upgrade

* bump django to 5.2.12, bump other packages with uv lock --upgrade

* bump factory-boy, django-ajax-selects, drf-extensions and replace deprecated django-cors-middleware with django-cors-headers

* replace bleach (deprecated) with nh3

* bump django-redis to 6.0.0

* bump rabbitmq to 4.2.4

* cleanup some Containerfiles

* Fixed missing code from #2237 that was not merged into this branch

* bump blessed and redis on instance; bump charset-normalizer and setuptools on compute worker

* fix logout that broke on django 5.2 upgrade

---------

Co-authored-by: Chris Harris <cjh@lbl.gov>
Co-authored-by: Obada Haddad <obada.haddad@lisn.fr>

Author: 3 people

.circleci/config.yml

@@ -69,7 +69,7 @@ jobs:
 
       - run:
           name: "Tests: Run unit/integration tests (excluding e2e)"
-          command: docker compose exec django py.test src/ -m "not e2e"
+          command: docker compose exec django py.test src/
 
       # We give the name of the test files manually because we need test_auth.py to be run before the others for state.json file to be created
       # CI="true" to skip some tests that fail in the CI for now

compute_worker/compute_worker.py

@@ -240,6 +240,8 @@ def rewrite_bundle_url_if_needed(url):
 # -----------------------------------------------------------------------------
 @shared_task(name="compute_worker_run")
 def run_wrapper(run_args):
+    # We need to convert the UUID given by celery into a byte like object otherwise things will break
+    run_args.update(secret=str(run_args["secret"]))
     logger.info(f"Received run arguments: \n {colorize_run_args(json.dumps(run_args))}")
     run = Run(run_args)
 
@@ -1183,14 +1185,14 @@ def start(self):
 
         logger.info("Running scoring program, and then ingestion program")
         loop = asyncio.new_event_loop()
+        # Set the event loop for the gather
+        asyncio.set_event_loop(loop)
         gathered_tasks = asyncio.gather(
             self._run_program_directory(program_dir, kind="program"),
             self._run_program_directory(ingestion_program_dir, kind="ingestion"),
             self.watch_detailed_results(),
-            loop=loop,
             return_exceptions=True,
         )
-
         task_results = []  # will store results/exceptions from gather
         signal.signal(signal.SIGALRM, alarm_handler)
         signal.alarm(self.execution_time_limit)
@@ -1278,6 +1280,9 @@ def start(self):
                 # set logs of this kind to None, since we handled them already
                 logger.info("Program finished")
         signal.alarm(0)
+        # Ensure loop is cleaned up
+        loop.close()
+        asyncio.set_event_loop(None)
 
         if self.is_scoring:
             # Check if scoring program failed

compute_worker/pyproject.toml

@@ -2,20 +2,21 @@
 authors = [
     {name = "codalab"},
 ]
-requires-python = "==3.9.20"
+requires-python = "==3.13.11"
 dependencies = [
-    "celery==5.2.2",
+    "celery==5.6.2",
     "requests>=2.32.4,<3",
-    "watchdog==2.1.1",
-    "argh==0.26.2",
-    "websockets==9.1",
-    "aiofiles==0.4.0",
-    "pyyaml==6.0.1",
+    "watchdog==6.0.0",
+    "argh==0.31.3",
+    "websockets==16.0.0",
+    "aiofiles==25.1.0",
+    "pyyaml==6.0.3",
     "loguru>=0.7.3,<0.8",
     "docker>=7.1.0,<8",
     "rich>=14.2.0,<15",
+    "setuptools>=82.0.0",
 ]
 
 name = "compute-worker"
 version = "0.1.0"
-description = ""
+description = ""

compute_worker/uv.lock

@@ -3,7 +3,7 @@ services:
   #   Web Services
   #----------------------------------------------------------------------------------------------------
   caddy:
-    image: caddy:2.10.0
+    image: caddy:2.11.1
     env_file: .env
     environment:
       - ACME_AGREE=true
@@ -22,9 +22,11 @@ services:
       - django
 
   django:
+    container_name: django
     build: 
       context: .
       dockerfile: packaging/container/Containerfile
+    image: django_site-worker
     # NOTE: We use watchmedo to reload gunicorn nicely, Uvicorn + Gunicorn reloads don't work well
     command: ["python manage.py migrate --no-input && python manage.py collectstatic --no-input && cd /app/src && watchmedo auto-restart -p '*.py' --recursive -- python3 ./gunicorn_run.py"]
     environment:
@@ -95,6 +97,7 @@ services:
   #   Local development helper, rebuilds RiotJS/Stylus on change
   #----------------------------------------------------------------------------------------------------
   builder:
+    container_name: builder
     build:
       context: .
       dockerfile: packaging/container/Containerfile.builder
@@ -115,6 +118,7 @@ services:
   #----------------------------------------------------------------------------------------------------
   db:
     image: postgres:18-alpine
+    container_name: db
     env_file: .env
     environment:
       - PGDATA=/var/lib/postgresql/18/docker
@@ -136,6 +140,7 @@ services:
   #   Rabbitmq & Flower monitoring tool
   #----------------------------------------------------------------------------------------------------
   rabbit:
+    container_name: rabbit
     build:
       context: .
       dockerfile: packaging/container/Containerfile.rabbitmq
@@ -161,6 +166,7 @@ services:
         max-file: "5"
 
   flower:
+    container_name: flower
     image: mher/flower
     env_file: .env
     environment:
@@ -179,6 +185,7 @@ services:
   #   Redis
   #----------------------------------------------------------------------------------------------------
   redis:
+    container_name: redis
     image: redis
     ports:
       - 6379:6379
@@ -195,9 +202,8 @@ services:
     # This auto-reloads
     command: ["watchmedo auto-restart -p '*.py' --recursive -- celery -A celery_config worker -B -Q site-worker -l info -n site-worker@%n --concurrency=2"]
     working_dir: /app/src
-    build:
-      context: .
-      dockerfile: packaging/container/Containerfile
+    container_name: site_worker
+    image: django_site-worker
     depends_on:
       - rabbit
       - db
@@ -219,6 +225,7 @@ services:
   compute_worker:
     command: ["celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n"]
     working_dir: /app
+    container_name: compute_worker
     build:
       context: .
       dockerfile: packaging/container/Containerfile.compute_worker

docker-compose.yml

@@ -2,11 +2,11 @@
   "name": "competitions-v2",
   "version": "0.0.1",
   "dependencies": {
-    "jquery": "^3.2.1",
-    "npm-watch": "^0.2.0",
-    "riot": "^3.6.3",
-    "stylus": "^0.54.5",
-    "uglify-js": "^3.0.28"
+    "jquery": "^4.0.0",
+    "npm-watch": "^0.13.0",
+    "riot": "^3.13.2",
+    "stylus": "^0.64.0",
+    "uglify-js": "^3.19.3"
   },
   "devDependencies": {},
   "watch": {

package.json

@@ -1,16 +1,16 @@
 FROM almalinux:10-minimal
+
 RUN microdnf install -y tar gzip
 
 ENV PYTHONUNBUFFERED=1
 
 # Install UV and add paths to PATH for uv and the future .venv created by uv
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh && microdnf remove -y tar gzip && microdnf clean all
 ENV PATH=$PATH:/root/.local/bin
 ENV PATH=$PATH:/.venv/bin
 
-
-COPY pyproject.toml ./
-COPY uv.lock ./
+# Copy pyproject and uv.lock
+COPY pyproject.toml uv.lock ./
 
 # Install dependencies
 RUN uv sync --all-extras --frozen

packaging/container/Containerfile

@@ -1,23 +1,21 @@
 FROM almalinux:10-minimal
+
 RUN microdnf install -y tar gzip
 
 # This makes output not buffer and return immediately, nice for seeing results in stdout
 ENV PYTHONUNBUFFERED=1
 
-COPY compute_worker/pyproject.toml ./
-COPY compute_worker/uv.lock ./
-COPY compute_worker/celery_config.py ./
-COPY compute_worker/compute_worker.py ./
-
+COPY compute_worker/pyproject.toml compute_worker/uv.lock compute_worker/celery_config.py compute_worker/compute_worker.py ./
 
 # Install UV and add paths to PATH for uv and the future .venv created by uv
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh && microdnf remove -y tar gzip && microdnf clean all
 ENV PATH=$PATH:/root/.local/bin
 ENV PATH=$PATH:/.venv/bin
 
 
 # Install dependencies
 RUN uv sync --frozen
 COPY src/settings/logs_loguru.py /.venv/bin
+
 ENTRYPOINT ["/bin/bash", "-c"]
 CMD ["celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@$HOSTNAME --concurrency=1"]

packaging/container/Containerfile.compute_worker

@@ -1,4 +1,4 @@
-FROM rabbitmq:4.2.0-management
+FROM rabbitmq:4.2.4-management
 
 ARG WORKER_CONNECTION_TIMEOUT
 RUN echo "consumer_timeout = $WORKER_CONNECTION_TIMEOUT" >> /etc/rabbitmq/conf.d/10-defaults.conf

packaging/container/Containerfile.rabbitmq

@@ -3,71 +3,69 @@ name = "codabench"
 version = "0.1.0"
 description = ""
 authors = [{ name = "Codalab" }]
-requires-python = ">=3.10,<3.11"
+requires-python = "==3.13.11"
 readme = "README.md"
 classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.10",
 ]
 dependencies = [
-    "django>=4.2.0,<5",
+    "django==5.2.12",
     "django-oauth-toolkit==1.6.3",
-    "django-cors-middleware==1.5.0",
-    "social-auth-core>=4.1.0,<5",
-    "social-auth-app-django>=5.0.0,<6",
-    "six==1.16.0",
-    "django-extensions>=3.2,<4",
-    "channels==4.2.0",
+    "social-auth-core==4.8.5",
+    "social-auth-app-django==5.4.3",
+    "django-extensions==4.1.0",
+    "channels==4.3.2",
     "channels-redis==4.0.0",
-    "pillow==10.3.0",
-    "celery==4.4.7",
-    "gunicorn==22.0.0",
-    "urllib3>=1.25.4,<1.27",
-    "uvicorn>=0.22.0,<0.23",
-    "pyyaml==5.3.1",
-    "watchdog==2.1.1",
-    "argh==0.26.2",
-    "python-dateutil==2.7.3",
-    "bpython>=0.21.0,<0.22",
-    "websockets>=10.4.0,<11",
-    "aiofiles==0.4.0",
-    "oyaml==0.7",
-    "factory-boy==2.11.1",
-    "bleach>=5.0.0",
-    "django-debug-toolbar==3.2",
-    "django-querycount==0.7.0",
+    "pillow==12.1.1",
+    "celery==5.6.2",
+    "gunicorn==23.0",
+    "urllib3==2.6.3",
+    "uvicorn==0.38",
+    "pyyaml==6.0.3",
+    "watchdog==6.0.0",
+    "argh==0.31.3",
+    "python-dateutil==2.9.0",
+    "bpython==0.26",
+    "websockets==16.0.0",
+    "aiofiles==25.1.0",
+    "oyaml==1.0",
+    "factory-boy==3.3.3",
     "blessings==1.7",
     "django-su>=1.0.0,<2",
-    "django-ajax-selects==2.0.0",
+    "django-ajax-selects==3.0.3",
     "dj-database-url==0.4.2",
     "psycopg2-binary>=2.9.9,<3",
-    "django-redis==4.12.1",
+    "django-redis==6.0.0",
     "django-storages[azure]>=1.14.6,<2",
     "azure-storage-blob>=12,<13",
     "azure-storage-common==2.1.0",
-    "boto3==1.26.76",
-    "whitenoise==5.2.0",
-    "djangorestframework>=3.13.0",
+    "boto3==1.42.50",
+    "whitenoise==6.11.0",
+    "djangorestframework==3.16.1",
     "djangorestframework-csv==3.0.1",
-    "drf-extensions==0.4.0",
-    "markdown==2.6.11",
-    "pygments==2.2.0",
-    "drf-writable-nested==0.6.2",
-    "django-filter==2.4.0",
-    "flex==6.12.0",
+    "drf-extensions==0.8.0",
+    "markdown==3.10.2",
+    "pygments==2.19.2",
+    "drf-writable-nested==0.7.2",
+    "flex==6.14.1",
     "pyrabbit2==1.0.7",
-    "django-enforce-host==1.0.1",
-    "twisted==24.7.0",
-    "ipdb==0.13",
-    "jinja2==3.1.4",
-    "requests==2.32.2",
-    "drf-extra-fields>=3.5.0",
-    "botocore==1.29.76",
-    "s3transfer==0.6.0",
+    "django-enforce-host==1.1.0",
+    "twisted==25.5.0",
+    "ipdb==0.13.13",
+    "jinja2==3.1.6",
+    "requests==2.32.5",
+    "drf-extra-fields==3.7.0",
+    "botocore==1.42.50",
+    "s3transfer==0.16.0",
     "drf-spectacular>=0.28.0,<0.29",
-    "coreapi>=2.3.3,<3",
     "loguru>=0.7.3,<0.8",
     "tzdata>=2025.3",
+    "setuptools==82.0.0",
+    "pytz>=2025.2",
+    "django-filter==25.1",
+    "django-cors-headers==4.9.0",
+    "nh3==0.3.3",
 ]
 
 [tool.uv]
@@ -79,9 +77,11 @@ build-backend = "uv_build"
 
 [dependency-groups]
 dev = [
-    "flake8>=3.8.4",
-    "pytest==7.4.4",
-    "pytest-django==4.11.1",
+    "django-querycount==0.7.0",
+    "django-debug-toolbar==6.2.0",
+    "flake8==7.3.0",
+    "pytest==9.0.2",
+    "pytest-django==4.12.0",
 ]
 [tool.pytest.ini_options]
 DJANGO_SETTINGS_MODULE = "settings.develop"  # Just "settings" since pytest will be running from src/