Diverse changes to sdkdocs generation

.github/workflows/publish_api_docs.yml

@@ -20,7 +20,26 @@ jobs:
         id: get_version
         run: |
           BRANCH="${GITHUB_REF_NAME}"
-          echo ::set-output name=VERSION::${BRANCH#release/}
+          VERSION="${BRANCH#release/}"
+
+          # Validate version looks like a semver (e.g. 0.71.0, 0.71.0rc1, 0.3.7.1rc0)
+          if ! echo "$VERSION" | grep -qE '^[0-9]+(\.[0-9]+){2,3}([a-zA-Z][a-zA-Z0-9]*)?$'; then
+            echo "::error::Invalid version '$VERSION' extracted from branch '$BRANCH'. Expected semver like 0.71.0"
+            exit 1
+          fi
+
+          echo "VERSION=${VERSION}" >> "$GITHUB_OUTPUT"
+
+          # Only tag as "latest" if this is the highest release version.
+          # This prevents backports to older release branches from stealing
+          # the latest alias on sdkdocs.zenml.io.
+          LATEST_TAG=$(git tag --list '[0-9]*' --sort=-version:refname | head -n1)
+          if [ "$(printf '%s\n%s' "$VERSION" "$LATEST_TAG" | sort -V | tail -n1)" = "$VERSION" ]; then
+            echo "IS_LATEST=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "IS_LATEST=false" >> "$GITHUB_OUTPUT"
+            echo "Skipping latest alias: $VERSION is older than $LATEST_TAG"
+          fi
       - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8  # v4.0.1
         with:
           node-version: '14'
@@ -36,6 +55,11 @@ jobs:
           git config --global user.email 'codemonkey@zenml.io'
       - name: Runs docs generation
         run: |-
-          bash scripts/generate-docs.sh -s src/zenml/ --push --latest -v ${STEPS_GET_VERSION_OUTPUTS_VERSION}
+          LATEST_FLAG=""
+          if [ "$IS_LATEST" = "true" ]; then
+            LATEST_FLAG="--latest"
+          fi
+          bash scripts/generate-docs.sh -s src/zenml/ --push $LATEST_FLAG -v "$VERSION"
         env:
-          STEPS_GET_VERSION_OUTPUTS_VERSION: ${{ steps.get_version.outputs.VERSION }}
+          VERSION: ${{ steps.get_version.outputs.VERSION }}
+          IS_LATEST: ${{ steps.get_version.outputs.IS_LATEST }}

docs/README.md

@@ -27,25 +27,12 @@ To build them locally follow these steps:
 pip3 install -e ".[server,dev]"
 pip3 install "Jinja2==3.0.3"
 ```
-* Modify `docs/mkdocs.yml` as follows:
-```yaml
-watch:
-  - ../src/zenml
-```
-```yaml
-watch:
-  - src/zenml
-```
-* Run `python3 docs/mkdocstrings_helper.py`
-* Run:
+* Run the serve script from the repository root:
 ```bash
-rm -rf src/zenml/zen_stores/migrations/env.py
-rm -rf src/zenml/zen_stores/migrations/versions
-rm -rf src/zenml/zen_stores/migrations/script.py.mako
+bash scripts/serve_api_docs.sh
 ```
-* Run `mkdocs serve -f docs/mkdocs.yml -a localhost:<PORT>` from the repository root - 
-running it from elsewhere can lead to unexpected errors. This script will compose the docs hierarchy
-and serve it (http://127.0.0.1:<PORT>/).
+This generates the docs structure and serves it locally. Alternatively, to build
+manually, see `scripts/generate-docs.sh` for the full sequence of steps.
 
 ## Link Checker Tool
 

docs/mkdocs.yml

@@ -10,7 +10,7 @@ theme:
   custom_dir: mkdocs/overrides
 
 repo_url: https://github.com/zenml-io/zenml
-edit_uri: https://github.com/zenml-io/zenml/docs/
+edit_uri: ""  # Disabled — SDK docs are auto-generated, not editable on GitHub
 
 plugins:
   - search
@@ -34,6 +34,7 @@ plugins:
               - "!^_"  # Exclude private members (starting with _)
               - "^__init__$"  # But include __init__ methods
               - "^__new__$"   # And __new__ methods
+              - "!^migrations$"  # Exclude alembic migrations (env.py has import side-effects)
             # Rendering options
             show_source: true
             show_root_heading: true

docs/mkdocs/cli.md

@@ -1,10 +1,7 @@
-# Cli
+# CLI
 
 ::: zenml.cli
-    handler: python
-    selection:
-      members: 
-        - "__init__.py"  
-    rendering:
+    options:
       show_root_heading: true
-      show_source: true
+      show_source: true
+      members: true

docs/mkdocs/overrides/main.html

@@ -150,7 +150,7 @@
 </script>
 
 <!-- Plausible and Reo continue unchanged -->
-<script defer data-domain="apidocs.zenml.io" src="https://plausible.io/js/plausible.js"></script>
+<script defer data-domain="sdkdocs.zenml.io" src="https://plausible.io/js/plausible.js"></script>
 
 <!-- Start of Reo Javascript -->
 <script type="text/javascript">

docs/mkdocstrings_helper.py

@@ -87,7 +87,17 @@ def create_entity_docs(
     for item in sources_path.iterdir():
         if item.name not in ignored_modules:
             is_python_file = item.is_file() and item.name.endswith(".py")
-            is_non_empty_dir = item.is_dir() and any(item.iterdir())
+            # Only consider directories that are valid Python packages
+            # with actual content beyond __init__.py
+            is_non_empty_dir = (
+                item.is_dir()
+                and (item / "__init__.py").exists()
+                and any(
+                    p
+                    for p in item.rglob("*.py")
+                    if p.name != "__init__.py"
+                )
+            )
 
             if is_python_file or is_non_empty_dir:
                 item_name = generate_title(item.stem)
@@ -199,12 +209,7 @@ def generate_docs(
         md_prefix="core",
     )
 
-    # Fix links in index file to point to index.md instead of just the directory
-    fixed_index_contents = []
-    for line in index_file_contents:
-        fixed_index_contents.append(line)
-
-    index_file_str = "\n".join(sorted(fixed_index_contents))
+    index_file_str = "\n".join(sorted(index_file_contents))
     to_md_file(
         index_file_str,
         "index.md",
@@ -244,10 +249,11 @@ def generate_docs(
     # Optional argument
     parser.add_argument(
         "--ignored_modules",
-        type=List[str],
+        nargs="*",
+        type=str,
         default=["VERSION", "README.md", "__init__.py", "__pycache__"],
         help="Top level entities that should not end up in "
-        "the sdk docs (e.g. README.md, __init__",
+        "the sdk docs (e.g. README.md, __init__)",
     )
 
     # Switch

scripts/generate-docs.sh

@@ -56,15 +56,6 @@ if [ -z "$SKIP_INSTALL" ]; then
   uv pip install --system "Jinja2==3.0.3"
 fi
 
-################################# Initialize DB and delete unnecessary alembic files ###################################
-
-# env.py leads to errors in the build as run_migrations() gets executed
-# the migration versions are not necessary parts of the api docs
-rm -rf src/zenml/zen_stores/migrations/env.py
-rm -rf src/zenml/zen_stores/migrations/versions
-rm -rf src/zenml/zen_stores/migrations/script.py.mako
-
-
 ########################################## Generate Structure of API docs ##############################################
 python3 docs/mkdocstrings_helper.py --path $SRC --output_path docs/mkdocs/
 

scripts/serve_api_docs.sh

@@ -6,8 +6,10 @@ SRC=${1:-"src/zenml"}
 
 export ZENML_DEBUG=1
 export ZENML_ANALYTICS_OPT_IN=false
-rm -rf docs/mkdocs/api_docs || true
-rm docs/mkdocs/index.md || true
+export DISABLE_DATABASE_MIGRATION=1
+rm -rf docs/mkdocs/core_code_docs || true
+rm -rf docs/mkdocs/integration_code_docs || true
+rm -f docs/mkdocs/index.md || true
 
 python docs/mkdocstrings_helper.py --path $SRC --output_path docs/mkdocs/
 cd docs

src/zenml/zen_stores/migrations/env.py

@@ -19,6 +19,7 @@
 migrations.
 """
 
+import os
 from typing import cast
 
 from alembic import context
@@ -47,4 +48,7 @@ def run_migrations() -> None:
     a.run_migrations(fn=None)
 
 
-run_migrations()
+# DISABLE_DATABASE_MIGRATION is set by scripts/generate-docs.sh to prevent
+# this call when mkdocstrings imports the module during SDK docs builds.
+if not os.environ.get("DISABLE_DATABASE_MIGRATION"):
+    run_migrations()