Add external snippet generated docs automation

.github/workflows/update-version-dashboard.yml

@@ -1,20 +1,29 @@
-name: Update version dashboard
+name: Update generated docs
 
 on:
   schedule:
-    - cron: "23 */6 * * *"
+    - cron: "*/10 * * * *"
   workflow_dispatch:
+    inputs:
+      targets:
+        description: "Generated-doc targets to run"
+        type: string
+        default: "all"
+      include_heavy_external_snippets:
+        description: "Run external snippet targets that require a heavier runner"
+        type: boolean
+        default: false
 
 permissions:
   contents: write
   pull-requests: write
 
 concurrency:
-  group: update-version-dashboard
+  group: update-generated-docs
   cancel-in-progress: false
 
 jobs:
-  update-version-dashboard:
+  update-generated-docs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -27,10 +36,21 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y direnv
-          direnv allow .
+          SKIP_NPM_INSTALL=1 direnv allow .
+
+      - name: Install npm dependencies
+        run: SKIP_NPM_INSTALL=1 direnv exec . npm ci
+
+      - name: Set up Daml tooling
+        run: |
+          curl -fsSL https://get.digitalasset.com/install/install.sh | sh
+          echo "$HOME/.dpm/bin" >> "$GITHUB_PATH"
+          echo "$HOME/.daml/bin" >> "$GITHUB_PATH"
 
       - name: Generate update pull requests
         env:
           GH_TOKEN: ${{ github.token }}
           GITHUB_TOKEN: ${{ github.token }}
-        run: python3 scripts/update_generated_reference_prs.py --targets all
+          GENERATED_DOCS_ENABLE_HEAVY_EXTERNAL_SNIPPETS: ${{ inputs.include_heavy_external_snippets && '1' || '' }}
+          GENERATED_DOCS_TARGETS: ${{ inputs.targets || 'all' }}
+        run: python3 scripts/update_generated_reference_prs.py --targets $GENERATED_DOCS_TARGETS

config/generated-docs/external-snippet-sources.json

@@ -0,0 +1,74 @@
+{
+  "sources": [
+    {
+      "key": "canton",
+      "label": "Canton external snippets",
+      "repository": "digital-asset/canton",
+      "ref": "main",
+      "version": "main",
+      "repo_arg": "canton",
+      "output_path": "docs-main/snippets/external/canton/main",
+      "requires_docker": true,
+      "requires_heavy_runner": true
+    },
+    {
+      "key": "cn-quickstart",
+      "label": "CN Quickstart external snippets",
+      "repository": "digital-asset/cn-quickstart",
+      "ref": "main",
+      "version": "main",
+      "repo_arg": "cn-quickstart",
+      "output_path": "docs-main/snippets/external/cn-quickstart/main"
+    },
+    {
+      "key": "daml",
+      "label": "Daml external snippets",
+      "repository": "digital-asset/daml",
+      "ref": "main",
+      "version": "main",
+      "repo_arg": "daml",
+      "output_path": "docs-main/snippets/external/daml/main"
+    },
+    {
+      "key": "daml-shell",
+      "label": "Daml Shell external snippets",
+      "repository": "DACH-NY/daml-shell",
+      "ref": "main",
+      "version": "main",
+      "repo_arg": "daml-shell",
+      "output_path": "docs-main/snippets/external/daml-shell/main",
+      "skip_if_unavailable": true
+    },
+    {
+      "key": "dpm",
+      "label": "DPM external snippets",
+      "repository": "digital-asset/dpm",
+      "ref": "main",
+      "version": "main",
+      "repo_arg": "dpm",
+      "output_path": "docs-main/snippets/external/dpm/main"
+    },
+    {
+      "key": "scribe",
+      "label": "Scribe external snippets",
+      "repository": "DACH-NY/scribe",
+      "ref": "main",
+      "version": "main",
+      "repo_arg": "scribe",
+      "output_path": "docs-main/snippets/external/scribe/main",
+      "skip_if_unavailable": true
+    },
+    {
+      "key": "splice",
+      "label": "Splice external snippets",
+      "repository": "canton-network/splice",
+      "ref": "main",
+      "version": "main",
+      "repo_arg": "splice",
+      "output_path": "docs-main/snippets/external/splice/main",
+      "preserve_paths": [
+        "common"
+      ]
+    }
+  ]
+}

config/snippet-config/update-workflows.md

@@ -136,7 +136,7 @@ The following token permission must be configured on these tokens:
 
 * repository scope: External repositories
   * `hyperledger-labs/splice-wallet-kernel/`
-  * `DACH-NY/canton`
+  * `digital-asset/canton`
   * `digital-asset/daml`
   * `hyperledger-labs/splice`
   * TODO: finalize list

config/x2mdx/grpc-ledger-api-reference/source-artifacts.json

@@ -3,8 +3,8 @@
   "release_url_template": "https://www.canton.io/releases/canton-open-source-{version}.tar.gz",
   "bundle_proto_dir": "protobuf",
   "repo": {
-    "remote": "https://github.com/DACH-NY/canton.git",
-    "web_url": "https://github.com/DACH-NY/canton"
+    "remote": "https://github.com/digital-asset/canton.git",
+    "web_url": "https://github.com/digital-asset/canton"
   },
   "min_version": "3.4.4",
   "metadata_path": "config/x2mdx/protobuf-history/metadata.json",

config/x2mdx/ledger-bindings/source-artifacts.json

@@ -7,7 +7,7 @@
       "language": "java",
       "status_manifest": "status/bindings-java.yaml",
       "include_prefixes": [
-        "com.daml"
+        "com.daml.ledger.javaapi"
       ],
       "versions": [
         "3.4.8",

config/x2mdx/ledger-bindings/status/bindings-java.yaml

@@ -267,6 +267,10 @@ types:
     status: stable
   com.daml.ledger.javaapi.data.User.Right.CanActAs:
     status: stable
+  com.daml.ledger.javaapi.data.User.Right.CanExecuteAs:
+    status: stable
+  com.daml.ledger.javaapi.data.User.Right.CanExecuteAsAnyParty:
+    status: stable
   com.daml.ledger.javaapi.data.User.Right.CanReadAs:
     status: stable
   com.daml.ledger.javaapi.data.User.Right.CanReadAsAnyParty:
@@ -341,6 +345,8 @@ types:
     status: stable
   com.daml.ledger.javaapi.data.codegen.Update:
     status: stable
+  com.daml.ledger.javaapi.data.codegen.UnknownTrailingFieldPolicy:
+    status: stable
   com.daml.ledger.javaapi.data.codegen.ValueDecoder:
     status: stable
   com.daml.ledger.javaapi.data.codegen.Variant:

config/x2mdx/protobuf-history/source-artifacts.json

@@ -3,8 +3,8 @@
   "release_url_template": "https://www.canton.io/releases/canton-open-source-{version}.tar.gz",
   "bundle_proto_dir": "protobuf",
   "repo": {
-    "remote": "https://github.com/DACH-NY/canton.git",
-    "web_url": "https://github.com/DACH-NY/canton"
+    "remote": "https://github.com/digital-asset/canton.git",
+    "web_url": "https://github.com/digital-asset/canton"
   },
   "min_version": "3.2.0",
   "excluded_versions": ["3.4.1"],

scripts/generate_canton_metrics_reference.py

@@ -20,8 +20,8 @@
 DEFAULT_CACHE_DIR = REPO_ROOT / ".internal" / "cache" / "canton-metrics-reference"
 DEFAULT_CANTON_DIR = DEFAULT_CACHE_DIR / "repos" / "canton"
 DEFAULT_OUTPUT = REPO_ROOT / "docs-main" / "global-synchronizer" / "reference" / "canton-metrics.mdx"
-DEFAULT_REMOTE = "https://github.com/DACH-NY/canton.git"
-DEFAULT_RELEASE_REPO = "DACH-NY/canton"
+DEFAULT_REMOTE = "https://github.com/digital-asset/canton.git"
+DEFAULT_RELEASE_REPO = "digital-asset/canton"
 METRICS_RST = Path("docs-open/src/sphinx/participant/reference/metrics.rst")
 GENERATED_INCLUDES_DIR = Path("docs-open/target/generated")
 USER_AGENT = "cf-docs-canton-metrics-reference/1.0"
@@ -152,11 +152,12 @@ def run_generation(*, canton_dir: Path, command: list[str], skip_direnv: bool) -
     if generated.exists():
         shutil.rmtree(generated)
     generated.mkdir(parents=True, exist_ok=True)
+    command_without_ci = ["env", "-u", "CI", *command]
     if skip_direnv or not (canton_dir / ".envrc").exists() or not shutil.which("direnv"):
-        run(command, cwd=canton_dir)
+        run(command_without_ci, cwd=canton_dir)
         return
     allow_direnv(canton_dir)
-    run(["direnv", "exec", str(canton_dir), *command], cwd=canton_dir)
+    run(["direnv", "exec", str(canton_dir), *command_without_ci], cwd=canton_dir)
 
 
 def resolve_generated_includes(template: str, *, generated_dir: Path) -> str:
@@ -193,7 +194,7 @@ def convert_rst_to_mdx(rst: str, *, source_ref: str) -> str:
         "",
         (
             "{/* GENERATED_FROM "
-            f'source="DACH-NY/canton" ref="{source_ref}" '
+            f'source="digital-asset/canton" ref="{source_ref}" '
             f'path="{METRICS_RST.as_posix()}" */}}'
         ),
         "",

scripts/generate_canton_protobuf_history.py

@@ -145,8 +145,10 @@ def ensure_repo(repo_dir: Path, *, remote: str, fetch: bool) -> Path:
     repo_dir.parent.mkdir(parents=True, exist_ok=True)
     if not repo_dir.exists():
         run(["git", "clone", "--bare", remote, str(repo_dir)])
+    else:
+        git(["remote", "set-url", "origin", remote], cwd=repo_dir)
     if fetch:
-        git(["fetch", "origin", "--tags", "--prune"], cwd=repo_dir)
+        git(["fetch", "origin", "--tags", "--prune", "--force"], cwd=repo_dir)
     return repo_dir
 
 

scripts/generate_daml_standard_library_json.sh

@@ -11,7 +11,7 @@ Usage: generate_daml_standard_library_json.sh --output-json PATH [options]
 Generate Daml Standard Library docs JSON using installed SDK artifacts.
 
 SDK source selection:
-- dpm (default): use DPM cache + `dpm damlc docs`.
+- dpm (default): use DPM cache + cached damlc binary.
 - auto: prefer DPM cache under ~/.dpm, fallback to DAML SDK installation
   under ~/.daml/sdk/<version>.
 - daml: use DAML SDK layout + damlc binary.
@@ -147,6 +147,10 @@ dpm_pkg_db_root() {
   printf '%s\n' "$DPM_HOME_DIR/cache/components/damlc/$SDK_VERSION/damlc-dist-dpm/resources/pkg-db_dir"
 }
 
+dpm_damlc_bin() {
+  printf '%s\n' "$DPM_HOME_DIR/cache/components/damlc/$SDK_VERSION/damlc-dist-dpm/damlc"
+}
+
 ensure_daml_sdk() {
   local pkg_db_root
   pkg_db_root="$(daml_pkg_db_root)"
@@ -237,11 +241,12 @@ configure_dpm_source() {
     return 1
   fi
   PKG_DB_ROOT="$(dpm_pkg_db_root)"
-  if ! command -v dpm >/dev/null 2>&1; then
-    echo "dpm not found in PATH." >&2
+  DPM_DAMLC_BIN="$(dpm_damlc_bin)"
+  if [[ ! -x "$DPM_DAMLC_BIN" ]]; then
+    echo "DPM damlc binary not found: $DPM_DAMLC_BIN" >&2
     return 1
   fi
-  DOCS_CMD=("dpm" "damlc" "docs")
+  DOCS_CMD=("$DPM_DAMLC_BIN" "docs")
   SDK_SOURCE="dpm"
   return 0
 }

scripts/generate_daml_standard_library_reference.py

@@ -12,6 +12,7 @@
 from typing import Any
 
 from docs_env import ensure_repo_direnv, repo_direnv_command
+import reference_nav
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 DEFAULT_CACHE_ROOT = Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")).expanduser() / "x2mdx"
@@ -229,15 +230,7 @@ def update_docs_navigation(
     output_dir: Path,
 ) -> Path:
     docs = load_json(docs_json_path)
-    dropdowns = docs.get("navigation", {}).get("dropdowns")
-    if not isinstance(dropdowns, list):
-        raise ValueError(f"docs.json navigation.dropdowns must be a list: {docs_json_path}")
-    dropdown = next((item for item in dropdowns if isinstance(item, dict) and item.get("dropdown") == dropdown_label), None)
-    if dropdown is None:
-        raise ValueError(f"Dropdown not found in docs.json: {dropdown_label}")
-    pages = dropdown.get("pages")
-    if not isinstance(pages, list):
-        raise ValueError(f"Dropdown does not expose a pages list: {dropdown_label}")
+    pages = reference_nav.navigation_pages(docs, label=dropdown_label, docs_json_path=docs_json_path)
 
     page_entries: list[tuple[str, str, Path]] = []
     for page in sorted(output_dir.glob("*.mdx")):
@@ -247,8 +240,10 @@ def update_docs_navigation(
     page_refs = {page_ref for _title, page_ref, _path in page_entries}
 
     existing_group_index = find_group_index(find_group_path(pages, parent_groups), GROUP_LABEL)
-    dropdown["pages"] = prune_nav_items(pages, page_refs=page_refs, group_labels={GROUP_LABEL})
-    target_pages = ensure_group_path(dropdown["pages"], parent_groups)
+    pruned_pages = prune_nav_items(pages, page_refs=page_refs, group_labels={GROUP_LABEL})
+    pages.clear()
+    pages.extend(pruned_pages)
+    target_pages = ensure_group_path(pages, parent_groups)
     overview_entry = next(((page_ref, path) for _title, page_ref, path in page_entries if path.name == "index.mdx"), None)
     module_refs = [page_ref for _title, page_ref, path in page_entries if path.name != "index.mdx"]
     group_pages: list[Any] = []