diff --git a/.github/workflows/update-version-dashboard.yml b/.github/workflows/update-version-dashboard.yml index bdced508..6db547d1 100644 --- a/.github/workflows/update-version-dashboard.yml +++ b/.github/workflows/update-version-dashboard.yml @@ -1,20 +1,29 @@ -name: Update version dashboard +name: Update generated docs on: schedule: - - cron: "23 */6 * * *" + - cron: "*/10 * * * *" workflow_dispatch: + inputs: + targets: + description: "Generated-doc targets to run" + type: string + default: "all" + include_heavy_external_snippets: + description: "Run external snippet targets that require a heavier runner" + type: boolean + default: false permissions: contents: write pull-requests: write concurrency: - group: update-version-dashboard + group: update-generated-docs cancel-in-progress: false jobs: - update-version-dashboard: + update-generated-docs: runs-on: ubuntu-latest steps: - name: Checkout repository @@ -27,10 +36,21 @@ jobs: run: | sudo apt-get update sudo apt-get install -y direnv - direnv allow . + SKIP_NPM_INSTALL=1 direnv allow . + + - name: Install npm dependencies + run: SKIP_NPM_INSTALL=1 direnv exec . npm ci + + - name: Set up Daml tooling + run: | + curl -fsSL https://get.digitalasset.com/install/install.sh | sh + echo "$HOME/.dpm/bin" >> "$GITHUB_PATH" + echo "$HOME/.daml/bin" >> "$GITHUB_PATH" - name: Generate update pull requests env: GH_TOKEN: ${{ github.token }} GITHUB_TOKEN: ${{ github.token }} - run: python3 scripts/update_generated_reference_prs.py --targets all + GENERATED_DOCS_ENABLE_HEAVY_EXTERNAL_SNIPPETS: ${{ inputs.include_heavy_external_snippets && '1' || '' }} + GENERATED_DOCS_TARGETS: ${{ inputs.targets || 'all' }} + run: python3 scripts/update_generated_reference_prs.py --targets $GENERATED_DOCS_TARGETS diff --git a/config/generated-docs/external-snippet-sources.json b/config/generated-docs/external-snippet-sources.json new file mode 100644 index 00000000..f794dd8b --- /dev/null +++ b/config/generated-docs/external-snippet-sources.json @@ -0,0 +1,74 @@ +{ + "sources": [ + { + "key": "canton", + "label": "Canton external snippets", + "repository": "digital-asset/canton", + "ref": "main", + "version": "main", + "repo_arg": "canton", + "output_path": "docs-main/snippets/external/canton/main", + "requires_docker": true, + "requires_heavy_runner": true + }, + { + "key": "cn-quickstart", + "label": "CN Quickstart external snippets", + "repository": "digital-asset/cn-quickstart", + "ref": "main", + "version": "main", + "repo_arg": "cn-quickstart", + "output_path": "docs-main/snippets/external/cn-quickstart/main" + }, + { + "key": "daml", + "label": "Daml external snippets", + "repository": "digital-asset/daml", + "ref": "main", + "version": "main", + "repo_arg": "daml", + "output_path": "docs-main/snippets/external/daml/main" + }, + { + "key": "daml-shell", + "label": "Daml Shell external snippets", + "repository": "DACH-NY/daml-shell", + "ref": "main", + "version": "main", + "repo_arg": "daml-shell", + "output_path": "docs-main/snippets/external/daml-shell/main", + "skip_if_unavailable": true + }, + { + "key": "dpm", + "label": "DPM external snippets", + "repository": "digital-asset/dpm", + "ref": "main", + "version": "main", + "repo_arg": "dpm", + "output_path": "docs-main/snippets/external/dpm/main" + }, + { + "key": "scribe", + "label": "Scribe external snippets", + "repository": "DACH-NY/scribe", + "ref": "main", + "version": "main", + "repo_arg": "scribe", + "output_path": "docs-main/snippets/external/scribe/main", + "skip_if_unavailable": true + }, + { + "key": "splice", + "label": "Splice external snippets", + "repository": "canton-network/splice", + "ref": "main", + "version": "main", + "repo_arg": "splice", + "output_path": "docs-main/snippets/external/splice/main", + "preserve_paths": [ + "common" + ] + } + ] +} diff --git a/config/snippet-config/update-workflows.md b/config/snippet-config/update-workflows.md index b23efb95..90595664 100644 --- a/config/snippet-config/update-workflows.md +++ b/config/snippet-config/update-workflows.md @@ -136,7 +136,7 @@ The following token permission must be configured on these tokens: * repository scope: External repositories * `hyperledger-labs/splice-wallet-kernel/` - * `DACH-NY/canton` + * `digital-asset/canton` * `digital-asset/daml` * `hyperledger-labs/splice` * TODO: finalize list diff --git a/config/x2mdx/grpc-ledger-api-reference/source-artifacts.json b/config/x2mdx/grpc-ledger-api-reference/source-artifacts.json index d9bf5992..5bf34d44 100644 --- a/config/x2mdx/grpc-ledger-api-reference/source-artifacts.json +++ b/config/x2mdx/grpc-ledger-api-reference/source-artifacts.json @@ -3,8 +3,8 @@ "release_url_template": "https://www.canton.io/releases/canton-open-source-{version}.tar.gz", "bundle_proto_dir": "protobuf", "repo": { - "remote": "https://github.com/DACH-NY/canton.git", - "web_url": "https://github.com/DACH-NY/canton" + "remote": "https://github.com/digital-asset/canton.git", + "web_url": "https://github.com/digital-asset/canton" }, "min_version": "3.4.4", "metadata_path": "config/x2mdx/protobuf-history/metadata.json", diff --git a/config/x2mdx/ledger-bindings/source-artifacts.json b/config/x2mdx/ledger-bindings/source-artifacts.json index c9422b93..02fac0c0 100644 --- a/config/x2mdx/ledger-bindings/source-artifacts.json +++ b/config/x2mdx/ledger-bindings/source-artifacts.json @@ -7,7 +7,7 @@ "language": "java", "status_manifest": "status/bindings-java.yaml", "include_prefixes": [ - "com.daml" + "com.daml.ledger.javaapi" ], "versions": [ "3.4.8", diff --git a/config/x2mdx/ledger-bindings/status/bindings-java.yaml b/config/x2mdx/ledger-bindings/status/bindings-java.yaml index 019c57e7..6a79e514 100644 --- a/config/x2mdx/ledger-bindings/status/bindings-java.yaml +++ b/config/x2mdx/ledger-bindings/status/bindings-java.yaml @@ -267,6 +267,10 @@ types: status: stable com.daml.ledger.javaapi.data.User.Right.CanActAs: status: stable + com.daml.ledger.javaapi.data.User.Right.CanExecuteAs: + status: stable + com.daml.ledger.javaapi.data.User.Right.CanExecuteAsAnyParty: + status: stable com.daml.ledger.javaapi.data.User.Right.CanReadAs: status: stable com.daml.ledger.javaapi.data.User.Right.CanReadAsAnyParty: @@ -341,6 +345,8 @@ types: status: stable com.daml.ledger.javaapi.data.codegen.Update: status: stable + com.daml.ledger.javaapi.data.codegen.UnknownTrailingFieldPolicy: + status: stable com.daml.ledger.javaapi.data.codegen.ValueDecoder: status: stable com.daml.ledger.javaapi.data.codegen.Variant: diff --git a/config/x2mdx/protobuf-history/source-artifacts.json b/config/x2mdx/protobuf-history/source-artifacts.json index 1c7365e4..c206cfe3 100644 --- a/config/x2mdx/protobuf-history/source-artifacts.json +++ b/config/x2mdx/protobuf-history/source-artifacts.json @@ -3,8 +3,8 @@ "release_url_template": "https://www.canton.io/releases/canton-open-source-{version}.tar.gz", "bundle_proto_dir": "protobuf", "repo": { - "remote": "https://github.com/DACH-NY/canton.git", - "web_url": "https://github.com/DACH-NY/canton" + "remote": "https://github.com/digital-asset/canton.git", + "web_url": "https://github.com/digital-asset/canton" }, "min_version": "3.2.0", "excluded_versions": ["3.4.1"], diff --git a/scripts/generate_canton_metrics_reference.py b/scripts/generate_canton_metrics_reference.py index 6d364d71..a5dbd69b 100644 --- a/scripts/generate_canton_metrics_reference.py +++ b/scripts/generate_canton_metrics_reference.py @@ -20,8 +20,8 @@ DEFAULT_CACHE_DIR = REPO_ROOT / ".internal" / "cache" / "canton-metrics-reference" DEFAULT_CANTON_DIR = DEFAULT_CACHE_DIR / "repos" / "canton" DEFAULT_OUTPUT = REPO_ROOT / "docs-main" / "global-synchronizer" / "reference" / "canton-metrics.mdx" -DEFAULT_REMOTE = "https://github.com/DACH-NY/canton.git" -DEFAULT_RELEASE_REPO = "DACH-NY/canton" +DEFAULT_REMOTE = "https://github.com/digital-asset/canton.git" +DEFAULT_RELEASE_REPO = "digital-asset/canton" METRICS_RST = Path("docs-open/src/sphinx/participant/reference/metrics.rst") GENERATED_INCLUDES_DIR = Path("docs-open/target/generated") USER_AGENT = "cf-docs-canton-metrics-reference/1.0" @@ -152,11 +152,12 @@ def run_generation(*, canton_dir: Path, command: list[str], skip_direnv: bool) - if generated.exists(): shutil.rmtree(generated) generated.mkdir(parents=True, exist_ok=True) + command_without_ci = ["env", "-u", "CI", *command] if skip_direnv or not (canton_dir / ".envrc").exists() or not shutil.which("direnv"): - run(command, cwd=canton_dir) + run(command_without_ci, cwd=canton_dir) return allow_direnv(canton_dir) - run(["direnv", "exec", str(canton_dir), *command], cwd=canton_dir) + run(["direnv", "exec", str(canton_dir), *command_without_ci], cwd=canton_dir) def resolve_generated_includes(template: str, *, generated_dir: Path) -> str: @@ -193,7 +194,7 @@ def convert_rst_to_mdx(rst: str, *, source_ref: str) -> str: "", ( "{/* GENERATED_FROM " - f'source="DACH-NY/canton" ref="{source_ref}" ' + f'source="digital-asset/canton" ref="{source_ref}" ' f'path="{METRICS_RST.as_posix()}" */}}' ), "", diff --git a/scripts/generate_canton_protobuf_history.py b/scripts/generate_canton_protobuf_history.py index 3aee1a1b..72da147a 100644 --- a/scripts/generate_canton_protobuf_history.py +++ b/scripts/generate_canton_protobuf_history.py @@ -145,8 +145,10 @@ def ensure_repo(repo_dir: Path, *, remote: str, fetch: bool) -> Path: repo_dir.parent.mkdir(parents=True, exist_ok=True) if not repo_dir.exists(): run(["git", "clone", "--bare", remote, str(repo_dir)]) + else: + git(["remote", "set-url", "origin", remote], cwd=repo_dir) if fetch: - git(["fetch", "origin", "--tags", "--prune"], cwd=repo_dir) + git(["fetch", "origin", "--tags", "--prune", "--force"], cwd=repo_dir) return repo_dir diff --git a/scripts/generate_daml_standard_library_json.sh b/scripts/generate_daml_standard_library_json.sh index f34b4ef1..911145f7 100644 --- a/scripts/generate_daml_standard_library_json.sh +++ b/scripts/generate_daml_standard_library_json.sh @@ -11,7 +11,7 @@ Usage: generate_daml_standard_library_json.sh --output-json PATH [options] Generate Daml Standard Library docs JSON using installed SDK artifacts. SDK source selection: -- dpm (default): use DPM cache + `dpm damlc docs`. +- dpm (default): use DPM cache + cached damlc binary. - auto: prefer DPM cache under ~/.dpm, fallback to DAML SDK installation under ~/.daml/sdk/. - daml: use DAML SDK layout + damlc binary. @@ -147,6 +147,10 @@ dpm_pkg_db_root() { printf '%s\n' "$DPM_HOME_DIR/cache/components/damlc/$SDK_VERSION/damlc-dist-dpm/resources/pkg-db_dir" } +dpm_damlc_bin() { + printf '%s\n' "$DPM_HOME_DIR/cache/components/damlc/$SDK_VERSION/damlc-dist-dpm/damlc" +} + ensure_daml_sdk() { local pkg_db_root pkg_db_root="$(daml_pkg_db_root)" @@ -237,11 +241,12 @@ configure_dpm_source() { return 1 fi PKG_DB_ROOT="$(dpm_pkg_db_root)" - if ! command -v dpm >/dev/null 2>&1; then - echo "dpm not found in PATH." >&2 + DPM_DAMLC_BIN="$(dpm_damlc_bin)" + if [[ ! -x "$DPM_DAMLC_BIN" ]]; then + echo "DPM damlc binary not found: $DPM_DAMLC_BIN" >&2 return 1 fi - DOCS_CMD=("dpm" "damlc" "docs") + DOCS_CMD=("$DPM_DAMLC_BIN" "docs") SDK_SOURCE="dpm" return 0 } diff --git a/scripts/generate_daml_standard_library_reference.py b/scripts/generate_daml_standard_library_reference.py index ba04d413..4d4008af 100644 --- a/scripts/generate_daml_standard_library_reference.py +++ b/scripts/generate_daml_standard_library_reference.py @@ -12,6 +12,7 @@ from typing import Any from docs_env import ensure_repo_direnv, repo_direnv_command +import reference_nav REPO_ROOT = Path(__file__).resolve().parents[1] DEFAULT_CACHE_ROOT = Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")).expanduser() / "x2mdx" @@ -229,15 +230,7 @@ def update_docs_navigation( output_dir: Path, ) -> Path: docs = load_json(docs_json_path) - dropdowns = docs.get("navigation", {}).get("dropdowns") - if not isinstance(dropdowns, list): - raise ValueError(f"docs.json navigation.dropdowns must be a list: {docs_json_path}") - dropdown = next((item for item in dropdowns if isinstance(item, dict) and item.get("dropdown") == dropdown_label), None) - if dropdown is None: - raise ValueError(f"Dropdown not found in docs.json: {dropdown_label}") - pages = dropdown.get("pages") - if not isinstance(pages, list): - raise ValueError(f"Dropdown does not expose a pages list: {dropdown_label}") + pages = reference_nav.navigation_pages(docs, label=dropdown_label, docs_json_path=docs_json_path) page_entries: list[tuple[str, str, Path]] = [] for page in sorted(output_dir.glob("*.mdx")): @@ -247,8 +240,10 @@ def update_docs_navigation( page_refs = {page_ref for _title, page_ref, _path in page_entries} existing_group_index = find_group_index(find_group_path(pages, parent_groups), GROUP_LABEL) - dropdown["pages"] = prune_nav_items(pages, page_refs=page_refs, group_labels={GROUP_LABEL}) - target_pages = ensure_group_path(dropdown["pages"], parent_groups) + pruned_pages = prune_nav_items(pages, page_refs=page_refs, group_labels={GROUP_LABEL}) + pages.clear() + pages.extend(pruned_pages) + target_pages = ensure_group_path(pages, parent_groups) overview_entry = next(((page_ref, path) for _title, page_ref, path in page_entries if path.name == "index.mdx"), None) module_refs = [page_ref for _title, page_ref, path in page_entries if path.name != "index.mdx"] group_pages: list[Any] = [] diff --git a/scripts/generate_external_snippet_target.py b/scripts/generate_external_snippet_target.py new file mode 100644 index 00000000..808668e6 --- /dev/null +++ b/scripts/generate_external_snippet_target.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_CONFIG = REPO_ROOT / "config" / "generated-docs" / "external-snippet-sources.json" +DEFAULT_CACHE_DIR = REPO_ROOT / ".internal" / "cache" / "external-snippets" +HEAVY_RUNNER_ENV = "GENERATED_DOCS_ENABLE_HEAVY_EXTERNAL_SNIPPETS" + + +@dataclass(frozen=True) +class ExternalSnippetSource: + key: str + label: str + repository: str + ref: str + version: str + repo_arg: str + output_path: str + requires_docker: bool = False + requires_heavy_runner: bool = False + skip_if_unavailable: bool = False + preserve_paths: tuple[str, ...] = () + + +class SourceUnavailableError(RuntimeError): + pass + + +def load_sources(config_path: Path) -> tuple[ExternalSnippetSource, ...]: + payload = json.loads(config_path.read_text(encoding="utf-8")) + items = payload.get("sources") if isinstance(payload, dict) else None + if not isinstance(items, list): + raise ValueError(f"Expected sources list in {config_path}") + sources: list[ExternalSnippetSource] = [] + for item in items: + if not isinstance(item, dict): + continue + preserve_paths = item.get("preserve_paths", []) + if not isinstance(preserve_paths, list) or not all(isinstance(path, str) for path in preserve_paths): + raise ValueError(f"preserve_paths must be a list of strings for external snippet source: {item}") + try: + sources.append( + ExternalSnippetSource( + key=str(item["key"]), + label=str(item["label"]), + repository=str(item["repository"]), + ref=str(item["ref"]), + version=str(item["version"]), + repo_arg=str(item["repo_arg"]), + output_path=str(item["output_path"]), + requires_docker=bool(item.get("requires_docker", False)), + requires_heavy_runner=bool(item.get("requires_heavy_runner", False)), + skip_if_unavailable=bool(item.get("skip_if_unavailable", False)), + preserve_paths=tuple(preserve_paths), + ) + ) + except KeyError as error: + raise ValueError(f"External snippet source missing field {error.args[0]!r}: {item}") from error + return tuple(sources) + + +def source_by_key(config_path: Path, key: str) -> ExternalSnippetSource: + for source in load_sources(config_path): + if source.key == key: + return source + raise SystemExit(f"Unknown external snippet source {key!r}") + + +def run(command: list[str], *, cwd: Path, dry_run: bool = False) -> str: + print("$ " + " ".join(command)) + if dry_run: + return "" + completed = subprocess.run( + command, + cwd=cwd, + check=True, + text=True, + stdout=subprocess.PIPE, + ) + if completed.stdout: + print(completed.stdout, end="") + return completed.stdout.strip() + + +def clone_url(repository: str) -> str: + return f"https://github.com/{repository}.git" + + +def source_dir(cache_dir: Path, source: ExternalSnippetSource) -> Path: + return cache_dir / source.key / "repo" + + +def ensure_source_available(source: ExternalSnippetSource, *, dry_run: bool) -> None: + if dry_run: + return + try: + run( + ["git", "ls-remote", "--exit-code", clone_url(source.repository), source.ref], + cwd=REPO_ROOT, + dry_run=False, + ) + except subprocess.CalledProcessError as error: + raise SourceUnavailableError( + f"{source.label} source {source.repository}@{source.ref} is not available to this runner" + ) from error + + +def ensure_checkout(source: ExternalSnippetSource, *, cache_dir: Path, dry_run: bool) -> Path: + ensure_source_available(source, dry_run=dry_run) + checkout = source_dir(cache_dir, source) + if not dry_run: + checkout.parent.mkdir(parents=True, exist_ok=True) + if not (checkout / ".git").exists(): + run(["git", "clone", clone_url(source.repository), str(checkout)], cwd=REPO_ROOT, dry_run=dry_run) + run(["git", "-c", "gc.auto=0", "-c", "maintenance.auto=false", "fetch", "origin"], cwd=checkout, dry_run=dry_run) + run(["git", "-c", "gc.auto=0", "-c", "maintenance.auto=false", "checkout", source.ref], cwd=checkout, dry_run=dry_run) + run(["git", "-c", "gc.auto=0", "-c", "maintenance.auto=false", "reset", "--hard", source.ref], cwd=checkout, dry_run=dry_run) + run(["git", "clean", "-ffd"], cwd=checkout, dry_run=dry_run) + return checkout + + +def allow_direnv(checkout: Path, *, dry_run: bool) -> None: + if not (checkout / ".envrc").is_file() or not shutil.which("direnv"): + return + run(["direnv", "allow"], cwd=checkout, dry_run=dry_run) + + +def check_docker(source: ExternalSnippetSource, *, dry_run: bool) -> None: + if not source.requires_docker: + return + run(["docker", "info", "--format", "{{.ServerVersion}}"], cwd=REPO_ROOT, dry_run=dry_run) + + +def heavy_runner_enabled() -> bool: + value = os.environ.get(HEAVY_RUNNER_ENV, "") + return value.lower() in {"1", "true", "yes", "on"} + + +def generate_source(source: ExternalSnippetSource, *, cache_dir: Path, dry_run: bool) -> None: + if source.requires_heavy_runner and not heavy_runner_enabled(): + print( + f"Skipping {source.key}: requires a heavy runner. " + f"Set {HEAVY_RUNNER_ENV}=1 to enable this target." + ) + return + try: + checkout = ensure_checkout(source, cache_dir=cache_dir, dry_run=dry_run) + except SourceUnavailableError as error: + if source.skip_if_unavailable: + print(f"Skipping {source.key}: {error}") + return + raise + allow_direnv(checkout, dry_run=dry_run) + check_docker(source, dry_run=dry_run) + with preserved_output_paths(source, dry_run=dry_run): + run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "generate_external_snippets.py"), + source.repo_arg, + "--source-dir", + str(checkout), + "--copy-output", + "--replace-output", + "--version", + source.version, + "--fetch", + ], + cwd=REPO_ROOT, + dry_run=dry_run, + ) + + +class preserved_output_paths: + def __init__(self, source: ExternalSnippetSource, *, dry_run: bool) -> None: + self.source = source + self.dry_run = dry_run + self.temp_dir: tempfile.TemporaryDirectory[str] | None = None + self.saved: list[tuple[Path, Path]] = [] + + def __enter__(self) -> None: + if self.dry_run or not self.source.preserve_paths: + return + target_root = REPO_ROOT / self.source.output_path + self.temp_dir = tempfile.TemporaryDirectory() + backup_root = Path(self.temp_dir.name) + for relative in self.source.preserve_paths: + if Path(relative).is_absolute() or ".." in Path(relative).parts: + raise ValueError(f"Invalid preserve path for {self.source.key}: {relative}") + source_path = target_root / relative + if not source_path.exists(): + continue + backup_path = backup_root / relative + backup_path.parent.mkdir(parents=True, exist_ok=True) + if source_path.is_dir(): + shutil.copytree(source_path, backup_path) + else: + shutil.copy2(source_path, backup_path) + self.saved.append((source_path, backup_path)) + + def __exit__(self, exc_type: object, exc: object, tb: object) -> None: + try: + for source_path, backup_path in self.saved: + source_path.parent.mkdir(parents=True, exist_ok=True) + if source_path.exists(): + if source_path.is_dir(): + shutil.rmtree(source_path) + else: + source_path.unlink() + if backup_path.is_dir(): + shutil.copytree(backup_path, source_path) + else: + shutil.copy2(backup_path, source_path) + finally: + if self.temp_dir is not None: + self.temp_dir.cleanup() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate one configured external snippet output tree.") + parser.add_argument("--source-key", required=True, help="External snippet source key from the manifest.") + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR) + parser.add_argument("--dry-run", action="store_true", help="Print clone/generation commands without running them.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + source = source_by_key(args.config, args.source_key) + generate_source(source, cache_dir=args.cache_dir, dry_run=args.dry_run) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generate_external_snippets.py b/scripts/generate_external_snippets.py index 77f4e27b..4be1d5b8 100644 --- a/scripts/generate_external_snippets.py +++ b/scripts/generate_external_snippets.py @@ -345,7 +345,7 @@ def prepare_repo(repo: SnippetRepo, source_dir: Path, skip_prepare: bool, dry_ru env = os.environ.copy() commands = repo.prepare if repo.name == "canton": - env["SBT_OPTS"] = os.environ.get("SNIPPET_CANTON_SBT_OPTS", "-Xmx8G -Xms2G") + env["SBT_OPTS"] = os.environ.get("SNIPPET_CANTON_SBT_OPTS", "-Xmx4G -Xms1G") commands = tuple(f'SBT_OPTS="{env["SBT_OPTS"]}" {command}' for command in commands) for command in commands: run(command_for_repo(source_dir, command), cwd=source_dir, dry_run=dry_run, env=env) @@ -376,9 +376,20 @@ def copy_output(repo: SnippetRepo, source_dir: Path, version: str, replace: bool shutil.rmtree(target) target.mkdir(parents=True, exist_ok=True) shutil.copytree(source_output, target, dirs_exist_ok=True) + normalize_generated_markdown(target) return target +def normalize_generated_markdown(path: Path) -> None: + for file_path in path.rglob("*.mdx"): + text = file_path.read_text(encoding="utf-8") + normalized = "\n".join(line.rstrip() for line in text.splitlines()) + if text.endswith("\n"): + normalized += "\n" + if normalized != text: + file_path.write_text(normalized, encoding="utf-8") + + def validate_inputs(repo: SnippetRepo) -> None: missing = [ path diff --git a/scripts/generate_grpc_ledger_api_reference.py b/scripts/generate_grpc_ledger_api_reference.py index 51ba24cb..c4b88018 100644 --- a/scripts/generate_grpc_ledger_api_reference.py +++ b/scripts/generate_grpc_ledger_api_reference.py @@ -44,6 +44,7 @@ DETAILS_LABEL = "Details and history" DEFAULT_INSERT_AFTER_GROUP = "Ledger API Endpoints" DEFAULT_SOURCE_NAME = "Canton Ledger API protobuf release bundles" +DEFAULT_NAV_GROUP = ["Ledger API"] LEDGER_API_PACKAGE_PREFIX = "com.daml.ledger.api." @@ -73,7 +74,10 @@ def parse_args() -> argparse.Namespace: "--version-filter", help="Version-filter label embedded in generated content.", ) - return parser.parse_args() + args = parser.parse_args() + if args.nav_group is None: + args.nav_group = DEFAULT_NAV_GROUP + return args def load_json(path: Path) -> dict[str, Any]: @@ -537,12 +541,12 @@ def update_docs_navigation( details_path=details_path, page_paths=page_paths, ) - pages[:] = canton_protobuf_history.prune_nav_items( - pages, + target_pages = canton_protobuf_history.ensure_group_path(pages, parent_groups) + target_pages[:] = canton_protobuf_history.prune_nav_items( + target_pages, page_refs=generated_refs, group_labels={GROUP_LABEL, LEGACY_GROUP_LABEL}, ) - target_pages = canton_protobuf_history.ensure_group_path(pages, parent_groups) insert_group(target_pages, group=nav_group, after_group=insert_after_group) docs_json_path.write_text(json.dumps(docs, indent=2) + "\n", encoding="utf-8") print(f"Updated docs navigation: {docs_json_path}") diff --git a/scripts/generate_json_api_asyncapi_reference.py b/scripts/generate_json_api_asyncapi_reference.py index f70f64ca..2e7539a5 100644 --- a/scripts/generate_json_api_asyncapi_reference.py +++ b/scripts/generate_json_api_asyncapi_reference.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import copy import html import json import os @@ -429,6 +430,23 @@ def normalize_nav_group_into_pages(*, docs_json_path: Path, dropdown_label: str, def build_command(args: argparse.Namespace, manifest_path: Path, publish_version: str, versions: list[str]) -> list[str]: + return build_command_with_docs_json( + args, + manifest_path=manifest_path, + publish_version=publish_version, + versions=versions, + docs_json_path=Path(args.docs_json).resolve(), + ) + + +def build_command_with_docs_json( + args: argparse.Namespace, + *, + manifest_path: Path, + publish_version: str, + versions: list[str], + docs_json_path: Path, +) -> list[str]: nav_groups = args.nav_group if args.nav_group is not None else [DEFAULT_NAV_GROUP] command = repo_direnv_command( REPO_ROOT, @@ -440,7 +458,7 @@ def build_command(args: argparse.Namespace, manifest_path: Path, publish_version "--fixture-root", str(REPO_ROOT), "--docs-json", - str(Path(args.docs_json).resolve()), + str(docs_json_path), "--nav-dropdown", args.nav_dropdown, "--publish-version", @@ -472,6 +490,41 @@ def build_command(args: argparse.Namespace, manifest_path: Path, publish_version return command +def with_legacy_dropdown_scratch(docs: dict[str, object], *, dropdown_label: str) -> dict[str, object]: + scratch = copy.deepcopy(docs) + navigation = scratch.get("navigation") + if not isinstance(navigation, dict) or isinstance(navigation.get("dropdowns"), list): + return scratch + products = navigation.get("products") + if not isinstance(products, list): + return scratch + product = next( + (item for item in products if isinstance(item, dict) and item.get("product") == dropdown_label), + None, + ) + if product is None or not isinstance(product.get("pages"), list): + return scratch + navigation["dropdowns"] = [ + { + "dropdown": dropdown_label, + "pages": copy.deepcopy(product["pages"]), + } + ] + return scratch + + +def x2mdx_docs_json_path(*, docs_json_path: Path, baseline_docs: dict[str, object], dropdown_label: str) -> Path: + navigation = baseline_docs.get("navigation") + if not isinstance(navigation, dict) or isinstance(navigation.get("dropdowns"), list): + return docs_json_path + scratch_path = docs_json_path.with_name(".docs-json-x2mdx-scratch.json") + scratch_path.write_text( + json.dumps(with_legacy_dropdown_scratch(baseline_docs, dropdown_label=dropdown_label), indent=2) + "\n", + encoding="utf-8", + ) + return scratch_path + + def remove_legacy_output(*, output_file: Path | None, output_dir: Path | None) -> None: legacy_output = LEGACY_OUTPUT_FILE.resolve() if output_file is not None and output_file == legacy_output: @@ -523,16 +576,26 @@ def main() -> int: publish_version=publish_version, ) - command = build_command( + docs_json_path = Path(args.docs_json).resolve() + baseline_docs = load_json(docs_json_path) + command_docs_json_path = x2mdx_docs_json_path( + docs_json_path=docs_json_path, + baseline_docs=baseline_docs, + dropdown_label=args.nav_dropdown, + ) + command = build_command_with_docs_json( args, manifest_path=manifest_path, publish_version=publish_version, versions=[entry["version"] for entry in selected_version_entries], + docs_json_path=command_docs_json_path, ) - docs_json_path = Path(args.docs_json).resolve() - baseline_docs = load_json(docs_json_path) print("Running:", " ".join(command)) - completed = subprocess.run(command, cwd=REPO_ROOT) + try: + completed = subprocess.run(command, cwd=REPO_ROOT) + finally: + if command_docs_json_path != docs_json_path and command_docs_json_path.exists(): + command_docs_json_path.unlink() if completed.returncode == 0: nav_groups = args.nav_group if args.nav_group is not None else [DEFAULT_NAV_GROUP] if not args.output_file: diff --git a/scripts/generate_ledger_bindings_api_reference.py b/scripts/generate_ledger_bindings_api_reference.py index e4b91030..779720d1 100644 --- a/scripts/generate_ledger_bindings_api_reference.py +++ b/scripts/generate_ledger_bindings_api_reference.py @@ -285,23 +285,7 @@ def update_docs_navigation( publish_root: Path, ) -> Path: docs = load_json(docs_json_path) - navigation = docs.get("navigation") - if not isinstance(navigation, dict): - raise ValueError(f"docs.json missing navigation object: {docs_json_path}") - dropdowns = navigation.get("dropdowns") - if not isinstance(dropdowns, list): - raise ValueError(f"docs.json navigation.dropdowns must be a list: {docs_json_path}") - - dropdown = next( - (item for item in dropdowns if isinstance(item, dict) and item.get("dropdown") == dropdown_label), - None, - ) - if dropdown is None: - raise ValueError(f"Dropdown not found in docs.json: {dropdown_label}") - - pages = dropdown.get("pages") - if not isinstance(pages, list): - raise ValueError(f"Dropdown does not expose a pages list: {dropdown_label}") + pages = reference_nav.navigation_pages(docs, label=dropdown_label, docs_json_path=docs_json_path) jvm_group, generated_refs = build_jvm_nav_group( publish_root=publish_root, @@ -313,12 +297,14 @@ def update_docs_navigation( jvm_pages = jvm_group.setdefault("pages", []) if isinstance(jvm_pages, list) and overview_ref not in jvm_pages: jvm_pages.append(overview_ref) - dropdown["pages"] = prune_nav_items( + pruned_pages = prune_nav_items( pages, page_refs=generated_refs, group_labels={group_label}, ) - target_pages = ensure_group_path(dropdown["pages"], parent_groups) + pages.clear() + pages.extend(pruned_pages) + target_pages = ensure_group_path(pages, parent_groups) target_pages.append(jvm_group) docs_json_path.write_text(json.dumps(docs, indent=2) + "\n", encoding="utf-8") diff --git a/scripts/generate_splice_mintlify_openapi.py b/scripts/generate_splice_mintlify_openapi.py index efcb97bd..4bf98060 100644 --- a/scripts/generate_splice_mintlify_openapi.py +++ b/scripts/generate_splice_mintlify_openapi.py @@ -39,13 +39,22 @@ def version_key(version: str) -> tuple[int, ...]: return tuple(int(part) for part in version.split(".")) +def request_headers(url: str) -> dict[str, str]: + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": USER_AGENT, + } + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token and url.startswith("https://api.github.com/"): + headers["Authorization"] = f"Bearer {token}" + headers["X-GitHub-Api-Version"] = "2022-11-28" + return headers + + def github_json(url: str) -> Any: request = urllib.request.Request( url, - headers={ - "Accept": "application/vnd.github+json", - "User-Agent": USER_AGENT, - }, + headers=request_headers(url), ) with urllib.request.urlopen(request, timeout=180) as response: return json.loads(response.read().decode("utf-8")) diff --git a/scripts/generated_reference_nav.py b/scripts/generated_reference_nav.py index e33e5550..71d45597 100644 --- a/scripts/generated_reference_nav.py +++ b/scripts/generated_reference_nav.py @@ -176,26 +176,43 @@ def build_protobuf_nav_group( def replace_group_in_dropdown(*, docs_json_path: Path, dropdown_label: str, group: MintlifyNavGroup) -> None: payload = load_json(docs_json_path) + nav_pages = navigation_pages(payload, label=dropdown_label, docs_json_path=docs_json_path) + if not _replace_group(nav_pages, group): + nav_pages.append(group) + docs_json_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def navigation_pages(payload: JsonObject, *, label: str, docs_json_path: Path) -> MintlifyNavItems: navigation = payload.get("navigation") if not isinstance(navigation, dict): raise ValueError(f"docs.json missing navigation object: {docs_json_path}") dropdowns = navigation.get("dropdowns") - if not isinstance(dropdowns, list): - raise ValueError(f"docs.json navigation.dropdowns must be a list: {docs_json_path}") - dropdown = next( - (item for item in dropdowns if isinstance(item, dict) and item.get("dropdown") == dropdown_label), - None, - ) - if dropdown is None: - raise ValueError(f"Dropdown not found in docs.json: {dropdown_label}") - pages = dropdown.get("pages") - if not isinstance(pages, list): - raise ValueError(f"Dropdown does not expose a pages list: {dropdown_label}") - - nav_pages = cast(MintlifyNavItems, pages) - if not _replace_group(nav_pages, group): - nav_pages.append(group) - docs_json_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + if isinstance(dropdowns, list): + dropdown = next( + (item for item in dropdowns if isinstance(item, dict) and item.get("dropdown") == label), + None, + ) + if dropdown is None: + raise ValueError(f"Dropdown not found in docs.json: {label}") + pages = dropdown.get("pages") + if not isinstance(pages, list): + raise ValueError(f"Dropdown does not expose a pages list: {label}") + return cast(MintlifyNavItems, pages) + + products = navigation.get("products") + if isinstance(products, list): + product = next( + (item for item in products if isinstance(item, dict) and item.get("product") == label), + None, + ) + if product is None: + raise ValueError(f"Product not found in docs.json: {label}") + pages = product.get("pages") + if not isinstance(pages, list): + raise ValueError(f"Product does not expose a pages list: {label}") + return cast(MintlifyNavItems, pages) + + raise ValueError(f"docs.json navigation must define dropdowns or products: {docs_json_path}") def _replace_group(items: MintlifyNavItems, group: MintlifyNavGroup) -> bool: diff --git a/scripts/generated_reference_pr_utils.py b/scripts/generated_reference_pr_utils.py index fa831380..3671eb62 100644 --- a/scripts/generated_reference_pr_utils.py +++ b/scripts/generated_reference_pr_utils.py @@ -53,17 +53,65 @@ def has_changes(paths: Sequence[str]) -> bool: def push_branch(branch: str) -> None: + branch_ref = f"refs/heads/{branch}" remote_output = git("ls-remote", "--heads", "origin", branch, capture=True) remote_sha = remote_output.split()[0] if remote_output else "" if remote_sha: git( "push", - f"--force-with-lease=refs/heads/{branch}:{remote_sha}", + f"--force-with-lease={branch_ref}:{remote_sha}", "origin", - f"HEAD:{branch}", + f"HEAD:{branch_ref}", ) else: - git("push", "origin", f"HEAD:{branch}") + git("push", "origin", f"HEAD:{branch_ref}") + + +def open_pull_request_number(*, branch: str, base_branch: str, repository: str) -> str: + return gh( + "pr", + "list", + "--repo", + repository, + "--head", + branch, + "--base", + base_branch, + "--state", + "open", + "--json", + "number", + "--jq", + ".[0].number // empty", + capture=True, + ) + + +def close_stale_pull_request( + *, + title: str, + branch: str, + base_branch: str, + repository: str, +) -> None: + existing_pr_number = open_pull_request_number( + branch=branch, + base_branch=base_branch, + repository=repository, + ) + if not existing_pr_number: + return + gh( + "pr", + "close", + existing_pr_number, + "--repo", + repository, + "--delete-branch", + "--comment", + f"Closing because the latest generated-docs automation run found no changes for {title}.", + ) + print(f"Closed stale PR #{existing_pr_number} for {title}") def create_or_update_pull_request( @@ -77,32 +125,25 @@ def create_or_update_pull_request( ) -> None: if not has_changes(paths): print(f"No changes for {title}") + close_stale_pull_request( + title=title, + branch=branch, + base_branch=base_branch, + repository=repository, + ) return git("status", "--short", "--", *paths) - git("switch", "-C", branch) git("add", "--", *paths) git("diff", "--cached", "--stat") git("diff", "--cached", "--check") git("commit", "--signoff", "-m", title) push_branch(branch) - existing_pr_number = gh( - "pr", - "list", - "--repo", - repository, - "--head", - branch, - "--base", - base_branch, - "--state", - "open", - "--json", - "number", - "--jq", - ".[0].number // empty", - capture=True, + existing_pr_number = open_pull_request_number( + branch=branch, + base_branch=base_branch, + repository=repository, ) if existing_pr_number: gh( diff --git a/scripts/generated_reference_sources/canton_release_bundles.py b/scripts/generated_reference_sources/canton_release_bundles.py new file mode 100644 index 00000000..13003806 --- /dev/null +++ b/scripts/generated_reference_sources/canton_release_bundles.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import os +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import generate_canton_protobuf_history as canton_protobuf_history + +from generated_reference_sources.common import SourceUpdate, load_json, write_json + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SOURCE_LABEL = "JSON Ledger API release bundle" +DEFAULT_CANTON_REMOTE = "https://github.com/digital-asset/canton.git" +DEFAULT_TIMEOUT_SECONDS = 20.0 +USER_AGENT = "cf-docs-generated-reference-source-updater" +DEFAULT_CACHE_ROOT = Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")).expanduser() / "x2mdx" +DEFAULT_REPO_DIR = DEFAULT_CACHE_ROOT / "protobuf-history" / "repos" / "canton" + + +@dataclass(frozen=True) +class LedgerApiVersionConfig: + raw: dict[str, object] + version: str + canton_version: str + + +@dataclass(frozen=True) +class LedgerApiSourceConfig: + raw: dict[str, object] + publish_version: str + release_url_template: str + versions: tuple[LedgerApiVersionConfig, ...] + + +def parse_source_config(path: Path) -> LedgerApiSourceConfig: + raw_json = load_json(path) + publish_version = raw_json.get("publish_version") + release_url_template = raw_json.get("release_url_template") + versions_json = raw_json.get("versions") + if not isinstance(publish_version, str) or not publish_version: + raise ValueError(f"{path} must define non-empty publish_version") + if not isinstance(release_url_template, str) or not release_url_template: + raise ValueError(f"{path} must define non-empty release_url_template") + if not isinstance(versions_json, list) or not versions_json: + raise ValueError(f"{path} must define a non-empty versions list") + + versions: list[LedgerApiVersionConfig] = [] + for index, entry_json in enumerate(versions_json): + if not isinstance(entry_json, dict): + raise ValueError(f"{path} versions[{index}] must be an object") + version = entry_json.get("version") + canton_version = entry_json.get("canton_version") + if not isinstance(version, str) or not version: + raise ValueError(f"{path} versions[{index}] must define version") + if not isinstance(canton_version, str) or not canton_version: + raise ValueError(f"{path} versions[{version}] must define canton_version") + versions.append( + LedgerApiVersionConfig( + raw=dict(entry_json), + version=version, + canton_version=canton_version, + ) + ) + return LedgerApiSourceConfig( + raw=raw_json, + publish_version=publish_version, + release_url_template=release_url_template, + versions=tuple(versions), + ) + + +def release_url(source_config: LedgerApiSourceConfig, *, canton_version: str) -> str: + return source_config.release_url_template.format(canton_version=canton_version) + + +def release_bundle_exists( + source_config: LedgerApiSourceConfig, + *, + canton_version: str, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> bool: + request = urllib.request.Request( + release_url(source_config, canton_version=canton_version), + method="HEAD", + headers={"User-Agent": USER_AGENT}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout): + return True + except urllib.error.HTTPError as error: + if error.code in {403, 405}: + fallback = urllib.request.Request( + release_url(source_config, canton_version=canton_version), + headers={"User-Agent": USER_AGENT, "Range": "bytes=0-0"}, + ) + try: + with urllib.request.urlopen(fallback, timeout=timeout): + return True + except urllib.error.URLError: + return False + if error.code == 404: + return False + raise + except urllib.error.URLError: + return False + + +def latest_public_canton_bundle_version( + source_config: LedgerApiSourceConfig, + *, + docs_version: str, + repo_dir: Path = DEFAULT_REPO_DIR, + remote: str = DEFAULT_CANTON_REMOTE, +) -> str: + repo = canton_protobuf_history.ensure_repo(repo_dir, remote=remote, fetch=True) + candidates = [ + version + for version, _tag in canton_protobuf_history.stable_tags( + repo, + min_version=f"{docs_version}.0", + include_versions=None, + ) + if version.startswith(f"{docs_version}.") + ] + for version in reversed(candidates): + if release_bundle_exists(source_config, canton_version=version): + return version + raise ValueError(f"No public Canton release bundle found for docs version {docs_version}") + + +def update_source( + *, + source_config_path: Path, + dry_run: bool, +) -> SourceUpdate | None: + source_config = parse_source_config(source_config_path) + publish_entry = next( + (entry for entry in source_config.versions if entry.version == source_config.publish_version), + None, + ) + if publish_entry is None: + available = ", ".join(entry.version for entry in source_config.versions) + raise ValueError(f"Publish version {source_config.publish_version} not found in versions: {available}") + + current_version = latest_public_canton_bundle_version( + source_config, + docs_version=source_config.publish_version, + ) + if publish_entry.canton_version == current_version: + return None + + update = SourceUpdate( + source=SOURCE_LABEL, + path=source_config_path, + field=f"versions[{publish_entry.version}].canton_version", + previous=publish_entry.canton_version, + current=current_version, + ) + if not dry_run: + updated_config = dict(source_config.raw) + updated_versions: list[dict[str, Any]] = [] + for entry in source_config.versions: + updated_entry = dict(entry.raw) + if entry.version == publish_entry.version: + updated_entry["canton_version"] = current_version + updated_versions.append(updated_entry) + updated_config["versions"] = updated_versions + write_json(source_config_path, updated_config) + return update diff --git a/scripts/generated_reference_sources/daml_standard_library.py b/scripts/generated_reference_sources/daml_standard_library.py new file mode 100644 index 00000000..85d0b00f --- /dev/null +++ b/scripts/generated_reference_sources/daml_standard_library.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Required, TypedDict + +from generated_reference_sources.common import SourceUpdate, load_json, write_json + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SOURCE_KEY = "daml-standard-library" +SOURCE_LABEL = "Daml Standard Library" +DEFAULT_SOURCE_CONFIG = REPO_ROOT / "config" / "x2mdx" / "daml-standard-library" / "source-artifacts.json" +DEFAULT_TIMEOUT_SECONDS = 20.0 +DPM_LATEST_URL = "https://get.digitalasset.com/install/latest" +USER_AGENT = "cf-docs-generated-reference-source-updater" + + +class DamlStandardLibrarySourceConfigPayload(TypedDict, total=False): + source: str + publish_version: Required[str] + package_set: str + sdk_source: str + versions: Required[list[str]] + + +@dataclass(frozen=True) +class DamlStandardLibrarySourceConfig: + raw: DamlStandardLibrarySourceConfigPayload + publish_version: str + versions: tuple[str, ...] + + +def parse_source_config(path: Path) -> DamlStandardLibrarySourceConfig: + raw_json = load_json(path) + publish_version = raw_json.get("publish_version") + versions = raw_json.get("versions") + if not isinstance(publish_version, str) or not publish_version: + raise ValueError(f"{path} must define non-empty publish_version") + if not isinstance(versions, list) or not all(isinstance(version, str) and version for version in versions): + raise ValueError(f"{path} must define a non-empty versions string list") + raw: DamlStandardLibrarySourceConfigPayload = {} + raw.update(raw_json) + return DamlStandardLibrarySourceConfig(raw=raw, publish_version=publish_version, versions=tuple(versions)) + + +def latest_dpm_version(*, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> str: + request = urllib.request.Request(DPM_LATEST_URL, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=timeout) as response: + version = response.read().decode("utf-8").strip() + if not version: + raise ValueError(f"{DPM_LATEST_URL} returned an empty latest version") + return version + + +def update_source( + *, + source_config_path: Path, + dry_run: bool, +) -> SourceUpdate | None: + source_config = parse_source_config(source_config_path) + current_version = latest_dpm_version() + if source_config.publish_version == current_version: + return None + + update = SourceUpdate( + source=SOURCE_LABEL, + path=source_config_path, + field="publish_version", + previous=source_config.publish_version, + current=current_version, + ) + if not dry_run: + updated_config = dict(source_config.raw) + versions = list(source_config.versions) + if current_version not in versions: + versions.append(current_version) + updated_config["publish_version"] = current_version + updated_config["versions"] = versions + write_json(source_config_path, updated_config) + return update diff --git a/scripts/generated_reference_sources/ledger_bindings.py b/scripts/generated_reference_sources/ledger_bindings.py new file mode 100644 index 00000000..6d337f91 --- /dev/null +++ b/scripts/generated_reference_sources/ledger_bindings.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import re +import urllib.request +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from generated_reference_sources.common import SourceUpdate, load_json, write_json + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SOURCE_KEY = "ledger-bindings" +SOURCE_LABEL = "Java ledger bindings" +DEFAULT_SOURCE_CONFIG = REPO_ROOT / "config" / "x2mdx" / "ledger-bindings" / "source-artifacts.json" +DEFAULT_TIMEOUT_SECONDS = 20.0 +USER_AGENT = "cf-docs-generated-reference-source-updater" +STABLE_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") + + +@dataclass(frozen=True) +class LedgerBindingArtifactConfig: + raw: dict[str, object] + group: str + artifact: str + language: str + versions: tuple[str, ...] + + +@dataclass(frozen=True) +class LedgerBindingsSourceConfig: + raw: dict[str, object] + repo_base: str + artifacts: tuple[LedgerBindingArtifactConfig, ...] + + +def parse_source_config(path: Path) -> LedgerBindingsSourceConfig: + raw_json = load_json(path) + repo_base = raw_json.get("repo_base") + artifacts_json = raw_json.get("artifacts") + if not isinstance(repo_base, str) or not repo_base: + raise ValueError(f"{path} must define non-empty repo_base") + if not isinstance(artifacts_json, list) or not artifacts_json: + raise ValueError(f"{path} must define a non-empty artifacts list") + + artifacts: list[LedgerBindingArtifactConfig] = [] + for index, artifact_json in enumerate(artifacts_json): + if not isinstance(artifact_json, dict): + raise ValueError(f"{path} artifacts[{index}] must be an object") + group = artifact_json.get("group") + artifact = artifact_json.get("artifact") + language = artifact_json.get("language") + versions = artifact_json.get("versions") + if not isinstance(group, str) or not group: + raise ValueError(f"{path} artifacts[{index}] must define group") + if not isinstance(artifact, str) or not artifact: + raise ValueError(f"{path} artifacts[{index}] must define artifact") + if not isinstance(language, str) or not language: + raise ValueError(f"{path} artifacts[{group}:{artifact}] must define language") + if not isinstance(versions, list) or not all(isinstance(version, str) and version for version in versions): + raise ValueError(f"{path} artifacts[{group}:{artifact}] must define a non-empty versions string list") + artifacts.append( + LedgerBindingArtifactConfig( + raw=dict(artifact_json), + group=group, + artifact=artifact, + language=language, + versions=tuple(versions), + ) + ) + return LedgerBindingsSourceConfig(raw=raw_json, repo_base=repo_base, artifacts=tuple(artifacts)) + + +def version_key(version: str) -> tuple[int, int, int]: + major, minor, patch = version.split(".") + return (int(major), int(minor), int(patch)) + + +def metadata_url(repo_base: str, *, group: str, artifact: str) -> str: + group_path = group.replace(".", "/") + return f"{repo_base.rstrip('/')}/{group_path}/{artifact}/maven-metadata.xml" + + +def latest_maven_version( + repo_base: str, + *, + group: str, + artifact: str, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> str: + request = urllib.request.Request( + metadata_url(repo_base, group=group, artifact=artifact), + headers={"User-Agent": USER_AGENT}, + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = response.read() + root = ET.fromstring(payload) + versions = [ + node.text.strip() + for node in root.findall("./versioning/versions/version") + if node.text and STABLE_VERSION_RE.fullmatch(node.text.strip()) + ] + if not versions: + raise ValueError(f"No stable Maven versions found for {group}:{artifact}") + return sorted(versions, key=version_key)[-1] + + +def update_source( + *, + source_config_path: Path, + dry_run: bool, +) -> list[SourceUpdate]: + source_config = parse_source_config(source_config_path) + updates: list[SourceUpdate] = [] + updated_artifacts: list[dict[str, Any]] = [] + + for artifact in source_config.artifacts: + current_version = latest_maven_version( + source_config.repo_base, + group=artifact.group, + artifact=artifact.artifact, + ) + updated_artifact = dict(artifact.raw) + if current_version not in artifact.versions: + updates.append( + SourceUpdate( + source=f"{SOURCE_LABEL} {artifact.group}:{artifact.artifact}", + path=source_config_path, + field="versions", + previous=", ".join(artifact.versions), + current=current_version, + ) + ) + updated_artifact["versions"] = [*artifact.versions, current_version] + updated_artifacts.append(updated_artifact) + + if updates and not dry_run: + updated_config = dict(source_config.raw) + updated_config["artifacts"] = updated_artifacts + write_json(source_config_path, updated_config) + return updates diff --git a/scripts/generated_reference_sources/typescript_bindings.py b/scripts/generated_reference_sources/typescript_bindings.py new file mode 100644 index 00000000..cdea9c0c --- /dev/null +++ b/scripts/generated_reference_sources/typescript_bindings.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Required, TypedDict +from urllib.parse import quote +from urllib.request import Request, urlopen + +from generated_reference_sources.common import SourceUpdate, load_json, write_json + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SOURCE_KEY = "typescript-bindings" +SOURCE_LABEL = "TypeScript bindings" +DEFAULT_SOURCE_CONFIG = REPO_ROOT / "config" / "x2mdx" / "typescript-bindings" / "source-artifacts.json" +DEFAULT_TIMEOUT_SECONDS = 20.0 +USER_AGENT = "cf-docs-generated-reference-source-updater" + + +class TypeScriptPackageConfigPayload(TypedDict, total=False): + package_name: Required[str] + source: str + version_filter: str + page_title: str + page_description: str + output_file: str + entry_point: str + typedoc_args: list[str] + typedoc_version: str + publish_version: Required[str] + versions: Required[list[str]] + + +@dataclass(frozen=True) +class TypeScriptPackageConfig: + raw: TypeScriptPackageConfigPayload + package_name: str + publish_version: str + versions: tuple[str, ...] + + +@dataclass(frozen=True) +class TypeScriptBindingsSourceConfig: + raw: dict[str, object] + packages: tuple[TypeScriptPackageConfig, ...] + + +def parse_source_config(path: Path) -> TypeScriptBindingsSourceConfig: + raw_json = load_json(path) + packages_json = raw_json.get("packages") + if not isinstance(packages_json, list) or not packages_json: + raise ValueError(f"{path} must define a non-empty packages list") + + packages: list[TypeScriptPackageConfig] = [] + for index, package_json in enumerate(packages_json): + if not isinstance(package_json, dict): + raise ValueError(f"{path} packages[{index}] must be an object") + package_name = package_json.get("package_name") + publish_version = package_json.get("publish_version") + versions = package_json.get("versions") + if not isinstance(package_name, str) or not package_name: + raise ValueError(f"{path} packages[{index}] must define package_name") + if not isinstance(publish_version, str) or not publish_version: + raise ValueError(f"{path} packages[{package_name}] must define publish_version") + if not isinstance(versions, list) or not all(isinstance(version, str) and version for version in versions): + raise ValueError(f"{path} packages[{package_name}] must define a non-empty versions string list") + + raw: TypeScriptPackageConfigPayload = {} + raw.update(package_json) + packages.append( + TypeScriptPackageConfig( + raw=raw, + package_name=package_name, + publish_version=publish_version, + versions=tuple(versions), + ) + ) + return TypeScriptBindingsSourceConfig(raw=raw_json, packages=tuple(packages)) + + +def latest_npm_version(package_name: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> str: + encoded_name = quote(package_name, safe="") + request = Request( + f"https://registry.npmjs.org/{encoded_name}", + headers={"User-Agent": USER_AGENT}, + ) + with urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + latest = payload.get("dist-tags", {}).get("latest") + if not isinstance(latest, str) or not latest: + raise ValueError(f"npm package {package_name} does not define a latest dist-tag") + return latest + + +def update_source( + *, + source_config_path: Path, + dry_run: bool, +) -> list[SourceUpdate]: + source_config = parse_source_config(source_config_path) + updates: list[SourceUpdate] = [] + updated_packages: list[dict[str, object]] = [] + + for package in source_config.packages: + current_version = latest_npm_version(package.package_name) + updated_package = dict(package.raw) + if package.publish_version != current_version: + updates.append( + SourceUpdate( + source=f"{SOURCE_LABEL} {package.package_name}", + path=source_config_path, + field="publish_version", + previous=package.publish_version, + current=current_version, + ) + ) + versions = list(package.versions) + if current_version not in versions: + versions.append(current_version) + updated_package["publish_version"] = current_version + updated_package["versions"] = versions + updated_packages.append(updated_package) + + if updates and not dry_run: + updated_config = dict(source_config.raw) + updated_config["packages"] = updated_packages + write_json(source_config_path, updated_config) + return updates diff --git a/scripts/splice_openapi_release_bundles.py b/scripts/splice_openapi_release_bundles.py index 007578c3..801a988d 100644 --- a/scripts/splice_openapi_release_bundles.py +++ b/scripts/splice_openapi_release_bundles.py @@ -36,13 +36,22 @@ def version_key(version: str) -> tuple[int, ...]: return tuple(int(part) for part in version.split(".")) +def request_headers(url: str) -> dict[str, str]: + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": USER_AGENT, + } + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token and url.startswith("https://api.github.com/"): + headers["Authorization"] = f"Bearer {token}" + headers["X-GitHub-Api-Version"] = "2022-11-28" + return headers + + def github_json(url: str) -> Any: request = urllib.request.Request( url, - headers={ - "Accept": "application/vnd.github+json", - "User-Agent": USER_AGENT, - }, + headers=request_headers(url), ) with urllib.request.urlopen(request, timeout=180) as response: return json.loads(response.read().decode("utf-8")) diff --git a/scripts/summarize_version_changes.py b/scripts/summarize_version_changes.py index dece2eb2..8253e1a3 100644 --- a/scripts/summarize_version_changes.py +++ b/scripts/summarize_version_changes.py @@ -141,6 +141,98 @@ def source_config_changes(before_path: Path, after_path: Path, *, label: str) -> return changes +def package_source_config_changes(before_path: Path, after_path: Path, *, label: str) -> list[str]: + before = load_json(before_path) + after = load_json(after_path) + before_packages = { + package["package_name"]: package + for package in object_items(before.get("packages")) + if isinstance(package.get("package_name"), str) + } + changes: list[str] = [] + for package in object_items(after.get("packages")): + package_name = package.get("package_name") + if not isinstance(package_name, str): + continue + before_package = before_packages.get(package_name) + if before_package is None: + continue + before_version = before_package.get("publish_version") + after_version = package.get("publish_version") + if before_version != after_version: + changes.append( + f"- {label} {package_name} publish_version: " + f"{format_value(before_version)} -> {format_value(after_version)}" + ) + return changes + + +def versioned_source_config_changes(before_path: Path, after_path: Path, *, label: str) -> list[str]: + before = load_json(before_path) + after = load_json(after_path) + before_versions = { + item["version"]: item + for item in object_items(before.get("versions")) + if isinstance(item.get("version"), str) + } + changes: list[str] = [] + for item in object_items(after.get("versions")): + version = item.get("version") + if not isinstance(version, str): + continue + before_item = before_versions.get(version) + if before_item is None: + continue + for field in ("canton_version",): + if before_item.get(field) != item.get(field): + changes.append( + f"- {label} {version} {field}: " + f"{format_value(before_item.get(field))} -> {format_value(item.get(field))}" + ) + return changes + + +def artifact_source_config_changes(before_path: Path, after_path: Path, *, label: str) -> list[str]: + before = load_json(before_path) + after = load_json(after_path) + before_artifacts = { + f"{item.get('group')}:{item.get('artifact')}": item + for item in object_items(before.get("artifacts")) + if isinstance(item.get("group"), str) and isinstance(item.get("artifact"), str) + } + changes: list[str] = [] + for item in object_items(after.get("artifacts")): + group = item.get("group") + artifact = item.get("artifact") + if not isinstance(group, str) or not isinstance(artifact, str): + continue + artifact_key = f"{group}:{artifact}" + before_item = before_artifacts.get(artifact_key) + if before_item is None: + continue + before_versions = tuple(version for version in before_item.get("versions", []) if isinstance(version, str)) + after_versions = tuple(version for version in item.get("versions", []) if isinstance(version, str)) + added_versions = [version for version in after_versions if version not in before_versions] + if added_versions: + changes.append(f"- {label} {artifact_key} versions: added {', '.join(added_versions)}") + return changes + + +def external_snippet_source_changes(config_path: Path, *, target_key: str, label: str) -> list[str]: + config = load_json(config_path) + sources = object_items(config.get("sources")) + source = next((item for item in sources if item.get("key") == target_key), None) + if source is None: + return [] + repository = source.get("repository") + ref = source.get("ref") + version = source.get("version") + details = f"{format_value(repository)}@{format_value(ref)}" + if version is not None: + details += f" -> output version {format_value(version)}" + return [f"- {label} source: {details}"] + + def print_changes(changes: list[str]) -> None: if changes: print("\n".join(changes)) @@ -160,6 +252,34 @@ def parse_args() -> argparse.Namespace: source_config.add_argument("before", type=Path) source_config.add_argument("after", type=Path) source_config.add_argument("--label", required=True) + package_source_config = subparsers.add_parser( + "package-source-config", + help="Summarize package-based generated-reference source config changes.", + ) + package_source_config.add_argument("before", type=Path) + package_source_config.add_argument("after", type=Path) + package_source_config.add_argument("--label", required=True) + versioned_source_config = subparsers.add_parser( + "versioned-source-config", + help="Summarize generated-reference source config entries keyed by docs version.", + ) + versioned_source_config.add_argument("before", type=Path) + versioned_source_config.add_argument("after", type=Path) + versioned_source_config.add_argument("--label", required=True) + artifact_source_config = subparsers.add_parser( + "artifact-source-config", + help="Summarize artifact-based generated-reference source config changes.", + ) + artifact_source_config.add_argument("before", type=Path) + artifact_source_config.add_argument("after", type=Path) + artifact_source_config.add_argument("--label", required=True) + external_snippet_source = subparsers.add_parser( + "external-snippet-source", + help="Summarize a configured external snippet source.", + ) + external_snippet_source.add_argument("config", type=Path) + external_snippet_source.add_argument("--target-key", required=True) + external_snippet_source.add_argument("--label", required=True) return parser.parse_args() @@ -169,6 +289,20 @@ def main() -> int: print_changes(dashboard_changes(args.before, args.after)) elif args.command == "source-config": print_changes(source_config_changes(args.before, args.after, label=args.label)) + elif args.command == "package-source-config": + print_changes(package_source_config_changes(args.before, args.after, label=args.label)) + elif args.command == "versioned-source-config": + print_changes(versioned_source_config_changes(args.before, args.after, label=args.label)) + elif args.command == "artifact-source-config": + print_changes(artifact_source_config_changes(args.before, args.after, label=args.label)) + elif args.command == "external-snippet-source": + print_changes( + external_snippet_source_changes( + args.config, + target_key=args.target_key, + label=args.label, + ) + ) else: raise AssertionError(f"Unhandled command: {args.command}") return 0 diff --git a/scripts/update_generated_reference_prs.py b/scripts/update_generated_reference_prs.py index d7c98588..6c1f16a4 100644 --- a/scripts/update_generated_reference_prs.py +++ b/scripts/update_generated_reference_prs.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import json import os import sys import tempfile @@ -15,6 +16,25 @@ REPO_ROOT = Path(__file__).resolve().parents[1] +EXTERNAL_SNIPPET_CONFIG = REPO_ROOT / "config" / "generated-docs" / "external-snippet-sources.json" +NETWORK_VARIABLE_TAB_PAGES = ( + "docs-main/appdev/deep-dives/token-standard.mdx", + "docs-main/global-synchronizer/canton-console/console-overview.mdx", + "docs-main/global-synchronizer/deployment/kubernetes-deployment.mdx", + "docs-main/global-synchronizer/deployment/onboarding-process.mdx", + "docs-main/global-synchronizer/deployment/required-network-parameters.mdx", + "docs-main/global-synchronizer/deployment/sv-network-resets.mdx", + "docs-main/global-synchronizer/deployment/synchronizer-traffic.mdx", + "docs-main/global-synchronizer/deployment/validator-docker-compose.mdx", + "docs-main/global-synchronizer/deployment/validator-kubernetes.mdx", + "docs-main/global-synchronizer/production-operations/validator-disaster-recovery.mdx", + "docs-main/global-synchronizer/reference/canton-console-reference.mdx", + "docs-main/global-synchronizer/understand/local-testing.mdx", + "docs-main/sdks-tools/api-reference/splice-daml-apis.mdx", + "docs-main/sdks-tools/api-reference/splice-http-apis.mdx", + "docs-main/sdks-tools/api-reference/splice-scan-bulk-data-api.mdx", + "docs-main/sdks-tools/api-reference/splice-scan-gs-connectivity-api.mdx", +) @dataclass(frozen=True) @@ -26,9 +46,54 @@ class UpdateTarget: generate_commands: tuple[tuple[str, ...], ...] paths: tuple[str, ...] summary_kind: str - summary_path: str + summary_path: str | None summary_label: str | None validation: tuple[str, ...] + source_update_commands: tuple[tuple[str, ...], ...] = () + source_update_paths: tuple[str, ...] = () + + +def load_external_snippet_sources() -> tuple[dict[str, object], ...]: + payload = json.loads(EXTERNAL_SNIPPET_CONFIG.read_text(encoding="utf-8")) + sources = payload.get("sources") if isinstance(payload, dict) else None + if not isinstance(sources, list): + raise ValueError(f"Expected sources list in {EXTERNAL_SNIPPET_CONFIG}") + return tuple(source for source in sources if isinstance(source, dict)) + + +def external_snippet_targets() -> tuple[UpdateTarget, ...]: + targets: list[UpdateTarget] = [] + for source in load_external_snippet_sources(): + key = str(source["key"]) + label = str(source["label"]) + output_path = str(source["output_path"]) + targets.append( + UpdateTarget( + key=f"external-snippets-{key}", + title=f"Update {label}", + branch=f"generated-docs/external-snippets-{key}/update", + description=( + f"Updates the checked-in {label} from " + f"{source['repository']}@{source['ref']}." + ), + generate_commands=( + ( + "nix-shell", + "--run", + f"python3 scripts/generate_external_snippet_target.py --source-key {key}", + ), + ), + paths=(output_path,), + summary_kind="external-snippet-source", + summary_path=str(EXTERNAL_SNIPPET_CONFIG.relative_to(REPO_ROOT)), + summary_label=label, + validation=( + f"python3 scripts/generate_external_snippet_target.py --source-key {key}", + "git diff --check", + ), + ) + ) + return tuple(targets) UPDATE_TARGETS = ( @@ -38,20 +103,60 @@ class UpdateTarget: branch="version-dashboard/update", description=( "Updates the committed Canton Network version dashboard data from public network, " - "package, and installer sources." + "package, and installer sources, then refreshes generated pages that render " + "network-specific values from that data." ), - generate_commands=(("nix-shell", "--run", "npm run generate:version-compatibility-dashboard"),), + generate_commands=(("nix-shell", "--run", "npm run generate:network-variable-tabs"),), paths=( "config/repo-version-config.json", "docs-main/snippets/generated/version-dashboard-data.mdx", + *NETWORK_VARIABLE_TAB_PAGES, ), summary_kind="dashboard", summary_path="config/repo-version-config.json", summary_label=None, validation=( "npm run generate:version-compatibility-dashboard", + "npm run generate:network-variable-tabs", + "git diff --check", + ), + source_update_commands=( + ("nix-shell", "--run", "npm run generate:version-compatibility-dashboard"), + ), + source_update_paths=( + "config/repo-version-config.json", + "docs-main/snippets/generated/version-dashboard-data.mdx", + ), + ), + UpdateTarget( + key="splice-openapi", + title="Update Splice OpenAPI reference", + branch="generated-references/splice-openapi/update", + description=( + "Updates the Splice OpenAPI source pin to the latest stable " + "decentralized-canton-sync release and regenerates the checked-in " + "Splice OpenAPI specifications and navigation." + ), + generate_commands=( + ("nix-shell", "--run", "npm run generate:splice-mintlify-openapi"), + ), + paths=( + "config/mintlify-openapi/splice-openapi/source-artifacts.json", + "docs-main/docs.json", + "docs-main/openapi/splice", + ), + summary_kind="source-config", + summary_path="config/mintlify-openapi/splice-openapi/source-artifacts.json", + summary_label="Splice OpenAPI", + validation=( + "npm run update:generated-reference-sources -- --source splice-openapi", + "npm run generate:splice-mintlify-openapi", "git diff --check", ), + source_update_commands=( + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source splice-openapi"), + ), + source_update_paths=("config/mintlify-openapi/splice-openapi/source-artifacts.json",), ), UpdateTarget( key="wallet-gateway-openrpc", @@ -63,7 +168,6 @@ class UpdateTarget: "OpenRPC reference pages." ), generate_commands=( - ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source wallet-gateway-openrpc"), ("nix-shell", "--run", "npm run generate:wallet-gateway-openrpc-reference"), ), paths=( @@ -79,7 +183,225 @@ class UpdateTarget: "npm run generate:wallet-gateway-openrpc-reference", "git diff --check", ), + source_update_commands=( + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source wallet-gateway-openrpc"), + ), + source_update_paths=("config/x2mdx/wallet-gateway-openrpc/source-artifacts.json",), ), + UpdateTarget( + key="json-api-reference", + title="Update JSON Ledger API OpenAPI reference", + branch="generated-references/json-api-reference/update", + description=( + "Updates the JSON Ledger API OpenAPI source pin to the latest public " + "Canton release bundle for the published docs version and regenerates " + "the checked-in OpenAPI reference." + ), + generate_commands=( + ("nix-shell", "--run", "npm run generate:json-api-reference"), + ), + paths=( + "config/x2mdx/ledger-api/source-artifacts.json", + "docs-main/docs.json", + "docs-main/openapi/json-ledger-api", + "docs-main/reference/json-api-reference", + ), + summary_kind="versioned-source-config", + summary_path="config/x2mdx/ledger-api/source-artifacts.json", + summary_label="JSON Ledger API OpenAPI", + validation=( + "npm run update:generated-reference-sources -- --source ledger-api", + "npm run generate:json-api-reference", + "git diff --check", + ), + source_update_commands=( + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source ledger-api"), + ), + source_update_paths=("config/x2mdx/ledger-api/source-artifacts.json",), + ), + UpdateTarget( + key="json-api-asyncapi-reference", + title="Update JSON Ledger API AsyncAPI reference", + branch="generated-references/json-api-asyncapi-reference/update", + description=( + "Updates the JSON Ledger API AsyncAPI source pin to the latest public " + "Canton release bundle for the published docs version and regenerates " + "the checked-in AsyncAPI reference." + ), + generate_commands=( + ("nix-shell", "--run", "npm run generate:json-api-asyncapi-reference"), + ), + paths=( + "config/x2mdx/ledger-api-asyncapi/source-artifacts.json", + "docs-main/docs.json", + "docs-main/reference/json-api-asyncapi-reference", + ), + summary_kind="versioned-source-config", + summary_path="config/x2mdx/ledger-api-asyncapi/source-artifacts.json", + summary_label="JSON Ledger API AsyncAPI", + validation=( + "npm run update:generated-reference-sources -- --source ledger-api-asyncapi", + "npm run generate:json-api-asyncapi-reference", + "git diff --check", + ), + source_update_commands=( + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source ledger-api-asyncapi"), + ), + source_update_paths=("config/x2mdx/ledger-api-asyncapi/source-artifacts.json",), + ), + UpdateTarget( + key="grpc-ledger-api-reference", + title="Update gRPC Ledger API reference", + branch="generated-references/grpc-ledger-api-reference/update", + description=( + "Regenerates the checked-in gRPC Ledger API reference from the latest " + "stable Canton protobuf release bundles selected by the existing source config." + ), + generate_commands=(("nix-shell", "--run", "npm run generate:grpc-ledger-api-reference"),), + paths=( + "docs-main/docs.json", + "docs-main/reference/grpc-ledger-api-reference", + ), + summary_kind="source-config", + summary_path="config/x2mdx/grpc-ledger-api-reference/source-artifacts.json", + summary_label="gRPC Ledger API", + validation=( + "npm run generate:grpc-ledger-api-reference", + "git diff --check", + ), + ), + UpdateTarget( + key="canton-protobuf-history", + title="Update Canton protobuf history reference", + branch="generated-references/canton-protobuf-history/update", + description=( + "Regenerates the checked-in Canton protobuf history references from the " + "latest stable Canton protobuf release bundles selected by the existing source config." + ), + generate_commands=(("nix-shell", "--run", "npm run generate:canton-protobuf-history"),), + paths=( + "config/x2mdx/protobuf-history/metadata.json", + "docs-main/docs.json", + "docs-main/appdev/reference/protobuf-history", + "docs-main/reference/admin-api/protobuf", + "docs-main/reference/protobuf", + ), + summary_kind="source-config", + summary_path="config/x2mdx/protobuf-history/source-artifacts.json", + summary_label="Canton protobuf history", + validation=( + "npm run generate:canton-protobuf-history", + "git diff --check", + ), + ), + UpdateTarget( + key="ledger-bindings", + title="Update Java ledger bindings reference", + branch="generated-references/ledger-bindings/update", + description=( + "Updates the Java ledger bindings source pins to the latest stable " + "Maven artifacts and regenerates the checked-in Java bindings reference pages." + ), + generate_commands=( + ("nix-shell", "--run", "npm run generate:ledger-bindings-api-reference"), + ), + paths=( + "config/x2mdx/ledger-bindings/source-artifacts.json", + "docs-main/docs.json", + "docs-main/reference/java-bindings.mdx", + "docs-main/reference/java", + ), + summary_kind="artifact-source-config", + summary_path="config/x2mdx/ledger-bindings/source-artifacts.json", + summary_label="Java ledger bindings", + validation=( + "npm run update:generated-reference-sources -- --source ledger-bindings", + "npm run generate:ledger-bindings-api-reference", + "git diff --check", + ), + source_update_commands=( + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source ledger-bindings"), + ), + source_update_paths=("config/x2mdx/ledger-bindings/source-artifacts.json",), + ), + UpdateTarget( + key="daml-standard-library", + title="Update Daml Standard Library reference", + branch="generated-references/daml-standard-library/update", + description=( + "Updates the Daml Standard Library source pin to the latest DPM SDK " + "version and regenerates the checked-in Daml Standard Library reference pages." + ), + generate_commands=( + ("nix-shell", "--run", "npm run generate:daml-standard-library-reference"), + ), + paths=( + "config/x2mdx/daml-standard-library/source-artifacts.json", + "docs-main/docs.json", + "docs-main/appdev/reference/daml-standard-library", + ), + summary_kind="source-config", + summary_path="config/x2mdx/daml-standard-library/source-artifacts.json", + summary_label="Daml Standard Library", + validation=( + "npm run update:generated-reference-sources -- --source daml-standard-library", + "npm run generate:daml-standard-library-reference", + "git diff --check", + ), + source_update_commands=( + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source daml-standard-library"), + ), + source_update_paths=("config/x2mdx/daml-standard-library/source-artifacts.json",), + ), + UpdateTarget( + key="typescript-bindings", + title="Update TypeScript bindings reference", + branch="generated-references/typescript-bindings/update", + description=( + "Updates the TypeScript bindings source pins to the latest stable npm " + "releases and regenerates the checked-in TypeScript bindings reference pages." + ), + generate_commands=( + ("nix-shell", "--run", "npm run generate:typescript-bindings-reference"), + ), + paths=( + "config/x2mdx/typescript-bindings/source-artifacts.json", + "docs-main/docs.json", + "docs-main/reference/typescript.mdx", + "docs-main/reference/typescript", + ), + summary_kind="package-source-config", + summary_path="config/x2mdx/typescript-bindings/source-artifacts.json", + summary_label="TypeScript bindings", + validation=( + "npm run update:generated-reference-sources -- --source typescript-bindings", + "npm run generate:typescript-bindings-reference", + "git diff --check", + ), + source_update_commands=( + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source typescript-bindings"), + ), + source_update_paths=("config/x2mdx/typescript-bindings/source-artifacts.json",), + ), + UpdateTarget( + key="canton-metrics-reference", + title="Update Canton metrics reference", + branch="generated-docs/canton-metrics-reference/update", + description=( + "Regenerates the checked-in Canton Metrics reference page from the latest " + "Canton release documentation source." + ), + generate_commands=(("nix-shell", "--run", "npm run generate:canton-metrics-reference"),), + paths=("docs-main/global-synchronizer/reference/canton-metrics.mdx",), + summary_kind="static", + summary_path=None, + summary_label=None, + validation=( + "npm run generate:canton-metrics-reference", + "git diff --check", + ), + ), + *external_snippet_targets(), ) @@ -113,6 +435,8 @@ def body_markdown(*, target: UpdateTarget, changes: list[str]) -> str: def summarize_target_changes(target: UpdateTarget, before_path: Path) -> list[str]: + if target.summary_path is None: + return [] after_path = REPO_ROOT / target.summary_path if target.summary_kind == "dashboard": return summarize_version_changes.dashboard_changes(before_path, after_path) @@ -124,6 +448,40 @@ def summarize_target_changes(target: UpdateTarget, before_path: Path) -> list[st after_path, label=target.summary_label, ) + if target.summary_kind == "package-source-config": + if target.summary_label is None: + raise ValueError(f"Update target {target.key} must define summary_label") + return summarize_version_changes.package_source_config_changes( + before_path, + after_path, + label=target.summary_label, + ) + if target.summary_kind == "versioned-source-config": + if target.summary_label is None: + raise ValueError(f"Update target {target.key} must define summary_label") + return summarize_version_changes.versioned_source_config_changes( + before_path, + after_path, + label=target.summary_label, + ) + if target.summary_kind == "artifact-source-config": + if target.summary_label is None: + raise ValueError(f"Update target {target.key} must define summary_label") + return summarize_version_changes.artifact_source_config_changes( + before_path, + after_path, + label=target.summary_label, + ) + if target.summary_kind == "external-snippet-source": + if target.summary_label is None: + raise ValueError(f"Update target {target.key} must define summary_label") + return summarize_version_changes.external_snippet_source_changes( + after_path, + target_key=target.key.removeprefix("external-snippets-"), + label=target.summary_label, + ) + if target.summary_kind == "static": + return [] raise ValueError(f"Unknown summary kind for {target.key}: {target.summary_kind}") @@ -150,12 +508,25 @@ def create_or_update_pull_request( def process_target(*, target: UpdateTarget, base_sha: str, base_branch: str, repository: str) -> None: reset_to_base(base_sha) - before_path = pr_utils.write_base_file(base_sha, target.summary_path) + before_path = pr_utils.write_base_file(base_sha, target.summary_path) if target.summary_path is not None else None + + for command in target.source_update_commands: + pr_utils.run(command) + + if target.source_update_commands and not pr_utils.has_changes(target.source_update_paths): + print(f"Source unchanged for {target.title}; skipping generation") + pr_utils.close_stale_pull_request( + title=target.title, + branch=target.branch, + base_branch=base_branch, + repository=repository, + ) + return for command in target.generate_commands: pr_utils.run(command) - changes = summarize_target_changes(target, before_path) + changes = summarize_target_changes(target, before_path) if before_path is not None else [] with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as body_file: body_path = Path(body_file.name) body_path.write_text(body_markdown(target=target, changes=changes), encoding="utf-8") @@ -186,11 +557,20 @@ def parse_args() -> argparse.Namespace: "--repository", help="GitHub repository for generated PRs. Defaults to the current gh repository.", ) + parser.add_argument( + "--dry-run", + action="store_true", + help="List selected generated-doc targets and commands without changing files or opening PRs.", + ) args = parser.parse_args() if "all" in args.targets and len(args.targets) > 1: parser.error("pass --targets all by itself, or list specific target keys") - args.base_branch = args.base_branch or current_base_branch() - args.repository = args.repository or pr_utils.current_repository() + if args.dry_run: + args.base_branch = args.base_branch or "" + args.repository = args.repository or "" + else: + args.base_branch = args.base_branch or current_base_branch() + args.repository = args.repository or pr_utils.current_repository() return args @@ -207,11 +587,21 @@ def targets_to_run(target_keys: Sequence[str]) -> tuple[UpdateTarget, ...]: def main() -> int: args = parse_args() + selected_targets = targets_to_run(args.targets) + if args.dry_run: + for target in selected_targets: + print(f"{target.key}: {target.title}") + for command in target.source_update_commands: + print(" source $ " + " ".join(command)) + for command in target.generate_commands: + print(" generate $ " + " ".join(command)) + return 0 + pr_utils.git("config", "user.name", "github-actions[bot]") pr_utils.git("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") base_sha = pr_utils.git("rev-parse", "HEAD", capture=True) - for target in targets_to_run(args.targets): + for target in selected_targets: process_target( target=target, base_sha=base_sha, diff --git a/scripts/update_generated_reference_sources.py b/scripts/update_generated_reference_sources.py index 62ffbec6..0a9b495a 100644 --- a/scripts/update_generated_reference_sources.py +++ b/scripts/update_generated_reference_sources.py @@ -6,13 +6,38 @@ import sys from pathlib import Path -from generated_reference_sources import splice_openapi, wallet_gateway_openrpc +from generated_reference_sources import ( + canton_release_bundles, + daml_standard_library, + ledger_bindings, + splice_openapi, + typescript_bindings, + wallet_gateway_openrpc, +) from generated_reference_sources.common import SourceUpdate +REPO_ROOT = Path(__file__).resolve().parents[1] +SOURCE_DAML_STANDARD_LIBRARY = daml_standard_library.SOURCE_KEY +SOURCE_LEDGER_API = "ledger-api" +SOURCE_LEDGER_API_ASYNCAPI = "ledger-api-asyncapi" +SOURCE_LEDGER_BINDINGS = ledger_bindings.SOURCE_KEY SOURCE_SPLICE_OPENAPI = splice_openapi.SOURCE_KEY SOURCE_WALLET_GATEWAY_OPENRPC = wallet_gateway_openrpc.SOURCE_KEY -ALL_SOURCES = (SOURCE_SPLICE_OPENAPI, SOURCE_WALLET_GATEWAY_OPENRPC) +SOURCE_TYPESCRIPT_BINDINGS = typescript_bindings.SOURCE_KEY +DEFAULT_LEDGER_API_SOURCE_CONFIG = REPO_ROOT / "config" / "x2mdx" / "ledger-api" / "source-artifacts.json" +DEFAULT_LEDGER_API_ASYNCAPI_SOURCE_CONFIG = ( + REPO_ROOT / "config" / "x2mdx" / "ledger-api-asyncapi" / "source-artifacts.json" +) +ALL_SOURCES = ( + SOURCE_SPLICE_OPENAPI, + SOURCE_WALLET_GATEWAY_OPENRPC, + SOURCE_TYPESCRIPT_BINDINGS, + SOURCE_LEDGER_API, + SOURCE_LEDGER_API_ASYNCAPI, + SOURCE_LEDGER_BINDINGS, + SOURCE_DAML_STANDARD_LIBRARY, +) def parse_args() -> argparse.Namespace: @@ -34,6 +59,45 @@ def parse_args() -> argparse.Namespace: f"Default: {wallet_gateway_openrpc.DEFAULT_SOURCE_CONFIG}" ), ) + parser.add_argument( + "--typescript-bindings-source-config", + type=Path, + default=typescript_bindings.DEFAULT_SOURCE_CONFIG, + help=( + "TypeScript bindings source-artifacts config. " + f"Default: {typescript_bindings.DEFAULT_SOURCE_CONFIG}" + ), + ) + parser.add_argument( + "--ledger-api-source-config", + type=Path, + default=DEFAULT_LEDGER_API_SOURCE_CONFIG, + help=f"JSON Ledger API OpenAPI source-artifacts config. Default: {DEFAULT_LEDGER_API_SOURCE_CONFIG}", + ) + parser.add_argument( + "--ledger-api-asyncapi-source-config", + type=Path, + default=DEFAULT_LEDGER_API_ASYNCAPI_SOURCE_CONFIG, + help=( + "JSON Ledger API AsyncAPI source-artifacts config. " + f"Default: {DEFAULT_LEDGER_API_ASYNCAPI_SOURCE_CONFIG}" + ), + ) + parser.add_argument( + "--ledger-bindings-source-config", + type=Path, + default=ledger_bindings.DEFAULT_SOURCE_CONFIG, + help=f"Ledger bindings source-artifacts config. Default: {ledger_bindings.DEFAULT_SOURCE_CONFIG}", + ) + parser.add_argument( + "--daml-standard-library-source-config", + type=Path, + default=daml_standard_library.DEFAULT_SOURCE_CONFIG, + help=( + "Daml Standard Library source-artifacts config. " + f"Default: {daml_standard_library.DEFAULT_SOURCE_CONFIG}" + ), + ) parser.add_argument( "--source", action="append", @@ -79,6 +143,41 @@ def main() -> int: ) if update is not None: updates.append(update) + if SOURCE_TYPESCRIPT_BINDINGS in sources: + updates.extend( + typescript_bindings.update_source( + source_config_path=args.typescript_bindings_source_config.resolve(), + dry_run=args.dry_run or args.check, + ) + ) + if SOURCE_LEDGER_API in sources: + update = canton_release_bundles.update_source( + source_config_path=args.ledger_api_source_config.resolve(), + dry_run=args.dry_run or args.check, + ) + if update is not None: + updates.append(update) + if SOURCE_LEDGER_API_ASYNCAPI in sources: + update = canton_release_bundles.update_source( + source_config_path=args.ledger_api_asyncapi_source_config.resolve(), + dry_run=args.dry_run or args.check, + ) + if update is not None: + updates.append(update) + if SOURCE_LEDGER_BINDINGS in sources: + updates.extend( + ledger_bindings.update_source( + source_config_path=args.ledger_bindings_source_config.resolve(), + dry_run=args.dry_run or args.check, + ) + ) + if SOURCE_DAML_STANDARD_LIBRARY in sources: + update = daml_standard_library.update_source( + source_config_path=args.daml_standard_library_source_config.resolve(), + dry_run=args.dry_run or args.check, + ) + if update is not None: + updates.append(update) if not updates: print("Generated reference source pins are up to date.") diff --git a/shell.nix b/shell.nix index 31722043..f99bf5e7 100644 --- a/shell.nix +++ b/shell.nix @@ -45,7 +45,7 @@ pkgs.mkShell { ;; esac - if [ -f package.json ] && [ ! -d node_modules ]; then + if [ "''${SKIP_NPM_INSTALL:-}" != "1" ] && [ -f package.json ] && [ ! -d node_modules ]; then echo "Installing npm dependencies..." if [ -f package-lock.json ]; then npm ci diff --git a/tests/harness/generate_daml_standard_library_json.sh b/tests/harness/generate_daml_standard_library_json.sh index 00691a9e..2ad747f3 100644 --- a/tests/harness/generate_daml_standard_library_json.sh +++ b/tests/harness/generate_daml_standard_library_json.sh @@ -112,6 +112,10 @@ dpm_pkg_db_root() { printf '%s\n' "$DPM_HOME_DIR/cache/components/damlc/$SDK_VERSION/damlc-dist-dpm/resources/pkg-db_dir" } +dpm_damlc_bin() { + printf '%s\n' "$DPM_HOME_DIR/cache/components/damlc/$SDK_VERSION/damlc-dist-dpm/damlc" +} + ensure_daml_sdk() { local pkg_db_root pkg_db_root="$(daml_pkg_db_root)" @@ -164,7 +168,12 @@ configure_daml_source() { configure_dpm_source() { ensure_dpm_sdk PKG_DB_ROOT="$(dpm_pkg_db_root)" - DOCS_CMD=("dpm" "damlc" "docs") + DPM_DAMLC_BIN="$(dpm_damlc_bin)" + if [[ ! -x "$DPM_DAMLC_BIN" ]]; then + echo "DPM damlc binary not found: $DPM_DAMLC_BIN" >&2 + return 1 + fi + DOCS_CMD=("$DPM_DAMLC_BIN" "docs") } case "$SDK_SOURCE" in diff --git a/tests/test_canton_metrics_reference.py b/tests/test_canton_metrics_reference.py index 64a9ec1b..831f8148 100644 --- a/tests/test_canton_metrics_reference.py +++ b/tests/test_canton_metrics_reference.py @@ -14,6 +14,10 @@ class CantonMetricsReferenceTests(unittest.TestCase): + def test_defaults_use_public_canton_source(self) -> None: + self.assertEqual(generator.DEFAULT_RELEASE_REPO, "digital-asset/canton") + self.assertEqual(generator.DEFAULT_REMOTE, "https://github.com/digital-asset/canton.git") + def test_resolve_generated_includes(self) -> None: with TemporaryDirectory() as tmp: generated_dir = Path(tmp) @@ -52,6 +56,7 @@ def test_convert_resolved_metrics_rst_to_mdx(self) -> None: mdx = generator.convert_rst_to_mdx(rst, source_ref="v1.2.3") + self.assertIn('source="digital-asset/canton"', mdx) self.assertIn('ref="v1.2.3"', mdx) self.assertIn("# Metrics", mdx) self.assertIn("[relevant Prometheus documentation](https://prometheus.io/docs/tutorials/understanding_metric_types/)", mdx) @@ -64,6 +69,37 @@ def test_unresolved_generatedinclude_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "generatedinclude"): generator.convert_rst_to_mdx(".. generatedinclude:: metrics.inc\n", source_ref="v1.2.3") + def test_run_generation_unsets_ci_for_canton_docs_generator(self) -> None: + with TemporaryDirectory() as tmp: + canton_dir = Path(tmp) + (canton_dir / ".envrc").write_text("use nix\n", encoding="utf-8") + calls: list[tuple[list[str], Path | None]] = [] + original_run = generator.run + original_which = generator.shutil.which + try: + generator.run = lambda command, cwd=None, capture=False: calls.append((command, cwd)) or "" + generator.shutil.which = lambda name: "/usr/bin/direnv" if name == "direnv" else None + + generator.run_generation( + canton_dir=canton_dir, + command=["sbt", "docs-open / generateIncludes"], + skip_direnv=False, + ) + finally: + generator.run = original_run + generator.shutil.which = original_which + + self.assertEqual( + calls, + [ + (["direnv", "allow"], canton_dir), + ( + ["direnv", "exec", str(canton_dir), "env", "-u", "CI", "sbt", "docs-open / generateIncludes"], + canton_dir, + ), + ], + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_canton_protobuf_generator.py b/tests/test_canton_protobuf_generator.py index 23b7b279..c6670fdf 100644 --- a/tests/test_canton_protobuf_generator.py +++ b/tests/test_canton_protobuf_generator.py @@ -30,6 +30,33 @@ def write_mdx(self, root: Path, relative: str, title: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(f'---\ntitle: "{title}"\n---\n', encoding="utf-8") + def test_ensure_repo_updates_cached_origin_remote_before_fetch(self) -> None: + repo_dir = self.root / "cached.git" + repo_dir.mkdir() + calls: list[tuple[tuple[str, ...], Path | None]] = [] + original_run = generator.run + try: + generator.run = lambda args, cwd=None, capture=False: calls.append((tuple(args), cwd)) or "" + + generator.ensure_repo( + repo_dir, + remote="https://github.com/digital-asset/canton.git", + fetch=True, + ) + finally: + generator.run = original_run + + self.assertEqual( + calls, + [ + ( + ("git", "remote", "set-url", "origin", "https://github.com/digital-asset/canton.git"), + repo_dir, + ), + (("git", "fetch", "origin", "--tags", "--prune", "--force"), repo_dir), + ], + ) + def test_bundle_selection_maps_only_ledger_and_admin_api_inputs(self) -> None: protobuf_root = self.root / "protobuf" self.write_proto(protobuf_root / "ledger-api", "com/daml/ledger/api/v2/command_service.proto") diff --git a/tests/test_daml_standard_library_json_script.py b/tests/test_daml_standard_library_json_script.py new file mode 100644 index 00000000..9d13757a --- /dev/null +++ b/tests/test_daml_standard_library_json_script.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_dpm_source_uses_cached_damlc_binary(tmp_path: Path) -> None: + sdk_version = "1.2.3" + lf_target = "2.2" + dpm_home = tmp_path / "dpm" + pkg_db_root = dpm_home / "cache/components/damlc" / sdk_version / "damlc-dist-dpm/resources/pkg-db_dir" + target_root = pkg_db_root / lf_target + (target_root / "daml-prim/DA").mkdir(parents=True) + (target_root / f"daml-stdlib-{sdk_version}/DA").mkdir(parents=True) + (target_root / "daml-prim/DA/Prim.daml").write_text("module DA.Prim where\n", encoding="utf-8") + (target_root / f"daml-stdlib-{sdk_version}/DA/List.daml").write_text("module DA.List where\n", encoding="utf-8") + + log_path = tmp_path / "damlc.log" + damlc_bin = dpm_home / "cache/components/damlc" / sdk_version / "damlc-dist-dpm/damlc" + damlc_bin.write_text( + """#!/usr/bin/env bash +set -euo pipefail +printf '%s\\n' "$0 $*" >> "$FAKE_DAMLC_LOG" +output="" +package="" +while [[ $# -gt 0 ]]; do + case "$1" in + --output) + output="$2" + shift 2 + ;; + --package-name) + package="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +python3 - "$output" "$package" <<'PY' +import json +import sys +from pathlib import Path + +Path(sys.argv[1]).write_text(json.dumps([{"md_name": sys.argv[2]}]) + "\\n", encoding="utf-8") +PY +""", + encoding="utf-8", + ) + damlc_bin.chmod(0o755) + + path_bin = tmp_path / "bin" + path_bin.mkdir() + dpm_bin = path_bin / "dpm" + dpm_bin.write_text("#!/usr/bin/env bash\nexit 42\n", encoding="utf-8") + dpm_bin.chmod(0o755) + + output_json = tmp_path / "base.json" + env = os.environ.copy() + env.update( + { + "DPM_HOME": str(dpm_home), + "FAKE_DAMLC_LOG": str(log_path), + "PATH": f"{path_bin}{os.pathsep}{env['PATH']}", + } + ) + + subprocess.run( + [ + "bash", + str(REPO_ROOT / "scripts/generate_daml_standard_library_json.sh"), + "--output-json", + str(output_json), + "--sdk-version", + sdk_version, + "--lf-target", + lf_target, + "--sdk-source", + "dpm", + "--skip-install", + ], + check=True, + cwd=REPO_ROOT, + env=env, + ) + + calls = log_path.read_text(encoding="utf-8") + assert str(damlc_bin) in calls + assert "dpm damlc" not in calls + assert json.loads(output_json.read_text(encoding="utf-8")) == [{"md_name": "daml-stdlib"}, {"md_name": "daml-prim"}] diff --git a/tests/test_generate_external_snippet_target.py b/tests/test_generate_external_snippet_target.py new file mode 100644 index 00000000..0a98db89 --- /dev/null +++ b/tests/test_generate_external_snippet_target.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path +from types import ModuleType + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def load_script_module() -> ModuleType: + script_path = REPO_ROOT / "scripts" / "generate_external_snippet_target.py" + spec = importlib.util.spec_from_file_location(script_path.stem, script_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[script_path.stem] = module + spec.loader.exec_module(module) + return module + + +def test_load_sources_reads_external_snippet_manifest() -> None: + module = load_script_module() + + sources = module.load_sources(module.DEFAULT_CONFIG) + + assert [source.key for source in sources] == [ + "canton", + "cn-quickstart", + "daml", + "daml-shell", + "dpm", + "scribe", + "splice", + ] + assert sources[0].repository == "digital-asset/canton" + assert sources[0].requires_docker is True + assert sources[0].requires_heavy_runner is True + assert next(source for source in sources if source.key == "daml-shell").skip_if_unavailable is True + assert next(source for source in sources if source.key == "scribe").skip_if_unavailable is True + assert next(source for source in sources if source.key == "splice").preserve_paths == ("common",) + + +def test_generate_source_clones_checks_out_and_delegates_to_wrapper( + monkeypatch, tmp_path: Path +) -> None: + module = load_script_module() + source = module.ExternalSnippetSource( + key="example", + label="Example snippets", + repository="example/repo", + ref="main", + version="main", + repo_arg="example", + output_path="docs-main/snippets/external/example/main", + ) + calls: list[tuple[tuple[str, ...], Path, bool]] = [] + + def fake_run(command: list[str], *, cwd: Path, dry_run: bool = False) -> str: + calls.append((tuple(command), cwd, dry_run)) + if command[:2] == ["git", "clone"]: + checkout = Path(command[-1]) + (checkout / ".git").mkdir(parents=True) + return "" + + monkeypatch.setattr(module, "run", fake_run) + monkeypatch.setattr(module, "allow_direnv", lambda checkout, *, dry_run: None) + monkeypatch.setattr(module, "check_docker", lambda source, *, dry_run: None) + + module.generate_source(source, cache_dir=tmp_path, dry_run=False) + + checkout = tmp_path / "example" / "repo" + assert (("git", "clone", "https://github.com/example/repo.git", str(checkout)), module.REPO_ROOT, False) in calls + assert any(call[0][:5] == ("git", "-c", "gc.auto=0", "-c", "maintenance.auto=false") for call in calls) + assert any( + call[0][:3] == (sys.executable, str(module.REPO_ROOT / "scripts" / "generate_external_snippets.py"), "example") + and "--copy-output" in call[0] + and "--replace-output" in call[0] + for call in calls + ) + + +def test_generate_source_skips_optional_unavailable_source(monkeypatch, tmp_path: Path) -> None: + module = load_script_module() + source = module.ExternalSnippetSource( + key="internal", + label="Internal snippets", + repository="example/internal", + ref="main", + version="main", + repo_arg="internal", + output_path="docs-main/snippets/external/internal/main", + skip_if_unavailable=True, + ) + calls: list[tuple[str, ...]] = [] + + def fake_run(command: list[str], *, cwd: Path, dry_run: bool = False) -> str: + calls.append(tuple(command)) + if command[:2] == ["git", "ls-remote"]: + raise subprocess.CalledProcessError(returncode=128, cmd=command) + raise AssertionError(f"Unexpected command after unavailable source: {command}") + + monkeypatch.setattr(module, "run", fake_run) + + module.generate_source(source, cache_dir=tmp_path, dry_run=False) + + assert calls == [ + ( + "git", + "ls-remote", + "--exit-code", + "https://github.com/example/internal.git", + "main", + ) + ] + + +def test_generate_source_skips_heavy_runner_source_without_opt_in( + monkeypatch, tmp_path: Path +) -> None: + module = load_script_module() + source = module.ExternalSnippetSource( + key="heavy", + label="Heavy snippets", + repository="example/heavy", + ref="main", + version="main", + repo_arg="heavy", + output_path="docs-main/snippets/external/heavy/main", + requires_heavy_runner=True, + ) + + monkeypatch.delenv(module.HEAVY_RUNNER_ENV, raising=False) + monkeypatch.setattr( + module, + "run", + lambda command, *, cwd, dry_run=False: (_ for _ in ()).throw( + AssertionError(f"Unexpected command for skipped heavy source: {command}") + ), + ) + + module.generate_source(source, cache_dir=tmp_path, dry_run=False) + + +def test_generate_source_runs_heavy_runner_source_with_opt_in( + monkeypatch, tmp_path: Path +) -> None: + module = load_script_module() + source = module.ExternalSnippetSource( + key="heavy", + label="Heavy snippets", + repository="example/heavy", + ref="main", + version="main", + repo_arg="heavy", + output_path="docs-main/snippets/external/heavy/main", + requires_heavy_runner=True, + ) + calls: list[tuple[str, ...]] = [] + + def fake_run(command: list[str], *, cwd: Path, dry_run: bool = False) -> str: + calls.append(tuple(command)) + if command[:2] == ["git", "clone"]: + checkout = Path(command[-1]) + (checkout / ".git").mkdir(parents=True) + return "" + + monkeypatch.setenv(module.HEAVY_RUNNER_ENV, "1") + monkeypatch.setattr(module, "run", fake_run) + monkeypatch.setattr(module, "allow_direnv", lambda checkout, *, dry_run: None) + monkeypatch.setattr(module, "check_docker", lambda source, *, dry_run: None) + + module.generate_source(source, cache_dir=tmp_path, dry_run=False) + + assert any(call[:2] == ("git", "ls-remote") for call in calls) + + +def test_generate_source_preserves_configured_output_paths( + monkeypatch, tmp_path: Path +) -> None: + module = load_script_module() + fake_root = tmp_path / "cf-docs" + monkeypatch.setattr(module, "REPO_ROOT", fake_root) + target = fake_root / "docs-main" / "snippets" / "external" / "splice" / "main" + preserved = target / "common" / "kms-config-aws.mdx" + stale = target / "stale-generated.mdx" + preserved.parent.mkdir(parents=True) + preserved.write_text("authored wrapper", encoding="utf-8") + stale.write_text("stale", encoding="utf-8") + + source = module.ExternalSnippetSource( + key="splice", + label="Splice external snippets", + repository="canton-network/splice", + ref="main", + version="main", + repo_arg="splice", + output_path="docs-main/snippets/external/splice/main", + preserve_paths=("common",), + ) + calls: list[tuple[str, ...]] = [] + + def fake_run(command: list[str], *, cwd: Path, dry_run: bool = False) -> str: + calls.append(tuple(command)) + if command[:2] == ["git", "clone"]: + checkout = Path(command[-1]) + (checkout / ".git").mkdir(parents=True) + if ( + len(command) >= 3 + and command[1] == str(fake_root / "scripts" / "generate_external_snippets.py") + and command[2] == "splice" + ): + import shutil + + shutil.rmtree(target) + target.mkdir(parents=True) + (target / "fresh-generated.mdx").write_text("fresh", encoding="utf-8") + return "" + + monkeypatch.setattr(module, "run", fake_run) + monkeypatch.setattr(module, "allow_direnv", lambda checkout, *, dry_run: None) + monkeypatch.setattr(module, "check_docker", lambda source, *, dry_run: None) + + module.generate_source(source, cache_dir=tmp_path / "cache", dry_run=False) + + assert preserved.read_text(encoding="utf-8") == "authored wrapper" + assert (target / "fresh-generated.mdx").read_text(encoding="utf-8") == "fresh" + assert not stale.exists() + + +def test_generate_source_fails_required_unavailable_source(monkeypatch, tmp_path: Path) -> None: + module = load_script_module() + source = module.ExternalSnippetSource( + key="required", + label="Required snippets", + repository="example/required", + ref="main", + version="main", + repo_arg="required", + output_path="docs-main/snippets/external/required/main", + ) + + def fake_run(command: list[str], *, cwd: Path, dry_run: bool = False) -> str: + raise subprocess.CalledProcessError(returncode=128, cmd=command) + + monkeypatch.setattr(module, "run", fake_run) + + try: + module.generate_source(source, cache_dir=tmp_path, dry_run=False) + except module.SourceUnavailableError as error: + assert "Required snippets source example/required@main is not available" in str(error) + else: + raise AssertionError("Expected required unavailable source to fail") diff --git a/tests/test_generate_external_snippets.py b/tests/test_generate_external_snippets.py index 550e5b15..98b0894c 100644 --- a/tests/test_generate_external_snippets.py +++ b/tests/test_generate_external_snippets.py @@ -75,6 +75,37 @@ def test_copy_output_targets_docs_main_snippets( assert not (fake_root / "snippets").exists() +def test_copy_output_strips_generated_mdx_trailing_whitespace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source_dir = tmp_path / "cn-quickstart" + docs_output = source_dir / "docs-output" + docs_output.mkdir(parents=True) + (docs_output / "example.mdx").write_text( + "# Admin users \n```bash\ncurl example\t\n```\n", + encoding="utf-8", + ) + (docs_output / "raw.txt").write_text("keep trailing whitespace \n", encoding="utf-8") + fake_root = tmp_path / "cf-docs" + + monkeypatch.setattr(generator, "CF_DOCS_ROOT", fake_root) + + target = generator.copy_output( + generator.REPOS["cn-quickstart"], + source_dir, + version="main", + replace=False, + dry_run=False, + ) + + assert (target / "example.mdx").read_text(encoding="utf-8") == ( + "# Admin users\n```bash\ncurl example\n```\n" + ) + assert (target / "raw.txt").read_text(encoding="utf-8") == ( + "keep trailing whitespace \n" + ) + + def test_wrapper_copies_helper_runs_extraction_and_copies_output( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -135,3 +166,67 @@ def test_wrapper_copies_helper_runs_extraction_and_copies_output( "```text\nhello\n```" ) assert (target / "example.mdx").read_text(encoding="utf-8") == "```text\nhello\n```" + + +def test_canton_prepare_uses_runner_sized_sbt_heap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[tuple[list[str], dict[str, str] | None]] = [] + + def fake_run( + argv: list[str], + *, + cwd: Path, + dry_run: bool, + env: dict[str, str] | None = None, + timeout: int | None = None, + ) -> None: + calls.append((argv, env)) + + monkeypatch.setattr(generator, "run", fake_run) + monkeypatch.setattr(generator, "check_docker", lambda dry_run: None) + monkeypatch.delenv("SNIPPET_CANTON_SBT_OPTS", raising=False) + + generator.prepare_repo( + generator.REPOS["canton"], + tmp_path, + skip_prepare=False, + dry_run=False, + ) + + assert len(calls) == 1 + assert 'SBT_OPTS="-Xmx4G -Xms1G"' in calls[0][0][-1] + assert calls[0][1] is not None + assert calls[0][1]["SBT_OPTS"] == "-Xmx4G -Xms1G" + + +def test_canton_prepare_allows_sbt_heap_override( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[tuple[list[str], dict[str, str] | None]] = [] + + def fake_run( + argv: list[str], + *, + cwd: Path, + dry_run: bool, + env: dict[str, str] | None = None, + timeout: int | None = None, + ) -> None: + calls.append((argv, env)) + + monkeypatch.setattr(generator, "run", fake_run) + monkeypatch.setattr(generator, "check_docker", lambda dry_run: None) + monkeypatch.setenv("SNIPPET_CANTON_SBT_OPTS", "-Xmx6G -Xms1G") + + generator.prepare_repo( + generator.REPOS["canton"], + tmp_path, + skip_prepare=False, + dry_run=False, + ) + + assert len(calls) == 1 + assert 'SBT_OPTS="-Xmx6G -Xms1G"' in calls[0][0][-1] + assert calls[0][1] is not None + assert calls[0][1]["SBT_OPTS"] == "-Xmx6G -Xms1G" diff --git a/tests/test_grpc_ledger_api_nav.py b/tests/test_grpc_ledger_api_nav.py new file mode 100644 index 00000000..8dbe87c7 --- /dev/null +++ b/tests/test_grpc_ledger_api_nav.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import sys +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def load_script(name: str): + scripts_dir = str(REPO_ROOT / "scripts") + if scripts_dir not in sys.path: + sys.path.insert(0, scripts_dir) + spec = importlib.util.spec_from_file_location(name, REPO_ROOT / "scripts" / f"{name}.py") + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[name] = module + previous = os.environ.get("DIGITAL_ASSET_DOCS_DIRENV") + os.environ["DIGITAL_ASSET_DOCS_DIRENV"] = "1" + try: + spec.loader.exec_module(module) + finally: + if previous is None: + os.environ.pop("DIGITAL_ASSET_DOCS_DIRENV", None) + else: + os.environ["DIGITAL_ASSET_DOCS_DIRENV"] = previous + return module + + +def write_mdx(path: Path, title: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f'---\ntitle: "{title}"\n---\n', encoding="utf-8") + + +def test_grpc_cli_defaults_to_ledger_api_nav_group(monkeypatch: pytest.MonkeyPatch) -> None: + generator = load_script("generate_grpc_ledger_api_reference") + + monkeypatch.setattr(sys, "argv", ["generate_grpc_ledger_api_reference.py"]) + + assert generator.parse_args().nav_group == ["Ledger API"] + + +def test_grpc_nav_update_preserves_admin_api_grpc_group(tmp_path: Path) -> None: + generator = load_script("generate_grpc_ledger_api_reference") + docs_json = tmp_path / "docs-main" / "docs.json" + docs_json.parent.mkdir(parents=True) + docs_json.write_text( + json.dumps( + { + "navigation": { + "products": [ + { + "product": "API Reference", + "pages": [ + {"group": "Ledger API", "pages": []}, + { + "group": "Admin API", + "pages": [ + { + "group": "gRPC API", + "pages": [ + "reference/admin-api/protobuf/packages/com-digitalasset-canton-admin" + ], + } + ], + }, + ], + } + ] + } + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + output_dir = docs_json.parent / "reference" / "grpc-ledger-api-reference" + details_path = output_dir / "details.mdx" + package_path = output_dir / "com-daml-ledger-api-v2.mdx" + operation_path = output_dir / "com-daml-ledger-api-v2" / "commandservice" / "submitandwait.mdx" + write_mdx(details_path, "Details and history") + write_mdx(package_path, "com.daml.ledger.api.v2") + write_mdx(operation_path, "SubmitAndWait") + + generator.update_docs_navigation( + docs_json_path=docs_json, + dropdown_label="API Reference", + parent_groups=["Ledger API"], + insert_after_group=None, + details_path=details_path, + page_paths=[package_path, operation_path], + ) + + docs = json.loads(docs_json.read_text(encoding="utf-8")) + api_pages = docs["navigation"]["products"][0]["pages"] + ledger = next(item for item in api_pages if item["group"] == "Ledger API") + admin = next(item for item in api_pages if item["group"] == "Admin API") + + assert next(item for item in ledger["pages"] if item["group"] == "gRPC API") + assert admin["pages"] == [ + { + "group": "gRPC API", + "pages": ["reference/admin-api/protobuf/packages/com-digitalasset-canton-admin"], + } + ] diff --git a/tests/test_ledger_bindings_nav.py b/tests/test_ledger_bindings_nav.py index 0447be48..44a215c8 100644 --- a/tests/test_ledger_bindings_nav.py +++ b/tests/test_ledger_bindings_nav.py @@ -85,6 +85,117 @@ def test_java_bindings_nav_includes_details_and_history_page(tmp_path: Path) -> } +def test_java_bindings_nav_supports_product_navigation(tmp_path: Path) -> None: + generate_ledger_bindings_api_reference = load_script("generate_ledger_bindings_api_reference") + docs_json = tmp_path / "docs-main" / "docs.json" + docs_json.parent.mkdir(parents=True) + docs_json.write_text( + json.dumps( + { + "navigation": { + "products": [ + { + "product": "API Reference", + "pages": [{"group": "Ledger API", "pages": []}], + } + ] + } + } + ), + encoding="utf-8", + ) + publish_root = docs_json.parent / "reference" + overview_file = publish_root / "java-bindings.mdx" + write_mdx(overview_file, "Details and history") + write_mdx( + publish_root / "java" / "com-example" / "index.mdx", + "com.example", + "## Package `com.example`\n", + ) + write_mdx(publish_root / "java" / "com-example" / "Client.mdx", "Client") + + generate_ledger_bindings_api_reference.update_docs_navigation( + docs_json_path=docs_json, + dropdown_label="API Reference", + parent_groups=["Ledger API"], + group_label="Java Bindings", + overview_file=overview_file, + publish_root=publish_root, + ) + + docs = json.loads(docs_json.read_text(encoding="utf-8")) + assert docs["navigation"]["products"][0]["pages"] == [ + { + "group": "Ledger API", + "pages": [ + { + "group": "Java Bindings", + "pages": [ + { + "group": "Javadocs", + "pages": [{"group": "com.example", "pages": ["reference/java/com-example/Client"]}], + }, + "reference/java-bindings", + ], + } + ], + } + ] + + +def test_daml_standard_library_nav_supports_product_navigation(tmp_path: Path) -> None: + generate_daml_standard_library_reference = load_script("generate_daml_standard_library_reference") + docs_json = tmp_path / "docs-main" / "docs.json" + docs_json.parent.mkdir(parents=True) + docs_json.write_text( + json.dumps( + { + "navigation": { + "products": [ + { + "product": "API Reference", + "pages": [{"group": "Daml APIs", "pages": []}], + } + ] + } + } + ), + encoding="utf-8", + ) + output_dir = docs_json.parent / "appdev" / "reference" / "daml-standard-library" + write_mdx(output_dir / "index.mdx", "Daml Standard Library") + write_mdx(output_dir / "da-list.mdx", "DA.List") + + generate_daml_standard_library_reference.update_docs_navigation( + docs_json_path=docs_json, + dropdown_label="API Reference", + parent_groups=["Daml APIs"], + output_dir=output_dir, + ) + + docs = json.loads(docs_json.read_text(encoding="utf-8")) + assert docs["navigation"]["products"][0]["pages"] == [ + { + "group": "Daml APIs", + "pages": [ + { + "group": "Daml Standard Library", + "pages": [ + { + "group": "Modules", + "pages": ["appdev/reference/daml-standard-library/da-list"], + }, + "appdev/reference/daml-standard-library/index", + ], + } + ], + } + ] + assert (output_dir / "index.mdx").read_text(encoding="utf-8").startswith( + '---\ntitle: "Details and history"\n---' + ) + + def test_java_bindings_overview_is_published_as_details_and_history() -> None: generate_ledger_bindings_api_reference = load_script("generate_ledger_bindings_api_reference") diff --git a/tests/test_splice_mintlify_openapi.py b/tests/test_splice_mintlify_openapi.py index 350af9be..fe96e0e0 100644 --- a/tests/test_splice_mintlify_openapi.py +++ b/tests/test_splice_mintlify_openapi.py @@ -28,6 +28,18 @@ def write_json(path: Path, payload: object) -> None: path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") +def test_splice_openapi_release_requests_use_github_token(monkeypatch) -> None: + module = load_script_module("generate_splice_mintlify_openapi.py") + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + + assert module.request_headers("https://api.github.com/repos/example/project/releases") == { + "Accept": "application/vnd.github+json", + "User-Agent": module.USER_AGENT, + "Authorization": "Bearer test-token", + "X-GitHub-Api-Version": "2022-11-28", + } + + def test_splice_openapi_rewrites_scan_server_examples(tmp_path: Path) -> None: module = load_script_module("generate_splice_mintlify_openapi.py") spec_bytes = b"""openapi: 3.0.0 diff --git a/tests/test_summarize_version_changes.py b/tests/test_summarize_version_changes.py index 7920b15e..c7087019 100644 --- a/tests/test_summarize_version_changes.py +++ b/tests/test_summarize_version_changes.py @@ -87,3 +87,114 @@ def test_source_config_changes_summarizes_publish_version(tmp_path: Path) -> Non assert module.source_config_changes(before, after, label="Wallet Gateway OpenRPC") == [ "- Wallet Gateway OpenRPC publish_version: 0.25.0 -> 1.4.0" ] + + +def test_package_source_config_changes_summarizes_package_publish_versions(tmp_path: Path) -> None: + module = load_script_module() + before = tmp_path / "before.json" + after = tmp_path / "after.json" + write_json( + before, + { + "packages": [ + {"package_name": "@daml/types", "publish_version": "3.4.11"}, + {"package_name": "@canton-network/dapp-sdk", "publish_version": "1.1.0"}, + ] + }, + ) + write_json( + after, + { + "packages": [ + {"package_name": "@daml/types", "publish_version": "3.5.2"}, + {"package_name": "@canton-network/dapp-sdk", "publish_version": "1.2.0"}, + ] + }, + ) + + assert module.package_source_config_changes(before, after, label="TypeScript bindings") == [ + "- TypeScript bindings @daml/types publish_version: 3.4.11 -> 3.5.2", + "- TypeScript bindings @canton-network/dapp-sdk publish_version: 1.1.0 -> 1.2.0", + ] + + +def test_versioned_source_config_changes_summarizes_canton_release_versions(tmp_path: Path) -> None: + module = load_script_module() + before = tmp_path / "before.json" + after = tmp_path / "after.json" + write_json( + before, + { + "versions": [ + {"version": "3.4", "canton_version": "3.4.11"}, + {"version": "3.5", "canton_version": "3.5.0-snapshot.20260405.18555.0.vbee160e5"}, + ] + }, + ) + write_json( + after, + { + "versions": [ + {"version": "3.4", "canton_version": "3.4.11"}, + {"version": "3.5", "canton_version": "3.5.5"}, + ] + }, + ) + + assert module.versioned_source_config_changes(before, after, label="JSON Ledger API OpenAPI") == [ + "- JSON Ledger API OpenAPI 3.5 canton_version: " + "3.5.0-snapshot.20260405.18555.0.vbee160e5 -> 3.5.5" + ] + + +def test_artifact_source_config_changes_summarizes_added_versions(tmp_path: Path) -> None: + module = load_script_module() + before = tmp_path / "before.json" + after = tmp_path / "after.json" + write_json( + before, + { + "artifacts": [ + {"group": "com.daml", "artifact": "bindings-java", "versions": ["3.4.11"]}, + ] + }, + ) + write_json( + after, + { + "artifacts": [ + {"group": "com.daml", "artifact": "bindings-java", "versions": ["3.4.11", "3.5.5"]}, + ] + }, + ) + + assert module.artifact_source_config_changes(before, after, label="Java ledger bindings") == [ + "- Java ledger bindings com.daml:bindings-java versions: added 3.5.5" + ] + + +def test_external_snippet_source_changes_reports_configured_source(tmp_path: Path) -> None: + module = load_script_module() + config = tmp_path / "external-snippet-sources.json" + write_json( + config, + { + "sources": [ + { + "key": "splice", + "label": "Splice external snippets", + "repository": "canton-network/splice", + "ref": "main", + "version": "main", + } + ] + }, + ) + + assert module.external_snippet_source_changes( + config, + target_key="splice", + label="Splice external snippets", + ) == [ + "- Splice external snippets source: canton-network/splice@main -> output version main", + ] diff --git a/tests/test_update_generated_reference_prs.py b/tests/test_update_generated_reference_prs.py index 9561b28e..e50b6f41 100644 --- a/tests/test_update_generated_reference_prs.py +++ b/tests/test_update_generated_reference_prs.py @@ -29,6 +29,124 @@ def test_targets_to_run_accepts_all() -> None: assert module.targets_to_run(["all"]) == module.UPDATE_TARGETS +def test_update_targets_cover_all_generated_doc_surfaces() -> None: + module = load_script_module() + + assert [target.key for target in module.UPDATE_TARGETS] == [ + "version-dashboard", + "splice-openapi", + "wallet-gateway-openrpc", + "json-api-reference", + "json-api-asyncapi-reference", + "grpc-ledger-api-reference", + "canton-protobuf-history", + "ledger-bindings", + "daml-standard-library", + "typescript-bindings", + "canton-metrics-reference", + "external-snippets-canton", + "external-snippets-cn-quickstart", + "external-snippets-daml", + "external-snippets-daml-shell", + "external-snippets-dpm", + "external-snippets-scribe", + "external-snippets-splice", + ] + + +def test_dashboard_target_runs_network_variable_tabs_after_dashboard_data_generation() -> None: + module = load_script_module() + target = next(target for target in module.UPDATE_TARGETS if target.key == "version-dashboard") + + assert target.source_update_commands == ( + ("nix-shell", "--run", "npm run generate:version-compatibility-dashboard"), + ) + assert target.generate_commands == ( + ("nix-shell", "--run", "npm run generate:network-variable-tabs"), + ) + assert target.source_update_paths == ( + "config/repo-version-config.json", + "docs-main/snippets/generated/version-dashboard-data.mdx", + ) + assert target.paths == ( + "config/repo-version-config.json", + "docs-main/snippets/generated/version-dashboard-data.mdx", + *module.NETWORK_VARIABLE_TAB_PAGES, + ) + + +def test_source_update_targets_skip_generation_when_source_is_unchanged(monkeypatch) -> None: + module = load_script_module() + target = next(target for target in module.UPDATE_TARGETS if target.key == "wallet-gateway-openrpc") + calls: list[tuple[str, ...]] = [] + + monkeypatch.setattr(module, "reset_to_base", lambda base_sha: calls.append(("reset", base_sha))) + monkeypatch.setattr(module.pr_utils, "write_base_file", lambda base_sha, path: Path("/tmp/before.json")) + monkeypatch.setattr(module.pr_utils, "has_changes", lambda paths: False) + monkeypatch.setattr( + module.pr_utils, + "close_stale_pull_request", + lambda **kwargs: calls.append(("close", kwargs["branch"])), + ) + monkeypatch.setattr(module, "create_or_update_pull_request", lambda **kwargs: calls.append(("pr",))) + + def fake_run(command: tuple[str, ...]) -> None: + calls.append(command) + + monkeypatch.setattr(module.pr_utils, "run", fake_run) + + module.process_target( + target=target, + base_sha="base-sha", + base_branch="main", + repository="canton-network/cf-docs", + ) + + assert calls == [ + ("reset", "base-sha"), + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source wallet-gateway-openrpc"), + ("close", "generated-references/wallet-gateway-openrpc/update"), + ] + + +def test_source_update_targets_generate_when_source_changed(monkeypatch, tmp_path: Path) -> None: + module = load_script_module() + target = next(target for target in module.UPDATE_TARGETS if target.key == "wallet-gateway-openrpc") + calls: list[tuple[str, ...]] = [] + body_paths: list[Path] = [] + + monkeypatch.setattr(module, "reset_to_base", lambda base_sha: calls.append(("reset", base_sha))) + monkeypatch.setattr(module.pr_utils, "write_base_file", lambda base_sha, path: tmp_path / "before.json") + monkeypatch.setattr(module.pr_utils, "has_changes", lambda paths: True) + monkeypatch.setattr(module, "summarize_target_changes", lambda target, before_path: ["- changed"]) + + def fake_pr(**kwargs) -> None: + calls.append(("pr", kwargs["target"].key)) + body_paths.append(kwargs["body_path"]) + + monkeypatch.setattr(module, "create_or_update_pull_request", fake_pr) + + def fake_run(command: tuple[str, ...]) -> None: + calls.append(command) + + monkeypatch.setattr(module.pr_utils, "run", fake_run) + + module.process_target( + target=target, + base_sha="base-sha", + base_branch="main", + repository="canton-network/cf-docs", + ) + + assert calls == [ + ("reset", "base-sha"), + ("nix-shell", "--run", "npm run update:generated-reference-sources -- --source wallet-gateway-openrpc"), + ("nix-shell", "--run", "npm run generate:wallet-gateway-openrpc-reference"), + ("pr", "wallet-gateway-openrpc"), + ] + assert body_paths + + def test_targets_to_run_requires_at_least_one_target() -> None: module = load_script_module() @@ -65,8 +183,28 @@ def test_generated_clean_paths_include_target_paths_and_internal_output() -> Non clean_paths = module.generated_clean_paths() assert ".internal" in clean_paths + assert "docs-main/openapi/splice" in clean_paths + assert "docs-main/openapi/json-ledger-api" in clean_paths + assert "docs-main/reference/grpc-ledger-api-reference" in clean_paths + assert "docs-main/reference/java" in clean_paths + assert "docs-main/appdev/reference/daml-standard-library" in clean_paths assert "docs-main/reference/wallet-gateway-json-rpc" in clean_paths + assert "docs-main/reference/typescript" in clean_paths assert "docs-main/snippets/generated/version-dashboard-data.mdx" in clean_paths + assert "docs-main/global-synchronizer/deployment/validator-kubernetes.mdx" in clean_paths + assert "docs-main/global-synchronizer/reference/canton-metrics.mdx" in clean_paths + assert "docs-main/snippets/external/canton/main" in clean_paths + + +def test_target_paths_exist_in_base_checkout() -> None: + module = load_script_module() + + missing_paths = { + target.key: [path for path in target.paths if not (REPO_ROOT / path).exists()] + for target in module.UPDATE_TARGETS + } + + assert {key: paths for key, paths in missing_paths.items() if paths} == {} def test_body_markdown_includes_description_changes_and_validation() -> None: @@ -92,6 +230,68 @@ def test_body_markdown_notes_when_no_versions_changed() -> None: assert "Version changes:\n- No version values changed." in body +def test_summarize_target_changes_supports_versioned_source_configs(monkeypatch, tmp_path: Path) -> None: + module = load_script_module() + target = next(target for target in module.UPDATE_TARGETS if target.key == "json-api-reference") + before = tmp_path / "before.json" + before.write_text("{}", encoding="utf-8") + monkeypatch.setattr(module, "REPO_ROOT", tmp_path) + after = tmp_path / target.summary_path + after.parent.mkdir(parents=True) + after.write_text("{}", encoding="utf-8") + monkeypatch.setattr( + module.summarize_version_changes, + "versioned_source_config_changes", + lambda before_path, after_path, *, label: [f"{label}:{before_path.name}:{after_path.name}"], + ) + + assert module.summarize_target_changes(target, before) == [ + "JSON Ledger API OpenAPI:before.json:source-artifacts.json" + ] + + +def test_summarize_target_changes_supports_artifact_source_configs(monkeypatch, tmp_path: Path) -> None: + module = load_script_module() + target = next(target for target in module.UPDATE_TARGETS if target.key == "ledger-bindings") + before = tmp_path / "before.json" + before.write_text("{}", encoding="utf-8") + monkeypatch.setattr(module, "REPO_ROOT", tmp_path) + after = tmp_path / target.summary_path + after.parent.mkdir(parents=True) + after.write_text("{}", encoding="utf-8") + monkeypatch.setattr( + module.summarize_version_changes, + "artifact_source_config_changes", + lambda before_path, after_path, *, label: [f"{label}:{before_path.name}:{after_path.name}"], + ) + + assert module.summarize_target_changes(target, before) == [ + "Java ledger bindings:before.json:source-artifacts.json" + ] + + +def test_summarize_target_changes_supports_external_snippet_sources( + monkeypatch, tmp_path: Path +) -> None: + module = load_script_module() + target = next(target for target in module.UPDATE_TARGETS if target.key == "external-snippets-splice") + before = tmp_path / "before.json" + before.write_text("{}", encoding="utf-8") + monkeypatch.setattr(module, "REPO_ROOT", tmp_path) + after = tmp_path / target.summary_path + after.parent.mkdir(parents=True) + after.write_text('{"sources":[]}', encoding="utf-8") + monkeypatch.setattr( + module.summarize_version_changes, + "external_snippet_source_changes", + lambda config_path, *, target_key, label: [f"{label}:{target_key}:{config_path.name}"], + ) + + assert module.summarize_target_changes(target, before) == [ + "Splice external snippets:splice:external-snippet-sources.json" + ] + + def test_parse_args_defaults_base_branch_and_repository_from_local_context(monkeypatch) -> None: module = load_script_module() monkeypatch.setattr( @@ -140,6 +340,55 @@ def test_parse_args_accepts_explicit_base_branch_and_repository(monkeypatch) -> assert args.repository == "canton-network/cf-docs" +def test_parse_args_dry_run_does_not_require_repository_context(monkeypatch) -> None: + module = load_script_module() + monkeypatch.setattr( + sys, + "argv", + [ + "update_generated_reference_prs.py", + "--targets", + "all", + "--dry-run", + ], + ) + monkeypatch.setattr( + module.pr_utils, + "current_repository", + lambda: (_ for _ in ()).throw(AssertionError("repository should not be resolved")), + ) + + args = module.parse_args() + + assert args.dry_run is True + assert args.repository == "" + + +def test_main_dry_run_lists_targets_without_git_or_gh(monkeypatch, capsys) -> None: + module = load_script_module() + monkeypatch.setattr( + sys, + "argv", + [ + "update_generated_reference_prs.py", + "--targets", + "version-dashboard", + "--dry-run", + ], + ) + monkeypatch.setattr( + module.pr_utils, + "git", + lambda *args, capture=False: (_ for _ in ()).throw(AssertionError("git should not run")), + ) + + assert module.main() == 0 + output = capsys.readouterr().out + assert "version-dashboard: Update generated docs" in output + assert "source $ nix-shell --run npm run generate:version-compatibility-dashboard" in output + assert "npm run generate:network-variable-tabs" in output + + def test_current_base_branch_uses_github_ref_name_for_detached_checkout(monkeypatch) -> None: module = load_script_module() monkeypatch.setattr( @@ -185,4 +434,61 @@ def fake_gh(*args: str, capture: bool = False) -> str: ) assert ("commit", "--signoff", "-m", "Update generated docs") in git_calls + assert not any(call[:1] == ("switch",) for call in git_calls) assert any(call[:2] == ("pr", "create") for call in gh_calls) + + +def test_create_or_update_pull_request_closes_stale_pr_when_no_changes( + monkeypatch, tmp_path: Path +) -> None: + load_script_module() + import generated_reference_pr_utils as pr_utils + + gh_calls: list[tuple[str, ...]] = [] + + monkeypatch.setattr(pr_utils, "has_changes", lambda paths: False) + + def fake_gh(*args: str, capture: bool = False) -> str: + gh_calls.append(args) + if args[:2] == ("pr", "list"): + return "825" + return "" + + monkeypatch.setattr(pr_utils, "gh", fake_gh) + body_path = tmp_path / "body.md" + body_path.write_text("body", encoding="utf-8") + + pr_utils.create_or_update_pull_request( + title="Update Splice external snippets", + branch="generated-docs/external-snippets-splice/update", + paths=("docs-main/snippets/external/splice/main",), + body_path=body_path, + base_branch="remaining-generated-reference-pr-targets", + repository="canton-network/cf-docs", + ) + + assert any(call[:2] == ("pr", "close") and call[2] == "825" for call in gh_calls) + assert any("--delete-branch" in call for call in gh_calls) + + +def test_push_branch_uses_full_ref_for_detached_head(monkeypatch) -> None: + load_script_module() + import generated_reference_pr_utils as pr_utils + + git_calls: list[tuple[str, ...]] = [] + + def fake_git(*args: str, capture: bool = False) -> str: + git_calls.append(args) + if args[:3] == ("ls-remote", "--heads", "origin"): + return "" + return "" + + monkeypatch.setattr(pr_utils, "git", fake_git) + + pr_utils.push_branch("version-dashboard/update") + + assert ( + "push", + "origin", + "HEAD:refs/heads/version-dashboard/update", + ) in git_calls diff --git a/tests/test_update_generated_reference_sources.py b/tests/test_update_generated_reference_sources.py index 87b65536..0450e0b5 100644 --- a/tests/test_update_generated_reference_sources.py +++ b/tests/test_update_generated_reference_sources.py @@ -61,6 +61,99 @@ def write_wallet_gateway_source_config(path: Path, *, publish_version: str) -> N ) +def write_typescript_bindings_source_config( + path: Path, + *, + daml_types_version: str = "3.4.11", + wallet_sdk_version: str = "1.3.1", + dapp_sdk_version: str = "1.1.0", +) -> None: + path.write_text( + json.dumps( + { + "typedoc_version": "0.27.9", + "packages": [ + { + "package_name": "@daml/types", + "publish_version": daml_types_version, + "versions": ["3.4.11"], + }, + { + "package_name": "@canton-network/wallet-sdk", + "publish_version": wallet_sdk_version, + "versions": ["1.3.1"], + }, + { + "package_name": "@canton-network/dapp-sdk", + "publish_version": dapp_sdk_version, + "versions": ["1.1.0"], + }, + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + +def write_ledger_api_source_config(path: Path, *, canton_version: str) -> None: + path.write_text( + json.dumps( + { + "source": "test", + "release_url_template": "https://www.canton.io/releases/canton-open-source-{canton_version}.tar.gz", + "publish_version": "3.5", + "versions": [ + {"version": "3.4", "canton_version": "3.4.11"}, + {"version": "3.5", "canton_version": canton_version}, + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + +def write_ledger_bindings_source_config(path: Path, *, versions: list[str] | None = None) -> None: + path.write_text( + json.dumps( + { + "repo_base": "https://repo1.maven.org/maven2", + "artifacts": [ + { + "group": "com.daml", + "artifact": "bindings-java", + "language": "java", + "versions": versions or ["3.4.11"], + } + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + +def write_daml_standard_library_source_config(path: Path, *, publish_version: str) -> None: + path.write_text( + json.dumps( + { + "source": "test", + "publish_version": publish_version, + "package_set": "base", + "sdk_source": "dpm", + "versions": ["3.4.11"], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + def test_update_splice_openapi_source_updates_stale_publish_version(tmp_path: Path) -> None: module = load_script_module() source_config_path = tmp_path / "source-artifacts.json" @@ -193,10 +286,202 @@ def test_update_wallet_gateway_openrpc_source_dry_run_does_not_write(tmp_path: P assert json.loads(source_config_path.read_text(encoding="utf-8"))["publish_version"] == "0.25.0" +def test_update_typescript_bindings_source_updates_stale_package_versions(tmp_path: Path) -> None: + module = load_script_module() + source_config_path = tmp_path / "source-artifacts.json" + write_typescript_bindings_source_config(source_config_path) + latest_versions = { + "@daml/types": "3.5.2", + "@canton-network/wallet-sdk": "1.3.1", + "@canton-network/dapp-sdk": "1.2.0", + } + module.typescript_bindings.latest_npm_version = lambda package_name: latest_versions[package_name] + + updates = module.typescript_bindings.update_source( + source_config_path=source_config_path, + dry_run=False, + ) + + assert updates == [ + module.SourceUpdate( + source="TypeScript bindings @daml/types", + path=source_config_path, + field="publish_version", + previous="3.4.11", + current="3.5.2", + ), + module.SourceUpdate( + source="TypeScript bindings @canton-network/dapp-sdk", + path=source_config_path, + field="publish_version", + previous="1.1.0", + current="1.2.0", + ), + ] + packages = json.loads(source_config_path.read_text(encoding="utf-8"))["packages"] + assert packages[0]["publish_version"] == "3.5.2" + assert packages[0]["versions"] == ["3.4.11", "3.5.2"] + assert packages[1]["publish_version"] == "1.3.1" + assert packages[1]["versions"] == ["1.3.1"] + assert packages[2]["publish_version"] == "1.2.0" + assert packages[2]["versions"] == ["1.1.0", "1.2.0"] + + +def test_update_typescript_bindings_source_noops_when_current(tmp_path: Path) -> None: + module = load_script_module() + source_config_path = tmp_path / "source-artifacts.json" + write_typescript_bindings_source_config( + source_config_path, + daml_types_version="3.5.2", + wallet_sdk_version="1.3.1", + dapp_sdk_version="1.2.0", + ) + latest_versions = { + "@daml/types": "3.5.2", + "@canton-network/wallet-sdk": "1.3.1", + "@canton-network/dapp-sdk": "1.2.0", + } + module.typescript_bindings.latest_npm_version = lambda package_name: latest_versions[package_name] + + assert module.typescript_bindings.update_source( + source_config_path=source_config_path, + dry_run=False, + ) == [] + + +def test_update_typescript_bindings_source_dry_run_does_not_write(tmp_path: Path) -> None: + module = load_script_module() + source_config_path = tmp_path / "source-artifacts.json" + write_typescript_bindings_source_config(source_config_path) + module.typescript_bindings.latest_npm_version = lambda package_name: { + "@daml/types": "3.5.2", + "@canton-network/wallet-sdk": "1.3.1", + "@canton-network/dapp-sdk": "1.2.0", + }[package_name] + + updates = module.typescript_bindings.update_source( + source_config_path=source_config_path, + dry_run=True, + ) + + assert [update.current for update in updates] == ["3.5.2", "1.2.0"] + packages = json.loads(source_config_path.read_text(encoding="utf-8"))["packages"] + assert packages[0]["publish_version"] == "3.4.11" + assert packages[2]["publish_version"] == "1.1.0" + + +def test_update_ledger_api_source_updates_publish_version_canton_release(tmp_path: Path) -> None: + module = load_script_module() + assert module.canton_release_bundles.DEFAULT_CANTON_REMOTE == "https://github.com/digital-asset/canton.git" + source_config_path = tmp_path / "source-artifacts.json" + write_ledger_api_source_config(source_config_path, canton_version="3.5.0-snapshot.20260405.18555.0.vbee160e5") + module.canton_release_bundles.latest_public_canton_bundle_version = lambda *_args, **_kwargs: "3.5.5" + + update = module.canton_release_bundles.update_source( + source_config_path=source_config_path, + dry_run=False, + ) + + assert update == module.SourceUpdate( + source="JSON Ledger API release bundle", + path=source_config_path, + field="versions[3.5].canton_version", + previous="3.5.0-snapshot.20260405.18555.0.vbee160e5", + current="3.5.5", + ) + versions = json.loads(source_config_path.read_text(encoding="utf-8"))["versions"] + assert versions[1]["canton_version"] == "3.5.5" + + +def test_update_ledger_api_source_noops_when_current(tmp_path: Path) -> None: + module = load_script_module() + source_config_path = tmp_path / "source-artifacts.json" + write_ledger_api_source_config(source_config_path, canton_version="3.5.5") + module.canton_release_bundles.latest_public_canton_bundle_version = lambda *_args, **_kwargs: "3.5.5" + + assert ( + module.canton_release_bundles.update_source( + source_config_path=source_config_path, + dry_run=False, + ) + is None + ) + + +def test_update_ledger_bindings_source_appends_latest_maven_version(tmp_path: Path) -> None: + module = load_script_module() + source_config_path = tmp_path / "source-artifacts.json" + write_ledger_bindings_source_config(source_config_path) + module.ledger_bindings.latest_maven_version = lambda *_args, **_kwargs: "3.5.5" + + updates = module.ledger_bindings.update_source( + source_config_path=source_config_path, + dry_run=False, + ) + + assert updates == [ + module.SourceUpdate( + source="Java ledger bindings com.daml:bindings-java", + path=source_config_path, + field="versions", + previous="3.4.11", + current="3.5.5", + ) + ] + artifact = json.loads(source_config_path.read_text(encoding="utf-8"))["artifacts"][0] + assert artifact["versions"] == ["3.4.11", "3.5.5"] + + +def test_update_ledger_bindings_source_noops_when_latest_is_configured(tmp_path: Path) -> None: + module = load_script_module() + source_config_path = tmp_path / "source-artifacts.json" + write_ledger_bindings_source_config(source_config_path, versions=["3.4.11", "3.5.5"]) + module.ledger_bindings.latest_maven_version = lambda *_args, **_kwargs: "3.5.5" + + assert ( + module.ledger_bindings.update_source( + source_config_path=source_config_path, + dry_run=False, + ) + == [] + ) + + +def test_update_daml_standard_library_source_updates_latest_dpm_version(tmp_path: Path) -> None: + module = load_script_module() + source_config_path = tmp_path / "source-artifacts.json" + write_daml_standard_library_source_config(source_config_path, publish_version="3.4.11") + module.daml_standard_library.latest_dpm_version = lambda: "3.5.1" + + update = module.daml_standard_library.update_source( + source_config_path=source_config_path, + dry_run=False, + ) + + assert update == module.SourceUpdate( + source="Daml Standard Library", + path=source_config_path, + field="publish_version", + previous="3.4.11", + current="3.5.1", + ) + payload = json.loads(source_config_path.read_text(encoding="utf-8")) + assert payload["publish_version"] == "3.5.1" + assert payload["versions"] == ["3.4.11", "3.5.1"] + + def test_requested_sources_defaults_to_all_sources() -> None: module = load_script_module() - assert module.requested_sources(type("Args", (), {"sources": None})()) == module.ALL_SOURCES + assert module.requested_sources(type("Args", (), {"sources": None})()) == ( + "splice-openapi", + "wallet-gateway-openrpc", + "typescript-bindings", + "ledger-api", + "ledger-api-asyncapi", + "ledger-bindings", + "daml-standard-library", + ) def test_requested_sources_preserves_order_and_deduplicates() -> None: @@ -209,9 +494,10 @@ def test_requested_sources_preserves_order_and_deduplicates() -> None: { "sources": [ "wallet-gateway-openrpc", + "typescript-bindings", "splice-openapi", "wallet-gateway-openrpc", ] }, )() - ) == ("wallet-gateway-openrpc", "splice-openapi") + ) == ("wallet-gateway-openrpc", "typescript-bindings", "splice-openapi") diff --git a/tests/test_wallet_kernel_nav.py b/tests/test_wallet_kernel_nav.py index 579c1919..63a11b3c 100644 --- a/tests/test_wallet_kernel_nav.py +++ b/tests/test_wallet_kernel_nav.py @@ -157,6 +157,101 @@ def test_openrpc_nav_group_helper_omits_redundant_spec_page_child(tmp_path: Path } +def test_generated_reference_nav_replaces_group_in_product_navigation(tmp_path: Path) -> None: + generated_reference_nav = load_script("generated_reference_nav") + docs_json = tmp_path / "docs-main" / "docs.json" + docs_json.parent.mkdir(parents=True) + docs_json.write_text( + json.dumps( + { + "navigation": { + "products": [ + { + "product": "API Reference", + "pages": [ + {"group": "Old", "pages": ["old"]}, + {"group": "AsyncAPI", "pages": ["stale"]}, + ], + } + ] + } + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + generated_reference_nav.replace_group_in_dropdown( + docs_json_path=docs_json, + dropdown_label="API Reference", + group={"group": "AsyncAPI", "pages": ["fresh"]}, + ) + + docs = json.loads(docs_json.read_text(encoding="utf-8")) + assert docs["navigation"]["products"][0]["pages"] == [ + {"group": "Old", "pages": ["old"]}, + {"group": "AsyncAPI", "pages": ["fresh"]}, + ] + + +def test_asyncapi_wrapper_builds_legacy_dropdown_scratch_for_product_navigation() -> None: + generate_json_api_asyncapi_reference = load_script("generate_json_api_asyncapi_reference") + docs = { + "navigation": { + "products": [ + { + "product": "API Reference", + "pages": [{"group": "Ledger API", "pages": ["reference/json-api-reference"]}], + } + ] + } + } + + scratch = generate_json_api_asyncapi_reference.with_legacy_dropdown_scratch( + docs, + dropdown_label="API Reference", + ) + + assert scratch["navigation"]["dropdowns"] == [ + { + "dropdown": "API Reference", + "pages": [{"group": "Ledger API", "pages": ["reference/json-api-reference"]}], + } + ] + assert docs["navigation"].get("dropdowns") is None + + +def test_asyncapi_wrapper_places_x2mdx_scratch_docs_under_docs_root(tmp_path: Path) -> None: + generate_json_api_asyncapi_reference = load_script("generate_json_api_asyncapi_reference") + docs_json = tmp_path / "docs-main" / "docs.json" + docs_json.parent.mkdir(parents=True) + baseline_docs = { + "navigation": { + "products": [ + { + "product": "API Reference", + "pages": ["reference/json-api-asyncapi-reference/index"], + } + ] + } + } + + scratch_path = generate_json_api_asyncapi_reference.x2mdx_docs_json_path( + docs_json_path=docs_json, + baseline_docs=baseline_docs, + dropdown_label="API Reference", + ) + + assert scratch_path == docs_json.parent / ".docs-json-x2mdx-scratch.json" + assert json.loads(scratch_path.read_text(encoding="utf-8"))["navigation"]["dropdowns"] == [ + { + "dropdown": "API Reference", + "pages": ["reference/json-api-asyncapi-reference/index"], + } + ] + + def test_aggregate_generation_rejects_duplicate_wallet_gateway_aliases(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: generate_all_reference_docs = load_script("generate_all_reference_docs") docs_json = tmp_path / "docs-main" / "docs.json"