Skip to content

Commit 26fec8f

Browse files
mehmetozguldevmehmetozguldev
committed
Add automated grammar build pipeline and improve CI/CD
Add grammar-sources.json as the single source of truth mapping each language to its tree-sitter repository and subdirectory path. Add build-grammars.ts script that clones, generates, and compiles parser.wasm files for all or specific languages. Add build-grammars GitHub Actions workflow that triggers on extension changes, builds missing wasm files, and deploys them to CDN via rsync. Improve deploy workflow to use direct rsync instead of SSH shell commands, and add grammar-sources.json validation to the validate workflow. This replaces the manual SSH-and-build approach with a fully automated pipeline that OSS contributors can trigger via PR.
1 parent 7a8fbda commit 26fec8f

6 files changed

Lines changed: 451 additions & 25 deletions

File tree

‎.github/workflows/build-grammars.yml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Build & Deploy Grammars
2+
3+
on:
4+
push:
5+
branches: [master]
6+
paths:
7+
- "extensions/*/extension.json"
8+
- "extensions/*/highlights.scm"
9+
- "grammar-sources.json"
10+
workflow_dispatch:
11+
inputs:
12+
languages:
13+
description: "Comma-separated language IDs to rebuild (empty = all missing)"
14+
required: false
15+
type: string
16+
17+
jobs:
18+
build-grammars:
19+
if: github.repository == 'athasdev/extensions'
20+
runs-on: ubuntu-latest
21+
steps:
22+
- uses: actions/checkout@v4
23+
24+
- uses: oven-sh/setup-bun@v2
25+
with:
26+
bun-version: latest
27+
28+
- name: Install tree-sitter CLI
29+
run: npm install -g tree-sitter-cli
30+
31+
- name: Build parser.wasm files
32+
run: bun run scripts/build-grammars.ts ${{ inputs.languages && format('--languages {0}', inputs.languages) || '' }}
33+
34+
- name: Upload grammar artifacts
35+
uses: actions/upload-artifact@v4
36+
with:
37+
name: parser-wasm-files
38+
path: extensions/*/parser.wasm
39+
retention-days: 30
40+
41+
deploy-grammars:
42+
needs: build-grammars
43+
runs-on: ubuntu-latest
44+
steps:
45+
- uses: actions/checkout@v4
46+
47+
- name: Download grammar artifacts
48+
uses: actions/download-artifact@v4
49+
with:
50+
name: parser-wasm-files
51+
path: extensions/
52+
53+
- name: List built grammars
54+
run: find extensions -name "parser.wasm" -exec ls -lh {} \;
55+
56+
- name: Deploy to CDN via SSH
57+
uses: appleboy/ssh-action@v1.0.3
58+
with:
59+
host: ${{ secrets.VPS_HOST }}
60+
username: ${{ secrets.VPS_USER }}
61+
key: ${{ secrets.VPS_SSH_KEY }}
62+
port: ${{ secrets.VPS_PORT || 22 }}
63+
script: echo "CDN ready for rsync"
64+
65+
- name: Sync grammars to CDN
66+
uses: burnett01/rsync-deployments@7.0.2
67+
with:
68+
switches: -avz --include='*/' --include='parser.wasm' --include='*.json' --include='*.scm' --exclude='*'
69+
path: extensions/
70+
remote_path: ${{ secrets.EXTENSIONS_CDN_ROOT }}/
71+
remote_host: ${{ secrets.VPS_HOST }}
72+
remote_user: ${{ secrets.VPS_USER }}
73+
remote_key: ${{ secrets.VPS_SSH_KEY }}
74+
remote_port: ${{ secrets.VPS_PORT || 22 }}
75+
76+
- name: Sync catalog files to CDN
77+
uses: burnett01/rsync-deployments@7.0.2
78+
with:
79+
switches: -avz
80+
path: registry.json index.json manifests.json
81+
remote_path: ${{ secrets.EXTENSIONS_CDN_ROOT }}/
82+
remote_host: ${{ secrets.VPS_HOST }}
83+
remote_user: ${{ secrets.VPS_USER }}
84+
remote_key: ${{ secrets.VPS_SSH_KEY }}
85+
remote_port: ${{ secrets.VPS_PORT || 22 }}

‎.github/workflows/deploy.yml‎

Lines changed: 39 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -3,36 +3,50 @@ name: Deploy Extensions CDN
33
on:
44
release:
55
types: [published]
6+
push:
7+
branches: [master]
8+
paths:
9+
- "extensions/*/extension.json"
10+
- "extensions/*/highlights.scm"
11+
- "registry.json"
12+
- "index.json"
13+
- "manifests.json"
614
workflow_dispatch:
715

816
jobs:
917
deploy:
1018
if: github.repository == 'athasdev/extensions'
1119
runs-on: ubuntu-latest
1220
steps:
13-
- name: Deploy via SSH
14-
uses: appleboy/ssh-action@v1.0.3
21+
- uses: actions/checkout@v4
22+
23+
- uses: oven-sh/setup-bun@v2
24+
with:
25+
bun-version: latest
26+
27+
- name: Regenerate catalog files
28+
run: |
29+
bun scripts/build-extensions-index.ts
30+
bun scripts/generate-manifests.ts
31+
32+
- name: Sync extension configs to CDN
33+
uses: burnett01/rsync-deployments@7.0.2
34+
with:
35+
switches: -avz --include='*/' --include='*.json' --include='*.scm' --exclude='*'
36+
path: extensions/
37+
remote_path: ${{ secrets.EXTENSIONS_CDN_ROOT }}/
38+
remote_host: ${{ secrets.VPS_HOST }}
39+
remote_user: ${{ secrets.VPS_USER }}
40+
remote_key: ${{ secrets.VPS_SSH_KEY }}
41+
remote_port: ${{ secrets.VPS_PORT || 22 }}
42+
43+
- name: Sync catalog files to CDN
44+
uses: burnett01/rsync-deployments@7.0.2
1545
with:
16-
host: ${{ secrets.VPS_HOST }}
17-
username: ${{ secrets.VPS_USER }}
18-
key: ${{ secrets.VPS_SSH_KEY }}
19-
port: ${{ secrets.VPS_PORT || 22 }}
20-
envs: EXTENSIONS_CDN_ROOT
21-
script: |
22-
set -e
23-
if [ ! -d /srv/extensions/.git ]; then
24-
git clone --depth 1 https://github.com/athasdev/extensions.git /srv/extensions
25-
fi
26-
git config --global --add safe.directory /srv/extensions
27-
cd /srv/extensions
28-
git stash push --include-untracked --message "auto-deploy-$(date +%s)" || true
29-
git pull --ff-only origin master
30-
/root/.bun/bin/bun scripts/build-extensions-index.ts
31-
/root/.bun/bin/bun scripts/generate-manifests.ts
32-
if [ -n "${EXTENSIONS_CDN_ROOT:-}" ]; then
33-
/root/.bun/bin/bun scripts/deploy-extensions-cdn.ts
34-
else
35-
echo "EXTENSIONS_CDN_ROOT not set, skipping extensions CDN sync."
36-
fi
37-
env:
38-
EXTENSIONS_CDN_ROOT: ${{ secrets.EXTENSIONS_CDN_ROOT }}
46+
switches: -avz
47+
path: registry.json index.json manifests.json
48+
remote_path: ${{ secrets.EXTENSIONS_CDN_ROOT }}/
49+
remote_host: ${{ secrets.VPS_HOST }}
50+
remote_user: ${{ secrets.VPS_USER }}
51+
remote_key: ${{ secrets.VPS_SSH_KEY }}
52+
remote_port: ${{ secrets.VPS_PORT || 22 }}

‎.github/workflows/validate.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,28 @@ jobs:
2929
3030
- name: Check registry.json and index.json are up to date
3131
run: bun run scripts/build-extensions-index.ts --check
32+
33+
- name: Validate grammar-sources.json
34+
run: |
35+
node -e "
36+
const sources = require('./grammar-sources.json');
37+
const fs = require('fs');
38+
const dirs = fs.readdirSync('extensions', { withFileTypes: true })
39+
.filter(d => d.isDirectory()).map(d => d.name);
40+
let errors = 0;
41+
for (const dir of dirs) {
42+
const hasHighlights = fs.existsSync('extensions/' + dir + '/highlights.scm');
43+
const hasSource = !!sources[dir];
44+
if (hasHighlights && !hasSource) {
45+
console.warn('Warning: ' + dir + ' has highlights.scm but no entry in grammar-sources.json');
46+
}
47+
}
48+
for (const [lang, src] of Object.entries(sources)) {
49+
if (!fs.existsSync('extensions/' + lang)) {
50+
console.error('Error: grammar-sources.json references ' + lang + ' but extensions/' + lang + '/ does not exist');
51+
errors++;
52+
}
53+
}
54+
if (errors > 0) process.exit(1);
55+
console.log('grammar-sources.json: OK (' + Object.keys(sources).length + ' grammars)');
56+
"

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ node_modules/
88
# Build artifacts
99
dist/
1010
*.tgz
11+
.grammar-build/
1112

1213
# WASM binaries (stored on CDN, not in git)
1314
*.wasm

‎grammar-sources.json‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
{
2+
"bash": {
3+
"repository": "tree-sitter-grammars/tree-sitter-bash",
4+
"path": "."
5+
},
6+
"c": {
7+
"repository": "tree-sitter/tree-sitter-c",
8+
"path": "."
9+
},
10+
"c_sharp": {
11+
"repository": "tree-sitter/tree-sitter-c-sharp",
12+
"path": "."
13+
},
14+
"cpp": {
15+
"repository": "tree-sitter/tree-sitter-cpp",
16+
"path": "."
17+
},
18+
"css": {
19+
"repository": "tree-sitter/tree-sitter-css",
20+
"path": "."
21+
},
22+
"dart": {
23+
"repository": "UserNobworker/tree-sitter-dart",
24+
"path": "."
25+
},
26+
"dockerfile": {
27+
"repository": "camdencheek/tree-sitter-dockerfile",
28+
"path": "."
29+
},
30+
"elisp": {
31+
"repository": "Wilfred/tree-sitter-elisp",
32+
"path": "."
33+
},
34+
"elixir": {
35+
"repository": "elixir-lang/tree-sitter-elixir",
36+
"path": "."
37+
},
38+
"elm": {
39+
"repository": "elm-tooling/tree-sitter-elm",
40+
"path": "."
41+
},
42+
"go": {
43+
"repository": "tree-sitter/tree-sitter-go",
44+
"path": "."
45+
},
46+
"graphql": {
47+
"repository": "bkegley/tree-sitter-graphql",
48+
"path": "."
49+
},
50+
"html": {
51+
"repository": "tree-sitter/tree-sitter-html",
52+
"path": "."
53+
},
54+
"java": {
55+
"repository": "tree-sitter/tree-sitter-java",
56+
"path": "."
57+
},
58+
"javascript": {
59+
"repository": "tree-sitter/tree-sitter-javascript",
60+
"path": "."
61+
},
62+
"json": {
63+
"repository": "tree-sitter/tree-sitter-json",
64+
"path": "."
65+
},
66+
"kotlin": {
67+
"repository": "fwcd/tree-sitter-kotlin",
68+
"path": "."
69+
},
70+
"lua": {
71+
"repository": "tree-sitter-grammars/tree-sitter-lua",
72+
"path": "."
73+
},
74+
"objc": {
75+
"repository": "amaanq/tree-sitter-objc",
76+
"path": "."
77+
},
78+
"ocaml": {
79+
"repository": "tree-sitter/tree-sitter-ocaml",
80+
"path": "grammars/ocaml"
81+
},
82+
"php": {
83+
"repository": "tree-sitter/tree-sitter-php",
84+
"path": "php"
85+
},
86+
"protobuf": {
87+
"repository": "treywood/tree-sitter-proto",
88+
"path": "."
89+
},
90+
"python": {
91+
"repository": "tree-sitter/tree-sitter-python",
92+
"path": "."
93+
},
94+
"ruby": {
95+
"repository": "tree-sitter/tree-sitter-ruby",
96+
"path": "."
97+
},
98+
"rust": {
99+
"repository": "tree-sitter/tree-sitter-rust",
100+
"path": "."
101+
},
102+
"scala": {
103+
"repository": "tree-sitter/tree-sitter-scala",
104+
"path": "."
105+
},
106+
"sql": {
107+
"repository": "m-novikov/tree-sitter-sql",
108+
"path": "."
109+
},
110+
"svelte": {
111+
"repository": "tree-sitter-grammars/tree-sitter-svelte",
112+
"path": "."
113+
},
114+
"swift": {
115+
"repository": "alex-pinkus/tree-sitter-swift",
116+
"path": "."
117+
},
118+
"terraform": {
119+
"repository": "tree-sitter-grammars/tree-sitter-hcl",
120+
"path": "dialects/terraform"
121+
},
122+
"toml": {
123+
"repository": "tree-sitter-grammars/tree-sitter-toml",
124+
"path": "."
125+
},
126+
"tsx": {
127+
"repository": "tree-sitter/tree-sitter-typescript",
128+
"path": "tsx"
129+
},
130+
"typescript": {
131+
"repository": "tree-sitter/tree-sitter-typescript",
132+
"path": "typescript"
133+
},
134+
"vue": {
135+
"repository": "tree-sitter-grammars/tree-sitter-vue",
136+
"path": "."
137+
},
138+
"xml": {
139+
"repository": "tree-sitter-grammars/tree-sitter-xml",
140+
"path": "xml"
141+
},
142+
"yaml": {
143+
"repository": "tree-sitter-grammars/tree-sitter-yaml",
144+
"path": "."
145+
},
146+
"zig": {
147+
"repository": "tree-sitter-grammars/tree-sitter-zig",
148+
"path": "."
149+
}
150+
}

0 commit comments

Comments
 (0)