Skip to content

Commit 8829525

Browse files
jacksonlesterjacksonlester
committed
Replace 3 independent data update workflows with single sequential pipeline
1 parent a66a69c commit 8829525

6 files changed

Lines changed: 159 additions & 254 deletions

File tree

.github/workflows/update-cache.yml renamed to .github/workflows/deploy-data.yml

Lines changed: 142 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,116 +1,142 @@
1-
name: Update Data Cache
2-
3-
on:
4-
push:
5-
branches:
6-
- main
7-
- staging
8-
paths:
9-
- 'events.csv'
10-
- 'geometries/**'
11-
12-
workflow_dispatch:
13-
inputs:
14-
environment:
15-
description: 'Environment to update'
16-
required: true
17-
default: 'staging'
18-
type: choice
19-
options:
20-
- staging
21-
- production
22-
23-
jobs:
24-
update-cache:
25-
runs-on: ubuntu-latest
26-
27-
steps:
28-
- name: Checkout repository
29-
uses: actions/checkout@v4
30-
31-
- name: Setup Python
32-
uses: actions/setup-python@v5
33-
with:
34-
python-version: '3.11'
35-
36-
- name: Install Python dependencies
37-
run: pip install -r .dev/requirements-dev.txt
38-
39-
- name: Setup Node.js
40-
uses: actions/setup-node@v4
41-
with:
42-
node-version: '20'
43-
cache: 'npm'
44-
cache-dependency-path: '.dev/package-lock.json'
45-
46-
- name: Install Node dependencies
47-
run: cd .dev && npm ci
48-
49-
- name: Determine environment
50-
id: env
51-
run: |
52-
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
53-
echo "environment=${{ github.event.inputs.environment }}" >> $GITHUB_OUTPUT
54-
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
55-
echo "environment=production" >> $GITHUB_OUTPUT
56-
else
57-
echo "environment=staging" >> $GITHUB_OUTPUT
58-
fi
59-
60-
- name: Import CSV to database
61-
env:
62-
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
63-
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
64-
STAGING: ${{ steps.env.outputs.environment == 'staging' }}
65-
GITHUB_ACTIONS: true
66-
run: |
67-
if [ "${{ steps.env.outputs.environment }}" = "staging" ]; then
68-
STAGING=true python3 .dev/import-csv.py
69-
else
70-
python3 .dev/import-csv.py
71-
fi
72-
73-
- name: Upload geometry files to storage
74-
env:
75-
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
76-
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
77-
STAGING: ${{ steps.env.outputs.environment == 'staging' }}
78-
GITHUB_ACTIONS: true
79-
run: |
80-
if [ "${{ steps.env.outputs.environment }}" = "staging" ]; then
81-
STAGING=true node .dev/upload-geometries.js
82-
else
83-
node .dev/upload-geometries.js
84-
fi
85-
86-
- name: Sync geometries table with storage
87-
env:
88-
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
89-
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
90-
STAGING: ${{ steps.env.outputs.environment == 'staging' }}
91-
GITHUB_ACTIONS: true
92-
run: |
93-
if [ "${{ steps.env.outputs.environment }}" = "staging" ]; then
94-
STAGING=true node .dev/sync-geometries-table.js
95-
else
96-
node .dev/sync-geometries-table.js
97-
fi
98-
99-
- name: Rebuild cache
100-
env:
101-
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
102-
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
103-
STAGING: ${{ steps.env.outputs.environment == 'staging' }}
104-
GITHUB_ACTIONS: true
105-
run: |
106-
if [ "${{ steps.env.outputs.environment }}" = "staging" ]; then
107-
STAGING=true node .dev/rebuild-cache.js
108-
else
109-
node .dev/rebuild-cache.js
110-
fi
111-
112-
- name: Summary
113-
run: |
114-
echo "✅ Cache updated successfully for ${{ steps.env.outputs.environment }}"
115-
echo "Branch: ${{ github.ref_name }}"
116-
echo "Commit: ${{ github.sha }}"
1+
name: Validate & Update Cache
2+
3+
on:
4+
push:
5+
branches: [main, staging]
6+
paths:
7+
- 'events.csv'
8+
- 'geometries/**'
9+
10+
pull_request:
11+
branches: [main, staging]
12+
paths:
13+
- 'events.csv'
14+
- 'geometries/**'
15+
16+
workflow_dispatch:
17+
inputs:
18+
environment:
19+
description: 'Environment to update'
20+
required: true
21+
default: 'staging'
22+
type: choice
23+
options:
24+
- staging
25+
- production
26+
27+
jobs:
28+
# ── Job 1: Validate ──────────────────────────────────────────────
29+
# Runs pytest. If this fails, nothing else runs.
30+
validate:
31+
runs-on: ubuntu-latest
32+
steps:
33+
- uses: actions/checkout@v4
34+
35+
- name: Setup Python
36+
uses: actions/setup-python@v5
37+
with:
38+
python-version: '3.11'
39+
40+
- name: Install dependencies
41+
run: pip install -r .dev/requirements-dev.txt
42+
43+
- name: Run validation tests
44+
run: pytest tests/ -v
45+
46+
# ── Job 2: Update Supabase cache ─────────────────────────────────
47+
# Runs only after validation passes. Skipped on pull requests.
48+
update-cache:
49+
needs: validate
50+
if: github.event_name != 'pull_request'
51+
runs-on: ubuntu-latest
52+
53+
steps:
54+
- name: Checkout repository
55+
uses: actions/checkout@v4
56+
57+
- name: Setup Python
58+
uses: actions/setup-python@v5
59+
with:
60+
python-version: '3.11'
61+
62+
- name: Install Python dependencies
63+
run: pip install -r .dev/requirements-dev.txt
64+
65+
- name: Setup Node.js
66+
uses: actions/setup-node@v4
67+
with:
68+
node-version: '20'
69+
cache: 'npm'
70+
cache-dependency-path: '.dev/package-lock.json'
71+
72+
- name: Install Node dependencies
73+
run: cd .dev && npm ci
74+
75+
- name: Determine environment
76+
id: env
77+
run: |
78+
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
79+
echo "environment=${{ github.event.inputs.environment }}" >> $GITHUB_OUTPUT
80+
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
81+
echo "environment=production" >> $GITHUB_OUTPUT
82+
else
83+
echo "environment=staging" >> $GITHUB_OUTPUT
84+
fi
85+
86+
- name: Import CSV to database
87+
env:
88+
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
89+
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
90+
STAGING: ${{ steps.env.outputs.environment == 'staging' }}
91+
GITHUB_ACTIONS: true
92+
run: |
93+
if [ "${{ steps.env.outputs.environment }}" = "staging" ]; then
94+
STAGING=true python3 .dev/import-csv.py
95+
else
96+
python3 .dev/import-csv.py
97+
fi
98+
99+
- name: Upload geometry files to storage
100+
env:
101+
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
102+
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
103+
STAGING: ${{ steps.env.outputs.environment == 'staging' }}
104+
GITHUB_ACTIONS: true
105+
run: |
106+
if [ "${{ steps.env.outputs.environment }}" = "staging" ]; then
107+
STAGING=true node .dev/upload-geometries.js
108+
else
109+
node .dev/upload-geometries.js
110+
fi
111+
112+
- name: Sync geometries table with storage
113+
env:
114+
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
115+
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
116+
STAGING: ${{ steps.env.outputs.environment == 'staging' }}
117+
GITHUB_ACTIONS: true
118+
run: |
119+
if [ "${{ steps.env.outputs.environment }}" = "staging" ]; then
120+
STAGING=true node .dev/sync-geometries-table.js
121+
else
122+
node .dev/sync-geometries-table.js
123+
fi
124+
125+
- name: Rebuild cache
126+
env:
127+
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
128+
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
129+
STAGING: ${{ steps.env.outputs.environment == 'staging' }}
130+
GITHUB_ACTIONS: true
131+
run: |
132+
if [ "${{ steps.env.outputs.environment }}" = "staging" ]; then
133+
STAGING=true node .dev/rebuild-cache.js
134+
else
135+
node .dev/rebuild-cache.js
136+
fi
137+
138+
- name: Summary
139+
run: |
140+
echo "✅ Cache updated successfully for ${{ steps.env.outputs.environment }}"
141+
echo "Branch: ${{ github.ref_name }}"
142+
echo "Commit: ${{ github.sha }}"

‎.github/workflows/update-avmap.yml‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎.github/workflows/validate.yml‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎CONTRIBUTING.md‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -308,13 +308,7 @@ pip install -r requirements-dev.txt
308308
pytest tests/ -v
309309
```
310310

311-
Or use the wrapper script:
312-
313-
```bash
314-
python3 scripts/validate.py
315-
```
316-
317-
**Don't worry if you skip this** - pull requests run tests automatically via GitHub Actions, and we'll help fix any issues.
311+
**Don't worry if you skip this** - pull requests run validation automatically via GitHub Actions, and we'll help fix any issues.
318312

319313
## Schema changes (Advanced)
320314

0 commit comments

Comments
 (0)