Skip to content

Commit 3771f62

Browse files
LayoricLayoric
committed
Automate typesense deployment and indexing.
1 parent 77459fe commit 3771f62

9 files changed

Lines changed: 2908 additions & 2116 deletions

File tree

‎.github/workflows/search-index-update.yml‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
name: Build Search Index
2+
3+
on:
4+
schedule:
5+
# Run once a day
6+
- cron: '10 0 * * *'
7+
workflow_dispatch:
8+
inputs:
9+
noop:
10+
description: 'noop'
11+
required: false
12+
default: 'noop'
13+
14+
jobs:
15+
build:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
run_update: ${{ steps.date_check.outputs.run_update }}
19+
steps:
20+
- uses: actions/checkout@v1
21+
- name: Setup node
22+
uses: actions/setup-node@v1
23+
with:
24+
node-version: 14
25+
26+
- name: Check commit date
27+
id: date_check
28+
run: |
29+
chmod +x ./search-server/commit-date-check.sh
30+
echo "$(./search-server/commit-date-check.sh)"
31+
echo "run_update=$(./search-server/commit-date-check.sh)" >> $GITHUB_ENV
32+
33+
- name: GH script check
34+
uses: actions/github-script@v2
35+
id: check
36+
with:
37+
script: |
38+
const { owner, repo } = context.repo
39+
const update = ${{ env.run_update == 'update' }}
40+
if (!update) {
41+
console.log('Cancelling ...');
42+
const run_id = "${{ github.run_id }}";
43+
await github.actions.cancelWorkflowRun({ owner, repo, run_id });
44+
return 'stop'
45+
} else {
46+
return 'build'
47+
}
48+
result-encoding: string
49+
50+
- name: Waiting for cancellation
51+
run: sleep 60
52+
if: steps.check.outputs.result == 'stop'
53+
54+
- name: Create search index
55+
run: |
56+
echo "TYPESENSE_API_KEY=${{ secrets.TYPESENSE_API_KEY }}" >> $GITHUB_ENV
57+
export TYPESENSE_API_KEY=${{ secrets.TYPESENSE_API_KEY }}
58+
mkdir -p ${GITHUB_WORKSPACE}/typesense-data
59+
cp ./search-server/typesense-server/Dockerfile ${GITHUB_WORKSPACE}/typesense-data/Dockerfile
60+
cp ./search-server/typesense-scraper/typesense-scraper-config.json typesense-scraper-config.json
61+
envsubst < "./search-server/typesense-scraper/typesense-scraper.env" > "typesense-scraper-updated.env"
62+
docker run -d -p 8108:8108 -v ${GITHUB_WORKSPACE}/typesense-data/data:/data typesense/typesense:0.21.0 --data-dir /data --api-key=${TYPESENSE_API_KEY} --enable-cors &
63+
sleep 5
64+
docker run -i --env-file typesense-scraper-updated.env -e "CONFIG=$(cat typesense-scraper-config.json | jq -r tostring)" typesense/docsearch-scraper
65+
curl 'http://172.17.0.2:8108/keys' -X POST -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" -H 'Content-Type: application/json' -d '{"value": "N4N8bF0XwyvzwCGwm3CKB0QcnwyWtygo","description": "Search only","actions": ["documents:search"],"collections":["*"]}'
66+
67+
- name: repository name fix
68+
run: echo "image_repository_name=docs-search" >> $GITHUB_ENV
69+
70+
- name: Configure AWS credentials
71+
uses: aws-actions/configure-aws-credentials@v1
72+
with:
73+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
74+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
75+
aws-region: ${{ secrets.AWS_REGION }}
76+
77+
- name: Login to Amazon ECR
78+
id: login_ecr
79+
uses: aws-actions/amazon-ecr-login@v1
80+
81+
- name: Build and push to ECR
82+
id: push_image_to_ecr
83+
uses: docker/build-push-action@v2.2.2
84+
with:
85+
file: ./typesense-data/Dockerfile
86+
context: ./typesense-data/
87+
push: true
88+
tags: ${{ steps.login_ecr.outputs.registry }}/${{ env.image_repository_name }}:latest
89+
90+
deploy_ecs:
91+
needs: build
92+
runs-on: ubuntu-20.04
93+
steps:
94+
- name: checkout
95+
uses: actions/checkout@v2
96+
97+
- name: Configure AWS credentials
98+
uses: aws-actions/configure-aws-credentials@v1
99+
with:
100+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
101+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
102+
aws-region: ${{ secrets.AWS_REGION }}
103+
104+
- name: Login to Amazon ECR
105+
id: login_ecr
106+
uses: aws-actions/amazon-ecr-login@v1
107+
108+
- name: Repository name fix and env values setup
109+
run: |
110+
echo "image_repository_name=docs-search" >> $GITHUB_ENV
111+
echo "domain=search.reference.servicestack.net" >> $GITHUB_ENV
112+
echo "letsencrypt_email=${{ secrets.LETSENCRYPT_EMAIL }}" >> $GITHUB_ENV
113+
echo "app_name=servicestack-docs-search" >> $GITHUB_ENV
114+
echo "cluster_name=default" >> $GITHUB_ENV
115+
echo "image_url=${{ steps.login_ecr.outputs.registry }}/docs-search:latest" >> $GITHUB_ENV
116+
echo "aws_region=${{ secrets.AWS_REGION }}" >> $GITHUB_ENV
117+
echo "typesense_api_key=${{ secrets.TYPESENSE_API_KEY }}" >> $GITHUB_ENV
118+
119+
- name: Populate task definition template
120+
uses: danielr1996/envsubst-action@1.0.0
121+
env:
122+
RELEASE_VERSION: latest
123+
APP_NAME: ${{ env.app_name }}
124+
IMAGE_URL: ${{ env.image_url }}
125+
HOST_DOMAIN: ${{ env.domain }}
126+
TYPESENSE_API_KEY: ${{ env.typesense_api_key }}
127+
LETSENCRYPT_EMAIL: ${{ env.letsencrypt_email }}
128+
AWS_REGION: ${{ env.aws_region }}
129+
CLUSTER_NAME: ${{ env.cluster_name }}
130+
with:
131+
input: search-server/typesense-server/typesense-task-def.json
132+
output: task-definition.json
133+
134+
- name: Create task definition if doesn't exist
135+
run: aws ecs describe-task-definition --task-definition ${{ env.app_name }} || aws ecs register-task-definition --cli-input-json file://task-definition.json
136+
137+
- name: Create ECS Service if not exists.
138+
run: aws ecs describe-services --cluster ${{ env.cluster_name }} --services ${{ env.app_name }} | jq '.services[0]' -e || aws ecs create-service --cluster ${{ env.cluster_name }} --service-name ${{ env.app_name }} --task-definition ${{ env.app_name }} --desired-count 1
139+
140+
- name: Deploy new revision of the task definition
141+
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
142+
with:
143+
task-definition: task-definition.json
144+
service: ${{ env.app_name }}
145+
cluster: ${{ env.cluster_name }}
146+
force-new-deployment: true

‎docusaurus.config.js‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,14 @@ const config = {
3434
}),
3535
],
3636
],
37-
37+
themes: ['docusaurus-theme-search-typesense'],
3838
themeConfig:
3939
/** @type {import('@docusaurus/preset-classic').ThemeConfig} */
4040
({
4141
navbar: {
42-
title: 'ServiceStack.Blazor',
42+
title: 'ServiceStack Reference',
4343
logo: {
44-
alt: 'ServiceStack.Blazor',
44+
alt: 'ServiceStack Reference',
4545
src: 'img/servicestack.svg',
4646
},
4747
items: [
@@ -71,6 +71,26 @@ const config = {
7171
darkTheme: darkCodeTheme,
7272
additionalLanguages: ["csharp"],
7373
},
74+
typesense: {
75+
typesenseCollectionName: 'docusaurus-2', // Replace with your own doc site's name. Should match the collection name in the scraper settings.
76+
77+
typesenseServerConfig: {
78+
nodes: [
79+
{
80+
host: 'search.reference.servicestack.net',
81+
port: 443,
82+
protocol: 'https',
83+
}
84+
],
85+
apiKey: 'xyz',
86+
},
87+
88+
// Optional: Typesense search parameters: https://typesense.org/docs/0.21.0/api/search.md#search-parameters
89+
typesenseSearchParameters: {},
90+
91+
// Optional
92+
contextualSearch: true,
93+
}
7494
}),
7595
};
7696

‎package.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@
1515
"write-heading-ids": "docusaurus write-heading-ids"
1616
},
1717
"dependencies": {
18-
"@docusaurus/core": "2.0.0-beta.14",
19-
"@docusaurus/preset-classic": "2.0.0-beta.14",
20-
"@docusaurus/types": "^2.0.0-beta.14",
18+
"@docusaurus/core": "^2.1.0",
19+
"@docusaurus/preset-classic": "^2.1.0",
20+
"@docusaurus/theme-classic": "^2.1.0",
21+
"@docusaurus/types": "^2.1.0",
2122
"@mdx-js/react": "^1.6.21",
2223
"clsx": "^1.1.1",
24+
"docusaurus-theme-search-typesense": "^0.7.0-0",
2325
"prism-react-renderer": "^1.2.1",
2426
"react": "^17.0.1",
2527
"react-dom": "^17.0.1"

‎search-server/commit-date-check.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
lastcommitdate=$(git log -n1 --pretty='format:%cd' --date=format:'%Y%m%d')
2+
yesterdaydate=$(date +%Y%m%d -d "yesterday")
3+
if [ ${lastcommitdate} -ge ${yesterdaydate} ]; then
4+
echo "update"
5+
fi
6+
7+

‎search-server/typesense-scraper/typesense-scraper-config.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"index_name": "typesense_docs",
3+
"allowed_domains": ["172.17.0.2","reference.servicestack.net"],
4+
"start_urls": [
5+
{
6+
"url": "https://reference.servicestack.net/"
7+
}
8+
],
9+
"selectors": {
10+
"default": {
11+
"lvl0": ".page h1",
12+
"lvl1": ".content h2",
13+
"lvl2": ".content h3",
14+
"lvl3": ".content h4",
15+
"lvl4": ".content h5",
16+
"text": ".content p, .content ul li, .content table tbody tr"
17+
}
18+
},
19+
"scrape_start_urls": false,
20+
"strip_chars": " .,;:#"
21+
}

‎search-server/typesense-scraper/typesense-scraper.env‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
TYPESENSE_API_KEY=${TYPESENSE_API_KEY}
2+
TYPESENSE_HOST=172.17.0.2
3+
TYPESENSE_PORT=8108
4+
TYPESENSE_PROTOCOL=http

‎search-server/typesense-server/Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM typesense/typesense:0.23.1
2+
3+
COPY ./data /data

‎search-server/typesense-server/typesense-task-def.json‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
{
2+
"family": "${APP_NAME}",
3+
"requiresCompatibilities": [
4+
"EC2"
5+
],
6+
"networkMode": "bridge",
7+
"containerDefinitions": [
8+
{
9+
"portMappings": [
10+
{
11+
"protocol": "tcp",
12+
"containerPort": 8108
13+
}
14+
],
15+
"environment": [
16+
{
17+
"name": "VIRTUAL_HOST",
18+
"value": "${HOST_DOMAIN}"
19+
},
20+
{
21+
"name": "LETSENCRYPT_HOST",
22+
"value": "${HOST_DOMAIN}"
23+
},
24+
{
25+
"name": "LETSENCRYPT_EMAIL",
26+
"value": "${LETSENCRYPT_EMAIL}"
27+
}
28+
],
29+
"command": [
30+
"--data-dir",
31+
"/data",
32+
"--api-key",
33+
"${TYPESENSE_API_KEY}",
34+
"--enable-cors"
35+
],
36+
"mountPoints": [],
37+
"memoryReservation": 512,
38+
"image": "${IMAGE_URL}",
39+
"essential": true,
40+
"name": "${APP_NAME}",
41+
"logConfiguration": {
42+
"logDriver": "awslogs",
43+
"options": {
44+
"awslogs-group": "${CLUSTER_NAME}-${APP_NAME}",
45+
"awslogs-region": "${AWS_REGION}",
46+
"awslogs-create-group": "true"
47+
}
48+
}
49+
}
50+
]
51+
}

0 commit comments

Comments
 (0)