|
| 1 | +import { prerender } from '$app/server'; |
| 2 | +import { env } from '$env/dynamic/private'; |
| 3 | + |
| 4 | +const POPULAR_STAR_THRESHOLD = 100; |
| 5 | +const OTHER_STAR_THRESHOLD = 10; |
| 6 | + |
| 7 | +export type Dependent = { |
| 8 | + name?: string; |
| 9 | + reponame?: string; |
| 10 | + owner?: string; |
| 11 | + description: string; |
| 12 | + repourl?: string; |
| 13 | + homepageurl?: string; |
| 14 | + stars?: number; |
| 15 | +}; |
| 16 | + |
| 17 | +export const getDependents = prerender(async () => { |
| 18 | + |
| 19 | + const featuredSites: Dependent[] = [ |
| 20 | + { |
| 21 | + name: 'Github Analysis', |
| 22 | + description: 'Analyze your GitHub repositories and NPM packages', |
| 23 | + repourl: 'https://github.com/techniq/github-analysis', |
| 24 | + homepageurl: 'https://github.techniq.dev' |
| 25 | + }, |
| 26 | + { |
| 27 | + name: 'Strava Analysis', |
| 28 | + description: 'Analyze your Strava activities', |
| 29 | + repourl: 'https://github.com/techniq/strava-analysis', |
| 30 | + homepageurl: 'https://strava.techniq.dev' |
| 31 | + }, |
| 32 | + { |
| 33 | + name: 'Zipline AI', |
| 34 | + description: 'Features, context and embeddings for real-time AI/ML', |
| 35 | + repourl: 'https://zipline.ai/', |
| 36 | + homepageurl: 'https://github.com/zipline-ai' |
| 37 | + } |
| 38 | + ]; |
| 39 | + |
| 40 | + const supporterSites: Dependent[] = [ |
| 41 | + // could this be automated? pull sponsers, check dependents by owner === sponser, and add them here? |
| 42 | + { |
| 43 | + name: 'Tenzir', |
| 44 | + description: 'Open source data pipelines for security teams', |
| 45 | + repourl: 'https://github.com/tenzir', |
| 46 | + homepageurl: 'https://tenzir.com/' |
| 47 | + }, |
| 48 | + { |
| 49 | + name: 'shadcn-svelte', |
| 50 | + description: 'shadcn/ui, but for Svelte.', |
| 51 | + repourl: 'https://github.com/huntabyte/shadcn-svelte', |
| 52 | + homepageurl: 'https://shadcn-svelte.com/' |
| 53 | + }, |
| 54 | + { |
| 55 | + name: 'Sky Zoo', |
| 56 | + description: 'Bluesky stats', |
| 57 | + repourl: 'https://skyzoo.blue/', |
| 58 | + homepageurl: 'https://github.com/jycouet/jyc.dev' |
| 59 | + } |
| 60 | + ]; |
| 61 | + |
| 62 | + // These do not have a GH repo, but will be promoted by adding to the top of popular sites. |
| 63 | + const highlightedSites: Dependent[] = [ |
| 64 | + { |
| 65 | + name: 'GEO audit', |
| 66 | + description: 'GEO / AI audit that tracks your visibility impact', |
| 67 | + homepageurl: 'https://www.geoaud.it/' |
| 68 | + }, |
| 69 | + { |
| 70 | + name: 'RetireNumber', |
| 71 | + description: 'Get a second opinion on your retirement number.', |
| 72 | + homepageurl: 'https://retirenumber.com/' |
| 73 | + }, |
| 74 | + { |
| 75 | + name: 'PowerOutage.com', |
| 76 | + description: 'Tracks, records, and aggregates power outage data across the World', |
| 77 | + homepageurl: 'https://poweroutage.com/' |
| 78 | + }, |
| 79 | + { |
| 80 | + name: 'IOM UN Migration: Ukraine Regional Response', |
| 81 | + description: 'Needs, Intentions, and Border Crossings', |
| 82 | + homepageurl: |
| 83 | + 'https://dtm.iom.int/online-interactive-resources/ukraine-regional-response-dashboard/index.html' |
| 84 | + }, |
| 85 | + { |
| 86 | + name: 'Loyola Chicago: Center for Criminal Justice', |
| 87 | + description: 'The First Year of the Pretrial Fairness Act', |
| 88 | + homepageurl: 'https://pfa-1yr.loyolaccj.org/' |
| 89 | + }, |
| 90 | + { |
| 91 | + name: 'ftop', |
| 92 | + description: 'Comperative performance metrics for Fortnite islands', |
| 93 | + homepageurl: 'https://ftop.app/' |
| 94 | + }, |
| 95 | + { |
| 96 | + name: 'Nocturne', |
| 97 | + description: 'A next-generation platform for diabetes management', |
| 98 | + homepageurl: 'https://nocturne.app/' |
| 99 | + } |
| 100 | + ]; |
| 101 | + |
| 102 | + const githubHeaders: Record<string, string> = { |
| 103 | + Accept: 'application/vnd.github.v3+json', |
| 104 | + 'User-Agent': 'LayerChart docs' |
| 105 | + }; |
| 106 | + |
| 107 | + if (env.GITHUB_API_TOKEN) { |
| 108 | + const prefix = env.GITHUB_API_TOKEN.startsWith('ghp_') ? 'token' : 'Bearer'; |
| 109 | + githubHeaders['Authorization'] = `${prefix} ${env.GITHUB_API_TOKEN}`; |
| 110 | + } |
| 111 | + |
| 112 | + const totalStart = performance.now(); |
| 113 | + |
| 114 | + // Step 1: Find repos with "layerchart" in package.json via code search |
| 115 | + // NOTE: Code search API has a strict rate limit, so pages are fetched sequentially |
| 116 | + const repoSet = new Set<string>(); |
| 117 | + let page = 1; |
| 118 | + const perPage = 100; |
| 119 | + |
| 120 | + const step1Start = performance.now(); |
| 121 | + while (true) { |
| 122 | + const searchUrl = `https://api.github.com/search/code?q=${encodeURIComponent('"layerchart" filename:package.json')}&per_page=${perPage}&page=${page}`; |
| 123 | + const res = await fetch(searchUrl, { headers: githubHeaders }); |
| 124 | + |
| 125 | + if (!res.ok) { |
| 126 | + console.error(`GitHub code search failed: ${res.status} ${res.statusText}`); |
| 127 | + break; |
| 128 | + } |
| 129 | + |
| 130 | + const data = await res.json(); |
| 131 | + const items = data.items ?? []; |
| 132 | + |
| 133 | + for (const item of items) { |
| 134 | + const fullName = item.repository?.full_name; |
| 135 | + if (fullName && fullName !== 'techniq/layerchart') { |
| 136 | + repoSet.add(fullName); |
| 137 | + } |
| 138 | + } |
| 139 | + |
| 140 | + if (items.length < perPage || repoSet.size >= data.total_count) break; |
| 141 | + page++; |
| 142 | + } |
| 143 | + console.log( |
| 144 | + `[getDependents] Step 1 - Code search: ${((performance.now() - step1Start) / 1000).toFixed(2)}s (${repoSet.size} repos found, ${page} pages)` |
| 145 | + ); |
| 146 | + |
| 147 | + // Step 2: Batch-fetch repo details via GitHub GraphQL (parallel) |
| 148 | + const step2Start = performance.now(); |
| 149 | + const repos = [...repoSet]; |
| 150 | + const batchSize = 50; |
| 151 | + |
| 152 | + const batchPromises = []; |
| 153 | + for (let i = 0; i < repos.length; i += batchSize) { |
| 154 | + const batch = repos.slice(i, i + batchSize); |
| 155 | + const fragments = batch |
| 156 | + .map((fullName, idx) => { |
| 157 | + const [owner, name] = fullName.split('/'); |
| 158 | + return `repo${idx}: repository(owner: ${JSON.stringify(owner)}, name: ${JSON.stringify(name)}) { stargazerCount description homepageUrl url owner { login } name }`; |
| 159 | + }) |
| 160 | + .join('\n'); |
| 161 | + |
| 162 | + batchPromises.push( |
| 163 | + fetch('https://api.github.com/graphql', { |
| 164 | + method: 'POST', |
| 165 | + headers: githubHeaders, |
| 166 | + body: JSON.stringify({ query: `{ ${fragments} }` }) |
| 167 | + }).then(async (res) => { |
| 168 | + if (!res.ok) return []; |
| 169 | + const { data } = await res.json(); |
| 170 | + if (!data) return []; |
| 171 | + return Object.values(data) |
| 172 | + .filter(Boolean) |
| 173 | + .map((repo: any) => ({ |
| 174 | + owner: repo.owner.login, |
| 175 | + reponame: repo.name, |
| 176 | + description: repo.description || null, |
| 177 | + repourl: repo.url, |
| 178 | + homepageurl: repo.homepageUrl || null, |
| 179 | + stars: repo.stargazerCount |
| 180 | + })); |
| 181 | + }) |
| 182 | + ); |
| 183 | + } |
| 184 | + |
| 185 | + const batchResults = await Promise.all(batchPromises); |
| 186 | + const dependents: Dependent[] = batchResults.flat(); |
| 187 | + |
| 188 | + console.log( |
| 189 | + `[getDependents] Step 2 - GraphQL details: ${((performance.now() - step2Start) / 1000).toFixed(2)}s (${dependents.length} repos, ${batchPromises.length} batches)` |
| 190 | + ); |
| 191 | + console.log(`[getDependents] Total: ${((performance.now() - totalStart) / 1000).toFixed(2)}s`); |
| 192 | + |
| 193 | + dependents |
| 194 | + .sort((a, b) => (b.stars ?? 0) - (a.stars ?? 0)) // Sort by stars descending |
| 195 | + .filter((d) => featuredSites.some((f) => f.reponame === d.reponame)) // Filter out any featured sites |
| 196 | + .filter((d) => supporterSites.some((s) => s.reponame === d.reponame)); // Filter out any supporter sites |
| 197 | + const popularSites = [ |
| 198 | + ...highlightedSites, |
| 199 | + ...dependents.filter((d) => (d.stars ?? 0) >= POPULAR_STAR_THRESHOLD) |
| 200 | + ]; |
| 201 | + const otherSites = dependents.filter( |
| 202 | + (d) => (d.stars ?? 0) >= OTHER_STAR_THRESHOLD && (d.stars ?? 0) < POPULAR_STAR_THRESHOLD |
| 203 | + ); |
| 204 | + |
| 205 | + return { featuredSites, supporterSites, popularSites, otherSites }; |
| 206 | +}); |
0 commit comments