From 051b978aa7c16cb189b55ec267b85d11f7b276b6 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 12 Jun 2026 20:52:14 +0200 Subject: [PATCH 01/45] Add backend code --- apps/api/alchemy.run.ts | 34 +- apps/api/src/app.ts | 23 + apps/api/src/lib/Env.ts | 12 + apps/api/src/routes/vcs-webhook.http.ts | 62 + apps/api/src/services/OAuthStateRepository.ts | 67 + apps/api/src/services/OrganizationService.ts | 6 + .../src/services/github/GithubAppClient.ts | 308 ++ .../api/src/services/github/GithubProvider.ts | 292 ++ .../api/src/services/vcs/VcsProviderClient.ts | 48 + .../src/services/vcs/VcsProviderRegistry.ts | 44 + apps/api/src/services/vcs/VcsRepository.ts | 481 ++ apps/api/src/services/vcs/VcsSyncQueue.ts | 61 + apps/api/src/services/vcs/VcsSyncService.ts | 239 + .../src/services/vcs/__tests__/vcs.test.ts | 470 ++ apps/api/src/vcs-sync-runtime.ts | 73 + apps/api/src/worker.ts | 21 +- apps/api/test/stubs/cloudflare-workers.ts | 7 + apps/api/vitest.config.ts | 5 + apps/api/wrangler.jsonc | 16 + packages/db/drizzle/0022_lumpy_iron_lad.sql | 65 + packages/db/drizzle/meta/0022_snapshot.json | 4607 +++++++++++++++++ packages/db/drizzle/meta/_journal.json | 7 + packages/db/src/d1-limits.ts | 37 + packages/db/src/index.ts | 1 + packages/db/src/schema/index.ts | 1 + packages/db/src/schema/vcs.ts | 118 + packages/domain/src/http/index.ts | 1 + packages/domain/src/http/vcs.ts | 284 + 28 files changed, 7388 insertions(+), 2 deletions(-) create mode 100644 apps/api/src/routes/vcs-webhook.http.ts create mode 100644 apps/api/src/services/OAuthStateRepository.ts create mode 100644 apps/api/src/services/github/GithubAppClient.ts create mode 100644 apps/api/src/services/github/GithubProvider.ts create mode 100644 apps/api/src/services/vcs/VcsProviderClient.ts create mode 100644 apps/api/src/services/vcs/VcsProviderRegistry.ts create mode 100644 apps/api/src/services/vcs/VcsRepository.ts create mode 100644 apps/api/src/services/vcs/VcsSyncQueue.ts create mode 100644 apps/api/src/services/vcs/VcsSyncService.ts create mode 100644 apps/api/src/services/vcs/__tests__/vcs.test.ts create mode 100644 apps/api/src/vcs-sync-runtime.ts create mode 100644 apps/api/test/stubs/cloudflare-workers.ts create mode 100644 packages/db/drizzle/0022_lumpy_iron_lad.sql create mode 100644 packages/db/drizzle/meta/0022_snapshot.json create mode 100644 packages/db/src/d1-limits.ts create mode 100644 packages/db/src/schema/vcs.ts create mode 100644 packages/domain/src/http/vcs.ts diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 2b111cf6e..28cce1fbe 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -1,6 +1,6 @@ import path from "node:path" import alchemy from "alchemy" -import { D1Database, KVNamespace, Worker, Workflow } from "alchemy/cloudflare" +import { D1Database, KVNamespace, Queue, Worker, Workflow } from "alchemy/cloudflare" import type { MapleDomains, MapleStage } from "@maple/infra/cloudflare" import { resolveD1Name, resolveDeploymentEnvironment, resolveWorkerName } from "@maple/infra/cloudflare" @@ -62,6 +62,19 @@ export const createMapleApi = async ({ stage, domains }: CreateMapleApiOptions) className: "AiTriageWorkflow", }) + // Vendor-agnostic VCS sync queue (commit backfill + webhook deltas). The same + // `api` worker is both producer (binding) and consumer (eventSources). Local + // dev is wired separately in wrangler.jsonc so miniflare runs it in-process. + const vcsSyncDlq = await Queue("vcs-sync-dlq", { + name: resolveWorkerName("vcs-sync-dlq", stage), + adopt: true, + }) + const vcsSyncQueue = await Queue("vcs-sync", { + name: resolveWorkerName("vcs-sync", stage), + adopt: true, + dlq: vcsSyncDlq, + }) + const worker = await Worker("api", { name: resolveWorkerName("api", stage), cwd: import.meta.dirname, @@ -71,9 +84,22 @@ export const createMapleApi = async ({ stage, domains }: CreateMapleApiOptions) url: true, adopt: true, routes: domains.api ? [{ pattern: `${domains.api}/*`, adopt: true }] : undefined, + eventSources: [ + { + queue: vcsSyncQueue, + settings: { + batchSize: 10, + maxConcurrency: 2, + maxRetries: 3, + maxWaitTimeMs: 5000, + deadLetterQueue: vcsSyncDlq, + }, + }, + ], bindings: { MAPLE_DB: mapleDb, MCP_SESSIONS: mcpSessions, + VCS_SYNC_QUEUE: vcsSyncQueue, CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: schemaApplyWorkflow, AI_TRIAGE_WORKFLOW: aiTriageWorkflow, TINYBIRD_HOST: requireEnv("TINYBIRD_HOST"), @@ -112,6 +138,12 @@ export const createMapleApi = async ({ stage, domains }: CreateMapleApiOptions) ...optionalPlain("HAZEL_OAUTH_CLIENT_ID"), ...optionalSecret("HAZEL_OAUTH_CLIENT_SECRET"), ...optionalPlain("HAZEL_OAUTH_SCOPES"), + ...optionalPlain("GITHUB_APP_ID"), + ...optionalSecret("GITHUB_APP_PRIVATE_KEY"), + ...optionalPlain("GITHUB_APP_CLIENT_ID"), + ...optionalSecret("GITHUB_APP_CLIENT_SECRET"), + ...optionalSecret("GITHUB_APP_WEBHOOK_SECRET"), + ...optionalPlain("GITHUB_API_BASE_URL"), }, }) diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 85d3de504..135f516a5 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -25,6 +25,7 @@ import { HttpOrgClickHouseSettingsLive } from "./routes/org-clickhouse-settings. import { HttpOrganizationsLive } from "./routes/organizations.http" import { PrometheusScrapeProxyRouter } from "./routes/prometheus-scrape-proxy.http" import { ScraperInternalRouter } from "./routes/scraper-internal.http" +import { VcsWebhookRouter } from "./routes/vcs-webhook.http" import { HttpQueryEngineLive } from "./routes/query-engine.http" import { HttpRecommendationIssuesLive } from "./routes/recommendation-issues.http" import { HttpScrapeTargetsLive } from "./routes/scrape-targets.http" @@ -59,6 +60,12 @@ import { RawSqlChartService } from "@maple/query-engine/runtime" import { PlanetScaleDiscoveryService } from "./services/PlanetScaleDiscoveryService" import { ScrapeTargetsService } from "./services/ScrapeTargetsService" import { WarehouseQueryService } from "./lib/WarehouseQueryService" +import { OAuthStateRepository } from "./services/OAuthStateRepository" +import { GithubAppClient } from "./services/github/GithubAppClient" +import { GithubProvider } from "./services/github/GithubProvider" +import { VcsProviderRegistry } from "./services/vcs/VcsProviderRegistry" +import { VcsRepository } from "./services/vcs/VcsRepository" +import { VcsSyncQueue } from "./services/vcs/VcsSyncQueue" export const HealthRouter = HttpRouter.use((router) => router.add("GET", "/health", HttpServerResponse.text("OK")), @@ -152,6 +159,20 @@ export const DigestServiceLive = DigestService.layer.pipe( Layer.provideMerge(Layer.mergeAll(InfraLive, WarehouseQueryServiceLive, EmailServiceLive)), ) +// Vendor-agnostic VCS services for the fetch path: the webhook router needs the +// provider registry + sync queue; the repo + state repo are ready for the +// Step-2 settings endpoints. The sync orchestrator (VcsSyncService) lives only +// in the queue-consumer runtime (vcs-sync-runtime.ts), not here. Database / +// WorkerEnvironment are satisfied at worker scope (like CoreServicesLive). +const GithubProviderLive = GithubProvider.layer.pipe(Layer.provide(GithubAppClient.layer)) + +export const VcsServicesLive = Layer.mergeAll( + VcsRepository.layer, + OAuthStateRepository.layer, + VcsSyncQueue.layer, + VcsProviderRegistry.layer.pipe(Layer.provide(GithubProviderLive)), +).pipe(Layer.provideMerge(InfraLive)) + export const MainLive = Layer.mergeAll( CoreServicesLive, WarehouseQueryServiceLive, @@ -163,6 +184,7 @@ export const MainLive = Layer.mergeAll( RecommendationIssueServiceLive, DigestServiceLive, DemoServiceLive, + VcsServicesLive, RawSqlChartService.layer, ) @@ -203,6 +225,7 @@ export const AllRoutes = Layer.mergeAll( OAuthDiscoveryRouter, PrometheusScrapeProxyRouter, ScraperInternalRouter, + VcsWebhookRouter, McpLive, HealthRouter, McpGetFallback, diff --git a/apps/api/src/lib/Env.ts b/apps/api/src/lib/Env.ts index ad1b7ee12..34e1da2da 100644 --- a/apps/api/src/lib/Env.ts +++ b/apps/api/src/lib/Env.ts @@ -37,6 +37,12 @@ export interface EnvShape { readonly HAZEL_OAUTH_CLIENT_ID: Option.Option readonly HAZEL_OAUTH_CLIENT_SECRET: Option.Option> readonly HAZEL_OAUTH_SCOPES: string + readonly GITHUB_APP_ID: Option.Option + readonly GITHUB_APP_PRIVATE_KEY: Option.Option> + readonly GITHUB_APP_CLIENT_ID: Option.Option + readonly GITHUB_APP_CLIENT_SECRET: Option.Option> + readonly GITHUB_APP_WEBHOOK_SECRET: Option.Option> + readonly GITHUB_API_BASE_URL: string } const stringWithDefault = (key: string, fallback: string) => @@ -95,6 +101,12 @@ const envConfig = Config.all({ "HAZEL_OAUTH_SCOPES", "openid email profile organizations:read channels:read channel-webhooks:write", ), + GITHUB_APP_ID: optionalString("GITHUB_APP_ID"), + GITHUB_APP_PRIVATE_KEY: optionalRedacted("GITHUB_APP_PRIVATE_KEY"), + GITHUB_APP_CLIENT_ID: optionalString("GITHUB_APP_CLIENT_ID"), + GITHUB_APP_CLIENT_SECRET: optionalRedacted("GITHUB_APP_CLIENT_SECRET"), + GITHUB_APP_WEBHOOK_SECRET: optionalRedacted("GITHUB_APP_WEBHOOK_SECRET"), + GITHUB_API_BASE_URL: stringWithDefault("GITHUB_API_BASE_URL", "https://api.github.com"), }) const makeEnv = Effect.gen(function* () { diff --git a/apps/api/src/routes/vcs-webhook.http.ts b/apps/api/src/routes/vcs-webhook.http.ts new file mode 100644 index 000000000..cf5ed780c --- /dev/null +++ b/apps/api/src/routes/vcs-webhook.http.ts @@ -0,0 +1,62 @@ +import { HttpRouter, type HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { Effect, Option } from "effect" +import type { VcsProviderClient } from "../services/vcs/VcsProviderClient" +import { VcsProviderRegistry } from "../services/vcs/VcsProviderRegistry" +import { VcsSyncQueue } from "../services/vcs/VcsSyncQueue" + +// --------------------------------------------------------------------------- +// Public webhook receiver, one static route per registered provider +// (`/api/integrations//webhook`). Generic pipeline: the provider +// verifies the signature + maps the event to jobs; this router just enqueues +// and returns 202. NOT behind auth — authenticity comes from the provider's +// signature check. +// --------------------------------------------------------------------------- + +const textResponse = (body: string, status: number) => HttpServerResponse.text(body, { status }) + +export const VcsWebhookRouter = HttpRouter.use((router) => + Effect.gen(function* () { + const registry = yield* VcsProviderRegistry + const queue = yield* VcsSyncQueue + + const makeHandler = + (provider: VcsProviderClient) => (req: HttpServerRequest.HttpServerRequest) => + Effect.gen(function* () { + const bodyOpt = yield* req.text.pipe(Effect.option) + if (Option.isNone(bodyOpt)) return textResponse("Missing request body", 400) + const headers = req.headers as Record + + return yield* provider.webhookToJobs({ headers, rawBody: bodyOpt.value }).pipe( + Effect.flatMap((jobs) => + Effect.forEach(jobs, (job) => queue.send(job), { discard: true }).pipe( + Effect.as(textResponse("accepted", 202)), + ), + ), + Effect.catchTags({ + "@maple/http/errors/VcsWebhookSignatureError": (error) => + Effect.succeed(textResponse(error.message, 401)), + "@maple/http/errors/VcsWebhookParseError": (error) => + Effect.succeed(textResponse(error.message, 400)), + "@maple/http/errors/VcsQueueError": (error) => + Effect.logError("Failed to enqueue VCS webhook jobs") + .pipe(Effect.annotateLogs({ error: error.message })) + .pipe(Effect.as(textResponse("enqueue failed", 500))), + }), + ) + }).pipe(Effect.withSpan("VcsWebhook.receive", { attributes: { "vcs.provider": provider.id } })) + + yield* Effect.forEach( + registry.ids, + (id) => + registry + .resolve(id) + .pipe( + Effect.orDie, + Effect.flatMap((provider) => + router.add("POST", `/api/integrations/${id}/webhook`, makeHandler(provider)), + ), + ), + { discard: true }, + ) + }), +) diff --git a/apps/api/src/services/OAuthStateRepository.ts b/apps/api/src/services/OAuthStateRepository.ts new file mode 100644 index 000000000..550aef2af --- /dev/null +++ b/apps/api/src/services/OAuthStateRepository.ts @@ -0,0 +1,67 @@ +import { OAuthStatePersistenceError } from "@maple/domain/http" +import { oauthAuthStates, type OAuthAuthStateInsert, type OAuthAuthStateRow } from "@maple/db" +import { eq, lt } from "drizzle-orm" +import { Context, Effect, Layer, Option } from "effect" +import { Database, type DatabaseError } from "../lib/DatabaseLive" + +// --------------------------------------------------------------------------- +// Generic, provider-agnostic repo over the shared `oauth_auth_states` table — +// the short-lived CSRF nonce store for any OAuth / App-install redirect flow. +// Callers supply `provider` in the insert row and verify it on read, so this +// repo is reusable across integrations (GitHub install, Hazel OAuth, …). +// +// Extracted from the inline state CRUD in HazelOAuthService so the GitHub +// install service (Step 2) can depend on it and swap it out. +// --------------------------------------------------------------------------- + +const toPersistenceError = (error: DatabaseError) => + new OAuthStatePersistenceError({ message: error.message }) + +export interface OAuthStateRepositoryShape { + readonly purgeExpired: (now: number) => Effect.Effect + readonly insert: (row: OAuthAuthStateInsert) => Effect.Effect + readonly findByState: ( + state: string, + ) => Effect.Effect, OAuthStatePersistenceError> + readonly deleteByState: (state: string) => Effect.Effect +} + +export class OAuthStateRepository extends Context.Service()( + "@maple/api/services/OAuthStateRepository", + { + make: Effect.gen(function* () { + const database = yield* Database + + const purgeExpired = Effect.fn("OAuthStateRepository.purgeExpired")(function* (now: number) { + yield* database + .execute((db) => db.delete(oauthAuthStates).where(lt(oauthAuthStates.expiresAt, now))) + .pipe(Effect.mapError(toPersistenceError)) + }) + + const insert = Effect.fn("OAuthStateRepository.insert")(function* (row: OAuthAuthStateInsert) { + yield* database + .execute((db) => db.insert(oauthAuthStates).values(row)) + .pipe(Effect.mapError(toPersistenceError)) + }) + + const findByState = Effect.fn("OAuthStateRepository.findByState")(function* (state: string) { + const rows = yield* database + .execute((db) => + db.select().from(oauthAuthStates).where(eq(oauthAuthStates.state, state)).limit(1), + ) + .pipe(Effect.mapError(toPersistenceError)) + return Option.fromNullishOr(rows[0]) + }) + + const deleteByState = Effect.fn("OAuthStateRepository.deleteByState")(function* (state: string) { + yield* database + .execute((db) => db.delete(oauthAuthStates).where(eq(oauthAuthStates.state, state))) + .pipe(Effect.mapError(toPersistenceError)) + }) + + return { purgeExpired, insert, findByState, deleteByState } satisfies OAuthStateRepositoryShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/OrganizationService.ts b/apps/api/src/services/OrganizationService.ts index 71548d7d9..97300744b 100644 --- a/apps/api/src/services/OrganizationService.ts +++ b/apps/api/src/services/OrganizationService.ts @@ -30,6 +30,9 @@ import { orgIngestKeys, orgOpenrouterSettings, scrapeTargets, + vcsCommits, + vcsInstallations, + vcsRepositories, } from "@maple/db" import { eq } from "drizzle-orm" import { Context, Effect, Layer, Option, Redacted, Schema } from "effect" @@ -75,6 +78,9 @@ const ORG_SCOPED_TABLES = [ errorIssues, errorNotificationPolicies, actors, + vcsInstallations, + vcsRepositories, + vcsCommits, ] as const export interface OrganizationServiceShape { diff --git a/apps/api/src/services/github/GithubAppClient.ts b/apps/api/src/services/github/GithubAppClient.ts new file mode 100644 index 000000000..a6a7c68d3 --- /dev/null +++ b/apps/api/src/services/github/GithubAppClient.ts @@ -0,0 +1,308 @@ +import { GitCommitSha } from "@maple/domain/http" +import { Clock, Context, Data, Effect, Layer, Option, Redacted, Schema } from "effect" +import { Env } from "../../lib/Env" + +// --------------------------------------------------------------------------- +// GitHub App REST client. Vendor-specific: mints a short-lived App JWT (RS256, +// Web Crypto), exchanges it for per-installation tokens, and calls the GitHub +// REST API. No Octokit (Worker bundle weight). This module never touches D1. +// +// `GithubAppError` is internal to the GitHub layer; `GithubProvider` maps it to +// the generic `VcsProviderError` at the port boundary. +// --------------------------------------------------------------------------- + +export class GithubAppError extends Data.TaggedError("GithubAppError")<{ + message: string + status?: number + cause?: unknown +}> {} + +const GITHUB_API_VERSION = "2022-11-28" +const USER_AGENT = "maple-vcs-integration" +const PER_PAGE = 100 +const MAX_PAGES = 20 // safety cap (≤2000 items) + +// ---- REST response schemas ------------------------------------------------ + +const GithubInstallationTokenResponse = Schema.Struct({ + token: Schema.String, + expires_at: Schema.String, +}) + +const GithubApiRepoSchema = Schema.Struct({ + id: Schema.Number, + name: Schema.String, + full_name: Schema.String, + private: Schema.Boolean, + archived: Schema.optionalKey(Schema.Boolean), + default_branch: Schema.optionalKey(Schema.String), + html_url: Schema.String, + owner: Schema.Struct({ login: Schema.String }), +}) +export type GithubApiRepo = Schema.Schema.Type + +const GithubInstallationReposResponse = Schema.Struct({ + total_count: Schema.Number, + repositories: Schema.Array(GithubApiRepoSchema), +}) + +const GithubApiCommitAuthor = Schema.Struct({ + name: Schema.optionalKey(Schema.NullOr(Schema.String)), + email: Schema.optionalKey(Schema.NullOr(Schema.String)), + date: Schema.optionalKey(Schema.NullOr(Schema.String)), +}) + +const GithubApiUser = Schema.Struct({ + login: Schema.String, + avatar_url: Schema.optionalKey(Schema.String), +}) + +const GithubApiCommitSchema = Schema.Struct({ + sha: GitCommitSha, // validated at decode — the 40-hex shape lives in the brand + html_url: Schema.String, + commit: Schema.Struct({ + message: Schema.String, + author: Schema.NullOr(GithubApiCommitAuthor), + committer: Schema.optionalKey(Schema.NullOr(GithubApiCommitAuthor)), + }), + author: Schema.NullOr(GithubApiUser), +}) +export type GithubApiCommit = Schema.Schema.Type + +const GithubApiCommitList = Schema.Array(GithubApiCommitSchema) + +const decodeInstallationToken = Schema.decodeUnknownEffect(GithubInstallationTokenResponse) +const decodeInstallationRepos = Schema.decodeUnknownEffect(GithubInstallationReposResponse) +const decodeCommitList = Schema.decodeUnknownEffect(GithubApiCommitList) +const decodeCommit = Schema.decodeUnknownEffect(GithubApiCommitSchema) + +// ---- JWT (RS256 via Web Crypto) ------------------------------------------- + +const base64UrlString = (value: string) => Buffer.from(value, "utf8").toString("base64url") +const base64UrlBytes = (value: ArrayBuffer) => Buffer.from(value).toString("base64url") + +const pemToPkcs8 = (pem: string): ArrayBuffer => { + const body = pem + .replace(/-----BEGIN[^-]+-----/g, "") + .replace(/-----END[^-]+-----/g, "") + .replace(/\s+/g, "") + const buf = Buffer.from(body, "base64") + return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) +} + +interface ResolvedAppConfig { + readonly appId: string + readonly privateKeyPem: string + readonly apiBaseUrl: string +} + +export class GithubAppClient extends Context.Service()( + "@maple/api/services/github/GithubAppClient", + { + make: Effect.gen(function* () { + const env = yield* Env + + const resolveConfig: Effect.Effect = Effect.gen(function* () { + const appId = Option.getOrUndefined(env.GITHUB_APP_ID) + const privateKey = Option.getOrUndefined(env.GITHUB_APP_PRIVATE_KEY) + if (!appId || !privateKey) { + return yield* new GithubAppError({ + message: "GitHub App is not configured (set GITHUB_APP_ID and GITHUB_APP_PRIVATE_KEY)", + }) + } + return { + appId, + privateKeyPem: Redacted.value(privateKey), + apiBaseUrl: env.GITHUB_API_BASE_URL.replace(/\/+$/, ""), + } + }) + + const importSigningKey = (pem: string) => + Effect.tryPromise({ + try: () => + crypto.subtle.importKey( + "pkcs8", + pemToPkcs8(pem), + { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, + false, + ["sign"], + ), + catch: (cause) => + new GithubAppError({ message: "Failed to import GitHub App private key", cause }), + }) + + const mintAppJwt = Effect.fn("GithubAppClient.mintAppJwt")(function* (config: ResolvedAppConfig) { + const nowSec = Math.floor((yield* Clock.currentTimeMillis) / 1000) + const header = base64UrlString(JSON.stringify({ alg: "RS256", typ: "JWT" })) + // iat back-dated 60s for clock skew; exp ≤ 10min per GitHub's limit. + const payload = base64UrlString( + JSON.stringify({ iat: nowSec - 60, exp: nowSec + 540, iss: config.appId }), + ) + const signingInput = `${header}.${payload}` + const key = yield* importSigningKey(config.privateKeyPem) + const signature = yield* Effect.tryPromise({ + try: () => + crypto.subtle.sign( + "RSASSA-PKCS1-v1_5", + key, + new TextEncoder().encode(signingInput), + ), + catch: (cause) => new GithubAppError({ message: "JWT signing failed", cause }), + }) + return `${signingInput}.${base64UrlBytes(signature)}` + }) + + // ---- HTTP helpers --------------------------------------------- + + const failure = (response: Response, context: string) => + Effect.gen(function* () { + const body = yield* Effect.tryPromise({ + try: () => response.text(), + catch: () => new GithubAppError({ message: `${context} failed`, status: response.status }), + }) + return yield* Effect.fail( + new GithubAppError({ + message: `${context} failed: ${response.status} ${body.slice(0, 300)}`, + status: response.status, + }), + ) + }) + + const parseJson = (response: Response, context: string) => + Effect.tryPromise({ + try: () => response.json() as Promise, + catch: (cause) => + new GithubAppError({ message: `${context} returned a non-JSON response`, cause }), + }) + + const mintInstallationToken = Effect.fn("GithubAppClient.mintInstallationToken")(function* ( + externalInstallationId: string, + ) { + const config = yield* resolveConfig + const jwt = yield* mintAppJwt(config) + const response = yield* Effect.tryPromise({ + try: () => + fetch(`${config.apiBaseUrl}/app/installations/${externalInstallationId}/access_tokens`, { + method: "POST", + headers: { + authorization: `Bearer ${jwt}`, + accept: "application/vnd.github+json", + "x-github-api-version": GITHUB_API_VERSION, + "user-agent": USER_AGENT, + }, + }), + catch: (cause) => + new GithubAppError({ message: "Installation token request failed", cause }), + }) + if (!response.ok) return yield* failure(response, "Installation token request") + const json = yield* parseJson(response, "Installation token request") + const decoded = yield* decodeInstallationToken(json).pipe( + Effect.mapError( + (cause) => + new GithubAppError({ message: "Unexpected installation token payload", cause }), + ), + ) + return decoded.token + }) + + const authedGet = (config: ResolvedAppConfig, token: string, url: string) => + Effect.tryPromise({ + try: () => + fetch(url, { + headers: { + authorization: `token ${token}`, + accept: "application/vnd.github+json", + "x-github-api-version": GITHUB_API_VERSION, + "user-agent": USER_AGENT, + }, + }), + catch: (cause) => new GithubAppError({ message: `GitHub request failed: ${url}`, cause }), + }) + + const listInstallationRepositories = Effect.fn("GithubAppClient.listInstallationRepositories")( + function* (externalInstallationId: string) { + const config = yield* resolveConfig + const token = yield* mintInstallationToken(externalInstallationId) + const repos: Array = [] + for (let page = 1; page <= MAX_PAGES; page++) { + const response = yield* authedGet( + config, + token, + `${config.apiBaseUrl}/installation/repositories?per_page=${PER_PAGE}&page=${page}`, + ) + if (!response.ok) return yield* failure(response, "List installation repositories") + const json = yield* parseJson(response, "List installation repositories") + const decoded = yield* decodeInstallationRepos(json).pipe( + Effect.mapError( + (cause) => + new GithubAppError({ + message: "Unexpected installation repositories payload", + cause, + }), + ), + ) + repos.push(...decoded.repositories) + if (decoded.repositories.length < PER_PAGE) break + } + return repos + }, + ) + + const listCommits = Effect.fn("GithubAppClient.listCommits")(function* ( + externalInstallationId: string, + owner: string, + repo: string, + params: { sha?: string; sinceIso?: string }, + ) { + const config = yield* resolveConfig + const token = yield* mintInstallationToken(externalInstallationId) + const base = `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits` + const commits: Array = [] + for (let page = 1; page <= MAX_PAGES; page++) { + const query = new URLSearchParams({ per_page: String(PER_PAGE), page: String(page) }) + if (params.sha) query.set("sha", params.sha) + if (params.sinceIso) query.set("since", params.sinceIso) + const response = yield* authedGet(config, token, `${base}?${query.toString()}`) + // 409 = empty repository, 404 = not found/no access → no commits. + if (response.status === 404 || response.status === 409) break + if (!response.ok) return yield* failure(response, "List commits") + const json = yield* parseJson(response, "List commits") + const decoded = yield* decodeCommitList(json).pipe( + Effect.mapError( + (cause) => new GithubAppError({ message: "Unexpected commits payload", cause }), + ), + ) + commits.push(...decoded) + if (decoded.length < PER_PAGE) break + } + return commits + }) + + const getCommit = Effect.fn("GithubAppClient.getCommit")(function* ( + externalInstallationId: string, + owner: string, + repo: string, + sha: string, + ) { + const config = yield* resolveConfig + const token = yield* mintInstallationToken(externalInstallationId) + const response = yield* authedGet( + config, + token, + `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits/${sha}`, + ) + if (!response.ok) return yield* failure(response, "Get commit") + const json = yield* parseJson(response, "Get commit") + return yield* decodeCommit(json).pipe( + Effect.mapError( + (cause) => new GithubAppError({ message: "Unexpected commit payload", cause }), + ), + ) + }) + + return { mintInstallationToken, listInstallationRepositories, listCommits, getCommit } + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/github/GithubProvider.ts b/apps/api/src/services/github/GithubProvider.ts new file mode 100644 index 000000000..bd8578502 --- /dev/null +++ b/apps/api/src/services/github/GithubProvider.ts @@ -0,0 +1,292 @@ +import { + type CommitUpsertInput, + GitCommitSha, + type RepoUpsertInput, + type VcsInstallation, + type VcsInstallationSyncReason, + VcsProviderError, + type VcsProviderId, + type VcsRepositoryRef, + type VcsSyncJob, + VcsWebhookParseError, + VcsWebhookSignatureError, +} from "@maple/domain/http" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { Env } from "../../lib/Env" +import type { VcsProviderClient, VcsWebhookRequest } from "../vcs/VcsProviderClient" +import { type GithubApiCommit, GithubAppClient, GithubAppError } from "./GithubAppClient" + +const PROVIDER: VcsProviderId = "github" + +// ---- Webhook payload schemas (minimal, permissive) ------------------------ + +const PushAuthor = Schema.Struct({ + name: Schema.optionalKey(Schema.NullOr(Schema.String)), + email: Schema.optionalKey(Schema.NullOr(Schema.String)), + username: Schema.optionalKey(Schema.NullOr(Schema.String)), +}) + +const PushCommit = Schema.Struct({ + id: GitCommitSha, // validated at decode — the 40-hex shape lives in the brand + message: Schema.String, + timestamp: Schema.optionalKey(Schema.String), + url: Schema.String, + author: Schema.optionalKey(PushAuthor), +}) + +const PushPayload = Schema.Struct({ + ref: Schema.String, + repository: Schema.Struct({ + id: Schema.Number, + owner: Schema.Struct({ + login: Schema.optionalKey(Schema.String), + name: Schema.optionalKey(Schema.NullOr(Schema.String)), + }), + }), + installation: Schema.Struct({ id: Schema.Number }), + commits: Schema.optionalKey(Schema.Array(PushCommit)), +}) + +const InstallationPayload = Schema.Struct({ + action: Schema.String, + installation: Schema.Struct({ id: Schema.Number }), +}) + +const decodePush = Schema.decodeUnknownEffect(PushPayload) +const decodeInstallationEvent = Schema.decodeUnknownEffect(InstallationPayload) + +const parseError = (message: string) => new VcsWebhookParseError({ message }) + +// Decode an event payload, logging the structured cause server-side (so schema +// drift is diagnosable) while returning a generic 400-mapped error to the caller. +const parsePayload = (event: string, decoded: Effect.Effect) => + decoded.pipe( + Effect.tapError((cause) => + Effect.logWarning("Invalid GitHub webhook payload").pipe( + Effect.annotateLogs({ provider: PROVIDER, event, cause: String(cause) }), + ), + ), + Effect.mapError(() => parseError(`Invalid ${event} payload`)), + ) + +const toProviderError = (error: GithubAppError) => + new VcsProviderError({ + message: error.message, + ...(error.status === undefined ? {} : { status: error.status }), + ...(error.cause === undefined ? {} : { cause: error.cause }), + }) + +const finiteOrNull = (value: number) => (Number.isFinite(value) ? value : null) + +const installationReason = (action: string): VcsInstallationSyncReason | null => { + switch (action) { + case "created": + return "created" + case "unsuspend": + return "unsuspend" + case "suspend": + return "suspend" + case "deleted": + return "deleted" + default: + return null + } +} + +const timingSafeEqual = (a: string, b: string): boolean => { + const ba = Buffer.from(a) + const bb = Buffer.from(b) + if (ba.length !== bb.length) return false + let mismatch = 0 + for (let i = 0; i < ba.length; i += 1) mismatch |= ba[i]! ^ bb[i]! + return mismatch === 0 +} + +const normalizeFetchedCommit = (commit: GithubApiCommit, branch: string, now: number): CommitUpsertInput => { + const authoredAt = commit.commit.author?.date ? finiteOrNull(Date.parse(commit.commit.author.date)) : null + const committedAt = commit.commit.committer?.date ? finiteOrNull(Date.parse(commit.commit.committer.date)) : null + return { + sha: commit.sha, + message: commit.commit.message, + authorName: commit.commit.author?.name ?? null, + authorEmail: commit.commit.author?.email ?? null, + authorLogin: commit.author?.login ?? null, + authorAvatarUrl: commit.author?.avatar_url ?? null, + authoredAt, + committedAt: committedAt ?? authoredAt ?? now, + htmlUrl: commit.html_url, + branch, + } +} + +export class GithubProvider extends Context.Service()( + "@maple/api/services/github/GithubProvider", + { + make: Effect.gen(function* () { + const env = yield* Env + const client = yield* GithubAppClient + + const verifySignature = (rawBody: string, signatureHeader: string | undefined) => + Effect.gen(function* () { + const secret = env.GITHUB_APP_WEBHOOK_SECRET + if (Option.isNone(secret)) { + return yield* new VcsWebhookSignatureError({ + message: "GitHub webhook secret is not configured (GITHUB_APP_WEBHOOK_SECRET)", + }) + } + if (!signatureHeader || !signatureHeader.startsWith("sha256=")) { + return yield* new VcsWebhookSignatureError({ + message: "Missing or malformed X-Hub-Signature-256 header", + }) + } + const enc = new TextEncoder() + const key = yield* Effect.tryPromise({ + try: () => + crypto.subtle.importKey( + "raw", + enc.encode(Redacted.value(secret.value)), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"], + ), + catch: () => + new VcsWebhookSignatureError({ message: "Failed to import webhook secret" }), + }) + const mac = yield* Effect.tryPromise({ + try: () => crypto.subtle.sign("HMAC", key, enc.encode(rawBody)), + catch: () => + new VcsWebhookSignatureError({ message: "Failed to compute webhook signature" }), + }) + const expected = `sha256=${Buffer.from(mac).toString("hex")}` + if (!timingSafeEqual(expected, signatureHeader)) { + return yield* new VcsWebhookSignatureError({ message: "Webhook signature mismatch" }) + } + }) + + const mapPush = (raw: unknown, now: number) => + Effect.gen(function* () { + const payload = yield* parsePayload("push", decodePush(raw)) + if (!payload.ref.startsWith("refs/heads/")) return [] // ignore tag/other refs + const branch = payload.ref.slice("refs/heads/".length) + const commits: ReadonlyArray = (payload.commits ?? []).map((c) => { + const ts = c.timestamp ? finiteOrNull(Date.parse(c.timestamp)) : null + return { + sha: c.id, + message: c.message, + authorName: c.author?.name ?? null, + authorEmail: c.author?.email ?? null, + authorLogin: c.author?.username ?? null, + authorAvatarUrl: null, + authoredAt: ts, + committedAt: ts ?? now, + htmlUrl: c.url, + branch, + } + }) + if (commits.length === 0) return [] + const job: VcsSyncJob = { + kind: "push-delta", + provider: PROVIDER, + externalInstallationId: String(payload.installation.id), + externalRepoId: String(payload.repository.id), + branch, + commits, + } + return [job] + }) + + const mapInstallationEvent = + (reasonFor: (action: string) => VcsInstallationSyncReason | null) => (raw: unknown) => + Effect.gen(function* () { + const payload = yield* parsePayload("installation", decodeInstallationEvent(raw)) + const reason = reasonFor(payload.action) + if (!reason) return [] + const job: VcsSyncJob = { + kind: "installation-sync", + provider: PROVIDER, + externalInstallationId: String(payload.installation.id), + reason, + } + return [job] + }) + + const mapInstallation = mapInstallationEvent(installationReason) + const mapInstallationRepositories = mapInstallationEvent((action) => + action === "added" + ? "repositories_added" + : action === "removed" + ? "repositories_removed" + : null, + ) + + const webhookToJobs = (input: VcsWebhookRequest) => + Effect.gen(function* () { + yield* verifySignature(input.rawBody, input.headers["x-hub-signature-256"]) + const parsed = yield* Effect.try({ + try: () => JSON.parse(input.rawBody) as unknown, + catch: () => parseError("Invalid JSON body"), + }) + const now = yield* Clock.currentTimeMillis + switch (input.headers["x-github-event"]) { + case "push": + return yield* mapPush(parsed, now) + case "installation": + return yield* mapInstallation(parsed) + case "installation_repositories": + return yield* mapInstallationRepositories(parsed) + default: + return [] // ping and unhandled events are accepted no-ops + } + }).pipe( + Effect.withSpan("GithubProvider.webhookToJobs", { + attributes: { + "vcs.provider": PROVIDER, + "vcs.webhook.event": input.headers["x-github-event"] ?? "unknown", + }, + }), + ) + + const fetchRepositories = (installation: VcsInstallation) => + client.listInstallationRepositories(installation.externalInstallationId).pipe( + Effect.map((repos): ReadonlyArray => + repos.map((r) => ({ + externalRepoId: String(r.id), + owner: r.owner.login, + name: r.name, + fullName: r.full_name, + defaultBranch: r.default_branch ?? "main", + htmlUrl: r.html_url, + isPrivate: r.private, + isArchived: r.archived ?? false, + })), + ), + Effect.mapError(toProviderError), + ) + + const fetchCommits = ( + installation: VcsInstallation, + repo: VcsRepositoryRef, + opts: { readonly sinceMs: number }, + ) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + const commits = yield* client + .listCommits(installation.externalInstallationId, repo.owner, repo.name, { + sha: repo.defaultBranch, + sinceIso: new Date(opts.sinceMs).toISOString(), + }) + .pipe(Effect.mapError(toProviderError)) + return commits.map((c) => normalizeFetchedCommit(c, repo.defaultBranch, now)) + }) + + return { + id: PROVIDER, + webhookToJobs, + fetchRepositories, + fetchCommits, + } satisfies VcsProviderClient + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/vcs/VcsProviderClient.ts b/apps/api/src/services/vcs/VcsProviderClient.ts new file mode 100644 index 000000000..da0b368d4 --- /dev/null +++ b/apps/api/src/services/vcs/VcsProviderClient.ts @@ -0,0 +1,48 @@ +import type { Effect } from "effect" +import type { + CommitUpsertInput, + RepoUpsertInput, + VcsInstallation, + VcsProviderError, + VcsProviderId, + VcsRepositoryRef, + VcsSyncJob, + VcsWebhookParseError, + VcsWebhookSignatureError, +} from "@maple/domain/http" + +// --------------------------------------------------------------------------- +// The single typed seam between the vendor-agnostic core and a VCS provider. +// +// Everything ABOVE this port (queue, orchestrator, webhook router, repo, tables) +// is provider-neutral and never imports a provider module. Everything BELOW it +// (GithubProvider, GithubAppClient, GitHub schemas) is provider-specific and +// never imports the vcs_* tables. The registry is the only place a provider id +// is wired to an implementation. +// --------------------------------------------------------------------------- + +export interface VcsWebhookRequest { + readonly headers: Record + readonly rawBody: string +} + +export interface VcsProviderClient { + readonly id: VcsProviderId + + /** Verify the webhook signature, parse the event, and map it to generic jobs. */ + readonly webhookToJobs: ( + input: VcsWebhookRequest, + ) => Effect.Effect, VcsWebhookSignatureError | VcsWebhookParseError> + + /** All repositories visible to an installation, normalized. */ + readonly fetchRepositories: ( + installation: VcsInstallation, + ) => Effect.Effect, VcsProviderError> + + /** Commits on a repo's default branch authored since `sinceMs`, normalized. */ + readonly fetchCommits: ( + installation: VcsInstallation, + repo: VcsRepositoryRef, + opts: { readonly sinceMs: number }, + ) => Effect.Effect, VcsProviderError> +} diff --git a/apps/api/src/services/vcs/VcsProviderRegistry.ts b/apps/api/src/services/vcs/VcsProviderRegistry.ts new file mode 100644 index 000000000..24fb6ec19 --- /dev/null +++ b/apps/api/src/services/vcs/VcsProviderRegistry.ts @@ -0,0 +1,44 @@ +import { UnknownVcsProviderError } from "@maple/domain/http" +import { Context, Effect, Layer } from "effect" +import { GithubProvider } from "../github/GithubProvider" +import type { VcsProviderClient } from "./VcsProviderClient" + +// --------------------------------------------------------------------------- +// Resolves a provider id → its `VcsProviderClient` implementation. This is the +// ONLY module that names a concrete provider. Adding a provider = implement the +// port + add one entry here; the generic orchestrator/webhook never change. +// --------------------------------------------------------------------------- + +export interface VcsProviderRegistryShape { + /** The ids of every registered provider (e.g. for static webhook routes). */ + readonly ids: ReadonlyArray + readonly resolve: ( + provider: string, + ) => Effect.Effect +} + +export class VcsProviderRegistry extends Context.Service()( + "@maple/api/services/vcs/VcsProviderRegistry", + { + make: Effect.gen(function* () { + const github = yield* GithubProvider + const byId: Record = { [github.id]: github } + + const resolve = (provider: string) => { + const impl = byId[provider] + return impl + ? Effect.succeed(impl) + : Effect.fail( + new UnknownVcsProviderError({ + provider, + message: `Unknown VCS provider: ${provider}`, + }), + ) + } + + return { ids: Object.keys(byId), resolve } satisfies VcsProviderRegistryShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/vcs/VcsRepository.ts b/apps/api/src/services/vcs/VcsRepository.ts new file mode 100644 index 000000000..f989b30b0 --- /dev/null +++ b/apps/api/src/services/vcs/VcsRepository.ts @@ -0,0 +1,481 @@ +import { randomUUID } from "node:crypto" +import { + type CommitUpsertInput, + GitCommitSha, + type OrgId, + type RepoUpsertInput, + type UserId, + VcsCommit, + VcsInstallation, + type VcsInstallStatus, + type VcsProviderId, + VcsRepo, + type VcsRepoSelection, + VcsRepoDecodeError, + VcsRepoPersistenceError, + type VcsAccountType, + type VcsRepoSyncStatus, +} from "@maple/domain/http" +import { + chunkRowsForInsert, + vcsCommits, + vcsInstallations, + type VcsCommitRow, + type VcsInstallationRow, + vcsRepositories, + type VcsRepositoryRow, +} from "@maple/db" +import { and, eq, sql } from "drizzle-orm" +import { Clock, Context, Effect, Layer, Option, Schema } from "effect" +import { Database, type DatabaseError } from "../../lib/DatabaseLive" + +const decodeInstallation = Schema.decodeUnknownSync(VcsInstallation) +const decodeRepo = Schema.decodeUnknownSync(VcsRepo) +const decodeCommit = Schema.decodeUnknownSync(VcsCommit) +// Validate the SHA shape via the branded type (the regex lives only there); +// a malformed SHA throws and is caught into a VcsRepoDecodeError on write. +const decodeGitSha = Schema.decodeUnknownSync(GitCommitSha) + +const toPersistenceError = (error: DatabaseError) => new VcsRepoPersistenceError({ message: error.message }) + +const decodeAll = (table: string, rows: ReadonlyArray, f: (row: Row) => A) => + Effect.try({ + try: () => rows.map(f), + catch: (err) => + new VcsRepoDecodeError({ message: err instanceof Error ? err.message : "row decode failed", table }), + }) + +const decodeOne = (table: string, row: Row, f: (row: Row) => A) => + Effect.try({ + try: () => f(row), + catch: (err) => + new VcsRepoDecodeError({ message: err instanceof Error ? err.message : "row decode failed", table }), + }) + +const rowToInstallation = (row: VcsInstallationRow): VcsInstallation => + decodeInstallation({ + id: row.id, + orgId: row.orgId, + provider: row.provider, + externalInstallationId: row.externalInstallationId, + accountLogin: row.accountLogin, + accountType: row.accountType, + externalAccountId: row.externalAccountId, + accountAvatarUrl: row.accountAvatarUrl ?? null, + repositorySelection: row.repositorySelection, + status: row.status, + suspendedAt: row.suspendedAt ?? null, + installedByUserId: row.installedByUserId, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }) + +const rowToRepo = (row: VcsRepositoryRow): VcsRepo => + decodeRepo({ + id: row.id, + orgId: row.orgId, + provider: row.provider, + externalInstallationId: row.externalInstallationId, + externalRepoId: row.externalRepoId, + owner: row.owner, + name: row.name, + fullName: row.fullName, + defaultBranch: row.defaultBranch, + htmlUrl: row.htmlUrl, + isPrivate: row.isPrivate === 1, + isArchived: row.isArchived === 1, + syncStatus: row.syncStatus, + lastSyncedAt: row.lastSyncedAt ?? null, + lastSyncCursor: row.lastSyncCursor ?? null, + lastSyncError: row.lastSyncError ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }) + +const rowToCommit = (row: VcsCommitRow): VcsCommit => + decodeCommit({ + id: row.id, + orgId: row.orgId, + provider: row.provider, + externalRepoId: row.externalRepoId, + sha: row.sha, + shortSha: row.shortSha, + message: row.message, + authorName: row.authorName ?? null, + authorEmail: row.authorEmail ?? null, + authorLogin: row.authorLogin ?? null, + authorAvatarUrl: row.authorAvatarUrl ?? null, + authoredAt: row.authoredAt ?? null, + committedAt: row.committedAt, + htmlUrl: row.htmlUrl, + branch: row.branch ?? null, + createdAt: row.createdAt, + }) + +export interface UpsertInstallationInput { + readonly orgId: OrgId + readonly provider: VcsProviderId + readonly externalInstallationId: string + readonly accountLogin: string + readonly accountType: VcsAccountType + readonly externalAccountId: string + readonly accountAvatarUrl: string | null + readonly repositorySelection: VcsRepoSelection + readonly status?: VcsInstallStatus + readonly installedByUserId: UserId +} + +export interface RepoSyncCursor { + readonly status: VcsRepoSyncStatus + readonly cursorSha?: string | null + readonly error?: string | null + readonly syncedAt?: number | null +} + +export class VcsRepository extends Context.Service()("@maple/api/services/vcs/VcsRepository", { + make: Effect.gen(function* () { + const database = yield* Database + + // ---- Installations ------------------------------------------------ + + const selectInstallationRow = (provider: VcsProviderId, externalInstallationId: string) => + database + .execute((db) => + db + .select() + .from(vcsInstallations) + .where( + and( + eq(vcsInstallations.provider, provider), + eq(vcsInstallations.externalInstallationId, externalInstallationId), + ), + ) + .limit(1), + ) + .pipe(Effect.mapError(toPersistenceError)) + + const getInstallation = Effect.fn("VcsRepository.getInstallation")(function* ( + provider: VcsProviderId, + externalInstallationId: string, + ) { + const rows = yield* selectInstallationRow(provider, externalInstallationId) + const row = Option.fromNullishOr(rows[0]) + if (Option.isNone(row)) return Option.none() + return Option.some(yield* decodeOne("vcs_installations", row.value, rowToInstallation)) + }) + + const listInstallationsByOrg = Effect.fn("VcsRepository.listInstallationsByOrg")(function* (orgId: OrgId) { + const rows = yield* database + .execute((db) => db.select().from(vcsInstallations).where(eq(vcsInstallations.orgId, orgId))) + .pipe(Effect.mapError(toPersistenceError)) + return yield* decodeAll("vcs_installations", rows, rowToInstallation) + }) + + const upsertInstallation = Effect.fn("VcsRepository.upsertInstallation")(function* ( + input: UpsertInstallationInput, + ) { + const now = yield* Clock.currentTimeMillis + yield* database + .execute((db) => + db + .insert(vcsInstallations) + .values({ + id: randomUUID() as VcsInstallation["id"], + orgId: input.orgId, + provider: input.provider, + externalInstallationId: input.externalInstallationId, + accountLogin: input.accountLogin, + accountType: input.accountType, + externalAccountId: input.externalAccountId, + accountAvatarUrl: input.accountAvatarUrl, + repositorySelection: input.repositorySelection, + status: input.status ?? "active", + suspendedAt: null, + installedByUserId: input.installedByUserId, + createdAt: now, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [vcsInstallations.provider, vcsInstallations.externalInstallationId], + // Ownership columns (org_id, installed_by_user_id, created_at) are + // immutable on conflict — only mutable provider metadata is refreshed. + set: { + accountLogin: sql`excluded.account_login`, + accountType: sql`excluded.account_type`, + externalAccountId: sql`excluded.external_account_id`, + accountAvatarUrl: sql`excluded.account_avatar_url`, + repositorySelection: sql`excluded.repository_selection`, + status: sql`excluded.status`, + suspendedAt: sql`excluded.suspended_at`, + updatedAt: sql`excluded.updated_at`, + }, + }), + ) + .pipe(Effect.mapError(toPersistenceError)) + + const rows = yield* selectInstallationRow(input.provider, input.externalInstallationId) + const row = Option.fromNullishOr(rows[0]) + if (Option.isNone(row)) { + return yield* new VcsRepoPersistenceError({ message: "Installation vanished after upsert" }) + } + return yield* decodeOne("vcs_installations", row.value, rowToInstallation) + }) + + const markInstallationStatus = Effect.fn("VcsRepository.markInstallationStatus")(function* ( + provider: VcsProviderId, + externalInstallationId: string, + status: VcsInstallStatus, + ) { + const now = yield* Clock.currentTimeMillis + yield* database + .execute((db) => + db + .update(vcsInstallations) + .set({ status, suspendedAt: status === "suspended" ? now : null, updatedAt: now }) + .where( + and( + eq(vcsInstallations.provider, provider), + eq(vcsInstallations.externalInstallationId, externalInstallationId), + ), + ), + ) + .pipe(Effect.mapError(toPersistenceError)) + }) + + // ---- Repositories ------------------------------------------------- + + const listRepositoriesByInstallation = Effect.fn("VcsRepository.listRepositoriesByInstallation")( + function* (provider: VcsProviderId, externalInstallationId: string) { + const rows = yield* database + .execute((db) => + db + .select() + .from(vcsRepositories) + .where( + and( + eq(vcsRepositories.provider, provider), + eq(vcsRepositories.externalInstallationId, externalInstallationId), + ), + ), + ) + .pipe(Effect.mapError(toPersistenceError)) + return yield* decodeAll("vcs_repositories", rows, rowToRepo) + }, + ) + + const upsertRepositories = Effect.fn("VcsRepository.upsertRepositories")(function* ( + orgId: OrgId, + provider: VcsProviderId, + externalInstallationId: string, + repos: ReadonlyArray, + ) { + if (repos.length === 0) return + const now = yield* Clock.currentTimeMillis + const values = repos.map((r) => ({ + id: randomUUID() as VcsRepo["id"], + orgId, + provider, + externalInstallationId, + externalRepoId: r.externalRepoId, + owner: r.owner, + name: r.name, + fullName: r.fullName, + defaultBranch: r.defaultBranch, + htmlUrl: r.htmlUrl, + isPrivate: r.isPrivate ? 1 : 0, + isArchived: r.isArchived ? 1 : 0, + createdAt: now, + updatedAt: now, + })) + yield* Effect.forEach( + chunkRowsForInsert(vcsRepositories, values), + (chunk) => + database + .execute((db) => + db + .insert(vcsRepositories) + .values(chunk) + .onConflictDoUpdate({ + target: [ + vcsRepositories.orgId, + vcsRepositories.provider, + vcsRepositories.externalRepoId, + ], + set: { + externalInstallationId: sql`excluded.external_installation_id`, + owner: sql`excluded.owner`, + name: sql`excluded.name`, + fullName: sql`excluded.full_name`, + defaultBranch: sql`excluded.default_branch`, + htmlUrl: sql`excluded.html_url`, + isPrivate: sql`excluded.is_private`, + isArchived: sql`excluded.is_archived`, + updatedAt: sql`excluded.updated_at`, + }, + }), + ) + .pipe(Effect.mapError(toPersistenceError)), + { discard: true }, + ) + }) + + const removeRepository = Effect.fn("VcsRepository.removeRepository")(function* ( + orgId: OrgId, + provider: VcsProviderId, + externalRepoId: string, + ) { + yield* database + .execute((db) => + db + .delete(vcsRepositories) + .where( + and( + eq(vcsRepositories.orgId, orgId), + eq(vcsRepositories.provider, provider), + eq(vcsRepositories.externalRepoId, externalRepoId), + ), + ), + ) + .pipe(Effect.mapError(toPersistenceError)) + }) + + const updateRepoSyncCursor = Effect.fn("VcsRepository.updateRepoSyncCursor")(function* ( + orgId: OrgId, + provider: VcsProviderId, + externalRepoId: string, + cursor: RepoSyncCursor, + ) { + const now = yield* Clock.currentTimeMillis + yield* database + .execute((db) => + db + .update(vcsRepositories) + .set({ + syncStatus: cursor.status, + lastSyncCursor: cursor.cursorSha ?? null, + lastSyncError: cursor.error ?? null, + lastSyncedAt: cursor.syncedAt ?? now, + updatedAt: now, + }) + .where( + and( + eq(vcsRepositories.orgId, orgId), + eq(vcsRepositories.provider, provider), + eq(vcsRepositories.externalRepoId, externalRepoId), + ), + ), + ) + .pipe(Effect.mapError(toPersistenceError)) + }) + + // ---- Commits ------------------------------------------------------ + + const upsertCommits = Effect.fn("VcsRepository.upsertCommits")(function* ( + orgId: OrgId, + provider: VcsProviderId, + externalRepoId: string, + commits: ReadonlyArray, + ) { + if (commits.length === 0) return 0 + const now = yield* Clock.currentTimeMillis + // Decode every SHA through the branded type before writing — a bad SHA + // throws here and is mapped to VcsRepoDecodeError below. + const values = yield* Effect.try({ + try: () => + commits.map((c) => { + const sha = decodeGitSha(c.sha) + return { + id: randomUUID() as VcsCommit["id"], + orgId, + provider, + externalRepoId, + sha, + shortSha: sha.slice(0, 7) as VcsCommit["shortSha"], + message: c.message, + authorName: c.authorName, + authorEmail: c.authorEmail, + authorLogin: c.authorLogin, + authorAvatarUrl: c.authorAvatarUrl, + authoredAt: c.authoredAt, + committedAt: c.committedAt, + htmlUrl: c.htmlUrl, + branch: c.branch, + createdAt: now, + } + }), + catch: (err) => + new VcsRepoDecodeError({ + message: err instanceof Error ? err.message : "commit decode failed", + table: "vcs_commits", + column: "sha", + }), + }) + + yield* Effect.forEach( + chunkRowsForInsert(vcsCommits, values), + (chunk) => + database + .execute((db) => + db + .insert(vcsCommits) + .values(chunk) + .onConflictDoUpdate({ + target: [ + vcsCommits.orgId, + vcsCommits.provider, + vcsCommits.externalRepoId, + vcsCommits.sha, + ], + set: { + message: sql`excluded.message`, + authorName: sql`excluded.author_name`, + authorEmail: sql`excluded.author_email`, + authorLogin: sql`excluded.author_login`, + authorAvatarUrl: sql`excluded.author_avatar_url`, + authoredAt: sql`excluded.authored_at`, + committedAt: sql`excluded.committed_at`, + htmlUrl: sql`excluded.html_url`, + branch: sql`excluded.branch`, + }, + }), + ) + .pipe(Effect.mapError(toPersistenceError)), + { discard: true }, + ) + return values.length + }) + + const findCommitBySha = Effect.fn("VcsRepository.findCommitBySha")(function* ( + orgId: OrgId, + sha: GitCommitSha, + ) { + const rows = yield* database + .execute((db) => + db + .select() + .from(vcsCommits) + .where(and(eq(vcsCommits.orgId, orgId), eq(vcsCommits.sha, sha))) + .limit(1), + ) + .pipe(Effect.mapError(toPersistenceError)) + const row = Option.fromNullishOr(rows[0]) + if (Option.isNone(row)) return Option.none() + return Option.some(yield* decodeOne("vcs_commits", row.value, rowToCommit)) + }) + + return { + getInstallation, + listInstallationsByOrg, + upsertInstallation, + markInstallationStatus, + listRepositoriesByInstallation, + upsertRepositories, + removeRepository, + updateRepoSyncCursor, + upsertCommits, + findCommitBySha, + } + }), +}) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/vcs/VcsSyncQueue.ts b/apps/api/src/services/vcs/VcsSyncQueue.ts new file mode 100644 index 000000000..16e3303c0 --- /dev/null +++ b/apps/api/src/services/vcs/VcsSyncQueue.ts @@ -0,0 +1,61 @@ +import type { Queue } from "@cloudflare/workers-types" +import { VcsQueueError, VcsSyncJob } from "@maple/domain/http" +import { WorkerEnvironment } from "@maple/effect-cloudflare" +import { Context, Effect, Layer, Schema } from "effect" + +// --------------------------------------------------------------------------- +// Vendor-agnostic queue producer. Reads the `VCS_SYNC_QUEUE` binding from the +// worker env and sends Schema-encoded `VcsSyncJob`s. The same queue carries +// jobs for every provider (discriminated by `job.provider`). +// --------------------------------------------------------------------------- + +const QUEUE_BINDING = "VCS_SYNC_QUEUE" +const encodeJob = Schema.encodeSync(VcsSyncJob) + +export interface VcsSyncQueueShape { + readonly send: (job: VcsSyncJob) => Effect.Effect + readonly sendBatch: (jobs: ReadonlyArray) => Effect.Effect +} + +export class VcsSyncQueue extends Context.Service()( + "@maple/api/services/vcs/VcsSyncQueue", + { + make: Effect.gen(function* () { + const workerEnv = yield* WorkerEnvironment + const queue = (workerEnv as Record)[QUEUE_BINDING] as Queue | undefined + + const missing = new VcsQueueError({ message: `Missing queue binding: ${QUEUE_BINDING}` }) + + const send = Effect.fn("VcsSyncQueue.send")(function* (job: VcsSyncJob) { + if (!queue) return yield* missing + const body = encodeJob(job) + yield* Effect.tryPromise({ + try: () => queue.send(body), + catch: (cause) => + new VcsQueueError({ + message: cause instanceof Error ? cause.message : "queue send failed", + }), + }) + }) + + const sendBatch = Effect.fn("VcsSyncQueue.sendBatch")(function* ( + jobs: ReadonlyArray, + ) { + if (jobs.length === 0) return + if (!queue) return yield* missing + const messages = jobs.map((job) => ({ body: encodeJob(job) })) + yield* Effect.tryPromise({ + try: () => queue.sendBatch(messages), + catch: (cause) => + new VcsQueueError({ + message: cause instanceof Error ? cause.message : "queue sendBatch failed", + }), + }) + }) + + return { send, sendBatch } satisfies VcsSyncQueueShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/vcs/VcsSyncService.ts b/apps/api/src/services/vcs/VcsSyncService.ts new file mode 100644 index 000000000..7c5dba345 --- /dev/null +++ b/apps/api/src/services/vcs/VcsSyncService.ts @@ -0,0 +1,239 @@ +import { + type CommitUpsertInput, + type UnknownVcsProviderError, + type VcsInstallation, + type VcsInstallationSyncReason, + type VcsProviderError, + type VcsQueueError, + type VcsRepoDecodeError, + type VcsRepoPersistenceError, + VcsSyncJob, +} from "@maple/domain/http" +import { Clock, Effect, Context, Layer, Option, Schema } from "effect" +import type { VcsProviderClient } from "./VcsProviderClient" +import { VcsProviderRegistry } from "./VcsProviderRegistry" +import { VcsRepository } from "./VcsRepository" +import { VcsSyncQueue } from "./VcsSyncQueue" + +// --------------------------------------------------------------------------- +// Vendor-agnostic sync orchestrator. Decodes a queue message, resolves the +// owning installation (→ orgId + provider auth), then dispatches by job kind: +// fetch via the provider port → persist via the repo. The provider port is the +// only provider-specific surface it touches. +// --------------------------------------------------------------------------- + +const BACKFILL_DAYS = 90 +const DAY_MS = 86_400_000 + +const decodeJob = Schema.decodeUnknownEffect(VcsSyncJob) + +type SyncError = + | VcsRepoPersistenceError + | VcsRepoDecodeError + | VcsProviderError + | VcsQueueError + | UnknownVcsProviderError + +export interface VcsSyncServiceShape { + readonly processMessage: (raw: unknown) => Effect.Effect +} + +export class VcsSyncService extends Context.Service()( + "@maple/api/services/vcs/VcsSyncService", + { + make: Effect.gen(function* () { + const repo = yield* VcsRepository + const registry = yield* VcsProviderRegistry + const queue = yield* VcsSyncQueue + + const syncInstallation = Effect.fn("VcsSyncService.syncInstallation")(function* ( + provider: VcsProviderClient, + installation: VcsInstallation, + reason: VcsInstallationSyncReason, + ) { + const now = yield* Clock.currentTimeMillis + const repos = yield* provider.fetchRepositories(installation) + yield* repo.upsertRepositories( + installation.orgId, + installation.provider, + installation.externalInstallationId, + repos, + ) + + // Reconcile removals: drop local repos no longer visible upstream. + if (reason === "repositories_removed") { + const remoteIds = new Set(repos.map((r) => r.externalRepoId)) + const local = yield* repo.listRepositoriesByInstallation( + installation.provider, + installation.externalInstallationId, + ) + yield* Effect.forEach( + local.filter((r) => !remoteIds.has(r.externalRepoId)), + (r) => + repo.removeRepository(installation.orgId, installation.provider, r.externalRepoId), + { discard: true }, + ) + } + + const sinceMs = now - BACKFILL_DAYS * DAY_MS + yield* queue.sendBatch( + repos.map((r) => ({ + kind: "backfill-repo" as const, + provider: installation.provider, + externalInstallationId: installation.externalInstallationId, + externalRepoId: r.externalRepoId, + owner: r.owner, + name: r.name, + defaultBranch: r.defaultBranch, + sinceMs, + })), + ) + }) + + const backfillRepo = ( + provider: VcsProviderClient, + installation: VcsInstallation, + job: { externalRepoId: string; owner: string; name: string; defaultBranch: string; sinceMs: number }, + ) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + const commits = yield* provider.fetchCommits( + installation, + { + externalRepoId: job.externalRepoId, + owner: job.owner, + name: job.name, + defaultBranch: job.defaultBranch, + }, + { sinceMs: job.sinceMs }, + ) + yield* repo.upsertCommits( + installation.orgId, + installation.provider, + job.externalRepoId, + commits, + ) + // GitHub returns newest-first; the first commit is the branch head. + yield* repo.updateRepoSyncCursor(installation.orgId, installation.provider, job.externalRepoId, { + status: "ready", + cursorSha: commits[0]?.sha ?? null, + error: null, + syncedAt: now, + }) + }).pipe( + // App uninstalled / repo gone → stop retrying, mark the installation disconnected. + Effect.catchTag( + "@maple/http/errors/VcsProviderError", + (error): Effect.Effect => + error.status === 404 || error.status === 410 + ? repo + .markInstallationStatus( + installation.provider, + installation.externalInstallationId, + "disconnected", + ) + .pipe( + Effect.flatMap(() => + Effect.logWarning("VCS installation no longer accessible").pipe( + Effect.annotateLogs({ + provider: installation.provider, + externalInstallationId: installation.externalInstallationId, + status: error.status, + }), + ), + ), + ) + : Effect.fail(error), + ), + Effect.withSpan("VcsSyncService.backfillRepo"), + ) + + const applyPushDelta = Effect.fn("VcsSyncService.applyPushDelta")(function* ( + installation: VcsInstallation, + job: { externalRepoId: string; commits: ReadonlyArray }, + ) { + const now = yield* Clock.currentTimeMillis + yield* repo.upsertCommits( + installation.orgId, + installation.provider, + job.externalRepoId, + job.commits, + ) + // Push commits are oldest→newest; the last is the new branch head. + const head = job.commits.length > 0 ? job.commits[job.commits.length - 1] : undefined + yield* repo.updateRepoSyncCursor(installation.orgId, installation.provider, job.externalRepoId, { + status: "ready", + cursorSha: head?.sha ?? null, + error: null, + syncedAt: now, + }) + }) + + const processMessage = Effect.fn("VcsSyncService.processMessage")(function* (raw: unknown) { + const jobOpt = yield* decodeJob(raw).pipe( + Effect.map(Option.some), + Effect.catch((cause) => + Effect.logWarning("Dropping undecodable VCS sync job").pipe( + Effect.annotateLogs({ error: String(cause) }), + Effect.as(Option.none()), + ), + ), + ) + if (Option.isNone(jobOpt)) return + const job = jobOpt.value + yield* Effect.annotateCurrentSpan({ + "vcs.provider": job.provider, + "vcs.job_kind": job.kind, + "vcs.installation.external_id": job.externalInstallationId, + }) + + // suspend/delete only flip status — no installation lookup needed. + if ( + job.kind === "installation-sync" && + (job.reason === "suspend" || job.reason === "deleted") + ) { + yield* repo.markInstallationStatus( + job.provider, + job.externalInstallationId, + job.reason === "suspend" ? "suspended" : "disconnected", + ) + return + } + + const installationOpt = yield* repo.getInstallation(job.provider, job.externalInstallationId) + if (Option.isNone(installationOpt)) { + yield* Effect.logInfo("Dropping VCS job for unknown installation").pipe( + Effect.annotateLogs({ + provider: job.provider, + externalInstallationId: job.externalInstallationId, + kind: job.kind, + }), + ) + return + } + const installation = installationOpt.value + if (installation.status === "disconnected") { + yield* Effect.logInfo("Dropping VCS job for disconnected installation").pipe( + Effect.annotateLogs({ externalInstallationId: job.externalInstallationId, kind: job.kind }), + ) + return + } + + const provider = yield* registry.resolve(job.provider) + + switch (job.kind) { + case "installation-sync": + return yield* syncInstallation(provider, installation, job.reason) + case "backfill-repo": + return yield* backfillRepo(provider, installation, job) + case "push-delta": + return yield* applyPushDelta(installation, job) + } + }) + + return { processMessage } satisfies VcsSyncServiceShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/vcs/__tests__/vcs.test.ts b/apps/api/src/services/vcs/__tests__/vcs.test.ts new file mode 100644 index 000000000..1fd7a07f0 --- /dev/null +++ b/apps/api/src/services/vcs/__tests__/vcs.test.ts @@ -0,0 +1,470 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { createHmac, randomUUID } from "node:crypto" +import { + OrgId, + UserId, + VcsRepoDecodeError, + VcsSyncJob, + VcsWebhookParseError, + VcsWebhookSignatureError, +} from "@maple/domain/http" +import { Cause, ConfigProvider, Effect, Exit, Layer, Option, Schema } from "effect" +import { DatabaseLibsqlLive } from "@/lib/DatabaseLibsqlLive" +import { Env } from "@/lib/Env" +import { cleanupTempDirs, createTempDbUrl, executeSql } from "@/lib/test-sqlite" +import { GithubAppClient } from "@/services/github/GithubAppClient" +import { GithubProvider } from "@/services/github/GithubProvider" +import type { VcsProviderClient } from "@/services/vcs/VcsProviderClient" +import { VcsProviderRegistry, type VcsProviderRegistryShape } from "@/services/vcs/VcsProviderRegistry" +import { VcsRepository } from "@/services/vcs/VcsRepository" +import { VcsSyncQueue, type VcsSyncQueueShape } from "@/services/vcs/VcsSyncQueue" +import { VcsSyncService } from "@/services/vcs/VcsSyncService" + +const dirs: string[] = [] +afterEach(() => cleanupTempDirs(dirs)) + +const WEBHOOK_SECRET = "testsecret" +const SHA = "abc1230000000000000000000000000000000def" + +const config = (url: string) => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_DB_URL: url, + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + GITHUB_APP_WEBHOOK_SECRET: WEBHOOK_SECRET, + }), + ) + +const envLayer = (url: string) => Env.layer.pipe(Layer.provide(config(url))) + +const repoLayer = (url: string) => + VcsRepository.layer.pipe(Layer.provide(DatabaseLibsqlLive), Layer.provide(envLayer(url))) + +const providerLayer = () => { + const env = envLayer("") + return GithubProvider.layer.pipe( + Layer.provide(Layer.mergeAll(env, GithubAppClient.layer.pipe(Layer.provide(env)))), + ) +} + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const asUserId = Schema.decodeUnknownSync(UserId) + +const sign = (body: string) => `sha256=${createHmac("sha256", WEBHOOK_SECRET).update(body).digest("hex")}` + +const findError = (exit: Exit.Exit): unknown => { + if (!Exit.isFailure(exit)) return undefined + const failure = Option.getOrUndefined(Exit.findErrorOption(exit)) + return failure ?? Cause.squash(exit.cause) +} + +describe("VcsSyncJob", () => { + it("round-trips through encode/decode", () => { + const job: VcsSyncJob = { + kind: "push-delta", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + branch: "main", + commits: [ + { + sha: SHA, + message: "hello", + authorName: "Octo", + authorEmail: "o@x.io", + authorLogin: "octocat", + authorAvatarUrl: null, + authoredAt: 1, + committedAt: 2, + htmlUrl: "https://github.com/o/r/commit/x", + branch: "main", + }, + ], + } + const wire = JSON.parse(JSON.stringify(Schema.encodeSync(VcsSyncJob)(job))) + assert.deepStrictEqual(Schema.decodeUnknownSync(VcsSyncJob)(wire), job) + }) +}) + +describe("GithubProvider.webhookToJobs", () => { + const pushBody = JSON.stringify({ + ref: "refs/heads/main", + repository: { id: 7, owner: { login: "octo" } }, + installation: { id: 42 }, + commits: [ + { + id: SHA, + message: "hello world", + timestamp: "2026-01-01T00:00:00Z", + url: `https://github.com/octo/repo/commit/${SHA}`, + author: { name: "Octo Cat", email: "octo@x.io", username: "octocat" }, + }, + ], + }) + + it.effect("maps a validly-signed push to a push-delta job", () => + Effect.gen(function* () { + const provider = yield* GithubProvider + const jobs = yield* provider.webhookToJobs({ + headers: { "x-github-event": "push", "x-hub-signature-256": sign(pushBody) }, + rawBody: pushBody, + }) + assert.strictEqual(jobs.length, 1) + const job = jobs[0]! + assert.strictEqual(job.kind, "push-delta") + if (job.kind !== "push-delta") return + assert.strictEqual(job.externalInstallationId, "42") + assert.strictEqual(job.externalRepoId, "7") + assert.strictEqual(job.branch, "main") + assert.strictEqual(job.commits.length, 1) + assert.strictEqual(job.commits[0]!.sha, SHA) + assert.strictEqual(job.commits[0]!.authorLogin, "octocat") + }).pipe(Effect.provide(providerLayer())), + ) + + it.effect("rejects an invalid signature with VcsWebhookSignatureError", () => + Effect.gen(function* () { + const provider = yield* GithubProvider + const exit = yield* provider + .webhookToJobs({ + headers: { "x-github-event": "push", "x-hub-signature-256": "sha256=deadbeef" }, + rawBody: pushBody, + }) + .pipe(Effect.exit) + assert.ok(Exit.isFailure(exit)) + assert.ok(findError(exit) instanceof VcsWebhookSignatureError) + }).pipe(Effect.provide(providerLayer())), + ) + + it.effect("maps an installation 'created' event to an installation-sync job", () => + Effect.gen(function* () { + const provider = yield* GithubProvider + const body = JSON.stringify({ action: "created", installation: { id: 99 } }) + const jobs = yield* provider.webhookToJobs({ + headers: { "x-github-event": "installation", "x-hub-signature-256": sign(body) }, + rawBody: body, + }) + assert.strictEqual(jobs.length, 1) + const job = jobs[0]! + assert.strictEqual(job.kind, "installation-sync") + if (job.kind !== "installation-sync") return + assert.strictEqual(job.reason, "created") + assert.strictEqual(job.externalInstallationId, "99") + }).pipe(Effect.provide(providerLayer())), + ) +}) + +describe("VcsRepository", () => { + it.effect("upserts + reads an installation and commits (validated)", () => { + const { url } = createTempDbUrl("maple-vcs-repo-", dirs) + return Effect.gen(function* () { + const repo = yield* VcsRepository + const orgId = asOrgId("org_test") + const installation = yield* repo.upsertInstallation({ + orgId, + provider: "github", + externalInstallationId: "42", + accountLogin: "octo", + accountType: "organization", + externalAccountId: "100", + accountAvatarUrl: null, + repositorySelection: "all", + installedByUserId: asUserId("user_1"), + }) + assert.strictEqual(installation.orgId, orgId) + assert.strictEqual(installation.accountType, "organization") + + const found = yield* repo.getInstallation("github", "42") + assert.ok(Option.isSome(found)) + assert.strictEqual(found.value.externalInstallationId, "42") + + const count = yield* repo.upsertCommits(orgId, "github", "7", [ + { + sha: SHA, + message: "hello", + authorName: "Octo", + authorEmail: null, + authorLogin: "octocat", + authorAvatarUrl: null, + authoredAt: null, + committedAt: 123, + htmlUrl: `https://github.com/octo/repo/commit/${SHA}`, + branch: "main", + }, + ]) + assert.strictEqual(count, 1) + + const commit = yield* repo.findCommitBySha(orgId, SHA as never) + assert.ok(Option.isSome(commit)) + assert.strictEqual(commit.value.shortSha, SHA.slice(0, 7)) + assert.strictEqual(commit.value.authorLogin, "octocat") + }).pipe(Effect.provide(repoLayer(url))) + }) + + it.effect("raises VcsRepoDecodeError when a row has an invalid enum", () => { + const { url, dbPath } = createTempDbUrl("maple-vcs-decode-", dirs) + return Effect.gen(function* () { + const repo = yield* VcsRepository + // Corrupt a row directly (account_type is not a valid VcsAccountType). + yield* Effect.promise(() => + executeSql( + dbPath, + `INSERT INTO vcs_installations + (id, org_id, provider, external_installation_id, account_login, account_type, + external_account_id, repository_selection, status, installed_by_user_id, created_at, updated_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`, + [randomUUID(), "org_x", "github", "55", "octo", "team", "1", "all", "active", "user_1", 0, 0], + ), + ) + const exit = yield* repo.getInstallation("github", "55").pipe(Effect.exit) + assert.ok(Exit.isFailure(exit)) + assert.ok(findError(exit) instanceof VcsRepoDecodeError) + }).pipe(Effect.provide(repoLayer(url))) + }) +}) + +describe("VcsSyncService orchestrator", () => { + const SHA_A = "a".repeat(40) + const SHA_B = "b".repeat(40) + + const commit = (sha: string, committedAt: number) => ({ + sha, + message: `commit ${sha.slice(0, 7)}`, + authorName: null, + authorEmail: null, + authorLogin: null, + authorAvatarUrl: null, + authoredAt: null, + committedAt, + htmlUrl: `https://github.com/o/r/commit/${sha}`, + branch: "main", + }) + + interface StubOpts { + readonly sent: Array + readonly repos?: ReadonlyArray<{ + externalRepoId: string + owner: string + name: string + fullName: string + defaultBranch: string + htmlUrl: string + isPrivate: boolean + isArchived: boolean + }> + readonly commits?: ReadonlyArray> + } + + // Real VcsRepository (temp D1) + stubbed provider/queue ports, so dispatch, + // cursor direction, and the drop guards are exercised against real persistence. + const orchestratorLayer = (url: string, opts: StubOpts) => { + const fakeProvider: VcsProviderClient = { + id: "github", + webhookToJobs: () => Effect.succeed([]), + fetchRepositories: () => Effect.succeed(opts.repos ?? []), + fetchCommits: () => Effect.succeed(opts.commits ?? []), + } + const registry = Layer.succeed(VcsProviderRegistry, { + ids: ["github"], + resolve: () => Effect.succeed(fakeProvider), + } satisfies VcsProviderRegistryShape) + const queue = Layer.succeed(VcsSyncQueue, { + send: (job) => Effect.sync(() => void opts.sent.push(job)), + sendBatch: (jobs) => Effect.sync(() => void opts.sent.push(...jobs)), + } satisfies VcsSyncQueueShape) + const repoLive = VcsRepository.layer.pipe( + Layer.provide(DatabaseLibsqlLive), + Layer.provide(envLayer(url)), + ) + return VcsSyncService.layer.pipe(Layer.provideMerge(Layer.mergeAll(repoLive, registry, queue))) + } + + const seedInstallation = (repo: VcsRepository, orgId: ReturnType) => + repo.upsertInstallation({ + orgId, + provider: "github", + externalInstallationId: "42", + accountLogin: "octo", + accountType: "organization", + externalAccountId: "100", + accountAvatarUrl: null, + repositorySelection: "all", + installedByUserId: asUserId("user_1"), + }) + + it.effect("drops a job for an unknown installation without persisting or failing", () => { + const { url } = createTempDbUrl("maple-vcs-orch-unknown-", dirs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + const job: VcsSyncJob = { + kind: "push-delta", + provider: "github", + externalInstallationId: "999", // never seeded + externalRepoId: "7", + branch: "main", + commits: [commit(SHA_A, 1)], + } + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) // must not fail + const found = yield* repo.findCommitBySha(orgId, SHA_A as never) + assert.ok(Option.isNone(found)) + assert.strictEqual(sent.length, 0) + }).pipe(Effect.provide(orchestratorLayer(url, { sent }))) + }) + + it.effect("push-delta upserts every commit", () => { + const { url } = createTempDbUrl("maple-vcs-orch-push-", dirs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + const job: VcsSyncJob = { + kind: "push-delta", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + branch: "main", + commits: [commit(SHA_A, 1), commit(SHA_B, 2)], + } + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) + const a = yield* repo.findCommitBySha(orgId, SHA_A as never) + const b = yield* repo.findCommitBySha(orgId, SHA_B as never) + assert.ok(Option.isSome(a) && Option.isSome(b)) + }).pipe(Effect.provide(orchestratorLayer(url, { sent }))) + }) + + it.effect("installation-sync upserts the provider's repos and enqueues a backfill per repo", () => { + const { url } = createTempDbUrl("maple-vcs-orch-inst-", dirs) + const sent: Array = [] + const repos = [ + { + externalRepoId: "7", + owner: "octo", + name: "repo", + fullName: "octo/repo", + defaultBranch: "main", + htmlUrl: "https://github.com/octo/repo", + isPrivate: true, + isArchived: false, + }, + ] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + const job: VcsSyncJob = { + kind: "installation-sync", + provider: "github", + externalInstallationId: "42", + reason: "created", + } + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) + const stored = yield* repo.listRepositoriesByInstallation("github", "42") + assert.strictEqual(stored.length, 1) + assert.strictEqual(stored[0]!.externalRepoId, "7") + assert.strictEqual(sent.length, 1) + assert.strictEqual(sent[0]!.kind, "backfill-repo") + }).pipe(Effect.provide(orchestratorLayer(url, { sent, repos }))) + }) + + it.effect("backfill sets the cursor to the head (first/newest) commit", () => { + const { url } = createTempDbUrl("maple-vcs-orch-backfill-", dirs) + const sent: Array = [] + // GitHub returns newest-first; fetchCommits[0] is the head. + const commits = [commit(SHA_B, 2), commit(SHA_A, 1)] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* repo.upsertRepositories(orgId, "github", "42", [ + { + externalRepoId: "7", + owner: "octo", + name: "repo", + fullName: "octo/repo", + defaultBranch: "main", + htmlUrl: "https://github.com/octo/repo", + isPrivate: true, + isArchived: false, + }, + ]) + const job: VcsSyncJob = { + kind: "backfill-repo", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + owner: "octo", + name: "repo", + defaultBranch: "main", + sinceMs: 0, + } + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) + const stored = yield* repo.listRepositoriesByInstallation("github", "42") + assert.strictEqual(stored[0]!.syncStatus, "ready") + assert.strictEqual(stored[0]!.lastSyncCursor, SHA_B) + }).pipe(Effect.provide(orchestratorLayer(url, { sent, commits }))) + }) +}) + +// The SHA-shape regex lives only in the GitCommitSha brand; these assert that +// validation fires at both the webhook decode boundary and on persistence. +describe("git SHA validation (branded type)", () => { + it.effect("webhook decode rejects a malformed commit SHA with VcsWebhookParseError", () => + Effect.gen(function* () { + const provider = yield* GithubProvider + const body = JSON.stringify({ + ref: "refs/heads/main", + repository: { id: 7, owner: { login: "octo" } }, + installation: { id: 42 }, + commits: [{ id: "not-a-real-sha", message: "x", url: "https://example.com" }], + }) + const exit = yield* provider + .webhookToJobs({ + headers: { "x-github-event": "push", "x-hub-signature-256": sign(body) }, + rawBody: body, + }) + .pipe(Effect.exit) + assert.ok(Exit.isFailure(exit)) + assert.ok(findError(exit) instanceof VcsWebhookParseError) + }).pipe(Effect.provide(providerLayer())), + ) + + it.effect("upsertCommits rejects a malformed SHA with VcsRepoDecodeError", () => { + const { url } = createTempDbUrl("maple-vcs-sha-", dirs) + return Effect.gen(function* () { + const repo = yield* VcsRepository + const orgId = asOrgId("org_sha") + const exit = yield* repo + .upsertCommits(orgId, "github", "7", [ + { + sha: "ABC", // not 40-char lowercase hex + message: "bad", + authorName: null, + authorEmail: null, + authorLogin: null, + authorAvatarUrl: null, + authoredAt: null, + committedAt: 1, + htmlUrl: "https://example.com", + branch: "main", + }, + ]) + .pipe(Effect.exit) + assert.ok(Exit.isFailure(exit)) + assert.ok(findError(exit) instanceof VcsRepoDecodeError) + }).pipe(Effect.provide(repoLayer(url))) + }) +}) diff --git a/apps/api/src/vcs-sync-runtime.ts b/apps/api/src/vcs-sync-runtime.ts new file mode 100644 index 000000000..9a283b63e --- /dev/null +++ b/apps/api/src/vcs-sync-runtime.ts @@ -0,0 +1,73 @@ +import type { MessageBatch } from "@cloudflare/workers-types" +import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" +import { WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" +import { Cause, Effect, Layer } from "effect" +import { DatabaseD1Live } from "./lib/DatabaseD1Live" +import { Env } from "./lib/Env" +import { GithubAppClient } from "./services/github/GithubAppClient" +import { GithubProvider } from "./services/github/GithubProvider" +import { VcsProviderRegistry } from "./services/vcs/VcsProviderRegistry" +import { VcsRepository } from "./services/vcs/VcsRepository" +import { VcsSyncQueue } from "./services/vcs/VcsSyncQueue" +import { VcsSyncService } from "./services/vcs/VcsSyncService" + +// --------------------------------------------------------------------------- +// Per-invocation runtime for the `VCS_SYNC_QUEUE` consumer. Mirrors the +// alerting worker's `buildLayer`: its own light layer graph (NOT the fetch +// path's MainLive) so the queue invocation stays within the startup CPU budget. +// --------------------------------------------------------------------------- + +const telemetry = MapleCloudflareSDK.make({ + serviceName: "maple-api", + serviceNamespace: "backend", + repositoryUrl: "https://github.com/Makisuo/maple", +}) + +export const buildVcsSyncLayer = (_env: Record) => { + const ConfigLive = WorkerConfigProviderLayer + const EnvLive = Env.layer.pipe(Layer.provide(ConfigLive)) + const DatabaseLive = DatabaseD1Live.pipe(Layer.provide(WorkerEnvironment.layer)) + const Base = Layer.mergeAll(EnvLive, DatabaseLive, WorkerEnvironment.layer) + + const VcsRepositoryLive = VcsRepository.layer.pipe(Layer.provide(Base)) + const GithubAppClientLive = GithubAppClient.layer.pipe(Layer.provide(EnvLive)) + const GithubProviderLive = GithubProvider.layer.pipe( + Layer.provide(Layer.mergeAll(EnvLive, GithubAppClientLive)), + ) + const VcsProviderRegistryLive = VcsProviderRegistry.layer.pipe(Layer.provide(GithubProviderLive)) + const VcsSyncQueueLive = VcsSyncQueue.layer.pipe(Layer.provide(WorkerEnvironment.layer)) + const VcsSyncServiceLive = VcsSyncService.layer.pipe( + Layer.provide(Layer.mergeAll(VcsRepositoryLive, VcsProviderRegistryLive, VcsSyncQueueLive)), + ) + + return VcsSyncServiceLive.pipe( + Layer.provideMerge(telemetry.layer), + Layer.provideMerge(ConfigLive), + ) +} + +export const flushVcsTelemetry = (env: Record) => telemetry.flush(env) + +export const processBatch = (batch: MessageBatch) => + Effect.gen(function* () { + const service = yield* VcsSyncService + yield* Effect.forEach( + batch.messages, + (message) => + service.processMessage(message.body).pipe( + Effect.matchCauseEffect({ + onFailure: (cause) => + Effect.logError("VCS sync message failed").pipe( + Effect.annotateLogs({ error: Cause.pretty(cause) }), + Effect.flatMap(() => Effect.sync(() => message.retry())), + ), + onSuccess: () => Effect.sync(() => message.ack()), + }), + ), + { discard: true }, + ) + }).pipe( + Effect.withSpan("VcsSyncQueue.processBatch", { + attributes: { "messaging.batch.message_count": batch.messages.length }, + }), + ) diff --git a/apps/api/src/worker.ts b/apps/api/src/worker.ts index 4e6e14a93..2d115919a 100644 --- a/apps/api/src/worker.ts +++ b/apps/api/src/worker.ts @@ -1,5 +1,6 @@ +import type { MessageBatch } from "@cloudflare/workers-types" import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" -import { WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" +import { runScheduledEffect, WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" import { Context, FileSystem, Layer, Path } from "effect" import { HttpMiddleware, HttpRouter } from "effect/unstable/http" import * as Etag from "effect/unstable/http/Etag" @@ -209,7 +210,25 @@ const handle = async ( export { ClickHouseSchemaApplyWorkflow } from "./workflows/ClickHouseSchemaApplyWorkflow" export { AiTriageWorkflow } from "./workflows/AiTriageWorkflow" +// VCS sync queue consumer. The runtime + layer graph are dynamic-imported (same +// startup-CPU-budget discipline as the route graph above): build a dedicated +// per-invocation layer and run the batch under it, then flush telemetry. +const handleQueue = async ( + batch: MessageBatch, + env: Record, + ctx: ExecutionContext, +): Promise => { + const { buildVcsSyncLayer, processBatch, flushVcsTelemetry } = await import("./vcs-sync-runtime") + try { + await runScheduledEffect(buildVcsSyncLayer(env), processBatch(batch), ctx) + } finally { + ctx.waitUntil(flushVcsTelemetry(env)) + } +} + export default { fetch: (request: Request, env: Record, ctx: ExecutionContext) => handle(request, env, ctx), + queue: (batch: MessageBatch, env: Record, ctx: ExecutionContext) => + handleQueue(batch, env, ctx), } diff --git a/apps/api/test/stubs/cloudflare-workers.ts b/apps/api/test/stubs/cloudflare-workers.ts new file mode 100644 index 000000000..1d969d766 --- /dev/null +++ b/apps/api/test/stubs/cloudflare-workers.ts @@ -0,0 +1,7 @@ +// Stub for the `cloudflare:workers` virtual module so worker-dependent code can +// be imported in the node test environment (vitest). Only the symbols that +// `@maple/effect-cloudflare`'s barrel statically imports are needed here +// (`DurableObject`, `WorkflowEntrypoint`); runtime worker behavior is never +// exercised in unit tests — services that read bindings are stubbed via layers. +export class DurableObject {} +export class WorkflowEntrypoint {} diff --git a/apps/api/vitest.config.ts b/apps/api/vitest.config.ts index deef11d14..d3f254af3 100644 --- a/apps/api/vitest.config.ts +++ b/apps/api/vitest.config.ts @@ -5,6 +5,11 @@ export default defineConfig({ resolve: { alias: { "@": fileURLToPath(new URL("./src", import.meta.url)), + // The `cloudflare:workers` virtual module only exists inside a Worker + // isolate; stub it so worker-dependent services can be imported in node. + "cloudflare:workers": fileURLToPath( + new URL("./test/stubs/cloudflare-workers.ts", import.meta.url), + ), }, }, test: { diff --git a/apps/api/wrangler.jsonc b/apps/api/wrangler.jsonc index 80b8d0bb3..e3d533cbf 100644 --- a/apps/api/wrangler.jsonc +++ b/apps/api/wrangler.jsonc @@ -35,4 +35,20 @@ "class_name": "AiTriageWorkflow", }, ], + // Declaring both a producer binding and a consumer on the same queue name + // makes miniflare run the VCS sync queue end-to-end in-process under + // `wrangler dev` — no remote queue, no deploy. (Mirrored in alchemy.run.ts + // for deploy.) DLQ is omitted locally so miniflare doesn't warn about an + // undeclared dead-letter queue. + "queues": { + "producers": [{ "binding": "VCS_SYNC_QUEUE", "queue": "maple-vcs-sync-local" }], + "consumers": [ + { + "queue": "maple-vcs-sync-local", + "max_batch_size": 10, + "max_batch_timeout": 5, + "max_retries": 3, + }, + ], + }, } diff --git a/packages/db/drizzle/0022_lumpy_iron_lad.sql b/packages/db/drizzle/0022_lumpy_iron_lad.sql new file mode 100644 index 000000000..3e3adb6cd --- /dev/null +++ b/packages/db/drizzle/0022_lumpy_iron_lad.sql @@ -0,0 +1,65 @@ +CREATE TABLE `vcs_commits` ( + `id` text PRIMARY KEY NOT NULL, + `org_id` text NOT NULL, + `provider` text NOT NULL, + `external_repo_id` text NOT NULL, + `sha` text NOT NULL, + `short_sha` text NOT NULL, + `message` text NOT NULL, + `author_name` text, + `author_email` text, + `author_login` text, + `author_avatar_url` text, + `authored_at` integer, + `committed_at` integer NOT NULL, + `html_url` text NOT NULL, + `branch` text, + `created_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `vcs_commits_org_repo_sha_idx` ON `vcs_commits` (`org_id`,`provider`,`external_repo_id`,`sha`);--> statement-breakpoint +CREATE INDEX `vcs_commits_org_sha_idx` ON `vcs_commits` (`org_id`,`sha`);--> statement-breakpoint +CREATE INDEX `vcs_commits_org_short_sha_idx` ON `vcs_commits` (`org_id`,`short_sha`);--> statement-breakpoint +CREATE TABLE `vcs_installations` ( + `id` text PRIMARY KEY NOT NULL, + `org_id` text NOT NULL, + `provider` text NOT NULL, + `external_installation_id` text NOT NULL, + `account_login` text NOT NULL, + `account_type` text NOT NULL, + `external_account_id` text NOT NULL, + `account_avatar_url` text, + `repository_selection` text DEFAULT 'all' NOT NULL, + `status` text DEFAULT 'active' NOT NULL, + `suspended_at` integer, + `installed_by_user_id` text NOT NULL, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `vcs_installations_provider_external_idx` ON `vcs_installations` (`provider`,`external_installation_id`);--> statement-breakpoint +CREATE INDEX `vcs_installations_org_idx` ON `vcs_installations` (`org_id`);--> statement-breakpoint +CREATE TABLE `vcs_repositories` ( + `id` text PRIMARY KEY NOT NULL, + `org_id` text NOT NULL, + `provider` text NOT NULL, + `external_installation_id` text NOT NULL, + `external_repo_id` text NOT NULL, + `owner` text NOT NULL, + `name` text NOT NULL, + `full_name` text NOT NULL, + `default_branch` text DEFAULT 'main' NOT NULL, + `html_url` text NOT NULL, + `is_private` integer DEFAULT 1 NOT NULL, + `is_archived` integer DEFAULT 0 NOT NULL, + `sync_status` text DEFAULT 'pending' NOT NULL, + `last_synced_at` integer, + `last_sync_cursor` text, + `last_sync_error` text, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `vcs_repositories_org_repo_idx` ON `vcs_repositories` (`org_id`,`provider`,`external_repo_id`);--> statement-breakpoint +CREATE INDEX `vcs_repositories_org_idx` ON `vcs_repositories` (`org_id`);--> statement-breakpoint +CREATE INDEX `vcs_repositories_installation_idx` ON `vcs_repositories` (`provider`,`external_installation_id`); \ No newline at end of file diff --git a/packages/db/drizzle/meta/0022_snapshot.json b/packages/db/drizzle/meta/0022_snapshot.json new file mode 100644 index 000000000..3edadc701 --- /dev/null +++ b/packages/db/drizzle/meta/0022_snapshot.json @@ -0,0 +1,4607 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "228d5a21-bac3-4115-8a05-bf1c7095686e", + "prevId": "31c9a36a-d77e-4e0f-a535-c71e671856bc", + "tables": { + "ai_triage_runs": { + "name": "ai_triage_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "incident_kind": { + "name": "incident_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "context_json": { + "name": "context_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "result_json": { + "name": "result_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "ai_triage_runs_incident_idx": { + "name": "ai_triage_runs_incident_idx", + "columns": [ + "org_id", + "incident_kind", + "incident_id" + ], + "isUnique": true + }, + "ai_triage_runs_org_issue_idx": { + "name": "ai_triage_runs_org_issue_idx", + "columns": [ + "org_id", + "issue_id" + ], + "isUnique": false + }, + "ai_triage_runs_org_created_idx": { + "name": "ai_triage_runs_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_triage_settings": { + "name": "ai_triage_settings", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "model_override": { + "name": "model_override", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_runs_per_day": { + "name": "max_runs_per_day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 20 + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "alert_delivery_events": { + "name": "alert_delivery_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt_number": { + "name": "attempt_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_message": { + "name": "provider_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_reference": { + "name": "provider_reference", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "alert_delivery_events_org_idx": { + "name": "alert_delivery_events_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "alert_delivery_events_org_incident_idx": { + "name": "alert_delivery_events_org_incident_idx", + "columns": [ + "org_id", + "incident_id" + ], + "isUnique": false + }, + "alert_delivery_events_due_idx": { + "name": "alert_delivery_events_due_idx", + "columns": [ + "status", + "scheduled_at" + ], + "isUnique": false + }, + "alert_delivery_events_claim_idx": { + "name": "alert_delivery_events_claim_idx", + "columns": [ + "status", + "claim_expires_at", + "scheduled_at" + ], + "isUnique": false + }, + "alert_delivery_events_delivery_attempt_idx": { + "name": "alert_delivery_events_delivery_attempt_idx", + "columns": [ + "delivery_key", + "attempt_number" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "alert_destinations": { + "name": "alert_destinations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "config_json": { + "name": "config_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_tested_at": { + "name": "last_tested_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_test_error": { + "name": "last_test_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "alert_destinations_org_idx": { + "name": "alert_destinations_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "alert_destinations_org_enabled_idx": { + "name": "alert_destinations_org_enabled_idx", + "columns": [ + "org_id", + "enabled" + ], + "isUnique": false + }, + "alert_destinations_org_name_idx": { + "name": "alert_destinations_org_name_idx", + "columns": [ + "org_id", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "alert_incidents": { + "name": "alert_incidents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "incident_key": { + "name": "incident_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rule_name": { + "name": "rule_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "threshold": { + "name": "threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivered_event_type": { + "name": "last_delivered_event_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_notified_at": { + "name": "last_notified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "alert_incidents_org_idx": { + "name": "alert_incidents_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "alert_incidents_org_status_idx": { + "name": "alert_incidents_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + }, + "alert_incidents_org_rule_idx": { + "name": "alert_incidents_org_rule_idx", + "columns": [ + "org_id", + "rule_id" + ], + "isUnique": false + }, + "alert_incidents_incident_key_idx": { + "name": "alert_incidents_incident_key_idx", + "columns": [ + "incident_key" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "alert_rule_states": { + "name": "alert_rule_states", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'__total__'" + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_value": { + "name": "last_value", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "alert_rule_states_org_idx": { + "name": "alert_rule_states_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "alert_rule_states_org_id_rule_id_group_key_pk": { + "columns": [ + "org_id", + "rule_id", + "group_key" + ], + "name": "alert_rule_states_org_id_rule_id_group_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "alert_rules": { + "name": "alert_rules", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notification_template_json": { + "name": "notification_template_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_names_json": { + "name": "service_names_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exclude_service_names_json": { + "name": "exclude_service_names_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "threshold": { + "name": "threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_minutes": { + "name": "window_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minimum_sample_count": { + "name": "minimum_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "consecutive_breaches_required": { + "name": "consecutive_breaches_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 2 + }, + "consecutive_healthy_required": { + "name": "consecutive_healthy_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 2 + }, + "renotify_interval_minutes": { + "name": "renotify_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 30 + }, + "metric_name": { + "name": "metric_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metric_type": { + "name": "metric_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metric_aggregation": { + "name": "metric_aggregation", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "apdex_threshold_ms": { + "name": "apdex_threshold_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "query_builder_draft_json": { + "name": "query_builder_draft_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_query_sql": { + "name": "raw_query_sql", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "group_by": { + "name": "group_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "query_spec_json": { + "name": "query_spec_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reducer": { + "name": "reducer", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sample_count_strategy": { + "name": "sample_count_strategy", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "no_data_behavior": { + "name": "no_data_behavior", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_scheduled_at": { + "name": "last_scheduled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "alert_rules_org_idx": { + "name": "alert_rules_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "alert_rules_org_enabled_idx": { + "name": "alert_rules_org_enabled_idx", + "columns": [ + "org_id", + "enabled" + ], + "isUnique": false + }, + "alert_rules_org_name_idx": { + "name": "alert_rules_org_name_idx", + "columns": [ + "org_id", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "anomaly_detector_settings": { + "name": "anomaly_detector_settings", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "sensitivity": { + "name": "sensitivity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'normal'" + }, + "muted_signals_json": { + "name": "muted_signals_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "last_tick_at": { + "name": "last_tick_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "anomaly_detector_states": { + "name": "anomaly_detector_states", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_value": { + "name": "last_value", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "baseline_median": { + "name": "baseline_median", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_resolved_at": { + "name": "last_resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "anomaly_detector_states_org_idx": { + "name": "anomaly_detector_states_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "anomaly_detector_states_open_incident_idx": { + "name": "anomaly_detector_states_open_incident_idx", + "columns": [ + "org_id", + "open_incident_id" + ], + "isUnique": false + }, + "anomaly_detector_states_evaluated_idx": { + "name": "anomaly_detector_states_evaluated_idx", + "columns": [ + "last_evaluated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "anomaly_detector_states_org_id_detector_key_pk": { + "columns": [ + "org_id", + "detector_key" + ], + "name": "anomaly_detector_states_org_id_detector_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "anomaly_incidents": { + "name": "anomaly_incidents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "opened_value": { + "name": "opened_value", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "baseline_median": { + "name": "baseline_median", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "baseline_sigma": { + "name": "baseline_sigma", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "threshold_value": { + "name": "threshold_value", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolve_reason": { + "name": "resolve_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "triage_status": { + "name": "triage_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "anomaly_incidents_org_status_idx": { + "name": "anomaly_incidents_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + }, + "anomaly_incidents_org_triggered_idx": { + "name": "anomaly_incidents_org_triggered_idx", + "columns": [ + "org_id", + "last_triggered_at" + ], + "isUnique": false + }, + "anomaly_incidents_org_detector_idx": { + "name": "anomaly_incidents_org_detector_idx", + "columns": [ + "org_id", + "detector_key" + ], + "isUnique": false + }, + "anomaly_incidents_org_issue_idx": { + "name": "anomaly_incidents_org_issue_idx", + "columns": [ + "org_id", + "error_issue_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_prefix": { + "name": "key_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revoked": { + "name": "revoked", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'standard'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by_email": { + "name": "created_by_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_org_id_idx": { + "name": "api_keys_org_id_idx", + "columns": [ + "org_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cloudflare_logpush_connectors": { + "name": "cloudflare_logpush_connectors", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'http_requests'" + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "last_received_at": { + "name": "last_received_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "secret_rotated_at": { + "name": "secret_rotated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cloudflare_logpush_connectors_org_idx": { + "name": "cloudflare_logpush_connectors_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "cloudflare_logpush_connectors_org_enabled_idx": { + "name": "cloudflare_logpush_connectors_org_enabled_idx", + "columns": [ + "org_id", + "enabled" + ], + "isUnique": false + }, + "cloudflare_logpush_connectors_secret_hash_unique": { + "name": "cloudflare_logpush_connectors_secret_hash_unique", + "columns": [ + "secret_hash" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dashboard_versions": { + "name": "dashboard_versions", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_number": { + "name": "version_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "change_kind": { + "name": "change_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_version_id": { + "name": "source_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "dashboard_versions_org_dashboard_idx": { + "name": "dashboard_versions_org_dashboard_idx", + "columns": [ + "org_id", + "dashboard_id", + "version_number" + ], + "isUnique": false + }, + "dashboard_versions_org_dashboard_version_unq": { + "name": "dashboard_versions_org_dashboard_version_unq", + "columns": [ + "org_id", + "dashboard_id", + "version_number" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboard_versions_org_id_id_pk": { + "columns": [ + "org_id", + "id" + ], + "name": "dashboard_versions_org_id_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dashboards": { + "name": "dashboards", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "dashboards_org_updated_idx": { + "name": "dashboards_org_updated_idx", + "columns": [ + "org_id", + "updated_at" + ], + "isUnique": false + }, + "dashboards_org_name_idx": { + "name": "dashboards_org_name_idx", + "columns": [ + "org_id", + "name" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboards_org_id_id_pk": { + "columns": [ + "org_id", + "id" + ], + "name": "dashboards_org_id_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "digest_subscriptions": { + "name": "digest_subscriptions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "day_of_week": { + "name": "day_of_week", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'UTC'" + }, + "last_sent_at": { + "name": "last_sent_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_attempted_at": { + "name": "last_attempted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "digest_subscriptions_org_user_idx": { + "name": "digest_subscriptions_org_user_idx", + "columns": [ + "org_id", + "user_id" + ], + "isUnique": true + }, + "digest_subscriptions_org_enabled_idx": { + "name": "digest_subscriptions_org_enabled_idx", + "columns": [ + "org_id", + "enabled" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "actors": { + "name": "actors", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent_name": { + "name": "agent_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "capabilities_json": { + "name": "capabilities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "actors_org_user_idx": { + "name": "actors_org_user_idx", + "columns": [ + "org_id", + "user_id" + ], + "isUnique": true + }, + "actors_org_agent_name_idx": { + "name": "actors_org_agent_name_idx", + "columns": [ + "org_id", + "agent_name" + ], + "isUnique": true + }, + "actors_org_type_idx": { + "name": "actors_org_type_idx", + "columns": [ + "org_id", + "type" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "error_incidents": { + "name": "error_incidents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "error_incidents_org_issue_idx": { + "name": "error_incidents_org_issue_idx", + "columns": [ + "org_id", + "issue_id" + ], + "isUnique": false + }, + "error_incidents_org_status_idx": { + "name": "error_incidents_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "error_issue_events": { + "name": "error_issue_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_state": { + "name": "from_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "to_state": { + "name": "to_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "error_issue_events_issue_idx": { + "name": "error_issue_events_issue_idx", + "columns": [ + "org_id", + "issue_id", + "created_at" + ], + "isUnique": false + }, + "error_issue_events_actor_idx": { + "name": "error_issue_events_actor_idx", + "columns": [ + "org_id", + "actor_id", + "created_at" + ], + "isUnique": false + }, + "error_issue_events_type_idx": { + "name": "error_issue_events_type_idx", + "columns": [ + "org_id", + "type", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "error_issue_states": { + "name": "error_issue_states", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_observed_occurrence_at": { + "name": "last_observed_occurrence_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "error_issue_states_org_idx": { + "name": "error_issue_states_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "error_issue_states_org_id_issue_id_pk": { + "columns": [ + "org_id", + "issue_id" + ], + "name": "error_issue_states_org_id_issue_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "error_issues": { + "name": "error_issues", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "exception_type": { + "name": "exception_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "exception_message": { + "name": "exception_message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error_label": { + "name": "error_label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "top_frame": { + "name": "top_frame", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workflow_state": { + "name": "workflow_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'triage'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 3 + }, + "assigned_actor_id": { + "name": "assigned_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_holder_actor_id": { + "name": "lease_holder_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_by_actor_id": { + "name": "resolved_by_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snooze_until": { + "name": "snooze_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archived_at": { + "name": "archived_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "error_issues_org_fp_idx": { + "name": "error_issues_org_fp_idx", + "columns": [ + "org_id", + "fingerprint_hash" + ], + "isUnique": true + }, + "error_issues_org_workflow_idx": { + "name": "error_issues_org_workflow_idx", + "columns": [ + "org_id", + "workflow_state" + ], + "isUnique": false + }, + "error_issues_org_last_seen_idx": { + "name": "error_issues_org_last_seen_idx", + "columns": [ + "org_id", + "last_seen_at" + ], + "isUnique": false + }, + "error_issues_org_assignee_idx": { + "name": "error_issues_org_assignee_idx", + "columns": [ + "org_id", + "assigned_actor_id" + ], + "isUnique": false + }, + "error_issues_lease_expiry_idx": { + "name": "error_issues_lease_expiry_idx", + "columns": [ + "lease_expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "error_notification_policies": { + "name": "error_notification_policies", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "notify_on_first_seen": { + "name": "notify_on_first_seen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "notify_on_regression": { + "name": "notify_on_regression", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "notify_on_resolve": { + "name": "notify_on_resolve", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "notify_on_transition_in_review": { + "name": "notify_on_transition_in_review", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "notify_on_transition_done": { + "name": "notify_on_transition_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "notify_on_claim": { + "name": "notify_on_claim", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "min_occurrence_count": { + "name": "min_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'warning'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_auth_states": { + "name": "oauth_auth_states", + "columns": { + "state": { + "name": "state", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "initiated_by_user_id": { + "name": "initiated_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_uri": { + "name": "redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "return_to": { + "name": "return_to", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "oauth_auth_states_expires_idx": { + "name": "oauth_auth_states_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_connections": { + "name": "oauth_connections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_user_id": { + "name": "external_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_user_email": { + "name": "external_user_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "access_token_ciphertext": { + "name": "access_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token_iv": { + "name": "access_token_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token_tag": { + "name": "access_token_tag", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token_ciphertext": { + "name": "refresh_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_iv": { + "name": "refresh_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_tag": { + "name": "refresh_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "oauth_connections_org_provider_idx": { + "name": "oauth_connections_org_provider_idx", + "columns": [ + "org_id", + "provider" + ], + "isUnique": true + }, + "oauth_connections_org_idx": { + "name": "oauth_connections_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_onboarding_state": { + "name": "org_onboarding_state", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "demo_data_requested": { + "name": "demo_data_requested", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checklist_dismissed_at": { + "name": "checklist_dismissed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "first_data_received_at": { + "name": "first_data_received_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "welcome_email_sent_at": { + "name": "welcome_email_sent_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "connect_nudge_email_sent_at": { + "name": "connect_nudge_email_sent_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalled_email_sent_at": { + "name": "stalled_email_sent_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "activation_email_sent_at": { + "name": "activation_email_sent_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_ingest_attribute_mappings": { + "name": "org_ingest_attribute_mappings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_context": { + "name": "source_context", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_key": { + "name": "target_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "org_ingest_attribute_mappings_org_idx": { + "name": "org_ingest_attribute_mappings_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_recommendation_issues": { + "name": "org_recommendation_issues", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recommendation_key": { + "name": "recommendation_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_key": { + "name": "canonical_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "usage_count": { + "name": "usage_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "opened_at": { + "name": "opened_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "org_recommendation_issues_org_idx": { + "name": "org_recommendation_issues_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "org_recommendation_issues_org_key_idx": { + "name": "org_recommendation_issues_org_key_idx", + "columns": [ + "org_id", + "recommendation_key" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_ingest_keys": { + "name": "org_ingest_keys", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_key_hash": { + "name": "public_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "private_key_ciphertext": { + "name": "private_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "private_key_iv": { + "name": "private_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "private_key_tag": { + "name": "private_key_tag", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "private_key_hash": { + "name": "private_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_rotated_at": { + "name": "public_rotated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "private_rotated_at": { + "name": "private_rotated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "org_ingest_keys_public_key_unique": { + "name": "org_ingest_keys_public_key_unique", + "columns": [ + "public_key" + ], + "isUnique": true + }, + "org_ingest_keys_public_key_hash_unique": { + "name": "org_ingest_keys_public_key_hash_unique", + "columns": [ + "public_key_hash" + ], + "isUnique": true + }, + "org_ingest_keys_private_key_hash_unique": { + "name": "org_ingest_keys_private_key_hash_unique", + "columns": [ + "private_key_hash" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_ingest_sampling_policies": { + "name": "org_ingest_sampling_policies", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "trace_sample_ratio": { + "name": "trace_sample_ratio", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "always_keep_error_spans": { + "name": "always_keep_error_spans", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "always_keep_slow_spans_ms": { + "name": "always_keep_slow_spans_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_openrouter_settings": { + "name": "org_openrouter_settings", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_tag": { + "name": "api_key_tag", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_clickhouse_settings": { + "name": "org_clickhouse_settings", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ch_url": { + "name": "ch_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ch_user": { + "name": "ch_user", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ch_password_ciphertext": { + "name": "ch_password_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ch_password_iv": { + "name": "ch_password_iv", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ch_password_tag": { + "name": "ch_password_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ch_database": { + "name": "ch_database", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schema_version": { + "name": "schema_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_clickhouse_schema_apply_runs": { + "name": "org_clickhouse_schema_apply_runs", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workflow_instance_id": { + "name": "workflow_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "current_migration": { + "name": "current_migration", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "steps_total": { + "name": "steps_total", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "steps_done": { + "name": "steps_done", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "applied_versions": { + "name": "applied_versions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "skipped": { + "name": "skipped", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scrape_target_checks": { + "name": "scrape_target_checks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sub_target_key": { + "name": "sub_target_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "samples_scraped": { + "name": "samples_scraped", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "samples_post_relabel": { + "name": "samples_post_relabel", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "scrape_target_checks_target_checked_idx": { + "name": "scrape_target_checks_target_checked_idx", + "columns": [ + "target_id", + "checked_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "scrape_target_checks_target_id_scrape_targets_id_fk": { + "name": "scrape_target_checks_target_id_scrape_targets_id_fk", + "tableFrom": "scrape_target_checks", + "tableTo": "scrape_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scrape_targets": { + "name": "scrape_targets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'prometheus'" + }, + "discovery_config_json": { + "name": "discovery_config_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scrape_interval_seconds": { + "name": "scrape_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 15 + }, + "labels_json": { + "name": "labels_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "auth_credentials_ciphertext": { + "name": "auth_credentials_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "auth_credentials_iv": { + "name": "auth_credentials_iv", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "auth_credentials_tag": { + "name": "auth_credentials_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "last_scrape_at": { + "name": "last_scrape_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_scrape_error": { + "name": "last_scrape_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "scrape_targets_org_idx": { + "name": "scrape_targets_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "scrape_targets_org_enabled_idx": { + "name": "scrape_targets_org_enabled_idx", + "columns": [ + "org_id", + "enabled" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "vcs_commits": { + "name": "vcs_commits", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_repo_id": { + "name": "external_repo_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "short_sha": { + "name": "short_sha", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author_name": { + "name": "author_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author_email": { + "name": "author_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author_avatar_url": { + "name": "author_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authored_at": { + "name": "authored_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "committed_at": { + "name": "committed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "vcs_commits_org_repo_sha_idx": { + "name": "vcs_commits_org_repo_sha_idx", + "columns": [ + "org_id", + "provider", + "external_repo_id", + "sha" + ], + "isUnique": true + }, + "vcs_commits_org_sha_idx": { + "name": "vcs_commits_org_sha_idx", + "columns": [ + "org_id", + "sha" + ], + "isUnique": false + }, + "vcs_commits_org_short_sha_idx": { + "name": "vcs_commits_org_short_sha_idx", + "columns": [ + "org_id", + "short_sha" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "vcs_installations": { + "name": "vcs_installations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_installation_id": { + "name": "external_installation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_login": { + "name": "account_login", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_account_id": { + "name": "external_account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_avatar_url": { + "name": "account_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repository_selection": { + "name": "repository_selection", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'all'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "suspended_at": { + "name": "suspended_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "installed_by_user_id": { + "name": "installed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "vcs_installations_provider_external_idx": { + "name": "vcs_installations_provider_external_idx", + "columns": [ + "provider", + "external_installation_id" + ], + "isUnique": true + }, + "vcs_installations_org_idx": { + "name": "vcs_installations_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "vcs_repositories": { + "name": "vcs_repositories", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_installation_id": { + "name": "external_installation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_repo_id": { + "name": "external_repo_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'main'" + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_private": { + "name": "is_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "is_archived": { + "name": "is_archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_sync_cursor": { + "name": "last_sync_cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "vcs_repositories_org_repo_idx": { + "name": "vcs_repositories_org_repo_idx", + "columns": [ + "org_id", + "provider", + "external_repo_id" + ], + "isUnique": true + }, + "vcs_repositories_org_idx": { + "name": "vcs_repositories_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "vcs_repositories_installation_idx": { + "name": "vcs_repositories_installation_idx", + "columns": [ + "provider", + "external_installation_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index bc9f4e80e..4d7352372 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -155,6 +155,13 @@ "when": 1781177263154, "tag": "0021_flimsy_random", "breakpoints": true + }, + { + "idx": 22, + "version": "6", + "when": 1781283099347, + "tag": "0022_lumpy_iron_lad", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/d1-limits.ts b/packages/db/src/d1-limits.ts new file mode 100644 index 000000000..e5f08ef6e --- /dev/null +++ b/packages/db/src/d1-limits.ts @@ -0,0 +1,37 @@ +import { getTableColumns } from "drizzle-orm" +import type { SQLiteTable } from "drizzle-orm/sqlite-core" + +// --------------------------------------------------------------------------- +// Cloudflare D1 bound-parameter limits. +// +// D1 caps bound parameters at 100 per SQL statement, and the limit applies to +// each statement (including each statement inside a `db.batch([...])`): +// https://developers.cloudflare.com/d1/platform/limits/ +// +// A multi-row `INSERT ... VALUES (...), (...)` binds (rows × columns) +// parameters, so bulk inserts must be chunked. Derive the chunk size from the +// table's LIVE column count rather than a hand-counted magic number — that way +// adding a column can never silently push a chunk past the cap. +// --------------------------------------------------------------------------- +const D1_MAX_BOUND_PARAMS = 100 + +/** + * Maximum number of rows that fit in a single multi-row INSERT into `table` + * without exceeding D1's bound-parameter cap. Conservative: uses the table's + * full column count (a superset of the columns any given row binds). + */ +export const maxRowsPerInsert = (table: SQLiteTable): number => + Math.max(1, Math.floor(D1_MAX_BOUND_PARAMS / Object.keys(getTableColumns(table)).length)) + +/** + * Split `rows` into chunks small enough that each multi-row INSERT into `table` + * stays within D1's bound-parameter cap. Use with `Effect.forEach(..., { discard: true })`. + */ +export const chunkRowsForInsert = (table: SQLiteTable, rows: ReadonlyArray): Array> => { + const size = maxRowsPerInsert(table) + const chunks: Array> = [] + for (let i = 0; i < rows.length; i += size) { + chunks.push(rows.slice(i, i + size)) + } + return chunks +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 8237300ae..ef420c652 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -2,6 +2,7 @@ export * from "./api-key-hash" export * from "./client" export * from "./cloudflare-logpush-secrets" export * from "./config" +export * from "./d1-limits" export * from "./ingest-key-hash" export * from "./migrate" export * from "./schema" diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index d754aeeb4..2faeb9bda 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -16,3 +16,4 @@ export * from "./org-openrouter-settings" export * from "./org-clickhouse-settings" export * from "./org-clickhouse-schema-apply-runs" export * from "./scrape-targets" +export * from "./vcs" diff --git a/packages/db/src/schema/vcs.ts b/packages/db/src/schema/vcs.ts new file mode 100644 index 000000000..3d029bb5f --- /dev/null +++ b/packages/db/src/schema/vcs.ts @@ -0,0 +1,118 @@ +import { index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import type { OrgId, UserId } from "@maple/domain/primitives" +import type { + GitCommitSha, + ShortCommitSha, + VcsAccountType, + VcsCommitRowId, + VcsInstallStatus, + VcsInstallationId, + VcsProviderId, + VcsRepoSelection, + VcsRepoSyncStatus, + VcsRepositoryId, +} from "@maple/domain/http" + +// --------------------------------------------------------------------------- +// Vendor-agnostic VCS integration tables. Every row carries a `provider` +// discriminator; GitHub-specific concepts never reach this layer. External +// provider ids (installation/repo/account) are stored as TEXT for +// cross-provider generality. Timestamps are epoch milliseconds. +// +// IMPORTANT: only `VcsRepository` (apps/api/src/services/vcs/VcsRepository.ts) +// may import these tables. All other code goes through that repo service. +// --------------------------------------------------------------------------- + +/** One row per provider App installation, owned by the initiating Maple org. */ +export const vcsInstallations = sqliteTable( + "vcs_installations", + { + id: text("id").$type().notNull().primaryKey(), + orgId: text("org_id").$type().notNull(), + provider: text("provider").$type().notNull(), + externalInstallationId: text("external_installation_id").notNull(), + accountLogin: text("account_login").notNull(), + accountType: text("account_type").$type().notNull(), + externalAccountId: text("external_account_id").notNull(), + accountAvatarUrl: text("account_avatar_url"), + repositorySelection: text("repository_selection").$type().notNull().default("all"), + status: text("status").$type().notNull().default("active"), + suspendedAt: integer("suspended_at", { mode: "number" }), + installedByUserId: text("installed_by_user_id").$type().notNull(), + createdAt: integer("created_at", { mode: "number" }).notNull(), + updatedAt: integer("updated_at", { mode: "number" }).notNull(), + }, + (table) => [ + uniqueIndex("vcs_installations_provider_external_idx").on(table.provider, table.externalInstallationId), + index("vcs_installations_org_idx").on(table.orgId), + ], +) + +/** Repositories accessible to an installation, plus a per-repo sync cursor. */ +export const vcsRepositories = sqliteTable( + "vcs_repositories", + { + id: text("id").$type().notNull().primaryKey(), + orgId: text("org_id").$type().notNull(), + provider: text("provider").$type().notNull(), + externalInstallationId: text("external_installation_id").notNull(), + externalRepoId: text("external_repo_id").notNull(), + owner: text("owner").notNull(), + name: text("name").notNull(), + fullName: text("full_name").notNull(), + defaultBranch: text("default_branch").notNull().default("main"), + htmlUrl: text("html_url").notNull(), + isPrivate: integer("is_private", { mode: "number" }).notNull().default(1), + isArchived: integer("is_archived", { mode: "number" }).notNull().default(0), + syncStatus: text("sync_status").$type().notNull().default("pending"), + lastSyncedAt: integer("last_synced_at", { mode: "number" }), + lastSyncCursor: text("last_sync_cursor"), + lastSyncError: text("last_sync_error"), + createdAt: integer("created_at", { mode: "number" }).notNull(), + updatedAt: integer("updated_at", { mode: "number" }).notNull(), + }, + (table) => [ + uniqueIndex("vcs_repositories_org_repo_idx").on(table.orgId, table.provider, table.externalRepoId), + index("vcs_repositories_org_idx").on(table.orgId), + index("vcs_repositories_installation_idx").on(table.provider, table.externalInstallationId), + ], +) + +/** + * Resolved commits. The dashboard resolver matches a trace's full 40-char SHA + * by `(org_id, sha)` — provider-agnostic. The row is self-contained + * (`html_url` + author fields) so the resolver needs no join. + */ +export const vcsCommits = sqliteTable( + "vcs_commits", + { + id: text("id").$type().notNull().primaryKey(), + orgId: text("org_id").$type().notNull(), + provider: text("provider").$type().notNull(), + externalRepoId: text("external_repo_id").notNull(), + sha: text("sha").$type().notNull(), + shortSha: text("short_sha").$type().notNull(), + message: text("message").notNull(), + authorName: text("author_name"), + authorEmail: text("author_email"), + authorLogin: text("author_login"), + authorAvatarUrl: text("author_avatar_url"), + authoredAt: integer("authored_at", { mode: "number" }), + committedAt: integer("committed_at", { mode: "number" }).notNull(), + htmlUrl: text("html_url").notNull(), + branch: text("branch"), + createdAt: integer("created_at", { mode: "number" }).notNull(), + }, + (table) => [ + uniqueIndex("vcs_commits_org_repo_sha_idx").on(table.orgId, table.provider, table.externalRepoId, table.sha), + index("vcs_commits_org_sha_idx").on(table.orgId, table.sha), + index("vcs_commits_org_short_sha_idx").on(table.orgId, table.shortSha), + ], +) + +export type VcsInstallationRow = typeof vcsInstallations.$inferSelect +export type VcsInstallationInsert = typeof vcsInstallations.$inferInsert +export type VcsRepositoryRow = typeof vcsRepositories.$inferSelect +export type VcsRepositoryInsert = typeof vcsRepositories.$inferInsert +export type VcsCommitRow = typeof vcsCommits.$inferSelect +export type VcsCommitInsert = typeof vcsCommits.$inferInsert diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index a9250936f..d53029a19 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -24,4 +24,5 @@ export * from "./recommendation-issues" export * from "./scrape-targets" export * from "./scraper-internal" export * from "./session-replay" +export * from "./vcs" export * from "./warehouse" diff --git a/packages/domain/src/http/vcs.ts b/packages/domain/src/http/vcs.ts new file mode 100644 index 000000000..ca2799d7a --- /dev/null +++ b/packages/domain/src/http/vcs.ts @@ -0,0 +1,284 @@ +import { Schema } from "effect" +import { OrgId, UserId } from "../primitives" + +// --------------------------------------------------------------------------- +// Vendor-agnostic VCS integration types. +// +// Everything here is provider-neutral: rows carry a `provider` discriminator +// and GitHub-specific concepts (App auth, REST/webhook payload shapes) live in +// the GitHub layer behind the `VcsProviderClient` port. Adding another provider +// means extending `VcsProviderId` + the enum normalizations — no new tables. +// --------------------------------------------------------------------------- + +// ---- Branded IDs ---------------------------------------------------------- + +export const VcsInstallationId = Schema.String.check(Schema.isUUID()).pipe( + Schema.brand("@maple/VcsInstallationId"), + Schema.annotate({ identifier: "@maple/VcsInstallationId", title: "VCS Installation ID" }), +) +export type VcsInstallationId = Schema.Schema.Type + +export const VcsRepositoryId = Schema.String.check(Schema.isUUID()).pipe( + Schema.brand("@maple/VcsRepositoryId"), + Schema.annotate({ identifier: "@maple/VcsRepositoryId", title: "VCS Repository ID" }), +) +export type VcsRepositoryId = Schema.Schema.Type + +export const VcsCommitRowId = Schema.String.check(Schema.isUUID()).pipe( + Schema.brand("@maple/VcsCommitRowId"), + Schema.annotate({ identifier: "@maple/VcsCommitRowId", title: "VCS Commit Row ID" }), +) +export type VcsCommitRowId = Schema.Schema.Type + +/** + * A full 40-char, lowercase-hex git commit SHA. Strict — unlike the permissive + * telemetry `CommitSha` brand (which must not throw on arbitrary OTel data) — + * so the SHA-shape regex lives in exactly this one declarative type. Decoding a + * value through it (at the webhook/REST boundary and on persistence) is the only + * SHA validation in the codebase. + */ +export const GitCommitSha = Schema.String.check(Schema.isPattern(/^[0-9a-f]{40}$/)).pipe( + Schema.brand("@maple/GitCommitSha"), + Schema.annotate({ identifier: "@maple/GitCommitSha", title: "Git Commit SHA" }), +) +export type GitCommitSha = Schema.Schema.Type + +/** First 7 hex chars of a commit SHA (display + abbreviated-input lookup). */ +export const ShortCommitSha = Schema.String.check(Schema.isPattern(/^[0-9a-f]{7}$/)).pipe( + Schema.brand("@maple/ShortCommitSha"), + Schema.annotate({ identifier: "@maple/ShortCommitSha", title: "Short Commit SHA" }), +) +export type ShortCommitSha = Schema.Schema.Type + +// ---- Provider + normalized enums ------------------------------------------ + +/** The set of supported VCS providers. Extend this array to add a provider. */ +export const VcsProviderId = Schema.Literals(["github"]).annotate({ + identifier: "@maple/VcsProviderId", + title: "VCS Provider", +}) +export type VcsProviderId = Schema.Schema.Type + +export const VcsAccountType = Schema.Literals(["organization", "user"]).annotate({ + identifier: "@maple/VcsAccountType", + title: "VCS Account Type", +}) +export type VcsAccountType = Schema.Schema.Type + +export const VcsInstallStatus = Schema.Literals(["active", "suspended", "disconnected"]).annotate({ + identifier: "@maple/VcsInstallStatus", + title: "VCS Installation Status", +}) +export type VcsInstallStatus = Schema.Schema.Type + +export const VcsRepoSelection = Schema.Literals(["all", "selected"]).annotate({ + identifier: "@maple/VcsRepoSelection", + title: "VCS Repository Selection", +}) +export type VcsRepoSelection = Schema.Schema.Type + +export const VcsRepoSyncStatus = Schema.Literals(["pending", "backfilling", "ready", "error"]).annotate({ + identifier: "@maple/VcsRepoSyncStatus", + title: "VCS Repository Sync Status", +}) +export type VcsRepoSyncStatus = Schema.Schema.Type + +// ---- Row → domain models (validated reads) -------------------------------- + +export class VcsInstallation extends Schema.Class("VcsInstallation")({ + id: VcsInstallationId, + orgId: OrgId, + provider: VcsProviderId, + externalInstallationId: Schema.String, + accountLogin: Schema.String, + accountType: VcsAccountType, + externalAccountId: Schema.String, + accountAvatarUrl: Schema.NullOr(Schema.String), + repositorySelection: VcsRepoSelection, + status: VcsInstallStatus, + suspendedAt: Schema.NullOr(Schema.Number), + installedByUserId: UserId, + createdAt: Schema.Number, + updatedAt: Schema.Number, +}) {} + +export class VcsRepo extends Schema.Class("VcsRepo")({ + id: VcsRepositoryId, + orgId: OrgId, + provider: VcsProviderId, + externalInstallationId: Schema.String, + externalRepoId: Schema.String, + owner: Schema.String, + name: Schema.String, + fullName: Schema.String, + defaultBranch: Schema.String, + htmlUrl: Schema.String, + isPrivate: Schema.Boolean, + isArchived: Schema.Boolean, + syncStatus: VcsRepoSyncStatus, + lastSyncedAt: Schema.NullOr(Schema.Number), + lastSyncCursor: Schema.NullOr(Schema.String), + lastSyncError: Schema.NullOr(Schema.String), + createdAt: Schema.Number, + updatedAt: Schema.Number, +}) {} + +export class VcsCommit extends Schema.Class("VcsCommit")({ + id: VcsCommitRowId, + orgId: OrgId, + provider: VcsProviderId, + externalRepoId: Schema.String, + sha: GitCommitSha, + shortSha: ShortCommitSha, + message: Schema.String, + authorName: Schema.NullOr(Schema.String), + authorEmail: Schema.NullOr(Schema.String), + authorLogin: Schema.NullOr(Schema.String), + authorAvatarUrl: Schema.NullOr(Schema.String), + authoredAt: Schema.NullOr(Schema.Number), + committedAt: Schema.Number, + htmlUrl: Schema.String, + branch: Schema.NullOr(Schema.String), + createdAt: Schema.Number, +}) {} + +// ---- Boundary input DTOs (provider → repo / queue) ------------------------ + +/** Normalized repository, returned by a provider and persisted by the repo. */ +export const RepoUpsertInput = Schema.Struct({ + externalRepoId: Schema.String, + owner: Schema.String, + name: Schema.String, + fullName: Schema.String, + defaultBranch: Schema.String, + htmlUrl: Schema.String, + isPrivate: Schema.Boolean, + isArchived: Schema.Boolean, +}) +export type RepoUpsertInput = Schema.Schema.Type + +/** Normalized commit, returned by a provider (or extracted from a push). */ +export const CommitUpsertInput = Schema.Struct({ + sha: Schema.String, + message: Schema.String, + authorName: Schema.NullOr(Schema.String), + authorEmail: Schema.NullOr(Schema.String), + authorLogin: Schema.NullOr(Schema.String), + authorAvatarUrl: Schema.NullOr(Schema.String), + authoredAt: Schema.NullOr(Schema.Number), + committedAt: Schema.Number, + htmlUrl: Schema.String, + branch: Schema.NullOr(Schema.String), +}) +export type CommitUpsertInput = Schema.Schema.Type + +/** Minimal repo identity a provider needs to fetch commits. */ +export const VcsRepositoryRef = Schema.Struct({ + externalRepoId: Schema.String, + owner: Schema.String, + name: Schema.String, + defaultBranch: Schema.String, +}) +export type VcsRepositoryRef = Schema.Schema.Type + +// ---- Queue jobs (vendor-agnostic; orgId resolved by the orchestrator) ------ + +export const VcsInstallationSyncReason = Schema.Literals([ + "created", + "unsuspend", + "repositories_added", + "repositories_removed", + "suspend", + "deleted", +]).annotate({ identifier: "@maple/VcsInstallationSyncReason", title: "VCS Installation Sync Reason" }) +export type VcsInstallationSyncReason = Schema.Schema.Type + +// Jobs carry only `externalInstallationId` (+ provider); the sync orchestrator +// resolves `orgId` from the installation row. A webhook handler has no DB +// access and cannot know the Maple org, so it must not be carried here. +export const InstallationSyncJob = Schema.Struct({ + kind: Schema.Literal("installation-sync"), + provider: VcsProviderId, + externalInstallationId: Schema.String, + reason: VcsInstallationSyncReason, +}) +export type InstallationSyncJob = Schema.Schema.Type + +export const BackfillRepoJob = Schema.Struct({ + kind: Schema.Literal("backfill-repo"), + provider: VcsProviderId, + externalInstallationId: Schema.String, + externalRepoId: Schema.String, + owner: Schema.String, + name: Schema.String, + defaultBranch: Schema.String, + sinceMs: Schema.Number, +}) +export type BackfillRepoJob = Schema.Schema.Type + +export const PushDeltaJob = Schema.Struct({ + kind: Schema.Literal("push-delta"), + provider: VcsProviderId, + externalInstallationId: Schema.String, + externalRepoId: Schema.String, + branch: Schema.String, + commits: Schema.Array(CommitUpsertInput), +}) +export type PushDeltaJob = Schema.Schema.Type + +export const VcsSyncJob = Schema.Union([InstallationSyncJob, BackfillRepoJob, PushDeltaJob]) +export type VcsSyncJob = Schema.Schema.Type + +// ---- Tagged errors -------------------------------------------------------- + +export class VcsRepoPersistenceError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsRepoPersistenceError", + { message: Schema.String }, + { httpApiStatus: 503 }, +) {} + +export class VcsRepoDecodeError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsRepoDecodeError", + { message: Schema.String, table: Schema.String, column: Schema.optional(Schema.String) }, + { httpApiStatus: 500 }, +) {} + +export class VcsQueueError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsQueueError", + { message: Schema.String }, + { httpApiStatus: 503 }, +) {} + +export class VcsProviderError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsProviderError", + { + message: Schema.String, + status: Schema.optional(Schema.Number), + cause: Schema.optionalKey(Schema.Defect), + }, + { httpApiStatus: 502 }, +) {} + +export class VcsWebhookSignatureError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsWebhookSignatureError", + { message: Schema.String }, + { httpApiStatus: 401 }, +) {} + +export class VcsWebhookParseError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsWebhookParseError", + { message: Schema.String }, + { httpApiStatus: 400 }, +) {} + +export class UnknownVcsProviderError extends Schema.TaggedErrorClass()( + "@maple/http/errors/UnknownVcsProviderError", + { provider: Schema.String, message: Schema.String }, + { httpApiStatus: 404 }, +) {} + +export class OAuthStatePersistenceError extends Schema.TaggedErrorClass()( + "@maple/http/errors/OAuthStatePersistenceError", + { message: Schema.String }, + { httpApiStatus: 503 }, +) {} From 99f0abc4abc78da92d6cc282bd64294ab3bc0706 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Sat, 13 Jun 2026 02:11:22 +0200 Subject: [PATCH 02/45] Refactor VscRepository --- apps/api/src/services/vcs/VcsRepository.ts | 17 ++++++++++------- apps/api/src/services/vcs/__tests__/vcs.test.ts | 2 ++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/apps/api/src/services/vcs/VcsRepository.ts b/apps/api/src/services/vcs/VcsRepository.ts index f989b30b0..92b7e2a3f 100644 --- a/apps/api/src/services/vcs/VcsRepository.ts +++ b/apps/api/src/services/vcs/VcsRepository.ts @@ -112,6 +112,10 @@ const rowToCommit = (row: VcsCommitRow): VcsCommit => createdAt: row.createdAt, }) +// Note: `status` is intentionally not part of the upsert input. A new row gets +// the schema column default ("active"); all status transitions (suspend / +// disconnect / unsuspend) go through `markInstallationStatus`, so a reconciling +// upsert never touches an existing installation's status. export interface UpsertInstallationInput { readonly orgId: OrgId readonly provider: VcsProviderId @@ -121,7 +125,6 @@ export interface UpsertInstallationInput { readonly externalAccountId: string readonly accountAvatarUrl: string | null readonly repositorySelection: VcsRepoSelection - readonly status?: VcsInstallStatus readonly installedByUserId: UserId } @@ -179,6 +182,9 @@ export class VcsRepository extends Context.Service()("@maple/api/ .execute((db) => db .insert(vcsInstallations) + // `status`/`suspended_at` are omitted: a new row takes the schema + // default ("active"), and on conflict they are left untouched so a + // reconcile can't un-suspend. Status is owned by markInstallationStatus. .values({ id: randomUUID() as VcsInstallation["id"], orgId: input.orgId, @@ -189,24 +195,21 @@ export class VcsRepository extends Context.Service()("@maple/api/ externalAccountId: input.externalAccountId, accountAvatarUrl: input.accountAvatarUrl, repositorySelection: input.repositorySelection, - status: input.status ?? "active", - suspendedAt: null, installedByUserId: input.installedByUserId, createdAt: now, updatedAt: now, }) .onConflictDoUpdate({ target: [vcsInstallations.provider, vcsInstallations.externalInstallationId], - // Ownership columns (org_id, installed_by_user_id, created_at) are - // immutable on conflict — only mutable provider metadata is refreshed. + // Ownership columns (org_id, installed_by_user_id, created_at) and + // status/suspended_at are immutable on conflict — only mutable + // provider metadata is refreshed. set: { accountLogin: sql`excluded.account_login`, accountType: sql`excluded.account_type`, externalAccountId: sql`excluded.external_account_id`, accountAvatarUrl: sql`excluded.account_avatar_url`, repositorySelection: sql`excluded.repository_selection`, - status: sql`excluded.status`, - suspendedAt: sql`excluded.suspended_at`, updatedAt: sql`excluded.updated_at`, }, }), diff --git a/apps/api/src/services/vcs/__tests__/vcs.test.ts b/apps/api/src/services/vcs/__tests__/vcs.test.ts index 1fd7a07f0..53e826f39 100644 --- a/apps/api/src/services/vcs/__tests__/vcs.test.ts +++ b/apps/api/src/services/vcs/__tests__/vcs.test.ts @@ -184,6 +184,8 @@ describe("VcsRepository", () => { const found = yield* repo.getInstallation("github", "42") assert.ok(Option.isSome(found)) assert.strictEqual(found.value.externalInstallationId, "42") + // status is not passed to upsertInstallation — it comes from the schema default. + assert.strictEqual(found.value.status, "active") const count = yield* repo.upsertCommits(orgId, "github", "7", [ { From 6413a120ef0b4eca3e0d9399d831eb5770164852 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Sat, 13 Jun 2026 15:32:45 +0200 Subject: [PATCH 03/45] Lots of refactors and bug fixes --- apps/api/src/services/OAuthStateRepository.ts | 3 - .../src/services/github/GithubAppClient.ts | 21 ++- .../api/src/services/github/GithubProvider.ts | 31 ++++- .../api/src/services/vcs/VcsProviderClient.ts | 22 ++- apps/api/src/services/vcs/VcsRepository.ts | 36 ++++- apps/api/src/services/vcs/VcsSyncQueue.ts | 12 +- apps/api/src/services/vcs/VcsSyncService.ts | 108 +++++++++------ .../src/services/vcs/__tests__/vcs.test.ts | 125 +++++++++++++++++- packages/db/src/schema/vcs.ts | 3 - packages/domain/src/http/vcs.ts | 67 ++++++++-- 10 files changed, 340 insertions(+), 88 deletions(-) diff --git a/apps/api/src/services/OAuthStateRepository.ts b/apps/api/src/services/OAuthStateRepository.ts index 550aef2af..4e5444ee6 100644 --- a/apps/api/src/services/OAuthStateRepository.ts +++ b/apps/api/src/services/OAuthStateRepository.ts @@ -9,9 +9,6 @@ import { Database, type DatabaseError } from "../lib/DatabaseLive" // the short-lived CSRF nonce store for any OAuth / App-install redirect flow. // Callers supply `provider` in the insert row and verify it on read, so this // repo is reusable across integrations (GitHub install, Hazel OAuth, …). -// -// Extracted from the inline state CRUD in HazelOAuthService so the GitHub -// install service (Step 2) can depend on it and swap it out. // --------------------------------------------------------------------------- const toPersistenceError = (error: DatabaseError) => diff --git a/apps/api/src/services/github/GithubAppClient.ts b/apps/api/src/services/github/GithubAppClient.ts index a6a7c68d3..ea556a53c 100644 --- a/apps/api/src/services/github/GithubAppClient.ts +++ b/apps/api/src/services/github/GithubAppClient.ts @@ -14,6 +14,9 @@ import { Env } from "../../lib/Env" export class GithubAppError extends Data.TaggedError("GithubAppError")<{ message: string status?: number + // Which resource the failing call addressed, so the provider can tell an + // installation-auth failure (the gone/suspended signal) from a repo-level one. + scope?: "installation" | "repository" cause?: unknown }> {} @@ -154,16 +157,22 @@ export class GithubAppClient extends Context.Service()( // ---- HTTP helpers --------------------------------------------- - const failure = (response: Response, context: string) => + const failure = ( + response: Response, + context: string, + scope?: "installation" | "repository", + ) => Effect.gen(function* () { const body = yield* Effect.tryPromise({ try: () => response.text(), - catch: () => new GithubAppError({ message: `${context} failed`, status: response.status }), + catch: () => + new GithubAppError({ message: `${context} failed`, status: response.status, scope }), }) return yield* Effect.fail( new GithubAppError({ message: `${context} failed: ${response.status} ${body.slice(0, 300)}`, status: response.status, + scope, }), ) }) @@ -194,7 +203,9 @@ export class GithubAppClient extends Context.Service()( catch: (cause) => new GithubAppError({ message: "Installation token request failed", cause }), }) - if (!response.ok) return yield* failure(response, "Installation token request") + // A failure here is the installation auth gate — the authoritative + // "installation gone / suspended" signal. + if (!response.ok) return yield* failure(response, "Installation token request", "installation") const json = yield* parseJson(response, "Installation token request") const decoded = yield* decodeInstallationToken(json).pipe( Effect.mapError( @@ -265,7 +276,7 @@ export class GithubAppClient extends Context.Service()( const response = yield* authedGet(config, token, `${base}?${query.toString()}`) // 409 = empty repository, 404 = not found/no access → no commits. if (response.status === 404 || response.status === 409) break - if (!response.ok) return yield* failure(response, "List commits") + if (!response.ok) return yield* failure(response, "List commits", "repository") const json = yield* parseJson(response, "List commits") const decoded = yield* decodeCommitList(json).pipe( Effect.mapError( @@ -291,7 +302,7 @@ export class GithubAppClient extends Context.Service()( token, `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits/${sha}`, ) - if (!response.ok) return yield* failure(response, "Get commit") + if (!response.ok) return yield* failure(response, "Get commit", "repository") const json = yield* parseJson(response, "Get commit") return yield* decodeCommit(json).pipe( Effect.mapError( diff --git a/apps/api/src/services/github/GithubProvider.ts b/apps/api/src/services/github/GithubProvider.ts index bd8578502..d29059d9f 100644 --- a/apps/api/src/services/github/GithubProvider.ts +++ b/apps/api/src/services/github/GithubProvider.ts @@ -3,10 +3,12 @@ import { GitCommitSha, type RepoUpsertInput, type VcsInstallation, + VcsInstallationGoneError, type VcsInstallationSyncReason, VcsProviderError, type VcsProviderId, type VcsRepositoryRef, + VcsRepoUnavailableError, type VcsSyncJob, VcsWebhookParseError, VcsWebhookSignatureError, @@ -44,6 +46,7 @@ const PushPayload = Schema.Struct({ }), }), installation: Schema.Struct({ id: Schema.Number }), + after: GitCommitSha, // the ref's new head SHA — authoritative, no ordering inference commits: Schema.optionalKey(Schema.Array(PushCommit)), }) @@ -69,12 +72,26 @@ const parsePayload = (event: string, decoded: Effect.Effect) => Effect.mapError(() => parseError(`Invalid ${event} payload`)), ) -const toProviderError = (error: GithubAppError) => - new VcsProviderError({ +// Classify a GitHub HTTP failure into a semantic VCS error. HTTP-status +// knowledge lives here, in the provider — the orchestrator only ever sees the +// semantic outcome. A gone/410 on the installation-auth call is the +// authoritative disconnect signal; on a repo call it means the repo is gone; +// everything else (incl. 401/403/429/5xx) is transient and retryable. +const isGone = (status?: number) => status === 404 || status === 410 + +const toVcsError = ( + error: GithubAppError, +): VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError => { + if (isGone(error.status)) { + if (error.scope === "installation") return new VcsInstallationGoneError({ message: error.message }) + if (error.scope === "repository") return new VcsRepoUnavailableError({ message: error.message }) + } + return new VcsProviderError({ message: error.message, ...(error.status === undefined ? {} : { status: error.status }), ...(error.cause === undefined ? {} : { cause: error.cause }), }) +} const finiteOrNull = (value: number) => (Number.isFinite(value) ? value : null) @@ -190,6 +207,7 @@ export class GithubProvider extends Context.Service normalizeFetchedCommit(c, repo.defaultBranch, now)) + .pipe(Effect.mapError(toVcsError)) + const normalized = commits.map((c) => normalizeFetchedCommit(c, repo.defaultBranch, now)) + // GitHub's /commits endpoint returns newest-first, so the first row is + // the branch head. This ordering knowledge stays inside the provider. + return { commits: normalized, headSha: normalized[0]?.sha ?? null } }) return { diff --git a/apps/api/src/services/vcs/VcsProviderClient.ts b/apps/api/src/services/vcs/VcsProviderClient.ts index da0b368d4..60721e77b 100644 --- a/apps/api/src/services/vcs/VcsProviderClient.ts +++ b/apps/api/src/services/vcs/VcsProviderClient.ts @@ -1,11 +1,13 @@ import type { Effect } from "effect" import type { - CommitUpsertInput, RepoUpsertInput, + VcsCommitFetch, VcsInstallation, + VcsInstallationGoneError, VcsProviderError, VcsProviderId, VcsRepositoryRef, + VcsRepoUnavailableError, VcsSyncJob, VcsWebhookParseError, VcsWebhookSignatureError, @@ -37,12 +39,24 @@ export interface VcsProviderClient { /** All repositories visible to an installation, normalized. */ readonly fetchRepositories: ( installation: VcsInstallation, - ) => Effect.Effect, VcsProviderError> + ) => Effect.Effect< + ReadonlyArray, + VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError + > - /** Commits on a repo's default branch authored since `sinceMs`, normalized. */ + /** + * Commits on a repo's default branch authored since `sinceMs`, normalized, + * plus the branch head SHA (the provider knows its own ordering — see + * `VcsCommitFetch`). The provider classifies failures: `VcsInstallationGoneError` + * (disconnect), `VcsRepoUnavailableError` (repo-scoped), else `VcsProviderError` + * (transient / retryable). + */ readonly fetchCommits: ( installation: VcsInstallation, repo: VcsRepositoryRef, opts: { readonly sinceMs: number }, - ) => Effect.Effect, VcsProviderError> + ) => Effect.Effect< + VcsCommitFetch, + VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError + > } diff --git a/apps/api/src/services/vcs/VcsRepository.ts b/apps/api/src/services/vcs/VcsRepository.ts index 92b7e2a3f..257c7c63e 100644 --- a/apps/api/src/services/vcs/VcsRepository.ts +++ b/apps/api/src/services/vcs/VcsRepository.ts @@ -178,7 +178,7 @@ export class VcsRepository extends Context.Service()("@maple/api/ input: UpsertInstallationInput, ) { const now = yield* Clock.currentTimeMillis - yield* database + const rows = yield* database .execute((db) => db .insert(vcsInstallations) @@ -212,14 +212,16 @@ export class VcsRepository extends Context.Service()("@maple/api/ repositorySelection: sql`excluded.repository_selection`, updatedAt: sql`excluded.updated_at`, }, - }), + }) + .returning(), ) .pipe(Effect.mapError(toPersistenceError)) - const rows = yield* selectInstallationRow(input.provider, input.externalInstallationId) + // `.returning()` hands back the upserted row in the same statement — one + // round-trip, and no read-after-write race with a concurrent status change. const row = Option.fromNullishOr(rows[0]) if (Option.isNone(row)) { - return yield* new VcsRepoPersistenceError({ message: "Installation vanished after upsert" }) + return yield* new VcsRepoPersistenceError({ message: "Installation upsert returned no row" }) } return yield* decodeOne("vcs_installations", row.value, rowToInstallation) }) @@ -371,6 +373,31 @@ export class VcsRepository extends Context.Service()("@maple/api/ .pipe(Effect.mapError(toPersistenceError)) }) + // Flag a single repo's sync as errored without touching its cursor / + // last-synced time (a failed fetch must not wipe prior progress). + const markRepoSyncError = Effect.fn("VcsRepository.markRepoSyncError")(function* ( + orgId: OrgId, + provider: VcsProviderId, + externalRepoId: string, + message: string, + ) { + const now = yield* Clock.currentTimeMillis + yield* database + .execute((db) => + db + .update(vcsRepositories) + .set({ syncStatus: "error", lastSyncError: message, updatedAt: now }) + .where( + and( + eq(vcsRepositories.orgId, orgId), + eq(vcsRepositories.provider, provider), + eq(vcsRepositories.externalRepoId, externalRepoId), + ), + ), + ) + .pipe(Effect.mapError(toPersistenceError)) + }) + // ---- Commits ------------------------------------------------------ const upsertCommits = Effect.fn("VcsRepository.upsertCommits")(function* ( @@ -475,6 +502,7 @@ export class VcsRepository extends Context.Service()("@maple/api/ upsertRepositories, removeRepository, updateRepoSyncCursor, + markRepoSyncError, upsertCommits, findCommitBySha, } diff --git a/apps/api/src/services/vcs/VcsSyncQueue.ts b/apps/api/src/services/vcs/VcsSyncQueue.ts index 16e3303c0..dd47850d3 100644 --- a/apps/api/src/services/vcs/VcsSyncQueue.ts +++ b/apps/api/src/services/vcs/VcsSyncQueue.ts @@ -22,12 +22,12 @@ export class VcsSyncQueue extends Context.Service)[QUEUE_BINDING] as Queue | undefined - - const missing = new VcsQueueError({ message: `Missing queue binding: ${QUEUE_BINDING}` }) + const queue = workerEnv[QUEUE_BINDING] as Queue | undefined const send = Effect.fn("VcsSyncQueue.send")(function* (job: VcsSyncJob) { - if (!queue) return yield* missing + if (!queue) { + return yield* new VcsQueueError({ message: `Missing queue binding: ${QUEUE_BINDING}` }) + } const body = encodeJob(job) yield* Effect.tryPromise({ try: () => queue.send(body), @@ -42,7 +42,9 @@ export class VcsSyncQueue extends Context.Service, ) { if (jobs.length === 0) return - if (!queue) return yield* missing + if (!queue) { + return yield* new VcsQueueError({ message: `Missing queue binding: ${QUEUE_BINDING}` }) + } const messages = jobs.map((job) => ({ body: encodeJob(job) })) yield* Effect.tryPromise({ try: () => queue.sendBatch(messages), diff --git a/apps/api/src/services/vcs/VcsSyncService.ts b/apps/api/src/services/vcs/VcsSyncService.ts index 7c5dba345..3364314f0 100644 --- a/apps/api/src/services/vcs/VcsSyncService.ts +++ b/apps/api/src/services/vcs/VcsSyncService.ts @@ -7,9 +7,10 @@ import { type VcsQueueError, type VcsRepoDecodeError, type VcsRepoPersistenceError, + type VcsRepoUnavailableError, VcsSyncJob, } from "@maple/domain/http" -import { Clock, Effect, Context, Layer, Option, Schema } from "effect" +import { Clock, Effect, Context, Layer, Option, Schema, Match } from "effect" import type { VcsProviderClient } from "./VcsProviderClient" import { VcsProviderRegistry } from "./VcsProviderRegistry" import { VcsRepository } from "./VcsRepository" @@ -27,10 +28,14 @@ const DAY_MS = 86_400_000 const decodeJob = Schema.decodeUnknownEffect(VcsSyncJob) +// VcsInstallationGoneError is handled internally (→ disconnect) and never +// surfaces here. VcsProviderError / VcsRepoUnavailableError that aren't caught +// propagate so the queue retries. type SyncError = | VcsRepoPersistenceError | VcsRepoDecodeError | VcsProviderError + | VcsRepoUnavailableError | VcsQueueError | UnknownVcsProviderError @@ -78,7 +83,7 @@ export class VcsSyncService extends Context.Service ({ - kind: "backfill-repo" as const, + kind: "backfill-repo", provider: installation.provider, externalInstallationId: installation.externalInstallationId, externalRepoId: r.externalRepoId, @@ -97,7 +102,7 @@ export class VcsSyncService extends Context.Service Effect.gen(function* () { const now = yield* Clock.currentTimeMillis - const commits = yield* provider.fetchCommits( + const { commits, headSha } = yield* provider.fetchCommits( installation, { externalRepoId: job.externalRepoId, @@ -113,44 +118,44 @@ export class VcsSyncService extends Context.Service => - error.status === 404 || error.status === 410 - ? repo - .markInstallationStatus( - installation.provider, - installation.externalInstallationId, - "disconnected", - ) - .pipe( - Effect.flatMap(() => - Effect.logWarning("VCS installation no longer accessible").pipe( - Effect.annotateLogs({ - provider: installation.provider, - externalInstallationId: installation.externalInstallationId, - status: error.status, - }), - ), - ), - ) - : Effect.fail(error), + // The provider classifies failures; the orchestrator dispatches on the + // semantic outcome, never on HTTP status: + // - VcsRepoUnavailableError (repo gone) → record on the repo and drain. + // - VcsInstallationGoneError → propagates to processMessage (disconnect). + // - VcsProviderError (transient) → propagates so the queue retries. + Effect.catchTag("@maple/http/errors/VcsRepoUnavailableError", (error) => + repo + .markRepoSyncError( + installation.orgId, + installation.provider, + job.externalRepoId, + error.message, + ) + .pipe( + Effect.flatMap(() => + Effect.logWarning("Repository unavailable — backfill skipped").pipe( + Effect.annotateLogs({ + provider: installation.provider, + externalRepoId: job.externalRepoId, + }), + ), + ), + ), ), Effect.withSpan("VcsSyncService.backfillRepo"), ) const applyPushDelta = Effect.fn("VcsSyncService.applyPushDelta")(function* ( installation: VcsInstallation, - job: { externalRepoId: string; commits: ReadonlyArray }, + job: { externalRepoId: string; headSha: string; commits: ReadonlyArray }, ) { const now = yield* Clock.currentTimeMillis yield* repo.upsertCommits( @@ -159,11 +164,10 @@ export class VcsSyncService extends Context.Service 0 ? job.commits[job.commits.length - 1] : undefined + // The provider already told us the head (the push's `after`). yield* repo.updateRepoSyncCursor(installation.orgId, installation.provider, job.externalRepoId, { status: "ready", - cursorSha: head?.sha ?? null, + cursorSha: job.headSha, error: null, syncedAt: now, }) @@ -221,14 +225,38 @@ export class VcsSyncService extends Context.Service backfillRepo(provider, installation, job)), + Match.discriminator("kind")("installation-sync", (job) => syncInstallation(provider, installation, job.reason)), + Match.discriminator("kind")("push-delta", (job) => applyPushDelta(installation, job)), + Match.exhaustive + ) + + // The ONE place an installation is disconnected, and only on the + // provider's authoritative gone signal — never on a raw HTTP status. + return yield* run.pipe( + Effect.catchTag("@maple/http/errors/VcsInstallationGoneError", () => + repo + .markInstallationStatus( + installation.provider, + installation.externalInstallationId, + "disconnected", + ) + .pipe( + Effect.flatMap(() => + Effect.logWarning( + "VCS installation reported gone by provider — marked disconnected", + ).pipe( + Effect.annotateLogs({ + provider: installation.provider, + externalInstallationId: installation.externalInstallationId, + }), + ), + ), + ), + ), + ) }) return { processMessage } satisfies VcsSyncServiceShape diff --git a/apps/api/src/services/vcs/__tests__/vcs.test.ts b/apps/api/src/services/vcs/__tests__/vcs.test.ts index 53e826f39..c7d74fbaa 100644 --- a/apps/api/src/services/vcs/__tests__/vcs.test.ts +++ b/apps/api/src/services/vcs/__tests__/vcs.test.ts @@ -3,7 +3,10 @@ import { createHmac, randomUUID } from "node:crypto" import { OrgId, UserId, + VcsInstallationGoneError, + VcsProviderError, VcsRepoDecodeError, + VcsRepoUnavailableError, VcsSyncJob, VcsWebhookParseError, VcsWebhookSignatureError, @@ -73,6 +76,7 @@ describe("VcsSyncJob", () => { externalInstallationId: "42", externalRepoId: "7", branch: "main", + headSha: SHA, commits: [ { sha: SHA, @@ -98,6 +102,7 @@ describe("GithubProvider.webhookToJobs", () => { ref: "refs/heads/main", repository: { id: 7, owner: { login: "octo" } }, installation: { id: 42 }, + after: SHA, commits: [ { id: SHA, @@ -123,6 +128,7 @@ describe("GithubProvider.webhookToJobs", () => { assert.strictEqual(job.externalInstallationId, "42") assert.strictEqual(job.externalRepoId, "7") assert.strictEqual(job.branch, "main") + assert.strictEqual(job.headSha, SHA) // from the push payload's `after` assert.strictEqual(job.commits.length, 1) assert.strictEqual(job.commits[0]!.sha, SHA) assert.strictEqual(job.commits[0]!.authorLogin, "octocat") @@ -262,6 +268,11 @@ describe("VcsSyncService orchestrator", () => { isArchived: boolean }> readonly commits?: ReadonlyArray> + readonly headSha?: string | null + readonly fetchCommitsError?: + | VcsProviderError + | VcsInstallationGoneError + | VcsRepoUnavailableError } // Real VcsRepository (temp D1) + stubbed provider/queue ports, so dispatch, @@ -271,7 +282,10 @@ describe("VcsSyncService orchestrator", () => { id: "github", webhookToJobs: () => Effect.succeed([]), fetchRepositories: () => Effect.succeed(opts.repos ?? []), - fetchCommits: () => Effect.succeed(opts.commits ?? []), + fetchCommits: () => + opts.fetchCommitsError + ? Effect.fail(opts.fetchCommitsError) + : Effect.succeed({ commits: opts.commits ?? [], headSha: opts.headSha ?? null }), } const registry = Layer.succeed(VcsProviderRegistry, { ids: ["github"], @@ -314,6 +328,7 @@ describe("VcsSyncService orchestrator", () => { externalInstallationId: "999", // never seeded externalRepoId: "7", branch: "main", + headSha: SHA_A, commits: [commit(SHA_A, 1)], } yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) // must not fail @@ -337,6 +352,7 @@ describe("VcsSyncService orchestrator", () => { externalInstallationId: "42", externalRepoId: "7", branch: "main", + headSha: SHA_B, commits: [commit(SHA_A, 1), commit(SHA_B, 2)], } yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) @@ -381,11 +397,12 @@ describe("VcsSyncService orchestrator", () => { }).pipe(Effect.provide(orchestratorLayer(url, { sent, repos }))) }) - it.effect("backfill sets the cursor to the head (first/newest) commit", () => { + it.effect("backfill sets the cursor to the provider's head, not commit array order", () => { const { url } = createTempDbUrl("maple-vcs-orch-backfill-", dirs) const sent: Array = [] - // GitHub returns newest-first; fetchCommits[0] is the head. - const commits = [commit(SHA_B, 2), commit(SHA_A, 1)] + // commits[0] is deliberately NOT the head — the cursor must come from the + // provider-supplied headSha, proving no array-order inference. + const commits = [commit(SHA_A, 1), commit(SHA_B, 2)] return Effect.gen(function* () { const svc = yield* VcsSyncService const repo = yield* VcsRepository @@ -417,7 +434,104 @@ describe("VcsSyncService orchestrator", () => { const stored = yield* repo.listRepositoriesByInstallation("github", "42") assert.strictEqual(stored[0]!.syncStatus, "ready") assert.strictEqual(stored[0]!.lastSyncCursor, SHA_B) - }).pipe(Effect.provide(orchestratorLayer(url, { sent, commits }))) + }).pipe(Effect.provide(orchestratorLayer(url, { sent, commits, headSha: SHA_B }))) + }) + + const seedRepo = (repo: VcsRepository, orgId: ReturnType) => + repo.upsertRepositories(orgId, "github", "42", [ + { + externalRepoId: "7", + owner: "octo", + name: "repo", + fullName: "octo/repo", + defaultBranch: "main", + htmlUrl: "https://github.com/octo/repo", + isPrivate: true, + isArchived: false, + }, + ]) + + const backfillJob: VcsSyncJob = { + kind: "backfill-repo", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + owner: "octo", + name: "repo", + defaultBranch: "main", + sinceMs: 0, + } + + it.effect("VcsRepoUnavailableError marks the repo errored and leaves the installation active", () => { + const { url } = createTempDbUrl("maple-vcs-orch-repo-gone-", dirs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* seedRepo(repo, orgId) + // A repo-scoped error must NOT fail the job (it drains). + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(backfillJob)) + const inst = yield* repo.getInstallation("github", "42") + assert.ok(Option.isSome(inst)) + assert.strictEqual(inst.value.status, "active") // never disconnected + const stored = yield* repo.listRepositoriesByInstallation("github", "42") + assert.strictEqual(stored[0]!.syncStatus, "error") + }).pipe( + Effect.provide( + orchestratorLayer(url, { + sent, + fetchCommitsError: new VcsRepoUnavailableError({ message: "repo gone" }), + }), + ), + ) + }) + + it.effect("VcsInstallationGoneError disconnects the installation and drains the job", () => { + const { url } = createTempDbUrl("maple-vcs-orch-inst-gone-", dirs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + // The provider's authoritative gone signal → disconnect, no failure. + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(backfillJob)) + const inst = yield* repo.getInstallation("github", "42") + assert.ok(Option.isSome(inst)) + assert.strictEqual(inst.value.status, "disconnected") + }).pipe( + Effect.provide( + orchestratorLayer(url, { + sent, + fetchCommitsError: new VcsInstallationGoneError({ message: "installation gone" }), + }), + ), + ) + }) + + it.effect("transient VcsProviderError fails the job so the queue retries, installation untouched", () => { + const { url } = createTempDbUrl("maple-vcs-orch-transient-", dirs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + const exit = yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(backfillJob)).pipe(Effect.exit) + assert.ok(Exit.isFailure(exit)) // transient → propagated so the queue retries + const inst = yield* repo.getInstallation("github", "42") + assert.ok(Option.isSome(inst)) + assert.strictEqual(inst.value.status, "active") + }).pipe( + Effect.provide( + orchestratorLayer(url, { + sent, + fetchCommitsError: new VcsProviderError({ message: "upstream unavailable", status: 503 }), + }), + ), + ) }) }) @@ -431,6 +545,7 @@ describe("git SHA validation (branded type)", () => { ref: "refs/heads/main", repository: { id: 7, owner: { login: "octo" } }, installation: { id: 42 }, + after: SHA, // valid head, so the parse failure is specifically the commit id commits: [{ id: "not-a-real-sha", message: "x", url: "https://example.com" }], }) const exit = yield* provider diff --git a/packages/db/src/schema/vcs.ts b/packages/db/src/schema/vcs.ts index 3d029bb5f..645b795de 100644 --- a/packages/db/src/schema/vcs.ts +++ b/packages/db/src/schema/vcs.ts @@ -2,7 +2,6 @@ import { index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqli import type { OrgId, UserId } from "@maple/domain/primitives" import type { GitCommitSha, - ShortCommitSha, VcsAccountType, VcsCommitRowId, VcsInstallStatus, @@ -91,7 +90,6 @@ export const vcsCommits = sqliteTable( provider: text("provider").$type().notNull(), externalRepoId: text("external_repo_id").notNull(), sha: text("sha").$type().notNull(), - shortSha: text("short_sha").$type().notNull(), message: text("message").notNull(), authorName: text("author_name"), authorEmail: text("author_email"), @@ -106,7 +104,6 @@ export const vcsCommits = sqliteTable( (table) => [ uniqueIndex("vcs_commits_org_repo_sha_idx").on(table.orgId, table.provider, table.externalRepoId, table.sha), index("vcs_commits_org_sha_idx").on(table.orgId, table.sha), - index("vcs_commits_org_short_sha_idx").on(table.orgId, table.shortSha), ], ) diff --git a/packages/domain/src/http/vcs.ts b/packages/domain/src/http/vcs.ts index ca2799d7a..9b5db7e45 100644 --- a/packages/domain/src/http/vcs.ts +++ b/packages/domain/src/http/vcs.ts @@ -1,4 +1,4 @@ -import { Schema } from "effect" +import { Schema, SchemaGetter } from "effect" import { OrgId, UserId } from "../primitives" // --------------------------------------------------------------------------- @@ -31,25 +31,28 @@ export const VcsCommitRowId = Schema.String.check(Schema.isUUID()).pipe( export type VcsCommitRowId = Schema.Schema.Type /** - * A full 40-char, lowercase-hex git commit SHA. Strict — unlike the permissive - * telemetry `CommitSha` brand (which must not throw on arbitrary OTel data) — - * so the SHA-shape regex lives in exactly this one declarative type. Decoding a - * value through it (at the webhook/REST boundary and on persistence) is the only - * SHA validation in the codebase. + * A full 40-char git commit SHA. Case-insensitive on input and normalized to + * lowercase during decode, so the same commit is identified regardless of the + * case a provider — or an OTel `deployment.commit_sha` attribute — emits it in. + * Strict — unlike the permissive telemetry `CommitSha` brand (which must not + * throw on arbitrary OTel data) — so the SHA-shape regex lives in exactly this + * one declarative type. Decoding a value through it (at the webhook/REST + * boundary and on persistence) is the only SHA validation in the codebase. + * + * (40 hex = git's SHA-1 object format; git's experimental SHA-256 object format + * — 64 hex — is a known, currently-unused limitation across every git host.) */ export const GitCommitSha = Schema.String.check(Schema.isPattern(/^[0-9a-f]{40}$/)).pipe( Schema.brand("@maple/GitCommitSha"), + // Lowercase on the way in so `aBc…` and `ABC…` resolve to the same row/lookup. + Schema.encode({ + decode: SchemaGetter.transform((s: string) => s.toLowerCase()), + encode: SchemaGetter.passthrough(), + }), Schema.annotate({ identifier: "@maple/GitCommitSha", title: "Git Commit SHA" }), ) export type GitCommitSha = Schema.Schema.Type -/** First 7 hex chars of a commit SHA (display + abbreviated-input lookup). */ -export const ShortCommitSha = Schema.String.check(Schema.isPattern(/^[0-9a-f]{7}$/)).pipe( - Schema.brand("@maple/ShortCommitSha"), - Schema.annotate({ identifier: "@maple/ShortCommitSha", title: "Short Commit SHA" }), -) -export type ShortCommitSha = Schema.Schema.Type - // ---- Provider + normalized enums ------------------------------------------ /** The set of supported VCS providers. Extend this array to add a provider. */ @@ -129,7 +132,6 @@ export class VcsCommit extends Schema.Class("VcsCommit")({ provider: VcsProviderId, externalRepoId: Schema.String, sha: GitCommitSha, - shortSha: ShortCommitSha, message: Schema.String, authorName: Schema.NullOr(Schema.String), authorEmail: Schema.NullOr(Schema.String), @@ -172,6 +174,17 @@ export const CommitUpsertInput = Schema.Struct({ }) export type CommitUpsertInput = Schema.Schema.Type +/** + * Result of a provider commit fetch: the commits plus the current head SHA of + * the fetched ref. The provider determines `headSha` (it alone knows its own + * response ordering / the authoritative tip); callers must never infer the head + * from commit array position. `null` when the ref has no commits. + */ +export interface VcsCommitFetch { + readonly commits: ReadonlyArray + readonly headSha: string | null +} + /** Minimal repo identity a provider needs to fetch commits. */ export const VcsRepositoryRef = Schema.Struct({ externalRepoId: Schema.String, @@ -222,6 +235,9 @@ export const PushDeltaJob = Schema.Struct({ externalInstallationId: Schema.String, externalRepoId: Schema.String, branch: Schema.String, + // The branch head after the push, as reported by the provider (e.g. GitHub's + // `after`). The generic layer never infers the head from commit order. + headSha: GitCommitSha, commits: Schema.Array(CommitUpsertInput), }) export type PushDeltaJob = Schema.Schema.Type @@ -259,6 +275,29 @@ export class VcsProviderError extends Schema.TaggedErrorClass( { httpApiStatus: 502 }, ) {} +/** + * The provider is certain the installation no longer exists / access is + * permanently revoked at the installation level (e.g. GitHub's installation + * token endpoint returning gone). The ONLY error the sync orchestrator treats + * as a disconnect — raw HTTP status never drives that decision. Providers must + * only raise this when the signal is unambiguous. + */ +export class VcsInstallationGoneError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsInstallationGoneError", + { message: Schema.String }, + { httpApiStatus: 410 }, +) {} + +/** + * The provider is certain a specific repository is permanently inaccessible + * (deleted / renamed / access lost). Scoped to the repo — never the installation. + */ +export class VcsRepoUnavailableError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsRepoUnavailableError", + { message: Schema.String }, + { httpApiStatus: 404 }, +) {} + export class VcsWebhookSignatureError extends Schema.TaggedErrorClass()( "@maple/http/errors/VcsWebhookSignatureError", { message: Schema.String }, From ec15b4a7d7d2e5f8225d9d42ee9ed14cfdc6a9a1 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Sat, 13 Jun 2026 17:02:24 +0200 Subject: [PATCH 04/45] More general cleanup --- .../src/services/github/GithubAppClient.ts | 29 ++++- .../api/src/services/github/GithubProvider.ts | 9 +- .../api/src/services/vcs/VcsProviderClient.ts | 10 +- apps/api/src/services/vcs/VcsRepository.ts | 2 - apps/api/src/services/vcs/VcsSyncService.ts | 80 ++++++++----- .../src/services/vcs/__tests__/vcs.test.ts | 108 +++++++++++++++--- ...ron_lad.sql => 0022_naive_enchantress.sql} | 2 - packages/db/drizzle/meta/0022_snapshot.json | 17 +-- packages/db/drizzle/meta/_journal.json | 4 +- packages/domain/src/http/vcs.ts | 36 ++++-- 10 files changed, 212 insertions(+), 85 deletions(-) rename packages/db/drizzle/{0022_lumpy_iron_lad.sql => 0022_naive_enchantress.sql} (94%) diff --git a/apps/api/src/services/github/GithubAppClient.ts b/apps/api/src/services/github/GithubAppClient.ts index ea556a53c..0b1aa7f53 100644 --- a/apps/api/src/services/github/GithubAppClient.ts +++ b/apps/api/src/services/github/GithubAppClient.ts @@ -23,7 +23,9 @@ export class GithubAppError extends Data.TaggedError("GithubAppError")<{ const GITHUB_API_VERSION = "2022-11-28" const USER_AGENT = "maple-vcs-integration" const PER_PAGE = 100 -const MAX_PAGES = 20 // safety cap (≤2000 items) +// Paginate effectively to the end (up to 100k items) while still bounding a +// pathological loop. Hitting this cap is logged — truncation is never silent. +const MAX_PAGES = 1000 // ---- REST response schemas ------------------------------------------------ @@ -235,7 +237,8 @@ export class GithubAppClient extends Context.Service()( const config = yield* resolveConfig const token = yield* mintInstallationToken(externalInstallationId) const repos: Array = [] - for (let page = 1; page <= MAX_PAGES; page++) { + let page = 1 + for (; page <= MAX_PAGES; page++) { const response = yield* authedGet( config, token, @@ -255,6 +258,12 @@ export class GithubAppClient extends Context.Service()( repos.push(...decoded.repositories) if (decoded.repositories.length < PER_PAGE) break } + // Exhausted the page cap without a short final page → likely truncated. + if (page > MAX_PAGES) { + yield* Effect.logWarning("GitHub installation repositories truncated at page cap").pipe( + Effect.annotateLogs({ externalInstallationId, maxPages: MAX_PAGES, fetched: repos.length }), + ) + } return repos }, ) @@ -269,13 +278,17 @@ export class GithubAppClient extends Context.Service()( const token = yield* mintInstallationToken(externalInstallationId) const base = `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits` const commits: Array = [] - for (let page = 1; page <= MAX_PAGES; page++) { + let page = 1 + for (; page <= MAX_PAGES; page++) { const query = new URLSearchParams({ per_page: String(PER_PAGE), page: String(page) }) if (params.sha) query.set("sha", params.sha) if (params.sinceIso) query.set("since", params.sinceIso) const response = yield* authedGet(config, token, `${base}?${query.toString()}`) - // 409 = empty repository, 404 = not found/no access → no commits. - if (response.status === 404 || response.status === 409) break + // 409 = empty repository → genuinely no commits, not an error. + if (response.status === 409) break + // Anything else non-2xx (incl. 404 = repo deleted / access lost) is + // surfaced as a repository-scoped failure so the orchestrator can mark + // the repo unavailable rather than mistaking it for an empty repo. if (!response.ok) return yield* failure(response, "List commits", "repository") const json = yield* parseJson(response, "List commits") const decoded = yield* decodeCommitList(json).pipe( @@ -286,6 +299,12 @@ export class GithubAppClient extends Context.Service()( commits.push(...decoded) if (decoded.length < PER_PAGE) break } + // Exhausted the page cap without a short final page → likely truncated. + if (page > MAX_PAGES) { + yield* Effect.logWarning("GitHub commit list truncated at page cap").pipe( + Effect.annotateLogs({ owner, repo, maxPages: MAX_PAGES, fetched: commits.length }), + ) + } return commits }) diff --git a/apps/api/src/services/github/GithubProvider.ts b/apps/api/src/services/github/GithubProvider.ts index d29059d9f..79555dec9 100644 --- a/apps/api/src/services/github/GithubProvider.ts +++ b/apps/api/src/services/github/GithubProvider.ts @@ -46,7 +46,6 @@ const PushPayload = Schema.Struct({ }), }), installation: Schema.Struct({ id: Schema.Number }), - after: GitCommitSha, // the ref's new head SHA — authoritative, no ordering inference commits: Schema.optionalKey(Schema.Array(PushCommit)), }) @@ -201,13 +200,14 @@ export class GithubProvider extends Context.Service Effect.gen(function* () { const now = yield* Clock.currentTimeMillis + // GitHub's `since` filters by *committer* date (matching the port's + // "committed since" contract). The newest-first ordering and the + // committer-date basis are both GitHub specifics that stay in here. const commits = yield* client .listCommits(installation.externalInstallationId, repo.owner, repo.name, { sha: repo.defaultBranch, diff --git a/apps/api/src/services/vcs/VcsProviderClient.ts b/apps/api/src/services/vcs/VcsProviderClient.ts index 60721e77b..3b9233626 100644 --- a/apps/api/src/services/vcs/VcsProviderClient.ts +++ b/apps/api/src/services/vcs/VcsProviderClient.ts @@ -45,11 +45,13 @@ export interface VcsProviderClient { > /** - * Commits on a repo's default branch authored since `sinceMs`, normalized, + * Commits on a repo's default branch *committed* since `sinceMs`, normalized, * plus the branch head SHA (the provider knows its own ordering — see - * `VcsCommitFetch`). The provider classifies failures: `VcsInstallationGoneError` - * (disconnect), `VcsRepoUnavailableError` (repo-scoped), else `VcsProviderError` - * (transient / retryable). + * `VcsCommitFetch`). The `sinceMs` filter is keyed on committer date; the exact + * basis and ordering are provider-defined and never assumed by callers. The + * provider classifies failures: `VcsInstallationGoneError` (disconnect), + * `VcsRepoUnavailableError` (repo-scoped), else `VcsProviderError` (transient / + * retryable). */ readonly fetchCommits: ( installation: VcsInstallation, diff --git a/apps/api/src/services/vcs/VcsRepository.ts b/apps/api/src/services/vcs/VcsRepository.ts index 257c7c63e..da59bdead 100644 --- a/apps/api/src/services/vcs/VcsRepository.ts +++ b/apps/api/src/services/vcs/VcsRepository.ts @@ -99,7 +99,6 @@ const rowToCommit = (row: VcsCommitRow): VcsCommit => provider: row.provider, externalRepoId: row.externalRepoId, sha: row.sha, - shortSha: row.shortSha, message: row.message, authorName: row.authorName ?? null, authorEmail: row.authorEmail ?? null, @@ -420,7 +419,6 @@ export class VcsRepository extends Context.Service()("@maple/api/ provider, externalRepoId, sha, - shortSha: sha.slice(0, 7) as VcsCommit["shortSha"], message: c.message, authorName: c.authorName, authorEmail: c.authorEmail, diff --git a/apps/api/src/services/vcs/VcsSyncService.ts b/apps/api/src/services/vcs/VcsSyncService.ts index 3364314f0..1801d3ebe 100644 --- a/apps/api/src/services/vcs/VcsSyncService.ts +++ b/apps/api/src/services/vcs/VcsSyncService.ts @@ -1,5 +1,6 @@ import { type CommitUpsertInput, + isInstallationProcessable, type UnknownVcsProviderError, type VcsInstallation, type VcsInstallationSyncReason, @@ -153,24 +154,47 @@ export class VcsSyncService extends Context.Service }, + job: { externalRepoId: string; commits: ReadonlyArray }, ) { - const now = yield* Clock.currentTimeMillis + // A push is incremental enrichment only: upsert the pushed commits and + // deliberately leave the sync cursor untouched. The cursor exclusively + // tracks the default-branch commit *list* backfill, which is the only + // path that resumes from it — a push can't authoritatively advance it + // (it may target any branch and its payload may be truncated), so there + // is nothing to gain by moving it here. yield* repo.upsertCommits( installation.orgId, installation.provider, job.externalRepoId, job.commits, ) - // The provider already told us the head (the push's `after`). - yield* repo.updateRepoSyncCursor(installation.orgId, installation.provider, job.externalRepoId, { - status: "ready", - cursorSha: job.headSha, - error: null, - syncedAt: now, + }) + + // THE gate: the single, vendor-agnostic answer to "should the sync engine + // act on this installation's data?" (rule lives in isInstallationProcessable). + // Every data-processing path runs through here; the decision is annotated on + // the current span (`vcs.installation.processable`) so it's traceable, and a + // skip is logged. Suspended / disconnected installations are skipped. + const ensureProcessable = (installation: VcsInstallation, kind: VcsSyncJob["kind"]) => + Effect.gen(function* () { + const processable = isInstallationProcessable(installation) + yield* Effect.annotateCurrentSpan({ + "vcs.installation.status": installation.status, + "vcs.installation.processable": processable, }) + if (!processable) { + yield* Effect.logInfo("Skipping VCS job: installation not processable").pipe( + Effect.annotateLogs({ + provider: installation.provider, + externalInstallationId: installation.externalInstallationId, + status: installation.status, + kind, + }), + ) + } + return processable }) const processMessage = Effect.fn("VcsSyncService.processMessage")(function* (raw: unknown) { @@ -191,17 +215,21 @@ export class VcsSyncService extends Context.Service backfillRepo(provider, installation, job)), Match.discriminator("kind")("installation-sync", (job) => syncInstallation(provider, installation, job.reason)), - Match.discriminator("kind")("push-delta", (job) => applyPushDelta(installation, job)), + Match.discriminator("kind")("push", (job) => applyPush(installation, job)), Match.exhaustive ) diff --git a/apps/api/src/services/vcs/__tests__/vcs.test.ts b/apps/api/src/services/vcs/__tests__/vcs.test.ts index c7d74fbaa..906b78331 100644 --- a/apps/api/src/services/vcs/__tests__/vcs.test.ts +++ b/apps/api/src/services/vcs/__tests__/vcs.test.ts @@ -1,6 +1,7 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { createHmac, randomUUID } from "node:crypto" import { + GitCommitSha, OrgId, UserId, VcsInstallationGoneError, @@ -71,12 +72,11 @@ const findError = (exit: Exit.Exit): unknown => { describe("VcsSyncJob", () => { it("round-trips through encode/decode", () => { const job: VcsSyncJob = { - kind: "push-delta", + kind: "push", provider: "github", externalInstallationId: "42", externalRepoId: "7", branch: "main", - headSha: SHA, commits: [ { sha: SHA, @@ -114,7 +114,7 @@ describe("GithubProvider.webhookToJobs", () => { ], }) - it.effect("maps a validly-signed push to a push-delta job", () => + it.effect("maps a validly-signed push to a push job (no head SHA — push never moves the cursor)", () => Effect.gen(function* () { const provider = yield* GithubProvider const jobs = yield* provider.webhookToJobs({ @@ -123,15 +123,16 @@ describe("GithubProvider.webhookToJobs", () => { }) assert.strictEqual(jobs.length, 1) const job = jobs[0]! - assert.strictEqual(job.kind, "push-delta") - if (job.kind !== "push-delta") return + assert.strictEqual(job.kind, "push") + if (job.kind !== "push") return assert.strictEqual(job.externalInstallationId, "42") assert.strictEqual(job.externalRepoId, "7") assert.strictEqual(job.branch, "main") - assert.strictEqual(job.headSha, SHA) // from the push payload's `after` assert.strictEqual(job.commits.length, 1) assert.strictEqual(job.commits[0]!.sha, SHA) assert.strictEqual(job.commits[0]!.authorLogin, "octocat") + // A push carries no headSha — the payload's `after` is intentionally ignored. + assert.ok(!("headSha" in job)) }).pipe(Effect.provide(providerLayer())), ) @@ -211,7 +212,6 @@ describe("VcsRepository", () => { const commit = yield* repo.findCommitBySha(orgId, SHA as never) assert.ok(Option.isSome(commit)) - assert.strictEqual(commit.value.shortSha, SHA.slice(0, 7)) assert.strictEqual(commit.value.authorLogin, "octocat") }).pipe(Effect.provide(repoLayer(url))) }) @@ -323,12 +323,11 @@ describe("VcsSyncService orchestrator", () => { const repo = yield* VcsRepository const orgId = asOrgId("org_orch") const job: VcsSyncJob = { - kind: "push-delta", + kind: "push", provider: "github", externalInstallationId: "999", // never seeded externalRepoId: "7", branch: "main", - headSha: SHA_A, commits: [commit(SHA_A, 1)], } yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) // must not fail @@ -338,7 +337,7 @@ describe("VcsSyncService orchestrator", () => { }).pipe(Effect.provide(orchestratorLayer(url, { sent }))) }) - it.effect("push-delta upserts every commit", () => { + it.effect("push upserts every commit and never moves the repo sync cursor", () => { const { url } = createTempDbUrl("maple-vcs-orch-push-", dirs) const sent: Array = [] return Effect.gen(function* () { @@ -346,19 +345,24 @@ describe("VcsSyncService orchestrator", () => { const repo = yield* VcsRepository const orgId = asOrgId("org_orch") yield* seedInstallation(repo, orgId) + yield* seedRepo(repo, orgId) // a freshly-discovered repo (pending, no cursor) const job: VcsSyncJob = { - kind: "push-delta", + kind: "push", provider: "github", externalInstallationId: "42", externalRepoId: "7", branch: "main", - headSha: SHA_B, commits: [commit(SHA_A, 1), commit(SHA_B, 2)], } yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) const a = yield* repo.findCommitBySha(orgId, SHA_A as never) const b = yield* repo.findCommitBySha(orgId, SHA_B as never) assert.ok(Option.isSome(a) && Option.isSome(b)) + // B2: a push is pure enrichment — the cursor/status stay exactly as the + // backfill left them (here: untouched since no backfill has run). + const stored = yield* repo.listRepositoriesByInstallation("github", "42") + assert.strictEqual(stored[0]!.lastSyncCursor, null) + assert.strictEqual(stored[0]!.syncStatus, "pending") }).pipe(Effect.provide(orchestratorLayer(url, { sent }))) }) @@ -533,11 +537,89 @@ describe("VcsSyncService orchestrator", () => { ), ) }) + + // C1: the processability gate. A non-active installation must process nothing. + it.effect("a suspended installation is skipped — no data processed, no failure", () => { + const { url } = createTempDbUrl("maple-vcs-orch-suspended-", dirs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* repo.markInstallationStatus("github", "42", "suspended") + const job: VcsSyncJob = { + kind: "push", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + branch: "main", + commits: [commit(SHA_A, 1)], + } + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) // gated → must not fail + const a = yield* repo.findCommitBySha(orgId, SHA_A as never) + assert.ok(Option.isNone(a)) // gate short-circuits before the upsert + const inst = yield* repo.getInstallation("github", "42") + assert.ok(Option.isSome(inst)) + assert.strictEqual(inst.value.status, "suspended") // status untouched + }).pipe(Effect.provide(orchestratorLayer(url, { sent, commits: [commit(SHA_A, 1)] }))) + }) + + // C1: unsuspend must flip the installation back to active and resume syncing. + it.effect("unsuspend reactivates a suspended installation and re-syncs its repos", () => { + const { url } = createTempDbUrl("maple-vcs-orch-unsuspend-", dirs) + const sent: Array = [] + const repos = [ + { + externalRepoId: "7", + owner: "octo", + name: "repo", + fullName: "octo/repo", + defaultBranch: "main", + htmlUrl: "https://github.com/octo/repo", + isPrivate: true, + isArchived: false, + }, + ] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* repo.markInstallationStatus("github", "42", "suspended") + const job: VcsSyncJob = { + kind: "installation-sync", + provider: "github", + externalInstallationId: "42", + reason: "unsuspend", + } + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) + const inst = yield* repo.getInstallation("github", "42") + assert.ok(Option.isSome(inst)) + assert.strictEqual(inst.value.status, "active") // reactivated + const stored = yield* repo.listRepositoriesByInstallation("github", "42") + assert.strictEqual(stored.length, 1) // re-sync ran + assert.strictEqual(sent.length, 1) + assert.strictEqual(sent[0]!.kind, "backfill-repo") + }).pipe(Effect.provide(orchestratorLayer(url, { sent, repos }))) + }) }) // The SHA-shape regex lives only in the GitCommitSha brand; these assert that // validation fires at both the webhook decode boundary and on persistence. describe("git SHA validation (branded type)", () => { + it("GitCommitSha accepts mixed-case input and normalizes it to lowercase", () => { + const decode = Schema.decodeUnknownSync(GitCommitSha) + // All-uppercase 40-hex is accepted and lowercased. + assert.strictEqual(decode("A".repeat(40)), "a".repeat(40)) + // Mixed case round-trips to its lowercase form (so case never splits a row). + const mixed = "AbCdEf0123456789aBcDeF0123456789AbCdEf01" + assert.strictEqual(decode(mixed), mixed.toLowerCase()) + // Non-hex / wrong length are still rejected (after lowercasing). + assert.throws(() => decode("Z".repeat(40))) + assert.throws(() => decode("abc")) + }) + it.effect("webhook decode rejects a malformed commit SHA with VcsWebhookParseError", () => Effect.gen(function* () { const provider = yield* GithubProvider @@ -567,7 +649,7 @@ describe("git SHA validation (branded type)", () => { const exit = yield* repo .upsertCommits(orgId, "github", "7", [ { - sha: "ABC", // not 40-char lowercase hex + sha: "ABC", // not 40-char hex (case-insensitive, but still invalid) message: "bad", authorName: null, authorEmail: null, diff --git a/packages/db/drizzle/0022_lumpy_iron_lad.sql b/packages/db/drizzle/0022_naive_enchantress.sql similarity index 94% rename from packages/db/drizzle/0022_lumpy_iron_lad.sql rename to packages/db/drizzle/0022_naive_enchantress.sql index 3e3adb6cd..620cf2f62 100644 --- a/packages/db/drizzle/0022_lumpy_iron_lad.sql +++ b/packages/db/drizzle/0022_naive_enchantress.sql @@ -4,7 +4,6 @@ CREATE TABLE `vcs_commits` ( `provider` text NOT NULL, `external_repo_id` text NOT NULL, `sha` text NOT NULL, - `short_sha` text NOT NULL, `message` text NOT NULL, `author_name` text, `author_email` text, @@ -19,7 +18,6 @@ CREATE TABLE `vcs_commits` ( --> statement-breakpoint CREATE UNIQUE INDEX `vcs_commits_org_repo_sha_idx` ON `vcs_commits` (`org_id`,`provider`,`external_repo_id`,`sha`);--> statement-breakpoint CREATE INDEX `vcs_commits_org_sha_idx` ON `vcs_commits` (`org_id`,`sha`);--> statement-breakpoint -CREATE INDEX `vcs_commits_org_short_sha_idx` ON `vcs_commits` (`org_id`,`short_sha`);--> statement-breakpoint CREATE TABLE `vcs_installations` ( `id` text PRIMARY KEY NOT NULL, `org_id` text NOT NULL, diff --git a/packages/db/drizzle/meta/0022_snapshot.json b/packages/db/drizzle/meta/0022_snapshot.json index 3edadc701..98ee81e43 100644 --- a/packages/db/drizzle/meta/0022_snapshot.json +++ b/packages/db/drizzle/meta/0022_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "228d5a21-bac3-4115-8a05-bf1c7095686e", + "id": "52f05d7c-c339-4714-804b-929419ba1dbd", "prevId": "31c9a36a-d77e-4e0f-a535-c71e671856bc", "tables": { "ai_triage_runs": { @@ -4191,13 +4191,6 @@ "notNull": true, "autoincrement": false }, - "short_sha": { - "name": "short_sha", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, "message": { "name": "message", "type": "text", @@ -4287,14 +4280,6 @@ "sha" ], "isUnique": false - }, - "vcs_commits_org_short_sha_idx": { - "name": "vcs_commits_org_short_sha_idx", - "columns": [ - "org_id", - "short_sha" - ], - "isUnique": false } }, "foreignKeys": {}, diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 4d7352372..27c7f6037 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -159,8 +159,8 @@ { "idx": 22, "version": "6", - "when": 1781283099347, - "tag": "0022_lumpy_iron_lad", + "when": 1781357626183, + "tag": "0022_naive_enchantress", "breakpoints": true } ] diff --git a/packages/domain/src/http/vcs.ts b/packages/domain/src/http/vcs.ts index 9b5db7e45..8a266464f 100644 --- a/packages/domain/src/http/vcs.ts +++ b/packages/domain/src/http/vcs.ts @@ -42,10 +42,13 @@ export type VcsCommitRowId = Schema.Schema.Type * (40 hex = git's SHA-1 object format; git's experimental SHA-256 object format * — 64 hex — is a known, currently-unused limitation across every git host.) */ -export const GitCommitSha = Schema.String.check(Schema.isPattern(/^[0-9a-f]{40}$/)).pipe( +const GitCommitShaBrand = Schema.String.check(Schema.isPattern(/^[0-9a-f]{40}$/)).pipe( Schema.brand("@maple/GitCommitSha"), - // Lowercase on the way in so `aBc…` and `ABC…` resolve to the same row/lookup. - Schema.encode({ +) +export const GitCommitSha = Schema.String.pipe( + // Lowercase on the way in (before the pattern check) so `aBc…` and `ABC…` + // resolve to the same branded value, hence the same row and the same lookup. + Schema.decodeTo(GitCommitShaBrand, { decode: SchemaGetter.transform((s: string) => s.toLowerCase()), encode: SchemaGetter.passthrough(), }), @@ -105,6 +108,16 @@ export class VcsInstallation extends Schema.Class("VcsInstallat updatedAt: Schema.Number, }) {} +/** + * The single, vendor-agnostic answer to "should the sync engine act on this + * installation's data?". Only `active` installations are processed — `suspended` + * (provider temporarily disabled it) and `disconnected` (uninstalled / access + * revoked) are both skipped. Every data-processing path gates on this so the + * rule lives in exactly one place. + */ +export const isInstallationProcessable = (installation: VcsInstallation): boolean => + installation.status === "active" + export class VcsRepo extends Schema.Class("VcsRepo")({ id: VcsRepositoryId, orgId: OrgId, @@ -229,20 +242,23 @@ export const BackfillRepoJob = Schema.Struct({ }) export type BackfillRepoJob = Schema.Schema.Type -export const PushDeltaJob = Schema.Struct({ - kind: Schema.Literal("push-delta"), +// A push event's commits, applied incrementally. NOT a cursor-advancing sync: +// the cursor only ever tracks the default-branch commit *list* (see +// `BackfillRepoJob`), so a push carries no head SHA and never moves it. A push +// payload may also be incomplete (GitHub caps `commits` at 2048 per delivery and +// sends one delivery per push, not many) — the authoritative fill-in is the +// default-branch backfill, so a push is purely best-effort enrichment. +export const PushJob = Schema.Struct({ + kind: Schema.Literal("push"), provider: VcsProviderId, externalInstallationId: Schema.String, externalRepoId: Schema.String, branch: Schema.String, - // The branch head after the push, as reported by the provider (e.g. GitHub's - // `after`). The generic layer never infers the head from commit order. - headSha: GitCommitSha, commits: Schema.Array(CommitUpsertInput), }) -export type PushDeltaJob = Schema.Schema.Type +export type PushJob = Schema.Schema.Type -export const VcsSyncJob = Schema.Union([InstallationSyncJob, BackfillRepoJob, PushDeltaJob]) +export const VcsSyncJob = Schema.Union([InstallationSyncJob, BackfillRepoJob, PushJob]) export type VcsSyncJob = Schema.Schema.Type // ---- Tagged errors -------------------------------------------------------- From 0c4d1aec238910f328792503ceebeb3c1bf0c204 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Sun, 14 Jun 2026 22:46:49 +0200 Subject: [PATCH 05/45] Cleanups, bugfixes --- apps/api/src/app.ts | 5 +- .../src/services/github/GithubAppClient.ts | 218 ++++++++--- apps/api/src/services/github/GithubHttp.ts | 20 + .../api/src/services/github/GithubProvider.ts | 107 +++++- .../api/src/services/vcs/VcsProviderClient.ts | 28 +- apps/api/src/services/vcs/VcsRepository.ts | 17 +- apps/api/src/services/vcs/VcsSyncQueue.ts | 34 +- apps/api/src/services/vcs/VcsSyncService.ts | 104 +++++- .../src/services/vcs/__tests__/vcs.test.ts | 345 +++++++++++++++++- apps/api/src/vcs-sync-runtime.ts | 39 +- .../db/drizzle/0022_naive_enchantress.sql | 1 - packages/db/drizzle/meta/0022_snapshot.json | 7 - packages/db/src/schema/vcs.ts | 3 +- packages/domain/src/http/vcs.ts | 46 ++- 14 files changed, 808 insertions(+), 166 deletions(-) create mode 100644 apps/api/src/services/github/GithubHttp.ts diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 135f516a5..5eed10665 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -62,6 +62,7 @@ import { ScrapeTargetsService } from "./services/ScrapeTargetsService" import { WarehouseQueryService } from "./lib/WarehouseQueryService" import { OAuthStateRepository } from "./services/OAuthStateRepository" import { GithubAppClient } from "./services/github/GithubAppClient" +import { GithubHttp } from "./services/github/GithubHttp" import { GithubProvider } from "./services/github/GithubProvider" import { VcsProviderRegistry } from "./services/vcs/VcsProviderRegistry" import { VcsRepository } from "./services/vcs/VcsRepository" @@ -164,7 +165,9 @@ export const DigestServiceLive = DigestService.layer.pipe( // Step-2 settings endpoints. The sync orchestrator (VcsSyncService) lives only // in the queue-consumer runtime (vcs-sync-runtime.ts), not here. Database / // WorkerEnvironment are satisfied at worker scope (like CoreServicesLive). -const GithubProviderLive = GithubProvider.layer.pipe(Layer.provide(GithubAppClient.layer)) +const GithubProviderLive = GithubProvider.layer.pipe( + Layer.provide(GithubAppClient.layer.pipe(Layer.provide(GithubHttp.layer))), +) export const VcsServicesLive = Layer.mergeAll( VcsRepository.layer, diff --git a/apps/api/src/services/github/GithubAppClient.ts b/apps/api/src/services/github/GithubAppClient.ts index 0b1aa7f53..e826b6d65 100644 --- a/apps/api/src/services/github/GithubAppClient.ts +++ b/apps/api/src/services/github/GithubAppClient.ts @@ -1,6 +1,7 @@ import { GitCommitSha } from "@maple/domain/http" -import { Clock, Context, Data, Effect, Layer, Option, Redacted, Schema } from "effect" +import { Clock, Context, Data, Duration, Effect, Layer, Option, Redacted, Schema } from "effect" import { Env } from "../../lib/Env" +import { GithubHttp } from "./GithubHttp" // --------------------------------------------------------------------------- // GitHub App REST client. Vendor-specific: mints a short-lived App JWT (RS256, @@ -17,6 +18,9 @@ export class GithubAppError extends Data.TaggedError("GithubAppError")<{ // Which resource the failing call addressed, so the provider can tell an // installation-auth failure (the gone/suspended signal) from a repo-level one. scope?: "installation" | "repository" + // Set when the failure is a rate limit too far out to wait through inline: + // seconds until the budget returns. The provider maps this to VcsRateLimitedError. + retryAfterSeconds?: number cause?: unknown }> {} @@ -26,6 +30,41 @@ const PER_PAGE = 100 // Paginate effectively to the end (up to 100k items) while still bounding a // pathological loop. Hitting this cap is logged — truncation is never silent. const MAX_PAGES = 1000 +// Ride out short rate limits inline; anything longer is surfaced so the caller +// can defer (backfill requeues from a cursor; other jobs get a delayed retry). +const INLINE_BACKOFF_CAP_S = 30 +// Cap inline rate-limit retries so a server stuck reporting tiny/zero waits (e.g. +// a past reset timestamp from clock skew) can't spin the consumer forever; once +// hit, we defer like any other long wait rather than looping. +const MAX_INLINE_RATE_LIMIT_RETRIES = 5 + +// A GitHub rate-limit response is a 429, or a 403 that carries `retry-after` / +// reports zero remaining (the secondary-limit shape). Plain 403s (permissions) +// are NOT rate limits. +const isRateLimited = (response: Response): boolean => + response.status === 429 || + (response.status === 403 && + (response.headers.get("retry-after") !== null || + response.headers.get("x-ratelimit-remaining") === "0")) + +// Seconds until the budget returns, per GitHub's guidance: prefer `retry-after`, +// else wait until the rate-limit reset (epoch seconds), else a conservative minute. +const rateLimitWaitSeconds = (response: Response, nowMs: number): number => { + const retryAfter = response.headers.get("retry-after") + if (retryAfter !== null) { + const secs = Number(retryAfter) + if (Number.isFinite(secs) && secs >= 0) return secs + // `retry-after` may be an HTTP-date instead of delta-seconds. + const dateMs = Date.parse(retryAfter) + if (Number.isFinite(dateMs)) return Math.max(0, Math.ceil((dateMs - nowMs) / 1000)) + } + const reset = response.headers.get("x-ratelimit-reset") + if (reset !== null) { + const resetSec = Number(reset) + if (Number.isFinite(resetSec)) return Math.max(0, Math.ceil(resetSec - nowMs / 1000)) + } + return 60 +} // ---- REST response schemas ------------------------------------------------ @@ -106,6 +145,41 @@ export class GithubAppClient extends Context.Service()( { make: Effect.gen(function* () { const env = yield* Env + const http = yield* GithubHttp + + // Run a request, riding out short rate limits inline and surfacing longer + // ones as a GithubAppError carrying `retryAfterSeconds`. The single place + // 429s are detected and turned into a rate-limit signal. + const rateLimitedFetch = (request: Effect.Effect) => + Effect.gen(function* () { + let inlineRetries = 0 + while (true) { + const response = yield* request + if (!isRateLimited(response)) return response + const waitS = rateLimitWaitSeconds(response, yield* Clock.currentTimeMillis) + // Defer (surface to the caller) when a single wait is longer than we'll + // ride out inline, OR when we've retried inline too many times. Floor + // the exhausted-case deferral so a tiny/zero-wait server can't drive an + // immediate-redelivery loop after we stop spinning. + const exhausted = inlineRetries >= MAX_INLINE_RATE_LIMIT_RETRIES + if (waitS > INLINE_BACKOFF_CAP_S || exhausted) { + return yield* new GithubAppError({ + message: `GitHub rate limited (retry after ${waitS}s)`, + status: response.status, + retryAfterSeconds: exhausted ? Math.max(waitS, 60) : waitS, + }) + } + inlineRetries += 1 + yield* Effect.logWarning("GitHub rate limit hit — waiting inline").pipe( + Effect.annotateLogs({ + waitSeconds: waitS, + status: response.status, + attempt: inlineRetries, + }), + ) + yield* Effect.sleep(Duration.seconds(waitS)) + } + }) const resolveConfig: Effect.Effect = Effect.gen(function* () { const appId = Option.getOrUndefined(env.GITHUB_APP_ID) @@ -191,22 +265,28 @@ export class GithubAppClient extends Context.Service()( ) { const config = yield* resolveConfig const jwt = yield* mintAppJwt(config) - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${config.apiBaseUrl}/app/installations/${externalInstallationId}/access_tokens`, { - method: "POST", - headers: { - authorization: `Bearer ${jwt}`, - accept: "application/vnd.github+json", - "x-github-api-version": GITHUB_API_VERSION, - "user-agent": USER_AGENT, - }, - }), - catch: (cause) => - new GithubAppError({ message: "Installation token request failed", cause }), - }) - // A failure here is the installation auth gate — the authoritative - // "installation gone / suspended" signal. + const response = yield* rateLimitedFetch( + Effect.tryPromise({ + try: () => + http.fetch( + `${config.apiBaseUrl}/app/installations/${externalInstallationId}/access_tokens`, + { + method: "POST", + headers: { + authorization: `Bearer ${jwt}`, + accept: "application/vnd.github+json", + "x-github-api-version": GITHUB_API_VERSION, + "user-agent": USER_AGENT, + }, + }, + ), + catch: (cause) => + new GithubAppError({ message: "Installation token request failed", cause }), + }), + ) + // A non-rate-limit failure here is the installation auth gate — the + // authoritative "installation gone / suspended" signal (rate limits were + // already split off by rateLimitedFetch above). if (!response.ok) return yield* failure(response, "Installation token request", "installation") const json = yield* parseJson(response, "Installation token request") const decoded = yield* decodeInstallationToken(json).pipe( @@ -218,19 +298,22 @@ export class GithubAppClient extends Context.Service()( return decoded.token }) - const authedGet = (config: ResolvedAppConfig, token: string, url: string) => - Effect.tryPromise({ - try: () => - fetch(url, { - headers: { - authorization: `token ${token}`, - accept: "application/vnd.github+json", - "x-github-api-version": GITHUB_API_VERSION, - "user-agent": USER_AGENT, - }, - }), - catch: (cause) => new GithubAppError({ message: `GitHub request failed: ${url}`, cause }), - }) + const authedGet = (_config: ResolvedAppConfig, token: string, url: string) => + rateLimitedFetch( + Effect.tryPromise({ + try: () => + http.fetch(url, { + headers: { + authorization: `token ${token}`, + accept: "application/vnd.github+json", + "x-github-api-version": GITHUB_API_VERSION, + "user-agent": USER_AGENT, + }, + }), + catch: (cause) => + new GithubAppError({ message: `GitHub request failed: ${url}`, cause }), + }), + ) const listInstallationRepositories = Effect.fn("GithubAppClient.listInstallationRepositories")( function* (externalInstallationId: string) { @@ -268,44 +351,61 @@ export class GithubAppClient extends Context.Service()( }, ) + // Returns commits page-by-page until the window is exhausted. A rate limit + // too far out to ride inline (from the token mint OR any page) is caught at + // the outer level and reported as a *partial* result with the commits already + // fetched, so the caller can checkpoint + requeue rather than refetch them. const listCommits = Effect.fn("GithubAppClient.listCommits")(function* ( externalInstallationId: string, owner: string, repo: string, - params: { sha?: string; sinceIso?: string }, + params: { sha?: string; sinceIso?: string; untilIso?: string }, ) { - const config = yield* resolveConfig - const token = yield* mintInstallationToken(externalInstallationId) - const base = `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits` const commits: Array = [] - let page = 1 - for (; page <= MAX_PAGES; page++) { - const query = new URLSearchParams({ per_page: String(PER_PAGE), page: String(page) }) - if (params.sha) query.set("sha", params.sha) - if (params.sinceIso) query.set("since", params.sinceIso) - const response = yield* authedGet(config, token, `${base}?${query.toString()}`) - // 409 = empty repository → genuinely no commits, not an error. - if (response.status === 409) break - // Anything else non-2xx (incl. 404 = repo deleted / access lost) is - // surfaced as a repository-scoped failure so the orchestrator can mark - // the repo unavailable rather than mistaking it for an empty repo. - if (!response.ok) return yield* failure(response, "List commits", "repository") - const json = yield* parseJson(response, "List commits") - const decoded = yield* decodeCommitList(json).pipe( - Effect.mapError( - (cause) => new GithubAppError({ message: "Unexpected commits payload", cause }), - ), - ) - commits.push(...decoded) - if (decoded.length < PER_PAGE) break - } - // Exhausted the page cap without a short final page → likely truncated. - if (page > MAX_PAGES) { + const outcome = yield* Effect.gen(function* () { + const config = yield* resolveConfig + const token = yield* mintInstallationToken(externalInstallationId) + const base = `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits` + let page = 1 + for (; page <= MAX_PAGES; page++) { + const query = new URLSearchParams({ per_page: String(PER_PAGE), page: String(page) }) + if (params.sha) query.set("sha", params.sha) + if (params.sinceIso) query.set("since", params.sinceIso) + if (params.untilIso) query.set("until", params.untilIso) + const response = yield* authedGet(config, token, `${base}?${query.toString()}`) + // 409 = empty repository → genuinely no commits, not an error. + if (response.status === 409) return { complete: true as const } + // Anything else non-2xx (incl. 404 = repo deleted / access lost) is + // surfaced as a repository-scoped failure so the orchestrator can mark + // the repo unavailable rather than mistaking it for an empty repo. + if (!response.ok) return yield* failure(response, "List commits", "repository") + const json = yield* parseJson(response, "List commits") + const decoded = yield* decodeCommitList(json).pipe( + Effect.mapError( + (cause) => new GithubAppError({ message: "Unexpected commits payload", cause }), + ), + ) + commits.push(...decoded) + if (decoded.length < PER_PAGE) return { complete: true as const } + } + // Exhausted the page cap without a short final page → likely truncated. yield* Effect.logWarning("GitHub commit list truncated at page cap").pipe( Effect.annotateLogs({ owner, repo, maxPages: MAX_PAGES, fetched: commits.length }), ) - } - return commits + return { complete: true as const } + }).pipe( + Effect.catch((error) => + error.retryAfterSeconds === undefined + ? Effect.fail(error) + : Effect.succeed({ + complete: false as const, + retryAfterSeconds: error.retryAfterSeconds, + }), + ), + ) + return outcome.complete + ? { commits, complete: true as const } + : { commits, complete: false as const, retryAfterSeconds: outcome.retryAfterSeconds } }) const getCommit = Effect.fn("GithubAppClient.getCommit")(function* ( diff --git a/apps/api/src/services/github/GithubHttp.ts b/apps/api/src/services/github/GithubHttp.ts new file mode 100644 index 000000000..270d8cfab --- /dev/null +++ b/apps/api/src/services/github/GithubHttp.ts @@ -0,0 +1,20 @@ +import { Context, Effect, Layer } from "effect" + +// --------------------------------------------------------------------------- +// Thin seam over the platform `fetch`, so the GitHub client's rate-limit and +// pagination handling can be unit-tested by injecting canned responses. The +// default layer is the global `fetch`; tests provide a stub via Layer.succeed. +// --------------------------------------------------------------------------- + +export interface GithubHttpShape { + readonly fetch: (url: string, init?: RequestInit) => Promise +} + +export class GithubHttp extends Context.Service()( + "@maple/api/services/github/GithubHttp", + { + make: Effect.sync((): GithubHttpShape => ({ fetch: (url, init) => fetch(url, init) })), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/github/GithubProvider.ts b/apps/api/src/services/github/GithubProvider.ts index 79555dec9..a8b9267c6 100644 --- a/apps/api/src/services/github/GithubProvider.ts +++ b/apps/api/src/services/github/GithubProvider.ts @@ -7,6 +7,7 @@ import { type VcsInstallationSyncReason, VcsProviderError, type VcsProviderId, + VcsRateLimitedError, type VcsRepositoryRef, VcsRepoUnavailableError, type VcsSyncJob, @@ -16,10 +17,21 @@ import { import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" import { Env } from "../../lib/Env" import type { VcsProviderClient, VcsWebhookRequest } from "../vcs/VcsProviderClient" +import { QUEUE_MESSAGE_LIMIT_BYTES } from "../vcs/VcsSyncQueue" import { type GithubApiCommit, GithubAppClient, GithubAppError } from "./GithubAppClient" const PROVIDER: VcsProviderId = "github" +// GitHub allows up to 2048 commits per push delivery and commit messages are +// unbounded, so neither a single inline job nor a fixed commit *count* can +// guarantee staying under the queue's message cap (a squash/merge commit alone +// can carry a multi-KB message). So commits are packed into jobs by encoded byte +// size, reserving headroom below the cap (QUEUE_MESSAGE_LIMIT_BYTES, owned by the +// queue layer) for the job envelope and the queue's own serialization. Pushes are +// independent and idempotent (commits upsert by unique index), so splitting across +// jobs is safe and order-independent. +const PUSH_JOB_MAX_BYTES = QUEUE_MESSAGE_LIMIT_BYTES - 16 * 1024 // 16 KB reserve ⇒ 112 KB target + // ---- Webhook payload schemas (minimal, permissive) ------------------------ const PushAuthor = Schema.Struct({ @@ -73,14 +85,21 @@ const parsePayload = (event: string, decoded: Effect.Effect) => // Classify a GitHub HTTP failure into a semantic VCS error. HTTP-status // knowledge lives here, in the provider — the orchestrator only ever sees the -// semantic outcome. A gone/410 on the installation-auth call is the +// semantic outcome. A rate limit (carrying `retryAfterSeconds`) becomes a +// VcsRateLimitedError; a gone/410 on the installation-auth call is the // authoritative disconnect signal; on a repo call it means the repo is gone; -// everything else (incl. 401/403/429/5xx) is transient and retryable. +// everything else (incl. 401/403/5xx) is transient and retryable. const isGone = (status?: number) => status === 404 || status === 410 const toVcsError = ( error: GithubAppError, -): VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError => { +): VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRateLimitedError => { + if (error.retryAfterSeconds !== undefined) { + return new VcsRateLimitedError({ + message: error.message, + retryAfterSeconds: error.retryAfterSeconds, + }) + } if (isGone(error.status)) { if (error.scope === "installation") return new VcsInstallationGoneError({ message: error.message }) if (error.scope === "repository") return new VcsRepoUnavailableError({ message: error.message }) @@ -92,6 +111,18 @@ const toVcsError = ( }) } +// Commit fetches fold rate limits into a partial result (see `VcsCommitFetch.next`), +// so a rate-limit error never reaches this path. Narrow the mapper accordingly so +// `fetchCommits` keeps the port's 3-way error channel (no VcsRateLimitedError). +const toVcsCommitError = ( + error: GithubAppError, +): VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError => { + const mapped = toVcsError(error) + return mapped._tag === "@maple/http/errors/VcsRateLimitedError" + ? new VcsProviderError({ message: mapped.message }) + : mapped +} + const finiteOrNull = (value: number) => (Number.isFinite(value) ? value : null) const installationReason = (action: string): VcsInstallationSyncReason | null => { @@ -200,17 +231,41 @@ export class GithubProvider extends Context.Service): VcsSyncJob => ({ kind: "push", provider: PROVIDER, - externalInstallationId: String(payload.installation.id), - externalRepoId: String(payload.repository.id), + externalInstallationId, + externalRepoId, branch, - commits, + commits: slice, + }) + // Greedily pack commits into jobs that each stay under the queue cap. + // `JSON.stringify` byte length is a conservative proxy for the wire size + // (CommitUpsertInput encodes 1:1, and the queue's v8 serialization is no + // larger for this string-heavy shape). Each commit is always placed in a + // job (guaranteed progress), so a lone commit bigger than the budget — a + // pathologically huge message, which the default-branch backfill re-fetches + // in full anyway — gets its own job rather than stalling the loop. + const envelopeBytes = Buffer.byteLength(JSON.stringify(makeJob([]))) + const jobs: VcsSyncJob[] = [] + let slice: CommitUpsertInput[] = [] + let sliceBytes = envelopeBytes + for (const c of commits) { + const commitBytes = Buffer.byteLength(JSON.stringify(c)) + 1 // +1: array comma + if (slice.length > 0 && sliceBytes + commitBytes > PUSH_JOB_MAX_BYTES) { + jobs.push(makeJob(slice)) + slice = [] + sliceBytes = envelopeBytes + } + slice.push(c) + sliceBytes += commitBytes } - return [job] + jobs.push(makeJob(slice)) + return jobs }) const mapInstallationEvent = @@ -284,23 +339,35 @@ export class GithubProvider extends Context.Service Effect.gen(function* () { const now = yield* Clock.currentTimeMillis - // GitHub's `since` filters by *committer* date (matching the port's - // "committed since" contract). The newest-first ordering and the - // committer-date basis are both GitHub specifics that stay in here. - const commits = yield* client + // GitHub's `since`/`until` filter by *committer* date (matching the + // port's "committed since" contract) — a GitHub specific that stays here. + const result = yield* client .listCommits(installation.externalInstallationId, repo.owner, repo.name, { sha: repo.defaultBranch, sinceIso: new Date(opts.sinceMs).toISOString(), + ...(opts.untilMs === undefined + ? {} + : { untilIso: new Date(opts.untilMs).toISOString() }), }) - .pipe(Effect.mapError(toVcsError)) - const normalized = commits.map((c) => normalizeFetchedCommit(c, repo.defaultBranch, now)) - // GitHub's /commits endpoint returns newest-first, so the first row is - // the branch head. This ordering knowledge stays inside the provider. - return { commits: normalized, headSha: normalized[0]?.sha ?? null } + .pipe(Effect.mapError(toVcsCommitError)) + const normalized = result.commits.map((c) => + normalizeFetchedCommit(c, repo.defaultBranch, now), + ) + if (result.complete) return { commits: normalized } + // Rate-limited mid-walk: resume from the oldest committer-date we got + // (a stable watermark — re-fetching only the boundary, idempotently). + const oldestMs = + normalized.length > 0 + ? normalized.reduce((min, c) => Math.min(min, c.committedAt), Number.POSITIVE_INFINITY) + : (opts.untilMs ?? now) + return { + commits: normalized, + next: { untilMs: oldestMs, retryAfterSeconds: result.retryAfterSeconds ?? 60 }, + } }) return { diff --git a/apps/api/src/services/vcs/VcsProviderClient.ts b/apps/api/src/services/vcs/VcsProviderClient.ts index 3b9233626..d5b30d47b 100644 --- a/apps/api/src/services/vcs/VcsProviderClient.ts +++ b/apps/api/src/services/vcs/VcsProviderClient.ts @@ -6,6 +6,7 @@ import type { VcsInstallationGoneError, VcsProviderError, VcsProviderId, + VcsRateLimitedError, VcsRepositoryRef, VcsRepoUnavailableError, VcsSyncJob, @@ -36,27 +37,34 @@ export interface VcsProviderClient { input: VcsWebhookRequest, ) => Effect.Effect, VcsWebhookSignatureError | VcsWebhookParseError> - /** All repositories visible to an installation, normalized. */ + /** + * All repositories visible to an installation, normalized. A rate limit too far + * out to ride inline surfaces as `VcsRateLimitedError` (the caller redelivers the + * whole job after the delay — repo lists are small, so refetch is cheap). + */ readonly fetchRepositories: ( installation: VcsInstallation, ) => Effect.Effect< ReadonlyArray, - VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError + VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRateLimitedError > /** - * Commits on a repo's default branch *committed* since `sinceMs`, normalized, - * plus the branch head SHA (the provider knows its own ordering — see - * `VcsCommitFetch`). The `sinceMs` filter is keyed on committer date; the exact - * basis and ordering are provider-defined and never assumed by callers. The - * provider classifies failures: `VcsInstallationGoneError` (disconnect), - * `VcsRepoUnavailableError` (repo-scoped), else `VcsProviderError` (transient / - * retryable). + * Commits on a repo's default branch *committed* in `(sinceMs, untilMs]`, + * normalized. `untilMs` resumes a rate-limited backfill from a watermark; omit + * it for a fresh walk from the tip. The `sinceMs`/`untilMs` filter is keyed on + * committer date; the exact basis and ordering are provider-defined and never + * assumed by callers. + * + * A rate limit is NOT an error here: the provider returns what it fetched plus + * `VcsCommitFetch.next` (resume cursor + delay). Failures are classified as + * `VcsInstallationGoneError` (disconnect), `VcsRepoUnavailableError` (repo-scoped), + * else `VcsProviderError` (transient / retryable). */ readonly fetchCommits: ( installation: VcsInstallation, repo: VcsRepositoryRef, - opts: { readonly sinceMs: number }, + opts: { readonly sinceMs: number; readonly untilMs?: number }, ) => Effect.Effect< VcsCommitFetch, VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError diff --git a/apps/api/src/services/vcs/VcsRepository.ts b/apps/api/src/services/vcs/VcsRepository.ts index da59bdead..bf3453349 100644 --- a/apps/api/src/services/vcs/VcsRepository.ts +++ b/apps/api/src/services/vcs/VcsRepository.ts @@ -86,7 +86,6 @@ const rowToRepo = (row: VcsRepositoryRow): VcsRepo => isArchived: row.isArchived === 1, syncStatus: row.syncStatus, lastSyncedAt: row.lastSyncedAt ?? null, - lastSyncCursor: row.lastSyncCursor ?? null, lastSyncError: row.lastSyncError ?? null, createdAt: row.createdAt, updatedAt: row.updatedAt, @@ -127,9 +126,8 @@ export interface UpsertInstallationInput { readonly installedByUserId: UserId } -export interface RepoSyncCursor { +export interface RepoSyncStatusUpdate { readonly status: VcsRepoSyncStatus - readonly cursorSha?: string | null readonly error?: string | null readonly syncedAt?: number | null } @@ -343,11 +341,11 @@ export class VcsRepository extends Context.Service()("@maple/api/ .pipe(Effect.mapError(toPersistenceError)) }) - const updateRepoSyncCursor = Effect.fn("VcsRepository.updateRepoSyncCursor")(function* ( + const updateRepoSyncStatus = Effect.fn("VcsRepository.updateRepoSyncStatus")(function* ( orgId: OrgId, provider: VcsProviderId, externalRepoId: string, - cursor: RepoSyncCursor, + update: RepoSyncStatusUpdate, ) { const now = yield* Clock.currentTimeMillis yield* database @@ -355,10 +353,9 @@ export class VcsRepository extends Context.Service()("@maple/api/ db .update(vcsRepositories) .set({ - syncStatus: cursor.status, - lastSyncCursor: cursor.cursorSha ?? null, - lastSyncError: cursor.error ?? null, - lastSyncedAt: cursor.syncedAt ?? now, + syncStatus: update.status, + lastSyncError: update.error ?? null, + lastSyncedAt: update.syncedAt ?? now, updatedAt: now, }) .where( @@ -499,7 +496,7 @@ export class VcsRepository extends Context.Service()("@maple/api/ listRepositoriesByInstallation, upsertRepositories, removeRepository, - updateRepoSyncCursor, + updateRepoSyncStatus, markRepoSyncError, upsertCommits, findCommitBySha, diff --git a/apps/api/src/services/vcs/VcsSyncQueue.ts b/apps/api/src/services/vcs/VcsSyncQueue.ts index dd47850d3..5e8d7bc40 100644 --- a/apps/api/src/services/vcs/VcsSyncQueue.ts +++ b/apps/api/src/services/vcs/VcsSyncQueue.ts @@ -12,8 +12,29 @@ import { Context, Effect, Layer, Schema } from "effect" const QUEUE_BINDING = "VCS_SYNC_QUEUE" const encodeJob = Schema.encodeSync(VcsSyncJob) +// Cloudflare Queues transport limits, owned here (the only module that talks to +// the binding). Producers that must pre-size their payloads — e.g. a provider +// splitting a large push so each job fits — import these rather than hardcoding +// the platform's magic numbers. +export const QUEUE_MESSAGE_LIMIT_BYTES = 128 * 1024 // max serialized message size +export const QUEUE_MAX_DELAY_SECONDS = 86_400 // max visibility delay (24h) + +// Coerce a requested delay into the range Cloudflare accepts: a whole number of +// seconds in [0, 86_400]. Out-of-range/fractional values would otherwise make +// the binding reject the send/retry outright. +export const clampQueueDelaySeconds = (seconds: number): number => + Math.min(Math.max(0, Math.floor(seconds)), QUEUE_MAX_DELAY_SECONDS) + export interface VcsSyncQueueShape { - readonly send: (job: VcsSyncJob) => Effect.Effect + /** + * Enqueue a job. `delaySeconds` (0–86,400) holds it invisible until the delay + * elapses — used to requeue a rate-limited backfill continuation only once the + * provider's budget is back. + */ + readonly send: ( + job: VcsSyncJob, + options?: { readonly delaySeconds?: number }, + ) => Effect.Effect readonly sendBatch: (jobs: ReadonlyArray) => Effect.Effect } @@ -24,13 +45,20 @@ export class VcsSyncQueue extends Context.Service | undefined - const send = Effect.fn("VcsSyncQueue.send")(function* (job: VcsSyncJob) { + const send = Effect.fn("VcsSyncQueue.send")(function* ( + job: VcsSyncJob, + options?: { readonly delaySeconds?: number }, + ) { if (!queue) { return yield* new VcsQueueError({ message: `Missing queue binding: ${QUEUE_BINDING}` }) } const body = encodeJob(job) + const sendOptions = + options?.delaySeconds === undefined + ? undefined + : { delaySeconds: clampQueueDelaySeconds(options.delaySeconds) } yield* Effect.tryPromise({ - try: () => queue.send(body), + try: () => queue.send(body, sendOptions), catch: (cause) => new VcsQueueError({ message: cause instanceof Error ? cause.message : "queue send failed", diff --git a/apps/api/src/services/vcs/VcsSyncService.ts b/apps/api/src/services/vcs/VcsSyncService.ts index 1801d3ebe..da5d4440e 100644 --- a/apps/api/src/services/vcs/VcsSyncService.ts +++ b/apps/api/src/services/vcs/VcsSyncService.ts @@ -1,4 +1,5 @@ import { + type BackfillRepoJob, type CommitUpsertInput, isInstallationProcessable, type UnknownVcsProviderError, @@ -6,6 +7,7 @@ import { type VcsInstallationSyncReason, type VcsProviderError, type VcsQueueError, + type VcsRateLimitedError, type VcsRepoDecodeError, type VcsRepoPersistenceError, type VcsRepoUnavailableError, @@ -26,17 +28,24 @@ import { VcsSyncQueue } from "./VcsSyncQueue" const BACKFILL_DAYS = 90 const DAY_MS = 86_400_000 +// How many consecutive continuations may fetch zero commits (rate-limited before +// any progress) before we give up. Bounds a permanently throttled installation: +// a transient limit clears long before this, but a wedged one stops requeuing. +const MAX_BACKFILL_STALL_RETRIES = 10 const decodeJob = Schema.decodeUnknownEffect(VcsSyncJob) // VcsInstallationGoneError is handled internally (→ disconnect) and never // surfaces here. VcsProviderError / VcsRepoUnavailableError that aren't caught -// propagate so the queue retries. +// propagate so the queue retries. VcsRateLimitedError propagates from a +// rate-limited fetchRepositories so the consumer redelivers after the delay +// (backfill handles its own rate limits via the resume cursor, not this error). type SyncError = | VcsRepoPersistenceError | VcsRepoDecodeError | VcsProviderError | VcsRepoUnavailableError + | VcsRateLimitedError | VcsQueueError | UnknownVcsProviderError @@ -99,11 +108,11 @@ export class VcsSyncService extends Context.Service Effect.gen(function* () { const now = yield* Clock.currentTimeMillis - const { commits, headSha } = yield* provider.fetchCommits( + const { commits, next } = yield* provider.fetchCommits( installation, { externalRepoId: job.externalRepoId, @@ -111,7 +120,7 @@ export class VcsSyncService extends Context.Service100 commits sharing the exact + // committer-second) would requeue itself forever. Stop and flag instead. + if (job.untilMs !== undefined && commits.length > 0 && next.untilMs >= job.untilMs) { + yield* repo.markRepoSyncError( + installation.orgId, + installation.provider, + job.externalRepoId, + "backfill stalled: commit-date watermark did not advance", + ) + yield* Effect.logError("VCS backfill stalled — watermark did not advance").pipe( + Effect.annotateLogs({ + provider: installation.provider, + externalRepoId: job.externalRepoId, + untilMs: job.untilMs, + }), + ) + return + } + + // Stall guard: a run that fetched no commits made no progress (rate-limited + // before page 1 / at the token mint). Count consecutive such runs and stop + // once they exceed the cap, so a permanently throttled installation can't + // requeue forever. Any productive run resets the counter. + const staleAttempts = commits.length > 0 ? 0 : (job.staleAttempts ?? 0) + 1 + if (staleAttempts > MAX_BACKFILL_STALL_RETRIES) { + yield* repo.markRepoSyncError( + installation.orgId, + installation.provider, + job.externalRepoId, + "backfill stalled: rate-limited before making progress", + ) + yield* Effect.logError("VCS backfill stalled — rate-limited before any progress").pipe( + Effect.annotateLogs({ + provider: installation.provider, + externalRepoId: job.externalRepoId, + staleAttempts, + }), + ) + return + } + + // Rate-limited mid-walk → checkpoint status + requeue a continuation + // that resumes from the watermark once the budget is back. A fresh job + // (not a queue retry) keeps the retry budget for genuine failures. + yield* repo.updateRepoSyncStatus(installation.orgId, installation.provider, job.externalRepoId, { + status: "backfilling", error: null, syncedAt: now, }) + yield* queue.send( + { + ...job, + untilMs: next.untilMs, + staleAttempts, + }, + { delaySeconds: next.retryAfterSeconds }, + ) + yield* Effect.logInfo("VCS backfill rate-limited — requeued continuation").pipe( + Effect.annotateLogs({ + provider: installation.provider, + externalRepoId: job.externalRepoId, + untilMs: next.untilMs, + delaySeconds: next.retryAfterSeconds, + staleAttempts, + }), + ) }).pipe( // The provider classifies failures; the orchestrator dispatches on the // semantic outcome, never on HTTP status: @@ -159,11 +239,9 @@ export class VcsSyncService extends Context.Service }, ) { // A push is incremental enrichment only: upsert the pushed commits and - // deliberately leave the sync cursor untouched. The cursor exclusively - // tracks the default-branch commit *list* backfill, which is the only - // path that resumes from it — a push can't authoritatively advance it - // (it may target any branch and its payload may be truncated), so there - // is nothing to gain by moving it here. + // deliberately leave the repo's sync state untouched. A push may target + // any branch and its payload may be truncated, so it is never treated as + // an authoritative sync — the default-branch backfill owns that. yield* repo.upsertCommits( installation.orgId, installation.provider, diff --git a/apps/api/src/services/vcs/__tests__/vcs.test.ts b/apps/api/src/services/vcs/__tests__/vcs.test.ts index 906b78331..2786e557b 100644 --- a/apps/api/src/services/vcs/__tests__/vcs.test.ts +++ b/apps/api/src/services/vcs/__tests__/vcs.test.ts @@ -1,11 +1,13 @@ import { afterEach, assert, describe, it } from "@effect/vitest" -import { createHmac, randomUUID } from "node:crypto" +import { createHmac, generateKeyPairSync, randomUUID } from "node:crypto" import { GitCommitSha, OrgId, UserId, + VcsInstallation, VcsInstallationGoneError, VcsProviderError, + VcsRateLimitedError, VcsRepoDecodeError, VcsRepoUnavailableError, VcsSyncJob, @@ -17,6 +19,7 @@ import { DatabaseLibsqlLive } from "@/lib/DatabaseLibsqlLive" import { Env } from "@/lib/Env" import { cleanupTempDirs, createTempDbUrl, executeSql } from "@/lib/test-sqlite" import { GithubAppClient } from "@/services/github/GithubAppClient" +import { GithubHttp, type GithubHttpShape } from "@/services/github/GithubHttp" import { GithubProvider } from "@/services/github/GithubProvider" import type { VcsProviderClient } from "@/services/vcs/VcsProviderClient" import { VcsProviderRegistry, type VcsProviderRegistryShape } from "@/services/vcs/VcsProviderRegistry" @@ -53,11 +56,79 @@ const repoLayer = (url: string) => const providerLayer = () => { const env = envLayer("") - return GithubProvider.layer.pipe( - Layer.provide(Layer.mergeAll(env, GithubAppClient.layer.pipe(Layer.provide(env)))), - ) + const client = GithubAppClient.layer.pipe(Layer.provide(Layer.mergeAll(env, GithubHttp.layer))) + return GithubProvider.layer.pipe(Layer.provide(Layer.mergeAll(env, client))) +} + +// A real RSA key so mintAppJwt's crypto.subtle.importKey succeeds; the App's REST +// calls are stubbed at the GithubHttp seam below. +const APP_PRIVATE_KEY = generateKeyPairSync("rsa", { + modulusLength: 2048, + publicKeyEncoding: { type: "spki", format: "pem" }, + privateKeyEncoding: { type: "pkcs8", format: "pem" }, +}).privateKey + +const appConfig = ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_DB_URL: "", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + GITHUB_APP_WEBHOOK_SECRET: WEBHOOK_SECRET, + GITHUB_APP_ID: "123456", + GITHUB_APP_PRIVATE_KEY: APP_PRIVATE_KEY, + }), +) + +// Build a GithubProvider whose HTTP responses are scripted in call order. The +// first call is always the installation-token mint. +const stubbedProviderLayer = (responders: ReadonlyArray<() => Response>) => { + let i = 0 + const http = Layer.succeed(GithubHttp, { + fetch: async () => { + const make = responders[Math.min(i, responders.length - 1)]! + i += 1 + return make() + }, + } satisfies GithubHttpShape) + const env = Env.layer.pipe(Layer.provide(appConfig)) + const client = GithubAppClient.layer.pipe(Layer.provide(Layer.mergeAll(env, http))) + return GithubProvider.layer.pipe(Layer.provide(Layer.mergeAll(env, client))) } +const jsonResponse = (body: unknown, init?: { status?: number; headers?: Record }) => + new Response(JSON.stringify(body), { + status: init?.status ?? 200, + headers: { "content-type": "application/json", ...init?.headers }, + }) + +const tokenResponse = () => jsonResponse({ token: "ghs_test", expires_at: "2099-01-01T00:00:00Z" }) + +const commitJson = (sha: string) => ({ + sha, + html_url: `https://github.com/octo/repo/commit/${sha}`, + commit: { + message: "m", + author: { name: "A", email: "a@x.io", date: "2026-01-01T00:00:00Z" }, + committer: { date: "2026-01-01T00:00:00Z" }, + }, + author: { login: "octo" }, +}) + +const commitsResponse = (shas: ReadonlyArray) => jsonResponse(shas.map(commitJson)) + +// 429 carrying retry-after (seconds): 0 ⇒ ride out inline; large ⇒ defer. +const rateLimited = (retryAfterSeconds: number) => + new Response("rate limited", { status: 429, headers: { "retry-after": String(retryAfterSeconds) } }) + +const hexShas = (count: number) => + Array.from({ length: count }, (_, n) => n.toString(16).padStart(40, "0")) + const asOrgId = Schema.decodeUnknownSync(OrgId) const asUserId = Schema.decodeUnknownSync(UserId) @@ -114,7 +185,7 @@ describe("GithubProvider.webhookToJobs", () => { ], }) - it.effect("maps a validly-signed push to a push job (no head SHA — push never moves the cursor)", () => + it.effect("maps a validly-signed push to a push job", () => Effect.gen(function* () { const provider = yield* GithubProvider const jobs = yield* provider.webhookToJobs({ @@ -131,8 +202,6 @@ describe("GithubProvider.webhookToJobs", () => { assert.strictEqual(job.commits.length, 1) assert.strictEqual(job.commits[0]!.sha, SHA) assert.strictEqual(job.commits[0]!.authorLogin, "octocat") - // A push carries no headSha — the payload's `after` is intentionally ignored. - assert.ok(!("headSha" in job)) }).pipe(Effect.provide(providerLayer())), ) @@ -150,6 +219,49 @@ describe("GithubProvider.webhookToJobs", () => { }).pipe(Effect.provide(providerLayer())), ) + // A Cloudflare Queue message caps at 128 KB and commit messages are unbounded, + // so a large push must split into multiple jobs packed by *byte size* — each + // independently enqueueable — rather than relying on a fixed commit count. + it.effect("splits a large push into multiple jobs that each stay under the 128 KB queue cap", () => + Effect.gen(function* () { + const QUEUE_MESSAGE_LIMIT = 128 * 1024 + const provider = yield* GithubProvider + const shas = hexShas(400) + const message = "x".repeat(1024) // ~1 KB messages ⇒ ~440 KB total ⇒ several jobs + const body = JSON.stringify({ + ref: "refs/heads/main", + repository: { id: 7, owner: { login: "octo" } }, + installation: { id: 42 }, + commits: shas.map((sha) => ({ + id: sha, + message, + timestamp: "2026-01-01T00:00:00Z", + url: `https://github.com/octo/repo/commit/${sha}`, + author: { name: "Octo Cat", email: "octo@x.io", username: "octocat" }, + })), + }) + const jobs = yield* provider.webhookToJobs({ + headers: { "x-github-event": "push", "x-hub-signature-256": sign(body) }, + rawBody: body, + }) + assert.ok(jobs.length > 1) // the push was split across multiple jobs + for (const job of jobs) { + assert.strictEqual(job.kind, "push") + if (job.kind !== "push") return + // Every job is independently enqueueable, regardless of the (count-blind) split. + const wireBytes = Buffer.byteLength(JSON.stringify(Schema.encodeSync(VcsSyncJob)(job))) + assert.ok(wireBytes < QUEUE_MESSAGE_LIMIT) + // All slices share the same provider/installation/repo/branch. + assert.strictEqual(job.externalInstallationId, "42") + assert.strictEqual(job.externalRepoId, "7") + assert.strictEqual(job.branch, "main") + } + // Every commit is preserved across the slices, in order — none dropped. + const splitShas = jobs.flatMap((job) => (job.kind === "push" ? job.commits.map((c) => c.sha) : [])) + assert.deepStrictEqual(splitShas, shas) + }).pipe(Effect.provide(providerLayer())), + ) + it.effect("maps an installation 'created' event to an installation-sync job", () => Effect.gen(function* () { const provider = yield* GithubProvider @@ -257,6 +369,7 @@ describe("VcsSyncService orchestrator", () => { interface StubOpts { readonly sent: Array + readonly sentDelays?: Array readonly repos?: ReadonlyArray<{ externalRepoId: string owner: string @@ -268,11 +381,12 @@ describe("VcsSyncService orchestrator", () => { isArchived: boolean }> readonly commits?: ReadonlyArray> - readonly headSha?: string | null + readonly commitFetchNext?: { untilMs: number; retryAfterSeconds: number } readonly fetchCommitsError?: | VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError + readonly fetchReposError?: VcsRateLimitedError | VcsProviderError | VcsInstallationGoneError } // Real VcsRepository (temp D1) + stubbed provider/queue ports, so dispatch, @@ -281,18 +395,28 @@ describe("VcsSyncService orchestrator", () => { const fakeProvider: VcsProviderClient = { id: "github", webhookToJobs: () => Effect.succeed([]), - fetchRepositories: () => Effect.succeed(opts.repos ?? []), + fetchRepositories: () => + opts.fetchReposError + ? Effect.fail(opts.fetchReposError) + : Effect.succeed(opts.repos ?? []), fetchCommits: () => opts.fetchCommitsError ? Effect.fail(opts.fetchCommitsError) - : Effect.succeed({ commits: opts.commits ?? [], headSha: opts.headSha ?? null }), + : Effect.succeed({ + commits: opts.commits ?? [], + ...(opts.commitFetchNext ? { next: opts.commitFetchNext } : {}), + }), } const registry = Layer.succeed(VcsProviderRegistry, { ids: ["github"], resolve: () => Effect.succeed(fakeProvider), } satisfies VcsProviderRegistryShape) const queue = Layer.succeed(VcsSyncQueue, { - send: (job) => Effect.sync(() => void opts.sent.push(job)), + send: (job, options) => + Effect.sync(() => { + opts.sent.push(job) + opts.sentDelays?.push(options?.delaySeconds) + }), sendBatch: (jobs) => Effect.sync(() => void opts.sent.push(...jobs)), } satisfies VcsSyncQueueShape) const repoLive = VcsRepository.layer.pipe( @@ -337,7 +461,7 @@ describe("VcsSyncService orchestrator", () => { }).pipe(Effect.provide(orchestratorLayer(url, { sent }))) }) - it.effect("push upserts every commit and never moves the repo sync cursor", () => { + it.effect("push upserts every commit and never changes repo sync state", () => { const { url } = createTempDbUrl("maple-vcs-orch-push-", dirs) const sent: Array = [] return Effect.gen(function* () { @@ -358,10 +482,9 @@ describe("VcsSyncService orchestrator", () => { const a = yield* repo.findCommitBySha(orgId, SHA_A as never) const b = yield* repo.findCommitBySha(orgId, SHA_B as never) assert.ok(Option.isSome(a) && Option.isSome(b)) - // B2: a push is pure enrichment — the cursor/status stay exactly as the - // backfill left them (here: untouched since no backfill has run). + // B2: a push is pure enrichment — the sync status stays exactly as the + // backfill left it (here: untouched since no backfill has run). const stored = yield* repo.listRepositoriesByInstallation("github", "42") - assert.strictEqual(stored[0]!.lastSyncCursor, null) assert.strictEqual(stored[0]!.syncStatus, "pending") }).pipe(Effect.provide(orchestratorLayer(url, { sent }))) }) @@ -401,11 +524,9 @@ describe("VcsSyncService orchestrator", () => { }).pipe(Effect.provide(orchestratorLayer(url, { sent, repos }))) }) - it.effect("backfill sets the cursor to the provider's head, not commit array order", () => { + it.effect("backfill persists fetched commits and marks the repo ready", () => { const { url } = createTempDbUrl("maple-vcs-orch-backfill-", dirs) const sent: Array = [] - // commits[0] is deliberately NOT the head — the cursor must come from the - // provider-supplied headSha, proving no array-order inference. const commits = [commit(SHA_A, 1), commit(SHA_B, 2)] return Effect.gen(function* () { const svc = yield* VcsSyncService @@ -435,10 +556,12 @@ describe("VcsSyncService orchestrator", () => { sinceMs: 0, } yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) + const a = yield* repo.findCommitBySha(orgId, SHA_A as never) + const b = yield* repo.findCommitBySha(orgId, SHA_B as never) + assert.ok(Option.isSome(a) && Option.isSome(b)) const stored = yield* repo.listRepositoriesByInstallation("github", "42") assert.strictEqual(stored[0]!.syncStatus, "ready") - assert.strictEqual(stored[0]!.lastSyncCursor, SHA_B) - }).pipe(Effect.provide(orchestratorLayer(url, { sent, commits, headSha: SHA_B }))) + }).pipe(Effect.provide(orchestratorLayer(url, { sent, commits }))) }) const seedRepo = (repo: VcsRepository, orgId: ReturnType) => @@ -603,6 +726,108 @@ describe("VcsSyncService orchestrator", () => { assert.strictEqual(sent[0]!.kind, "backfill-repo") }).pipe(Effect.provide(orchestratorLayer(url, { sent, repos }))) }) + + // A rate-limited backfill checkpoints + requeues a delayed continuation rather + // than failing — no retry budget spent, finished pages not refetched. + it.effect("a rate-limited backfill requeues a continuation with a cursor + delay", () => { + const { url } = createTempDbUrl("maple-vcs-orch-backfill-rl-", dirs) + const sent: Array = [] + const sentDelays: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* seedRepo(repo, orgId) + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(backfillJob)) // must not fail + // The fetched commit was persisted… + assert.ok(Option.isSome(yield* repo.findCommitBySha(orgId, SHA_A as never))) + // …the repo is marked backfilling (in progress, not ready)… + const stored = yield* repo.listRepositoriesByInstallation("github", "42") + assert.strictEqual(stored[0]!.syncStatus, "backfilling") + // …and a continuation was requeued from the watermark, delayed until reset. + assert.strictEqual(sent.length, 1) + const continuation = sent[0]! + assert.strictEqual(continuation.kind, "backfill-repo") + if (continuation.kind !== "backfill-repo") return + assert.strictEqual(continuation.untilMs, 5000) + assert.strictEqual(sentDelays[0], 600) + }).pipe( + Effect.provide( + orchestratorLayer(url, { + sent, + sentDelays, + commits: [commit(SHA_A, 1)], + commitFetchNext: { untilMs: 5000, retryAfterSeconds: 600 }, + }), + ), + ) + }) + + // A backfill that keeps getting rate-limited *before any commit* must not + // requeue forever — after the stall cap it errors the repo instead. + it.effect("a backfill with no progress stops after the stall cap", () => { + const STALL_CAP = 10 // mirrors MAX_BACKFILL_STALL_RETRIES in VcsSyncService + const { url } = createTempDbUrl("maple-vcs-orch-stall-", dirs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* seedRepo(repo, orgId) + // Drive the continuation back through the consumer; every run fetches zero + // commits (still throttled), so the watermark never moves. + let job: VcsSyncJob = backfillJob + for (let i = 0; i <= STALL_CAP; i++) { + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)) + if (sent.length > 0) job = sent[sent.length - 1]! + } + // It requeued exactly the cap's worth of continuations, then gave up. + assert.strictEqual(sent.length, STALL_CAP) + const stored = yield* repo.listRepositoriesByInstallation("github", "42") + assert.strictEqual(stored[0]!.syncStatus, "error") + }).pipe( + Effect.provide( + orchestratorLayer(url, { + sent, + commits: [], // zero progress on every run + commitFetchNext: { untilMs: 5000, retryAfterSeconds: 600 }, + }), + ), + ) + }) + + // A rate-limited installation-sync isn't resumable — it propagates so the + // consumer redelivers the whole (small) job after the delay. + it.effect("a rate-limited installation-sync propagates VcsRateLimitedError", () => { + const { url } = createTempDbUrl("maple-vcs-orch-inst-rl-", dirs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + const job: VcsSyncJob = { + kind: "installation-sync", + provider: "github", + externalInstallationId: "42", + reason: "created", + } + const exit = yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(job)).pipe(Effect.exit) + assert.ok(Exit.isFailure(exit)) + const error = findError(exit) + assert.ok(error instanceof VcsRateLimitedError) + assert.strictEqual((error as VcsRateLimitedError).retryAfterSeconds, 600) + }).pipe( + Effect.provide( + orchestratorLayer(url, { + sent, + fetchReposError: new VcsRateLimitedError({ message: "rate limited", retryAfterSeconds: 600 }), + }), + ), + ) + }) }) // The SHA-shape regex lives only in the GitCommitSha brand; these assert that @@ -667,3 +892,83 @@ describe("git SHA validation (branded type)", () => { }).pipe(Effect.provide(repoLayer(url))) }) }) + +// The centralized GitHub fetch detects 429s and decides: ride out short waits +// inline; surface long ones (backfill → partial `next`; repos → VcsRateLimitedError). +describe("GithubProvider rate-limit handling", () => { + const REPO = { externalRepoId: "7", owner: "octo", name: "repo", defaultBranch: "main" } + const installation = Schema.decodeUnknownSync(VcsInstallation)({ + id: randomUUID(), + orgId: "org_test", + provider: "github", + externalInstallationId: "123456", + accountLogin: "octo", + accountType: "organization", + externalAccountId: "1", + accountAvatarUrl: null, + repositorySelection: "all", + status: "active", + suspendedAt: null, + installedByUserId: "user_1", + createdAt: 0, + updatedAt: 0, + }) + + it.effect("rides out a short rate limit inline, then completes", () => + Effect.gen(function* () { + const provider = yield* GithubProvider + const result = yield* provider.fetchCommits(installation, REPO, { sinceMs: 0 }) + assert.strictEqual(result.commits.length, 1) + assert.strictEqual(result.next, undefined) // retried past the 429 → window complete + }).pipe( + // token mint → page 1 (429, retry-after 0 → inline retry) → page 1 (commits) + Effect.provide( + stubbedProviderLayer([tokenResponse, () => rateLimited(0), () => commitsResponse(["a".repeat(40)])]), + ), + ), + ) + + it.effect("surfaces a long rate limit mid-walk as a partial result with `next`", () => + Effect.gen(function* () { + const provider = yield* GithubProvider + const result = yield* provider.fetchCommits(installation, REPO, { sinceMs: 0 }) + assert.strictEqual(result.commits.length, 100) // page 1 kept, not thrown away + assert.ok(result.next !== undefined) + assert.strictEqual(result.next?.retryAfterSeconds, 600) + }).pipe( + // token → page 1 (full) → page 2 (429, retry-after 600 → defer) + Effect.provide( + stubbedProviderLayer([tokenResponse, () => commitsResponse(hexShas(100)), () => rateLimited(600)]), + ), + ), + ) + + it.effect("a long rate limit on fetchRepositories raises VcsRateLimitedError", () => + Effect.gen(function* () { + const provider = yield* GithubProvider + const exit = yield* provider.fetchRepositories(installation).pipe(Effect.exit) + assert.ok(Exit.isFailure(exit)) + const error = findError(exit) + assert.ok(error instanceof VcsRateLimitedError) + assert.strictEqual((error as VcsRateLimitedError).retryAfterSeconds, 600) + }).pipe( + // token → repos page 1 (429, retry-after 600 → surfaced, not resumable) + Effect.provide(stubbedProviderLayer([tokenResponse, () => rateLimited(600)])), + ), + ) + + it.effect("stops riding out a rate limit after the inline-retry cap and defers", () => + Effect.gen(function* () { + const provider = yield* GithubProvider + // Every page replies with a 0s-wait 429; without a retry cap this would spin + // forever. The cap surfaces it as a deferral instead, floored off 0. + const result = yield* provider.fetchCommits(installation, REPO, { sinceMs: 0 }) + assert.strictEqual(result.commits.length, 0) + assert.ok(result.next !== undefined) + assert.strictEqual(result.next?.retryAfterSeconds, 60) + }).pipe( + // token → page 1 (429 retry-after 0, repeated past the inline cap) + Effect.provide(stubbedProviderLayer([tokenResponse, () => rateLimited(0)])), + ), + ) +}) diff --git a/apps/api/src/vcs-sync-runtime.ts b/apps/api/src/vcs-sync-runtime.ts index 9a283b63e..317eac97f 100644 --- a/apps/api/src/vcs-sync-runtime.ts +++ b/apps/api/src/vcs-sync-runtime.ts @@ -1,14 +1,15 @@ import type { MessageBatch } from "@cloudflare/workers-types" import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" import { WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" -import { Cause, Effect, Layer } from "effect" +import { Cause, Effect, Layer, Option } from "effect" import { DatabaseD1Live } from "./lib/DatabaseD1Live" import { Env } from "./lib/Env" import { GithubAppClient } from "./services/github/GithubAppClient" +import { GithubHttp } from "./services/github/GithubHttp" import { GithubProvider } from "./services/github/GithubProvider" import { VcsProviderRegistry } from "./services/vcs/VcsProviderRegistry" import { VcsRepository } from "./services/vcs/VcsRepository" -import { VcsSyncQueue } from "./services/vcs/VcsSyncQueue" +import { clampQueueDelaySeconds, VcsSyncQueue } from "./services/vcs/VcsSyncQueue" import { VcsSyncService } from "./services/vcs/VcsSyncService" // --------------------------------------------------------------------------- @@ -30,7 +31,9 @@ export const buildVcsSyncLayer = (_env: Record) => { const Base = Layer.mergeAll(EnvLive, DatabaseLive, WorkerEnvironment.layer) const VcsRepositoryLive = VcsRepository.layer.pipe(Layer.provide(Base)) - const GithubAppClientLive = GithubAppClient.layer.pipe(Layer.provide(EnvLive)) + const GithubAppClientLive = GithubAppClient.layer.pipe( + Layer.provide(Layer.mergeAll(EnvLive, GithubHttp.layer)), + ) const GithubProviderLive = GithubProvider.layer.pipe( Layer.provide(Layer.mergeAll(EnvLive, GithubAppClientLive)), ) @@ -56,11 +59,31 @@ export const processBatch = (batch: MessageBatch) => (message) => service.processMessage(message.body).pipe( Effect.matchCauseEffect({ - onFailure: (cause) => - Effect.logError("VCS sync message failed").pipe( - Effect.annotateLogs({ error: Cause.pretty(cause) }), - Effect.flatMap(() => Effect.sync(() => message.retry())), - ), + onFailure: (cause) => { + // A rate limit too far out to ride inline → redeliver this message + // only once the VCS's budget is back, instead of an immediate retry. + const failure = Option.getOrUndefined(Cause.findErrorOption(cause)) + + const delaySeconds = + failure?._tag === "@maple/http/errors/VcsRateLimitedError" + ? clampQueueDelaySeconds(failure.retryAfterSeconds) + : undefined + const isDelaySecondsSet = delaySeconds !== undefined + + return Effect.logError("VCS sync message failed").pipe( + Effect.annotateLogs({ + error: Cause.pretty(cause), + ...(isDelaySecondsSet ? { retryDelaySeconds: delaySeconds } : {}), + }), + Effect.flatMap(() => + Effect.sync(() => + isDelaySecondsSet + ? message.retry({ delaySeconds }) + : message.retry(), + ), + ), + ) + }, onSuccess: () => Effect.sync(() => message.ack()), }), ), diff --git a/packages/db/drizzle/0022_naive_enchantress.sql b/packages/db/drizzle/0022_naive_enchantress.sql index 620cf2f62..b1b4f7f71 100644 --- a/packages/db/drizzle/0022_naive_enchantress.sql +++ b/packages/db/drizzle/0022_naive_enchantress.sql @@ -52,7 +52,6 @@ CREATE TABLE `vcs_repositories` ( `is_archived` integer DEFAULT 0 NOT NULL, `sync_status` text DEFAULT 'pending' NOT NULL, `last_synced_at` integer, - `last_sync_cursor` text, `last_sync_error` text, `created_at` integer NOT NULL, `updated_at` integer NOT NULL diff --git a/packages/db/drizzle/meta/0022_snapshot.json b/packages/db/drizzle/meta/0022_snapshot.json index 98ee81e43..03db861ea 100644 --- a/packages/db/drizzle/meta/0022_snapshot.json +++ b/packages/db/drizzle/meta/0022_snapshot.json @@ -4518,13 +4518,6 @@ "notNull": false, "autoincrement": false }, - "last_sync_cursor": { - "name": "last_sync_cursor", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, "last_sync_error": { "name": "last_sync_error", "type": "text", diff --git a/packages/db/src/schema/vcs.ts b/packages/db/src/schema/vcs.ts index 645b795de..98ccc23e9 100644 --- a/packages/db/src/schema/vcs.ts +++ b/packages/db/src/schema/vcs.ts @@ -47,7 +47,7 @@ export const vcsInstallations = sqliteTable( ], ) -/** Repositories accessible to an installation, plus a per-repo sync cursor. */ +/** Repositories accessible to an installation, plus per-repo sync state. */ export const vcsRepositories = sqliteTable( "vcs_repositories", { @@ -65,7 +65,6 @@ export const vcsRepositories = sqliteTable( isArchived: integer("is_archived", { mode: "number" }).notNull().default(0), syncStatus: text("sync_status").$type().notNull().default("pending"), lastSyncedAt: integer("last_synced_at", { mode: "number" }), - lastSyncCursor: text("last_sync_cursor"), lastSyncError: text("last_sync_error"), createdAt: integer("created_at", { mode: "number" }).notNull(), updatedAt: integer("updated_at", { mode: "number" }).notNull(), diff --git a/packages/domain/src/http/vcs.ts b/packages/domain/src/http/vcs.ts index 8a266464f..748b9c0de 100644 --- a/packages/domain/src/http/vcs.ts +++ b/packages/domain/src/http/vcs.ts @@ -133,7 +133,6 @@ export class VcsRepo extends Schema.Class("VcsRepo")({ isArchived: Schema.Boolean, syncStatus: VcsRepoSyncStatus, lastSyncedAt: Schema.NullOr(Schema.Number), - lastSyncCursor: Schema.NullOr(Schema.String), lastSyncError: Schema.NullOr(Schema.String), createdAt: Schema.Number, updatedAt: Schema.Number, @@ -188,14 +187,18 @@ export const CommitUpsertInput = Schema.Struct({ export type CommitUpsertInput = Schema.Schema.Type /** - * Result of a provider commit fetch: the commits plus the current head SHA of - * the fetched ref. The provider determines `headSha` (it alone knows its own - * response ordering / the authoritative tip); callers must never infer the head - * from commit array position. `null` when the ref has no commits. + * Result of a provider commit fetch: the normalized commits, plus an optional + * resume cursor. The branch head is deliberately NOT reported — incremental sync + * derives its watermark from `max(committed_at)` of the persisted commits, so no + * provider has to claim a head and no caller infers one from array position. + * + * `next` is present iff a rate limit cut the walk short before the requested + * window was fully fetched: resume the backfill from `untilMs` (a committer-date + * watermark) after waiting `retryAfterSeconds`. Absent ⇒ the window is complete. */ export interface VcsCommitFetch { readonly commits: ReadonlyArray - readonly headSha: string | null + readonly next?: { readonly untilMs: number; readonly retryAfterSeconds: number } } /** Minimal repo identity a provider needs to fetch commits. */ @@ -239,15 +242,22 @@ export const BackfillRepoJob = Schema.Struct({ name: Schema.String, defaultBranch: Schema.String, sinceMs: Schema.Number, + // Resume cursor for a continuation requeued after a rate limit: fetch commits + // committed at-or-before `untilMs` (a committer-date watermark). Absent on a + // fresh backfill. + untilMs: Schema.optionalKey(Schema.Number), + // Count of consecutive continuations that fetched zero commits (rate-limited + // before any progress). Reset to 0 whenever a run makes progress; bounded so a + // permanently throttled installation can't requeue forever. Absent ⇒ 0. + staleAttempts: Schema.optionalKey(Schema.Number), }) export type BackfillRepoJob = Schema.Schema.Type -// A push event's commits, applied incrementally. NOT a cursor-advancing sync: -// the cursor only ever tracks the default-branch commit *list* (see -// `BackfillRepoJob`), so a push carries no head SHA and never moves it. A push -// payload may also be incomplete (GitHub caps `commits` at 2048 per delivery and -// sends one delivery per push, not many) — the authoritative fill-in is the -// default-branch backfill, so a push is purely best-effort enrichment. +// A push event's commits, applied incrementally — purely best-effort enrichment. +// A push may target any branch and its payload may be incomplete (GitHub caps +// `commits` at 2048 per delivery and sends one delivery per push, not many), so +// it is never treated as an authoritative sync: the default-branch backfill is +// the source of truth and re-fetches the full history regardless. export const PushJob = Schema.Struct({ kind: Schema.Literal("push"), provider: VcsProviderId, @@ -314,6 +324,18 @@ export class VcsRepoUnavailableError extends Schema.TaggedErrorClass()( + "@maple/http/errors/VcsRateLimitedError", + { message: Schema.String, retryAfterSeconds: Schema.Number }, + { httpApiStatus: 429 }, +) {} + export class VcsWebhookSignatureError extends Schema.TaggedErrorClass()( "@maple/http/errors/VcsWebhookSignatureError", { message: Schema.String }, From eb2cfce73d2c662cfd3bc174be11c926d4ba5e65 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Sun, 14 Jun 2026 23:06:53 +0200 Subject: [PATCH 06/45] hadnle page max buget to avoid hitting wall clock limit --- .../src/services/github/GithubAppClient.ts | 46 +++++++++---- .../api/src/services/github/GithubProvider.ts | 12 +++- .../api/src/services/vcs/VcsProviderClient.ts | 10 +-- apps/api/src/services/vcs/VcsSyncService.ts | 16 +++-- .../src/services/vcs/__tests__/vcs.test.ts | 69 ++++++++++++++++++- packages/domain/src/http/vcs.ts | 17 +++-- 6 files changed, 138 insertions(+), 32 deletions(-) diff --git a/apps/api/src/services/github/GithubAppClient.ts b/apps/api/src/services/github/GithubAppClient.ts index e826b6d65..30f0b500a 100644 --- a/apps/api/src/services/github/GithubAppClient.ts +++ b/apps/api/src/services/github/GithubAppClient.ts @@ -30,6 +30,14 @@ const PER_PAGE = 100 // Paginate effectively to the end (up to 100k items) while still bounding a // pathological loop. Hitting this cap is logged — truncation is never silent. const MAX_PAGES = 1000 +// Commit pages walked per consumer invocation before yielding a continuation. +// Each page is a sequential GitHub round-trip; a single backfill can span an +// unbounded history, so we cap the work per invocation to keep its wall-clock +// far under Cloudflare Queues' 15-min consumer limit. The remainder resumes from +// a committer-date watermark in a follow-up job, so a full history is walked +// across many short invocations — there is no per-invocation history cap here +// (unlike `MAX_PAGES`); the walk simply continues rather than truncating. +const COMMIT_PAGES_PER_INVOCATION = 25 // Ride out short rate limits inline; anything longer is surfaced so the caller // can defer (backfill requeues from a cursor; other jobs get a delayed retry). const INLINE_BACKOFF_CAP_S = 30 @@ -351,10 +359,14 @@ export class GithubAppClient extends Context.Service()( }, ) - // Returns commits page-by-page until the window is exhausted. A rate limit - // too far out to ride inline (from the token mint OR any page) is caught at - // the outer level and reported as a *partial* result with the commits already - // fetched, so the caller can checkpoint + requeue rather than refetch them. + // Returns commits page-by-page until the window is exhausted OR the + // per-invocation page budget is hit. Two ways a walk is cut short, both + // reported as a *partial* result (commits kept, never refetched) so the + // caller can checkpoint + requeue: + // - `"rate-limited"`: a rate limit too far out to ride inline (from the + // token mint OR any page), caught at the outer level. + // - `"page-budget"`: `COMMIT_PAGES_PER_INVOCATION` full pages fetched with + // more to come — yield so one invocation stays under the queue's limit. const listCommits = Effect.fn("GithubAppClient.listCommits")(function* ( externalInstallationId: string, owner: string, @@ -366,8 +378,7 @@ export class GithubAppClient extends Context.Service()( const config = yield* resolveConfig const token = yield* mintInstallationToken(externalInstallationId) const base = `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits` - let page = 1 - for (; page <= MAX_PAGES; page++) { + for (let page = 1; page <= COMMIT_PAGES_PER_INVOCATION; page++) { const query = new URLSearchParams({ per_page: String(PER_PAGE), page: String(page) }) if (params.sha) query.set("sha", params.sha) if (params.sinceIso) query.set("since", params.sinceIso) @@ -388,24 +399,31 @@ export class GithubAppClient extends Context.Service()( commits.push(...decoded) if (decoded.length < PER_PAGE) return { complete: true as const } } - // Exhausted the page cap without a short final page → likely truncated. - yield* Effect.logWarning("GitHub commit list truncated at page cap").pipe( - Effect.annotateLogs({ owner, repo, maxPages: MAX_PAGES, fetched: commits.length }), - ) - return { complete: true as const } + // Hit the per-invocation page budget with a full final page → more + // remain. Yield a continuation (NOT a truncation): the caller resumes + // from a committer-date watermark in a follow-up job, keeping each + // invocation's wall-clock far under the Queues 15-min consumer limit. + return { complete: false as const, reason: "page-budget" as const } }).pipe( Effect.catch((error) => error.retryAfterSeconds === undefined ? Effect.fail(error) : Effect.succeed({ complete: false as const, + reason: "rate-limited" as const, retryAfterSeconds: error.retryAfterSeconds, }), ), ) - return outcome.complete - ? { commits, complete: true as const } - : { commits, complete: false as const, retryAfterSeconds: outcome.retryAfterSeconds } + if (outcome.complete) return { commits, complete: true as const } + return outcome.reason === "rate-limited" + ? { + commits, + complete: false as const, + reason: "rate-limited" as const, + retryAfterSeconds: outcome.retryAfterSeconds, + } + : { commits, complete: false as const, reason: "page-budget" as const } }) const getCommit = Effect.fn("GithubAppClient.getCommit")(function* ( diff --git a/apps/api/src/services/github/GithubProvider.ts b/apps/api/src/services/github/GithubProvider.ts index a8b9267c6..dc3608f0c 100644 --- a/apps/api/src/services/github/GithubProvider.ts +++ b/apps/api/src/services/github/GithubProvider.ts @@ -358,15 +358,21 @@ export class GithubProvider extends Context.Service 0 ? normalized.reduce((min, c) => Math.min(min, c.committedAt), Number.POSITIVE_INFINITY) : (opts.untilMs ?? now) return { commits: normalized, - next: { untilMs: oldestMs, retryAfterSeconds: result.retryAfterSeconds ?? 60 }, + next: { + untilMs: oldestMs, + reason: result.reason, + retryAfterSeconds: result.reason === "rate-limited" ? result.retryAfterSeconds : 0, + }, } }) diff --git a/apps/api/src/services/vcs/VcsProviderClient.ts b/apps/api/src/services/vcs/VcsProviderClient.ts index d5b30d47b..ace1b3f87 100644 --- a/apps/api/src/services/vcs/VcsProviderClient.ts +++ b/apps/api/src/services/vcs/VcsProviderClient.ts @@ -56,10 +56,12 @@ export interface VcsProviderClient { * committer date; the exact basis and ordering are provider-defined and never * assumed by callers. * - * A rate limit is NOT an error here: the provider returns what it fetched plus - * `VcsCommitFetch.next` (resume cursor + delay). Failures are classified as - * `VcsInstallationGoneError` (disconnect), `VcsRepoUnavailableError` (repo-scoped), - * else `VcsProviderError` (transient / retryable). + * Being cut short is NOT an error here: on a rate limit, OR after a bounded + * number of pages (so one invocation's wall-clock stays under the queue limit), + * the provider returns what it fetched plus `VcsCommitFetch.next` (resume cursor + * + delay + reason). Failures are classified as `VcsInstallationGoneError` + * (disconnect), `VcsRepoUnavailableError` (repo-scoped), else `VcsProviderError` + * (transient / retryable). */ readonly fetchCommits: ( installation: VcsInstallation, diff --git a/apps/api/src/services/vcs/VcsSyncService.ts b/apps/api/src/services/vcs/VcsSyncService.ts index da5d4440e..8d635ca95 100644 --- a/apps/api/src/services/vcs/VcsSyncService.ts +++ b/apps/api/src/services/vcs/VcsSyncService.ts @@ -181,9 +181,12 @@ export class VcsSyncService extends Context.Service { isArchived: boolean }> readonly commits?: ReadonlyArray> - readonly commitFetchNext?: { untilMs: number; retryAfterSeconds: number } + readonly commitFetchNext?: { + untilMs: number + retryAfterSeconds: number + reason: "rate-limited" | "page-budget" + } readonly fetchCommitsError?: | VcsProviderError | VcsInstallationGoneError @@ -758,7 +762,45 @@ describe("VcsSyncService orchestrator", () => { sent, sentDelays, commits: [commit(SHA_A, 1)], - commitFetchNext: { untilMs: 5000, retryAfterSeconds: 600 }, + commitFetchNext: { untilMs: 5000, retryAfterSeconds: 600, reason: "rate-limited" }, + }), + ), + ) + }) + + // A page-budget continuation (the walk yielded to stay under the queue's 15-min + // limit, NOT throttled) checkpoints and requeues to continue *immediately*. + it.effect("a page-budget backfill requeues a continuation with no delay", () => { + const { url } = createTempDbUrl("maple-vcs-orch-backfill-budget-", dirs) + const sent: Array = [] + const sentDelays: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* seedRepo(repo, orgId) + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(backfillJob)) // must not fail + // The fetched page was persisted and the repo marked backfilling… + assert.ok(Option.isSome(yield* repo.findCommitBySha(orgId, SHA_A as never))) + const stored = yield* repo.listRepositoriesByInstallation("github", "42") + assert.strictEqual(stored[0]!.syncStatus, "backfilling") + // …and a continuation was requeued from the watermark with NO delay… + assert.strictEqual(sent.length, 1) + const continuation = sent[0]! + assert.strictEqual(continuation.kind, "backfill-repo") + if (continuation.kind !== "backfill-repo") return + assert.strictEqual(continuation.untilMs, 5000) + assert.strictEqual(sentDelays[0], 0) + // …and it never counts against the stall cap (it made progress). + assert.strictEqual(continuation.staleAttempts, 0) + }).pipe( + Effect.provide( + orchestratorLayer(url, { + sent, + sentDelays, + commits: [commit(SHA_A, 1)], + commitFetchNext: { untilMs: 5000, retryAfterSeconds: 0, reason: "page-budget" }, }), ), ) @@ -792,7 +834,7 @@ describe("VcsSyncService orchestrator", () => { orchestratorLayer(url, { sent, commits: [], // zero progress on every run - commitFetchNext: { untilMs: 5000, retryAfterSeconds: 600 }, + commitFetchNext: { untilMs: 5000, retryAfterSeconds: 600, reason: "rate-limited" }, }), ), ) @@ -971,4 +1013,25 @@ describe("GithubProvider rate-limit handling", () => { Effect.provide(stubbedProviderLayer([tokenResponse, () => rateLimited(0)])), ), ) + + // Not throttled — the walk voluntarily yields after the per-invocation page + // budget so one consumer invocation can't approach the Queues 15-min limit. + it.effect("yields a page-budget continuation when the per-invocation page cap is hit", () => + Effect.gen(function* () { + // Mirrors COMMIT_PAGES_PER_INVOCATION in GithubAppClient: every page comes + // back full (100), so the pager never sees a short page and stops only at + // the budget, handing back a continuation instead of walking everything. + const COMMIT_PAGES_PER_INVOCATION = 25 + const provider = yield* GithubProvider + const result = yield* provider.fetchCommits(installation, REPO, { sinceMs: 0 }) + assert.strictEqual(result.commits.length, COMMIT_PAGES_PER_INVOCATION * 100) + assert.ok(result.next !== undefined) + assert.strictEqual(result.next?.reason, "page-budget") + assert.strictEqual(result.next?.retryAfterSeconds, 0) // continue immediately, no wait + }).pipe( + // token → full page on every fetch (the last responder repeats for all + // subsequent pages), so the only stop condition is the page budget. + Effect.provide(stubbedProviderLayer([tokenResponse, () => commitsResponse(hexShas(100))])), + ), + ) }) diff --git a/packages/domain/src/http/vcs.ts b/packages/domain/src/http/vcs.ts index 748b9c0de..0a42a06c7 100644 --- a/packages/domain/src/http/vcs.ts +++ b/packages/domain/src/http/vcs.ts @@ -192,13 +192,22 @@ export type CommitUpsertInput = Schema.Schema.Type * derives its watermark from `max(committed_at)` of the persisted commits, so no * provider has to claim a head and no caller infers one from array position. * - * `next` is present iff a rate limit cut the walk short before the requested - * window was fully fetched: resume the backfill from `untilMs` (a committer-date - * watermark) after waiting `retryAfterSeconds`. Absent ⇒ the window is complete. + * `next` is present iff the walk was cut short before the requested window was + * fully fetched, for one of two reasons: + * - `"rate-limited"`: the provider throttled us — resume after `retryAfterSeconds`. + * - `"page-budget"`: the provider voluntarily yielded after a bounded number of + * pages, to keep a single consumer invocation's wall-clock under the queue's + * limit — resume immediately (`retryAfterSeconds` is 0). + * Either way, resume the backfill from `untilMs` (a committer-date watermark). + * Absent ⇒ the window is complete. */ export interface VcsCommitFetch { readonly commits: ReadonlyArray - readonly next?: { readonly untilMs: number; readonly retryAfterSeconds: number } + readonly next?: { + readonly untilMs: number + readonly retryAfterSeconds: number + readonly reason: "rate-limited" | "page-budget" + } } /** Minimal repo identity a provider needs to fetch commits. */ From 68d0d73003cb0b8f620d6ccb23cf5c90e52fb295 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Mon, 15 Jun 2026 19:25:53 +0200 Subject: [PATCH 07/45] Add basic frontend features --- .gitignore | 3 + apps/api/src/app.ts | 15 +- apps/api/src/lib/Env.ts | 2 + apps/api/src/routes/integrations.http.ts | 176 +- .../src/services/github/GithubAppClient.ts | 57 +- .../services/github/GithubConnectService.ts | 312 ++ .../__tests__/GithubConnectService.test.ts | 389 ++ apps/api/src/services/vcs/VcsRepository.ts | 237 +- apps/api/src/services/vcs/VcsSyncService.ts | 75 +- .../src/services/vcs/__tests__/vcs.test.ts | 293 +- apps/web/src/components/icons/github.tsx | 19 + apps/web/src/components/icons/index.ts | 1 + .../integrations/github-integration-card.tsx | 367 ++ .../integrations/integration-catalog.tsx | 37 +- apps/web/src/routes/integrations.tsx | 5 +- .../db/drizzle/0023_vcs_repository_status.sql | 1 + .../0024_vcs_commits_repository_id.sql | 34 + packages/db/drizzle/meta/0023_snapshot.json | 4593 +++++++++++++++++ packages/db/drizzle/meta/0024_snapshot.json | 4591 ++++++++++++++++ packages/db/drizzle/meta/_journal.json | 14 + packages/db/src/schema/vcs.ts | 24 +- packages/domain/src/http/integrations.ts | 92 + packages/domain/src/http/vcs.ts | 18 +- patches/effect@4.0.0-beta.70.patch | 23 + 24 files changed, 11330 insertions(+), 48 deletions(-) create mode 100644 apps/api/src/services/github/GithubConnectService.ts create mode 100644 apps/api/src/services/github/__tests__/GithubConnectService.test.ts create mode 100644 apps/web/src/components/icons/github.tsx create mode 100644 apps/web/src/components/integrations/github-integration-card.tsx create mode 100644 packages/db/drizzle/0023_vcs_repository_status.sql create mode 100644 packages/db/drizzle/0024_vcs_commits_repository_id.sql create mode 100644 packages/db/drizzle/meta/0023_snapshot.json create mode 100644 packages/db/drizzle/meta/0024_snapshot.json diff --git a/.gitignore b/.gitignore index 0eec107e4..7b6a9a712 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ playwright/.cache/ /apps/ingest/apps/ingest/.wal /apps/ingest/target /apps/ingest/.wal + +# local-only Drizzle Studio config pointing at miniflare D1 +drizzle.config.local.ts diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 5eed10665..77566b070 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -62,6 +62,7 @@ import { ScrapeTargetsService } from "./services/ScrapeTargetsService" import { WarehouseQueryService } from "./lib/WarehouseQueryService" import { OAuthStateRepository } from "./services/OAuthStateRepository" import { GithubAppClient } from "./services/github/GithubAppClient" +import { GithubConnectService } from "./services/github/GithubConnectService" import { GithubHttp } from "./services/github/GithubHttp" import { GithubProvider } from "./services/github/GithubProvider" import { VcsProviderRegistry } from "./services/vcs/VcsProviderRegistry" @@ -165,15 +166,17 @@ export const DigestServiceLive = DigestService.layer.pipe( // Step-2 settings endpoints. The sync orchestrator (VcsSyncService) lives only // in the queue-consumer runtime (vcs-sync-runtime.ts), not here. Database / // WorkerEnvironment are satisfied at worker scope (like CoreServicesLive). -const GithubProviderLive = GithubProvider.layer.pipe( - Layer.provide(GithubAppClient.layer.pipe(Layer.provide(GithubHttp.layer))), -) +const GithubAppClientLive = GithubAppClient.layer.pipe(Layer.provide(GithubHttp.layer)) +const GithubProviderLive = GithubProvider.layer.pipe(Layer.provide(GithubAppClientLive)) + +const VcsDataLive = Layer.mergeAll(VcsRepository.layer, OAuthStateRepository.layer, VcsSyncQueue.layer) export const VcsServicesLive = Layer.mergeAll( - VcsRepository.layer, - OAuthStateRepository.layer, - VcsSyncQueue.layer, + VcsDataLive, VcsProviderRegistry.layer.pipe(Layer.provide(GithubProviderLive)), + // The dashboard connect flow: needs the repo + state repo + sync queue + // (VcsDataLive) plus the GitHub App client (App-JWT installation lookup). + GithubConnectService.layer.pipe(Layer.provide(Layer.mergeAll(VcsDataLive, GithubAppClientLive))), ).pipe(Layer.provideMerge(InfraLive)) export const MainLive = Layer.mergeAll( diff --git a/apps/api/src/lib/Env.ts b/apps/api/src/lib/Env.ts index 34e1da2da..8f11fb1e7 100644 --- a/apps/api/src/lib/Env.ts +++ b/apps/api/src/lib/Env.ts @@ -38,6 +38,7 @@ export interface EnvShape { readonly HAZEL_OAUTH_CLIENT_SECRET: Option.Option> readonly HAZEL_OAUTH_SCOPES: string readonly GITHUB_APP_ID: Option.Option + readonly GITHUB_APP_SLUG: Option.Option readonly GITHUB_APP_PRIVATE_KEY: Option.Option> readonly GITHUB_APP_CLIENT_ID: Option.Option readonly GITHUB_APP_CLIENT_SECRET: Option.Option> @@ -102,6 +103,7 @@ const envConfig = Config.all({ "openid email profile organizations:read channels:read channel-webhooks:write", ), GITHUB_APP_ID: optionalString("GITHUB_APP_ID"), + GITHUB_APP_SLUG: optionalString("GITHUB_APP_SLUG"), GITHUB_APP_PRIVATE_KEY: optionalRedacted("GITHUB_APP_PRIVATE_KEY"), GITHUB_APP_CLIENT_ID: optionalString("GITHUB_APP_CLIENT_ID"), GITHUB_APP_CLIENT_SECRET: optionalRedacted("GITHUB_APP_CLIENT_SECRET"), diff --git a/apps/api/src/routes/integrations.http.ts b/apps/api/src/routes/integrations.http.ts index ee289bb3b..21ea1f55b 100644 --- a/apps/api/src/routes/integrations.http.ts +++ b/apps/api/src/routes/integrations.http.ts @@ -3,6 +3,10 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { CurrentTenant, ExternalUserId, + GithubDeleteRepositoryResponse, + GithubDisconnectResponse, + GithubIntegrationStatus, + GithubStartConnectResponse, HazelChannelsListResponse, HazelDisconnectResponse, HazelIntegrationStatus, @@ -14,6 +18,7 @@ import { UserId, } from "@maple/domain/http" import { Effect, Option, Schema } from "effect" +import { GithubConnectService } from "../services/github/GithubConnectService" import { HazelOAuthService } from "../services/HazelOAuthService" import { requireAdmin as requireAdminRole } from "../lib/auth" @@ -21,6 +26,9 @@ const asExternalUserId = Schema.decodeUnknownSync(ExternalUserId) const asUserId = Schema.decodeUnknownSync(UserId) const HAZEL_CALLBACK_PATH = "/api/integrations/hazel/callback" +const GITHUB_CALLBACK_PATH = "/api/integrations/github/callback" +const HAZEL_MESSAGE_TYPE = "maple:integration:hazel" +const GITHUB_MESSAGE_TYPE = "maple:integration:github" const resolveRequestOrigin = (req: HttpServerRequest.HttpServerRequest): string => { const headers = req.headers as Record @@ -42,6 +50,9 @@ const resolveRequestOrigin = (req: HttpServerRequest.HttpServerRequest): string const resolveCallbackUrl = (req: HttpServerRequest.HttpServerRequest): string => `${resolveRequestOrigin(req)}${HAZEL_CALLBACK_PATH}` +const resolveGithubCallbackUrl = (req: HttpServerRequest.HttpServerRequest): string => + `${resolveRequestOrigin(req)}${GITHUB_CALLBACK_PATH}` + const requireAdmin = (roles: ReadonlyArray) => requireAdminRole( roles, @@ -51,6 +62,7 @@ const requireAdmin = (roles: ReadonlyArray) => export const HttpIntegrationsLive = HttpApiBuilder.group(MapleApi, "integrations", (handlers) => Effect.gen(function* () { const hazel = yield* HazelOAuthService + const github = yield* GithubConnectService return handlers .handle("hazelStatus", () => @@ -123,6 +135,47 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleApi, "integrations return new HazelDisconnectResponse(result) }), ) + .handle("githubStatus", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const status = yield* github.getStatus(tenant.orgId) + return new GithubIntegrationStatus({ + connected: status.connected, + accountLogin: status.accountLogin, + accountType: status.accountType, + repositorySelection: status.repositorySelection, + repositories: status.repositories, + }) + }), + ) + .handle("githubStart", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles) + const req = yield* HttpServerRequest.HttpServerRequest + const result = yield* github.startConnect(tenant.orgId, tenant.userId, { + callbackUrl: resolveGithubCallbackUrl(req), + returnTo: payload.returnTo, + }) + return new GithubStartConnectResponse(result) + }), + ) + .handle("githubDisconnect", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles) + const result = yield* github.disconnect(tenant.orgId) + return new GithubDisconnectResponse(result) + }), + ) + .handle("githubDeleteRepository", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles) + const result = yield* github.deleteRepository(tenant.orgId, params.repositoryId) + return new GithubDeleteRepositoryResponse(result) + }), + ) }), ) @@ -155,12 +208,14 @@ const renderCallbackPage = (params: { status: "success" | "error" message: string returnTo: string | null + messageType: string + label: string }) => { const safeMessage = escapeHtml(params.message) const safeReturn = params.returnTo ? escapeHtml(params.returnTo) : null const payload = escapeJsonInHtml( JSON.stringify({ - type: "maple:integration:hazel", + type: params.messageType, status: params.status, message: params.message, }), @@ -169,7 +224,7 @@ const renderCallbackPage = (params: { - Maple — Hazel integration + Maple — ${params.label} integration