diff --git a/README.md b/README.md index 95a814f..f4a7fb3 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ After the test completes, you can run `npx playwright show-report` to see a deta - **Core Execution** — `runSteps()` and `runUserFlow()` for flexible test orchestration in natural language, with smart caching and auto-healing - **Multi-Model Assertion Engine** — Consensus-based validation using Claude and Gemini, with an arbiter model to resolve disagreements -- **Redis-Based Step Caching** — Cache-first execution with AI fallback and automatic self-healing when cached steps fail +- **Pluggable Step Caching** — Cache-first execution with AI fallback and automatic self-healing. Supports Redis, file-based, or custom cache backends. - **Configurable AI Models** — 8 dedicated model slots for step execution, assertions, extraction, and more - **AI Gateway Support** — Route requests through Vercel AI Gateway, OpenRouter, Cloudflare AI Gateway, or connect directly to provider SDKs - **Dynamic Placeholders** — Inject values at runtime with `{{run.*}}`, `{{global.*}}`, `{{data.*}}`, and `{{email.*}}` expressions for repeatable and data-driven tests @@ -184,6 +184,8 @@ configure({ | Variable | Required | Default | Description | |----------|----------|---------|-------------| +| `CACHE_PROVIDER` | No | - | Cache backend: `redis`, `file`, or `none`. Falls back to Redis if `REDIS_URL` is set. | +| `CACHE_DIR` | No | `.passmark-cache` | Directory for file-based cache (when `CACHE_PROVIDER=file`) | | `REDIS_URL` | No | - | Redis connection URL for step caching and global state | | `ANTHROPIC_API_KEY` | Yes | - | Anthropic API key for Claude models | | `GOOGLE_GENERATIVE_AI_API_KEY` | Yes | - | Google API key for Gemini models | @@ -212,7 +214,35 @@ All models are configurable via `configure({ ai: { models: { ... } } })`: ## Caching -Passmark caches successful step actions in Redis. On subsequent runs, cached steps execute directly without AI calls, dramatically reducing latency and cost. +Passmark caches successful step actions so that subsequent runs execute directly without AI calls, dramatically reducing latency and cost. The cache backend is pluggable — choose between Redis, file-based, or no caching at all. + +### Cache Providers + +Set the `CACHE_PROVIDER` environment variable to select a backend: + +| Provider | `CACHE_PROVIDER` | Additional Config | Description | +|----------|-------------------|-------------------|-------------| +| **Redis** | `redis` | `REDIS_URL` | Uses Redis via ioredis. Default when `REDIS_URL` is set. | +| **File** | `file` | `CACHE_DIR` (optional, defaults to `.passmark-cache`) | JSON files on disk. No external dependencies — great for local development and CI. | +| **None** | `none` | — | Disables caching entirely. Every step uses AI execution. | + +For backwards compatibility, if `CACHE_PROVIDER` is not set, Passmark will use Redis when `REDIS_URL` is present, otherwise caching is disabled. + +### Custom Cache Store + +You can implement a custom cache backend by conforming to the `CacheStore` interface: + +```typescript +import { CacheStore } from "passmark"; + +interface CacheStore { + hgetall(key: string): Promise>; + hset(key: string, values: Record): Promise; + expire(key: string, seconds: number): Promise; +} +``` + +### Caching Behavior - Steps are cached by `userFlow` + `step.description` - Set `bypassCache: true` on individual steps or the entire run to force AI execution diff --git a/src/__tests__/data-cache.test.ts b/src/__tests__/data-cache.test.ts index 69be5ec..04e29c5 100644 --- a/src/__tests__/data-cache.test.ts +++ b/src/__tests__/data-cache.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -vi.mock("../redis", () => ({ - redis: { hgetall: vi.fn(), hset: vi.fn(), expire: vi.fn() }, +vi.mock("../cache", () => ({ + cache: { hgetall: vi.fn(), hset: vi.fn(), expire: vi.fn() }, })); vi.mock("../email", () => ({ diff --git a/src/__tests__/integration/run-steps.test.ts b/src/__tests__/integration/run-steps.test.ts index c3edc40..aa7c6d8 100644 --- a/src/__tests__/integration/run-steps.test.ts +++ b/src/__tests__/integration/run-steps.test.ts @@ -3,12 +3,12 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; // Mock instrumentation (imported as side effect) vi.mock("../../instrumentation", () => ({ axiomEnabled: false })); -// Mock Redis -vi.mock("../../redis", () => ({ - redis: { +// Mock Cache +vi.mock("../../cache", () => ({ + cache: { hgetall: vi.fn().mockResolvedValue({}), - hset: vi.fn().mockResolvedValue("OK"), - expire: vi.fn().mockResolvedValue(1), + hset: vi.fn().mockResolvedValue(undefined), + expire: vi.fn().mockResolvedValue(undefined), }, })); @@ -89,7 +89,7 @@ vi.mock("../../utils/secure-script-runner", () => ({ import { runSteps } from "../../index"; import { resetConfig } from "../../config"; -import { redis } from "../../redis"; +import { cache } from "../../cache"; import { generateText } from "ai"; import type { Page } from "@playwright/test"; import type { Step } from "../../types"; @@ -121,8 +121,8 @@ describe("runSteps", () => { beforeEach(() => { vi.clearAllMocks(); resetConfig(); - // Reset redis mock to default empty - vi.mocked(redis!.hgetall).mockResolvedValue({}); + // Reset cache mock to default empty + vi.mocked(cache!.hgetall).mockResolvedValue({}); }); it("executes a simple step", async () => { @@ -198,8 +198,8 @@ describe("runSteps", () => { const page = createMockPage(); const steps: Step[] = [{ description: "Click submit" }]; - // Mock redis to return cached step data - vi.mocked(redis!.hgetall).mockResolvedValue({ + // Mock cache to return cached step data + vi.mocked(cache!.hgetall).mockResolvedValue({ locator: 'getByRole("button", { name: "Submit" })', action: "click", description: "Submit button", @@ -220,8 +220,8 @@ describe("runSteps", () => { const page = createMockPage(); const steps: Step[] = [{ description: "Click submit" }]; - // Mock redis to return cached step data - vi.mocked(redis!.hgetall).mockResolvedValue({ + // Mock cache to return cached step data + vi.mocked(cache!.hgetall).mockResolvedValue({ locator: 'getByRole("button", { name: "Submit" })', action: "click", description: "Submit button", @@ -291,8 +291,8 @@ describe("runSteps", () => { it("bypasses cache for individual step when step.bypassCache is true", async () => { const page = createMockPage(); - // Mock redis to return cached data - vi.mocked(redis!.hgetall).mockResolvedValue({ + // Mock cache to return cached data + vi.mocked(cache!.hgetall).mockResolvedValue({ locator: 'getByRole("button", { name: "Go" })', action: "click", description: "Go button", diff --git a/src/cache.ts b/src/cache.ts new file mode 100644 index 0000000..8075921 --- /dev/null +++ b/src/cache.ts @@ -0,0 +1,160 @@ +import { logger } from "./logger"; + +// ============================================================================= +// Cache Store Interface +// ============================================================================= + +/** + * Interface for a hash-based cache store. + * Implementations must support hash get/set and key expiration. + */ +export interface CacheStore { + hgetall(key: string): Promise>; + hset(key: string, values: Record): Promise; + expire(key: string, seconds: number): Promise; +} + +// ============================================================================= +// Redis Store +// ============================================================================= + +class RedisStore implements CacheStore { + private client: import("ioredis").default; + + constructor(url: string) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const Redis = require("ioredis") as typeof import("ioredis").default; + this.client = new Redis(url); + } + + async hgetall(key: string): Promise> { + return this.client.hgetall(key); + } + + async hset(key: string, values: Record): Promise { + await this.client.hset(key, values); + } + + async expire(key: string, seconds: number): Promise { + await this.client.expire(key, seconds); + } +} + +// ============================================================================= +// File Store +// ============================================================================= + +import * as fs from "fs"; +import * as path from "path"; + +class FileStore implements CacheStore { + private dir: string; + + constructor(dir: string) { + this.dir = dir; + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } + + private filePath(key: string): string { + // Encode key to a safe filename + const safeKey = encodeURIComponent(key); + return path.join(this.dir, `${safeKey}.json`); + } + + private read(key: string): { data: Record; expiresAt?: number } | null { + const fp = this.filePath(key); + if (!fs.existsSync(fp)) return null; + + try { + const raw = JSON.parse(fs.readFileSync(fp, "utf-8")); + + // Check expiration + if (raw.expiresAt && Date.now() > raw.expiresAt) { + fs.unlinkSync(fp); + return null; + } + + return raw; + } catch { + return null; + } + } + + private write(key: string, entry: { data: Record; expiresAt?: number }): void { + const fp = this.filePath(key); + fs.writeFileSync(fp, JSON.stringify(entry), "utf-8"); + } + + async hgetall(key: string): Promise> { + const entry = this.read(key); + return entry?.data ?? {}; + } + + async hset(key: string, values: Record): Promise { + const existing = this.read(key); + const merged = { ...(existing?.data ?? {}), ...values }; + this.write(key, { data: merged, expiresAt: existing?.expiresAt }); + } + + async expire(key: string, seconds: number): Promise { + const existing = this.read(key); + if (!existing) return; + this.write(key, { ...existing, expiresAt: Date.now() + seconds * 1000 }); + } +} + +// ============================================================================= +// Factory +// ============================================================================= + +/** + * Creates the cache store based on environment variables. + * + * CACHE_PROVIDER selects the backend: + * - "redis" (default when REDIS_URL is set): uses Redis via ioredis + * - "file": uses JSON files on disk at CACHE_DIR (defaults to .passmark-cache) + * - "none": disables caching entirely + * + * For backwards compatibility, if CACHE_PROVIDER is not set: + * - If REDIS_URL is set → uses Redis + * - Otherwise → caching is disabled (null) + */ +function createCacheStore(): CacheStore | null { + const provider = process.env.CACHE_PROVIDER?.toLowerCase(); + + if (provider === "none") { + logger.warn("Cache provider set to 'none'. Caching is disabled."); + return null; + } + + if (provider === "file") { + const dir = process.env.CACHE_DIR || ".passmark-cache"; + logger.info(`Using file-based cache at: ${dir}`); + return new FileStore(dir); + } + + if (provider === "redis" || (!provider && process.env.REDIS_URL)) { + if (!process.env.REDIS_URL) { + logger.warn("CACHE_PROVIDER is 'redis' but REDIS_URL is not set. Caching is disabled."); + return null; + } + logger.info("Using Redis cache."); + return new RedisStore(process.env.REDIS_URL); + } + + if (provider) { + logger.warn(`Unknown CACHE_PROVIDER '${provider}'. Caching is disabled.`); + return null; + } + + // No CACHE_PROVIDER and no REDIS_URL + logger.warn( + "No cache provider configured. Set CACHE_PROVIDER=redis|file|none or REDIS_URL. " + + "Step caching, global placeholders, and project data are disabled.", + ); + return null; +} + +export const cache: CacheStore | null = createCacheStore(); diff --git a/src/constants.ts b/src/constants.ts index 195521f..384ccd9 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -22,5 +22,5 @@ export const MAX_RETRIES = 3; // Thinking budgets (tokens) export const THINKING_BUDGET_DEFAULT = 1024; -// Redis +// Cache export const GLOBAL_VALUES_TTL_SECONDS = 86400; diff --git a/src/data-cache.ts b/src/data-cache.ts index 01ecec2..f7b415b 100644 --- a/src/data-cache.ts +++ b/src/data-cache.ts @@ -4,7 +4,7 @@ import { getConfig } from "./config"; import { extractEmailContent } from "./email"; import { GLOBAL_VALUES_TTL_SECONDS } from "./constants"; import { logger } from "./logger"; -import { redis } from "./redis"; +import { cache } from "./cache"; import { Step } from "./types"; import { generatePhoneNumber } from "./utils"; @@ -26,7 +26,7 @@ export type LocalPlaceholders = { /** * Global placeholders that are shared across all tests within an execution. - * These values are persisted to Redis and loaded for subsequent runSteps calls + * These values are persisted to the cache and loaded for subsequent runSteps calls * with the same executionId. */ export type GlobalPlaceholders = { @@ -39,7 +39,7 @@ export type GlobalPlaceholders = { /** * Project data placeholders for {{data.key}} syntax. - * These are stored in Redis and managed via project settings. + * These are stored in the cache and managed via project settings. */ export type ProjectDataPlaceholders = Record; @@ -95,26 +95,26 @@ const GLOBAL_PLACEHOLDER_PATTERN = /\{\{global\.\w+\}\}/; const PROJECT_DATA_PLACEHOLDER_PATTERN = /\{\{data\.(\w+)\}\}/g; // ============================================================================= -// Redis Operations (Global Values) +// Cache Operations (Global Values) // ============================================================================= /** - * Generates a Redis key for storing global values for an execution. + * Generates a cache key for storing global values for an execution. */ -function getRedisKey(executionId: string): string { +function getCacheKey(executionId: string): string { return `execution:${executionId}:globals`; } /** - * Fetches global values from Redis for a given execution ID. + * Fetches global values from the cache for a given execution ID. * Returns null if no values exist. */ export async function getGlobalValues( executionId: string, ): Promise | null> { - if (!redis) return null; - const key = getRedisKey(executionId); - const values = await redis.hgetall(key); + if (!cache) return null; + const key = getCacheKey(executionId); + const values = await cache.hgetall(key); if (!values || Object.keys(values).length === 0) { return null; @@ -124,45 +124,45 @@ export async function getGlobalValues( } /** - * Saves global values to Redis for a given execution ID. + * Saves global values to the cache for a given execution ID. * Sets a 24-hour TTL on the key. */ export async function saveGlobalValues( executionId: string, values: GlobalPlaceholders, ): Promise { - if (!redis) return; + if (!cache) return; - const key = getRedisKey(executionId); + const key = getCacheKey(executionId); // Save all values as a hash - await redis.hset(key, values); + await cache.hset(key, values); // Set TTL - await redis.expire(key, GLOBAL_VALUES_TTL_SECONDS); + await cache.expire(key, GLOBAL_VALUES_TTL_SECONDS); - logger.debug(`Saved global values to Redis for execution: ${executionId}`); + logger.debug(`Saved global values to cache for execution: ${executionId}`); } // ============================================================================= -// Redis Operations (Project Data) +// Cache Operations (Project Data) // ============================================================================= /** - * Generates a Redis key for storing project data. + * Generates a cache key for storing project data. */ -function getProjectDataRedisKey(projectId: string): string { +function getProjectDataCacheKey(projectId: string): string { return `project:${projectId}:data`; } /** - * Fetches project data from Redis for a given project ID. + * Fetches project data from the cache for a given project ID. * Returns an empty object if no data exists. */ export async function getProjectData(projectId: string): Promise { - if (!redis) return {}; - const key = getProjectDataRedisKey(projectId); - const values = await redis.hgetall(key); + if (!cache) return {}; + const key = getProjectDataCacheKey(projectId); + const values = await cache.hgetall(key); if (!values || Object.keys(values).length === 0) { return {}; @@ -384,7 +384,7 @@ export function replacePlaceholders( /** * Processes steps and assertions to replace dynamic placeholders with consistent values. * Handles {{run.*}} placeholders (fresh per call), {{global.*}} placeholders - * (shared across execution via Redis), and {{data.*}} placeholders (project data from Redis). + * (shared across execution via cache), and {{data.*}} placeholders (project data from cache). * Returns the processed steps and assertions along with the generated values. */ export async function processPlaceholders( diff --git a/src/index.ts b/src/index.ts index 820ff31..572b8eb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,7 +22,7 @@ async function maybeWithSpan( } import { z } from "zod"; import { buildRunStepsPrompt, buildRunUserFlowPrompt } from "./prompts"; -import { redis } from "./redis"; +import { cache } from "./cache"; import { getAItools } from "./tools"; import { RunStepsOptions, UserFlowOptions } from "./types"; import { @@ -59,7 +59,7 @@ import { /** * Executes a sequence of test steps using AI with intelligent caching. * Each step is described in natural language and executed via browser automation. - * Successfully executed steps are cached in Redis for faster subsequent runs. + * Successfully executed steps are cached for faster subsequent runs. * * @param options - Configuration including page, steps, assertions, and callbacks * @param options.page - The Playwright page instance @@ -109,13 +109,13 @@ export const runSteps = async ({ // when a new tab opens, or explicitly via the `switchToTab` step field. const tabManager = createTabManager(page); - if (!redis) { + if (!cache) { logger.warn( - "Redis not configured. Step caching is disabled — all steps will use AI execution.", + "Cache not configured. Step caching is disabled — all steps will use AI execution.", ); if (executionId) { logger.warn( - "{{global.*}} placeholders will not persist across runSteps calls without Redis.", + "{{global.*}} placeholders will not persist across runSteps calls without a cache provider.", ); } } @@ -262,7 +262,7 @@ export const runSteps = async ({ } // First check if the step is cached on redis - const cachedStep = redis ? await redis.hgetall(`step:${userFlow}:${step.description}`) : {}; + const cachedStep = cache ? await cache.hgetall(`step:${userFlow}:${step.description}`) : {}; if ( !bypassCache && @@ -443,10 +443,10 @@ export const runSteps = async ({ .flatMap((s) => s.toolCalls) .filter((tool) => ["browser_snapshot", "browser_stop"].indexOf(tool.toolName) === -1); - if (allToolCalls.length === 1 && redis) { + if (allToolCalls.length === 1 && cache) { const cacheData = getPendingCacheData(); if (cacheData) { - await redis.hset(`step:${userFlow}:${step.description}`, cacheData); + await cache.hset(`step:${userFlow}:${step.description}`, cacheData); logger.debug(`Cached step action: ${step.description}`); } } @@ -712,4 +712,5 @@ export { extractEmailContent, generateEmail } from "./email"; export { assert } from "./assertion"; export type { AssertionResult } from "./types"; +export type { CacheStore } from "./cache"; export { PassmarkError, StepExecutionError, ValidationError, AIModelError, CacheError, ConfigurationError } from "./errors"; diff --git a/src/redis.ts b/src/redis.ts deleted file mode 100644 index bf3f34d..0000000 --- a/src/redis.ts +++ /dev/null @@ -1,14 +0,0 @@ -import Redis from "ioredis"; -import { logger } from "./logger"; - -let redis: Redis | null = null; - -if (process.env.REDIS_URL) { - redis = new Redis(process.env.REDIS_URL); -} else { - logger.warn( - "REDIS_URL not set. Step caching, global placeholders, and project data are disabled.", - ); -} - -export { redis }; diff --git a/src/types.ts b/src/types.ts index abd4597..27d7d9e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -111,7 +111,7 @@ export type RunStepsOptions = { /** * Execution ID to link multiple runSteps calls together. - * When provided, {{global.*}} placeholders are persisted to Redis + * When provided, {{global.*}} placeholders are persisted to the cache * and shared across all runSteps calls with the same executionId. * Required when using {{global.*}} placeholders. */