internal: refactor benchmark runner structure (#4677)

...into multiple files.

Author: yaacovCR

resources/benchmark.ts

@@ -0,0 +1,46 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { localRepoPath } from '../utils.js';
+
+import { LOCAL } from './config.js';
+import { bold } from './output.js';
+
+export interface BenchmarkArguments {
+  benchmarks: Array<string>;
+  revisions: Array<string>;
+}
+
+export function getArguments(argv: ReadonlyArray<string>): BenchmarkArguments {
+  const revsIndex = argv.indexOf('--revs');
+  const revisions = revsIndex === -1 ? [] : argv.slice(revsIndex + 1);
+  const benchmarks = revsIndex === -1 ? [...argv] : argv.slice(0, revsIndex);
+
+  switch (revisions.length) {
+    case 0:
+      revisions.unshift('HEAD');
+    // fall through
+    case 1: {
+      revisions.unshift(LOCAL);
+
+      const assumeArgv = ['benchmark', ...benchmarks, '--revs', ...revisions];
+      console.warn('Assuming you meant: ' + bold(assumeArgv.join(' ')));
+      break;
+    }
+  }
+
+  if (benchmarks.length === 0) {
+    benchmarks.push(...findAllBenchmarks());
+  }
+
+  return { benchmarks, revisions };
+}
+
+function findAllBenchmarks(): Array<string> {
+  return fs
+    .readdirSync(localRepoPath('benchmark'), { withFileTypes: true })
+    .filter((dirent) => dirent.isFile())
+    .map((dirent) => dirent.name)
+    .filter((name) => name.endsWith('-benchmark.js'))
+    .map((name) => path.join('benchmark', name));
+}

resources/benchmark/args.ts

@@ -0,0 +1,7 @@
+export const NS_PER_SEC = 1e9;
+export const LOCAL = 'local';
+
+// The maximum time in seconds a benchmark is allowed to run before finishing.
+export const maxTime = 5;
+// The minimum sample size required to perform statistical analysis.
+export const minSamples = 5;

resources/benchmark/config.ts

@@ -0,0 +1,93 @@
+import type { BenchmarkResult } from './types.js';
+
+export function printBenchmarkResults(
+  results: ReadonlyArray<BenchmarkResult>,
+): void {
+  const nameMaxLen = maxBy(results, ({ name }) => name.length);
+  const opsTop = maxBy(results, ({ ops }) => ops);
+  const opsMaxLen = maxBy(results, ({ ops }) => beautifyNumber(ops).length);
+  const memPerOpMaxLen = maxBy(
+    results,
+    ({ memPerOp }) => beautifyBytes(memPerOp).length,
+  );
+
+  for (const result of results) {
+    printBench(result);
+  }
+
+  function printBench(bench: BenchmarkResult): void {
+    const { name, memPerOp, ops, deviation, numSamples } = bench;
+    console.log(
+      '  ' +
+        nameStr() +
+        grey(' x ') +
+        opsStr() +
+        ' ops/sec ' +
+        grey('\xb1') +
+        deviationStr() +
+        cyan('%') +
+        grey(' x ') +
+        memPerOpStr() +
+        '/op' +
+        grey(' (' + numSamples + ' runs sampled)'),
+    );
+
+    function nameStr(): string {
+      const nameFmt = name.padEnd(nameMaxLen);
+      return ops === opsTop ? green(nameFmt) : nameFmt;
+    }
+
+    function opsStr(): string {
+      const percent = ops / opsTop;
+      const colorFn = percent > 0.95 ? green : percent > 0.8 ? yellow : red;
+      return colorFn(beautifyNumber(ops).padStart(opsMaxLen));
+    }
+
+    function deviationStr(): string {
+      const colorFn = deviation > 5 ? red : deviation > 2 ? yellow : green;
+      return colorFn(deviation.toFixed(2));
+    }
+
+    function memPerOpStr(): string {
+      return beautifyBytes(memPerOp).padStart(memPerOpMaxLen);
+    }
+  }
+}
+
+function beautifyBytes(bytes: number): string {
+  const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+  const i = Math.floor(Math.log2(bytes) / 10);
+  return beautifyNumber(bytes / 2 ** (i * 10)) + ' ' + sizes[i];
+}
+
+function beautifyNumber(num: number): string {
+  return Number(num.toFixed(num > 100 ? 0 : 2)).toLocaleString();
+}
+
+function maxBy<T>(array: ReadonlyArray<T>, fn: (obj: T) => number): number {
+  return Math.max(...array.map(fn));
+}
+
+export function bold(str: number | string): string {
+  return '\u001b[1m' + str + '\u001b[0m';
+}
+
+export function red(str: number | string): string {
+  return '\u001b[31m' + str + '\u001b[0m';
+}
+
+export function green(str: number | string): string {
+  return '\u001b[32m' + str + '\u001b[0m';
+}
+
+export function yellow(str: number | string): string {
+  return '\u001b[33m' + str + '\u001b[0m';
+}
+
+export function cyan(str: number | string): string {
+  return '\u001b[36m' + str + '\u001b[0m';
+}
+
+export function grey(str: number | string): string {
+  return '\u001b[90m' + str + '\u001b[0m';
+}

resources/benchmark/output.ts

@@ -0,0 +1,78 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { git, localRepoPath, makeTmpDir, npm } from '../utils.js';
+
+import { LOCAL } from './config.js';
+import type { BenchmarkProject } from './types.js';
+
+// Build a benchmark-friendly environment for each revision.
+export function prepareBenchmarkProjects(
+  revisionList: ReadonlyArray<string>,
+): Array<BenchmarkProject> {
+  const { tmpDirPath } = makeTmpDir('graphql-js-benchmark');
+
+  return revisionList.map((revision) => {
+    console.log(`\u{1F373}  Preparing ${revision}...`);
+    const projectPath = tmpDirPath('setup', revision);
+    fs.rmSync(projectPath, { recursive: true, force: true });
+    fs.mkdirSync(projectPath, { recursive: true });
+
+    fs.cpSync(localRepoPath('benchmark'), path.join(projectPath, 'benchmark'), {
+      recursive: true,
+    });
+
+    fs.writeFileSync(
+      path.join(projectPath, 'package.json'),
+      JSON.stringify(
+        {
+          private: true,
+          type: 'module',
+          dependencies: {
+            graphql: prepareNPMPackage(revision),
+          },
+        },
+        null,
+        2,
+      ),
+    );
+    npm({ cwd: projectPath, quiet: true }).install('--ignore-scripts');
+
+    return { revision, projectPath };
+  });
+
+  function prepareNPMPackage(revision: string): string {
+    if (revision === LOCAL) {
+      const repoDir = localRepoPath();
+      const archivePath = tmpDirPath('graphql-local.tgz');
+      fs.renameSync(buildNPMArchive(repoDir), archivePath);
+      return archivePath;
+    }
+
+    // Returns the complete git hash for a given git revision reference.
+    const hash = git().revParse(revision);
+
+    const archivePath = tmpDirPath(`graphql-${hash}.tgz`);
+    if (fs.existsSync(archivePath)) {
+      return archivePath;
+    }
+
+    const repoDir = tmpDirPath(hash);
+    fs.rmSync(repoDir, { recursive: true, force: true });
+    fs.mkdirSync(repoDir);
+    git({ quiet: true }).clone(localRepoPath(), repoDir);
+    git({ cwd: repoDir, quiet: true }).checkout('--detach', hash);
+    npm({ cwd: repoDir, quiet: true }).ci('--ignore-scripts');
+    fs.renameSync(buildNPMArchive(repoDir), archivePath);
+    fs.rmSync(repoDir, { recursive: true });
+    return archivePath;
+  }
+
+  function buildNPMArchive(repoDir: string): string {
+    npm({ cwd: repoDir, quiet: true }).run('build:npm');
+
+    const distDir = path.join(repoDir, 'npmDist');
+    const archiveName = npm({ cwd: repoDir, quiet: true }).pack(distDir);
+    return path.join(repoDir, archiveName);
+  }
+}

resources/benchmark/projects.ts

@@ -0,0 +1,50 @@
+import path from 'node:path';
+
+import { getArguments } from './args.js';
+import { cyan, printBenchmarkResults, red } from './output.js';
+import { prepareBenchmarkProjects } from './projects.js';
+import { collectSamples, sampleModule } from './sampling.js';
+import { computeStats } from './statistics.js';
+import type { BenchmarkProject, BenchmarkResult } from './types.js';
+
+export function runBenchmarks(): void {
+  // Get the revisions and make things happen!
+  const { benchmarks, revisions } = getArguments(process.argv.slice(2));
+  const benchmarkProjects = prepareBenchmarkProjects(revisions);
+
+  for (const benchmark of benchmarks) {
+    runBenchmark(benchmark, benchmarkProjects);
+  }
+}
+
+// Prepare all revisions and run benchmarks matching a pattern against them.
+function runBenchmark(
+  benchmark: string,
+  benchmarkProjects: ReadonlyArray<BenchmarkProject>,
+): void {
+  const results: Array<BenchmarkResult> = [];
+  for (let i = 0; i < benchmarkProjects.length; ++i) {
+    const { revision, projectPath } = benchmarkProjects[i];
+    const modulePath = path.join(projectPath, benchmark);
+
+    if (i === 0) {
+      const { name } = sampleModule(modulePath);
+      console.log('\u23F1   ' + name);
+    }
+
+    try {
+      const samples = collectSamples(modulePath);
+
+      results.push(computeStats(revision, samples));
+      process.stdout.write('  ' + cyan(i + 1) + ' tests completed.\u000D');
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      console.log('  ' + revision + ': ' + red(errorMessage));
+    }
+  }
+  console.log('\n');
+
+  printBenchmarkResults(results);
+  console.log('');
+}

resources/benchmark/run.ts

@@ -0,0 +1,106 @@
+import assert from 'node:assert';
+import cp from 'node:child_process';
+import url from 'node:url';
+
+import { maxTime, minSamples } from './config.js';
+import { yellow } from './output.js';
+import type { BenchmarkSample } from './types.js';
+
+export function collectSamples(modulePath: string): Array<BenchmarkSample> {
+  let numOfConsequentlyRejectedSamples = 0;
+  const samples: Array<BenchmarkSample> = [];
+
+  // If time permits, increase sample size to reduce the margin of error.
+  const start = Date.now();
+  while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
+    const sample = sampleModule(modulePath);
+
+    if (sample.involuntaryContextSwitches > 0) {
+      numOfConsequentlyRejectedSamples++;
+      if (numOfConsequentlyRejectedSamples === 5) {
+        console.error(
+          yellow(
+            '  Five or more consequent runs beings rejected because of context switching.\n' +
+              '  Measurement can take a significantly longer time and its correctness can also be impacted.',
+          ),
+        );
+      }
+      continue;
+    }
+    numOfConsequentlyRejectedSamples = 0;
+
+    assert(sample.clocked > 0);
+    assert(sample.memUsed > 0);
+    samples.push(sample);
+  }
+  return samples;
+}
+
+export function sampleModule(modulePath: string): BenchmarkSample {
+  // To support Windows we need to use URL instead of path
+  const moduleURL = url.pathToFileURL(modulePath);
+
+  const sampleCode = `
+    import fs from 'node:fs';
+
+    import { benchmark } from '${moduleURL}';
+
+    // warm up, it looks like 7 is a magic number to reliably trigger JIT
+    await benchmark.measure();
+    await benchmark.measure();
+    await benchmark.measure();
+    await benchmark.measure();
+    await benchmark.measure();
+    await benchmark.measure();
+    await benchmark.measure();
+
+    const memBaseline = process.memoryUsage().heapUsed;
+
+    const resourcesStart = process.resourceUsage();
+    const startTime = process.hrtime.bigint();
+    for (let i = 0; i < benchmark.count; ++i) {
+      await benchmark.measure();
+    }
+    const timeDiff = Number(process.hrtime.bigint() - startTime);
+    const resourcesEnd = process.resourceUsage();
+
+    const result = {
+      name: benchmark.name,
+      clocked: timeDiff / benchmark.count,
+      memUsed: (process.memoryUsage().heapUsed - memBaseline) / benchmark.count,
+      involuntaryContextSwitches:
+        resourcesEnd.involuntaryContextSwitches - resourcesStart.involuntaryContextSwitches,
+    };
+    fs.writeFileSync(3, JSON.stringify(result));
+  `;
+
+  const result = cp.spawnSync(
+    process.execPath,
+    [
+      // V8 flags
+      '--predictable',
+      '--no-concurrent-sweeping',
+      '--no-minor-gc-task',
+      '--min-semi-space-size=1280', // 1.25GB
+      '--max-semi-space-size=1280', // 1.25GB
+      '--trace-gc', // no gc calls should happen during benchmark, so trace them
+
+      // Node.js flags
+      '--input-type=module',
+      '--eval',
+      sampleCode,
+    ],
+    {
+      stdio: ['inherit', 'inherit', 'inherit', 'pipe'],
+      env: { NODE_ENV: 'production' },
+    },
+  );
+
+  if (result.status !== 0) {
+    throw new Error(`Benchmark failed with "${result.status}" status.`);
+  }
+
+  const resultStr = result.output[3]?.toString();
+  assert(resultStr != null);
+  return JSON.parse(resultStr);
+}