internal: separate benchmark timing and memory sampling (#4680)

Author: yaacovCR

resources/benchmark/config.ts

@@ -6,7 +6,9 @@ export const maxTime = 5;
 // The minimum sample size required to perform statistical analysis.
 export const minSamples = 5;
 
-export const nodeFlags: ReadonlyArray<string> = [
+export const memorySamplesPerBenchmark = 10;
+
+export const memoryBenchmarkNodeFlags: ReadonlyArray<string> = [
   '--predictable',
   '--no-concurrent-sweeping',
   '--no-minor-gc-task',

resources/benchmark/run.ts

@@ -3,7 +3,7 @@ import path from 'node:path';
 import { getArguments } from './args.js';
 import { cyan, printBenchmarkResults, red } from './output.js';
 import { prepareBenchmarkProjects } from './projects.js';
-import { collectSamples } from './sampling.js';
+import { collectMemorySamples, collectTimingSamples } from './sampling.js';
 import { computeStats } from './statistics.js';
 import type { BenchmarkProject, BenchmarkResult } from './types.js';
 import { getBenchmarkName } from './workers.js';
@@ -33,9 +33,10 @@ function runBenchmark(
     }
 
     try {
-      const samples = collectSamples(modulePath);
+      const timingSamples = collectTimingSamples(modulePath);
+      const memorySamples = collectMemorySamples(modulePath);
 
-      results.push(computeStats(revision, samples));
+      results.push(computeStats(revision, timingSamples, memorySamples));
       process.stdout.write('  ' + cyan(i + 1) + ' tests completed.\u000D');
     } catch (error) {
       const errorMessage =

resources/benchmark/sampling.ts

@@ -1,18 +1,20 @@
 import assert from 'node:assert';
 
-import { maxTime, minSamples } from './config.js';
+import { maxTime, memorySamplesPerBenchmark, minSamples } from './config.js';
 import { yellow } from './output.js';
-import type { BenchmarkSample } from './types.js';
-import { sampleModule } from './workers.js';
+import type { BenchmarkTimingSample } from './types.js';
+import { sampleMemoryModule, sampleTimingModule } from './workers.js';
 
-export function collectSamples(modulePath: string): Array<BenchmarkSample> {
+export function collectTimingSamples(
+  modulePath: string,
+): Array<BenchmarkTimingSample> {
   let numOfConsequentlyRejectedSamples = 0;
-  const samples: Array<BenchmarkSample> = [];
+  const samples: Array<BenchmarkTimingSample> = [];
 
   // If time permits, increase sample size to reduce the margin of error.
   const start = Date.now();
   while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
-    const sample = sampleModule(modulePath);
+    const sample = sampleTimingModule(modulePath);
 
     if (sample.involuntaryContextSwitches > 0) {
       numOfConsequentlyRejectedSamples++;
@@ -29,7 +31,20 @@ export function collectSamples(modulePath: string): Array<BenchmarkSample> {
     numOfConsequentlyRejectedSamples = 0;
 
     assert(sample.clocked > 0);
-    assert(sample.memUsed > 0);
+    samples.push(sample);
+  }
+  return samples;
+}
+
+export function collectMemorySamples(modulePath: string): Array<number> {
+  const samples: Array<number> = [];
+  for (
+    let sampleIndex = 0;
+    sampleIndex < memorySamplesPerBenchmark;
+    ++sampleIndex
+  ) {
+    const sample = sampleMemoryModule(modulePath);
+    assert(sample > 0);
     samples.push(sample);
   }
   return samples;

resources/benchmark/statistics.ts

@@ -1,7 +1,7 @@
 import assert from 'node:assert';
 
 import { NS_PER_SEC } from './config.js';
-import type { BenchmarkResult, BenchmarkSample } from './types.js';
+import type { BenchmarkResult, BenchmarkTimingSample } from './types.js';
 
 // T-Distribution two-tailed critical values for 95% confidence.
 // See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
@@ -18,35 +18,40 @@ const tTableInfinity = 1.96;
 // Computes stats on benchmark results.
 export function computeStats(
   name: string,
-  samples: ReadonlyArray<BenchmarkSample>,
+  timingSamples: ReadonlyArray<BenchmarkTimingSample>,
+  memorySamples: ReadonlyArray<number>,
 ): BenchmarkResult {
-  assert(samples.length > 1);
+  assert(timingSamples.length > 1);
+  assert(memorySamples.length > 0);
 
   // Compute the sample mean (estimate of the population mean).
   let mean = 0;
-  let meanMemUsed = 0;
-  for (const { clocked, memUsed } of samples) {
+  for (const { clocked } of timingSamples) {
     mean += clocked;
+  }
+  mean /= timingSamples.length;
+
+  let meanMemUsed = 0;
+  for (const memUsed of memorySamples) {
     meanMemUsed += memUsed;
   }
-  mean /= samples.length;
-  meanMemUsed /= samples.length;
+  meanMemUsed /= memorySamples.length;
 
   // Compute the sample variance (estimate of the population variance).
   let variance = 0;
-  for (const { clocked } of samples) {
+  for (const { clocked } of timingSamples) {
     variance += (clocked - mean) ** 2;
   }
-  variance /= samples.length - 1;
+  variance /= timingSamples.length - 1;
 
   // Compute the sample standard deviation (estimate of the population standard deviation).
   const sd = Math.sqrt(variance);
 
   // Compute the standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean).
-  const sem = sd / Math.sqrt(samples.length);
+  const sem = sd / Math.sqrt(timingSamples.length);
 
   // Compute the degrees of freedom.
-  const df = samples.length - 1;
+  const df = timingSamples.length - 1;
 
   // Compute the critical value.
   const critical = tTable[df] ?? tTableInfinity;
@@ -62,6 +67,6 @@ export function computeStats(
     memPerOp: Math.floor(meanMemUsed),
     ops: NS_PER_SEC / mean,
     deviation: rme,
-    numSamples: samples.length,
+    numSamples: timingSamples.length,
   };
 }

resources/benchmark/types.ts

@@ -3,9 +3,8 @@ export interface BenchmarkProject {
   projectPath: string;
 }
 
-export interface BenchmarkSample {
+export interface BenchmarkTimingSample {
   clocked: number;
-  memUsed: number;
   involuntaryContextSwitches: number;
 }
 

resources/benchmark/worker-memory.js

@@ -0,0 +1,29 @@
+import {
+  loadBenchmark,
+  readModulePath,
+  runWorker,
+  writeResult,
+} from './worker-utils.js';
+
+runWorker(async () => {
+  const benchmark = await loadBenchmark(readModulePath());
+  await warmUp(benchmark);
+
+  const memBaseline = process.memoryUsage().heapUsed;
+  for (let i = 0; i < benchmark.count; ++i) {
+    // eslint-disable-next-line no-await-in-loop
+    await benchmark.measure();
+  }
+  writeResult((process.memoryUsage().heapUsed - memBaseline) / benchmark.count);
+});
+
+async function warmUp(benchmark) {
+  // It looks like 7 is a magic number to reliably trigger JIT.
+  await benchmark.measure();
+  await benchmark.measure();
+  await benchmark.measure();
+  await benchmark.measure();
+  await benchmark.measure();
+  await benchmark.measure();
+  await benchmark.measure();
+}

resources/benchmark/worker-timing.js

@@ -9,8 +9,6 @@ runWorker(async () => {
   const benchmark = await loadBenchmark(readModulePath());
   await warmUp(benchmark);
 
-  const memBaseline = process.memoryUsage().heapUsed;
-
   const resourcesStart = process.resourceUsage();
   const startTime = process.hrtime.bigint();
   for (let i = 0; i < benchmark.count; ++i) {
@@ -22,7 +20,6 @@ runWorker(async () => {
 
   writeResult({
     clocked: timeDiff / benchmark.count,
-    memUsed: (process.memoryUsage().heapUsed - memBaseline) / benchmark.count,
     involuntaryContextSwitches:
       resourcesEnd.involuntaryContextSwitches -
       resourcesStart.involuntaryContextSwitches,

resources/benchmark/workers.ts

@@ -3,8 +3,8 @@ import childProcess from 'node:child_process';
 
 import { localRepoPath } from '../utils.js';
 
-import { nodeFlags } from './config.js';
-import type { BenchmarkSample } from './types.js';
+import { memoryBenchmarkNodeFlags } from './config.js';
+import type { BenchmarkTimingSample } from './types.js';
 
 export function getBenchmarkName(modulePath: string): string {
   return runWorkerFile(
@@ -13,17 +13,24 @@ export function getBenchmarkName(modulePath: string): string {
   ) as string;
 }
 
-export function sampleModule(modulePath: string): BenchmarkSample {
+export function sampleTimingModule(modulePath: string): BenchmarkTimingSample {
   return runWorkerFile(
     localRepoPath('resources/benchmark/worker-timing.js'),
     modulePath,
-  ) as BenchmarkSample;
+  ) as BenchmarkTimingSample;
+}
+
+export function sampleMemoryModule(modulePath: string): number {
+  return runWorkerFile(
+    localRepoPath('resources/benchmark/worker-memory.js'),
+    modulePath,
+  ) as number;
 }
 
 function runWorkerFile(workerPath: string, modulePath: string): unknown {
   const result = childProcess.spawnSync(
     process.execPath,
-    [...nodeFlags, workerPath, modulePath],
+    [...memoryBenchmarkNodeFlags, workerPath, modulePath],
     {
       stdio: ['inherit', 'inherit', 'inherit', 'pipe'],
       env: { NODE_ENV: 'production' },