Skip to content

Commit 7bfcc49

Browse files
yaacovCRyaacovCR
authored
internal: run benchmarks through worker files (#4678)
motivation: to separate workers for name/timing/memory
1 parent 226c289 commit 7bfcc49

5 files changed

Lines changed: 119 additions & 71 deletions

File tree

‎resources/benchmark/config.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,12 @@ export const LOCAL = 'local';
55
export const maxTime = 5;
66
// The minimum sample size required to perform statistical analysis.
77
export const minSamples = 5;
8+
9+
export const nodeFlags: ReadonlyArray<string> = [
10+
'--predictable',
11+
'--no-concurrent-sweeping',
12+
'--no-minor-gc-task',
13+
'--min-semi-space-size=1280', // 1.25GB
14+
'--max-semi-space-size=1280', // 1.25GB
15+
'--trace-gc', // no gc calls should happen during benchmark, so trace them
16+
];

‎resources/benchmark/sampling.ts‎

Lines changed: 3 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import assert from 'node:assert';
2-
import cp from 'node:child_process';
3-
import url from 'node:url';
42

53
import { maxTime, minSamples } from './config.js';
64
import { yellow } from './output.js';
75
import type { BenchmarkSample } from './types.js';
6+
import { sampleModule } from './workers.js';
7+
8+
export { sampleModule };
89

910
export function collectSamples(modulePath: string): Array<BenchmarkSample> {
1011
let numOfConsequentlyRejectedSamples = 0;
@@ -35,72 +36,3 @@ export function collectSamples(modulePath: string): Array<BenchmarkSample> {
3536
}
3637
return samples;
3738
}
38-
39-
export function sampleModule(modulePath: string): BenchmarkSample {
40-
// To support Windows we need to use URL instead of path
41-
const moduleURL = url.pathToFileURL(modulePath);
42-
43-
const sampleCode = `
44-
import fs from 'node:fs';
45-
46-
import { benchmark } from '${moduleURL}';
47-
48-
// warm up, it looks like 7 is a magic number to reliably trigger JIT
49-
await benchmark.measure();
50-
await benchmark.measure();
51-
await benchmark.measure();
52-
await benchmark.measure();
53-
await benchmark.measure();
54-
await benchmark.measure();
55-
await benchmark.measure();
56-
57-
const memBaseline = process.memoryUsage().heapUsed;
58-
59-
const resourcesStart = process.resourceUsage();
60-
const startTime = process.hrtime.bigint();
61-
for (let i = 0; i < benchmark.count; ++i) {
62-
await benchmark.measure();
63-
}
64-
const timeDiff = Number(process.hrtime.bigint() - startTime);
65-
const resourcesEnd = process.resourceUsage();
66-
67-
const result = {
68-
name: benchmark.name,
69-
clocked: timeDiff / benchmark.count,
70-
memUsed: (process.memoryUsage().heapUsed - memBaseline) / benchmark.count,
71-
involuntaryContextSwitches:
72-
resourcesEnd.involuntaryContextSwitches - resourcesStart.involuntaryContextSwitches,
73-
};
74-
fs.writeFileSync(3, JSON.stringify(result));
75-
`;
76-
77-
const result = cp.spawnSync(
78-
process.execPath,
79-
[
80-
// V8 flags
81-
'--predictable',
82-
'--no-concurrent-sweeping',
83-
'--no-minor-gc-task',
84-
'--min-semi-space-size=1280', // 1.25GB
85-
'--max-semi-space-size=1280', // 1.25GB
86-
'--trace-gc', // no gc calls should happen during benchmark, so trace them
87-
88-
// Node.js flags
89-
'--input-type=module',
90-
'--eval',
91-
sampleCode,
92-
],
93-
{
94-
stdio: ['inherit', 'inherit', 'inherit', 'pipe'],
95-
env: { NODE_ENV: 'production' },
96-
},
97-
);
98-
99-
if (result.status !== 0) {
100-
throw new Error(`Benchmark failed with "${result.status}" status.`);
101-
}
102-
103-
const resultStr = result.output[3]?.toString();
104-
assert(resultStr != null);
105-
return JSON.parse(resultStr);
106-
}

‎resources/benchmark/worker-timing.js‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import {
2+
loadBenchmark,
3+
readModulePath,
4+
runWorker,
5+
writeResult,
6+
} from './worker-utils.js';
7+
8+
runWorker(async () => {
9+
const benchmark = await loadBenchmark(readModulePath());
10+
await warmUp(benchmark);
11+
12+
const memBaseline = process.memoryUsage().heapUsed;
13+
14+
const resourcesStart = process.resourceUsage();
15+
const startTime = process.hrtime.bigint();
16+
for (let i = 0; i < benchmark.count; ++i) {
17+
// eslint-disable-next-line no-await-in-loop
18+
await benchmark.measure();
19+
}
20+
const timeDiff = Number(process.hrtime.bigint() - startTime);
21+
const resourcesEnd = process.resourceUsage();
22+
23+
writeResult({
24+
name: benchmark.name,
25+
clocked: timeDiff / benchmark.count,
26+
memUsed: (process.memoryUsage().heapUsed - memBaseline) / benchmark.count,
27+
involuntaryContextSwitches:
28+
resourcesEnd.involuntaryContextSwitches -
29+
resourcesStart.involuntaryContextSwitches,
30+
});
31+
});
32+
33+
async function warmUp(benchmark) {
34+
// It looks like 7 is a magic number to reliably trigger JIT.
35+
await benchmark.measure();
36+
await benchmark.measure();
37+
await benchmark.measure();
38+
await benchmark.measure();
39+
await benchmark.measure();
40+
await benchmark.measure();
41+
await benchmark.measure();
42+
}

‎resources/benchmark/worker-utils.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import assert from 'node:assert';
2+
import fs from 'node:fs';
3+
import url from 'node:url';
4+
5+
export function readModulePath() {
6+
const [modulePath] = process.argv.slice(2);
7+
assert(modulePath != null);
8+
return modulePath;
9+
}
10+
11+
export async function loadBenchmark(modulePath) {
12+
const moduleURL = url.pathToFileURL(modulePath);
13+
const module = await import(moduleURL.href);
14+
const benchmark = module.benchmark;
15+
if (benchmark?.name == null) {
16+
throw new Error(`Benchmark at ${modulePath} must define a name.`);
17+
}
18+
assert(typeof benchmark.measure === 'function');
19+
return benchmark;
20+
}
21+
22+
export function writeResult(result) {
23+
fs.writeFileSync(3, JSON.stringify(result));
24+
}
25+
26+
export function runWorker(main) {
27+
main().catch((error) => {
28+
const errorMessage =
29+
error instanceof Error ? (error.stack ?? error.message) : String(error);
30+
process.stderr.write(errorMessage + '\n');
31+
process.exitCode = 1;
32+
});
33+
}

‎resources/benchmark/workers.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import assert from 'node:assert';
2+
import childProcess from 'node:child_process';
3+
4+
import { localRepoPath } from '../utils.js';
5+
6+
import { nodeFlags } from './config.js';
7+
import type { BenchmarkSample } from './types.js';
8+
9+
export function sampleModule(modulePath: string): BenchmarkSample {
10+
return runWorkerFile(
11+
localRepoPath('resources/benchmark/worker-timing.js'),
12+
modulePath,
13+
) as BenchmarkSample;
14+
}
15+
16+
function runWorkerFile(workerPath: string, modulePath: string): unknown {
17+
const result = childProcess.spawnSync(
18+
process.execPath,
19+
[...nodeFlags, workerPath, modulePath],
20+
{
21+
stdio: ['inherit', 'inherit', 'inherit', 'pipe'],
22+
env: { NODE_ENV: 'production' },
23+
},
24+
);
25+
if (result.status !== 0) {
26+
throw new Error(`Benchmark worker failed with "${result.status}" status.`);
27+
}
28+
29+
const resultStr = result.output[3]?.toString();
30+
assert(resultStr != null);
31+
return JSON.parse(resultStr);
32+
}

0 commit comments

Comments
 (0)