diff --git a/test/benchmarks/driver_bench/readme.md b/test/benchmarks/driver_bench/readme.md index 81f8b859745..d660a997672 100644 --- a/test/benchmarks/driver_bench/readme.md +++ b/test/benchmarks/driver_bench/readme.md @@ -51,6 +51,8 @@ type BenchmarkModule = { run: () => Promise; afterEach?: () => Promise; after?: () => Promise; + + tags?: string[]; }; ``` @@ -58,6 +60,21 @@ Just like mocha we have once before and once after as well as before each and af The `driver.mts` module is intended to hold various helpers for setup and teardown and help abstract some of the driver API. +## Benchmark tags +The `tags` property of `BenchmarkModule` is where a benchmark's tags should be added to facilitate +performance alerting and filter of results via our internal tools. + +Tags are defined in `driver.mts` under the `TAG` enum. +Whenever a new tag is defined it should be documented in the table below . + +| tag variable name | tag string value | purpose | +|-------------------|------------------------|--------------------------------------------------------------------------------------------------------------------------------------| +| `TAG.spec` | `'spec-benchmark'` | Special tag that marks a benchmark as a spec-required benchmark | +| `TAG.alert` | `'alerting-benchmark'` | Special tag that enables our perf monitoring tooling to create alerts when regressions in this benchmark's performance are detected | +| `TAG.cursor` | `'cursor-benchmark'` | Tag marking a benchmark as being related to cursor performance | +| `TAG.read` | `'read-benchmark'` | Tag marking a benchmark as being related to read performance | +| `TAG.write` | `'write-benchmark'` | Tag marking a benchmark as being related to write performance | + ## Wishlist - Make it so runner can handle: `./lib/suites/multi_bench/grid_fs_upload.mjs` as an argument so shell path autocomplete makes it easier to pick a benchmark diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index ab172fe7bd6..dffe48408d6 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -7,6 +7,20 @@ import process from 'node:process'; const __dirname = import.meta.dirname; const require = module.createRequire(__dirname); +export const TAG = { + // Special tag that marks a benchmark as a spec-required benchmark + spec: 'spec-benchmark', + // Special tag that enables our perf monitoring tooling to create alerts when regressions in this + // benchmark's performance are detected + alert: 'alerting-benchmark', + // Tag marking a benchmark as being related to cursor performance + cursor: 'cursor-benchmark', + // Tag marking a benchmark as being related to read performance + read: 'read-benchmark', + // Tag marking a benchmark as being related to write performance + write: 'write-benchmark' +}; + /** * The path to the MongoDB Node.js driver. * This MUST be set to the directory the driver is installed in @@ -118,19 +132,23 @@ export const PARALLEL_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'parallel'); export const TEMP_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'tmp'); export type Metric = { - name: 'megabytes_per_second'; + name: 'megabytes_per_second' | 'normalized_throughput'; value: number; + metadata: { + improvement_direction: 'up' | 'down'; + }; }; export type MetricInfo = { info: { test_name: string; args: Record; + tags?: string[]; }; metrics: Metric[]; }; -export function metrics(test_name: string, result: number): MetricInfo { +export function metrics(test_name: string, result: number, tags?: string[]): MetricInfo { return { info: { test_name, @@ -141,9 +159,13 @@ export function metrics(test_name: string, result: number): MetricInfo { key, typeof value === 'number' ? value : value ? 1 : 0 ]) - ) + ), + tags }, - metrics: [{ name: 'megabytes_per_second', value: result }] + // FIXME(NODE-6781): For now all of our metrics are of throughput so their improvement_direction is up, + metrics: [ + { name: 'megabytes_per_second', value: result, metadata: { improvement_direction: 'up' } } + ] } as const; } diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 67309b7d49a..04e573b7734 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -85,7 +85,7 @@ console.log(systemInfo()); const runnerPath = path.join(__dirname, 'runner.mjs'); -const results: MetricInfo[] = []; +let results: MetricInfo[] = []; for (const [suite, benchmarks] of Object.entries(tests)) { console.group(snakeToCamel(suite)); @@ -198,6 +198,42 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { return [...results, ...compositeResults]; } -const finalResults = calculateCompositeBenchmarks(results); +function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { + const baselineBench = results.find(r => r.info.test_name === 'cpuBaseline'); + const pingBench = results.find(r => r.info.test_name === 'ping'); + + assert.ok(pingBench, 'ping bench results not found!'); + assert.ok(baselineBench, 'baseline results not found!'); + const pingThroughput = pingBench.metrics[0].value; + const cpuBaseline = baselineBench.metrics[0].value; + + for (const bench of results) { + if (bench.info.test_name === 'cpuBaseline') continue; + if (bench.info.test_name === 'ping') { + bench.metrics.push({ + name: 'normalized_throughput', + value: bench.metrics[0].value / cpuBaseline, + metadata: { + improvement_direction: 'up' + } + }); + } + // Compute normalized_throughput of benchmarks against ping bench + else { + bench.metrics.push({ + name: 'normalized_throughput', + value: bench.metrics[0].value / pingThroughput, + metadata: { + improvement_direction: 'up' + } + }); + } + } + + return results; +} + +results = calculateCompositeBenchmarks(results); +results = calculateNormalizedResults(results); -await fs.writeFile('results.json', JSON.stringify(finalResults, undefined, 2), 'utf8'); +await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8'); diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index f360b2ef0ac..b68bc9cfe49 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -14,6 +14,7 @@ type BenchmarkModule = { run: () => Promise; afterEach?: () => Promise; after?: () => Promise; + tags?: string[]; }; const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs')); @@ -80,6 +81,14 @@ function percentileIndex(percentile: number, count: number) { const medianExecution = durations[percentileIndex(50, count)]; const megabytesPerSecond = benchmark.taskSize / medianExecution; +const tags = benchmark.tags; +if ( + tags && + (!Array.isArray(tags) || (tags.length > 0 && !tags.every(t => typeof t === 'string'))) +) { + throw new Error('If tags is specified, it MUST be an array of strings'); +} + console.log( ' '.repeat(3), ...['total time:', totalDuration, 'sec,'], @@ -91,6 +100,6 @@ console.log( await fs.writeFile( `results_${path.basename(benchmarkFile, '.mjs')}.json`, - JSON.stringify(metrics(benchmarkName, megabytesPerSecond), undefined, 2) + '\n', + JSON.stringify(metrics(benchmarkName, megabytesPerSecond, tags), undefined, 2) + '\n', 'utf8' ); diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts index 09c4cadd97a..deb745ea646 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts @@ -1,8 +1,10 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 16.22; +export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read]; + let collection: mongodb.Collection; export async function before() { diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts index 31b85447858..67b1c802517 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts @@ -1,10 +1,12 @@ import { Readable, Writable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 52.43; +export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read]; + let bucket: mongodb.GridFSBucket; let bin: Uint8Array; let _id: mongodb.ObjectId; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts index 86361639d3c..002c530ec01 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts @@ -1,9 +1,10 @@ import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 52.43; +export const tags = [TAG.alert, TAG.spec, TAG.write]; let bucket: mongodb.GridFSBucket; let uploadStream: mongodb.GridFSBucketWriteStream; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts index 5b913aefe6c..6ca9edaaf64 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 27.31; +export const tags = [TAG.alert, TAG.spec, TAG.write]; let collection: mongodb.Collection; let documents: any[]; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts index 922002f49b4..0047731fe1b 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 2.75; +export const tags = [TAG.alert, TAG.spec, TAG.write]; let collection: mongodb.Collection; let documents: any[]; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts index 9a9e3f92fdd..87f24ef3fee 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 16; +export const tags = [TAG.alert, TAG.cursor, TAG.read]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts index da2ba5cc416..4432f40a4ff 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 1500; +export const tags = [TAG.alert, TAG.cursor, TAG.read]; let db: mongodb.Db; let tweet: Record; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/primes.mts b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts similarity index 84% rename from test/benchmarks/driver_bench/src/suites/node_specific/primes.mts rename to test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts index 6b5906408cc..9760f90921b 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/primes.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts @@ -6,12 +6,9 @@ const expectedPrimes = 78_498; // byteLength of // BSON.serialize({ primes: Buffer.from(new Int32Array(sieveOfEratosthenes(findPrimesBelow)).buffer) }).byteLength) // a bin data of int32s -const byteLength = 314_010; - -export const taskSize = 3.1401000000000003; // ~3MB worth of work - -assert.equal(taskSize, byteLength * 10e-6); // taskSize should stay hardcoded, checking here the math is done right. +const stableRegionMean = 42.82; +export const taskSize = 3.1401000000000003 / stableRegionMean; // ~3MB worth of work scaled down by the mean of the current stable region in CI to bring this value to roughly 1 /** @see https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes */ export function sieveOfEratosthenes(n: number) { // Create a boolean array "prime[0..n]" and initialize diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts index 778d44c3a3d..f435995ea98 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts @@ -1,7 +1,9 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 16.22; +export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read]; + let collection: mongodb.Collection; export async function before() { diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts index ac8e462381a..2f95eea344d 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts @@ -1,7 +1,8 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; // { ping: 1 } is 15 bytes of BSON x 10,000 iterations export const taskSize = 0.15; +export const tags = [TAG.alert]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts index 9345826a604..656919ed9b2 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts @@ -2,9 +2,10 @@ import { createReadStream, createWriteStream, promises as fs } from 'node:fs'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; -import { driver, type mongodb, PARALLEL_DIRECTORY, TEMP_DIRECTORY } from '../../driver.mjs'; +import { driver, type mongodb, PARALLEL_DIRECTORY, TAG, TEMP_DIRECTORY } from '../../driver.mjs'; export const taskSize = 262.144; +export const tags = [TAG.spec, TAG.alert, TAG.read, TAG.cursor]; let bucket: mongodb.GridFSBucket; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts index 6c29c18ea15..0942f6bc414 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts @@ -3,9 +3,10 @@ import path from 'node:path'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs'; +import { driver, type mongodb, PARALLEL_DIRECTORY, TAG } from '../../driver.mjs'; export const taskSize = 262.144; +export const tags = [TAG.spec, TAG.alert, TAG.write]; let bucket: mongodb.GridFSBucket; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts index 81d05dcb526..62fef2aa600 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts @@ -3,9 +3,17 @@ import path from 'node:path'; import readline from 'node:readline/promises'; import stream from 'node:stream/promises'; -import { driver, EJSON, type mongodb, PARALLEL_DIRECTORY, TEMP_DIRECTORY } from '../../driver.mjs'; +import { + driver, + EJSON, + type mongodb, + PARALLEL_DIRECTORY, + TAG, + TEMP_DIRECTORY +} from '../../driver.mjs'; export const taskSize = 565; +export const tags = [TAG.spec, TAG.alert, TAG.write]; let collection: mongodb.Collection; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts index 5f5dab24f90..46a53823924 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts @@ -2,9 +2,10 @@ import { createReadStream, promises as fs } from 'node:fs'; import path from 'node:path'; import readline from 'node:readline/promises'; -import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs'; +import { driver, type mongodb, PARALLEL_DIRECTORY, TAG } from '../../driver.mjs'; export const taskSize = 565; +export const tags = [TAG.spec, TAG.alert, TAG.write]; const directory = path.resolve(PARALLEL_DIRECTORY, 'ldjson_multi'); let collection: mongodb.Collection; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts index d99ac5f9bfe..e079c723c3e 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 16.22; +export const tags = [TAG.spec, TAG.alert, TAG.cursor, TAG.read]; let collection: mongodb.Collection<{ _id: number }>; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts index 4dcae894903..70cf758888b 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 27.31; +export const tags = [TAG.spec, TAG.alert, TAG.write]; let collection: mongodb.Collection; let documents: Record[]; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts b/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts index a2884ebb343..0a55c4fae37 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts @@ -1,7 +1,8 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; // { hello: true } is 13 bytes of BSON x 10,000 iterations export const taskSize = 0.13; +export const tags = [TAG.spec, TAG.alert]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts index 4995a9b2539..070b0fff7e5 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 2.75; +export const tags = [TAG.spec, TAG.alert, TAG.write]; let collection: mongodb.Collection; let documents: Record[];