ci(NODE-6755): add tags to benchmarks

test/benchmarks/driver_bench/readme.md

@@ -51,13 +51,30 @@ type BenchmarkModule = {
   run: () => Promise<void>;
   afterEach?: () => Promise<void>;
   after?: () => Promise<void>;
+
+  tags?: string[];
 };
 ```
 
 Just like mocha we have once before and once after as well as before each and after each hooks.
 
 The `driver.mts` module is intended to hold various helpers for setup and teardown and help abstract some of the driver API.
 
+## Benchmark tags
+The `tags` property of `BenchmarkModule` is where a benchmark's tags should be added to facilitate
+performance alerting and filter of results via our internal tools.
+
+Tags are defined in `driver.mts` and should end with a _TAG suffix.
+Whenever a new tag is defined it should be documented in the table below .
+
+| tag variable name |    tag string value    |                                                               purpose                                                                |
+|-------------------|------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+|     `SPEC_TAG`    | `'spec-benchmark'`     | Special tag that marks a benchmark as a spec-required benchmark                                                                      |
+|     `ALERT_TAG`   | `'alerting-benchmark'` | Special tag that enables our perf monitoring tooling to create alerts when regressions in this benchmark's performance are detected  |
+|     `CURSOR_TAG`  | `'cursor-benchmark'`   | Tag marking a benchmark as being related to cursor performance                                                                       |
+|     `READ_TAG`    | `'read-benchmark'`     | Tag marking a benchmark as being related to read performance                                                                         |
+|     `WRITE_TAG`   | `'write-benchmark'`    | Tag marking a benchmark as being related to write performance                                                                        |
+
 ## Wishlist
 
 - Make it so runner can handle: `./lib/suites/multi_bench/grid_fs_upload.mjs` as an argument so shell path autocomplete makes it easier to pick a benchmark

test/benchmarks/driver_bench/src/driver.mts

@@ -7,6 +7,20 @@ import process from 'node:process';
 const __dirname = import.meta.dirname;
 const require = module.createRequire(__dirname);
 
+// Special tag that marks a benchmark as a spec-required benchmark
+export const SPEC_TAG = 'spec-benchmark';
+// Special tag that enables our perf monitoring tooling to create alerts when regressions in this
+// benchmark's performance are detected
+export const ALERT_TAG = 'alerting-benchmark';
+// Tag marking a benchmark as being related to cursor performance
+export const CURSOR_TAG = 'cursor-benchmark';
+// Tag marking a benchmark as being related to read performance
+export const READ_TAG = 'read-benchmark';
+// Tag marking a benchmark as being related to write performance
+export const WRITE_TAG = 'write-benchmark';
+
+export const NORMALIZED_PING_SCALING_CONST = 1000;
+
 /**
  * The path to the MongoDB Node.js driver.
  * This MUST be set to the directory the driver is installed in
@@ -118,19 +132,20 @@ export const PARALLEL_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'parallel');
 export const TEMP_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'tmp');
 
 export type Metric = {
-  name: 'megabytes_per_second';
+  name: 'megabytes_per_second' | 'normalized_throughput';
   value: number;
 };
 
 export type MetricInfo = {
   info: {
     test_name: string;
     args: Record<string, number>;
+    tags?: string[]
   };
   metrics: Metric[];
 };
 
-export function metrics(test_name: string, result: number): MetricInfo {
+export function metrics(test_name: string, result: number, tags?: string[]): MetricInfo {
   return {
     info: {
       test_name,
@@ -141,7 +156,8 @@ export function metrics(test_name: string, result: number): MetricInfo {
           key,
           typeof value === 'number' ? value : value ? 1 : 0
         ])
-      )
+      ),
+      tags
     },
     metrics: [{ name: 'megabytes_per_second', value: result }]
   } as const;

test/benchmarks/driver_bench/src/main.mts

@@ -18,6 +18,7 @@ import {
   MONGODB_DRIVER_PATH,
   MONGODB_DRIVER_REVISION,
   MONGODB_DRIVER_VERSION,
+  NORMALIZED_PING_SCALING_CONST,
   snakeToCamel
 } from './driver.mjs';
 
@@ -85,7 +86,7 @@ console.log(systemInfo());
 
 const runnerPath = path.join(__dirname, 'runner.mjs');
 
-const results: MetricInfo[] = [];
+let results: MetricInfo[] = [];
 
 for (const [suite, benchmarks] of Object.entries(tests)) {
   console.group(snakeToCamel(suite));
@@ -146,8 +147,8 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) {
 
   const aMetricInfo =
     (testName: string) =>
-    ({ info: { test_name } }: MetricInfo) =>
-      test_name === testName;
+      ({ info: { test_name } }: MetricInfo) =>
+        test_name === testName;
 
   const anMBsMetric = ({ name }: Metric) => name === 'megabytes_per_second';
 
@@ -198,6 +199,33 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) {
   return [...results, ...compositeResults];
 }
 
-const finalResults = calculateCompositeBenchmarks(results);
+function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] {
+  const primesBench = results.find(r => r.info.test_name === 'primes');
+  const pingBench = results.find(r => r.info.test_name === 'ping');
+
+  if (pingBench) {
+    const pingThroughput = pingBench.metrics[0].value;
+    for (const bench of results) {
+      if (bench.info.test_name === 'ping') {
+        // Compute ping's normalized_throughput against the primes bench if present
+        if (primesBench) {
+          const primesThroughput = primesBench.metrics[0].value;
+          const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: NORMALIZED_PING_SCALING_CONST * bench.metrics[0].value / primesThroughput };
+          bench.metrics.push(normalizedThroughput);
+        }
+      } else {
+        // Compute normalized_throughput of benchmarks against ping bench
+        const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: bench.metrics[0].value / pingThroughput };
+        bench.metrics.push(normalizedThroughput);
+      }
+    }
+  }
+
+  return results;
+}
+
+results = calculateCompositeBenchmarks(results);
+results = calculateNormalizedResults(results);
+
 
-await fs.writeFile('results.json', JSON.stringify(finalResults, undefined, 2), 'utf8');
+await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8');

test/benchmarks/driver_bench/src/runner.mts

@@ -14,8 +14,10 @@ type BenchmarkModule = {
   run: () => Promise<void>;
   afterEach?: () => Promise<void>;
   after?: () => Promise<void>;
+  tags?: string[];
 };
 
+
 const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs'));
 const benchmark: BenchmarkModule = await import(`./${benchmarkFile}`);
 
@@ -79,6 +81,7 @@ function percentileIndex(percentile: number, count: number) {
 
 const medianExecution = durations[percentileIndex(50, count)];
 const megabytesPerSecond = benchmark.taskSize / medianExecution;
+const tags = benchmark.tags;
 
 console.log(
   ' '.repeat(3),
@@ -91,6 +94,6 @@ console.log(
 
 await fs.writeFile(
   `results_${path.basename(benchmarkFile, '.mjs')}.json`,
-  JSON.stringify(metrics(benchmarkName, megabytesPerSecond), undefined, 2) + '\n',
+  JSON.stringify(metrics(benchmarkName, megabytesPerSecond, tags), undefined, 2) + '\n',
   'utf8'
 );

test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts

@@ -1,8 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, ALERT_TAG, SPEC_TAG, type mongodb, CURSOR_TAG, READ_TAG } from '../../driver.mjs';
 
 export const taskSize = 16.22;
 
+export const tags = [ALERT_TAG, SPEC_TAG, CURSOR_TAG, READ_TAG];
+
 let collection: mongodb.Collection;
 
 export async function before() {

test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts

@@ -1,10 +1,12 @@
 import { Readable, Writable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, SPEC_TAG, ALERT_TAG, type mongodb, CURSOR_TAG, READ_TAG } from '../../driver.mjs';
 
 export const taskSize = 52.43;
 
+export const tags = [ALERT_TAG, SPEC_TAG, CURSOR_TAG, READ_TAG];
+
 let bucket: mongodb.GridFSBucket;
 let bin: Uint8Array;
 let _id: mongodb.ObjectId;

test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts

@@ -1,9 +1,10 @@
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 52.43;
+export const tags = [ALERT_TAG, SPEC_TAG, WRITE_TAG];
 
 let bucket: mongodb.GridFSBucket;
 let uploadStream: mongodb.GridFSBucketWriteStream;

test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 27.31;
+export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG];
 
 let collection: mongodb.Collection;
 let documents: any[];

test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 2.75;
+export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG];
 
 let collection: mongodb.Collection;
 let documents: any[];

test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 16;
+export const tags = [ALERT_TAG, CURSOR_TAG, READ_TAG];
 
 let db: mongodb.Db;
 

test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 1500;
+export const tags = [ALERT_TAG, CURSOR_TAG, READ_TAG];
 
 let db: mongodb.Db;
 let tweet: Record<string, any>;

test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts

@@ -1,7 +1,9 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, SPEC_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 16.22;
 
+export const tags = [SPEC_TAG, ALERT_TAG,READ_TAG, CURSOR_TAG]
+
 let collection: mongodb.Collection;
 
 export async function before() {

test/benchmarks/driver_bench/src/suites/node_specific/ping.mts

@@ -1,7 +1,8 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, driver, type mongodb } from '../../driver.mjs';
 
 // { ping: 1 } is 15 bytes of BSON x 10,000 iterations
 export const taskSize = 0.15;
+export const tags = [ALERT_TAG];
 
 let db: mongodb.Db;
 

test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts

@@ -2,9 +2,10 @@ import { createReadStream, createWriteStream, promises as fs } from 'node:fs';
 import path from 'node:path';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb, PARALLEL_DIRECTORY, TEMP_DIRECTORY } from '../../driver.mjs';
+import { ALERT_TAG, CURSOR_TAG, driver, type mongodb, PARALLEL_DIRECTORY, READ_TAG, SPEC_TAG, TEMP_DIRECTORY } from '../../driver.mjs';
 
 export const taskSize = 262.144;
+export const tags = [SPEC_TAG, ALERT_TAG, READ_TAG, CURSOR_TAG];
 
 let bucket: mongodb.GridFSBucket;
 

test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts

@@ -3,9 +3,10 @@ import path from 'node:path';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs';
+import { ALERT_TAG, driver, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, WRITE_TAG } from '../../driver.mjs';
 
 export const taskSize = 262.144;
+export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG];
 
 let bucket: mongodb.GridFSBucket;
 

test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts

@@ -3,9 +3,10 @@ import path from 'node:path';
 import readline from 'node:readline/promises';
 import stream from 'node:stream/promises';
 
-import { driver, EJSON, type mongodb, PARALLEL_DIRECTORY, TEMP_DIRECTORY } from '../../driver.mjs';
+import { ALERT_TAG, driver, EJSON, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, TEMP_DIRECTORY, WRITE_TAG } from '../../driver.mjs';
 
 export const taskSize = 565;
+export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG];
 
 let collection: mongodb.Collection;
 

test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts

@@ -2,9 +2,10 @@ import { createReadStream, promises as fs } from 'node:fs';
 import path from 'node:path';
 import readline from 'node:readline/promises';
 
-import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs';
+import { ALERT_TAG, driver, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, WRITE_TAG } from '../../driver.mjs';
 
 export const taskSize = 565;
+export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG];
 
 const directory = path.resolve(PARALLEL_DIRECTORY, 'ldjson_multi');
 let collection: mongodb.Collection;

test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, SPEC_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 16.22;
+export const tags = [SPEC_TAG, ALERT_TAG, CURSOR_TAG, READ_TAG];
 
 let collection: mongodb.Collection<{ _id: number }>;
 

test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 27.31;
+export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG];
 
 let collection: mongodb.Collection;
 let documents: Record<string, any>[];

test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts

@@ -1,7 +1,8 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, driver, SPEC_TAG, type mongodb } from '../../driver.mjs';
 
 // { hello: true } is 13 bytes of BSON x 10,000 iterations
 export const taskSize = 0.13;
+export const tags = [SPEC_TAG, ALERT_TAG];
 
 let db: mongodb.Db;
 

test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs';
 
 export const taskSize = 2.75;
+export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG];
 
 let collection: mongodb.Collection;
 let documents: Record<string, any>[];