fix: divan walltime metric

Author: not-matthias

crates/codspeed/src/walltime.rs

@@ -26,13 +26,13 @@ impl RawWallTimeData {
         uri: String,
         iter_per_round: u32,
         max_time_ns: Option<u128>,
-        times_ns: Vec<u128>,
+        avg_round_times_ns: Vec<u128>,
     ) -> Self {
         RawWallTimeData {
             metadata: BenchmarkMetadata { name, uri },
             iter_per_round,
             max_time_ns,
-            times_ns,
+            times_ns: avg_round_times_ns,
         }
     }
 
@@ -50,13 +50,39 @@ impl RawWallTimeData {
 /// Entry point called in patched integration to harvest raw walltime data
 ///
 /// `CODSPEED_CARGO_WORKSPACE_ROOT` is expected to be set for this to work
+///
+/// # Arguments
+///
+/// - `scope`: The used integration, e.g. "divan" or "criterion"
+/// - `name`: The name of the benchmark
+/// - `uri`: The URI of the benchmark
+/// - `iter_per_round`: The number of iterations per round for the benchmark (=sample_size)
+/// - `max_time_ns`: The time limit for the benchmark in nanoseconds (if defined)
+/// - `avg_round_times_ns`: The **average** time per iteration in nanoseconds. Calculated as `total_round_time_ns / iter_per_round`
+///
+/// # Terminology
+///
+/// - `sample_count`: The number of times the benchmark was executed.
+/// - `sample_size` (=iter_per_round/iterations): The number of iterations within each round.
+///
+/// ```
+/// for round in 0..sample_count {
+///     let total_round_time_ns = 0;
+///     for iteration in 0..iter_per_round {
+///         run_benchmark();
+///         total_round_time_ns += /* measured execution time */;
+///     }
+///     let avg_round_times_ns = total_round_time_ns / iter_per_round;
+/// }
+/// ```
+///
 pub fn collect_raw_walltime_results(
     scope: &str,
     name: String,
     uri: String,
     iter_per_round: u32,
     max_time_ns: Option<u128>,
-    times_ns: Vec<u128>,
+    avg_round_times_ns: Vec<u128>,
 ) {
     if !crate::utils::running_with_codspeed_runner() {
         return;
@@ -66,7 +92,13 @@ pub fn collect_raw_walltime_results(
         eprintln!("codspeed failed to get workspace root. skipping");
         return;
     };
-    let data = RawWallTimeData::from_runtime_data(name, uri, iter_per_round, max_time_ns, times_ns);
+    let data = RawWallTimeData::from_runtime_data(
+        name,
+        uri,
+        iter_per_round,
+        max_time_ns,
+        avg_round_times_ns,
+    );
     data.dump_to_results(&workspace_root, scope);
 }
 

crates/divan_compat/Cargo.toml

@@ -25,3 +25,7 @@ codspeed-divan-compat-macros = { version = "=2.10.1", path = './macros' }
 [[bench]]
 name = "basic_example"
 harness = false
+
+[[bench]]
+name = "sleep_benches"
+harness = false

crates/divan_compat/benches/sleep_benches.rs

@@ -0,0 +1,49 @@
+#[divan::bench]
+fn sleep_1ns() {
+    std::thread::sleep(std::time::Duration::from_nanos(1));
+}
+
+#[divan::bench]
+fn sleep_100ns() {
+    std::thread::sleep(std::time::Duration::from_nanos(100));
+}
+
+#[divan::bench]
+fn sleep_1us() {
+    std::thread::sleep(std::time::Duration::from_micros(1));
+}
+
+#[divan::bench]
+fn sleep_100us() {
+    std::thread::sleep(std::time::Duration::from_micros(100));
+}
+
+#[divan::bench]
+fn sleep_1ms() {
+    std::thread::sleep(std::time::Duration::from_millis(1));
+}
+
+#[divan::bench]
+fn sleep_10ms() {
+    std::thread::sleep(std::time::Duration::from_millis(10));
+}
+
+#[divan::bench]
+fn sleep_50ms() {
+    std::thread::sleep(std::time::Duration::from_millis(50));
+}
+
+#[divan::bench]
+fn sleep_100ms() {
+    std::thread::sleep(std::time::Duration::from_millis(100));
+}
+
+// Tests COD-1044, do not modify the sample size or count!
+#[divan::bench(sample_size = 3, sample_count = 6)]
+fn sleep_100ms_with_custom_sample() {
+    std::thread::sleep(std::time::Duration::from_millis(100));
+}
+
+fn main() {
+    codspeed_divan_compat::main();
+}

crates/divan_compat/divan_fork/src/divan.rs

@@ -424,8 +424,13 @@ mod codspeed {
         };
 
         let iter_per_round = bench_context.samples.sample_size;
+
+        // Note: times_ns = total time of the round execution, but we need the time per iteration.
+        // See: BenchContext::compute_stats in bench/mod.rs
         let times_ns: Vec<_> =
             bench_context.samples.time_samples.iter().map(|s| s.duration.picos / 1_000).collect();
+        let avg_times_ns: Vec<_> = times_ns.iter().map(|&t| t / iter_per_round as u128).collect();
+
         let max_time_ns = bench_context.options.max_time.map(|t| t.as_nanos());
 
         if let Err(error) = ::codspeed::fifo::send_cmd(codspeed::fifo::Command::CurrentBenchmark {
@@ -443,7 +448,7 @@ mod codspeed {
             uri,
             iter_per_round,
             max_time_ns,
-            times_ns,
+            avg_times_ns,
         );
     }
 }