Update google-benchmark to 1.7.0 (#3151)

Co-authored-by: Nicole Mazzuca <[email protected]>

Author: AZero13

CMakeLists.txt

@@ -42,11 +42,16 @@ if(TARGET run-validate)
 endif()
 
 option(BUILD_TESTING "Enable testing" ON)
-option(STL_BUILD_BENCHMARKING "Enable benchmarking" OFF)
 set(VCLIBS_SUFFIX "_oss" CACHE STRING "suffix for built DLL names to avoid conflicts with distributed DLLs")
 
 option(STL_USE_ANALYZE "Pass the /analyze flag to MSVC" OFF)
 
+if(STL_BUILD_BENCHMARKING)
+    message(WARNING "Due to issues with building google benchmark, "
+        "STL_BUILD_BENCHMARKING is no longer allowed as part of the STL build. "
+        "Instead, build the `benchmarks` directory with the STL_BINARY_DIR option.")
+endif()
+
 if("${VCLIBS_TARGET_ARCHITECTURE}" MATCHES "^x86$")
     set(VCLIBS_TARGET_ARCHITECTURE "x86")
     set(VCLIBS_I386_OR_AMD64 "i386")
@@ -115,9 +120,3 @@ if(BUILD_TESTING)
     enable_testing()
     add_subdirectory(tests)
 endif()
-if(STL_BUILD_BENCHMARKING)
-    if(VCLIBS_TARGET_ARCHITECTURE STREQUAL "arm")
-        message(WARNING "google benchmark does not build on arm32 - this build will likely fail.")
-    endif()
-    add_subdirectory(benchmarks)
-endif()

README.md

@@ -413,17 +413,19 @@ You may also modify an existing benchmark file. We use Google's [Benchmark][gben
 so you may find [their documentation][gbenchmark:docs] helpful, and you can also read the existing code
 for how _we_ use it.
 
-To run benchmarks, you'll need to configure the STL with the `-DSTL_BUILD_BENCHMARKING=ON` option:
+To run benchmarks, you'll need to first build the STL, then build the benchmarks:
 
 ```cmd
-cmake -B out\bench -S . -G Ninja -DSTL_BUILD_BENCHMARKING=ON
-cmake --build out\bench
+cmake -B out\x64 -S . -G Ninja
+cmake --build out\x64
+cmake -B out\benchmark -S benchmarks -G Ninja -DSTL_BINARY_DIR=out\x64
+cmake --build out\benchmark
 ```
 
 You can then run your benchmark with:
 
 ```cmd
-out\bench\benchmarks\benchmark-<benchmark-name> --benchmark_out=<file> --benchmark_out_format=csv
+out\benchmark\benchmark-<benchmark-name> --benchmark_out=<file> --benchmark_out_format=csv
 ```
 
 And then you can copy this csv file into Excel, or another spreadsheet program. For example:

azure-devops/cmake-configure-build.yml

@@ -11,9 +11,9 @@ parameters:
 - name: buildOutputLocationVar
   type: string
   default: buildOutputLocation
-- name: buildBenchmarking
+- name: benchmarkBuildOutputLocationVar
   type: string
-  default: 'ON'
+  default: benchmarkBuildOutputLocation
 - name: cmakeAdditionalFlags
   type: string
   default: ''
@@ -35,7 +35,6 @@ steps:
     cmake ${{ parameters.cmakeAdditionalFlags}} -G Ninja ^
     -DCMAKE_CXX_COMPILER=cl ^
     -DCMAKE_BUILD_TYPE=Release ^
-    -DSTL_BUILD_BENCHMARKING=${{ parameters.buildBenchmarking }} ^
     -DLIT_FLAGS=$(litFlags) ^
     -DSTL_USE_ANALYZE=ON ^
     -S $(Build.SourcesDirectory) -B $(${{ parameters.buildOutputLocationVar }})
@@ -49,3 +48,28 @@ steps:
   displayName: 'Build the STL'
   timeoutInMinutes: 10
   env: { TMP: $(tmpDir), TEMP: $(tmpDir) }
+- script: |
+    if exist "$(${{ parameters.benchmarkBuildOutputLocationVar }})" (
+      rmdir /S /Q "$(${{ parameters.benchmarkBuildOutputLocationVar }})"
+    )
+    call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\Common7\Tools\VsDevCmd.bat" ^
+    -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo
+    cmake ${{ parameters.cmakeAdditionalFlags}} -G Ninja ^
+    -DCMAKE_CXX_COMPILER=cl ^
+    -DCMAKE_BUILD_TYPE=Release ^
+    -DLIT_FLAGS=$(litFlags) ^
+    -DSTL_USE_ANALYZE=ON ^
+    -DSTL_BINARY_DIR=$(${{ parameters.buildOutputLocationVar }}) ^
+    -S $(Build.SourcesDirectory)/benchmarks -B $(${{ parameters.benchmarkBuildOutputLocationVar }})
+  displayName: 'Configure the benchmarks'
+  timeoutInMinutes: 2
+  env: { TMP: $(tmpDir), TEMP: $(tmpDir) }
+  condition: ne('${{ parameters.benchmarkBuildOutputLocationVar }}', '')
+- script: |
+    call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\Common7\Tools\VsDevCmd.bat" ^
+    -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo
+    cmake --build $(${{ parameters.benchmarkBuildOutputLocationVar }})
+  displayName: 'Build the benchmarks'
+  timeoutInMinutes: 10
+  env: { TMP: $(tmpDir), TEMP: $(tmpDir) }
+  condition: ne('${{ parameters.benchmarkBuildOutputLocationVar }}', '')

azure-devops/cross-build.yml

@@ -12,12 +12,12 @@ parameters:
 - name: buildOutputLocationVar
   type: string
   default: buildOutputLocation
+- name: benchmarkBuildOutputLocationVar
+  type: string
+  default: benchmarkBuildOutputLocation
 - name: numShards
   type: number
   default: 8
-- name: buildBenchmarking
-  type: string
-  default: 'ON'
 jobs:
 - job: '${{ parameters.targetPlatform }}'
   variables:
@@ -41,8 +41,8 @@ jobs:
       targetPlatform: ${{ parameters.targetPlatform }}
       hostArch: ${{ parameters.hostArch }}
       targetArch: ${{ parameters.vsDevCmdArch }}
-      buildBenchmarking: ${{ parameters.buildBenchmarking }}
       cmakeAdditionalFlags: '-DTESTS_BUILD_ONLY=ON'
+      benchmarkBuildOutputLocationVar: ${{ parameters.benchmarkBuildOutputLocationVar }}
   - template: run-tests.yml
     parameters:
       hostArch: ${{ parameters.hostArch }}

azure-devops/native-build-test.yml

@@ -9,6 +9,9 @@ parameters:
 - name: buildOutputLocationVar
   type: string
   default: buildOutputLocation
+- name: benchmarkBuildOutputLocationVar
+  type: string
+  default: benchmarkBuildOutputLocation
 - name: numShards
   type: number
   default: 8

azure-pipelines.yml

@@ -6,6 +6,7 @@
 variables:
   tmpDir: 'D:\Temp'
   buildOutputLocation: 'D:\build'
+  benchmarkBuildOutputLocation: 'D:\benchmark'
 
 pool:
   name: 'StlBuild-2022-10-11T1916-Pool'
@@ -91,7 +92,7 @@ stages:
         parameters:
           targetPlatform: arm
           vsDevCmdArch: arm
-          buildBenchmarking: 'OFF'
+          benchmarkBuildOutputLocationVar: ''
 
   - stage: Build_ARM64
     dependsOn: Build_And_Test_x64

benchmarks/CMakeLists.txt

@@ -1,6 +1,42 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+cmake_minimum_required(VERSION 3.24)
+project(msvc_standard_libraries_benchmarks LANGUAGES CXX)
+
+if(DEFINED STL_BINARY_DIR)
+    cmake_path(ABSOLUTE_PATH STL_BINARY_DIR
+        BASE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/.."
+        NORMALIZE
+    )
+    if(NOT EXISTS "${STL_BINARY_DIR}/out")
+        message(FATAL_ERROR "Invalid STL_BINARY_DIR '${STL_BINARY_DIR}'")
+    endif()
+
+    if(NOT DEFINED VCLIBS_TARGET_ARCHITECTURE)
+        set(VCLIBS_TARGET_ARCHITECTURE "${CMAKE_CXX_COMPILER_ARCHITECTURE_ID}")
+    endif()
+
+    string(TOLOWER "${VCLIBS_TARGET_ARCHITECTURE}" VCLIBS_TARGET_ARCHITECTURE)
+
+    if("${VCLIBS_TARGET_ARCHITECTURE}" MATCHES "^x86$")
+        set(VCLIBS_I386_OR_AMD64 "i386")
+    elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^x64$")
+        set(VCLIBS_I386_OR_AMD64 "amd64")
+    elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^armv7$")
+        set(VCLIBS_I386_OR_AMD64 "arm")
+    elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^arm64$")
+        set(VCLIBS_I386_OR_AMD64 "arm64")
+    else()
+        message(FATAL_ERROR "Could not determine target architecture: VCLIBS_TARGET_ARCHITECTURE: ${VCLIBS_TARGET_ARCHITECTURE}")
+    endif()
+
+    include_directories(BEFORE "${STL_BINARY_DIR}/out/inc")
+    link_directories(BEFORE "${STL_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
+else()
+    message(WARNING "STL_BINARY_DIR not set; benchmarking the globally installed standard library")
+endif()
+
 set(STL_BENCHMARK_MSVC_RUNTIME_LIBRARY
     MultiThreaded
     CACHE STRING "The flavor of the standard library to use; see https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html for more information.")
@@ -32,13 +68,9 @@ endif()
 set(BENCHMARK_ENABLE_DOXYGEN OFF)
 set(BENCHMARK_ENABLE_INSTALL OFF)
 set(BENCHMARK_ENABLE_TESTING OFF)
-set(BUILD_SHARED_LIBS OFF)
-
-# TRANSITION, GH-2816: on some machines, librt is found, despite it being a unix-only library
-set(HAVE_LIB_RT OFF)
 
-include_directories(BEFORE "${CMAKE_BINARY_DIR}/out/inc")
-link_directories(BEFORE "${STL_LIBRARY_OUTPUT_DIRECTORY}")
+set(HAVE_GNU_POSIX_REGEX OFF)
+set(HAVE_POSIX_REGEX OFF)
 
 add_subdirectory(google-benchmark EXCLUDE_FROM_ALL)
 
@@ -69,6 +101,8 @@ function(add_benchmark name)
     target_compile_features(benchmark-${name} PRIVATE cxx_std_${arg_CXX_STANDARD})
     target_include_directories(benchmark-${name} PRIVATE inc)
     target_link_libraries(benchmark-${name} PRIVATE benchmark::benchmark)
+    # TRANSITION, google/benchmark#1450
+    target_compile_definitions(benchmark-${name} PRIVATE BENCHMARK_STATIC_DEFINE)
 endfunction()
 
 add_benchmark(std_copy src/std_copy.cpp)

benchmarks/google-benchmark

@@ -87,38 +87,38 @@ namespace {
     }
 } // namespace
 
-BENCHMARK_TEMPLATE1(handwritten_loop, char)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(handwritten_loop_n, char)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(memcpy_call, char)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(std_copy_call, char)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(std_copy_n_call, char)->Range(0, 1 << 18);
-
-BENCHMARK_TEMPLATE1(handwritten_loop, aggregate<char>)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(handwritten_loop_n, aggregate<char>)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(memcpy_call, aggregate<char>)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(std_copy_call, aggregate<char>)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(std_copy_n_call, aggregate<char>)->Range(0, 1 << 18);
-
-BENCHMARK_TEMPLATE1(handwritten_loop, non_trivial<char>)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(handwritten_loop_n, non_trivial<char>)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(std_copy_call, non_trivial<char>)->Range(0, 1 << 18);
-BENCHMARK_TEMPLATE1(std_copy_n_call, non_trivial<char>)->Range(0, 1 << 18);
-
-BENCHMARK_TEMPLATE1(handwritten_loop, int)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(handwritten_loop_n, int)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(memcpy_call, int)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(std_copy_call, int)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(std_copy_n_call, int)->Range(0, 1 << 15);
-
-BENCHMARK_TEMPLATE1(handwritten_loop, aggregate<int>)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(handwritten_loop_n, aggregate<int>)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(memcpy_call, aggregate<int>)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(std_copy_call, aggregate<int>)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(std_copy_n_call, aggregate<int>)->Range(0, 1 << 15);
-
-BENCHMARK_TEMPLATE1(handwritten_loop, non_trivial<int>)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(handwritten_loop_n, non_trivial<int>)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(std_copy_call, non_trivial<int>)->Range(0, 1 << 15);
-BENCHMARK_TEMPLATE1(std_copy_n_call, non_trivial<int>)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(handwritten_loop, char)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(handwritten_loop_n, char)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(memcpy_call, char)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(std_copy_call, char)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(std_copy_n_call, char)->Range(0, 1 << 18);
+
+BENCHMARK_TEMPLATE(handwritten_loop, aggregate<char>)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(handwritten_loop_n, aggregate<char>)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(memcpy_call, aggregate<char>)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(std_copy_call, aggregate<char>)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(std_copy_n_call, aggregate<char>)->Range(0, 1 << 18);
+
+BENCHMARK_TEMPLATE(handwritten_loop, non_trivial<char>)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(handwritten_loop_n, non_trivial<char>)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(std_copy_call, non_trivial<char>)->Range(0, 1 << 18);
+BENCHMARK_TEMPLATE(std_copy_n_call, non_trivial<char>)->Range(0, 1 << 18);
+
+BENCHMARK_TEMPLATE(handwritten_loop, int)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(handwritten_loop_n, int)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(memcpy_call, int)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(std_copy_call, int)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(std_copy_n_call, int)->Range(0, 1 << 15);
+
+BENCHMARK_TEMPLATE(handwritten_loop, aggregate<int>)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(handwritten_loop_n, aggregate<int>)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(memcpy_call, aggregate<int>)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(std_copy_call, aggregate<int>)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(std_copy_n_call, aggregate<int>)->Range(0, 1 << 15);
+
+BENCHMARK_TEMPLATE(handwritten_loop, non_trivial<int>)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(handwritten_loop_n, non_trivial<int>)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(std_copy_call, non_trivial<int>)->Range(0, 1 << 15);
+BENCHMARK_TEMPLATE(std_copy_n_call, non_trivial<int>)->Range(0, 1 << 15);
 
 BENCHMARK_MAIN();