Add the beginnings of a fuzzing system for CLI11. This commit adds the fuzzing code, a simple test, and two fixes to issues(seg faults) found by the initial round of fuzzing. It also adds a few tests and coverage issues uncovered in the process of developing the fuzz tests. As a side effect adjusts some of the azure tests to specify the vmImage which was being changed on azure.

Author: pre-commit-ci[bot]

.codecov.yml

@@ -4,3 +4,4 @@ ignore:
   - "book"
   - "docs"
   - "test_package"
+  - "fuzz"

.github/codecov.yml

@@ -1,6 +1,6 @@
 codecov:
   notify:
-    after_n_builds: 4
+    after_n_builds: 8
 coverage:
   status:
     project:

.github/workflows/fuzz.yml

@@ -0,0 +1,54 @@
+name: Fuzz
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - v*
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  quick_fuzz1:
+    name: quickfuzz1
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure
+        run: |
+          cmake -S . -B build \
+            -DCMAKE_CXX_STANDARD=17 \
+            -DCLI11_SINGLE_FILE_TESTS=OFF \
+            -DCLI11_BUILD_EXAMPLES=OFF \
+            -DCLI11_FUZZ_TARGET=ON \
+            -DCLI11_BUILD_TESTS=OFF \
+            -DCLI11_BUILD_DOCS=OFF \
+            -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_COMPILER_FORCED=ON \
+            -DCMAKE_CXX_FLAGS="-g -O1 -fsanitize=fuzzer,undefined,address"
+
+      - name: Build
+        run: cmake --build build -j4
+
+      - name: Test
+        run: |
+          cd build
+          make QUICK_CLI11_APP_FUZZ
+
+      - name: Test2
+        run: |
+          cd build
+          make QUICK_CLI11_FILE_FUZZ
+
+
+      - name: artifacts
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: file_failure
+          path: ./build/fuzz/cli11_*_fail_artifact.txt

.github/workflows/tests.yml

@@ -48,7 +48,7 @@ jobs:
       - name: Prepare coverage
         run: |
           lcov --directory . --capture --output-file coverage.info
-          lcov --remove coverage.info '*/tests/*' '*/examples/*' '/usr/*' '*/book/*' --output-file coverage.info
+          lcov --remove coverage.info '*/tests/*' '*/examples/*' '/usr/*' '*/book/*' '*/fuzz/*' --output-file coverage.info
           lcov --list coverage.info
         working-directory: build
 

.pre-commit-config.yaml

@@ -1,10 +1,11 @@
+exclude: ^(.github/workflows/|docs/img/)
 ci:
   autoupdate_commit_msg: "chore(deps): pre-commit.ci autoupdate"
   autofix_commit_msg: "style: pre-commit.ci fixes"
 
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.1.0
     hooks:
       - id: black
 

CMakeLists.txt

@@ -132,6 +132,9 @@ endif()
 
 include(CLI11Warnings)
 
+# build the fuzzing example or fuzz entry point
+add_subdirectory(fuzz)
+
 add_subdirectory(src)
 
 # Allow tests to be run on CUDA

azure-pipelines.yml

@@ -28,7 +28,6 @@ jobs:
       - bash: cpplint --counting=detailed --recursive examples include/CLI tests
         displayName: Checking against google style guide
 
-  # TODO: Fix macOS error and windows warning in c++17 mode
   - job: Native
     strategy:
       matrix:
@@ -38,13 +37,13 @@ jobs:
           vmImage: "ubuntu-latest"
           cli11.precompile: ON
         macOS17:
-          vmImage: "macOS-latest"
+          vmImage: "macOS-12"
           cli11.std: 17
         macOS11:
-          vmImage: "macOS-latest"
+          vmImage: "macOS-11"
           cli11.std: 11
         macOS11PC:
-          vmImage: "macOS-latest"
+          vmImage: "macOS-11"
           cli11.std: 11
           cli11.precompile: ON
         Windows17:

conanfile.py

@@ -28,6 +28,7 @@ class CLI11Conan(ConanFile):
         "README.md",
         "include/*",
         "src/*",
+        "fuzz/*",
         "extern/*",
         "cmake/*",
         "CMakeLists.txt",

fuzz/CMakeLists.txt

@@ -0,0 +1,42 @@
+if(CMAKE_CXX_STANDARD GREATER 16)
+  if(CLI11_FUZZ_TARGET)
+
+    add_executable(cli11_app_fuzzer cli11_app_fuzz.cpp fuzzApp.cpp fuzzApp.hpp)
+    target_link_libraries(cli11_app_fuzzer PUBLIC CLI11)
+    set_property(TARGET cli11_app_fuzzer PROPERTY FOLDER "Tests")
+
+    add_executable(cli11_file_fuzzer cli11_file_fuzz.cpp fuzzApp.cpp fuzzApp.hpp)
+    target_link_libraries(cli11_file_fuzzer PUBLIC CLI11)
+    set_property(TARGET cli11_file_fuzzer PROPERTY FOLDER "Tests")
+
+    if(NOT CLI11_FUZZ_ARTIFACT_PATH)
+      set(CLI11_FUZZ_ARTIFACT_PATH ${PROJECT_BINARY_DIR}/fuzz)
+    endif()
+
+    if(NOT CLI11_FUZZ_TIME)
+      set(CLI11_FUZZ_TIME 360)
+    endif()
+    add_custom_target(
+      QUICK_CLI11_APP_FUZZ
+      COMMAND ${CMAKE_COMMAND} -E make_directory corp
+      COMMAND
+        cli11_app_fuzzer corp -max_total_time=${CLI11_FUZZ_TIME} -max_len=2048
+        -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary1.txt
+        -exact_artifact_path=${CLI11_FUZZ_ARTIFACT_PATH}/cli11_app_fail_artifact.txt)
+
+    add_custom_target(
+      QUICK_CLI11_FILE_FUZZ
+      COMMAND ${CMAKE_COMMAND} -E make_directory corp
+      COMMAND
+        cli11_file_fuzzer corp -max_total_time=${CLI11_FUZZ_TIME} -max_len=2048
+        -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary2.txt
+        -exact_artifact_path=${CLI11_FUZZ_ARTIFACT_PATH}/cli11_file_fail_artifact.txt)
+
+  else()
+    if(CLI11_BUILD_EXAMPLES)
+      add_executable(cli11Fuzz fuzzCommand.cpp fuzzApp.cpp fuzzApp.hpp)
+      target_link_libraries(cli11Fuzz PUBLIC CLI11)
+      set_property(TARGET cli11Fuzz PROPERTY FOLDER "Examples")
+    endif()
+  endif()
+endif()

fuzz/cli11_app_fuzz.cpp

@@ -0,0 +1,31 @@
+/*
+Copyright (c) 2019-2022,
+Lawrence Livermore National Security, LLC;
+See the top-level NOTICE for additional details. All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+*/
+
+#include "fuzzApp.hpp"
+#include <CLI/CLI.hpp>
+#include <cstring>
+#include <exception>
+#include <string>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+    if(Size == 0) {
+        return 0;
+    }
+    std::string parseString(reinterpret_cast<const char *>(Data), Size);
+
+    CLI::FuzzApp fuzzdata;
+
+    auto app = fuzzdata.generateApp();
+    try {
+        app->parse(parseString);
+    } catch(const CLI::ParseError &e) {
+        //(app)->exit(e);
+        // this just indicates we caught an error known by CLI
+    }
+
+    return 0;  // Non-zero return values are reserved for future use.
+}