CLI11 Fuzzing

.codecov.yml

@@ -4,3 +4,4 @@ ignore:
   - "book"
   - "docs"
   - "test_package"
+  - "fuzz"

.github/codecov.yml

@@ -1,6 +1,6 @@
 codecov:
   notify:
-    after_n_builds: 4
+    after_n_builds: 8
 coverage:
   status:
     project:

.github/workflows/fuzz.yml

@@ -0,0 +1,54 @@
+name: Fuzz
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - v*
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  quick_fuzz1:
+    name: quickfuzz1
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure
+        run: |
+          cmake -S . -B build \
+            -DCMAKE_CXX_STANDARD=17 \
+            -DCLI11_SINGLE_FILE_TESTS=OFF \
+            -DCLI11_BUILD_EXAMPLES=OFF \
+            -DCLI11_FUZZ_TARGET=ON \
+            -DCLI11_BUILD_TESTS=OFF \
+            -DCLI11_BUILD_DOCS=OFF \
+            -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_COMPILER_FORCED=ON \
+            -DCMAKE_CXX_FLAGS="-g -O1 -fsanitize=fuzzer,undefined,address"
+
+      - name: Build
+        run: cmake --build build -j4
+
+      - name: Test
+        run: |
+          cd build
+          make QUICK_CLI11_APP_FUZZ
+
+      - name: Test2
+        run: |
+          cd build
+          make QUICK_CLI11_FILE_FUZZ
+
+
+      - name: artifacts
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: file_failure
+          path: ./build/fuzz/cli11_*_fail_artifact.txt

.github/workflows/tests.yml

@@ -48,7 +48,7 @@ jobs:
       - name: Prepare coverage
         run: |
           lcov --directory . --capture --output-file coverage.info
-          lcov --remove coverage.info '*/tests/*' '*/examples/*' '/usr/*' '*/book/*' --output-file coverage.info
+          lcov --remove coverage.info '*/tests/*' '*/examples/*' '/usr/*' '*/book/*' '*/fuzz/*' --output-file coverage.info
           lcov --list coverage.info
         working-directory: build
 

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+exclude: ^(.github/workflows/|docs/img/)
 ci:
   autoupdate_commit_msg: "chore(deps): pre-commit.ci autoupdate"
   autofix_commit_msg: "style: pre-commit.ci fixes"

CMakeLists.txt

@@ -132,6 +132,9 @@ endif()
 
 include(CLI11Warnings)
 
+# build the fuzzing example or fuzz entry point
+add_subdirectory(fuzz)
+
 add_subdirectory(src)
 
 # Allow tests to be run on CUDA

azure-pipelines.yml

@@ -28,7 +28,6 @@ jobs:
       - bash: cpplint --counting=detailed --recursive examples include/CLI tests
         displayName: Checking against google style guide
 
-  # TODO: Fix macOS error and windows warning in c++17 mode
   - job: Native
     strategy:
       matrix:
@@ -38,13 +37,13 @@ jobs:
           vmImage: "ubuntu-latest"
           cli11.precompile: ON
         macOS17:
-          vmImage: "macOS-latest"
+          vmImage: "macOS-12"
           cli11.std: 17
         macOS11:
-          vmImage: "macOS-latest"
+          vmImage: "macOS-11"
           cli11.std: 11
         macOS11PC:
-          vmImage: "macOS-latest"
+          vmImage: "macOS-11"
           cli11.std: 11
           cli11.precompile: ON
         Windows17:

fuzz/CMakeLists.txt

@@ -0,0 +1,48 @@
+# Copyright (c) 2017-2023, University of Cincinnati, developed by Henry Schreiner
+# under NSF AWARD 1414736 and by the respective contributors.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+if(CMAKE_CXX_STANDARD GREATER 16)
+  if(CLI11_FUZZ_TARGET)
+
+    add_executable(cli11_app_fuzzer cli11_app_fuzz.cpp fuzzApp.cpp fuzzApp.hpp)
+    target_link_libraries(cli11_app_fuzzer PUBLIC CLI11)
+    set_property(TARGET cli11_app_fuzzer PROPERTY FOLDER "Tests")
+
+    add_executable(cli11_file_fuzzer cli11_file_fuzz.cpp fuzzApp.cpp fuzzApp.hpp)
+    target_link_libraries(cli11_file_fuzzer PUBLIC CLI11)
+    set_property(TARGET cli11_file_fuzzer PROPERTY FOLDER "Tests")
+
+    if(NOT CLI11_FUZZ_ARTIFACT_PATH)
+      set(CLI11_FUZZ_ARTIFACT_PATH ${PROJECT_BINARY_DIR}/fuzz)
+    endif()
+
+    if(NOT CLI11_FUZZ_TIME)
+      set(CLI11_FUZZ_TIME 360)
+    endif()
+    add_custom_target(
+      QUICK_CLI11_APP_FUZZ
+      COMMAND ${CMAKE_COMMAND} -E make_directory corp
+      COMMAND
+        cli11_app_fuzzer corp -max_total_time=${CLI11_FUZZ_TIME} -max_len=2048
+        -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary1.txt
+        -exact_artifact_path=${CLI11_FUZZ_ARTIFACT_PATH}/cli11_app_fail_artifact.txt)
+
+    add_custom_target(
+      QUICK_CLI11_FILE_FUZZ
+      COMMAND ${CMAKE_COMMAND} -E make_directory corp
+      COMMAND
+        cli11_file_fuzzer corp -max_total_time=${CLI11_FUZZ_TIME} -max_len=2048
+        -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary2.txt
+        -exact_artifact_path=${CLI11_FUZZ_ARTIFACT_PATH}/cli11_file_fail_artifact.txt)
+
+  else()
+    if(CLI11_BUILD_EXAMPLES)
+      add_executable(cli11Fuzz fuzzCommand.cpp fuzzApp.cpp fuzzApp.hpp)
+      target_link_libraries(cli11Fuzz PUBLIC CLI11)
+      set_property(TARGET cli11Fuzz PROPERTY FOLDER "Examples")
+    endif()
+  endif()
+endif()

fuzz/cli11_app_fuzz.cpp

@@ -0,0 +1,30 @@
+// Copyright (c) 2017-2023, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include "fuzzApp.hpp"
+#include <CLI/CLI.hpp>
+#include <cstring>
+#include <exception>
+#include <string>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+    if(Size == 0) {
+        return 0;
+    }
+    std::string parseString(reinterpret_cast<const char *>(Data), Size);
+
+    CLI::FuzzApp fuzzdata;
+
+    auto app = fuzzdata.generateApp();
+    try {
+        app->parse(parseString);
+    } catch(const CLI::ParseError &e) {
+        //(app)->exit(e);
+        // this just indicates we caught an error known by CLI
+    }
+
+    return 0;  // Non-zero return values are reserved for future use.
+}

fuzz/cli11_file_fuzz.cpp

@@ -0,0 +1,31 @@
+// Copyright (c) 2017-2023, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include "fuzzApp.hpp"
+#include <CLI/CLI.hpp>
+#include <cstring>
+#include <exception>
+#include <sstream>
+#include <string>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+    if(Size == 0) {
+        return 0;
+    }
+    std::string parseString(reinterpret_cast<const char *>(Data), Size);
+    std::stringstream out(parseString);
+    CLI::FuzzApp fuzzdata;
+
+    auto app = fuzzdata.generateApp();
+    try {
+        app->parse_from_stream(out);
+    } catch(const CLI::ParseError &e) {
+        (app)->exit(e);
+        // this just indicates we caught an error known by CLI
+    }
+
+    return 0;  // Non-zero return values are reserved for future use.
+}

fuzz/fuzzApp.cpp

@@ -0,0 +1,85 @@
+// Copyright (c) 2017-2023, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include "fuzzApp.hpp"
+
+namespace CLI {
+/*
+int32_t val32{0};
+    int16_t val16{0};
+    int8_t val8{0};
+    int64_t val64{0};
+
+    uint32_t uval32{0};
+    uint16_t uval16{0};
+    uint8_t uval8{0};
+    uint64_t uval64{0};
+
+    std::atomic<int64_t> atomicval64{0};
+    std::atomic<uint64_t> atomicuval64{0};
+
+    double v1{0};
+    float v2{0};
+
+    std::vector<double> vv1;
+    std::vector<std::string> vstr;
+    std::vector<std::vector<double>> vecvecd;
+    std::vector<std::vector<std::string>> vvs;
+    std::optional<double> od1;
+    std::optional<std::string> ods;
+    std::pair<double, std::string> p1;
+    std::pair<std::vector<double>, std::string> p2;
+    std::tuple<int64_t, uint16_t, std::optional<double>> t1;
+    std::tuple<std::tuple<std::tuple<std::string, double, std::vector<int>>,std::string, double>,std::vector<int>,
+std::optional<std::string>> tcomplex; std::string_view vstrv;
+
+    bool flag1{false};
+    int flagCnt{0};
+    std::atomic<bool> flagAtomic{false};
+    */
+std::shared_ptr<CLI::App> FuzzApp::generateApp() {
+    auto fApp = std::make_shared<CLI::App>("fuzzing App", "fuzzer");
+    fApp->set_config("--config");
+    fApp->add_flag("-a,--flag");
+    fApp->add_flag("-b,--flag2", flag1);
+    fApp->add_flag("-c{34},--flag3{1}", flagCnt)->disable_flag_override();
+    fApp->add_flag("-e,--flagA", flagAtomic);
+
+    fApp->add_option("-d,--opt1", val8);
+    fApp->add_option("--opt2", val16);
+    fApp->add_option("--opt3", val32);
+    fApp->add_option("--opt4", val64);
+
+    fApp->add_option("--opt5", uval8);
+    fApp->add_option("--opt6", uval16);
+    fApp->add_option("--opt7", uval32);
+    fApp->add_option("--opt8", uval64);
+
+    fApp->add_option("--aopt1", atomicval64);
+    fApp->add_option("--aopt2", atomicuval64);
+
+    fApp->add_option("--dopt1", v1);
+    fApp->add_option("--dopt2", v2);
+
+    fApp->add_option("--vopt1", vv1);
+    fApp->add_option("--vopt2", vvs);
+    fApp->add_option("--vopt3", vstr);
+    fApp->add_option("--vopt4", vecvecd);
+
+    fApp->add_option("--oopt1", od1);
+    fApp->add_option("--oopt2", ods);
+
+    fApp->add_option("--tup1", p1);
+    fApp->add_option("--tup2", t1);
+    fApp->add_option("--tup4", tcomplex);
+
+    fApp->add_option("--dwrap", dwrap);
+    fApp->add_option("--iwrap", iwrap);
+
+    return fApp;
+}
+
+}  // namespace CLI