add some reduction methods to the options on the fuzz tests (#930)

This adds a round trip test for config file generation to the fuzzer. 

(the next step after this PR will be a fuzzer that verifies that the
round trip actually matches the results.
This change ended up requiring quite a few minor changes to fix the
ambiguities between the config file generation and config file reader.

1). There was a number of potential conflicts between positional names
and regular option names that could be triggered in config files, this
required a number of additional checks on the positional naming to
ensure no conflicts.
2). flag options with disable flag override can produce output results
that are not valid by themselves, resolving this required flag input to
be able to handle an array and output the original value set of results.
3). strings with non-printable characters could cause all sorts of chaos
in the config files. This was resolved by generating a binary string
conversion format and handling multiline comments and characters, and
handling escaped characters. Note; I think a better solution is to move
to fully supporting string formatting and escaping along with the binary
strings from TOML now that TOML 1.0 is finalized. That will not be this
PR though, maybe the next one.
4). Lot of ambiguities and edge cases in the string splitter, this was
reworked
5). handling of comments was not done well, especially comment characters in the
name of the option which is allowed.
6). non printable characters in the option naming. This would be weird
in practice but it also cause some big holes in the config file
generation, so the restricted character set for option naming was
expanded. (don't allow spaces or control characters).

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: phlptp

.codacy.yml

@@ -0,0 +1,18 @@
+---
+engines:
+  rubocop:
+    enabled: true
+  duplication:
+    enabled: true
+  metrics:
+    enabled: true
+  coverage:
+    enabled: false
+languages:
+
+exclude_paths:
+  - "fuzz/**/*"
+  - "fuzz/*"
+  - "scripts/**/*"
+  - "scripts/*"
+  - "**.md"

CPPLINT.cfg

@@ -9,6 +9,7 @@ filter=-readability/nolint  # Conflicts with clang-tidy
 filter=-readability/check  # Catch uses CHECK(a == b) (Tests only)
 filter=-build/namespaces  # Currently using it for one test (Tests only)
 filter=-runtime/references  # Requires fundamental change of API, don't see need for this
+filter=-runtime/string  # Requires not using static const strings which makes thing really annoying
 filter=-whitespace/blank_line  # Unnecessarily strict with blank lines that otherwise help with readability
 filter=-whitespace/indent  # Requires strange 3-space indent of private/protected/public markers
 filter=-whitespace/parens,-whitespace/braces  # Conflict with clang-format

book/chapters/config.md

@@ -206,6 +206,16 @@ str3 = """\
 The key is that the closing of the multiline string must be at the end of a line
 and match the starting 3 quote sequence.
 
+### Binary Strings
+
+Config files have a binary conversion capability, this is mainly to support
+writing config files but can be used by user generated files as well. Strings
+with the form `B"(XXXXX)"` will convert any characters inside the parenthesis
+with the form \xHH to the equivalent binary value. The HH are hexadecimal
+characters. Characters not in this form will be translated as given. If argument
+values with unprintable characters are used to generate a config file this
+binary form will be used in the output string.
+
 ## Multiple configuration files
 
 If it is desired that multiple configuration be allowed. Use

fuzz/cli11_app_fuzz.cpp

@@ -40,10 +40,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
 
     try {
         app->parse(parseString);
+
     } catch(const CLI::ParseError &e) {
         //(app)->exit(e);
         // this just indicates we caught an error known by CLI
+        return 0;  // Non-zero return values are reserved for future use.
     }
-
-    return 0;  // Non-zero return values are reserved for future use.
+    // should be able to write the config to a file and read from it again
+    std::string configOut = app->config_to_str();
+    app->clear();
+    std::stringstream out(configOut);
+    app->parse_from_stream(out);
+    return 0;
 }

fuzz/cli11_file_fuzz.cpp

@@ -22,6 +22,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     auto app = fuzzdata.generateApp();
     try {
         app->parse_from_stream(out);
+        // should be able to write the config to a file and read from it again
+        std::string configOut = app->config_to_str();
+
+        app->clear();
+        std::stringstream out(configOut);
+        app->parse_from_stream(out);
+
     } catch(const CLI::ParseError &e) {
         // (app)->exit(e);
         // this just indicates we caught an error known by CLI

fuzz/fuzzApp.cpp

@@ -68,9 +68,9 @@ std::shared_ptr<CLI::App> FuzzApp::generateApp() {
     auto *vgroup = fApp->add_option_group("vectors");
 
     vgroup->add_option("--vopt1", vv1);
-    vgroup->add_option("--vopt2", vvs);
+    vgroup->add_option("--vopt2", vvs)->inject_separator();
     vgroup->add_option("--vopt3", vstr);
-    vgroup->add_option("--vopt4", vecvecd);
+    vgroup->add_option("--vopt4", vecvecd)->inject_separator();
 
     fApp->add_option("--oopt1", od1);
     fApp->add_option("--oopt2", ods);
@@ -121,6 +121,30 @@ std::shared_ptr<CLI::App> FuzzApp::generateApp() {
     sub->add_option("--sdwrap", dwrap);
     sub->add_option("--siwrap", iwrap);
 
+    auto *resgroup = fApp->add_option_group("outputOrder");
+
+    resgroup->add_option("--vA", vstrA)->expected(0, 2)->multi_option_policy(CLI::MultiOptionPolicy::TakeAll);
+    resgroup->add_option("--vB", vstrB)->expected(0, 2)->multi_option_policy(CLI::MultiOptionPolicy::TakeLast);
+    resgroup->add_option("--vC", vstrC)->expected(0, 2)->multi_option_policy(CLI::MultiOptionPolicy::TakeFirst);
+    resgroup->add_option("--vD", vstrD)->expected(0, 2)->multi_option_policy(CLI::MultiOptionPolicy::Reverse);
+    resgroup->add_option("--vS", val32)->expected(0, 2)->multi_option_policy(CLI::MultiOptionPolicy::Sum);
+    resgroup->add_option("--vM", mergeBuffer)->expected(0, 2)->multi_option_policy(CLI::MultiOptionPolicy::Join);
+    resgroup->add_option("--vE", vstrE)->expected(2, 4)->delimiter(',');
+
+    auto *vldtr = fApp->add_option_group("validators");
+
+    validator_strings.resize(10);
+    vldtr->add_option("--vdtr1", validator_strings[0])->join()->check(CLI::PositiveNumber);
+    vldtr->add_option("--vdtr2", validator_strings[1])->join()->check(CLI::NonNegativeNumber);
+    vldtr->add_option("--vdtr3", validator_strings[2])->join()->check(CLI::NonexistentPath);
+    vldtr->add_option("--vdtr4", validator_strings[3])->join()->check(CLI::Range(7, 3456));
+    vldtr->add_option("--vdtr5", validator_strings[4])
+        ->join()
+        ->check(CLI::Range(std::string("aa"), std::string("zz"), "string range"));
+    vldtr->add_option("--vdtr6", validator_strings[5])->join()->check(CLI::TypeValidator<double>());
+    vldtr->add_option("--vdtr7", validator_strings[6])->join()->check(CLI::TypeValidator<bool>());
+    vldtr->add_option("--vdtr8", validator_strings[7])->join()->check(CLI::ValidIPV4);
+    vldtr->add_option("--vdtr9", validator_strings[8])->join()->transform(CLI::Bound(2, 255));
     return fApp;
 }
 

fuzz/fuzzApp.hpp

@@ -75,6 +75,7 @@ class FuzzApp {
 
     std::vector<double> vv1{};
     std::vector<std::string> vstr{};
+
     std::vector<std::vector<double>> vecvecd{};
     std::vector<std::vector<std::string>> vvs{};
     std::optional<double> od1{};
@@ -103,5 +104,15 @@ class FuzzApp {
     std::string buffer{};
     int intbuffer{0};
     std::atomic<double> doubleAtomic{0.0};
+
+    // for testing restrictions and reduction methods
+    std::vector<std::string> vstrA{};
+    std::vector<std::string> vstrB{};
+    std::vector<std::string> vstrC{};
+    std::vector<std::string> vstrD{};
+    std::vector<std::string> vstrE{};
+    std::vector<std::string> vstrF{};
+    std::string mergeBuffer{};
+    std::vector<std::string> validator_strings{};
 };
 }  // namespace CLI

fuzz/fuzzCommand.cpp

@@ -20,5 +20,6 @@ int main(int argc, char **argv) {
         (app)->exit(e);
         // this just indicates we caught an error known by CLI
     }
+
     return 0;
 }

fuzz/fuzz_dictionary1.txt

@@ -66,6 +66,14 @@
 "--siwrap"
 "--svtup"
 "--satd"
+"--vA"
+"--vB"
+"--vC"
+"--vD"
+"--vS"
+"--vM"
+"--vE"
+"--vdtr"
 "nflag2"
 "stup1"
 "svtup"
@@ -143,6 +151,7 @@
 "stup4"
 "sdwrap"
 "siwrap"
+"vdtr"
 "svtup"
 "satd"
 "%%"
@@ -156,3 +165,10 @@
 "!"
 "{"
 "}"
+"vA"
+"vB"
+"vC"
+"vD"
+"vS"
+"vM"
+"vE"

fuzz/fuzz_dictionary2.txt

@@ -72,6 +72,7 @@
 "svopt4"
 "config"
 "nflag2"
+"vdtr"
 "--"
 "fuzzer"
 "t-"
@@ -82,3 +83,10 @@
 "dexists"
 "fexists"
 "fnexists"
+"vA"
+"vB"
+"vC"
+"vD"
+"vS"
+"vM"
+"vE"