Improve logging of environment variables (#3543)

Author: ssbarnea

docs/changelog/3542.bugfix.rst

@@ -0,0 +1,2 @@
+Improves logging of environment variables by sorting them by key and redacting
+the values for the ones that are likely to contain secrets.

docs/user_guide.rst

@@ -441,6 +441,15 @@ CLI
   have a stale Python environment; e.g. ``tox run -e py310 -r`` would clean the run environment and recreate it from
   scratch.
 
+Logging
+~~~~~~~
+
+Tox logs its activity inside ``.tox/<env_name>/log`` which can prove to be a good source of information when debugging
+its behavior. It should be noted that some of the environment variables with names containing one of the words
+``access``, ``api``, ``auth``, ``client``, ``cred``, ``key``, ``passwd``, ``password``, ``private``, ``pwd``,
+``secret`` and ``token`` will be logged with their values redacted with ``*`` to prevent accidental secret leaking when
+tox is used in CI/CD environments (as log collection is common).
+
 Config files
 ~~~~~~~~~~~~
 

pyproject.toml

@@ -83,6 +83,7 @@ dev = [
 test = [
   "build[virtualenv]>=1.2.2.post1",
   "covdefaults>=2.3",
+  "coverage>=7.9.1",
   "detect-test-pollution>=1.2",
   "devpi-process>=1.0.2",
   "diff-cover>=9.2",

src/tox/tox_env/api.py

@@ -31,6 +31,30 @@
     from tox.tox_env.installer import Installer
 
 LOGGER = logging.getLogger(__name__)
+# Based on original gitleaks rule named generic-api-key
+# See: https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml#L587
+SECRET_KEYWORDS = [
+    "access",
+    "api",
+    "auth",
+    "client",
+    "cred",
+    "key",
+    "passwd",
+    "password",
+    "private",
+    "pwd",
+    "secret",
+    "token",
+]
+SECRET_ENV_VAR_REGEX = re.compile(".*(" + "|".join(SECRET_KEYWORDS) + ").*", re.IGNORECASE)
+
+
+def redact_value(name: str, value: str) -> str:
+    """Returns a redacted text if the key name looks like a secret."""
+    if SECRET_ENV_VAR_REGEX.match(name):
+        return "*" * len(value)
+    return value
 
 
 class ToxEnvCreateArgs(NamedTuple):
@@ -461,8 +485,11 @@ def _write_execute_log(env_name: str, log_file: Path, request: ExecuteRequest, s
         with log_file.open("wt", encoding="utf-8") as file:
             file.write(f"name: {env_name}\n")
             file.write(f"run_id: {request.run_id}\n")
-            for env_key, env_value in request.env.items():
-                file.write(f"env {env_key}: {env_value}\n")
+            msg = ""
+            for env_key, env_value in sorted(request.env.items()):
+                redacted_value = redact_value(name=env_key, value=env_value)
+                msg += f"env {env_key}: {redacted_value}\n"
+            file.write(msg)
             for meta_key, meta_value in status.metadata.items():
                 file.write(f"metadata {meta_key}: {meta_value}\n")
             file.write(f"cwd: {request.cwd}\n")

tests/tox_env/test_api.py

@@ -2,6 +2,10 @@
 
 from typing import TYPE_CHECKING
 
+import pytest
+
+from tox.tox_env.api import redact_value
+
 if TYPE_CHECKING:
     from pathlib import Path
 
@@ -32,3 +36,31 @@ def test_setenv_section_substitution(tox_project: ToxProjectCreator) -> None:
     project = tox_project({"tox.ini": ini})
     result = project.run()
     result.assert_success()
+
+
+@pytest.mark.parametrize(
+    ("key", "do_redact"),
+    [
+        pytest.param("SOME_KEY", True, id="key"),
+        pytest.param("API_FOO", True, id="api"),
+        pytest.param("AUTH", True, id="auth"),
+        pytest.param("CLIENT", True, id="client"),
+        pytest.param("DB_PASSWORD", True, id="password"),
+        pytest.param("FOO", False, id="foo"),
+        pytest.param("GITHUB_TOKEN", True, id="token"),
+        pytest.param("NORMAL_VAR", False, id="other"),
+        pytest.param("S_PASSWD", True, id="passwd"),
+        pytest.param("SECRET", True, id="secret"),
+        pytest.param("SOME_ACCESS", True, id="access"),
+        pytest.param("MY_CRED", True, id="cred"),
+        pytest.param("MY_PRIVATE", True, id="private"),
+        pytest.param("MY_PWD", True, id="pwd"),
+    ],
+)
+def test_redact(key: str, do_redact: bool) -> None:
+    """Ensures that redact_value works as expected."""
+    result = redact_value(key, "foo")
+    if do_redact:
+        assert result == "***"
+    else:
+        assert result == "foo"