tests: Move sanic under toxgen & add option to control how many releases to pick (#4767)

- Move sanic under toxgen
- This gets rid of the `-latest` group for Web 2 as all Web 2 test
suites are now governed by toxgen
- Add `num_versions` as a new option to optionally slim down the number
of tested versions (useful for sanic as it has a lot of "majors" due to
calver)
- When a package had multiple majors, we used to always pick the lowest
as well as the highest release in the last major -- removed the lowest
now to not over-test. All majors will still be tested by default, we
just won't be testing two releases in the last major.

Ref #4506

Author: sentrivana

.github/workflows/test-integrations-web-2.yml

@@ -22,101 +22,6 @@ env:
   CACHED_BUILD_PATHS: |
     ${{ github.workspace }}/dist-serverless
 jobs:
-  test-web_2-latest:
-    name: Web 2 (latest)
-    timeout-minutes: 30
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.9","3.12","3.13"]
-        # python3.6 reached EOL and is no longer being supported on
-        # new versions of hosted runners on Github Actions
-        # ubuntu-20.04 is the last version that supported python3.6
-        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
-        os: [ubuntu-22.04]
-    # Use Docker container only for Python 3.6
-    container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }}
-    steps:
-      - uses: actions/[email protected]
-      - uses: actions/setup-python@v6
-        if: ${{ matrix.python-version != '3.6' }}
-        with:
-          python-version: ${{ matrix.python-version }}
-          allow-prereleases: true
-      - name: Setup Test Env
-        run: |
-          pip install "coverage[toml]" tox
-      - name: Erase coverage
-        run: |
-          coverage erase
-      - name: Test aiohttp latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-aiohttp-latest"
-      - name: Test asgi latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-asgi-latest"
-      - name: Test bottle latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-bottle-latest"
-      - name: Test falcon latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-falcon-latest"
-      - name: Test litestar latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-litestar-latest"
-      - name: Test pyramid latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-pyramid-latest"
-      - name: Test quart latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-quart-latest"
-      - name: Test sanic latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-sanic-latest"
-      - name: Test starlite latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-starlite-latest"
-      - name: Test tornado latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-tornado-latest"
-      - name: Generate coverage XML (Python 3.6)
-        if: ${{ !cancelled() && matrix.python-version == '3.6' }}
-        run: |
-          export COVERAGE_RCFILE=.coveragerc36
-          coverage combine .coverage-sentry-*
-          coverage xml --ignore-errors
-      - name: Generate coverage XML
-        if: ${{ !cancelled() && matrix.python-version != '3.6' }}
-        run: |
-          coverage combine .coverage-sentry-*
-          coverage xml
-      - name: Upload coverage to Codecov
-        if: ${{ !cancelled() }}
-        uses: codecov/[email protected]
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: coverage.xml
-          # make sure no plugins alter our coverage reports
-          plugins: noop
-          verbose: true
-      - name: Upload test results to Codecov
-        if: ${{ !cancelled() }}
-        uses: codecov/test-results-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: .junitxml
-          verbose: true
   test-web_2-pinned:
     name: Web 2 (pinned)
     timeout-minutes: 30

scripts/populate_tox/README.md

@@ -18,8 +18,7 @@ then determining which versions make sense to test to get good coverage.
 
 The lowest supported and latest version of a framework are always tested, with
 a number of releases in between:
-- If the package has majors, we pick the highest version of each major. For the
-  latest major, we also pick the lowest version in that major.
+- If the package has majors, we pick the highest version of each major.
 - If the package doesn't have multiple majors, we pick two versions in between
   lowest and highest.
 
@@ -46,6 +45,8 @@ integration_name: {
      },
      "python": python_version_specifier,
      "include": package_version_specifier,
+     "integration_name": integration_name,
+     "num_versions": int,
 }
 ```
 
@@ -161,6 +162,10 @@ of which are actually testing the `openai` integration. If this is the case, you
 
 Linking an integration to a test suite allows the script to access integration configuration like for example the minimum version defined in `sentry_sdk/integrations/__init__.py`.
 
+### `num_versions`
+
+With this option you can tweak the default version picking behavior by specifying how many package versions should be tested. It accepts an integer equal to or greater than 2, as the oldest and latest supported versions will always be picked. Additionally, if there is a recent prerelease, it'll also always be picked (this doesn't count towards `num_versions`).
+
 
 ## How-Tos
 

scripts/populate_tox/config.py

@@ -259,6 +259,16 @@
     "requests": {
         "package": "requests",
     },
+    "sanic": {
+        "package": "sanic",
+        "deps": {
+            "*": ["websockets<11.0", "aiohttp"],
+            ">=22": ["sanic-testing"],
+            "py3.6": ["aiocontextvars==0.2.1"],
+            "py3.8": ["tracerite<1.1.2"],
+        },
+        "num_versions": 4,
+    },
     "spark": {
         "package": "pyspark",
         "python": ">=3.8",

scripts/populate_tox/populate_tox.py

@@ -74,7 +74,6 @@
     "redis",
     "requests",
     "rq",
-    "sanic",
 }
 
 
@@ -258,47 +257,48 @@ def _supports_lowest(release: Version) -> bool:
 
 
 def pick_releases_to_test(
-    releases: list[Version], last_prerelease: Optional[Version]
+    integration: str, releases: list[Version], last_prerelease: Optional[Version]
 ) -> list[Version]:
     """Pick a handful of releases to test from a sorted list of supported releases."""
     # If the package has majors (or major-like releases, even if they don't do
-    # semver), we want to make sure we're testing them all. If not, we just pick
-    # the oldest, the newest, and a couple in between.
+    # semver), we want to make sure we're testing them all (unless there's too
+    # many). If not, we just pick the oldest, the newest, and a couple
+    # in between.
     #
     # If there is a relevant prerelease, also test that in addition to the above.
-    has_majors = len(set([v.major for v in releases])) > 1
+    num_versions = TEST_SUITE_CONFIG[integration].get("num_versions")
+    if num_versions is not None and (
+        not isinstance(num_versions, int) or num_versions < 2
+    ):
+        print("  Integration has invalid `num_versions`: must be an int >= 2")
+        num_versions = None
+
+    has_majors = len({v.major for v in releases}) > 1
     filtered_releases = set()
 
     if has_majors:
         # Always check the very first supported release
         filtered_releases.add(releases[0])
 
-        # Find out the min and max release by each major
+        # Find out the max release by each major
         releases_by_major = {}
         for release in releases:
-            if release.major not in releases_by_major:
-                releases_by_major[release.major] = [release, release]
-            if release < releases_by_major[release.major][0]:
-                releases_by_major[release.major][0] = release
-            if release > releases_by_major[release.major][1]:
-                releases_by_major[release.major][1] = release
-
-        for i, (min_version, max_version) in enumerate(releases_by_major.values()):
+            if (
+                release.major not in releases_by_major
+                or release > releases_by_major[release.major]
+            ):
+                releases_by_major[release.major] = release
+
+        # Add the highest release in each major
+        for max_version in releases_by_major.values():
             filtered_releases.add(max_version)
-            if i == len(releases_by_major) - 1:
-                # If this is the latest major release, also check the lowest
-                # version of this version
-                filtered_releases.add(min_version)
+
+        # If num_versions was provided, slim down the selection
+        if num_versions is not None:
+            filtered_releases = _pick_releases(sorted(filtered_releases), num_versions)
 
     else:
-        filtered_releases = {
-            releases[0],  # oldest version supported
-            releases[len(releases) // 3],
-            releases[
-                len(releases) // 3 * 2
-            ],  # two releases in between, roughly evenly spaced
-            releases[-1],  # latest
-        }
+        filtered_releases = _pick_releases(releases, num_versions)
 
     filtered_releases = sorted(filtered_releases)
     if last_prerelease is not None:
@@ -307,6 +307,25 @@ def pick_releases_to_test(
     return filtered_releases
 
 
+def _pick_releases(
+    releases: list[Version], num_versions: Optional[int]
+) -> set[Version]:
+    num_versions = num_versions or 4
+
+    versions = {
+        releases[0],  # oldest version supported
+        releases[-1],  # latest
+    }
+
+    for i in range(1, num_versions - 1):
+        try:
+            versions.add(releases[len(releases) // (num_versions - 1) * i])
+        except IndexError:
+            pass
+
+    return versions
+
+
 def supported_python_versions(
     package_python_versions: Union[SpecifierSet, list[Version]],
     custom_supported_versions: Optional[SpecifierSet] = None,
@@ -631,7 +650,9 @@ def main(fail_on_changes: bool = False) -> None:
             # Pick a handful of the supported releases to actually test against
             # and fetch the PyPI data for each to determine which Python versions
             # to test it on
-            test_releases = pick_releases_to_test(releases, latest_prerelease)
+            test_releases = pick_releases_to_test(
+                integration, releases, latest_prerelease
+            )
 
             for release in test_releases:
                 _add_python_versions_to_release(integration, package, release)

scripts/populate_tox/tox.jinja

@@ -81,12 +81,6 @@ envlist =
     {py3.7,py3.11,py3.12}-rq-v{1.15,1.16}
     {py3.7,py3.12,py3.13}-rq-latest
 
-    # Sanic
-    {py3.6,py3.7}-sanic-v{0.8}
-    {py3.6,py3.8}-sanic-v{20}
-    {py3.8,py3.11,py3.12}-sanic-v{24.6}
-    {py3.9,py3.12,py3.13}-sanic-latest
-
     # === Integrations - Auto-generated ===
     # These come from the populate_tox.py script. Eventually we should move all
     # integration tests there.
@@ -210,18 +204,6 @@ deps =
     rq-v1.16: rq~=1.16.0
     rq-latest: rq
 
-    # Sanic
-    sanic: websockets<11.0
-    sanic: aiohttp
-    sanic-v{24.6}: sanic_testing
-    sanic-latest: sanic_testing
-    {py3.6}-sanic: aiocontextvars==0.2.1
-    {py3.8}-sanic: tracerite<1.1.2
-    sanic-v0.8: sanic~=0.8.0
-    sanic-v20: sanic~=20.0
-    sanic-v24.6: sanic~=24.6.0
-    sanic-latest: sanic
-
     # === Integrations - Auto-generated ===
     # These come from the populate_tox.py script. Eventually we should move all
     # integration tests there.