Allow running the acceptance tests in pulumi/examples as part of the test suite (#823)

This change adds a new optional flag `testPulumiExamples` that can be
set in .ci-mgmt.yaml. When set, the acceptance tests in pulumi/examples
will be run as part of the test suite.

The goal is to increase test coverage in provider PRs or releases.
`pulumi/examples` has a rich set of realistic programs that can be used
for this via ProgramTest.

Providers don't necessarily need to gate PRs on these tests, they can be
informational only.

This requires that the provider's test suite in `examples/` is enabled
for this. See pulumi/pulumi-azure#1717 for an example. It's not enough
to just run the test suite in pulumi/examples because we want to use or
inject the locally built provider and SDKs of the current PR.

I chose the approach using the test job's matrix because the new
pulumi/examples tests should run concurrently to the regular ones, but
to run them in a separate job would require copying the lengthy setup
steps of the test job, or extracting the setup into a composite action
which I thought could be avoided.

Author: thomas11

provider-ci/internal/pkg/templates/bridged-provider.config.yaml

@@ -61,6 +61,8 @@ extraTests:
 #   your_test_name:
 #     name: My Test
 #     ...
+# Run e2e tests using the examples and test suite in the pulumi/examples repo.
+testPulumiExamples: false
 runner:
   default: ubuntu-latest
   prerequisites: ubuntu-latest

provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/run-acceptance-tests.yml

@@ -186,6 +186,7 @@ jobs:
       run: exit 1
     - name: Workflow is a success
       run: echo "🎉🎈🎉🎈🎉"
+
   test:
     if: github.event_name == 'repository_dispatch' ||
       github.event.pull_request.head.repo.full_name == github.repository
@@ -209,6 +210,12 @@ jobs:
         path: ci-scripts
         repository: pulumi/scripts
         ref: #{{ .Config.pulumiScriptsRef }}#
+    - name: Checkout p/examples
+      if: matrix.testTarget == 'pulumiExamples'
+      uses: #{{ .Config.actionVersions.checkout }}#
+      with:
+        repository: pulumi/examples
+        path: p-examples
     - name: Unshallow clone for tags
       run: git fetch --prune --unshallow --tags
 #{{ .Config.actions.setupGo | toYaml | indent 4 }}#
@@ -315,15 +322,20 @@ jobs:
 #{{ .Config.actions.preTest | toYaml | indent 4 }}#
 #{{- end }}#
     - name: Run tests
+      if: matrix.testTarget == 'local'
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
-    - if: failure() && github.event_name == 'push'
+        matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    - if: failure() && github.event_name == 'push' && matrix.testTarget == 'local'
       name: Notify Slack
       uses: #{{ .Config.actionVersions.notifySlack }}#
       with:
         author_name: Failure in running ${{ matrix.language }} tests
         fields: repo,commit,author,action
         status: ${{ job.status }}
+    - name: Run pulumi/examples tests
+      if: matrix.testTarget == 'pulumiExamples'
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
+        matrix.language }} -run TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:
@@ -333,6 +345,11 @@ jobs:
         - dotnet
         - go
         - java
+        #{{- if .Config.testPulumiExamples }}#
+        testTarget: [local, pulumiExamples]
+        #{{- else }}#
+        testTarget: [local]
+        #{{- end }}#
   license_check:
     name: License Check
     uses: ./.github/workflows/license.yml

provider-ci/test-workflows/aws/.github/workflows/run-acceptance-tests.yml

@@ -289,6 +289,7 @@ jobs:
       run: exit 1
     - name: Workflow is a success
       run: echo "🎉🎈🎉🎈🎉"
+
   test:
     if: github.event_name == 'repository_dispatch' ||
       github.event.pull_request.head.repo.full_name == github.repository
@@ -310,6 +311,12 @@ jobs:
         path: ci-scripts
         repository: pulumi/scripts
         ref: deca2c5c6015ad7aaea6f572a1c2b198ca323592
+    - name: Checkout p/examples
+      if: matrix.testTarget == 'pulumiExamples'
+      uses: actions/checkout@v4
+      with:
+        repository: pulumi/examples
+        path: p-examples
     - name: Unshallow clone for tags
       run: git fetch --prune --unshallow --tags
     - name: Install Go
@@ -395,15 +402,20 @@ jobs:
     - name: Make upstream
       run: make upstream
     - name: Run tests
+      if: matrix.testTarget == 'local'
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
-    - if: failure() && github.event_name == 'push'
+        matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    - if: failure() && github.event_name == 'push' && matrix.testTarget == 'local'
       name: Notify Slack
       uses: 8398a7/action-slack@v3
       with:
         author_name: Failure in running ${{ matrix.language }} tests
         fields: repo,commit,author,action
         status: ${{ job.status }}
+    - name: Run pulumi/examples tests
+      if: matrix.testTarget == 'pulumiExamples'
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
+        matrix.language }} -run TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:
@@ -413,6 +425,7 @@ jobs:
         - dotnet
         - go
         - java
+        testTarget: [local]
   license_check:
     name: License Check
     uses: ./.github/workflows/license.yml

provider-ci/test-workflows/cloudflare/.github/workflows/run-acceptance-tests.yml

@@ -283,6 +283,7 @@ jobs:
       run: exit 1
     - name: Workflow is a success
       run: echo "🎉🎈🎉🎈🎉"
+
   test:
     if: github.event_name == 'repository_dispatch' ||
       github.event.pull_request.head.repo.full_name == github.repository
@@ -303,6 +304,12 @@ jobs:
         path: ci-scripts
         repository: pulumi/scripts
         ref: deca2c5c6015ad7aaea6f572a1c2b198ca323592
+    - name: Checkout p/examples
+      if: matrix.testTarget == 'pulumiExamples'
+      uses: actions/checkout@v4
+      with:
+        repository: pulumi/examples
+        path: p-examples
     - name: Unshallow clone for tags
       run: git fetch --prune --unshallow --tags
     - name: Install Go
@@ -377,15 +384,20 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
         version: v2.5.0
     - name: Run tests
+      if: matrix.testTarget == 'local'
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
-    - if: failure() && github.event_name == 'push'
+        matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    - if: failure() && github.event_name == 'push' && matrix.testTarget == 'local'
       name: Notify Slack
       uses: 8398a7/action-slack@v3
       with:
         author_name: Failure in running ${{ matrix.language }} tests
         fields: repo,commit,author,action
         status: ${{ job.status }}
+    - name: Run pulumi/examples tests
+      if: matrix.testTarget == 'pulumiExamples'
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
+        matrix.language }} -run TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:
@@ -395,6 +407,7 @@ jobs:
         - dotnet
         - go
         - java
+        testTarget: [local]
   license_check:
     name: License Check
     uses: ./.github/workflows/license.yml

provider-ci/test-workflows/docker/.github/workflows/run-acceptance-tests.yml

@@ -296,6 +296,7 @@ jobs:
       run: exit 1
     - name: Workflow is a success
       run: echo "🎉🎈🎉🎈🎉"
+
   test:
     if: github.event_name == 'repository_dispatch' ||
       github.event.pull_request.head.repo.full_name == github.repository
@@ -316,6 +317,12 @@ jobs:
         path: ci-scripts
         repository: pulumi/scripts
         ref: deca2c5c6015ad7aaea6f572a1c2b198ca323592
+    - name: Checkout p/examples
+      if: matrix.testTarget == 'pulumiExamples'
+      uses: actions/checkout@v4
+      with:
+        repository: pulumi/examples
+        path: p-examples
     - name: Unshallow clone for tags
       run: git fetch --prune --unshallow --tags
     - name: Install Go
@@ -419,15 +426,20 @@ jobs:
     - name: Expose GitHub Actions runtime
       uses: crazy-max/ghaction-github-runtime@v3
     - name: Run tests
+      if: matrix.testTarget == 'local'
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
-    - if: failure() && github.event_name == 'push'
+        matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    - if: failure() && github.event_name == 'push' && matrix.testTarget == 'local'
       name: Notify Slack
       uses: 8398a7/action-slack@v3
       with:
         author_name: Failure in running ${{ matrix.language }} tests
         fields: repo,commit,author,action
         status: ${{ job.status }}
+    - name: Run pulumi/examples tests
+      if: matrix.testTarget == 'pulumiExamples'
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
+        matrix.language }} -run TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:
@@ -437,6 +449,7 @@ jobs:
         - dotnet
         - go
         - java
+        testTarget: [local]
   license_check:
     name: License Check
     uses: ./.github/workflows/license.yml