test: skip tests with known error messages

Author: ruifigueira

packages/playwright-cloudflare/tests/src/server/testsServer.ts

@@ -1,8 +1,8 @@
-import { TestRunner, TestEndPayload, isUnderTest } from '@cloudflare/playwright/internal';
+import { TestRunner, TestEndPayload, isUnderTest, TestInfoError, TestResult } from '@cloudflare/playwright/internal';
 import { DurableObject } from 'cloudflare:workers';
 import '@workerTests/index';
 
-import { skipTests } from '../skipTests';
+import { skipTests, skipErrorMessages } from '../skipTests';
 import { BrowserBindingName } from '../utils';
 
 export type TestRequestPayload = {
@@ -17,6 +17,17 @@ const log = console.log.bind(console);
 
 const skipTestsFullTitles = new Set(skipTests.map(t => t.join(' > ')));
 
+function formatError(error: TestInfoError | Error | string) {
+  if (typeof error === 'string')
+    return error;
+  return `${error.message}${error.stack ? `\n${error.stack}` : ''}`;
+}
+
+function shouldSkipTestResult(testResult: TestResult) {
+  const errorText = testResult.errors.map(e => formatError(e)).join('\n');
+  return skipErrorMessages.some(msg => typeof msg === 'string' ? errorText.includes(msg) : msg.test(errorText));
+}
+
 export class TestsServer extends DurableObject<Env> {
   constructor(state: DurableObjectState, env: Env) {
     super(state, env);
@@ -61,9 +72,28 @@ export class TestsServer extends DurableObject<Env> {
     log(`🧪 Running ${fullTitle}${retry ? ` (retry #${retry})` : ''}`);
 
     const result = await testRunner.runTest(file, testId);
+
     if (!['passed', 'skipped'].includes(result.status)) {
-      const error = result.errors[0];
-      log(`❌ ${fullTitle} failed with status ${result.status}${error ? `: ${error.message}` : ''}${error.stack ? `\n${error.stack}` : ''}`);
+      if (shouldSkipTestResult(result)) {
+        log(`🚫 Skipping ${fullTitle} because it failed with a known error message`);
+        return Response.json({
+          testId,
+          status: 'skipped',
+          errors: result.errors,
+          annotations: [
+            {
+              type: 'skip',
+              description: `Test skipped because it failed with a known error message`,
+            }
+          ],
+          duration: 0,
+          hasNonRetriableError: false,
+          timeout,
+          expectedStatus: 'skipped'
+        } satisfies TestEndPayload);
+      }
+      const [error] = result.errors;       
+      log(`❌ ${fullTitle} failed with status ${result.status}${error ? `: ${formatError(error)}` : ''}`);
     }
     return Response.json(result);
   }

packages/playwright-cloudflare/tests/src/server/workerFixtures.ts

@@ -111,6 +111,10 @@ class TestServer {
   sendOnWebSocketConnection() {
     this._testInfo.skip(true, 'sendOnWebSocketConnection not supported, skipping');
   }
+
+  setExtraHeaders() {
+    this._testInfo.skip(true, 'setExtraHeaders not supported, skipping');
+  }
 }
 
 export type ServerFixtures = {

packages/playwright-cloudflare/tests/src/skipTests.ts

@@ -11,3 +11,13 @@ export const skipTests: string[][] = [
   ['page/page-event-popup.spec.ts', 'should work @smoke'],
   ['page/page-history.spec.ts', 'page.goBack should work @smoke'],
 ];
+
+export const skipErrorMessages: Array<string | RegExp> = [
+  'Cloudflare Workers does not support',
+  '__filename is not defined',
+  '__dirname is not defined',
+  '\(\) => {}', // Cloudflare serializes empty arrow function with new lines
+  'Expected substring: "Mozilla"',
+  'createHttp2Server is not defined',
+  /\[unenv\] .* is not implemented yet!/,
+];