WIP Mi/pw exp

.github/workflows/positron-merge-to-main.yml

@@ -27,6 +27,56 @@ permissions:
   contents: read
 
 jobs:
+  e2e-playwright:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      POSITRON_BUILD_NUMBER: 0 # CI skips building releases
+      _R_CHECK_FUTURE_FILE_TIMESTAMPS_: false # this check can be flaky in the R pkg tests
+      _R_CHECK_CRAN_INCOMING_: false
+      _R_CHECK_SYSTEM_CLOCK_: false
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+
+      - name: Cache node_modules, build, extensions, and remote
+        uses: ./.github/actions/cache-multi-paths
+
+      - name: Setup Build and Compile
+        uses: ./.github/actions/setup-build-env
+
+      - name: Setup E2E Test Environment
+        uses: ./.github/actions/setup-test-env
+        with:
+          aws-role-to-assume: ${{ secrets.QA_AWS_RO_ROLE }}
+          aws-region: ${{ secrets.QA_AWS_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Compile E2E Tests
+        run: yarn --cwd test/automation compile && yarn --cwd test/smoke compile
+
+      # - name: Setup tmate session
+      #   uses: mxschmitt/action-tmate@v3
+
+      - name: Run Tests (Electron)
+        env:
+          POSITRON_PY_VER_SEL: 3.10.12
+          POSITRON_R_VER_SEL: 4.4.0
+          TEST: "marie123!"
+        id: e2e-playwright-tests
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --grep @pr --workers 2
+
+      - name: Upload Playwright Report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 7
   linux:
     name: Tests on Linux
     runs-on: ubuntu-latest-8x

.gitignore

@@ -26,4 +26,6 @@ product.overrides.json
 # --- Start Positron ---
 .Rproj.user
 # --- End Positron ---
-
+/playwright-report/
+/blob-report/
+/playwright/.cache/

package.json

@@ -130,7 +130,7 @@
     "yazl": "^2.4.3"
   },
   "devDependencies": {
-    "@playwright/test": "^1.46.1",
+    "@playwright/test": "^1.48.0",
     "@swc/core": "1.3.62",
     "@types/cookie": "^0.3.3",
     "@types/debug": "^4.1.5",

playwright.config.ts

@@ -0,0 +1,73 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2024 Posit Software, PBC. All rights reserved.
+ *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+type TestOptions = {
+	web: boolean;
+	artifactDir: string;
+};
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig<TestOptions>({
+	globalSetup: require.resolve('./test/smoke/src/e2e/_global.setup.ts'),
+	testDir: './test/smoke/src/e2e',
+	testMatch: '*.test.ts',
+	fullyParallel: false,
+	forbidOnly: !!process.env.CI,
+	retries: process.env.CI ? 0 : 0,
+	workers: 3, // Number of parallel workers (tests will run in parallel)
+	reporter: process.env.CI
+		? [
+			['github'],
+			['junit', { outputFile: 'test-results/results.xml' }],
+			['html']
+		]
+		: [
+			['list'],
+			['html']
+		],
+	timeout: 2 * 60 * 1000, // 2 minutes
+
+	/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+	use: {
+		/* Base URL to use in actions like `await page.goto('/')`. */
+		// baseURL: 'http://127.0.0.1:3000',
+		headless: false,
+		trace: 'off',
+		// video: 'on'
+	},
+
+	/* Configure projects for major browsers */
+	projects: [
+		{
+			name: 'e2e-electron',
+			use: {
+				...devices['Desktop Chrome'],
+				web: false,
+				artifactDir: 'e2e-electron'
+			},
+		},
+		{
+			name: 'e2e-browser',
+			use: {
+				...devices['Desktop Chrome'],
+				web: true,
+				artifactDir: 'e2e-browser'
+			},
+			grep: /@web/
+		},
+	],
+});

test/automation/src/application.ts

@@ -104,8 +104,8 @@ export class Application {
 		await this._code?.startTracing(name);
 	}
 
-	async stopTracing(name: string, persist: boolean): Promise<void> {
-		await this._code?.stopTracing(name, persist);
+	async stopTracing(name: string, persist: boolean, customPath?: string): Promise<void> {
+		await this._code?.stopTracing(name, persist, customPath);
 	}
 
 	private async startApplication(extraArgs: string[] = []): Promise<Code> {

test/automation/src/code.ts

@@ -121,8 +121,8 @@ export class Code {
 		return await this.driver.startTracing(name);
 	}
 
-	async stopTracing(name: string, persist: boolean): Promise<void> {
-		return await this.driver.stopTracing(name, persist);
+	async stopTracing(name: string, persist: boolean, customPath?: string): Promise<void> {
+		return await this.driver.stopTracing(name, persist, customPath);
 	}
 
 	async dispatchKeybinding(keybinding: string): Promise<void> {

test/automation/src/playwrightBrowser.ts

@@ -105,7 +105,7 @@ async function launchBrowser(options: LaunchOptions, endpoint: string) {
 
 	if (tracing) {
 		try {
-			await measureAndLog(() => context.tracing.start({ screenshots: true, /* remaining options are off for perf reasons */ }), 'context.tracing.start()', logger);
+			await measureAndLog(() => context.tracing.start({ screenshots: true, snapshots: true, /* remaining options are off for perf reasons */ }), 'context.tracing.start()', logger);
 		} catch (error) {
 			logger.log(`Playwright (Browser): Failed to start playwright tracing (${error})`); // do not fail the build when this fails
 		}

test/automation/src/playwrightDriver.ts

@@ -35,8 +35,8 @@ export class PlaywrightDriver {
 
 	constructor(
 		private readonly application: playwright.Browser | playwright.ElectronApplication,
-		private readonly context: playwright.BrowserContext,
 		// --- Start Positron ---
+		readonly context: playwright.BrowserContext,
 		readonly page: playwright.Page,
 		// --- End Positron ---
 		private readonly serverProcess: ChildProcess | undefined,
@@ -49,15 +49,14 @@ export class PlaywrightDriver {
 		if (!this.options.tracing) {
 			return; // tracing disabled
 		}
-
 		try {
 			await measureAndLog(() => this.context.tracing.startChunk({ title: name }), `startTracing for ${name}`, this.options.logger);
 		} catch (error) {
 			// Ignore
 		}
 	}
 
-	async stopTracing(name: string, persist: boolean): Promise<void> {
+	async stopTracing(name: string, persist: boolean = true, customPath?: string): Promise<void> {
 		if (!this.options.tracing) {
 			return; // tracing disabled
 		}
@@ -67,21 +66,10 @@ export class PlaywrightDriver {
 			if (persist) {
 				// --- Start Positron ---
 				// Positron: Windows has issues with long paths, shortened the name
-				persistPath = join(this.options.logsPath, `trace-${PlaywrightDriver.traceCounter++}-${name.replace(/\s+/g, '-')}.zip`);
+				persistPath = customPath || join(this.options.logsPath, `trace-${PlaywrightDriver.traceCounter++}-${name.replace(/\s+/g, '-')}.zip`);
 				// --- End Positron ---
 			}
-
 			await measureAndLog(() => this.context.tracing.stopChunk({ path: persistPath }), `stopTracing for ${name}`, this.options.logger);
-
-			// To ensure we have a screenshot at the end where
-			// it failed, also trigger one explicitly. Tracing
-			// does not guarantee to give us a screenshot unless
-			// some driver action ran before.
-			if (persist) {
-				// --- Start Positron ---
-				await this.takeScreenshot(`${name}`);
-				// --- End Positron ---
-			}
 		} catch (error) {
 			// Ignore
 		}

test/automation/src/playwrightElectron.ts

@@ -45,7 +45,7 @@ async function launchElectron(configuration: IElectronConfiguration, options: La
 
 	if (tracing) {
 		try {
-			await measureAndLog(() => context.tracing.start({ screenshots: true, /* remaining options are off for perf reasons */ }), 'context.tracing.start()', logger);
+			await measureAndLog(() => context.tracing.start({ screenshots: true, snapshots: true /* remaining options are off for perf reasons */ }), 'context.tracing.start()', logger);
 		} catch (error) {
 			logger.log(`Playwright (Electron): Failed to start playwright tracing (${error})`); // do not fail the build when this fails
 		}

test/smoke/README.md

@@ -167,7 +167,7 @@ Make sure that you have followed the [Machine Setup](https://connect.posit.it/po
 
 ### Test Dependencies
 
-Several tests use [QA Content Examples](https://github.com/posit-dev/qa-example-content). You will need to install the dependencies for those projects.  A few current tests also use additional packages. You can look in the [positron-full-test.yml](https://github.com/posit-dev/positron/blob/39a01b71064e2ef3ef5822c95691a034b7e0194f/.github/workflows/positron-full-test.yml) Github action for the full list.
+Several tests use [QA Content Examples](https://github.com/posit-dev/qa-example-content). You will need to install the dependencies for those projects. A few current tests also use additional packages. You can look in the [positron-full-test.yml](https://github.com/posit-dev/positron/blob/39a01b71064e2ef3ef5822c95691a034b7e0194f/.github/workflows/positron-full-test.yml) Github action for the full list.
 
 ## Running Tests
 
@@ -235,9 +235,9 @@ yarn smoketest-pr --build /Applications/Positron.app --parallel --jobs 3
 
 ## Test Project
 
-Before any of the tests start executing the test framework clones down the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo.  This repo contains R and Python files that are run by the automated tests and also includes data files (such as Excel, SQLite, & parquet) that support the test scripts.  If you make additions to QA Content Examples for a test, please be sure that the data files are free to use in a public repository.
+Before any of the tests start executing the test framework clones down the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo. This repo contains R and Python files that are run by the automated tests and also includes data files (such as Excel, SQLite, & parquet) that support the test scripts. If you make additions to QA Content Examples for a test, please be sure that the data files are free to use in a public repository.
 
-For Python, add any package requirements to the `requirements.txt` file in the root of the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo.  We generally do NOT pin them to a specific version, as test can be run against different versions of python and conflicts could arise.  If this becomes a problem, we can revisit this mechanism.
+For Python, add any package requirements to the `requirements.txt` file in the root of the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo. We generally do NOT pin them to a specific version, as test can be run against different versions of python and conflicts could arise. If this becomes a problem, we can revisit this mechanism.
 
 For R, add any package requirements to the "imports" section of the `DESCRIPTION` file in the root of the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo.
 
@@ -262,26 +262,26 @@ await app.workbench.quickaccess.runCommand('workbench.action.toggleDevTools');`
 
 ### Playwright Traces
 
-Note that in launch.json for `Launch Smoke Test` we are passing the `--tracing` argument for you.  This will result in Playwright traces being generated locally for you when tests fail at `.build/logs/smoke-tests-electron/{testCase}`.  Note that for command line runs you will need to pass this arg yourself to get the trace file(s).
+Note that in launch.json for `Launch Smoke Test` we are passing the `--tracing` argument for you. This will result in Playwright traces being generated locally for you when tests fail at `.build/logs/smoke-tests-electron/{testCase}`. Note that for command line runs you will need to pass this arg yourself to get the trace file(s).
 
 ## Running Tests in Github Actions
 
-New tests are not complete until they run successfully across operating systems (Mac, Windows, & Ubuntu) and in [Github Actions](https://github.com/posit-dev/positron/actions/workflows/positron-full-test.yml).  In Github Actions we use an Ubuntu instance to run the tests, so if you are developing your tests using a Mac or on Windows, this is an opportunity to test a different operating system.  Also, you can easily run your new tests against a branch to verify them before merge.  Simply pick the branch after you click on "Run Workflow".  Note that you can also temporarily modify the workflow itself to get your new tests executed more quickly.  To do this, skip the runs of the unit and integration tests.
+New tests are not complete until they run successfully across operating systems (Mac, Windows, & Ubuntu) and in [Github Actions](https://github.com/posit-dev/positron/actions/workflows/positron-full-test.yml). In Github Actions we use an Ubuntu instance to run the tests, so if you are developing your tests using a Mac or on Windows, this is an opportunity to test a different operating system. Also, you can easily run your new tests against a branch to verify them before merge. Simply pick the branch after you click on "Run Workflow". Note that you can also temporarily modify the workflow itself to get your new tests executed more quickly. To do this, skip the runs of the unit and integration tests.
 
 ### Github Actions Test Artifacts
 
-When a run is complete, you can debug any test failures that occurred using the uploaded run artifacts.  The artifacts are available as a ZIP file from inside the workflow run.  Each artifact zip contains: a folder for each test file and an overall run log.  Inside the folder corresponding to each test file, you will find zip files that are Playwright traces.  Note that the trace files are only present for failed cases.
+When a run is complete, you can debug any test failures that occurred using the uploaded run artifacts. The artifacts are available as a ZIP file from inside the workflow run. Each artifact zip contains: a folder for each test file and an overall run log. Inside the folder corresponding to each test file, you will find zip files that are Playwright traces. Note that the trace files are only present for failed cases.
 
-Playwright traces can be drag and dropped to the [Trace Viewer](https://trace.playwright.dev/).  The trace will usually give you a good visualization of the failed test, but they can be sparse on details.  More details are available from the run log (smoke-test-runner.log).  It has a start and end marker for each test case.
+Playwright traces can be drag and dropped to the [Trace Viewer](https://trace.playwright.dev/). The trace will usually give you a good visualization of the failed test, but they can be sparse on details. More details are available from the run log (e2e-test-runner.log). It has a start and end marker for each test case.
 
 ## Notes About Updating Specific Tests
 
 ### Plot Tests That Use Resemblejs
 
-In order to get the "golden screenshots" used for plot comparison is CI, you will need to temporarily uncomment the line of code marked with `capture master image in CI` or add a similar line of code for a new case.  We must use CI taken snapshots because if the "golden screenshots" are taken locally, they will differ too much from the CI images to be useable with a proper threshold.  You can't compare the current runtime plot against a snapshot until you have established a baseline screenshot from CI that is saved to `test/smoke/plots`.
+In order to get the "golden screenshots" used for plot comparison is CI, you will need to temporarily uncomment the line of code marked with `capture master image in CI` or add a similar line of code for a new case. We must use CI taken snapshots because if the "golden screenshots" are taken locally, they will differ too much from the CI images to be useable with a proper threshold. You can't compare the current runtime plot against a snapshot until you have established a baseline screenshot from CI that is saved to `test/smoke/plots`.
 
 ## Tests run on PRs
 
-If you think your test should be run when PRs are created, add the string `#pr` to its name.  The existing #pr cases were selected to give good overall coverage while keeping the overall execution time down to ten minutes or less.  If your new test functionality covers a part of the application that no other tests cover, it is probably a good idea to include it in the #pr set.
+If you think your test should be run when PRs are created, add the string `#pr` to its name. The existing #pr cases were selected to give good overall coverage while keeping the overall execution time down to ten minutes or less. If your new test functionality covers a part of the application that no other tests cover, it is probably a good idea to include it in the #pr set.
 
 <!-- End Positron -->

test/smoke/package.json

@@ -26,6 +26,7 @@
     "@types/node-fetch": "^2.5.10",
     "@types/resemblejs": "4.1.3",
     "@types/rimraf": "3.0.2",
+    "archiver": "^7.0.1",
     "npm-run-all": "^4.1.5",
     "watch": "^1.0.2"
   }

test/smoke/src/e2e/_global.setup.ts

@@ -0,0 +1,22 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2024 Posit Software, PBC. All rights reserved.
+ *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { join } from 'path';
+import * as os from 'os';
+import * as fs from 'fs';
+import { cloneTestRepo, prepareTestEnv } from '../test-runner';
+
+const ROOT_PATH = join(__dirname, '..', '..', '..', '..');
+const LOGS_ROOT_PATH = join(ROOT_PATH, '.build', 'logs');
+const TEST_DATA_PATH = join(os.tmpdir(), 'vscsmoke');
+const WORKSPACE_PATH = join(TEST_DATA_PATH, 'qa-example-content');
+
+async function globalSetup() {
+	fs.rmSync(LOGS_ROOT_PATH, { recursive: true, force: true });
+	prepareTestEnv(ROOT_PATH);
+	cloneTestRepo(WORKSPACE_PATH);
+}
+
+export default globalSetup;