diff --git a/.github/workflows/ci_mac.yml b/.github/workflows/ci_mac.yml index 40acb5f1a..04a7ce613 100644 --- a/.github/workflows/ci_mac.yml +++ b/.github/workflows/ci_mac.yml @@ -23,3 +23,4 @@ jobs: platform: mac checkout-ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.target-ref || github.ref }} yarn-args: --network-timeout 100000 + diff --git a/.github/workflows/job-compile-and-test.yml b/.github/workflows/job-compile-and-test.yml index bbdea87f1..cf96f22e0 100644 --- a/.github/workflows/job-compile-and-test.yml +++ b/.github/workflows/job-compile-and-test.yml @@ -38,6 +38,12 @@ jobs: run: yarn install ${{ inputs.yarn-args }} working-directory: Extension + - name: Install gdb (linux) + if: ${{ inputs.platform == 'linux' }} + run: | + sudo apt-get update + sudo apt-get install -y gdb + - name: Compile Sources run: yarn run compile working-directory: Extension @@ -50,53 +56,46 @@ jobs: run: yarn test working-directory: Extension - # These tests don't require the binary. - # On Linux, it is failing (before the tests actually run) with: Test run terminated with signal SIGSEGV. - # But it works on Linux during the E2E test. - - name: Run SingleRootProject tests - if: ${{ inputs.platform != 'linux' }} - run: yarn test --scenario=SingleRootProject --skipCheckBinaries + - name: Acquire Native Binaries + run: yarn install-and-copy-binaries-for-test working-directory: Extension - # NOTE : We can't run the test that require the native binary files - # yet -- there will be an update soon that allows the tester to - # acquire them on-the-fly - # - name: Run languageServer integration tests - # if: ${{ inputs.platform == 'windows' }} - # run: yarn test --scenario=SingleRootProject - # working-directory: Extension + - name: Run languageServer integration tests (Windows) + if: ${{ inputs.platform == 'windows' }} + run: yarn test --scenario=SingleRootProject + working-directory: Extension - # - name: Run E2E IntelliSense features tests - # if: ${{ inputs.platform == 'windows' }} - # run: yarn test --scenario=MultirootDeadlockTest - # working-directory: Extension + - name: Run E2E IntelliSense features tests (Windows) + if: ${{ inputs.platform == 'windows' }} + run: yarn test --scenario=MultirootDeadlockTest + working-directory: Extension - # - name: Run E2E IntelliSense features tests - # if: ${{ inputs.platform == 'windows' }} - # run: yarn test --scenario=RunWithoutDebugging - # working-directory: Extension + - name: Run RunWithoutDebugging tests (Windows) + if: ${{ inputs.platform == 'windows' }} + run: yarn test --scenario=RunWithoutDebugging + working-directory: Extension # NOTE: For mac/linux run the tests with xvfb-action for UI support. # Another way to start xvfb https://github.com/microsoft/vscode-test/blob/master/sample/azure-pipelines.yml - # - name: Run languageServer integration tests (xvfb) - # if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }} - # uses: coactions/setup-xvfb@v1 - # with: - # run: yarn test --scenario=SingleRootProject - # working-directory: Extension - - # - name: Run E2E IntelliSense features tests (xvfb) - # if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }} - # uses: coactions/setup-xvfb@v1 - # with: - # run: yarn test --scenario=MultirootDeadlockTest - # working-directory: Extension - - # - name: Run E2E IntelliSense features tests (xvfb) - # if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }} - # uses: coactions/setup-xvfb@v1 - # with: - # run: yarn test --scenario=RunWithoutDebugging - # working-directory: Extension + - name: Run languageServer integration tests (linux/macOS) + if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }} + uses: coactions/setup-xvfb@v1 + with: + run: yarn test --scenario=SingleRootProject + working-directory: Extension + + - name: Run E2E IntelliSense features tests (linux/macOS) + if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }} + uses: coactions/setup-xvfb@v1 + with: + run: yarn test --scenario=MultirootDeadlockTest + working-directory: Extension + + - name: Run RunWithoutDebugging tests (linux/macOS) + if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }} + uses: coactions/setup-xvfb@v1 + with: + run: yarn test --scenario=RunWithoutDebugging --scenario-arg=skipExternalConsole + working-directory: Extension diff --git a/Extension/.scripts/common.ts b/Extension/.scripts/common.ts index 29758c269..b6e5d69bb 100644 --- a/Extension/.scripts/common.ts +++ b/Extension/.scripts/common.ts @@ -20,9 +20,18 @@ import { verbose } from '../src/Utility/Text/streams'; export const $root = resolve(`${__dirname}/..`); export let $cmd = 'main'; export let $scenario = ''; +export const $scenarioArgs: string[] = []; // loop through the args and pick out --scenario=... and remove it from the $args and set $scenario process.argv.slice(2).filter(each => !(each.startsWith('--scenario=') && ($scenario = each.substring('--scenario='.length)))); +// parse out the scenario arguments. +process.argv.slice(2).reduce((acc, arg) => { + if (arg.startsWith('--scenario-arg=')) { + acc.push(arg.substring('--scenario-arg='.length)); + } + return acc; +}, $scenarioArgs); + export const $args = process.argv.slice(2).filter(each => !each.startsWith('--')); export const $switches = process.argv.slice(2).filter(each => each.startsWith('--')); diff --git a/Extension/.scripts/copyExtensionBinaries.ts b/Extension/.scripts/copyExtensionBinaries.ts index 0ebf078be..23c9b13e0 100644 --- a/Extension/.scripts/copyExtensionBinaries.ts +++ b/Extension/.scripts/copyExtensionBinaries.ts @@ -5,7 +5,8 @@ import { cp, readdir, rm, stat } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { join } from 'node:path'; +import { basename, join } from 'node:path'; +import { verbose } from '../src/Utility/Text/streams'; import { $args, $root, green, heading, note } from './common'; const extensionPrefix = 'ms-vscode.cpptools-'; @@ -73,17 +74,47 @@ async function getInstalledExtensions(root: string): Promise { - if (providedPath) { - return providedPath; +async function findExtensionsFolder(root: string): Promise { + try { + const entries = await readdir(root, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isDirectory()) { + if (entry.name === 'extensions') { + const extensionEntries = await readdir(join(root, entry.name), { withFileTypes: true }); + for (const extensionEntry of extensionEntries) { + if (extensionEntry.isDirectory() && extensionEntry.name.startsWith(extensionPrefix)) { + return join(root, entry.name); + } + } + } else { + const result = await findExtensionsFolder(join(root, entry.name)); + if (result) { + return result; + } + } + } + } + } catch { + // Ignore errors (permission denied, etc.) } + return undefined; +} +async function findLatestInstalledExtension(providedPath?: string): Promise { const searchRoots: string[] = [ join(homedir(), '.vscode', 'extensions'), join(homedir(), '.vscode-insiders', 'extensions'), join(homedir(), '.vscode-server', 'extensions'), join(homedir(), '.vscode-server-insiders', 'extensions') ]; + if (providedPath) { + // find a folder called 'extensions' recursively under the provided path and add it to the front of the search roots + const extensionsFolderPath = await findExtensionsFolder(providedPath); + if (extensionsFolderPath) { + verbose(`Found extensions folder under provided path: ${extensionsFolderPath}`); + searchRoots.unshift(extensionsFolderPath); + } + } const installed: InstalledExtension[] = (await Promise.all(searchRoots.map(each => getInstalledExtensions(each)))).flat(); if (!installed.length) { @@ -94,7 +125,7 @@ async function findLatestInstalledExtension(providedPath?: string): Promise { console.log(heading('Copy installed extension binaries')); const installedExtensionPath: string = await findLatestInstalledExtension(sourcePath); @@ -110,4 +141,7 @@ export async function main(sourcePath = $args[0]) { } note(`Copied installed binaries into ${$root}`); + + const installedVersion = tryParseVersion(basename(installedExtensionPath)); + return installedVersion?.join('.'); } diff --git a/Extension/.scripts/installAndCopyBinaries.ts b/Extension/.scripts/installAndCopyBinaries.ts new file mode 100644 index 000000000..982c6b067 --- /dev/null +++ b/Extension/.scripts/installAndCopyBinaries.ts @@ -0,0 +1,31 @@ +/* -------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All Rights Reserved. + * See 'LICENSE' in the project root for license information. + * ------------------------------------------------------------------------------------------ */ + +import { runVSCodeCommand } from '@vscode/test-electron'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { $root, error, heading, note } from './common'; +import * as copy from './copyExtensionBinaries'; +import { install, isolated, options } from "./vscode"; + +export async function main() { + console.log(heading(`Install VS Code`)); + const vscode = await install(); + + console.log(heading('Install latest C/C++ Extension')); + const result = await runVSCodeCommand([...vscode?.args ?? [], '--install-extension', 'ms-vscode.cpptools', '--pre-release'], options); + if (result.stdout) { + console.log(result.stdout.toString()); + } + if (result.stderr) { + error(result.stderr.toString()); + } + + const binaryVersion = await copy.main(isolated); + if (binaryVersion) { + await writeFile(join($root, 'bin', 'binaryVersion.json'), JSON.stringify({ version: binaryVersion })); + note(`Wrote binary version ${binaryVersion} to bin/binaryVersion.json`); + } +} diff --git a/Extension/.scripts/test.ts b/Extension/.scripts/test.ts index c6b2a9a2c..033d7010c 100644 --- a/Extension/.scripts/test.ts +++ b/Extension/.scripts/test.ts @@ -14,7 +14,7 @@ import { filepath } from '../src/Utility/Filesystem/filepath'; import { is } from '../src/Utility/System/guards'; import { verbose } from '../src/Utility/Text/streams'; import { getTestInfo } from '../test/common/selectTests'; -import { $args, $root, $scenario, assertAnyFile, assertAnyFolder, brightGreen, checkBinaries, cmdSwitch, cyan, error, gray, green, readJson, red, writeJson } from './common'; +import { $args, $root, $scenario, $scenarioArgs, assertAnyFile, assertAnyFolder, brightGreen, checkBinaries, cmdSwitch, cyan, error, gray, green, readJson, red, writeJson } from './common'; import { install, isolated, options } from './vscode'; export { install, reset } from './vscode'; @@ -90,7 +90,8 @@ async function scenarioTests(assets: string, name: string, workspace: string) { extensionTestsPath: resolve($root, 'dist/test/common/selectTests'), launchArgs: workspace ? [...options.launchArgs, workspace] : options.launchArgs, extensionTestsEnv: { - SCENARIO: assets + SCENARIO: assets, + SCENARIO_ARGS: $scenarioArgs.join(',') } }); } diff --git a/Extension/.scripts/vscode.ts b/Extension/.scripts/vscode.ts index fcd5ac7e3..9be75a371 100644 --- a/Extension/.scripts/vscode.ts +++ b/Extension/.scripts/vscode.ts @@ -17,7 +17,7 @@ export const settings = resolve(userDir, "User", 'settings.json'); export const options = { cachePath: `${isolated}/cache`, - launchArgs: ['--no-sandbox', '--disable-updates', '--skip-welcome', '--skip-release-notes', `--extensions-dir=${extensionsDir}`, `--user-data-dir=${userDir}`, '--disable-workspace-trust'] + launchArgs: ['--no-sandbox', '--disable-updates', '--skip-welcome', '--skip-release-notes', '--disable-extensions', `--extensions-dir=${extensionsDir}`, `--user-data-dir=${userDir}`, '--disable-workspace-trust'] }; export async function install() { @@ -34,9 +34,9 @@ export async function install() { args.push(`--extensions-dir=${extensionsDir}`, `--user-data-dir=${userDir}`); // install the appropriate extensions - // spawnSync(cli, [...args, '--install-extension', 'ms-vscode.cpptools'], { encoding: 'utf-8', stdio: 'ignore' }); - // spawnSync(cli, [...args, '--install-extension', 'twxs.cmake'], { encoding: 'utf-8', stdio: 'ignore' }); - // spawnSync(cli, [...args, '--install-extension', 'ms-vscode.cmake-tools'], { encoding: 'utf-8', stdio: 'ignore' }); + // runVSCodeCommand([...args, '--install-extension', 'ms-vscode.cpptools'], options); + // runVSCodeCommand([...args, '--install-extension', 'twxs.cmake'], options); + // runVSCodeCommand([...args, '--install-extension', 'ms-vscode.cmake-tools'], options); const settingsJson = await readJson(settings, {}); if (!settingsJson["workbench.colorTheme"]) { settingsJson["workbench.colorTheme"] = "Tomorrow Night Blue"; diff --git a/Extension/package.json b/Extension/package.json index c9e0d6958..bc98e2b33 100644 --- a/Extension/package.json +++ b/Extension/package.json @@ -6806,6 +6806,7 @@ "generate-options-schema": "ts-node -T ./.scripts/generateOptionsSchema.ts", "copy-walkthrough-media": "ts-node -T ./.scripts/copyWalkthruMedia.ts", "copy-extension-binaries": "ts-node -T ./.scripts/copyExtensionBinaries.ts", + "install-and-copy-binaries-for-test": "ts-node -T ./.scripts/installAndCopyBinaries.ts", "translations-export": "yarn install && yarn prep && yarn generate-native-strings && gulp translations-export", "translations-generate": "gulp translations-generate", "translations-import": "gulp translations-import", diff --git a/Extension/test/common/selectTests.ts b/Extension/test/common/selectTests.ts index 1ad470056..14969e606 100644 --- a/Extension/test/common/selectTests.ts +++ b/Extension/test/common/selectTests.ts @@ -3,8 +3,8 @@ * See 'LICENSE' in the project root for license information. * ------------------------------------------------------------------------------------------ */ -import { readdir } from 'fs/promises'; -import { IOptions, glob as globSync } from 'glob'; +import { readdir, readFile } from 'fs/promises'; +import { glob as globSync, IOptions } from 'glob'; import * as Mocha from 'mocha'; import { basename, dirname, resolve } from 'path'; import { env } from 'process'; @@ -73,10 +73,45 @@ export async function getTestInfo(...scenarioOptions: (string | undefined)[]) { return undefined; } -export function run (testsRoot: string, cb: (error: any, failures?: number) => void): void { /** - * This code runs in the extension host process, and not in the launch (main.ts) process. + * When running tests on GitHub, this function determines if the tests should be skipped based on + * whether the binary version copied for tests is compatible with the minimum required version. + * The minimum required binary version is defined in the `binaryCompat.json` file and changes when + * there are breaking changes in the communication protocol or messages. + * + * When running locally, the function will always return false since you're expected to have the + * correct binaries available. + * @returns A promise that resolves to a boolean indicating whether the tests should be skipped. */ +async function shouldSkipTests(): Promise { + try { + const binaryVersion = JSON.parse(await readFile(`${$root}/bin/binaryVersion.json`, 'utf-8')) as { version: string } | undefined; + const binaryCompat = JSON.parse(await readFile(`${$root}/test/minBinaryVersion.json`, 'utf-8')) as { minBinaryVersion: string } | undefined; + if (binaryCompat?.minBinaryVersion && binaryVersion?.version) { + const minParts = binaryCompat.minBinaryVersion.split('.').map(Number); + const actualParts = binaryVersion.version.split('.').map(Number); + const maxLen = Math.max(minParts.length, actualParts.length); + let tooOld = false; + for (let i = 0; i < maxLen; i++) { + const diff = (actualParts[i] ?? 0) - (minParts[i] ?? 0); + if (diff < 0) { tooOld = true; break; } + if (diff > 0) { break; } + } + if (tooOld) { + console.warn(`\nBinary-dependent tests SKIPPED: installed binary version ${binaryVersion.version} is below the required minimum ${binaryCompat.minBinaryVersion}.`); + console.warn(`Tests will re-enable automatically once binaries >= ${binaryCompat.minBinaryVersion} are installed.\n`); + return true; + } + } + } catch { + } + return false; +} + +export function run(testsRoot: string, cb: (error: any, failures?: number) => void): void { + /** + * This code runs in the extension host process, and not in the launch (main.ts) process. + */ let location = ''; // scan through the $args to find the --scenario=... @@ -87,34 +122,37 @@ export function run (testsRoot: string, cb: (error: any, failures?: number) => v console.error(`The Scenario folder must be specified either by '--scenario=...' or an environment variable 'SCENARIO=...'`); process.exit(1); } - const { name} = testInfo; - - void glob(`${$root}/dist/test/scenarios/${name}/tests/**/**.test.js`).then((files) => { - - try { - if (!files.length) { - throw new Error(`Unable to find unit tests for ${name} at '${$root}/dist/test/scenarios/${name}/tests/**/**.test.js'`); - } - const mocha = new Mocha({ - ui: 'tdd', - timeout: 500000, - require: ['source-map-support/register'], - color: true - }); - - // Add files to the test suite - files.forEach(f => mocha.addFile(resolve(testsRoot, f))); - - console.log('\n\n=============================================\n Test Output\n\n'); - // Run the mocha test - mocha.run((failures: any) => { - cb(null, failures); - console.log('\n\n=============================================\n\n'); - }); - } catch (err) { - console.error(err); - cb(err); + const { name } = testInfo; + + if (await shouldSkipTests()) { + cb(null, 0); + return; + } + + const files = await glob(`${$root}/dist/test/scenarios/${name}/tests/**/**.test.js`); + try { + if (!files.length) { + throw new Error(`Unable to find unit tests for ${name} at '${$root}/dist/test/scenarios/${name}/tests/**/**.test.js'`); } - }); + const mocha = new Mocha({ + ui: 'tdd', + timeout: 500000, + require: ['source-map-support/register'], + color: true + }); + + // Add files to the test suite + files.forEach(f => mocha.addFile(resolve(testsRoot, f))); + + console.log('\n\n=============================================\n Test Output\n\n'); + // Run the mocha test + mocha.run((failures: any) => { + cb(null, failures); + console.log('\n\n=============================================\n\n'); + }); + } catch (err) { + console.error(err); + cb(err); + } }); } diff --git a/Extension/test/minBinaryVersion.json b/Extension/test/minBinaryVersion.json new file mode 100644 index 000000000..0f97e8d05 --- /dev/null +++ b/Extension/test/minBinaryVersion.json @@ -0,0 +1,3 @@ +{ + "minBinaryVersion": "1.33.0" +} \ No newline at end of file diff --git a/Extension/test/scenarios/RunWithoutDebugging/tests/runWithoutDebugging.terminals.test.ts b/Extension/test/scenarios/RunWithoutDebugging/tests/runWithoutDebugging.terminals.test.ts index 57b27d975..5e59d6105 100644 --- a/Extension/test/scenarios/RunWithoutDebugging/tests/runWithoutDebugging.terminals.test.ts +++ b/Extension/test/scenarios/RunWithoutDebugging/tests/runWithoutDebugging.terminals.test.ts @@ -108,6 +108,7 @@ suite('Run Without Debugging Terminal and Arguments Test', function (this: Mocha 'two words', path.join(workspacePath, 'input folder', 'three words.txt') ]; + const skipExternalConsole = process.env.SCENARIO_ARGS?.includes('skipExternalConsole'); suiteSetup(async function (): Promise { const extension: vscode.Extension = vscode.extensions.getExtension('ms-vscode.cpptools') || assert.fail('Extension not found'); @@ -170,6 +171,10 @@ suite('Run Without Debugging Terminal and Arguments Test', function (this: Mocha for (const profile of profiles) { const profileSuffix = profile ? ` with ${profile} as the default terminal` : consoleCase.consoleMode === 'integratedTerminal' ? ' with default terminal' : ''; test(`No-debug launch via ${consoleCase.label} handles ${programCase.label}${profileSuffix}`, async () => { + if (skipExternalConsole && consoleCase.consoleMode === 'externalTerminal') { + console.log(`\tSkipping external terminal test for ${programCase.label}`); + return; + } await setWindowsDefaultTerminalProfile(profile); disposeTerminals(executablePaths);