test.cjs

/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

// @ts-check

/**
 * Deterministic test runner — replays .commands.json files generated by generate.cjs.
 *
 * No LLM calls. Just sequential playwright-cli commands and assertions.
 *
 * Usage:
 *   node test.cjs                    # run all compiled scenarios
 *   node test.cjs 01-repo-picker     # run matching scenario(s)
 */

const fs = require('fs');
const path = require('path');
const cp = require('child_process');
const {
	APP_ROOT,
	SCENARIOS_DIR,
	runPlaywrightCli,
	getSnapshot,
	startServer,
	waitForServer,
} = require('./common.cjs');

const PORT = 9100 + Math.floor(Math.random() * 900);
const BASE_URL = `http://localhost:${PORT}/?skip-sessions-welcome`;

// ---------------------------------------------------------------------------
// Discover compiled command files
// ---------------------------------------------------------------------------

function discoverCommandFiles(filter) {
	const compiledDir = path.join(SCENARIOS_DIR, 'generated');
	if (!fs.existsSync(compiledDir)) { return []; }
	return fs.readdirSync(compiledDir)
		.filter(f => f.endsWith('.commands.json'))
		.filter(f => !filter || f.includes(filter))
		.sort()
		.map(f => path.join(compiledDir, f));
}

// ---------------------------------------------------------------------------
// Normalize label for assertion matching (strip codicons + trim + lowercase)
// ---------------------------------------------------------------------------

function normalizeLabel(text) {
	return text.replace(/[\uE000-\uF8FF]/g, '').trim().toLowerCase();
}

// ---------------------------------------------------------------------------
// Semantic command resolver — resolves role+label selectors to current refs
// ---------------------------------------------------------------------------

/**
 * Resolve a semantic command like `click button "Send"` to a ref-based
 * command like `click e170` using a fresh snapshot of the current page.
 *
 * Commands that don't match the semantic pattern (type, press, comments)
 * are returned as-is.
 */
function resolveSemanticCommand(cmd, snapshotText) {
	// Match: <action> <role> "<label>"
	const match = cmd.match(/^(click|focus)\s+(\w+)\s+"([^"]+)"$/);
	if (!match) { return { resolved: cmd, ok: true }; }

	const [, action, role, label] = match;
	const needle = normalizeLabel(label);

	for (const line of snapshotText.split('\n')) {
		const refMatch = line.match(/\[ref=(e\d+)\]/);
		if (!refMatch) { continue; }
		if (!line.includes(role)) { continue; }
		const labelMatch = line.match(/"([^"]+)"/);
		if (!labelMatch) { continue; }
		const lineLabel = normalizeLabel(labelMatch[1]);
		if (lineLabel.includes(needle) || needle.includes(lineLabel)) {
			return { resolved: `${action} ${refMatch[1]}`, ok: true };
		}
	}

	return { resolved: cmd, ok: false, message: `Could not find ${role} "${label}" in snapshot` };
}

// ---------------------------------------------------------------------------
// Polling helper — retries a check function until it passes or times out
// ---------------------------------------------------------------------------

const ASSERT_TIMEOUT_MS = 10_000;
const ASSERT_POLL_MS = 500;

/**
 * @param {() => { ok: boolean; message?: string }} checkFn
 * @returns {{ ok: boolean; message?: string }}
 */
function pollAssertion(checkFn) {
	const deadline = Date.now() + ASSERT_TIMEOUT_MS;
	let lastResult = checkFn();
	while (!lastResult.ok && Date.now() < deadline) {
		cp.spawnSync('sleep', [(ASSERT_POLL_MS / 1000).toString()]);
		lastResult = checkFn();
	}
	return lastResult;
}

// ---------------------------------------------------------------------------
// Execute a single command line and handle assertions
// ---------------------------------------------------------------------------

function executeCommand(cmd) {
	// Assertion comments — poll with retries to handle async rendering
	if (cmd.startsWith('# ASSERT_VISIBLE:')) {
		const text = cmd.slice('# ASSERT_VISIBLE:'.length).trim();
		return pollAssertion(() => {
			const snap = getSnapshot();
			if (!snap.stdout) { return { ok: false, message: 'Failed to get snapshot for assertion' }; }
			if (!snap.stdout.toLowerCase().includes(text.toLowerCase())) {
				return { ok: false, message: `Expected "${text}" to be visible in snapshot` };
			}
			return { ok: true };
		});
	}

	if (cmd.startsWith('# ASSERT_DISABLED:')) {
		const label = cmd.slice('# ASSERT_DISABLED:'.length).trim();
		return pollAssertion(() => {
			const snap = getSnapshot();
			if (!snap.stdout) { return { ok: false, message: 'Failed to get snapshot for assertion' }; }
			const needle = normalizeLabel(label);
			const buttonLine = snap.stdout.split('\n').find(l =>
				l.includes('button') && l.match(/"([^"]+)"/) &&
				normalizeLabel(l.match(/"([^"]+)"/)[1]) === needle
			);
			if (!buttonLine) { return { ok: false, message: `Button "${label}" not found in snapshot` }; }
			if (!buttonLine.includes('[disabled]')) {
				return { ok: false, message: `Expected button "${label}" to be disabled` };
			}
			return { ok: true };
		});
	}

	if (cmd.startsWith('# ASSERT_ENABLED:')) {
		const label = cmd.slice('# ASSERT_ENABLED:'.length).trim();
		return pollAssertion(() => {
			const snap = getSnapshot();
			if (!snap.stdout) { return { ok: false, message: 'Failed to get snapshot for assertion' }; }
			const needle = normalizeLabel(label);
			const buttonLine = snap.stdout.split('\n').find(l =>
				l.includes('button') && l.match(/"([^"]+)"/) &&
				normalizeLabel(l.match(/"([^"]+)"/)[1]) === needle
			);
			if (!buttonLine) { return { ok: false, message: `Button "${label}" not found in snapshot` }; }
			if (buttonLine.includes('[disabled]')) {
				return { ok: false, message: `Expected button "${label}" to be enabled` };
			}
			return { ok: true };
		});
	}

	// Skip other comments
	if (cmd.startsWith('#')) { return { ok: true }; }

	// Semantic commands (e.g. `click button "Send"`) — resolve to ref from live snapshot
	const semanticMatch = cmd.match(/^(click|focus)\s+\w+\s+"[^"]+"$/);
	if (semanticMatch) {
		// Poll: the element might not be rendered yet
		return pollAssertion(() => {
			const snap = getSnapshot();
			if (!snap.stdout) { return { ok: false, message: 'Failed to get snapshot for command resolution' }; }
			const { resolved, ok, message } = resolveSemanticCommand(cmd, snap.stdout);
			if (!ok) { return { ok: false, message: message || `Could not resolve: ${cmd}` }; }
			console.log(`    [resolve] ${cmd} → ${resolved}`);
			const result = runPlaywrightCli(resolved);
			if (!result.ok) {
				return { ok: false, message: `playwright-cli ${resolved} failed:\n${result.stderr || result.stdout}` };
			}
			return { ok: true };
		});
	}

	// Regular playwright-cli command (type, press, snapshot, etc.)
	const result = runPlaywrightCli(cmd);
	if (!result.ok) {
		return { ok: false, message: `playwright-cli ${cmd} failed:\n${result.stderr || result.stdout}` };
	}
	return { ok: true };
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

async function main() {
	const filter = process.argv[2] || '';
	const commandFiles = discoverCommandFiles(filter);

	if (commandFiles.length === 0) {
		console.error('No .commands.json files found' + (filter ? ` matching "${filter}"` : ''));
		console.error('Run "npm run generate" first to compile scenarios.');
		process.exit(1);
	}

	console.log(`Found ${commandFiles.length} compiled scenario(s)\n`);

	// Start web server
	console.log(`Starting sessions web server on port ${PORT}…`);
	const server = startServer(PORT, { mock: true });
	await waitForServer(`http://localhost:${PORT}/`, 30_000);
	console.log('Server ready.\n');

	// Open browser
	// --headed if you want to run the test locally and verify the UI interactions
	// const openResult = runPlaywrightCli(['open', '--headed']);
	const openResult = runPlaywrightCli(['open', '--headed']);
	if (!openResult.ok) {
		console.error('Failed to open browser:', openResult.stdout, openResult.stderr);
		cleanup(server);
		process.exit(1);
	}
	const gotoResult = runPlaywrightCli(['goto', BASE_URL]);
	if (!gotoResult.ok) {
		console.error('Failed to navigate:', gotoResult.stdout, gotoResult.stderr);
		cleanup(server);
		process.exit(1);
	}

	// Wait for workbench to render
	cp.spawnSync('sleep', ['5']);

	let totalPassed = 0;
	let totalFailed = 0;

	for (const filePath of commandFiles) {
		const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
		console.log(`▶ ${data.scenario}`);

		// Reset state between scenarios
		runPlaywrightCli(['press', 'Escape']);
		runPlaywrightCli(['goto', BASE_URL]);
		cp.spawnSync('sleep', ['3']);

		let scenarioPassed = true;

		for (const [i, step] of data.steps.entries()) {
			// Give the UI time to settle after each step (matches generate.cjs behavior)
			cp.spawnSync('sleep', ['1']);
			const label = `  step ${i + 1}: ${step.description}`;

			if (step.error) {
				console.error(`  ❌ ${label}`);
				console.error(`     Compilation error: ${step.error}`);
				totalFailed++;
				scenarioPassed = false;
				continue;
			}

			let stepPassed = true;
			for (const cmd of step.commands) {
				const result = executeCommand(cmd);
				if (!result.ok) {
					console.error(`  ❌ ${label}`);
					console.error(`     ${result.message}`);
					// Screenshot on failure
					const basename = path.basename(filePath, '.commands.json');
					runPlaywrightCli(`screenshot --filename=out/failure-${basename}-step${i + 1}.png`);
					totalFailed++;
					stepPassed = false;
					scenarioPassed = false;
					break;
				}
			}

			if (stepPassed) {
				console.log(`  ✅ ${label}`);
				totalPassed++;
			}
		}

		console.log();
	}

	cleanup(server);

	console.log(`Results: ${totalPassed} passed, ${totalFailed} failed`);
	process.exit(totalFailed > 0 ? 1 : 0);
}

function cleanup(server) {
	runPlaywrightCli('close');
	server.kill('SIGTERM');
}

main();