diff --git a/.github/workflows/language-smoke.yml b/.github/workflows/language-smoke.yml index 0441fecf6..f8d14c61f 100644 --- a/.github/workflows/language-smoke.yml +++ b/.github/workflows/language-smoke.yml @@ -2,6 +2,8 @@ name: Language smoke # Everyday code intelligence (completion, go to definition, Find Usages, hover, diagnostics # with a quick fix, an intention, formatting, rename) per language, against the language # server the app starts by default: crates/ide-host/tests/language_smoke.rs. +# Then running and testing each fixture through the run store and openidea-server +# (Run, Run tests, the test tree, Rerun Failed Tests): smoke/run.test.ts. on: push: branches: [main] @@ -14,12 +16,18 @@ on: - "fixtures/**" - "Cargo.toml" - "Cargo.lock" + - "src/features/run/**" + - "src/platform/**" + - "smoke/**" + - "package.json" + - "pnpm-lock.yaml" - ".github/workflows/language-smoke.yml" workflow_dispatch: concurrency: group: language-smoke-${{ github.ref }} - cancel-in-progress: true + # A new push supersedes a PR's run; on main every merge runs to the end (and saves the cache). + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: CARGO_TERM_COLOR: always @@ -53,9 +61,10 @@ jobs: shared-key: language-smoke save-if: ${{ github.ref == 'refs/heads/main' && matrix.language == 'rust' }} - - if: matrix.language == 'typescript' - uses: actions/setup-node@v4 - with: { node-version: 22 } + - uses: pnpm/action-setup@v4 + with: { version: 11 } + - uses: actions/setup-node@v4 + with: { node-version: 22, cache: pnpm } - if: matrix.language == 'typescript' run: npm install --global "@vtsls/language-server@$VTSLS" "typescript@$TYPESCRIPT" @@ -95,3 +104,7 @@ jobs: - run: cargo test --locked -p ide-host --test language_smoke --no-run - run: cargo test --locked -p ide-host --test language_smoke -- --ignored --exact ${{ matrix.language }} + + - run: pnpm install --frozen-lockfile + - run: cargo build --locked -p ide-host --bin openidea-server + - run: pnpm smoke smoke/run.test.ts -t "^${{ matrix.language }}$" diff --git a/AGENTS.md b/AGENTS.md index 689bbba32..adcb49b51 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,6 +69,8 @@ Shared agent skills live in `.agents/skills/` (`.claude/skills/` links to them): - `cargo nextest run --workspace` (or `cargo test`), `cargo clippy --workspace --all-targets -- -D warnings`, `cargo fmt --all`. - `cargo test -p ide-host --test language_smoke -- --ignored --exact ` — code intelligence on `fixtures/` with that language's real server installed (CI: `Language smoke`, pinned versions). +- `pnpm smoke smoke/run.test.ts -t ` — run and test that language's fixture through the + run store and a real `openidea-server` (build it first: `cargo build -p ide-host --bin openidea-server`). - `pnpm test`, `pnpm e2e`, `pnpm lint`, `pnpm format:check`, `pnpm knip`, `pnpm typecheck`, `pnpm gen:bindings`. - `pnpm perf` (UI interaction latency / frames / leaks), `pnpm perf:rust` (criterion diff --git a/fixtures/README.md b/fixtures/README.md index 1966e5825..80a5a5eef 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -20,7 +20,9 @@ The fixtures are standalone: `rust-app` declares its own `[workspace]` and The language smoke test (`crates/ide-host/tests/language_smoke.rs`) runs completion, navigation, diagnostics, quick fixes, formatting and rename on these projects with real -language servers: when you change a fixture, run it for that language. +language servers, and `smoke/run.test.ts` runs each project's run and test configurations +(the intentionally failing test included, then Rerun Failed Tests): when you change a +fixture, run both for that language. ## Setup diff --git a/knip.ts b/knip.ts index 600d52246..572c234ad 100644 --- a/knip.ts +++ b/knip.ts @@ -9,6 +9,7 @@ const config: KnipConfig = { "scripts/**/*.ts", "perf/**/*.ts", "e2e/**/*.ts", + "smoke/**/*.ts", ], // Plugins' web pages are projects of their own (own package.json, checked by plugins/build.sh and CI). ignore: ["plugins/**/web-src/**"], diff --git a/package.json b/package.json index c266ac699..5ba6e561e 100644 --- a/package.json +++ b/package.json @@ -12,11 +12,12 @@ "tauri": "node scripts/tauri.ts", "test": "vitest run", "test:watch": "vitest", + "smoke": "vitest run --config smoke/vitest.config.ts", "lint": "oxlint", "format": "oxfmt", "format:check": "oxfmt --check", "knip": "knip", - "typecheck": "tsrs --noEmit && tsrs -p tsconfig.node.json --noEmit && tsrs -p perf --noEmit && tsrs -p scripts --noEmit", + "typecheck": "tsrs --noEmit && tsrs -p tsconfig.node.json --noEmit && tsrs -p perf --noEmit && tsrs -p scripts --noEmit && tsrs -p smoke --noEmit", "gen:bindings": "cargo test -p openidea --test export_bindings", "gen:windows-icons": "node scripts/gen-windows-icons.ts", "gen:angular-grammar": "lezer-generator src/editor/core/cm/angular/angular.grammar --typeScript -o src/editor/core/cm/angular/parser.ts", diff --git a/smoke/desktop.ts b/smoke/desktop.ts new file mode 100644 index 000000000..768fe8e43 --- /dev/null +++ b/smoke/desktop.ts @@ -0,0 +1,73 @@ +/** + * The frontend's view of the desktop app, with a real backend: the window's Tauri runtime + * (callbacks, `Channel`s, the event plugin) in-process, and every app command sent to an + * `openidea-server` through the route remote development uses (`src/platform/tauriCore.ts`). + */ +import { setInvokeRoute } from "@/platform/tauriCore"; +import { encodeArgs } from "@/platform/remote"; +import { Server } from "./server"; + +type Callback = (message: unknown) => void; + +// One runtime per page, as in the app: the frontend registers its event listeners once +// (`src/platform/ipc.ts`), so they outlive the servers connected one after another. +const callbacks = new Map(); +const listeners: { event: string; handler: number }[] = []; +const channels = new Map void }>(); +let nextCallback = 1; +let installed = false; + +function install() { + if (installed) return; + installed = true; + const internals = { + metadata: { currentWindow: { label: "main" }, currentWebview: { label: "main", windowLabel: "main" } }, + transformCallback(cb: Callback) { + const id = nextCallback++; + callbacks.set(id, cb); + return id; + }, + unregisterCallback(id: number) { + callbacks.delete(id); + }, + convertFileSrc: (path: string) => path, + // Only the window's own plugins reach this: app commands take the route below. + async invoke(cmd: string, args: Record = {}) { + if (cmd === "plugin:event|listen") { + listeners.push({ event: args.event as string, handler: args.handler as number }); + return listeners.length; + } + if (cmd === "plugin:window|get_all_windows") return []; + if (cmd.startsWith("plugin:")) return null; + throw { kind: "unavailable", message: `${cmd} has no route`, detail: null }; + }, + }; + Object.assign(window, { + __TAURI_INTERNALS__: internals, + __TAURI_EVENT_PLUGIN_INTERNALS__: { unregisterListener: () => {} }, + }); +} + +/** Connects this page's frontend to `server`; returns a function that disconnects it. */ +export function connectDesktop(server: Server): () => void { + install(); + channels.clear(); + server.onChannel = (id, data) => channels.get(id)?.onmessage(data); + server.onEvent = (target, event) => { + if (target !== null && target !== "main") return; + for (const l of listeners) { + if (l.event === "ide://event") callbacks.get(l.handler)?.({ event: l.event, id: 0, payload: event }); + } + }; + setInvokeRoute((cmd, args) => { + if (cmd.startsWith("plugin:")) return null; + const { json, channels: used } = encodeArgs(args); + for (const [id, channel] of used) channels.set(id, channel as unknown as { onmessage: (data: unknown) => void }); + return server.invoke(cmd, JSON.parse(json)); + }); + return () => { + setInvokeRoute(null); + server.onChannel = () => {}; + server.onEvent = () => {}; + }; +} diff --git a/smoke/run.test.ts b/smoke/run.test.ts new file mode 100644 index 000000000..22dcce330 --- /dev/null +++ b/smoke/run.test.ts @@ -0,0 +1,232 @@ +/** + * Running and testing each language's `fixtures/` project the way the Run tool window does: + * the backend detects the configurations, the run store starts them (with its test + * reporting), parses the output into the test tree, and reruns the failed tests. + */ +import { execFileSync } from "node:child_process"; +import { cpSync, mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { commands, type RunConfig } from "@/bindings"; +import { onIdeEvent } from "@/platform/ipc"; +import { openProject, useProject } from "@/platform/project"; +import { + failedTests, + loadConfigs, + onRunExited, + rerunFailed, + resetRuns, + runtime, + startRun, + stopSession, + useRun, + waitForExit, +} from "@/features/run/store"; +import { nodeStatus } from "@/features/run/testing/results"; +import { connectDesktop } from "./desktop"; +import { repoRoot, Server } from "./server"; + +interface Case { + fixture: string; + /** Runs in the project copy first (installs its dependencies). */ + setup?: (root: string) => void; + /** A configuration that runs to completion printing this, or a server stopped once it prints it. */ + run?: { config: string; prints: string; stop?: boolean }; + tests: { + config: string; + /** The configuration with the fixture's intentionally failing test switched on. */ + failing: (config: RunConfig) => RunConfig; + passed: number; + failed: string; + /** The failure's location ends with this file. */ + file: string; + }; +} + +const env = (config: RunConfig, vars: Record): RunConfig => ({ + ...config, + env: { ...config.env, ...vars }, +}); +const args = (config: RunConfig, extra: string[]): RunConfig => ({ ...config, args: [...config.args, ...extra] }); + +const CASES: Record = { + typescript: { + fixture: "ts-app", + setup: (root) => execFileSync("npm", ["install", "--no-audit", "--no-fund"], { cwd: root, stdio: "ignore" }), + run: { config: "npm:start", prints: "Total: $134.97" }, + tests: { + config: "npm:test", + failing: (c) => env(c, { OPENIDEA_FAIL: "1" }), + passed: 4, + failed: "intentionally fails (set OPENIDEA_FAIL=1)", + file: "tests/cart.test.ts", + }, + }, + python: { + fixture: "python-app", + // A project venv with pytest, as the fixture's README sets it up. + setup: (root) => { + execFileSync("python3", ["-m", "venv", ".venv"], { cwd: root, stdio: "ignore" }); + execFileSync(join(root, ".venv", "bin", "pip"), ["install", "-q", "pytest"], { cwd: root, stdio: "ignore" }); + }, + tests: { + config: "pytest", + failing: (c) => env(c, { OPENIDEA_FAIL: "1" }), + passed: 4, + failed: "test_intentionally_failing", + file: "tests/test_models.py", + }, + }, + rust: { + fixture: "rust-app", + run: { config: "cargo:run:inventory", prints: "Sold 3 mice, 9 left" }, + tests: { + config: "cargo:test", + failing: (c) => args(c, ["--features", "failing-tests"]), + // `cargo test` stops after the failing unit tests: the integration tests don't run. + passed: 7, + failed: "intentionally_failing", + file: "src/lib.rs", + }, + }, + go: { + fixture: "go-app", + run: { config: "go:run:server", prints: "listening on", stop: true }, + tests: { + config: "go:test", + failing: (c) => env(c, { OPENIDEA_FAIL: "1" }), + passed: 3, + failed: "TestIntentionallyFailing", + file: "store_test.go", + }, + }, + java: { + fixture: "java-app", + tests: { + config: "maven:test", + failing: (c) => args(c, ["-DopenideaFail=true"]), + passed: 2, + failed: "intentionallyFails", + file: "OrderBookTest.java", + }, + }, + kotlin: { + fixture: "kotlin-gradle", + run: { config: "gradle:run", prints: "Lent 'Kotlin in Action' until 2024-01-15" }, + tests: { + config: "gradle:test", + failing: (c) => args(c, ["-PopenideaFail"]), + passed: 3, + failed: "intentionally failing", + file: "LibraryTest.kt", + }, + }, +}; + +let server: Server | null = null; +let disconnect: (() => void) | null = null; +let base = ""; + +afterEach(() => { + // As switching projects does: run ids restart with each server. + resetRuns(); + disconnect?.(); + server?.close(); + if (base) rmSync(base, { recursive: true, force: true }); +}); + +async function open(c: Case): Promise { + base = realpathSync(mkdtempSync(join(tmpdir(), "openidea-smoke-"))); + const root = join(base, c.fixture); + cpSync(join(repoRoot, "fixtures", c.fixture), root, { recursive: true }); + c.setup?.(root); + server = await Server.start(join(base, "home")); + disconnect = connectDesktop(server); + const unsubscribe = onIdeEvent("runExited", (e) => onRunExited(e.run, e.code)); + const previous = disconnect; + disconnect = () => { + unsubscribe(); + previous(); + }; + await openProject(root); + const trusted = await commands.projectTrustSet(root, true, false); + if (trusted.status !== "ok") throw new Error(trusted.error.message); + useProject.setState({ project: { ...useProject.getState().project!, trusted: true } }); + await loadConfigs(); + return root; +} + +function config(id: string): RunConfig { + const found = useRun.getState().configs.find((c) => c.id === id); + if (!found) + throw new Error( + `no configuration ${id}: ${useRun + .getState() + .configs.map((c) => c.id) + .join(", ")}`, + ); + return found; +} + +/** Runs `config` to its end; returns its session id and output. */ +async function runToEnd(config: RunConfig, timeoutMs = 8 * 60_000): Promise<{ id: string; output: string[] }> { + const output: string[] = []; + const session = await startRun(config, undefined, undefined, { prepared: true, onLine: (l) => output.push(l) }); + if (!session) throw new Error(`${config.id} didn't start`); + await waitForExit([session.id], timeoutMs); + const ended = useRun.getState().sessions.find((s) => s.id === session.id)!; + expect(ended.status, `${config.id} is still running:\n${output.join("\n")}`).not.toBe("running"); + return { id: session.id, output }; +} + +async function smoke(language: string) { + const c = CASES[language]!; + await open(c); + + if (c.run) { + const output: string[] = []; + const session = await startRun(config(c.run.config), undefined, undefined, { + prepared: true, + onLine: (l) => output.push(l), + }); + expect(session, "the run starts").toBeTruthy(); + await expect.poll(() => output.join("\n"), { timeout: 5 * 60_000 }).toContain(c.run.prints); + if (c.run.stop) await stopSession(session!.id); + await waitForExit([session!.id], 60_000); + const ended = useRun.getState().sessions.find((s) => s.id === session!.id)!; + expect(ended.status, output.join("\n")).toBe("finished"); + if (!c.run.stop) expect(ended.exitCode, output.join("\n")).toBe(0); + } + + // Tests, with the intentionally failing one switched on: one failure, located, the rest passed. + const { id, output } = await runToEnd(c.tests.failing(config(c.tests.config))); + const tests = runtime(id)!.tests!; + const log = `\n--- output ---\n${output.slice(-80).join("\n")}`; + expect(tests.counts(), log).toMatchObject({ passed: c.tests.passed, failed: 1 }); + const failure = tests.firstFailure()!; + expect(failure.name, log).toBe(c.tests.failed); + expect(failure.location?.path ?? "", `${JSON.stringify(failure.location)}${log}`).toMatch( + new RegExp(`${c.tests.file.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`), + ); + expect(failure.output.join("\n"), "the failure's message is kept").not.toBe(""); + expect(failedTests(id)).toHaveLength(1); + + // Rerun Failed Tests runs that one test only. + const rerunOutput: string[] = []; + const before = useRun.getState().sessions.find((s) => s.id === id)!.startedAt; + expect(rerunFailed(id), "Rerun Failed Tests is offered").toBe(true); + const ended = () => { + const s = useRun.getState().sessions.find((s) => s.id === id)!; + return s.startedAt > before && s.endedAt !== null && s.endedAt >= s.startedAt; + }; + await expect.poll(ended, { timeout: 8 * 60_000 }).toBe(true); + const again = runtime(id)!.tests!; + rerunOutput.push(...again.roots.map((r) => `${r.name}: ${nodeStatus(r)}`)); + expect(again.counts(), rerunOutput.join("\n")).toMatchObject({ passed: 0, failed: 1 }); + expect(again.firstFailure()?.name).toBe(c.tests.failed); +} + +for (const language of Object.keys(CASES)) { + test(language, () => smoke(language)); +} diff --git a/smoke/server.ts b/smoke/server.ts new file mode 100644 index 000000000..97250b00e --- /dev/null +++ b/smoke/server.ts @@ -0,0 +1,132 @@ +/** + * `openidea-server --stdio`, spoken to as the desktop app does in remote development: + * JSON-RPC with `Content-Length` framing over the server's stdin / stdout. + */ +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { dirname, join } from "node:path"; + +const PROTOCOL_VERSION = 2; +const WINDOW = "main"; + +// Not `new URL(…, import.meta.url)`: the jsdom environment's `URL` isn't Node's. +export const repoRoot = join(dirname(import.meta.filename), ".."); + +/** The server binary: `OPENIDEA_SERVER`, else the workspace's debug build. */ +function serverBinary(): string { + const exe = process.platform === "win32" ? ".exe" : ""; + return process.env.OPENIDEA_SERVER ?? join(repoRoot, "target", "debug", `openidea-server${exe}`); +} + +interface Pending { + resolve: (value: unknown) => void; + reject: (error: unknown) => void; +} + +export class Server { + private nextId = 1; + private readonly pending = new Map(); + private buffer = Buffer.alloc(0); + private readonly stderr: string[] = []; + /** `channel` notifications: a value streamed to the `Channel` with this id. */ + onChannel: (id: number, data: unknown) => void = () => {}; + /** `event` notifications: an `IdeEvent` for a window (`null`: every window). */ + onEvent: (window: string | null, event: unknown) => void = () => {}; + + private constructor(private readonly child: ChildProcessWithoutNullStreams) { + child.stdout.on("data", (chunk: Buffer) => this.receive(chunk)); + child.stderr.on("data", (chunk: Buffer) => { + this.stderr.push(chunk.toString()); + if (this.stderr.length > 200) this.stderr.shift(); + }); + child.on("exit", (code) => { + for (const p of this.pending.values()) p.reject(new Error(`openidea-server exited (${code}): ${this.log()}`)); + this.pending.clear(); + }); + } + + /** Starts a server whose settings, caches and tools live under `home`. */ + static async start(home: string, env: Record = {}): Promise { + const child = spawn(serverBinary(), ["--stdio"], { + env: { ...process.env, OPENIDEA_SERVER_HOME: home, ...env }, + stdio: ["pipe", "pipe", "pipe"], + }); + const server = new Server(child); + await server.request("initialize", { version: PROTOCOL_VERSION }); + return server; + } + + /** The tail of the server's log, for failure messages. */ + log(): string { + return this.stderr.join("").split("\n").slice(-40).join("\n"); + } + + /** Runs an IPC command; rejects with the command's `IdeError`. */ + async invoke(cmd: string, args: unknown): Promise { + try { + return await this.request("invoke", { window: WINDOW, cmd, args }); + } catch (e) { + const message = (e as { message?: string }).message ?? String(e); + try { + throw JSON.parse(message); + } catch (parsed) { + if (parsed instanceof SyntaxError) throw { kind: "protocol", message, detail: null }; + throw parsed; + } + } + } + + close() { + this.child.stdin.end(); + this.child.kill(); + } + + private request(method: string, params: unknown): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.send({ jsonrpc: "2.0", id, method, params }); + }); + } + + private send(message: unknown) { + const body = Buffer.from(JSON.stringify(message)); + this.child.stdin.write(`Content-Length: ${body.length}\r\n\r\n`); + this.child.stdin.write(body); + } + + private receive(chunk: Buffer) { + this.buffer = Buffer.concat([this.buffer, chunk]); + for (;;) { + const end = this.buffer.indexOf("\r\n\r\n"); + if (end < 0) return; + const length = /content-length:\s*(\d+)/i.exec(this.buffer.subarray(0, end).toString())?.[1]; + if (!length) throw new Error(`bad header: ${this.buffer.subarray(0, end)}`); + const start = end + 4; + if (this.buffer.length < start + Number(length)) return; + const message = JSON.parse(this.buffer.subarray(start, start + Number(length)).toString()); + this.buffer = this.buffer.subarray(start + Number(length)); + this.dispatch(message); + } + } + + private dispatch(message: { + id?: number; + method?: string; + params?: { id?: number; data?: unknown; window?: string | null; event?: unknown }; + result?: unknown; + error?: { message: string }; + }) { + if (message.method && message.id !== undefined) { + this.send({ jsonrpc: "2.0", id: message.id, error: { code: -32601, message: "not supported" } }); + } else if (message.method === "channel") { + this.onChannel(message.params!.id!, message.params!.data); + } else if (message.method === "event") { + this.onEvent(message.params!.window ?? null, message.params!.event); + } else if (message.id !== undefined) { + const pending = this.pending.get(message.id); + this.pending.delete(message.id); + if (message.error) pending?.reject(message.error); + else pending?.resolve(message.result); + } + } +} diff --git a/smoke/tsconfig.json b/smoke/tsconfig.json new file mode 100644 index 000000000..b83899532 --- /dev/null +++ b/smoke/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "types": ["node", "vite/client"], + "paths": { "@/*": ["../src/*"] } + }, + "include": ["./**/*.ts", "../src/**/*.d.ts"] +} diff --git a/smoke/vitest.config.ts b/smoke/vitest.config.ts new file mode 100644 index 000000000..a4fb5c3b1 --- /dev/null +++ b/smoke/vitest.config.ts @@ -0,0 +1,20 @@ +/** + * Backend smoke tests (`pnpm smoke`): the frontend's own modules driving a real + * `openidea-server` and real toolchains. Not part of `pnpm test`: they need the server + * built (`cargo build -p ide-host --bin openidea-server`) and each language's tools. + */ +import { defineConfig, mergeConfig } from "vitest/config"; +import viteConfig from "../vite.config.ts"; + +export default mergeConfig( + viteConfig({ command: "serve", mode: "test" }), + defineConfig({ + test: { + include: ["smoke/**/*.test.ts"], + testTimeout: 10 * 60_000, + hookTimeout: 5 * 60_000, + // Each file starts a server and toolchains of its own. + fileParallelism: false, + }, + }), +);