test(run): smoke-test running and testing each language's fixture (#1332)

* test(run): smoke-test running and testing each language's fixture

smoke/run.test.ts drives the frontend's run store against a real
openidea-server for TypeScript, Python, Rust, Go, Java and Kotlin: the
detected configurations, a run to its output, the tests with the
fixture's failing test switched on (counts, failure name, location and
message in the test tree), and Rerun Failed Tests running only that test.

`pnpm smoke` runs it; the Language smoke workflow runs it per language
after the code intelligence smoke.

* ci(language-smoke): let main's runs finish

Merges to main came faster than a run, so each one cancelled the run
before it: main never finished the smoke nor saved the build cache PRs
restore. Only a PR's runs are superseded now.

.github/workflows/language-smoke.yml

@@ -2,6 +2,8 @@ name: Language smoke
 # Everyday code intelligence (completion, go to definition, Find Usages, hover, diagnostics
 # with a quick fix, an intention, formatting, rename) per language, against the language
 # server the app starts by default: crates/ide-host/tests/language_smoke.rs.
+# Then running and testing each fixture through the run store and openidea-server
+# (Run, Run tests, the test tree, Rerun Failed Tests): smoke/run.test.ts.
 on:
   push:
     branches: [main]
@@ -14,12 +16,18 @@ on:
       - "fixtures/**"
       - "Cargo.toml"
       - "Cargo.lock"
+      - "src/features/run/**"
+      - "src/platform/**"
+      - "smoke/**"
+      - "package.json"
+      - "pnpm-lock.yaml"
       - ".github/workflows/language-smoke.yml"
   workflow_dispatch:
 
 concurrency:
   group: language-smoke-${{ github.ref }}
-  cancel-in-progress: true
+  # A new push supersedes a PR's run; on main every merge runs to the end (and saves the cache).
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   CARGO_TERM_COLOR: always
@@ -53,9 +61,10 @@ jobs:
           shared-key: language-smoke
           save-if: ${{ github.ref == 'refs/heads/main' && matrix.language == 'rust' }}
 
-      - if: matrix.language == 'typescript'
-        uses: actions/setup-node@v4
-        with: { node-version: 22 }
+      - uses: pnpm/action-setup@v4
+        with: { version: 11 }
+      - uses: actions/setup-node@v4
+        with: { node-version: 22, cache: pnpm }
       - if: matrix.language == 'typescript'
         run: npm install --global "@vtsls/language-server@$VTSLS" "typescript@$TYPESCRIPT"
 
@@ -95,3 +104,7 @@ jobs:
 
       - run: cargo test --locked -p ide-host --test language_smoke --no-run
       - run: cargo test --locked -p ide-host --test language_smoke -- --ignored --exact ${{ matrix.language }}
+
+      - run: pnpm install --frozen-lockfile
+      - run: cargo build --locked -p ide-host --bin openidea-server
+      - run: pnpm smoke smoke/run.test.ts -t "^${{ matrix.language }}$"

AGENTS.md

@@ -69,6 +69,8 @@ Shared agent skills live in `.agents/skills/` (`.claude/skills/` links to them):
 - `cargo nextest run --workspace` (or `cargo test`), `cargo clippy --workspace --all-targets -- -D warnings`, `cargo fmt --all`.
 - `cargo test -p ide-host --test language_smoke -- --ignored --exact <language>` — code intelligence
   on `fixtures/` with that language's real server installed (CI: `Language smoke`, pinned versions).
+- `pnpm smoke smoke/run.test.ts -t <language>` — run and test that language's fixture through the
+  run store and a real `openidea-server` (build it first: `cargo build -p ide-host --bin openidea-server`).
 - `pnpm test`, `pnpm e2e`, `pnpm lint`, `pnpm format:check`, `pnpm knip`, `pnpm typecheck`,
   `pnpm gen:bindings`.
 - `pnpm perf` (UI interaction latency / frames / leaks), `pnpm perf:rust` (criterion

fixtures/README.md

@@ -20,7 +20,9 @@ The fixtures are standalone: `rust-app` declares its own `[workspace]` and
 
 The language smoke test (`crates/ide-host/tests/language_smoke.rs`) runs completion,
 navigation, diagnostics, quick fixes, formatting and rename on these projects with real
-language servers: when you change a fixture, run it for that language.
+language servers, and `smoke/run.test.ts` runs each project's run and test configurations
+(the intentionally failing test included, then Rerun Failed Tests): when you change a
+fixture, run both for that language.
 
 ## Setup
 

knip.ts

@@ -9,6 +9,7 @@ const config: KnipConfig = {
     "scripts/**/*.ts",
     "perf/**/*.ts",
     "e2e/**/*.ts",
+    "smoke/**/*.ts",
   ],
   // Plugins' web pages are projects of their own (own package.json, checked by plugins/build.sh and CI).
   ignore: ["plugins/**/web-src/**"],

package.json

@@ -12,11 +12,12 @@
     "tauri": "node scripts/tauri.ts",
     "test": "vitest run",
     "test:watch": "vitest",
+    "smoke": "vitest run --config smoke/vitest.config.ts",
     "lint": "oxlint",
     "format": "oxfmt",
     "format:check": "oxfmt --check",
     "knip": "knip",
-    "typecheck": "tsrs --noEmit && tsrs -p tsconfig.node.json --noEmit && tsrs -p perf --noEmit && tsrs -p scripts --noEmit",
+    "typecheck": "tsrs --noEmit && tsrs -p tsconfig.node.json --noEmit && tsrs -p perf --noEmit && tsrs -p scripts --noEmit && tsrs -p smoke --noEmit",
     "gen:bindings": "cargo test -p openidea --test export_bindings",
     "gen:windows-icons": "node scripts/gen-windows-icons.ts",
     "gen:angular-grammar": "lezer-generator src/editor/core/cm/angular/angular.grammar --typeScript -o src/editor/core/cm/angular/parser.ts",

smoke/desktop.ts

@@ -0,0 +1,73 @@
+/**
+ * The frontend's view of the desktop app, with a real backend: the window's Tauri runtime
+ * (callbacks, `Channel`s, the event plugin) in-process, and every app command sent to an
+ * `openidea-server` through the route remote development uses (`src/platform/tauriCore.ts`).
+ */
+import { setInvokeRoute } from "@/platform/tauriCore";
+import { encodeArgs } from "@/platform/remote";
+import { Server } from "./server";
+
+type Callback = (message: unknown) => void;
+
+// One runtime per page, as in the app: the frontend registers its event listeners once
+// (`src/platform/ipc.ts`), so they outlive the servers connected one after another.
+const callbacks = new Map<number, Callback>();
+const listeners: { event: string; handler: number }[] = [];
+const channels = new Map<number, { onmessage: (data: unknown) => void }>();
+let nextCallback = 1;
+let installed = false;
+
+function install() {
+  if (installed) return;
+  installed = true;
+  const internals = {
+    metadata: { currentWindow: { label: "main" }, currentWebview: { label: "main", windowLabel: "main" } },
+    transformCallback(cb: Callback) {
+      const id = nextCallback++;
+      callbacks.set(id, cb);
+      return id;
+    },
+    unregisterCallback(id: number) {
+      callbacks.delete(id);
+    },
+    convertFileSrc: (path: string) => path,
+    // Only the window's own plugins reach this: app commands take the route below.
+    async invoke(cmd: string, args: Record<string, unknown> = {}) {
+      if (cmd === "plugin:event|listen") {
+        listeners.push({ event: args.event as string, handler: args.handler as number });
+        return listeners.length;
+      }
+      if (cmd === "plugin:window|get_all_windows") return [];
+      if (cmd.startsWith("plugin:")) return null;
+      throw { kind: "unavailable", message: `${cmd} has no route`, detail: null };
+    },
+  };
+  Object.assign(window, {
+    __TAURI_INTERNALS__: internals,
+    __TAURI_EVENT_PLUGIN_INTERNALS__: { unregisterListener: () => {} },
+  });
+}
+
+/** Connects this page's frontend to `server`; returns a function that disconnects it. */
+export function connectDesktop(server: Server): () => void {
+  install();
+  channels.clear();
+  server.onChannel = (id, data) => channels.get(id)?.onmessage(data);
+  server.onEvent = (target, event) => {
+    if (target !== null && target !== "main") return;
+    for (const l of listeners) {
+      if (l.event === "ide://event") callbacks.get(l.handler)?.({ event: l.event, id: 0, payload: event });
+    }
+  };
+  setInvokeRoute((cmd, args) => {
+    if (cmd.startsWith("plugin:")) return null;
+    const { json, channels: used } = encodeArgs(args);
+    for (const [id, channel] of used) channels.set(id, channel as unknown as { onmessage: (data: unknown) => void });
+    return server.invoke(cmd, JSON.parse(json));
+  });
+  return () => {
+    setInvokeRoute(null);
+    server.onChannel = () => {};
+    server.onEvent = () => {};
+  };
+}

smoke/run.test.ts

@@ -0,0 +1,232 @@
+/**
+ * Running and testing each language's `fixtures/` project the way the Run tool window does:
+ * the backend detects the configurations, the run store starts them (with its test
+ * reporting), parses the output into the test tree, and reruns the failed tests.
+ */
+import { execFileSync } from "node:child_process";
+import { cpSync, mkdtempSync, realpathSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "vitest";
+import { commands, type RunConfig } from "@/bindings";
+import { onIdeEvent } from "@/platform/ipc";
+import { openProject, useProject } from "@/platform/project";
+import {
+  failedTests,
+  loadConfigs,
+  onRunExited,
+  rerunFailed,
+  resetRuns,
+  runtime,
+  startRun,
+  stopSession,
+  useRun,
+  waitForExit,
+} from "@/features/run/store";
+import { nodeStatus } from "@/features/run/testing/results";
+import { connectDesktop } from "./desktop";
+import { repoRoot, Server } from "./server";
+
+interface Case {
+  fixture: string;
+  /** Runs in the project copy first (installs its dependencies). */
+  setup?: (root: string) => void;
+  /** A configuration that runs to completion printing this, or a server stopped once it prints it. */
+  run?: { config: string; prints: string; stop?: boolean };
+  tests: {
+    config: string;
+    /** The configuration with the fixture's intentionally failing test switched on. */
+    failing: (config: RunConfig) => RunConfig;
+    passed: number;
+    failed: string;
+    /** The failure's location ends with this file. */
+    file: string;
+  };
+}
+
+const env = (config: RunConfig, vars: Record<string, string>): RunConfig => ({
+  ...config,
+  env: { ...config.env, ...vars },
+});
+const args = (config: RunConfig, extra: string[]): RunConfig => ({ ...config, args: [...config.args, ...extra] });
+
+const CASES: Record<string, Case> = {
+  typescript: {
+    fixture: "ts-app",
+    setup: (root) => execFileSync("npm", ["install", "--no-audit", "--no-fund"], { cwd: root, stdio: "ignore" }),
+    run: { config: "npm:start", prints: "Total: $134.97" },
+    tests: {
+      config: "npm:test",
+      failing: (c) => env(c, { OPENIDEA_FAIL: "1" }),
+      passed: 4,
+      failed: "intentionally fails (set OPENIDEA_FAIL=1)",
+      file: "tests/cart.test.ts",
+    },
+  },
+  python: {
+    fixture: "python-app",
+    // A project venv with pytest, as the fixture's README sets it up.
+    setup: (root) => {
+      execFileSync("python3", ["-m", "venv", ".venv"], { cwd: root, stdio: "ignore" });
+      execFileSync(join(root, ".venv", "bin", "pip"), ["install", "-q", "pytest"], { cwd: root, stdio: "ignore" });
+    },
+    tests: {
+      config: "pytest",
+      failing: (c) => env(c, { OPENIDEA_FAIL: "1" }),
+      passed: 4,
+      failed: "test_intentionally_failing",
+      file: "tests/test_models.py",
+    },
+  },
+  rust: {
+    fixture: "rust-app",
+    run: { config: "cargo:run:inventory", prints: "Sold 3 mice, 9 left" },
+    tests: {
+      config: "cargo:test",
+      failing: (c) => args(c, ["--features", "failing-tests"]),
+      // `cargo test` stops after the failing unit tests: the integration tests don't run.
+      passed: 7,
+      failed: "intentionally_failing",
+      file: "src/lib.rs",
+    },
+  },
+  go: {
+    fixture: "go-app",
+    run: { config: "go:run:server", prints: "listening on", stop: true },
+    tests: {
+      config: "go:test",
+      failing: (c) => env(c, { OPENIDEA_FAIL: "1" }),
+      passed: 3,
+      failed: "TestIntentionallyFailing",
+      file: "store_test.go",
+    },
+  },
+  java: {
+    fixture: "java-app",
+    tests: {
+      config: "maven:test",
+      failing: (c) => args(c, ["-DopenideaFail=true"]),
+      passed: 2,
+      failed: "intentionallyFails",
+      file: "OrderBookTest.java",
+    },
+  },
+  kotlin: {
+    fixture: "kotlin-gradle",
+    run: { config: "gradle:run", prints: "Lent 'Kotlin in Action' until 2024-01-15" },
+    tests: {
+      config: "gradle:test",
+      failing: (c) => args(c, ["-PopenideaFail"]),
+      passed: 3,
+      failed: "intentionally failing",
+      file: "LibraryTest.kt",
+    },
+  },
+};
+
+let server: Server | null = null;
+let disconnect: (() => void) | null = null;
+let base = "";
+
+afterEach(() => {
+  // As switching projects does: run ids restart with each server.
+  resetRuns();
+  disconnect?.();
+  server?.close();
+  if (base) rmSync(base, { recursive: true, force: true });
+});
+
+async function open(c: Case): Promise<string> {
+  base = realpathSync(mkdtempSync(join(tmpdir(), "openidea-smoke-")));
+  const root = join(base, c.fixture);
+  cpSync(join(repoRoot, "fixtures", c.fixture), root, { recursive: true });
+  c.setup?.(root);
+  server = await Server.start(join(base, "home"));
+  disconnect = connectDesktop(server);
+  const unsubscribe = onIdeEvent("runExited", (e) => onRunExited(e.run, e.code));
+  const previous = disconnect;
+  disconnect = () => {
+    unsubscribe();
+    previous();
+  };
+  await openProject(root);
+  const trusted = await commands.projectTrustSet(root, true, false);
+  if (trusted.status !== "ok") throw new Error(trusted.error.message);
+  useProject.setState({ project: { ...useProject.getState().project!, trusted: true } });
+  await loadConfigs();
+  return root;
+}
+
+function config(id: string): RunConfig {
+  const found = useRun.getState().configs.find((c) => c.id === id);
+  if (!found)
+    throw new Error(
+      `no configuration ${id}: ${useRun
+        .getState()
+        .configs.map((c) => c.id)
+        .join(", ")}`,
+    );
+  return found;
+}
+
+/** Runs `config` to its end; returns its session id and output. */
+async function runToEnd(config: RunConfig, timeoutMs = 8 * 60_000): Promise<{ id: string; output: string[] }> {
+  const output: string[] = [];
+  const session = await startRun(config, undefined, undefined, { prepared: true, onLine: (l) => output.push(l) });
+  if (!session) throw new Error(`${config.id} didn't start`);
+  await waitForExit([session.id], timeoutMs);
+  const ended = useRun.getState().sessions.find((s) => s.id === session.id)!;
+  expect(ended.status, `${config.id} is still running:\n${output.join("\n")}`).not.toBe("running");
+  return { id: session.id, output };
+}
+
+async function smoke(language: string) {
+  const c = CASES[language]!;
+  await open(c);
+
+  if (c.run) {
+    const output: string[] = [];
+    const session = await startRun(config(c.run.config), undefined, undefined, {
+      prepared: true,
+      onLine: (l) => output.push(l),
+    });
+    expect(session, "the run starts").toBeTruthy();
+    await expect.poll(() => output.join("\n"), { timeout: 5 * 60_000 }).toContain(c.run.prints);
+    if (c.run.stop) await stopSession(session!.id);
+    await waitForExit([session!.id], 60_000);
+    const ended = useRun.getState().sessions.find((s) => s.id === session!.id)!;
+    expect(ended.status, output.join("\n")).toBe("finished");
+    if (!c.run.stop) expect(ended.exitCode, output.join("\n")).toBe(0);
+  }
+
+  // Tests, with the intentionally failing one switched on: one failure, located, the rest passed.
+  const { id, output } = await runToEnd(c.tests.failing(config(c.tests.config)));
+  const tests = runtime(id)!.tests!;
+  const log = `\n--- output ---\n${output.slice(-80).join("\n")}`;
+  expect(tests.counts(), log).toMatchObject({ passed: c.tests.passed, failed: 1 });
+  const failure = tests.firstFailure()!;
+  expect(failure.name, log).toBe(c.tests.failed);
+  expect(failure.location?.path ?? "", `${JSON.stringify(failure.location)}${log}`).toMatch(
+    new RegExp(`${c.tests.file.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`),
+  );
+  expect(failure.output.join("\n"), "the failure's message is kept").not.toBe("");
+  expect(failedTests(id)).toHaveLength(1);
+
+  // Rerun Failed Tests runs that one test only.
+  const rerunOutput: string[] = [];
+  const before = useRun.getState().sessions.find((s) => s.id === id)!.startedAt;
+  expect(rerunFailed(id), "Rerun Failed Tests is offered").toBe(true);
+  const ended = () => {
+    const s = useRun.getState().sessions.find((s) => s.id === id)!;
+    return s.startedAt > before && s.endedAt !== null && s.endedAt >= s.startedAt;
+  };
+  await expect.poll(ended, { timeout: 8 * 60_000 }).toBe(true);
+  const again = runtime(id)!.tests!;
+  rerunOutput.push(...again.roots.map((r) => `${r.name}: ${nodeStatus(r)}`));
+  expect(again.counts(), rerunOutput.join("\n")).toMatchObject({ passed: 0, failed: 1 });
+  expect(again.firstFailure()?.name).toBe(c.tests.failed);
+}
+
+for (const language of Object.keys(CASES)) {
+  test(language, () => smoke(language));
+}

smoke/server.ts

@@ -0,0 +1,132 @@
+/**
+ * `openidea-server --stdio`, spoken to as the desktop app does in remote development:
+ * JSON-RPC with `Content-Length` framing over the server's stdin / stdout.
+ */
+import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process";
+import { dirname, join } from "node:path";
+
+const PROTOCOL_VERSION = 2;
+const WINDOW = "main";
+
+// Not `new URL(…, import.meta.url)`: the jsdom environment's `URL` isn't Node's.
+export const repoRoot = join(dirname(import.meta.filename), "..");
+
+/** The server binary: `OPENIDEA_SERVER`, else the workspace's debug build. */
+function serverBinary(): string {
+  const exe = process.platform === "win32" ? ".exe" : "";
+  return process.env.OPENIDEA_SERVER ?? join(repoRoot, "target", "debug", `openidea-server${exe}`);
+}
+
+interface Pending {
+  resolve: (value: unknown) => void;
+  reject: (error: unknown) => void;
+}
+
+export class Server {
+  private nextId = 1;
+  private readonly pending = new Map<number, Pending>();
+  private buffer = Buffer.alloc(0);
+  private readonly stderr: string[] = [];
+  /** `channel` notifications: a value streamed to the `Channel` with this id. */
+  onChannel: (id: number, data: unknown) => void = () => {};
+  /** `event` notifications: an `IdeEvent` for a window (`null`: every window). */
+  onEvent: (window: string | null, event: unknown) => void = () => {};
+
+  private constructor(private readonly child: ChildProcessWithoutNullStreams) {
+    child.stdout.on("data", (chunk: Buffer) => this.receive(chunk));
+    child.stderr.on("data", (chunk: Buffer) => {
+      this.stderr.push(chunk.toString());
+      if (this.stderr.length > 200) this.stderr.shift();
+    });
+    child.on("exit", (code) => {
+      for (const p of this.pending.values()) p.reject(new Error(`openidea-server exited (${code}): ${this.log()}`));
+      this.pending.clear();
+    });
+  }
+
+  /** Starts a server whose settings, caches and tools live under `home`. */
+  static async start(home: string, env: Record<string, string> = {}): Promise<Server> {
+    const child = spawn(serverBinary(), ["--stdio"], {
+      env: { ...process.env, OPENIDEA_SERVER_HOME: home, ...env },
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+    const server = new Server(child);
+    await server.request("initialize", { version: PROTOCOL_VERSION });
+    return server;
+  }
+
+  /** The tail of the server's log, for failure messages. */
+  log(): string {
+    return this.stderr.join("").split("\n").slice(-40).join("\n");
+  }
+
+  /** Runs an IPC command; rejects with the command's `IdeError`. */
+  async invoke(cmd: string, args: unknown): Promise<unknown> {
+    try {
+      return await this.request("invoke", { window: WINDOW, cmd, args });
+    } catch (e) {
+      const message = (e as { message?: string }).message ?? String(e);
+      try {
+        throw JSON.parse(message);
+      } catch (parsed) {
+        if (parsed instanceof SyntaxError) throw { kind: "protocol", message, detail: null };
+        throw parsed;
+      }
+    }
+  }
+
+  close() {
+    this.child.stdin.end();
+    this.child.kill();
+  }
+
+  private request(method: string, params: unknown): Promise<unknown> {
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject });
+      this.send({ jsonrpc: "2.0", id, method, params });
+    });
+  }
+
+  private send(message: unknown) {
+    const body = Buffer.from(JSON.stringify(message));
+    this.child.stdin.write(`Content-Length: ${body.length}\r\n\r\n`);
+    this.child.stdin.write(body);
+  }
+
+  private receive(chunk: Buffer) {
+    this.buffer = Buffer.concat([this.buffer, chunk]);
+    for (;;) {
+      const end = this.buffer.indexOf("\r\n\r\n");
+      if (end < 0) return;
+      const length = /content-length:\s*(\d+)/i.exec(this.buffer.subarray(0, end).toString())?.[1];
+      if (!length) throw new Error(`bad header: ${this.buffer.subarray(0, end)}`);
+      const start = end + 4;
+      if (this.buffer.length < start + Number(length)) return;
+      const message = JSON.parse(this.buffer.subarray(start, start + Number(length)).toString());
+      this.buffer = this.buffer.subarray(start + Number(length));
+      this.dispatch(message);
+    }
+  }
+
+  private dispatch(message: {
+    id?: number;
+    method?: string;
+    params?: { id?: number; data?: unknown; window?: string | null; event?: unknown };
+    result?: unknown;
+    error?: { message: string };
+  }) {
+    if (message.method && message.id !== undefined) {
+      this.send({ jsonrpc: "2.0", id: message.id, error: { code: -32601, message: "not supported" } });
+    } else if (message.method === "channel") {
+      this.onChannel(message.params!.id!, message.params!.data);
+    } else if (message.method === "event") {
+      this.onEvent(message.params!.window ?? null, message.params!.event);
+    } else if (message.id !== undefined) {
+      const pending = this.pending.get(message.id);
+      this.pending.delete(message.id);
+      if (message.error) pending?.reject(message.error);
+      else pending?.resolve(message.result);
+    }
+  }
+}

smoke/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "types": ["node", "vite/client"],
+    "paths": { "@/*": ["../src/*"] }
+  },
+  "include": ["./**/*.ts", "../src/**/*.d.ts"]
+}

smoke/vitest.config.ts

@@ -0,0 +1,20 @@
+/**
+ * Backend smoke tests (`pnpm smoke`): the frontend's own modules driving a real
+ * `openidea-server` and real toolchains. Not part of `pnpm test`: they need the server
+ * built (`cargo build -p ide-host --bin openidea-server`) and each language's tools.
+ */
+import { defineConfig, mergeConfig } from "vitest/config";
+import viteConfig from "../vite.config.ts";
+
+export default mergeConfig(
+  viteConfig({ command: "serve", mode: "test" }),
+  defineConfig({
+    test: {
+      include: ["smoke/**/*.test.ts"],
+      testTimeout: 10 * 60_000,
+      hookTimeout: 5 * 60_000,
+      // Each file starts a server and toolchains of its own.
+      fileParallelism: false,
+    },
+  }),
+);