feat(run): load a Minitest reporter into Ruby test runs (#1547)

Parsing Minitest's console output needs -v, gives no file or line for passing tests and depends on how output interleaves. Ruby test runs (ruby and rake test configurations in the console) now load a small Minitest reporter through RUBYLIB/RUBYOPT that prints one record per test on stdout with its status, time, failure text and real file and line. The console strips the records and the test tree is built from them, so jump to source gets exact locations; the text parser stays as the fallback. It works on Minitest 5 and 6 (6 no longer loads plugins on its own, so a small hook registers the reporter once Minitest is loaded).

Co-authored-by: kjeldahl <[email protected]>

src/features/run/store.ts

@@ -43,6 +43,7 @@ import {
 } from "./testing";
 import { GoTestOutput } from "./testing/go";
 import { moveToReportedClasses } from "./testing/jvm";
+import { RubyTestOutput } from "./testing/ruby";
 import { TeamcityOutput } from "./testing/teamcity";
 import type { TestStdout } from "./testing/types";
 import { recordTestStates } from "./testing/states";
@@ -87,7 +88,7 @@ interface SessionRuntime {
   console: ConsoleModel;
   tests: TestResults | null;
   parser: TestOutputParser | null;
-  /** Machine-readable stdout (`go test -json`, `--teamcity`): fed to the parser, shown decoded. */
+  /** Machine-readable stdout (`go test -json`, `--teamcity`, the Ruby reporter): fed to the parser, shown decoded. */
   stdout: TestStdout | null;
   queue: OutputChunk[];
   /** rAF id while the page is visible, timeout id while hidden (rAF pauses in background tabs). */
@@ -567,13 +568,16 @@ function newRuntime(config: RunConfig): SessionRuntime {
     if (tests && parser) tests.apply(parser.feed(line));
   };
   const framework = tests && parser ? frameworkFor(config) : null;
-  // PHPUnit `--teamcity` and Gradle's test init script report through service messages.
+  // PHPUnit `--teamcity` and Gradle's test init script report through service messages, Minitest through
+  // the IDE's own reporter (RSpec has none: its progress dots must show as they come).
   const stdout =
     framework === "go"
       ? new GoTestOutput(feed)
       : framework === "phpunit" || framework === "jvm"
         ? new TeamcityOutput(feed)
-        : null;
+        : framework === "ruby" && config.kind !== "rspec"
+          ? new RubyTestOutput(feed)
+          : null;
   const runtime: SessionRuntime = {
     stdout,
     tests,

src/features/run/testing/results.ts

@@ -128,7 +128,9 @@ export class TestResults {
         }
         case "location": {
           const n = this.node(e.path);
-          n.location ??= e.location;
+          // The first location wins, except that a place to search for (Ruby tests) gives way to a real one.
+          if (!n.location || (n.location.resolve === "rubyTest" && e.location.resolve !== "rubyTest"))
+            n.location = e.location;
           break;
         }
         case "output": {

src/features/run/testing/ruby.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from "vitest";
 import { parseAll, withTestReporting, frameworkFor } from "./index";
 import { TestResults, nodeStatus } from "./results";
-import { rubyParser } from "./ruby";
+import { RubyTestOutput, rubyParser } from "./ruby";
 
 // Captured from `ruby -Ilib test/mixed_test.rb -v --seed 1` (Minitest 6.0.6).
 const MINITEST = `Run options: -v --seed 1
@@ -125,3 +125,98 @@ describe("Ruby test reporting", () => {
     expect(frameworkFor(cfg("rake", "bundle", ["exec", "rake", "test"]))).toBe("ruby");
   });
 });
+
+// Captured from a run with the IDE's reporter (Minitest 6.0.6): records end with their own newline, so the
+// progress dots stay on one line once they are taken out.
+const M = "@@OPENIDEA@@";
+const F = "/p/test/mixed_test.rb";
+const REPORTER_STREAM = [
+  "# Running:",
+  "",
+  `${M}{"t":"start","c":"MixedTest","n":"test_pass","f":"${F}","l":4}`,
+  `.${M}{"t":"result","c":"MixedTest","n":"test_pass","s":"passed","d":0.000063,"f":"${F}","l":4,"m":""}`,
+  `${M}{"t":"start","c":"MixedTest","n":"test_fail","f":"${F}","l":5}`,
+  `F${M}{"t":"result","c":"MixedTest","n":"test_fail","s":"failed","d":0.000054,"f":"${F}","l":5,"m":"Expected: 4\\n  Actual: 3"}`,
+  `${M}{"t":"start","c":"MixedTest","n":"test_prints","f":"${F}","l":8}`,
+  "hello from test",
+  `.${M}{"t":"result","c":"MixedTest","n":"test_prints","s":"passed","d":0.000007,"f":"${F}","l":8,"m":""}`,
+  `${M}{"t":"start","c":"MixedTest","n":"test_skip","f":"${F}","l":7}`,
+  `S${M}{"t":"result","c":"MixedTest","n":"test_skip","s":"skipped","d":0.000006,"f":"${F}","l":7,"m":"later"}`,
+  "",
+  `${M}{"t":"summary","runs":4,"failures":1,"errors":0,"skips":1,"d":0.0011}`,
+  "Finished in 0.001124s",
+  "",
+].join("\n");
+
+describe("the IDE's Minitest reporter", () => {
+  const collect = (chunks: string[]) => {
+    const lines: string[] = [];
+    const out = new RubyTestOutput((line) => lines.push(line));
+    const text = chunks.map((c) => out.feed(c)).join("") + out.end();
+    return { text, records: lines.filter((l) => l.includes(M)), lines };
+  };
+
+  it("takes the records out of the console, exactly as Minitest prints them without it", () => {
+    const { text, records } = collect([REPORTER_STREAM]);
+    expect(text).toBe("# Running:\n\n.Fhello from test\n.S\nFinished in 0.001124s\n");
+    expect(records).toHaveLength(9);
+    expect(records.every((r) => r.includes(M))).toBe(true);
+  });
+
+  it("is the same however the output is cut into chunks", () => {
+    const whole = collect([REPORTER_STREAM]);
+    const chars = collect(REPORTER_STREAM.split(""));
+    const odd = collect(REPORTER_STREAM.match(/[\s\S]{1,7}/g)!);
+    expect(chars).toEqual(whole);
+    expect(odd).toEqual(whole);
+  });
+
+  it("drops a record cut off by the end of the process, not the text before it", () => {
+    expect(collect([`abc${M}{"t":"sta`]).text).toBe("abc");
+    expect(collect(["no marker @@OPEN"]).text).toBe("no marker @@OPEN\n");
+  });
+
+  describe("results", () => {
+    const parser = rubyParser();
+    const results = new TestResults();
+    for (const l of collect([REPORTER_STREAM]).lines) results.apply(parser.feed(l));
+    // The console's own text arrives too: the same results, printed.
+    for (const l of ["MixedTest#test_pass = 0.00 s = .", "1 runs, 1 assertions, 0 failures, 0 errors, 0 skips"])
+      results.apply(parser.feed(l));
+    const node = (name: string) => results.get(["MixedTest", name].join("\u0000"))!;
+
+    it("builds the tree with statuses, times and failure text", () => {
+      expect(nodeStatus(node("test_pass"))).toBe("passed");
+      expect(nodeStatus(node("test_fail"))).toBe("failed");
+      expect(nodeStatus(node("test_skip"))).toBe("skipped");
+      expect(node("test_fail").output).toEqual(["Expected: 4", "  Actual: 3"]);
+      expect(node("test_pass").durationMs).toBeCloseTo(0.063, 3);
+    });
+
+    it("gives every test its real file and line, passed or not", () => {
+      for (const [name, line] of [
+        ["test_pass", 4],
+        ["test_fail", 5],
+        ["test_prints", 8],
+        ["test_skip", 7],
+      ] as const)
+        expect(node(name).location).toEqual({ path: F, line, resolve: "path" });
+    });
+
+    it("summarises from the reporter, and ignores the repeated text", () => {
+      expect(results.summary).toMatchObject({ passed: 2, failed: 1, skipped: 1 });
+      expect(results.get("MixedTest")!.children).toHaveLength(4);
+    });
+  });
+
+  it("a real location replaces the place to search for", () => {
+    const results = new TestResults();
+    results.apply(parseAll(rubyParser(), "MixedTest#test_pass = 0.00 s = ."));
+    const id = ["MixedTest", "test_pass"].join("\u0000");
+    expect(results.get(id)!.location?.resolve).toBe("rubyTest");
+    results.apply([
+      { type: "location", path: ["MixedTest", "test_pass"], location: { path: F, line: 4, resolve: "path" } },
+    ]);
+    expect(results.get(id)!.location).toEqual({ path: F, line: 4, resolve: "path" });
+  });
+});

src/features/run/testing/ruby.ts

@@ -15,7 +15,7 @@
  * 7 runs, 5 assertions, 1 failures, 1 errors, 1 skips
  * ```
  */
-import { parseDuration, type TestEvent, type TestOutputParser, type TestStatus } from "./types";
+import { parseDuration, TestStdout, type TestEvent, type TestOutputParser, type TestStatus } from "./types";
 
 // A class may be a spec description with spaces (`Spec style#test_0001_works`).
 const RESULT_RE = /^(.+?)#(\S+) = ([\d.]+ s) = ([.FES])$/;
@@ -44,7 +44,81 @@ const located = (path: string[]): TestEvent => ({
   location: { path: path[1]!, className: path[0], resolve: "rubyTest" },
 });
 
+/** Starts a record of the IDE's own Minitest reporter (`ide-run/src/ruby_reporter`); a JSON object follows. */
+export const RECORD_MARK = "@@OPENIDEA@@";
+
+interface ReporterRecord {
+  t: "start" | "result" | "summary";
+  c?: string;
+  n?: string;
+  s?: "passed" | "failed" | "skipped";
+  /** Seconds. */
+  d?: number | null;
+  f?: string | null;
+  l?: number | null;
+  m?: string;
+  runs?: number;
+  failures?: number;
+  errors?: number;
+  skips?: number;
+}
+
+/** What a reporter record says: the test's real file and line, its status, time and failure text. */
+function fromRecord(json: string): TestEvent[] {
+  let r: ReporterRecord;
+  try {
+    r = JSON.parse(json) as ReporterRecord;
+  } catch {
+    return [];
+  }
+  if (r.t === "summary") {
+    const failed = (r.failures ?? 0) + (r.errors ?? 0);
+    const skipped = r.skips ?? 0;
+    return [
+      {
+        type: "summary",
+        passed: Math.max(0, (r.runs ?? 0) - failed - skipped),
+        failed,
+        skipped,
+        durationMs: typeof r.d === "number" ? Math.round(r.d * 1000 * 1000) / 1000 : undefined,
+      },
+    ];
+  }
+  if (!r.c || !r.n) return [];
+  const path = [r.c, r.n];
+  const events: TestEvent[] = [];
+  if (r.t === "start") events.push({ type: "start", path });
+  if (r.t === "result" && r.s) {
+    events.push({
+      type: "result",
+      path,
+      status: r.s,
+      durationMs: typeof r.d === "number" ? Math.round(r.d * 1000 * 1000) / 1000 : undefined,
+    });
+    for (const line of (r.m ?? "").split("\n")) if (line) events.push({ type: "output", path, line });
+  }
+  if (r.f && r.l) {
+    events.push({ type: "location", path, location: { path: r.f, line: r.l, resolve: "path" } });
+  }
+  return events;
+}
+
+/**
+ * The console side of the reporter: a record ends with its own newline, so what the line holds before the
+ * marker (a progress dot) is all the console shows of it.
+ */
+export class RubyTestOutput extends TestStdout {
+  constructor(feedLine: (line: string) => void) {
+    super(feedLine, (line) => {
+      const at = line.indexOf(RECORD_MARK);
+      return at < 0 ? line + "\n" : line.slice(0, at);
+    });
+  }
+}
+
 export function rubyParser(): TestOutputParser {
+  /** The reporter's records are the source of results; the text is then ignored (it would repeat them). */
+  let records = false;
   const seen = new Set<string>();
   /** The test's own location, once. */
   const locate = (path: string[]): TestEvent[] => {
@@ -60,6 +134,12 @@ export function rubyParser(): TestOutputParser {
 
   return {
     feed(line) {
+      const mark = line.indexOf(RECORD_MARK);
+      if (mark >= 0) {
+        records = true;
+        return fromRecord(line.slice(mark + RECORD_MARK.length));
+      }
+      if (records) return [];
       const events: TestEvent[] = [];
       let m: RegExpExecArray | null;