feat(run): load a Minitest reporter into Ruby test runs (#1547)

Parsing Minitest's console output needs -v, gives no file or line for passing tests and depends on how output interleaves. Ruby test runs (ruby and rake test configurations in the console) now load a small Minitest reporter through RUBYLIB/RUBYOPT that prints one record per test on stdout with its status, time, failure text and real file and line. The console strips the records and the test tree is built from them, so jump to source gets exact locations; the text parser stays as the fallback. It works on Minitest 5 and 6 (6 no longer loads plugins on its own, so a small hook registers the reporter once Minitest is loaded).

Co-authored-by: kjeldahl <[email protected]>

crates/ide-host/src/ops/run.rs

@@ -198,6 +198,7 @@ pub async fn start(host: &Host, window: &str, config: RunConfig, on_output: Emit
     let options = StartOptions {
         encoding: Some(encoding).filter(|e| !e.trim().is_empty()),
         python_paths: if *root != project.root { vec![root.clone()] } else { super::python::paths(&project) },
+        ruby_reporter: ruby_reporter_dir(host, &config),
     };
     project.require_trust()?;
     let mut sink = project.sink.clone();
@@ -214,6 +215,21 @@ pub async fn start(host: &Host, window: &str, config: RunConfig, on_output: Emit
     host.processes.start(&project.root, &config, &options, sink, Box::new(move |c| on_output(c)))
 }
 
+/// The Ruby test reporter's directory, installed on demand, for runs that take it.
+fn ruby_reporter_dir(host: &Host, config: &RunConfig) -> Option<std::path::PathBuf> {
+    if !ide_run::ruby_reporter::applies(config) {
+        return None;
+    }
+    let dir = host.local_data_dir.join("ruby-reporter");
+    match ide_run::ruby_reporter::install(&dir) {
+        Ok(()) => Some(dir),
+        Err(e) => {
+            tracing::warn!("cannot install the Ruby test reporter in {}: {e}", dir.display());
+            None
+        }
+    }
+}
+
 /// Stops a run (interrupt, then kill after Settings › Run's termination timeout).
 pub fn stop(host: &Host, window: &str, id: RunId) -> IdeResult<()> {
     let settings = match host.project(window) {

crates/ide-run/src/lib.rs

@@ -13,6 +13,7 @@ pub mod junit;
 mod process;
 pub mod python_path;
 pub mod ruby_preload;
+pub mod ruby_reporter;
 pub mod tomcat;
 mod utf8;
 pub mod which;

crates/ide-run/src/process.rs

@@ -80,6 +80,8 @@ pub struct StartOptions {
     pub encoding: Option<String>,
     /// Project content and source roots added to Python launch environments.
     pub python_paths: Vec<PathBuf>,
+    /// Where the Ruby test reporter is installed (see [`crate::ruby_reporter`]); Ruby test runs load it.
+    pub ruby_reporter: Option<PathBuf>,
 }
 
 #[derive(Default)]
@@ -355,6 +357,10 @@ impl ProcessManager {
         if matches!(config.kind.as_str(), "python" | "pytest" | "django-test") {
             crate::python_path::extend(&mut env.vars, &options.python_paths, !env.clear)?;
         }
+        if let Some(dir) = options.ruby_reporter.as_deref().filter(|_| crate::ruby_reporter::applies(config)) {
+            crate::ruby_preload::add(&mut env.vars, dir, crate::ruby_reporter::FEATURE, !env.clear);
+            env.vars.insert("OPENIDEA_TEST_REPORT".into(), "1".into());
+        }
         let on_output: OutputFn = Arc::from(on_output);
         let on_output = match extras.output_file.as_deref() {
             Some(file) => tee_output(&resolve_cwd(root, Some(Path::new(file))), on_output)?,

crates/ide-run/src/ruby_reporter.rs

@@ -0,0 +1,52 @@
+//! A Minitest reporter the IDE loads into test runs (see `ruby_reporter/`): it prints one record per
+//! test on stdout, which the IDE strips from the console and turns into the test tree. Tests then
+//! report their real source file and line, and the output no longer needs `-v` to be parsed.
+use crate::RunConfig;
+use std::io;
+use std::path::Path;
+
+/// The Ruby file `RUBYOPT` loads first (`<dir>/openidea_test_hook.rb`).
+pub const FEATURE: &str = "openidea_test_hook";
+
+const FILES: [(&str, &str); 2] = [
+    ("openidea_test_hook.rb", include_str!("ruby_reporter/openidea_test_hook.rb")),
+    ("openidea/minitest_reporter.rb", include_str!("ruby_reporter/minitest_reporter.rb")),
+];
+
+/// Whether a run takes the reporter: Minitest-style test runs through `ruby` or `rake`, with their output
+/// in the console (a terminal would show the records).
+pub fn applies(config: &RunConfig) -> bool {
+    config.is_test && matches!(config.kind.as_str(), "ruby" | "rake") && !config.extras.emulate_terminal
+}
+
+/// Writes the reporter files under `dir` (unchanged files stay).
+pub fn install(dir: &Path) -> io::Result<()> {
+    for (name, text) in FILES {
+        let path = dir.join(name);
+        if std::fs::read_to_string(&path).is_ok_and(|current| current == text) {
+            continue;
+        }
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(&path, text)?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn installs_the_files_once_and_only_for_ruby_test_runs() {
+        let tmp = tempfile::tempdir().unwrap();
+        let dir = tmp.path().join("ruby reporter");
+        install(&dir).unwrap();
+        install(&dir).unwrap();
+        for (name, text) in FILES {
+            assert_eq!(std::fs::read_to_string(dir.join(name)).unwrap(), text);
+        }
+        assert!(FILES[0].1.contains("OPENIDEA_TEST_REPORT") && FILES[1].1.contains("@@OPENIDEA@@"));
+    }
+}

crates/ide-run/src/ruby_reporter/minitest_reporter.rb

@@ -0,0 +1,78 @@
+# Written by OpenIDEA for test runs; registered by `openidea_test_hook.rb` once Minitest is loaded.
+# Nothing in the project is changed.
+#
+# One record per line on stdout, which the IDE strips from the console again:
+#   @@OPENIDEA@@{"t":"start","c":"MyTest","n":"test_x","f":"/abs/test.rb","l":12}
+#   @@OPENIDEA@@{"t":"result","c":"MyTest","n":"test_x","s":"passed","d":0.001,"f":...,"l":...,"m":"failure text"}
+#   @@OPENIDEA@@{"t":"summary","runs":7,"failures":1,"errors":1,"skips":1,"d":0.4}
+require "json"
+
+module Minitest
+  class OpenIdeaReporter < AbstractReporter
+    MARK = "@@OPENIDEA@@"
+
+    def initialize(io)
+      super()
+      @io = io
+      @cwd = Dir.pwd
+      @lock = Mutex.new
+      @runs = @failures = @errors = @skips = 0
+      @started = nil
+    end
+
+    def start
+      @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def prerecord(klass, name)
+      emit(t: "start", c: klass.to_s, n: name.to_s, **where(klass, name))
+    end
+
+    def record(result)
+      @runs += 1
+      status = result.skipped? ? "skipped" : result.passed? ? "passed" : "failed"
+      @skips += 1 if status == "skipped"
+      if status == "failed"
+        result.failures.any? { |f| f.is_a?(UnexpectedError) } ? @errors += 1 : @failures += 1
+      end
+      loc = result.source_location
+      emit(t: "result", c: result.klass.to_s, n: result.name.to_s, s: status, d: result.time,
+           f: loc && absolute(loc[0]), l: loc && loc[1], m: result.failures.map(&:message).join("\n"))
+    end
+
+    def report
+      d = @started ? Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started : nil
+      emit(t: "summary", runs: @runs, failures: @failures, errors: @errors, skips: @skips, d: d)
+    end
+
+    private
+
+    # Where the test is defined, known before it runs (so the tree can show it while running).
+    def where(klass, name)
+      f, l = klass.instance_method(name).source_location
+      { f: absolute(f), l: l }
+    rescue StandardError
+      {}
+    end
+
+    # Test files are often loaded by a path relative to the working directory.
+    def absolute(file)
+      File.expand_path(file, @cwd)
+    end
+
+    def emit(record)
+      line = "#{MARK}#{JSON.generate(record)}\n"
+      @lock.synchronize { @io.write(line) }
+    rescue StandardError
+      nil # reporting must never fail a test run
+    end
+  end
+
+  def self.plugin_openidea_init(options)
+    return unless ENV["OPENIDEA_TEST_REPORT"]
+
+    io = options[:io] || $stdout
+    io.sync = true if io.respond_to?(:sync=)
+    reporter << OpenIdeaReporter.new(io)
+  end
+end

crates/ide-run/src/ruby_reporter/openidea_test_hook.rb

@@ -0,0 +1,32 @@
+# Written by OpenIDEA for test runs. Loaded first in every Ruby the run starts
+# (RUBYLIB=<this directory>, RUBYOPT=-ropenidea_test_hook).
+#
+# Registers the test reporter once a test framework is loaded. Minitest 5 would find a plugin on
+# its own, Minitest 6 does not load plugins any more, so both are registered the same way.
+if ENV["OPENIDEA_TEST_REPORT"]
+  module OpenIdeaTestHook
+    @registered = false
+
+    def self.register
+      return if @registered
+      return unless defined?(::Minitest) && ::Minitest.respond_to?(:extensions) && defined?(::Minitest::AbstractReporter)
+
+      @registered = true
+      require "openidea/minitest_reporter"
+      ::Minitest.extensions << "openidea" unless ::Minitest.extensions.include?("openidea")
+    rescue StandardError, LoadError
+      nil # reporting must never break a test run
+    end
+
+    # `require "minitest/autorun"` loads Minitest through require_relative, so look after every require.
+    module Watch
+      def require(path)
+        result = super
+        OpenIdeaTestHook.register
+        result
+      end
+    end
+  end
+
+  Kernel.prepend(OpenIdeaTestHook::Watch)
+end

crates/ide-run/tests/process.rs

@@ -479,6 +479,36 @@ fn python_scripts_import_sibling_packages_and_preserve_env_file_paths() {
     }
 }
 
+#[test]
+fn ruby_test_runs_load_the_reporter_and_other_runs_do_not() {
+    let reporter = std::env::temp_dir().join("ruby reporter");
+    let run = |kind: &str, is_test: bool| {
+        let h = harness();
+        let mut config =
+            cfg("ruby-env", "/bin/sh", &["-c", r#"echo "lib=$RUBYLIB opt=$RUBYOPT report=$OPENIDEA_TEST_REPORT""#]);
+        config.kind = kind.into();
+        config.is_test = is_test;
+        let options = StartOptions { ruby_reporter: Some(reporter.clone()), ..Default::default() };
+        let chunks: Chunks = Arc::default();
+        let sink = chunks.clone();
+        let info =
+            h.pm.start(h.root.path(), &config, &options, h.sink.clone(), Box::new(move |c| sink.lock().push(c)))
+                .unwrap();
+        assert_eq!(wait_exit(&h, info.id, Duration::from_secs(10)), Some(0));
+        text(&chunks, stdout)
+    };
+    for kind in ["ruby", "rake"] {
+        let out = run(kind, true);
+        // The directory has a space: it is on RUBYLIB and only the bare feature name is in RUBYOPT.
+        assert!(out.contains(&format!("lib={}", reporter.display())), "{out}");
+        assert!(out.contains("-ropenidea_test_hook") && !out.contains(&format!("-r{}", reporter.display())), "{out}");
+        assert!(out.contains("report=1"), "{out}");
+    }
+    for (kind, is_test) in [("ruby", false), ("rspec", true), ("pytest", true)] {
+        assert!(run(kind, is_test).contains("report=\n"), "{kind} {is_test}");
+    }
+}
+
 #[test]
 fn revoking_one_project_stops_only_its_runs() {
     let h = harness();

src/features/run/store.ts

@@ -43,6 +43,7 @@ import {
 } from "./testing";
 import { GoTestOutput } from "./testing/go";
 import { moveToReportedClasses } from "./testing/jvm";
+import { RubyTestOutput } from "./testing/ruby";
 import { TeamcityOutput } from "./testing/teamcity";
 import type { TestStdout } from "./testing/types";
 import { recordTestStates } from "./testing/states";
@@ -87,7 +88,7 @@ interface SessionRuntime {
   console: ConsoleModel;
   tests: TestResults | null;
   parser: TestOutputParser | null;
-  /** Machine-readable stdout (`go test -json`, `--teamcity`): fed to the parser, shown decoded. */
+  /** Machine-readable stdout (`go test -json`, `--teamcity`, the Ruby reporter): fed to the parser, shown decoded. */
   stdout: TestStdout | null;
   queue: OutputChunk[];
   /** rAF id while the page is visible, timeout id while hidden (rAF pauses in background tabs). */
@@ -567,13 +568,16 @@ function newRuntime(config: RunConfig): SessionRuntime {
     if (tests && parser) tests.apply(parser.feed(line));
   };
   const framework = tests && parser ? frameworkFor(config) : null;
-  // PHPUnit `--teamcity` and Gradle's test init script report through service messages.
+  // PHPUnit `--teamcity` and Gradle's test init script report through service messages, Minitest through
+  // the IDE's own reporter (RSpec has none: its progress dots must show as they come).
   const stdout =
     framework === "go"
       ? new GoTestOutput(feed)
       : framework === "phpunit" || framework === "jvm"
         ? new TeamcityOutput(feed)
-        : null;
+        : framework === "ruby" && config.kind !== "rspec"
+          ? new RubyTestOutput(feed)
+          : null;
   const runtime: SessionRuntime = {
     stdout,
     tests,

src/features/run/testing/results.ts

@@ -128,7 +128,9 @@ export class TestResults {
         }
         case "location": {
           const n = this.node(e.path);
-          n.location ??= e.location;
+          // The first location wins, except that a place to search for (Ruby tests) gives way to a real one.
+          if (!n.location || (n.location.resolve === "rubyTest" && e.location.resolve !== "rubyTest"))
+            n.location = e.location;
           break;
         }
         case "output": {

src/features/run/testing/ruby.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from "vitest";
 import { parseAll, withTestReporting, frameworkFor } from "./index";
 import { TestResults, nodeStatus } from "./results";
-import { rubyParser } from "./ruby";
+import { RubyTestOutput, rubyParser } from "./ruby";
 
 // Captured from `ruby -Ilib test/mixed_test.rb -v --seed 1` (Minitest 6.0.6).
 const MINITEST = `Run options: -v --seed 1
@@ -125,3 +125,98 @@ describe("Ruby test reporting", () => {
     expect(frameworkFor(cfg("rake", "bundle", ["exec", "rake", "test"]))).toBe("ruby");
   });
 });
+
+// Captured from a run with the IDE's reporter (Minitest 6.0.6): records end with their own newline, so the
+// progress dots stay on one line once they are taken out.
+const M = "@@OPENIDEA@@";
+const F = "/p/test/mixed_test.rb";
+const REPORTER_STREAM = [
+  "# Running:",
+  "",
+  `${M}{"t":"start","c":"MixedTest","n":"test_pass","f":"${F}","l":4}`,
+  `.${M}{"t":"result","c":"MixedTest","n":"test_pass","s":"passed","d":0.000063,"f":"${F}","l":4,"m":""}`,
+  `${M}{"t":"start","c":"MixedTest","n":"test_fail","f":"${F}","l":5}`,
+  `F${M}{"t":"result","c":"MixedTest","n":"test_fail","s":"failed","d":0.000054,"f":"${F}","l":5,"m":"Expected: 4\\n  Actual: 3"}`,
+  `${M}{"t":"start","c":"MixedTest","n":"test_prints","f":"${F}","l":8}`,
+  "hello from test",
+  `.${M}{"t":"result","c":"MixedTest","n":"test_prints","s":"passed","d":0.000007,"f":"${F}","l":8,"m":""}`,
+  `${M}{"t":"start","c":"MixedTest","n":"test_skip","f":"${F}","l":7}`,
+  `S${M}{"t":"result","c":"MixedTest","n":"test_skip","s":"skipped","d":0.000006,"f":"${F}","l":7,"m":"later"}`,
+  "",
+  `${M}{"t":"summary","runs":4,"failures":1,"errors":0,"skips":1,"d":0.0011}`,
+  "Finished in 0.001124s",
+  "",
+].join("\n");
+
+describe("the IDE's Minitest reporter", () => {
+  const collect = (chunks: string[]) => {
+    const lines: string[] = [];
+    const out = new RubyTestOutput((line) => lines.push(line));
+    const text = chunks.map((c) => out.feed(c)).join("") + out.end();
+    return { text, records: lines.filter((l) => l.includes(M)), lines };
+  };
+
+  it("takes the records out of the console, exactly as Minitest prints them without it", () => {
+    const { text, records } = collect([REPORTER_STREAM]);
+    expect(text).toBe("# Running:\n\n.Fhello from test\n.S\nFinished in 0.001124s\n");
+    expect(records).toHaveLength(9);
+    expect(records.every((r) => r.includes(M))).toBe(true);
+  });
+
+  it("is the same however the output is cut into chunks", () => {
+    const whole = collect([REPORTER_STREAM]);
+    const chars = collect(REPORTER_STREAM.split(""));
+    const odd = collect(REPORTER_STREAM.match(/[\s\S]{1,7}/g)!);
+    expect(chars).toEqual(whole);
+    expect(odd).toEqual(whole);
+  });
+
+  it("drops a record cut off by the end of the process, not the text before it", () => {
+    expect(collect([`abc${M}{"t":"sta`]).text).toBe("abc");
+    expect(collect(["no marker @@OPEN"]).text).toBe("no marker @@OPEN\n");
+  });
+
+  describe("results", () => {
+    const parser = rubyParser();
+    const results = new TestResults();
+    for (const l of collect([REPORTER_STREAM]).lines) results.apply(parser.feed(l));
+    // The console's own text arrives too: the same results, printed.
+    for (const l of ["MixedTest#test_pass = 0.00 s = .", "1 runs, 1 assertions, 0 failures, 0 errors, 0 skips"])
+      results.apply(parser.feed(l));
+    const node = (name: string) => results.get(["MixedTest", name].join("\u0000"))!;
+
+    it("builds the tree with statuses, times and failure text", () => {
+      expect(nodeStatus(node("test_pass"))).toBe("passed");
+      expect(nodeStatus(node("test_fail"))).toBe("failed");
+      expect(nodeStatus(node("test_skip"))).toBe("skipped");
+      expect(node("test_fail").output).toEqual(["Expected: 4", "  Actual: 3"]);
+      expect(node("test_pass").durationMs).toBeCloseTo(0.063, 3);
+    });
+
+    it("gives every test its real file and line, passed or not", () => {
+      for (const [name, line] of [
+        ["test_pass", 4],
+        ["test_fail", 5],
+        ["test_prints", 8],
+        ["test_skip", 7],
+      ] as const)
+        expect(node(name).location).toEqual({ path: F, line, resolve: "path" });
+    });
+
+    it("summarises from the reporter, and ignores the repeated text", () => {
+      expect(results.summary).toMatchObject({ passed: 2, failed: 1, skipped: 1 });
+      expect(results.get("MixedTest")!.children).toHaveLength(4);
+    });
+  });
+
+  it("a real location replaces the place to search for", () => {
+    const results = new TestResults();
+    results.apply(parseAll(rubyParser(), "MixedTest#test_pass = 0.00 s = ."));
+    const id = ["MixedTest", "test_pass"].join("\u0000");
+    expect(results.get(id)!.location?.resolve).toBe("rubyTest");
+    results.apply([
+      { type: "location", path: ["MixedTest", "test_pass"], location: { path: F, line: 4, resolve: "path" } },
+    ]);
+    expect(results.get(id)!.location).toEqual({ path: F, line: 4, resolve: "path" });
+  });
+});

src/features/run/testing/ruby.ts

@@ -15,7 +15,7 @@
  * 7 runs, 5 assertions, 1 failures, 1 errors, 1 skips
  * ```
  */
-import { parseDuration, type TestEvent, type TestOutputParser, type TestStatus } from "./types";
+import { parseDuration, TestStdout, type TestEvent, type TestOutputParser, type TestStatus } from "./types";
 
 // A class may be a spec description with spaces (`Spec style#test_0001_works`).
 const RESULT_RE = /^(.+?)#(\S+) = ([\d.]+ s) = ([.FES])$/;
@@ -44,7 +44,81 @@ const located = (path: string[]): TestEvent => ({
   location: { path: path[1]!, className: path[0], resolve: "rubyTest" },
 });
 
+/** Starts a record of the IDE's own Minitest reporter (`ide-run/src/ruby_reporter`); a JSON object follows. */
+export const RECORD_MARK = "@@OPENIDEA@@";
+
+interface ReporterRecord {
+  t: "start" | "result" | "summary";
+  c?: string;
+  n?: string;
+  s?: "passed" | "failed" | "skipped";
+  /** Seconds. */
+  d?: number | null;
+  f?: string | null;
+  l?: number | null;
+  m?: string;
+  runs?: number;
+  failures?: number;
+  errors?: number;
+  skips?: number;
+}
+
+/** What a reporter record says: the test's real file and line, its status, time and failure text. */
+function fromRecord(json: string): TestEvent[] {
+  let r: ReporterRecord;
+  try {
+    r = JSON.parse(json) as ReporterRecord;
+  } catch {
+    return [];
+  }
+  if (r.t === "summary") {
+    const failed = (r.failures ?? 0) + (r.errors ?? 0);
+    const skipped = r.skips ?? 0;
+    return [
+      {
+        type: "summary",
+        passed: Math.max(0, (r.runs ?? 0) - failed - skipped),
+        failed,
+        skipped,
+        durationMs: typeof r.d === "number" ? Math.round(r.d * 1000 * 1000) / 1000 : undefined,
+      },
+    ];
+  }
+  if (!r.c || !r.n) return [];
+  const path = [r.c, r.n];
+  const events: TestEvent[] = [];
+  if (r.t === "start") events.push({ type: "start", path });
+  if (r.t === "result" && r.s) {
+    events.push({
+      type: "result",
+      path,
+      status: r.s,
+      durationMs: typeof r.d === "number" ? Math.round(r.d * 1000 * 1000) / 1000 : undefined,
+    });
+    for (const line of (r.m ?? "").split("\n")) if (line) events.push({ type: "output", path, line });
+  }
+  if (r.f && r.l) {
+    events.push({ type: "location", path, location: { path: r.f, line: r.l, resolve: "path" } });
+  }
+  return events;
+}
+
+/**
+ * The console side of the reporter: a record ends with its own newline, so what the line holds before the
+ * marker (a progress dot) is all the console shows of it.
+ */
+export class RubyTestOutput extends TestStdout {
+  constructor(feedLine: (line: string) => void) {
+    super(feedLine, (line) => {
+      const at = line.indexOf(RECORD_MARK);
+      return at < 0 ? line + "\n" : line.slice(0, at);
+    });
+  }
+}
+
 export function rubyParser(): TestOutputParser {
+  /** The reporter's records are the source of results; the text is then ignored (it would repeat them). */
+  let records = false;
   const seen = new Set<string>();
   /** The test's own location, once. */
   const locate = (path: string[]): TestEvent[] => {
@@ -60,6 +134,12 @@ export function rubyParser(): TestOutputParser {
 
   return {
     feed(line) {
+      const mark = line.indexOf(RECORD_MARK);
+      if (mark >= 0) {
+        records = true;
+        return fromRecord(line.slice(mark + RECORD_MARK.length));
+      }
+      if (records) return [];
       const events: TestEvent[] = [];
       let m: RegExpExecArray | null;