diff --git a/crates/ide-host/src/ops/run.rs b/crates/ide-host/src/ops/run.rs index a71a8da5e..7409f0a32 100644 --- a/crates/ide-host/src/ops/run.rs +++ b/crates/ide-host/src/ops/run.rs @@ -198,6 +198,7 @@ pub async fn start(host: &Host, window: &str, config: RunConfig, on_output: Emit let options = StartOptions { encoding: Some(encoding).filter(|e| !e.trim().is_empty()), python_paths: if *root != project.root { vec![root.clone()] } else { super::python::paths(&project) }, + ruby_reporter: ruby_reporter_dir(host, &config), }; project.require_trust()?; let mut sink = project.sink.clone(); @@ -214,6 +215,21 @@ pub async fn start(host: &Host, window: &str, config: RunConfig, on_output: Emit host.processes.start(&project.root, &config, &options, sink, Box::new(move |c| on_output(c))) } +/// The Ruby test reporter's directory, installed on demand, for runs that take it. +fn ruby_reporter_dir(host: &Host, config: &RunConfig) -> Option { + if !ide_run::ruby_reporter::applies(config) { + return None; + } + let dir = host.local_data_dir.join("ruby-reporter"); + match ide_run::ruby_reporter::install(&dir) { + Ok(()) => Some(dir), + Err(e) => { + tracing::warn!("cannot install the Ruby test reporter in {}: {e}", dir.display()); + None + } + } +} + /// Stops a run (interrupt, then kill after Settings › Run's termination timeout). pub fn stop(host: &Host, window: &str, id: RunId) -> IdeResult<()> { let settings = match host.project(window) { diff --git a/crates/ide-run/src/lib.rs b/crates/ide-run/src/lib.rs index 97e313423..4b2493041 100644 --- a/crates/ide-run/src/lib.rs +++ b/crates/ide-run/src/lib.rs @@ -13,6 +13,7 @@ pub mod junit; mod process; pub mod python_path; pub mod ruby_preload; +pub mod ruby_reporter; pub mod tomcat; mod utf8; pub mod which; diff --git a/crates/ide-run/src/process.rs b/crates/ide-run/src/process.rs index 2899c026a..9deb76d46 100644 --- a/crates/ide-run/src/process.rs +++ b/crates/ide-run/src/process.rs @@ -80,6 +80,8 @@ pub struct StartOptions { pub encoding: Option, /// Project content and source roots added to Python launch environments. pub python_paths: Vec, + /// Where the Ruby test reporter is installed (see [`crate::ruby_reporter`]); Ruby test runs load it. + pub ruby_reporter: Option, } #[derive(Default)] @@ -355,6 +357,10 @@ impl ProcessManager { if matches!(config.kind.as_str(), "python" | "pytest" | "django-test") { crate::python_path::extend(&mut env.vars, &options.python_paths, !env.clear)?; } + if let Some(dir) = options.ruby_reporter.as_deref().filter(|_| crate::ruby_reporter::applies(config)) { + crate::ruby_preload::add(&mut env.vars, dir, crate::ruby_reporter::FEATURE, !env.clear); + env.vars.insert("OPENIDEA_TEST_REPORT".into(), "1".into()); + } let on_output: OutputFn = Arc::from(on_output); let on_output = match extras.output_file.as_deref() { Some(file) => tee_output(&resolve_cwd(root, Some(Path::new(file))), on_output)?, diff --git a/crates/ide-run/src/ruby_reporter.rs b/crates/ide-run/src/ruby_reporter.rs new file mode 100644 index 000000000..c689d4ddd --- /dev/null +++ b/crates/ide-run/src/ruby_reporter.rs @@ -0,0 +1,52 @@ +//! A Minitest reporter the IDE loads into test runs (see `ruby_reporter/`): it prints one record per +//! test on stdout, which the IDE strips from the console and turns into the test tree. Tests then +//! report their real source file and line, and the output no longer needs `-v` to be parsed. +use crate::RunConfig; +use std::io; +use std::path::Path; + +/// The Ruby file `RUBYOPT` loads first (`/openidea_test_hook.rb`). +pub const FEATURE: &str = "openidea_test_hook"; + +const FILES: [(&str, &str); 2] = [ + ("openidea_test_hook.rb", include_str!("ruby_reporter/openidea_test_hook.rb")), + ("openidea/minitest_reporter.rb", include_str!("ruby_reporter/minitest_reporter.rb")), +]; + +/// Whether a run takes the reporter: Minitest-style test runs through `ruby` or `rake`, with their output +/// in the console (a terminal would show the records). +pub fn applies(config: &RunConfig) -> bool { + config.is_test && matches!(config.kind.as_str(), "ruby" | "rake") && !config.extras.emulate_terminal +} + +/// Writes the reporter files under `dir` (unchanged files stay). +pub fn install(dir: &Path) -> io::Result<()> { + for (name, text) in FILES { + let path = dir.join(name); + if std::fs::read_to_string(&path).is_ok_and(|current| current == text) { + continue; + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, text)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn installs_the_files_once_and_only_for_ruby_test_runs() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("ruby reporter"); + install(&dir).unwrap(); + install(&dir).unwrap(); + for (name, text) in FILES { + assert_eq!(std::fs::read_to_string(dir.join(name)).unwrap(), text); + } + assert!(FILES[0].1.contains("OPENIDEA_TEST_REPORT") && FILES[1].1.contains("@@OPENIDEA@@")); + } +} diff --git a/crates/ide-run/src/ruby_reporter/minitest_reporter.rb b/crates/ide-run/src/ruby_reporter/minitest_reporter.rb new file mode 100644 index 000000000..a657ad4e9 --- /dev/null +++ b/crates/ide-run/src/ruby_reporter/minitest_reporter.rb @@ -0,0 +1,78 @@ +# Written by OpenIDEA for test runs; registered by `openidea_test_hook.rb` once Minitest is loaded. +# Nothing in the project is changed. +# +# One record per line on stdout, which the IDE strips from the console again: +# @@OPENIDEA@@{"t":"start","c":"MyTest","n":"test_x","f":"/abs/test.rb","l":12} +# @@OPENIDEA@@{"t":"result","c":"MyTest","n":"test_x","s":"passed","d":0.001,"f":...,"l":...,"m":"failure text"} +# @@OPENIDEA@@{"t":"summary","runs":7,"failures":1,"errors":1,"skips":1,"d":0.4} +require "json" + +module Minitest + class OpenIdeaReporter < AbstractReporter + MARK = "@@OPENIDEA@@" + + def initialize(io) + super() + @io = io + @cwd = Dir.pwd + @lock = Mutex.new + @runs = @failures = @errors = @skips = 0 + @started = nil + end + + def start + @started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def prerecord(klass, name) + emit(t: "start", c: klass.to_s, n: name.to_s, **where(klass, name)) + end + + def record(result) + @runs += 1 + status = result.skipped? ? "skipped" : result.passed? ? "passed" : "failed" + @skips += 1 if status == "skipped" + if status == "failed" + result.failures.any? { |f| f.is_a?(UnexpectedError) } ? @errors += 1 : @failures += 1 + end + loc = result.source_location + emit(t: "result", c: result.klass.to_s, n: result.name.to_s, s: status, d: result.time, + f: loc && absolute(loc[0]), l: loc && loc[1], m: result.failures.map(&:message).join("\n")) + end + + def report + d = @started ? Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started : nil + emit(t: "summary", runs: @runs, failures: @failures, errors: @errors, skips: @skips, d: d) + end + + private + + # Where the test is defined, known before it runs (so the tree can show it while running). + def where(klass, name) + f, l = klass.instance_method(name).source_location + { f: absolute(f), l: l } + rescue StandardError + {} + end + + # Test files are often loaded by a path relative to the working directory. + def absolute(file) + File.expand_path(file, @cwd) + end + + def emit(record) + line = "#{MARK}#{JSON.generate(record)}\n" + @lock.synchronize { @io.write(line) } + rescue StandardError + nil # reporting must never fail a test run + end + end + + def self.plugin_openidea_init(options) + return unless ENV["OPENIDEA_TEST_REPORT"] + + io = options[:io] || $stdout + io.sync = true if io.respond_to?(:sync=) + reporter << OpenIdeaReporter.new(io) + end +end diff --git a/crates/ide-run/src/ruby_reporter/openidea_test_hook.rb b/crates/ide-run/src/ruby_reporter/openidea_test_hook.rb new file mode 100644 index 000000000..57a72084e --- /dev/null +++ b/crates/ide-run/src/ruby_reporter/openidea_test_hook.rb @@ -0,0 +1,32 @@ +# Written by OpenIDEA for test runs. Loaded first in every Ruby the run starts +# (RUBYLIB=, RUBYOPT=-ropenidea_test_hook). +# +# Registers the test reporter once a test framework is loaded. Minitest 5 would find a plugin on +# its own, Minitest 6 does not load plugins any more, so both are registered the same way. +if ENV["OPENIDEA_TEST_REPORT"] + module OpenIdeaTestHook + @registered = false + + def self.register + return if @registered + return unless defined?(::Minitest) && ::Minitest.respond_to?(:extensions) && defined?(::Minitest::AbstractReporter) + + @registered = true + require "openidea/minitest_reporter" + ::Minitest.extensions << "openidea" unless ::Minitest.extensions.include?("openidea") + rescue StandardError, LoadError + nil # reporting must never break a test run + end + + # `require "minitest/autorun"` loads Minitest through require_relative, so look after every require. + module Watch + def require(path) + result = super + OpenIdeaTestHook.register + result + end + end + end + + Kernel.prepend(OpenIdeaTestHook::Watch) +end diff --git a/crates/ide-run/tests/process.rs b/crates/ide-run/tests/process.rs index 176be79c9..cbb2ba914 100644 --- a/crates/ide-run/tests/process.rs +++ b/crates/ide-run/tests/process.rs @@ -479,6 +479,36 @@ fn python_scripts_import_sibling_packages_and_preserve_env_file_paths() { } } +#[test] +fn ruby_test_runs_load_the_reporter_and_other_runs_do_not() { + let reporter = std::env::temp_dir().join("ruby reporter"); + let run = |kind: &str, is_test: bool| { + let h = harness(); + let mut config = + cfg("ruby-env", "/bin/sh", &["-c", r#"echo "lib=$RUBYLIB opt=$RUBYOPT report=$OPENIDEA_TEST_REPORT""#]); + config.kind = kind.into(); + config.is_test = is_test; + let options = StartOptions { ruby_reporter: Some(reporter.clone()), ..Default::default() }; + let chunks: Chunks = Arc::default(); + let sink = chunks.clone(); + let info = + h.pm.start(h.root.path(), &config, &options, h.sink.clone(), Box::new(move |c| sink.lock().push(c))) + .unwrap(); + assert_eq!(wait_exit(&h, info.id, Duration::from_secs(10)), Some(0)); + text(&chunks, stdout) + }; + for kind in ["ruby", "rake"] { + let out = run(kind, true); + // The directory has a space: it is on RUBYLIB and only the bare feature name is in RUBYOPT. + assert!(out.contains(&format!("lib={}", reporter.display())), "{out}"); + assert!(out.contains("-ropenidea_test_hook") && !out.contains(&format!("-r{}", reporter.display())), "{out}"); + assert!(out.contains("report=1"), "{out}"); + } + for (kind, is_test) in [("ruby", false), ("rspec", true), ("pytest", true)] { + assert!(run(kind, is_test).contains("report=\n"), "{kind} {is_test}"); + } +} + #[test] fn revoking_one_project_stops_only_its_runs() { let h = harness(); diff --git a/src/features/run/store.ts b/src/features/run/store.ts index 6c542050c..5ed72249f 100644 --- a/src/features/run/store.ts +++ b/src/features/run/store.ts @@ -43,6 +43,7 @@ import { } from "./testing"; import { GoTestOutput } from "./testing/go"; import { moveToReportedClasses } from "./testing/jvm"; +import { RubyTestOutput } from "./testing/ruby"; import { TeamcityOutput } from "./testing/teamcity"; import type { TestStdout } from "./testing/types"; import { recordTestStates } from "./testing/states"; @@ -87,7 +88,7 @@ interface SessionRuntime { console: ConsoleModel; tests: TestResults | null; parser: TestOutputParser | null; - /** Machine-readable stdout (`go test -json`, `--teamcity`): fed to the parser, shown decoded. */ + /** Machine-readable stdout (`go test -json`, `--teamcity`, the Ruby reporter): fed to the parser, shown decoded. */ stdout: TestStdout | null; queue: OutputChunk[]; /** rAF id while the page is visible, timeout id while hidden (rAF pauses in background tabs). */ @@ -567,13 +568,16 @@ function newRuntime(config: RunConfig): SessionRuntime { if (tests && parser) tests.apply(parser.feed(line)); }; const framework = tests && parser ? frameworkFor(config) : null; - // PHPUnit `--teamcity` and Gradle's test init script report through service messages. + // PHPUnit `--teamcity` and Gradle's test init script report through service messages, Minitest through + // the IDE's own reporter (RSpec has none: its progress dots must show as they come). const stdout = framework === "go" ? new GoTestOutput(feed) : framework === "phpunit" || framework === "jvm" ? new TeamcityOutput(feed) - : null; + : framework === "ruby" && config.kind !== "rspec" + ? new RubyTestOutput(feed) + : null; const runtime: SessionRuntime = { stdout, tests, diff --git a/src/features/run/testing/results.ts b/src/features/run/testing/results.ts index e0c75aca7..9b67583f9 100644 --- a/src/features/run/testing/results.ts +++ b/src/features/run/testing/results.ts @@ -128,7 +128,9 @@ export class TestResults { } case "location": { const n = this.node(e.path); - n.location ??= e.location; + // The first location wins, except that a place to search for (Ruby tests) gives way to a real one. + if (!n.location || (n.location.resolve === "rubyTest" && e.location.resolve !== "rubyTest")) + n.location = e.location; break; } case "output": { diff --git a/src/features/run/testing/ruby.test.ts b/src/features/run/testing/ruby.test.ts index 769ce3f27..1139bc54d 100644 --- a/src/features/run/testing/ruby.test.ts +++ b/src/features/run/testing/ruby.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { parseAll, withTestReporting, frameworkFor } from "./index"; import { TestResults, nodeStatus } from "./results"; -import { rubyParser } from "./ruby"; +import { RubyTestOutput, rubyParser } from "./ruby"; // Captured from `ruby -Ilib test/mixed_test.rb -v --seed 1` (Minitest 6.0.6). const MINITEST = `Run options: -v --seed 1 @@ -125,3 +125,98 @@ describe("Ruby test reporting", () => { expect(frameworkFor(cfg("rake", "bundle", ["exec", "rake", "test"]))).toBe("ruby"); }); }); + +// Captured from a run with the IDE's reporter (Minitest 6.0.6): records end with their own newline, so the +// progress dots stay on one line once they are taken out. +const M = "@@OPENIDEA@@"; +const F = "/p/test/mixed_test.rb"; +const REPORTER_STREAM = [ + "# Running:", + "", + `${M}{"t":"start","c":"MixedTest","n":"test_pass","f":"${F}","l":4}`, + `.${M}{"t":"result","c":"MixedTest","n":"test_pass","s":"passed","d":0.000063,"f":"${F}","l":4,"m":""}`, + `${M}{"t":"start","c":"MixedTest","n":"test_fail","f":"${F}","l":5}`, + `F${M}{"t":"result","c":"MixedTest","n":"test_fail","s":"failed","d":0.000054,"f":"${F}","l":5,"m":"Expected: 4\\n Actual: 3"}`, + `${M}{"t":"start","c":"MixedTest","n":"test_prints","f":"${F}","l":8}`, + "hello from test", + `.${M}{"t":"result","c":"MixedTest","n":"test_prints","s":"passed","d":0.000007,"f":"${F}","l":8,"m":""}`, + `${M}{"t":"start","c":"MixedTest","n":"test_skip","f":"${F}","l":7}`, + `S${M}{"t":"result","c":"MixedTest","n":"test_skip","s":"skipped","d":0.000006,"f":"${F}","l":7,"m":"later"}`, + "", + `${M}{"t":"summary","runs":4,"failures":1,"errors":0,"skips":1,"d":0.0011}`, + "Finished in 0.001124s", + "", +].join("\n"); + +describe("the IDE's Minitest reporter", () => { + const collect = (chunks: string[]) => { + const lines: string[] = []; + const out = new RubyTestOutput((line) => lines.push(line)); + const text = chunks.map((c) => out.feed(c)).join("") + out.end(); + return { text, records: lines.filter((l) => l.includes(M)), lines }; + }; + + it("takes the records out of the console, exactly as Minitest prints them without it", () => { + const { text, records } = collect([REPORTER_STREAM]); + expect(text).toBe("# Running:\n\n.Fhello from test\n.S\nFinished in 0.001124s\n"); + expect(records).toHaveLength(9); + expect(records.every((r) => r.includes(M))).toBe(true); + }); + + it("is the same however the output is cut into chunks", () => { + const whole = collect([REPORTER_STREAM]); + const chars = collect(REPORTER_STREAM.split("")); + const odd = collect(REPORTER_STREAM.match(/[\s\S]{1,7}/g)!); + expect(chars).toEqual(whole); + expect(odd).toEqual(whole); + }); + + it("drops a record cut off by the end of the process, not the text before it", () => { + expect(collect([`abc${M}{"t":"sta`]).text).toBe("abc"); + expect(collect(["no marker @@OPEN"]).text).toBe("no marker @@OPEN\n"); + }); + + describe("results", () => { + const parser = rubyParser(); + const results = new TestResults(); + for (const l of collect([REPORTER_STREAM]).lines) results.apply(parser.feed(l)); + // The console's own text arrives too: the same results, printed. + for (const l of ["MixedTest#test_pass = 0.00 s = .", "1 runs, 1 assertions, 0 failures, 0 errors, 0 skips"]) + results.apply(parser.feed(l)); + const node = (name: string) => results.get(["MixedTest", name].join("\u0000"))!; + + it("builds the tree with statuses, times and failure text", () => { + expect(nodeStatus(node("test_pass"))).toBe("passed"); + expect(nodeStatus(node("test_fail"))).toBe("failed"); + expect(nodeStatus(node("test_skip"))).toBe("skipped"); + expect(node("test_fail").output).toEqual(["Expected: 4", " Actual: 3"]); + expect(node("test_pass").durationMs).toBeCloseTo(0.063, 3); + }); + + it("gives every test its real file and line, passed or not", () => { + for (const [name, line] of [ + ["test_pass", 4], + ["test_fail", 5], + ["test_prints", 8], + ["test_skip", 7], + ] as const) + expect(node(name).location).toEqual({ path: F, line, resolve: "path" }); + }); + + it("summarises from the reporter, and ignores the repeated text", () => { + expect(results.summary).toMatchObject({ passed: 2, failed: 1, skipped: 1 }); + expect(results.get("MixedTest")!.children).toHaveLength(4); + }); + }); + + it("a real location replaces the place to search for", () => { + const results = new TestResults(); + results.apply(parseAll(rubyParser(), "MixedTest#test_pass = 0.00 s = .")); + const id = ["MixedTest", "test_pass"].join("\u0000"); + expect(results.get(id)!.location?.resolve).toBe("rubyTest"); + results.apply([ + { type: "location", path: ["MixedTest", "test_pass"], location: { path: F, line: 4, resolve: "path" } }, + ]); + expect(results.get(id)!.location).toEqual({ path: F, line: 4, resolve: "path" }); + }); +}); diff --git a/src/features/run/testing/ruby.ts b/src/features/run/testing/ruby.ts index a73fe6cbf..bc7d49ce2 100644 --- a/src/features/run/testing/ruby.ts +++ b/src/features/run/testing/ruby.ts @@ -15,7 +15,7 @@ * 7 runs, 5 assertions, 1 failures, 1 errors, 1 skips * ``` */ -import { parseDuration, type TestEvent, type TestOutputParser, type TestStatus } from "./types"; +import { parseDuration, TestStdout, type TestEvent, type TestOutputParser, type TestStatus } from "./types"; // A class may be a spec description with spaces (`Spec style#test_0001_works`). const RESULT_RE = /^(.+?)#(\S+) = ([\d.]+ s) = ([.FES])$/; @@ -44,7 +44,81 @@ const located = (path: string[]): TestEvent => ({ location: { path: path[1]!, className: path[0], resolve: "rubyTest" }, }); +/** Starts a record of the IDE's own Minitest reporter (`ide-run/src/ruby_reporter`); a JSON object follows. */ +export const RECORD_MARK = "@@OPENIDEA@@"; + +interface ReporterRecord { + t: "start" | "result" | "summary"; + c?: string; + n?: string; + s?: "passed" | "failed" | "skipped"; + /** Seconds. */ + d?: number | null; + f?: string | null; + l?: number | null; + m?: string; + runs?: number; + failures?: number; + errors?: number; + skips?: number; +} + +/** What a reporter record says: the test's real file and line, its status, time and failure text. */ +function fromRecord(json: string): TestEvent[] { + let r: ReporterRecord; + try { + r = JSON.parse(json) as ReporterRecord; + } catch { + return []; + } + if (r.t === "summary") { + const failed = (r.failures ?? 0) + (r.errors ?? 0); + const skipped = r.skips ?? 0; + return [ + { + type: "summary", + passed: Math.max(0, (r.runs ?? 0) - failed - skipped), + failed, + skipped, + durationMs: typeof r.d === "number" ? Math.round(r.d * 1000 * 1000) / 1000 : undefined, + }, + ]; + } + if (!r.c || !r.n) return []; + const path = [r.c, r.n]; + const events: TestEvent[] = []; + if (r.t === "start") events.push({ type: "start", path }); + if (r.t === "result" && r.s) { + events.push({ + type: "result", + path, + status: r.s, + durationMs: typeof r.d === "number" ? Math.round(r.d * 1000 * 1000) / 1000 : undefined, + }); + for (const line of (r.m ?? "").split("\n")) if (line) events.push({ type: "output", path, line }); + } + if (r.f && r.l) { + events.push({ type: "location", path, location: { path: r.f, line: r.l, resolve: "path" } }); + } + return events; +} + +/** + * The console side of the reporter: a record ends with its own newline, so what the line holds before the + * marker (a progress dot) is all the console shows of it. + */ +export class RubyTestOutput extends TestStdout { + constructor(feedLine: (line: string) => void) { + super(feedLine, (line) => { + const at = line.indexOf(RECORD_MARK); + return at < 0 ? line + "\n" : line.slice(0, at); + }); + } +} + export function rubyParser(): TestOutputParser { + /** The reporter's records are the source of results; the text is then ignored (it would repeat them). */ + let records = false; const seen = new Set(); /** The test's own location, once. */ const locate = (path: string[]): TestEvent[] => { @@ -60,6 +134,12 @@ export function rubyParser(): TestOutputParser { return { feed(line) { + const mark = line.indexOf(RECORD_MARK); + if (mark >= 0) { + records = true; + return fromRecord(line.slice(mark + RECORD_MARK.length)); + } + if (records) return []; const events: TestEvent[] = []; let m: RegExpExecArray | null;