feat: Syntax-colour Find in Files and Show Usages result lines (#143)

Result lines in the Find in Files popup and tool window and in the Show Usages popup and Usages tool window are coloured like the editor, using the whole file as context, so a line inside a block comment or a multi-line string gets the right colour. The colouring lives in editor/lineTokens and all four lists share it; each file is parsed once and cached, and the parse runs in 8 ms time slices so large files don't block the UI thread. Lines that moved since the search, and files over 500k characters, are coloured on their own.

Claude-Session: https://claude.ai/code/session_01Dq6jQQn8ZhRfkkyj3PRutA
Co-authored-by: jaw52 <[email protected]>

src/editor/lineTokens.test.ts

@@ -0,0 +1,82 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { EditorState } from "@codemirror/state";
+
+const disk = new Map<string, string>();
+const editors = new Map<string, EditorState>();
+
+vi.mock(import("@/platform/ipc"), async (importOriginal) => ({
+  ...(await importOriginal()),
+  commands: { fsReadText: (path: string) => path } as never,
+  call: (async (path: string) => disk.get(path) ?? Promise.reject(new Error("missing"))) as never,
+}));
+vi.mock("@/editor/api", () => ({
+  editorsFor: (path: string) => {
+    const state = editors.get(path);
+    return state ? [{ view: { state } }] : [];
+  },
+}));
+
+const { clearLineTokens, lineTokens, peekLineTokens } = await import("./lineTokens");
+
+const SOURCE = ["/**", " * A read-only virtual document (library", " */", 'const lib = "x";'].join("\n");
+const classesOf = (text: string, tokens: Array<{ from: number; to: number; cls: string }>) =>
+  tokens.map((t) => `${text.slice(t.from, t.to)}:${t.cls}`);
+
+describe("lineTokens", () => {
+  beforeEach(() => {
+    clearLineTokens();
+    disk.clear();
+    editors.clear();
+  });
+
+  it("colours a line inside a block comment with the whole file as context", async () => {
+    disk.set("/p/a.ts", SOURCE);
+    const text = "* A read-only virtual document (library";
+    const tokens = await lineTokens({ path: "/p/a.ts", line: 1, col: 1 }, text);
+    expect(classesOf(text, tokens)).toEqual([`${text}:tok-comment`]);
+    expect(peekLineTokens({ path: "/p/a.ts", line: 1, col: 1 }, text)).toBe(tokens);
+  });
+
+  it("keeps positions relative to the shown text", async () => {
+    disk.set("/p/a.ts", SOURCE);
+    const text = 'lib = "x";';
+    const tokens = await lineTokens({ path: "/p/a.ts", line: 3, col: 6 }, text);
+    expect(classesOf(text, tokens)).toContain('"x":tok-string');
+  });
+
+  it("parses the line on its own when the file no longer has it there", async () => {
+    disk.set("/p/a.ts", SOURCE);
+    const text = "* A read-only virtual document (library";
+    const tokens = await lineTokens({ path: "/p/a.ts", line: 0, col: 0 }, text);
+    expect(classesOf(text, tokens)).not.toContain(`${text}:tok-comment`);
+  });
+
+  it("parses again when the open editor's document changes", async () => {
+    editors.set("/p/a.ts", EditorState.create({ doc: SOURCE.replace("/**", "//") }));
+    const text = "* A read-only virtual document (library";
+    expect(classesOf(text, await lineTokens({ path: "/p/a.ts", line: 1, col: 1 }, text))).not.toContain(
+      `${text}:tok-comment`,
+    );
+    editors.set("/p/a.ts", EditorState.create({ doc: SOURCE }));
+    expect(classesOf(text, await lineTokens({ path: "/p/a.ts", line: 1, col: 1 }, text))).toEqual([
+      `${text}:tok-comment`,
+    ]);
+  });
+
+  it("parses a big file in slices, letting other tasks run in between", async () => {
+    const body = "const a = 1; // filler line for the parser\n".repeat(8000);
+    disk.set("/p/big.ts", `${body}const lib = "x";`);
+    let ticks = 0;
+    const timer = setInterval(() => ticks++, 0);
+    const text = 'lib = "x";';
+    const tokens = await lineTokens({ path: "/p/big.ts", line: 8000, col: 6 }, text);
+    clearInterval(timer);
+    expect(classesOf(text, tokens)).toContain('"x":tok-string');
+    expect(ticks).toBeGreaterThan(0);
+  });
+
+  it("leaves files of unknown languages plain", async () => {
+    disk.set("/p/a.unknownext", SOURCE);
+    expect(await lineTokens({ path: "/p/a.unknownext", line: 1, col: 1 }, "* A read-only")).toEqual([]);
+  });
+});

src/editor/lineTokens.tsx

@@ -0,0 +1,167 @@
+/**
+ * Syntax colours for result lines (Find in Files, Show Usages), like IDEA's
+ * `ChunkExtractor`: each file is parsed whole once, so a line inside a block
+ * comment or a multi-line string is coloured as such.
+ */
+import { useEffect, useState, type ReactNode } from "react";
+import { Text } from "@codemirror/state";
+import { editorsFor } from "@/editor/api";
+import { codeHighlighter, highlightTokens, languageForPath, type CodeToken } from "@/editor/lsp/markdown";
+import { call, commands } from "@/platform/ipc";
+import { isVirtualUri, virtualDocument } from "@/platform/virtualDocs";
+
+export type { CodeToken };
+
+/** Where a result line's `text` sits in its file: 0-based `line`, column `col`. */
+export interface LineSource {
+  path: string;
+  line: number;
+  col: number;
+}
+
+const texts = new Map<string, Promise<string>>();
+
+/**
+ * Text of a file that isn't open (a library source or a read-only backend
+ * read that doesn't open a document), cached until `clearLineTokens`.
+ */
+export function readFileText(path: string): Promise<string> {
+  let p = texts.get(path);
+  if (!p) {
+    p = isVirtualUri(path) ? virtualDocument(path).then((d) => d.text) : call(commands.fsReadText(path));
+    p.catch(() => texts.delete(path));
+    texts.set(path, p);
+    if (texts.size > 40) texts.delete(texts.keys().next().value!);
+  }
+  return p;
+}
+
+/** Drops cached file texts and parses (a new search reads the files again). */
+export function clearLineTokens() {
+  texts.clear();
+  highlighted.clear();
+}
+
+/** Bigger files aren't parsed whole: their result lines are coloured on their own. */
+const MAX_PARSE = 500_000;
+
+interface Parsed {
+  doc: Text;
+  highlight: (from: number, to: number) => CodeToken[];
+}
+
+/**
+ * One file parsed whole: `source` is the editor document or the text it was
+ * parsed from, so an edit (a new `Text`) or a re-read (a new string) parses
+ * again.
+ */
+interface Highlighted {
+  source: Text | string;
+  parsed: Promise<Parsed | null>;
+  /** Tokens per result line (`lineKey`). */
+  lines: Map<string, CodeToken[]>;
+}
+const highlighted = new Map<string, Highlighted>();
+
+const lineKey = (line: number, col: number, text: string) => `${line}\u0000${col}\u0000${text}`;
+
+/** `lineTokens` already computed for the file as last parsed (for a first render without flicker). */
+export function peekLineTokens({ path, line, col }: LineSource, text: string): CodeToken[] | null {
+  return highlighted.get(path)?.lines.get(lineKey(line, col, text)) ?? null;
+}
+
+/**
+ * Syntax tokens of a result line (`text` at `source`), positions within
+ * `text`, coloured with the whole file as context. When the file no longer
+ * has `text` there (changed since the search) or is too big, the line is
+ * parsed on its own.
+ */
+export async function lineTokens({ path, line, col }: LineSource, text: string): Promise<CodeToken[]> {
+  const language = languageForPath(path);
+  if (!language || !text) return [];
+  const source = editorsFor(path)[0]?.view.state.doc ?? (await readFileText(path).catch(() => null));
+  if (source == null) return highlightTokens(text, language);
+  let h = highlighted.get(path);
+  if (h?.source !== source) {
+    h = { source, parsed: parseWhole(source, language), lines: new Map() };
+    highlighted.delete(path);
+    highlighted.set(path, h);
+    if (highlighted.size > 40) highlighted.delete(highlighted.keys().next().value!);
+  }
+  const key = lineKey(line, col, text);
+  const known = h.lines.get(key);
+  if (known) return known;
+  const parsed = await h.parsed;
+  let tokens: CodeToken[] | null = null;
+  if (parsed && line < parsed.doc.lines) {
+    const l = parsed.doc.line(line + 1);
+    if (l.text.slice(col, col + text.length) === text) {
+      const from = l.from + col;
+      tokens = parsed.highlight(from, from + text.length).map((t) => ({ ...t, from: t.from - from, to: t.to - from }));
+    }
+  }
+  tokens ??= await highlightTokens(text, language);
+  h.lines.set(key, tokens);
+  return tokens;
+}
+
+async function parseWhole(source: Text | string, language: string): Promise<Parsed | null> {
+  if (source.length > MAX_PARSE) return null;
+  const text = typeof source === "string" ? source : source.toString();
+  const highlight = await codeHighlighter(text, language);
+  if (!highlight) return null;
+  // Splitting on "\n" alone keeps a "\r" in its line, so offsets match `text`.
+  return { doc: typeof source === "string" ? Text.of(text.split("\n")) : source, highlight };
+}
+
+/** `lineTokens` for a rendered line; null until known (and without a `source`). */
+export function useLineTokens(source: LineSource | undefined, text: string): CodeToken[] | null {
+  const path = source?.path ?? null;
+  const line = source?.line ?? 0;
+  const col = source?.col ?? 0;
+  const key = `${path}\u0000${line}\u0000${col}\u0000${text}`;
+  const [loaded, setLoaded] = useState<{ key: string; tokens: CodeToken[] } | null>(null);
+  useEffect(() => {
+    if (path == null) return;
+    let alive = true;
+    void lineTokens({ path, line, col }, text).then((tokens) => {
+      if (alive) setLoaded({ key, tokens });
+    });
+    return () => {
+      alive = false;
+    };
+  }, [key, path, line, col, text]);
+  if (path == null) return null;
+  return loaded?.key === key ? loaded.tokens : peekLineTokens({ path, line, col }, text);
+}
+
+/**
+ * `text` between `from` and `to` as spans with the tokens' `tok-*` classes
+ * (styled by the caller); `key` prefixes the React keys.
+ */
+export function tokenSpans(
+  text: string,
+  from: number,
+  to: number,
+  tokens: readonly CodeToken[] | null,
+  key = "",
+): ReactNode[] {
+  if (!tokens?.length) return [text.slice(from, to)];
+  const out: ReactNode[] = [];
+  let pos = from;
+  for (const t of tokens) {
+    if (t.to <= pos) continue;
+    if (t.from >= to) break;
+    const s = Math.max(t.from, pos);
+    const e = Math.min(t.to, to);
+    if (s > pos) out.push(text.slice(pos, s));
+    out.push(
+      <span key={`${key}${s}`} className={t.cls}>
+        {text.slice(s, e)}
+      </span>,
+    );
+    pos = e;
+  }
+  if (pos < to) out.push(text.slice(pos, to));
+  return out;
+}

src/editor/lsp/markdown.ts

@@ -7,9 +7,9 @@
  * is prefetched when the app goes idle; until then markup shows as plain text.
  */
 import { useMemo, useSyncExternalStore } from "react";
-import { LanguageDescription } from "@codemirror/language";
+import { LanguageDescription, type Language } from "@codemirror/language";
 import { languages } from "@codemirror/language-data";
-import { classHighlighter, highlightCode } from "@lezer/highlight";
+import { classHighlighter, highlightTree } from "@lezer/highlight";
 import type { Markup } from "@/bindings";
 import { maybeEditorService } from "@/workbench/contributions";
 import { isTauri } from "@/platform/ipc";
@@ -114,26 +114,55 @@ export interface CodeToken {
 
 /** Highlighted ranges of `text`; empty when the language is unknown or its grammar fails to load. */
 export async function highlightTokens(text: string, language: string | null): Promise<CodeToken[]> {
+  return (await codeHighlighter(text, language))?.(0, text.length) ?? [];
+}
+
+/**
+ * Parses `text` once; the result gives the highlighted ranges (positions in
+ * `text`) between `from` and `to`, so a line is coloured with the whole file
+ * as context. Null when the language is unknown or its grammar fails to load.
+ */
+export async function codeHighlighter(
+  text: string,
+  language: string | null,
+): Promise<((from: number, to: number) => CodeToken[]) | null> {
   const desc = language ? findLanguage(language) : null;
-  if (!desc) return [];
+  if (!desc) return null;
   try {
     const support = desc.support ?? (await desc.load());
-    const tree = support.language.parser.parse(text);
-    const out: CodeToken[] = [];
-    let pos = 0;
-    highlightCode(
-      text,
-      tree,
-      classHighlighter,
-      (t, cls) => {
-        if (cls) out.push({ from: pos, to: pos + t.length, cls });
-        pos += t.length;
-      },
-      () => pos++,
-    );
-    return out;
+    const tree = await parseSliced(support.language.parser, text);
+    return (from, to) => {
+      const out: CodeToken[] = [];
+      highlightTree(
+        tree,
+        classHighlighter,
+        (a, b, cls) => {
+          const s = Math.max(a, from);
+          const e = Math.min(b, to);
+          if (e > s) out.push({ from: s, to: e, cls });
+        },
+        from,
+        to,
+      );
+      return out;
+    };
   } catch {
-    return [];
+    return null;
+  }
+}
+
+/** Parse budget per task: a whole file (up to ~1s) yields to input and rendering in between. */
+const PARSE_SLICE_MS = 8;
+
+async function parseSliced(parser: Language["parser"], text: string): Promise<ReturnType<Language["parser"]["parse"]>> {
+  const parse = parser.startParse(text);
+  for (;;) {
+    const start = performance.now();
+    while (performance.now() - start < PARSE_SLICE_MS) {
+      const tree = parse.advance();
+      if (tree) return tree;
+    }
+    await new Promise((r) => setTimeout(r, 0));
   }
 }