feat(vcs): IntelliJ's diff algorithm and change colouring in the diff viewer (#242)

* feat(vcs): IDEA's diff algorithm and change colouring in the diff viewer

Port IntelliJ's text comparison to TypeScript (src/features/vcs/comparison):
Myers/patience LCS with unique-element discarding, ByLine (smart matching
that skips unimportant lines, chunk optimizer, second-step correction),
ByWord (word chunks, punctuation matching, whitespace policies, line block
splitting) and ComparisonManager's compareLinesInner / processBlocks. The
port runs IntelliJ's own LineComparisonUtilTest, WordComparisonUtilTest and
SplitComparisonUtilTest cases (459 runs), converted to ideaCases.json.

MergeView chunks now come straight from these line fragments, with the
inner word fragments as chunk changes, instead of @codemirror/merge's
presentable diff. Revert and >> / << replace lines as
DiffUtil.applyModification does.

Colour changes as IDEA does: modified (blue), inserted (green) and deleted
(grey) by chunk type, a lighter line background for chunks whose words were
compared, each word fragment in its own type at full line height, a thin
bar for a fragment empty on one side, and the colours carried into the
gutters and the divider bands. New --diff-modified / --diff-inserted /
--diff-deleted theme tokens, imported from .icls DIFF_* attributes.

Add a Playwright suite over a generated git repository of diff cases,
including 10,000-line files, in every viewer mode.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

* fix(vcs): preserve unified revert boundaries and bound large diffs

---------

Co-authored-by: Claude Opus 5.5 <[email protected]>
Co-authored-by: l0ng-ai <[email protected]>

src/features/vcs/diffOptions.ts

@@ -1,10 +1,12 @@
 /**
  * Diff viewer settings (IDEA's diff toolbar): viewer kind, whitespace policy,
  * highlighting, collapsed unchanged fragments and synchronized scrolling, plus
- * the whitespace-ignoring diff algorithm and per-chunk transfer helpers.
+ * chunks computed by IDEA's comparison and per-chunk transfer helpers.
  */
-import { Change, diff, type Chunk } from "@codemirror/merge";
-import type { Text } from "@codemirror/state";
+import { Change, Chunk, type DiffConfig } from "@codemirror/merge";
+import type { ChangeDesc, Text } from "@codemirror/state";
+import { compareTexts, type ComparisonPolicy, type LineFragment } from "./comparison";
+import { DiffTooBigError } from "./comparison/lcs";
 
 export type DiffMode = "split" | "unified";
 export type IgnorePolicy = "none" | "trim" | "whitespace" | "whitespaceAndLines";
@@ -82,151 +84,125 @@ export function saveDiffOptions(o: DiffViewOptions) {
   }
 }
 
-interface Line {
-  /** Start offset. */
-  from: number;
-  /** End offset including the line break. */
-  to: number;
-  key: string;
+const IDEA = Symbol("ideaDiff");
+
+interface IdeaDiff {
+  policy: ComparisonPolicy;
+  words: boolean;
+  trimChunks: boolean;
 }
 
-function lineKey(text: string, policy: IgnorePolicy): string {
-  if (policy === "trim") return text.trim();
-  if (policy === "none") return text;
-  return text.replace(/\s+/g, "");
-}
-
-function splitLines(s: string, policy: IgnorePolicy): Line[] {
-  const out: Line[] = [];
-  let from = 0;
-  for (;;) {
-    const nl = s.indexOf("\n", from);
-    const end = nl < 0 ? s.length : nl;
-    const key = lineKey(s.slice(from, end), policy) + (policy === "none" && nl >= 0 ? "\n" : "");
-    if (!(policy === "whitespaceAndLines" && key === "")) out.push({ from, to: nl < 0 ? s.length : nl + 1, key });
-    if (nl < 0) return out;
-    from = nl + 1;
-  }
-}
-
-/** First char code used to encode lines (skips control chars), and the surrogate gap. */
-const BASE = 0x21;
-
-function encoder() {
-  const codes = new Map<string, string>();
-  let next = BASE;
-  return (key: string): string | null => {
-    let c = codes.get(key);
-    if (c === undefined) {
-      if (next === 0xd800) next = 0xe000;
-      if (next > 0xffff) return null;
-      c = String.fromCharCode(next++);
-      codes.set(key, c);
-    }
-    return c;
-  };
-}
-
-/** Bounds the expensive diff search; larger inputs fall back to a cheaper, coarser match. */
-const DIFF_CONFIG = { scanLimit: 500 };
+const COMPARISON_POLICY: Record<IgnorePolicy, ComparisonPolicy> = {
+  none: "default",
+  trim: "trim",
+  whitespace: "ignore",
+  whitespaceAndLines: "ignore",
+};
 
 /**
- * Diff of encoded lines (one char per line). Lines that occur exactly once on
- * each side anchor the match (patience diff), and only the gaps between the
- * anchors get a regular diff, so distant edits in long files stay separate
- * without a quadratic search.
+ * A MergeView `diffConfig` whose chunks come from IDEA's comparison
+ * (`./comparison`) instead of `@codemirror/merge`'s own diff: lines matched as
+ * IDEA does, word fragments by IDEA's rules, no merging of nearby changes.
  */
-function diffLines(a: string, b: string): Change[] {
-  const seen = new Map<string, { inA: number; inB: number; at: number }>();
-  for (let i = 0; i < a.length; i++) {
-    const e = seen.get(a[i]!);
-    if (e) e.inA++;
-    else seen.set(a[i]!, { inA: 1, inB: 0, at: i });
-  }
-  for (const c of b) {
-    const e = seen.get(c);
-    if (e) e.inB++;
-  }
-  // Unique common lines as [indexA, indexB] in A order, then their longest increasing run in B.
-  const pairs: [number, number][] = [];
-  for (let j = 0; j < b.length; j++) {
-    const e = seen.get(b[j]!);
-    if (e && e.inA === 1 && e.inB === 1) pairs.push([e.at, j]);
-  }
-  pairs.sort((x, y) => x[0] - y[0]);
-  const tails: number[] = [];
-  const prev = new Array<number>(pairs.length);
-  for (let i = 0; i < pairs.length; i++) {
-    let lo = 0;
-    let hi = tails.length;
-    while (lo < hi) {
-      const mid = (lo + hi) >> 1;
-      if (pairs[tails[mid]!]![1] < pairs[i]![1]) lo = mid + 1;
-      else hi = mid;
-    }
-    prev[i] = lo > 0 ? tails[lo - 1]! : -1;
-    tails[lo] = i;
-  }
-  const anchors: [number, number][] = [];
-  for (let i = tails.length ? tails[tails.length - 1]! : -1; i >= 0; i = prev[i]!) anchors.push(pairs[i]!);
-  anchors.reverse().push([a.length, b.length]);
-  const out: Change[] = [];
-  let fromA = 0;
-  let fromB = 0;
-  for (const [toA, toB] of anchors) {
-    if (toA > fromA || toB > fromB)
-      for (const c of diff(a.slice(fromA, toA), b.slice(fromB, toB), DIFF_CONFIG))
-        out.push(new Change(c.fromA + fromA, c.toA + fromA, c.fromB + fromB, c.toB + fromB));
-    fromA = toA + 1;
-    fromB = toB + 1;
-  }
-  return out;
+export function ideaDiffConfig(ignore: IgnorePolicy, highlight: HighlightPolicy): DiffConfig {
+  const idea: IdeaDiff = {
+    policy: COMPARISON_POLICY[ignore],
+    words: highlight === "words",
+    trimChunks: ignore === "whitespaceAndLines",
+  };
+  const config: DiffConfig = {};
+  (config as Record<symbol, IdeaDiff>)[IDEA] = idea;
+  return config;
 }
 
+/** The line fragment a chunk was built from. */
+const fragments = new WeakMap<Chunk, LineFragment>();
+
 /**
- * Match lines first, then compare the contents of changed blocks for word
- * highlighting. The line keys also implement the whitespace policy.
+ * Whether the chunk has IDEA's inner fragments (its words were compared):
+ * such a chunk gets the lighter line background (TextDiffType "ignored"
+ * colour) with its `changes` in full colour.
  */
-export function whitespaceDiff(policy: IgnorePolicy): (a: string, b: string) => readonly Change[] {
-  return (a, b) => {
-    const la = splitLines(a, policy);
-    const lb = splitLines(b, policy);
-    const enc = encoder();
-    const ea: string[] = [];
-    const eb: string[] = [];
-    for (const l of la) {
-      const c = enc(l.key);
-      if (c === null) return diff(a, b, DIFF_CONFIG);
-      ea.push(c);
-    }
-    for (const l of lb) {
-      const c = enc(l.key);
-      if (c === null) return diff(a, b, DIFF_CONFIG);
-      eb.push(c);
-    }
-    const out: Change[] = [];
-    const at = (lines: Line[], i: number) => (i > 0 ? lines[i - 1]!.to : 0);
-    for (const c of diffLines(ea.join(""), eb.join(""))) {
-      const fromA = c.toA > c.fromA ? la[c.fromA]!.from : at(la, c.fromA);
-      const toA = c.toA > c.fromA ? la[c.toA - 1]!.to : fromA;
-      const fromB = c.toB > c.fromB ? lb[c.fromB]!.from : at(lb, c.fromB);
-      const toB = c.toB > c.fromB ? lb[c.toB - 1]!.to : fromB;
-      if (fromA === toA || fromB === toB) {
-        out.push(new Change(fromA, toA, fromB, toB));
-        continue;
-      }
-      for (const inner of diff(a.slice(fromA, toA), b.slice(fromB, toB), DIFF_CONFIG))
-        out.push(new Change(inner.fromA + fromA, inner.toA + fromA, inner.fromB + fromB, inner.toB + fromB));
-    }
-    return out;
-  };
+export const hasInnerFragments = (chunk: Chunk) => !!fragments.get(chunk)?.inner;
+
+/** IDEA's line fragments as chunks; `changes` hold the inner fragments, or the whole block. */
+export function ideaChunks(a: Text, b: Text, ignore: IgnorePolicy, highlight: HighlightPolicy): readonly Chunk[] {
+  return toChunks(a, b, ideaOf(ideaDiffConfig(ignore, highlight))!);
 }
 
+function toChunks(a: Text, b: Text, idea: IdeaDiff): readonly Chunk[] {
+  let compared: LineFragment[];
+  try {
+    compared = compareTexts(a.toString(), b.toString(), idea);
+  } catch (error) {
+    if (!(error instanceof DiffTooBigError)) throw error;
+    // Repeated lines can exhaust IDEA's search. Keep the viewer usable with a bounded, coarser diff.
+    return build.call(Chunk, a, b, { scanLimit: 500 });
+  }
+  return compared.map((f) => {
+    // `to` is one past the last line's end, past the document for its last line.
+    const toA = f.startLine1 === f.endLine1 ? f.startOffset1 : f.endLine1 === a.lines ? a.length + 1 : f.endOffset1;
+    const toB = f.startLine2 === f.endLine2 ? f.startOffset2 : f.endLine2 === b.lines ? b.length + 1 : f.endOffset2;
+    const changes = f.inner
+      ? f.inner.map((r) => new Change(r.start1, r.end1, r.start2, r.end2))
+      : [new Change(0, f.endOffset1 - f.startOffset1, 0, f.endOffset2 - f.startOffset2)];
+    const chunk = new Chunk(changes, f.startOffset1, toA, f.startOffset2, toB);
+    fragments.set(chunk, f);
+    return chunk;
+  });
+}
+
+// Every chunk computation of MergeView and unifiedMergeView goes through these.
+const ideaOf = (conf?: DiffConfig) => (conf as Record<symbol, IdeaDiff | undefined> | undefined)?.[IDEA];
+const { build, updateA, updateB } = Chunk;
+Chunk.build = (a: Text, b: Text, conf?: DiffConfig) => {
+  const idea = ideaOf(conf);
+  return idea ? toChunks(a, b, idea) : build.call(Chunk, a, b, conf);
+};
+Chunk.updateA = (chunks: readonly Chunk[], a: Text, b: Text, changes: ChangeDesc, conf?: DiffConfig) => {
+  const idea = ideaOf(conf);
+  return idea ? toChunks(a, b, idea) : updateA.call(Chunk, chunks, a, b, changes, conf);
+};
+Chunk.updateB = (chunks: readonly Chunk[], a: Text, b: Text, changes: ChangeDesc, conf?: DiffConfig) => {
+  const idea = ideaOf(conf);
+  return idea ? toChunks(a, b, idea) : updateB.call(Chunk, chunks, a, b, changes, conf);
+};
+
+/** Text of lines `[from, to)` (0-based), without the last line break. */
+const linesContent = (doc: Text, from: number, to: number) => doc.sliceString(doc.line(from + 1).from, doc.line(to).to);
+
 /**
  * The change that copies a chunk's content from one side to the other
- * (IDEA's ">>" / "<<" and Revert), in the destination's coordinates.
+ * (IDEA's ">>" / "<<" and Revert), in the destination's coordinates. Lines are
+ * replaced as DiffUtil.applyModification does, so a missing final line break
+ * stays missing.
  */
 export function transferChunk(chunk: Chunk, source: "a" | "b", src: Text, dst: Text) {
+  const f = fragments.get(chunk);
+  if (!f) return transferByOffsets(chunk, source, src, dst);
+  const [srcFrom, srcTo, dstFrom, dstTo] =
+    source === "a"
+      ? [f.startLine1, f.endLine1, f.startLine2, f.endLine2]
+      : [f.startLine2, f.endLine2, f.startLine1, f.endLine1];
+  if (dstFrom === dstTo) {
+    const text = linesContent(src, srcFrom, srcTo);
+    return dstFrom === dst.lines
+      ? { from: dst.length, to: dst.length, insert: "\n" + text }
+      : { from: dst.line(dstFrom + 1).from, to: dst.line(dstFrom + 1).from, insert: text + "\n" };
+  }
+  let from = dst.line(dstFrom + 1).from;
+  let to = dst.line(dstTo).to;
+  if (srcFrom === srcTo) {
+    // Lines removed with a line break: the one before them, or else the one after.
+    if (from > 0) from--;
+    else if (to < dst.length) to++;
+    return { from, to, insert: "" };
+  }
+  return { from, to, insert: linesContent(src, srcFrom, srcTo) };
+}
+
+/** `transferChunk` for chunks not built from line fragments. */
+function transferByOffsets(chunk: Chunk, source: "a" | "b", src: Text, dst: Text) {
   const [srcFrom, srcTo, dstFrom, dstTo] =
     source === "a"
       ? [chunk.fromA, chunk.toA, chunk.fromB, chunk.toB]