feat(editor): Highlight languages without a CodeMirror mode from tree-sitter

Elixir files opened as plain text: highlighting only came from
@codemirror/language-data, which has no Elixir mode. Grammars now carry
their own highlights.scm; `doc_highlights` folds its captures into a small
kind vocabulary (inner nodes and later patterns win, interpolation clears
the string colour), memoised per document version on the incremental
tree. The editor maps kinds onto Lezer tags, so colours follow the active
scheme, and requests them only when no CodeMirror mode exists.

Enabled for Elixir; other grammar-only languages need their query wired.

Claude-Session: https://claude.ai/code/session_01B65qLEKnkeYTGdbwhLpjCu

src/editor/core/backend.ts

@@ -18,6 +18,7 @@ import {
   type LineEnding,
   type OutlineItem,
   type Range,
+  type SyntaxHighlights,
   type TextChange,
 } from "@/platform/ipc";
 
@@ -43,6 +44,8 @@ export interface DocBackend {
   setEncoding(doc: DocId, encoding: string, reload: boolean): Promise<string>;
   outline(doc: DocId): Promise<OutlineItem[]>;
   folding(doc: DocId): Promise<FoldRange[]>;
+  /** Tree-sitter highlighting, for languages without a CodeMirror mode. */
+  highlights(doc: DocId): Promise<SyntaxHighlights>;
   selectionRanges(doc: DocId, offset: number): Promise<Range[]>;
   languages(): Promise<LanguageInfo[]>;
   onPatched(handler: (e: DocPatchedEvent) => void): () => void;
@@ -62,6 +65,7 @@ export const ipcBackend: DocBackend = {
   setEncoding: (doc, encoding, reload) => call(commands.docSetEncoding(doc, encoding, reload)),
   outline: (doc) => call(commands.docOutline(doc)),
   folding: (doc) => call(commands.docFolding(doc)),
+  highlights: (doc) => call(commands.docHighlights(doc)),
   selectionRanges: (doc, offset) => call(commands.docSelectionRanges(doc, offset)),
   languages: () => commands.languages(),
   onPatched: (h) => onIdeEvent("docPatched", (e) => h(e)),
@@ -343,6 +347,9 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
       }
       return items;
     },
+    async highlights() {
+      return { kinds: [], data: [] };
+    },
     async folding(doc) {
       const d = get(doc);
       const out: FoldRange[] = [];

src/editor/core/cm/setup.ts

@@ -25,6 +25,7 @@ import { openideaTheme } from "./theme";
 import { caretRowHighlight } from "./caretRow";
 import { changeLane, overviewLane } from "./lanes";
 import { foldingExtension } from "./folding";
+import { syntaxHighlightField } from "./syntaxHighlight";
 import { indentGuides } from "./indentGuides";
 import { stickyMaxLines, stickyScroll } from "./stickyScroll";
 import { findExtension } from "./find";
@@ -267,6 +268,7 @@ export function baseExtensions(o: SetupOptions): Extension {
     clipboardHistoryExtension,
     indentOnInput(),
     syntaxHighlighting(openideaHighlightStyle),
+    syntaxHighlightField,
     closeBrackets(),
     caretRowHighlight,
     fontZoom,

src/editor/core/cm/syntaxHighlight.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import { EditorState } from "@codemirror/state";
+import { highlightingFor, syntaxHighlighting } from "@codemirror/language";
+import { tags as t } from "@lezer/highlight";
+import { openideaHighlightStyle } from "./highlight";
+import { setSyntaxHighlights, syntaxHighlightField } from "./syntaxHighlight";
+
+function marks(state: EditorState) {
+  const out: { text: string; cls: string }[] = [];
+  state.field(syntaxHighlightField).between(0, state.doc.length, (from, to, d) => {
+    out.push({ text: state.doc.sliceString(from, to), cls: d.spec.class as string });
+  });
+  return out;
+}
+
+describe("syntaxHighlightField", () => {
+  const create = () =>
+    EditorState.create({ doc: "def hi, do: :ok", extensions: [syntaxHighlighting(openideaHighlightStyle), syntaxHighlightField] });
+
+  it("styles backend ranges with the active highlight style and maps them through edits", () => {
+    let state = create();
+    state = state.update({
+      effects: setSyntaxHighlights.of({ kinds: ["keyword", "atom", "functionCall"], data: [0, 3, 0, 4, 6, 2, 12, 15, 1, 99, 120, 0] }),
+    }).state;
+    const keyword = highlightingFor(state, [t.keyword]);
+    const atom = highlightingFor(state, [t.atom]);
+    expect(keyword).toBeTruthy();
+    expect(marks(state)).toEqual([
+      { text: "def", cls: keyword },
+      { text: "hi", cls: highlightingFor(state, [t.function(t.variableName)]) },
+      { text: ":ok", cls: atom },
+    ]);
+
+    state = state.update({ changes: { from: 0, insert: "  " } }).state;
+    expect(marks(state).map((m) => m.text)).toEqual(["def", "hi", ":ok"]);
+  });
+});

src/editor/core/cm/syntaxHighlight.ts

@@ -0,0 +1,66 @@
+/**
+ * Tree-sitter highlighting (`doc_highlights`) for languages without a
+ * CodeMirror mode (Elixir, ...): backend kinds → Lezer tags, so the colours
+ * come from the same highlight style and Color Scheme settings as Lezer
+ * highlighting. Refreshed with folding; in between, marks are mapped through
+ * the changes.
+ */
+import { StateEffect, StateField, type EditorState } from "@codemirror/state";
+import { Decoration, EditorView, type DecorationSet } from "@codemirror/view";
+import { highlightingFor } from "@codemirror/language";
+import { tags as t, type Tag } from "@lezer/highlight";
+import type { SyntaxHighlightKind, SyntaxHighlights } from "@/platform/ipc";
+
+const TAGS: Record<SyntaxHighlightKind, Tag> = {
+  keyword: t.keyword,
+  string: t.string,
+  stringSpecial: t.special(t.string),
+  escape: t.escape,
+  regexp: t.regexp,
+  atom: t.atom,
+  number: t.number,
+  constant: t.constant(t.variableName),
+  comment: t.comment,
+  docComment: t.docComment,
+  function: t.function(t.definition(t.variableName)),
+  functionCall: t.function(t.variableName),
+  builtin: t.standard(t.variableName),
+  type: t.typeName,
+  namespace: t.namespace,
+  variable: t.variableName,
+  selfVariable: t.self,
+  property: t.propertyName,
+  attribute: t.annotation,
+  macro: t.macroName,
+  label: t.labelName,
+  tag: t.tagName,
+  operator: t.operator,
+  punctuation: t.punctuation,
+};
+
+export const setSyntaxHighlights = StateEffect.define<SyntaxHighlights>();
+
+function decorations(state: EditorState, h: SyntaxHighlights): DecorationSet {
+  const marks = h.kinds.map((k) => {
+    const cls = highlightingFor(state, [TAGS[k]]);
+    return cls ? Decoration.mark({ class: cls }) : null;
+  });
+  const len = state.doc.length;
+  const ranges = [];
+  for (let i = 0; i + 2 < h.data.length; i += 3) {
+    const from = h.data[i]!;
+    const to = Math.min(h.data[i + 1]!, len);
+    const mark = marks[h.data[i + 2]!];
+    if (mark && from < to) ranges.push(mark.range(from, to));
+  }
+  return Decoration.set(ranges, true);
+}
+
+export const syntaxHighlightField = StateField.define<DecorationSet>({
+  create: () => Decoration.none,
+  update(deco, tr) {
+    for (const e of tr.effects) if (e.is(setSyntaxHighlights)) return decorations(tr.state, e.value);
+    return tr.docChanged ? deco.map(tr.changes) : deco;
+  },
+  provide: (f) => EditorView.decorations.from(f),
+});

src/editor/core/model/document.ts

@@ -7,7 +7,7 @@ import { EditorView } from "@codemirror/view";
 import type { LanguageDescription, LanguageSupport } from "@codemirror/language";
 import { create } from "zustand";
 import type { EditorConfigProps } from "@/bindings";
-import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings } from "@/platform/ipc";
+import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings, SyntaxHighlights } from "@/platform/ipc";
 import {
   _emitEditorEvent,
   _saveParticipants,
@@ -22,6 +22,7 @@ import type { DocBackend, LanguageInfo } from "../backend";
 import { findLanguageDescription, languageInfo, loadLanguageInfos, loadSupport, loadedSupport, commentTokens } from "../cm/languages";
 import { foldEffect } from "@codemirror/language";
 import { backendFolds, setBackendFolds } from "../cm/folding";
+import { setSyntaxHighlights } from "../cm/syntaxHighlight";
 import {
   caretFor,
   compartments,
@@ -265,6 +266,7 @@ export class DocModel {
 
   addSlot(slot: EditorSlot) {
     this.slots.add(slot);
+    if (this.highlights?.counter === this.changeCounter) slot.reconfigure([setSyntaxHighlights.of(this.highlights.value)]);
   }
 
   removeSlot(slot: EditorSlot) {
@@ -332,6 +334,8 @@ export class DocModel {
   indentOverride: { tabSize: number; insertSpaces: boolean } | null = null;
   /** Indentation detected in the file's text at open time. */
   detectedIndent: { insertSpaces: boolean; size?: number } | null = null;
+  /** Last tree-sitter highlighting, for slots opened before the next refresh. */
+  private highlights: { counter: number; value: SyntaxHighlights } | null = null;
   /** Fold-by-default has been applied (first folding ranges of the document). */
   private initialFolds = false;
   /** Signature of settings that require contributions to be rebuilt. */
@@ -423,7 +427,12 @@ export class DocModel {
     await this.sync.whenIdle();
     const counter = this.changeCounter;
     const backend = this.manager.backend;
-    const [folds, outline] = await Promise.allSettled([backend.folding(this.doc), backend.outline(this.doc)]);
+    const [folds, outline, highlights] = await Promise.allSettled([
+      backend.folding(this.doc),
+      backend.outline(this.doc),
+      // Without a CodeMirror mode, colours come from the grammar's tree-sitter query.
+      this.desc ? null : backend.highlights(this.doc),
+    ]);
     if (this.closed) return;
     if (counter !== this.changeCounter) {
       this.scheduleRefresh();
@@ -436,6 +445,10 @@ export class DocModel {
         this.foldByDefault();
       }
     }
+    if (highlights.status === "fulfilled" && highlights.value) {
+      this.highlights = { counter, value: highlights.value };
+      for (const s of this.slots) s.reconfigure([setSyntaxHighlights.of(highlights.value)]);
+    }
     if (outline.status === "fulfilled") {
       useOutline.setState({ [this.path]: outline.value });
       const lines = methodLines(outline.value);