fix(editor): honor semantic token scheme styles (#2098)

Semantic tokens ignored the background, weight, italic and effect set for them in Color Scheme; only syntax tokens honoured them. Semantic roles now use the same attributes as syntax tokens, package references get their own attribute and theme token, and modifier styles keep winning over role styles, including deprecated static fields.

Default rendering changes: semantic constants are italic, as Color Scheme already declared, and package references in the light theme use the editor text colour.

Siblings covered: find matches stay above token backgrounds, and Package reference has a preview sample in Color Scheme.

Co-authored-by: shohruh-genius <[email protected]>

docs/plugins/themes.md

@@ -324,7 +324,7 @@ a token.
 
 <!-- BEGIN GENERATED TOKEN REFERENCE: pnpm gen:theme-docs -->
 
-242 themeable tokens: 121 in the UI theme file, 121 in the editor colour scheme file.
+243 themeable tokens: 121 in the UI theme file, 122 in the editor colour scheme file.
 
 ### Surfaces (UI theme)
 
@@ -431,6 +431,7 @@ a token.
 | `--syn-punct` | `#bcbec4` | `#080808` |
 | `--syn-variable` | `#bcbec4` | `#080808` |
 | `--syn-global` | `var(--syn-variable)` | `var(--syn-variable)` |
+| `--syn-namespace` | `var(--fg-editor)` | `var(--fg-editor)` |
 | `--syn-param` | `#bcbec4` | `#080808` |
 | `--syn-property` | `#c77dbb` | `#871094` |
 | `--syn-constant` | `#c77dbb` | `#871094` |

e2e/semanticStyle.fixture.ts

@@ -0,0 +1,47 @@
+import { EditorView } from "@codemirror/view";
+import { commands } from "../src/platform/ipc";
+import { setInvokeRoute } from "../src/platform/tauriCore";
+import { schemeCss } from "../src/platform/colorScheme";
+import { lspDoc } from "../src/editor/lsp/lsp";
+import { semanticExtension } from "../src/editor/lsp/semantic";
+import { lspTheme } from "../src/editor/lsp/theme";
+
+setInvokeRoute(() => null);
+commands.lspSemanticTokens = async () => ({
+  status: "ok",
+  data: {
+    tokenTypes: ["namespace", "class", "function", "parameter", "variable", "property", "selfTypeKeyword"],
+    tokenModifiers: ["declaration", "static", "deprecated", "constant"],
+    data: [
+      0, 0, 7, 0, 0, 1, 0, 6, 1, 0, 2, 0, 6, 2, 1, 3, 0, 5, 3, 0, 4, 0, 5, 4, 8, 5, 0, 6, 2, 7, 6, 0, 5, 5, 6, 7, 0, 4,
+      6, 0,
+    ],
+  },
+});
+const style = document.createElement("style");
+style.textContent = schemeCss("dark", {
+  dark: {
+    keyword: { foreground: "#99bbdd", background: "#ddeeff", italic: true },
+    packageReference: { foreground: "#446688", background: "#ddeeff", italic: true },
+    className: { foreground: "#557799", background: "#ddeeff", bold: true },
+    functionDeclaration: {
+      foreground: "#6688aa",
+      background: "#ddeeff",
+      bold: true,
+      italic: false,
+      effect: "underline",
+    },
+    parameter: { foreground: "#7799bb", background: "#ddeeff", bold: true },
+    constant: { foreground: "#88aacc", background: "#ddeeff", bold: true },
+  },
+});
+document.head.append(style);
+new EditorView({
+  parent: document.body,
+  doc: "package\nSample\nrender\ninput\nLimit\nlegacy\nfield\nSelf",
+  extensions: [
+    lspTheme,
+    lspDoc.of({ doc: 1, path: "demo.ts", language: "typescript", uiMode: "TypeScript", readOnly: false }),
+    semanticExtension(),
+  ],
+});

e2e/semanticStyle.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="UTF-8" />
+  </head>
+  <body>
+    <script type="module" src="./semanticStyle.fixture.ts"></script>
+  </body>
+</html>

e2e/semanticStyle.spec.ts

@@ -0,0 +1,27 @@
+import { expect, test } from "@playwright/test";
+
+test("semantic roles honor all Color Scheme attributes and keep modifier styles", async ({ page }) => {
+  await page.goto("/e2e/semanticStyle.html");
+  for (const [selector, color] of [
+    [".cm-sem-keyword", "rgb(153, 187, 221)"],
+    [".cm-sem-namespace", "rgb(68, 102, 136)"],
+    [".cm-sem-type", "rgb(85, 119, 153)"],
+    [".cm-sem-function:not(.cm-sem-static)", "rgb(102, 136, 170)"],
+    [".cm-sem-parameter", "rgb(119, 153, 187)"],
+    [".cm-sem-constant", "rgb(136, 170, 204)"],
+  ]) {
+    const role = page.locator(selector);
+    await expect(role).toHaveCSS("color", color!);
+    await expect(role).toHaveCSS("background-color", "rgb(221, 238, 255)");
+  }
+  await expect(page.locator(".cm-sem-namespace")).toHaveCSS("font-style", "italic");
+  for (const selector of [".cm-sem-type", ".cm-sem-function", ".cm-sem-parameter", ".cm-sem-constant"]) {
+    await expect(page.locator(selector).first()).toHaveCSS("font-weight", "700");
+  }
+  await expect(page.locator(".cm-sem-function:not(.cm-sem-static)")).toHaveCSS("font-style", "normal");
+  await expect(page.locator(".cm-sem-function:not(.cm-sem-static)")).toHaveCSS("text-decoration-line", "underline");
+  await expect(page.locator(".cm-sem-function.cm-sem-static")).toHaveCSS("font-style", "italic");
+  for (const role of [".cm-sem-function.cm-sem-deprecated", ".cm-sem-property.cm-sem-deprecated"]) {
+    await expect(page.locator(role)).toHaveCSS("text-decoration-line", "line-through");
+  }
+});

plugins/official/harbor-night/themes/harbor-night.scheme.json

@@ -19,6 +19,7 @@
     "--syn-punct": "#a9b4c2",
     "--syn-variable": "#d9dfe8",
     "--syn-global": "#d9dfe8",
+    "--syn-namespace": "var(--fg-editor)",
     "--syn-param": "#d9dfe8",
     "--syn-property": "#b9a3f0",
     "--syn-constant": "#e58fa3",

src/editor/core/cm/find.test.ts

@@ -1,9 +1,10 @@
 import { afterEach, describe, expect, it } from "vitest";
-import { EditorState } from "@codemirror/state";
-import { EditorView } from "@codemirror/view";
+import { EditorSelection, EditorState, Prec } from "@codemirror/state";
+import { Decoration, EditorView } from "@codemirror/view";
 import { setSearchQuery } from "@codemirror/search";
 import { StreamLanguage } from "@codemirror/language";
 import { literalKindAt } from "./literals";
+import { selectionMarks } from "./selectionMarks";
 import {
   countMatches,
   findResultLines,
@@ -20,6 +21,7 @@ import {
   replaceAll,
   replaceNext,
   setFindScope,
+  setFindOpen,
   type FindOptions,
 } from "./find";
 
@@ -39,6 +41,36 @@ function make(doc: string, o: Partial<FindOptions>, scope?: { from: number; to:
 
 const sel = (v: EditorView) => [v.state.selection.main.from, v.state.selection.main.to];
 
+describe("find backgrounds", () => {
+  it.each([
+    ["semantic", Prec.highest],
+    ["syntax", Prec.high],
+    ["ordinary", Prec.default],
+  ])("keeps matches inside %s backgrounds and selections inside matches", (_name, precedence) => {
+    const view = new EditorView({
+      parent: document.body,
+      doc: "Session",
+      extensions: [
+        selectionMarks,
+        findExtension(),
+        precedence(EditorView.decorations.of(Decoration.set(Decoration.mark({ class: "token" }).range(0, 7)))),
+      ],
+    });
+    views.push(view);
+    view.dispatch({
+      effects: [
+        setFindOpen.of(true),
+        setSearchQuery.of(
+          makeQuery({ search: "Session", caseSensitive: true, wholeWord: false, regexp: false, replace: "" }),
+        ),
+      ],
+      selection: EditorSelection.single(1, 6),
+    });
+    expect(view.contentDOM.querySelector(".token .cm-openidea-findMatch")?.textContent).toBe("Session");
+    expect(view.contentDOM.querySelector(".cm-openidea-findMatch .cm-openidea-selected")?.textContent).toBe("essio");
+  });
+});
+
 describe("find in selection", () => {
   const doc = "a1 a2\na3 a4\na5 a6\n";
   // Scope: the second line only.

src/editor/core/cm/find.ts

@@ -199,7 +199,7 @@ export function findExtension(): Extension {
     findExcluded,
     findMarks,
     literalsExtension,
-    findHighlighter,
+    Prec.highest(findHighlighter),
   ];
 }
 

src/editor/core/cm/selectionMarks.ts

@@ -4,8 +4,8 @@ import { Decoration, EditorView, ViewPlugin, type DecorationSet, type ViewUpdate
 /**
  * `drawSelection` paints the selection in a layer behind the text, so any mark with its own
  * background (light-scheme warnings, find matches, usages, semantic backgrounds…) hides it.
- * This repaints selected text in the innermost span: higher-precedence marks nest closer to the
- * text, and the only other `Prec.highest` marks (LSP semantic tokens) set colour, not background.
+ * This repaints selected text in the innermost span. At equal precedence the earlier source
+ * nests inside, so this must precede find and semantic decorations at `Prec.highest`.
  */
 const selectedMark = Decoration.mark({ class: "cm-openidea-selected" });
 

src/editor/lsp/semantic/index.ts

@@ -314,8 +314,8 @@ const semanticPlugin = ViewPlugin.fromClass(
 
 export function semanticExtension(): Extension {
   // Higher-precedence decoration sources create the *inner* DOM nodes, so this
-  // must outrank Lezer's highlighter for semantic colours to win; they only set
-  // colour / style, never backgrounds. `Prec.highest`, not `Prec.high`: the
+  // must outrank Lezer's highlighter for Color Scheme attributes to win.
+  // `Prec.highest`, not `Prec.high`: the
   // highlighter (`treeHighlighter` in @codemirror/language) is itself
   // `Prec.high`, and at equal precedence the later source ends up outside —
   // its inner span then re-set the colour (a field's `variableName` grey hid

src/editor/lsp/theme.ts

@@ -69,13 +69,14 @@ const markdown = {
   ".tok-invalid": { color: "var(--syn-invalid)" },
 };
 
-function pythonStyle(token: string, fallback: string) {
+function semanticStyle(token: string, fallback: string) {
   return {
     color: `var(--syn-${token}, var(--syn-${fallback}))`,
     fontWeight: `var(--syn-${token}-bold, inherit)`,
     fontStyle: `var(--syn-${token}-italic, inherit)`,
     backgroundColor: `var(--syn-${token}-bg, transparent)`,
     textDecoration: `var(--syn-${token}-effect, none)`,
+    WebkitTextDecoration: `var(--syn-${token}-effect, none)`,
   };
 }
 
@@ -358,44 +359,46 @@ export const lspTheme = EditorView.theme({
   ".cm-lsp-inlay-pl": { marginLeft: "4px" },
   ".cm-lsp-inlay-pr": { marginRight: "4px" },
 
-  ".cm-sem-python-method-call": pythonStyle("python-method-call", "function-call"),
-  ".cm-sem-python-annotation": pythonStyle("python-annotation", "type"),
-  ".cm-sem-python-parentheses": pythonStyle("python-parentheses", "punct"),
-  ".cm-sem-python-brackets": pythonStyle("python-brackets", "punct"),
-  ".cm-sem-python-braces": pythonStyle("python-braces", "punct"),
-  ".cm-sem-python-definition": pythonStyle("function", "function"),
-  ".cm-sem-python-nested": pythonStyle("python-nested-function", "function"),
-  ".cm-sem-python-predefined-definition": pythonStyle("python-predefined-definition", "predefined"),
-  ".cm-sem-python-predefined-usage": pythonStyle("python-predefined-usage", "predefined"),
-  ".cm-sem-python-docstring, .cm-sem-python-docstring *": pythonStyle("docstring", "string"),
-  ".cm-sem-python-bytes": pythonStyle("python-bytes", "string"),
-  ".cm-sem-python-fstring, .cm-sem-python-fstring .tok-string, .cm-sem-python-fstring .tok-string2": pythonStyle(
+  ".cm-sem-python-method-call": semanticStyle("python-method-call", "function-call"),
+  ".cm-sem-python-annotation": semanticStyle("python-annotation", "type"),
+  ".cm-sem-python-parentheses": semanticStyle("python-parentheses", "punct"),
+  ".cm-sem-python-brackets": semanticStyle("python-brackets", "punct"),
+  ".cm-sem-python-braces": semanticStyle("python-braces", "punct"),
+  ".cm-sem-python-definition": semanticStyle("function", "function"),
+  ".cm-sem-python-nested": semanticStyle("python-nested-function", "function"),
+  ".cm-sem-python-predefined-definition": semanticStyle("python-predefined-definition", "predefined"),
+  ".cm-sem-python-predefined-usage": semanticStyle("python-predefined-usage", "predefined"),
+  ".cm-sem-python-docstring, .cm-sem-python-docstring *": semanticStyle("docstring", "string"),
+  ".cm-sem-python-bytes": semanticStyle("python-bytes", "string"),
+  ".cm-sem-python-fstring, .cm-sem-python-fstring .tok-string, .cm-sem-python-fstring .tok-string2": semanticStyle(
     "python-fstring",
     "string",
   ),
-  ".cm-sem-python-escape": pythonStyle("escape", "escape"),
-  ".cm-sem-python-format": pythonStyle("python-format", "escape"),
-  ".cm-sem-python-keyword-arg": pythonStyle("keyword-arg", "param"),
+  ".cm-sem-python-escape": semanticStyle("escape", "escape"),
+  ".cm-sem-python-format": semanticStyle("python-format", "escape"),
+  ".cm-sem-python-keyword-arg": semanticStyle("keyword-arg", "param"),
   // --- semantic tokens -----------------------------------------------------
-  ".cm-sem-type, .cm-sem-typeParameter": { color: "var(--syn-type)" },
+  ".cm-sem-type, .cm-sem-typeParameter": semanticStyle("type", "type"),
   // IDEA colours declarations, calls stay (nearly) plain text.
-  ".cm-sem-function": { color: "var(--syn-function-call, var(--syn-function))" },
-  ".cm-sem-function.cm-sem-declaration": { color: "var(--syn-function)" },
-  ".cm-sem-function.cm-sem-builtin": { color: "var(--syn-builtin, var(--syn-function))" },
-  ".cm-sem-function.cm-sem-python-method-call": pythonStyle("python-method-call", "function-call"),
-  ".cm-sem-macro": { color: "var(--syn-annotation)" },
-  ".cm-sem-annotation": { color: "var(--syn-decorator, var(--syn-annotation))" },
-  ".cm-sem-constant": { color: "var(--syn-constant)" },
-  ".cm-sem-variable": { color: "var(--syn-variable)" },
-  ".cm-sem-property": { color: "var(--syn-property)" },
-  ".cm-sem-parameter": { color: "var(--syn-param, var(--syn-variable))" },
+  ".cm-sem-function": semanticStyle("function-call", "function"),
+  // Keep role selectors at one class so static/deprecated modifiers can override their styles.
+  ".cm-sem-function:where(.cm-sem-declaration)": semanticStyle("function", "function"),
+  ".cm-sem-function:where(.cm-sem-builtin)": semanticStyle("builtin", "function"),
+  ".cm-sem-function.cm-sem-python-method-call": semanticStyle("python-method-call", "function-call"),
+  ".cm-sem-macro": semanticStyle("annotation", "annotation"),
+  ".cm-sem-annotation": semanticStyle("decorator", "annotation"),
+  ".cm-sem-namespace": semanticStyle("namespace", "type"),
+  ".cm-sem-constant": { ...semanticStyle("constant", "constant"), fontStyle: "var(--syn-constant-italic, italic)" },
+  ".cm-sem-variable": semanticStyle("variable", "variable"),
+  ".cm-sem-property": semanticStyle("property", "property"),
+  ".cm-sem-parameter": semanticStyle("param", "variable"),
   ".cm-sem-parameter.cm-sem-unused": { color: "var(--fg-unused, var(--fg-faint))" },
-  ".cm-sem-self": { color: "var(--syn-self, var(--syn-keyword))" },
-  ".cm-sem-keyword": { color: "var(--syn-keyword)", fontWeight: "var(--syn-keyword-bold, inherit)" },
+  ".cm-sem-self": semanticStyle("self", "keyword"),
+  ".cm-sem-keyword": semanticStyle("keyword", "keyword"),
   // Before the builtin rule: library globals (`print`, `string`) keep the predefined colour.
-  ".cm-sem-global:not(.cm-sem-function)": { color: "var(--syn-global, inherit)" },
-  ".cm-sem-builtin:not(.cm-sem-function)": { color: "var(--syn-builtin, inherit)" },
-  ".cm-sem-builtin.cm-sem-python-builtin": pythonStyle("python-builtin", "builtin"),
+  ".cm-sem-global:where(:not(.cm-sem-function))": semanticStyle("global", "variable"),
+  ".cm-sem-builtin:where(:not(.cm-sem-function))": semanticStyle("builtin", "variable"),
+  ".cm-sem-builtin.cm-sem-python-builtin": semanticStyle("python-builtin", "builtin"),
   ".cm-sem-static": { fontStyle: "var(--syn-static-italic, italic)" },
   ".cm-sem-property.cm-sem-static, .cm-sem-property.cm-sem-static *": {
     color: "var(--syn-static-property, var(--syn-property))",
@@ -411,14 +414,15 @@ export const lspTheme = EditorView.theme({
     textDecorationColor: "var(--fg-faint)",
     textUnderlineOffset: "3px",
   },
-  ".cm-sem-deprecated": { textDecoration: "line-through" },
-  ".cm-sem-python-local-variable": pythonStyle("python-local-variable", "variable"),
-  ".cm-sem-global.cm-sem-python-global-variable": pythonStyle("python-global-variable", "global"),
-  ".cm-sem-python-instance-attribute": pythonStyle("python-instance-attribute", "property"),
-  ".cm-sem-python-class-attribute": pythonStyle("python-class-attribute", "property"),
-  ".cm-sem-python-constant": pythonStyle("python-constant", "constant"),
-  ".cm-sem-python-property": pythonStyle("python-property", "property"),
-  ".cm-sem-python-type-parameter": pythonStyle("python-type-parameter", "type"),
+  ".cm-sem-python-local-variable": semanticStyle("python-local-variable", "variable"),
+  ".cm-sem-global.cm-sem-python-global-variable": semanticStyle("python-global-variable", "global"),
+  ".cm-sem-python-instance-attribute": semanticStyle("python-instance-attribute", "property"),
+  ".cm-sem-python-class-attribute": semanticStyle("python-class-attribute", "property"),
+  ".cm-sem-python-constant": semanticStyle("python-constant", "constant"),
+  ".cm-sem-python-property": semanticStyle("python-property", "property"),
+  ".cm-sem-python-type-parameter": semanticStyle("python-type-parameter", "type"),
+  // Deprecation wins over role effects, including static fields.
+  ".cm-content .cm-sem-deprecated, .cm-content .cm-sem-deprecated *": { textDecoration: "line-through" },
   // Semantic highlighting: after the base classes so a parameter's colour gives way, and over the
   // syntax highlighter's span inside.
   ...Object.fromEntries(

src/features/settings/pages/editor/colorScheme.tsx

@@ -279,6 +279,8 @@ const SAMPLES: Record<string, CodeSample> = {
         ["name", "parameter"],
         [", ", "operator"],
         ["GLOBAL", "globalVariable"],
+        [", ", "operator"],
+        ["com.example", "packageReference"],
         ["):", "operator"],
       ],
       [

src/platform/colorScheme.test.ts

@@ -73,6 +73,15 @@ describe("color schemes", () => {
     });
   });
 
+  it("resolves package references from the editor foreground and emits their overrides", () => {
+    for (const theme of ["light", "dark", "dark-hc"] as const) {
+      expect(resolvedAttribute(theme, "packageReference").foreground).toBe(schemeTokens(theme).get("--fg-editor"));
+    }
+    const css = schemeCss("dark", { dark: { packageReference: { foreground: "#446688", italic: true } } });
+    expect(css).toContain("--syn-namespace: #446688 !important");
+    expect(css).toContain("--syn-namespace-italic: italic !important");
+  });
+
   it("emits scheme and override declarations", () => {
     const root = document.documentElement;
     root.dataset.theme = "light";

src/platform/colorScheme.ts

@@ -125,6 +125,7 @@ const BASE_ATTRIBUTES: readonly AttributeDef[] = [
   syn("identifier", "Identifiers › Local variable", [LD], "variable"),
   syn("parameter", "Identifiers › Parameter", [LD], "param"),
   syn("globalVariable", "Identifiers › Global variable", [LD], "global"),
+  syn("packageReference", "Identifiers › Package reference", [LD], "namespace"),
   syn("constant", "Identifiers › Constant", [LD], "constant", { defaultItalic: true }),
   syn("field", "Identifiers › Instance field", [LD], "property"),
   { id: "staticSymbol", name: "Identifiers › Static symbol", path: [LD], token: "static", italicOnly: true },

src/platform/locales/text/en.json

@@ -2,6 +2,7 @@
   "Cannot remove a worktree containing the open project.": "Cannot remove a worktree containing the open project.",
   "Delete Remote Branches": "Delete Remote Branches",
   "Delete the {length} branches '{names}' on the remote? The remote-tracking branches are removed as well.": "Delete the {length} branches '{names}' on the remote? The remote-tracking branches are removed as well.",
+  "Identifiers › Package reference": "Identifiers › Package reference",
   "Add or Remove Caret": "Add or Remove Caret",
   "Select Python Environment Folder": "Select Python Environment Folder",
   "Checking interpreter": "Checking interpreter",

src/platform/locales/text/zh.json

@@ -1,4 +1,5 @@
 {
+  "Identifiers › Package reference": "标识符 › 包引用",
   "Cannot remove a worktree containing the open project.": "无法移除包含当前打开项目的工作树。",
   "Select Python Environment Folder": "选择 Python 环境文件夹",
   "Checking interpreter": "正在检查解释器",

src/styles/tokens.css

@@ -123,6 +123,7 @@ html {
   --syn-punct: #bcbec4;
   --syn-variable: #bcbec4;
   --syn-global: var(--syn-variable);
+  --syn-namespace: var(--fg-editor);
   --syn-param: #bcbec4;
   --syn-property: #c77dbb;
   --syn-constant: #c77dbb;
@@ -524,6 +525,7 @@ html {
   --syn-punct: #080808;
   --syn-variable: #080808;
   --syn-global: var(--syn-variable);
+  --syn-namespace: var(--fg-editor);
   --syn-param: #080808;
   --syn-property: #871094;
   --syn-constant: #871094;
@@ -797,6 +799,7 @@ html {
   --syn-punct: #e3e4e8;
   --syn-variable: #e3e4e8;
   --syn-global: var(--syn-variable);
+  --syn-namespace: var(--fg-editor);
   --syn-param: #e3e4e8;
   --syn-property: #e0a0d6;
   --syn-constant: #e0a0d6;
@@ -1155,6 +1158,7 @@ html {
   --syn-punct: var(--palette-fg);
   --syn-variable: var(--palette-variable);
   --syn-global: var(--syn-variable);
+  --syn-namespace: var(--fg-editor);
   --syn-param: var(--palette-fg);
   --syn-property: var(--palette-property);
   --syn-constant: var(--palette-number);