fix(kotlin): highlight declarations and semantic roles (#2097)

Kotlin files showed declarations, parameters, fields, annotations, KDoc and escapes in generic lexer colours, because the CodeMirror mode alone drove their highlighting.

Kotlin now gets supplemental backend marks from its tree-sitter grammar, which take precedence over lexer marks while LSP semantic marks stay innermost. The semantic provider also maps the mutable and operator-declaration tokens.

Siblings covered: other grammars keep their parameter colours, and constructor default values are highlighted as values.

Co-authored-by: shohruh-genius <[email protected]>
Цей коміт міститься в:
Shohruh Ismoilovтаshohruh-genius authored and GitHub committed 2026-11-08 21:56:26 +08:00
1 джерело 99100e48bd
коміт c20f4c0e84
17 files changed
+324 -12

No files matched your search

+21
Переглянути файл
@@ -0,0 +1,21 @@
(class_declaration name: (identifier) @type)
(object_declaration name: (identifier) @type)
(companion_object name: (identifier) @type)
(type_alias type: (identifier) @type)
(user_type (identifier) @type)
(type_parameter (identifier) @type)
(function_declaration name: (identifier) @function.definition)
(parameter (identifier) @parameter.kotlin)
(class_parameter (identifier) @parameter.kotlin ":")
(property_declaration (variable_declaration (identifier) @variable))
(class_body (property_declaration (variable_declaration (identifier) @property)))
(class_parameter ["val" "var"] (identifier) @property ":")
(enum_entry (identifier) @constant)
(annotation (user_type (identifier) @attribute))
(annotation (constructor_invocation (user_type (identifier) @attribute)))
(call_expression . (identifier) @function)
(call_expression . (navigation_expression (identifier) @function .))
(class_modifier) @keyword
(block_comment) @comment
((block_comment) @comment.documentation (#match? @comment.documentation "^/\\*\\*"))
(escape_sequence) @string.escape
+1 -1
Переглянути файл
@@ -120,7 +120,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "kotlin",
language: tree_sitter_kotlin_ng::LANGUAGE,
outline: outline!("kotlin"),
highlights: None,
highlights: Some(Highlights { query: highlights!("kotlin"), first_wins: false }),
fold_nodes: &["string_literal"],
fold_bodies: NONE,
imports: &["import"],
+60 -1
Переглянути файл
@@ -42,11 +42,12 @@ pub enum SyntaxHighlightKind {
Tag,
Operator,
Punctuation,
Parameter,
}
impl SyntaxHighlightKind {
/// Every kind, in `as u32` order: the legend for [`SyntaxHighlights::data`].
pub const ALL: [Self; 24] = [
pub const ALL: [Self; 25] = [
Self::Keyword,
Self::String,
Self::StringSpecial,
@@ -71,6 +72,7 @@ impl SyntaxHighlightKind {
Self::Tag,
Self::Operator,
Self::Punctuation,
Self::Parameter,
];
/// The kebab-case name plugins use (`doc-comment`, `function-call`, …).
@@ -162,6 +164,7 @@ const CAPTURES: &[(&str, Paint)] = {
("variable.member", Kind(K::Property)),
("variable.field", Kind(K::Property)),
("variable", Kind(K::Variable)),
("parameter.kotlin", Kind(K::Parameter)),
("parameter", Kind(K::Variable)),
("property", Kind(K::Property)),
("field", Kind(K::Property)),
@@ -361,6 +364,9 @@ mod tests {
assert_eq!(paint_for("string.special.path"), Some(Paint::Kind(K::StringSpecial)));
assert_eq!(paint_for("comment.doc.__attribute__"), Some(Paint::Kind(K::DocComment)));
assert_eq!(paint_for("punctuation.bracket"), Some(Paint::Kind(K::Punctuation)));
assert_eq!(paint_for("parameter.kotlin"), Some(Paint::Kind(K::Parameter)));
assert_eq!(paint_for("parameter"), Some(Paint::Kind(K::Variable)));
assert_eq!(paint_for("variable.parameter"), Some(Paint::Kind(K::Variable)));
assert_eq!(paint_for("embedded"), Some(Paint::Clear));
assert_eq!(paint_for("__name__"), None);
assert_eq!(paint_for("spell"), None);
@@ -376,6 +382,59 @@ mod tests {
.collect()
}
#[test]
fn kotlin_supplements_stream_highlighting_with_declaration_roles() {
use SyntaxHighlightKind as K;
let src = "package palette\n/** Docs */\n@Deprecated(\"old\")\nclass Greeter(val name: String) {\n var count: Int = 0\n fun greet(prefix: String): String { val message = \"hello\\n\"; println(message); return prefix }\n}\ntypealias Handler = (String) -> Unit\nvalue class UserId(val value: String)\n";
let got = spans("kotlin", src);
for (text, expected) in [
("Deprecated", K::Attribute),
("Greeter", K::Type),
("name", K::Property),
("count", K::Property),
("greet", K::Function),
("prefix", K::Parameter),
("message", K::Variable),
("println", K::FunctionCall),
("Handler", K::Type),
("value", K::Keyword),
("/** Docs */", K::DocComment),
("\\n", K::Escape),
] {
assert!(
got.iter().any(|(word, kind)| word == text && *kind == expected),
"missing {text}: {expected:?} in {got:?}"
);
}
}
#[test]
fn non_kotlin_parameters_keep_variable_highlighting() {
for (lang, src) in [
("graphql", "query Hero { hero(id: 1) { name } }"),
("gdscript", "func greet(id: int):\n\tpass\n"),
("zig", "fn greet(id: u32) void {}"),
("nix", "{ id }: id"),
] {
let got = spans(lang, src);
assert!(
got.iter().any(|(text, kind)| text == "id" && *kind == SyntaxHighlightKind::Variable),
"{lang}: {got:?}"
);
assert!(!got.iter().any(|(_, kind)| *kind == SyntaxHighlightKind::Parameter), "{lang}: {got:?}");
}
}
#[test]
fn kotlin_constructor_defaults_are_not_declarations() {
let got = spans("kotlin", "class Example(val name: String = fallback, count: Int = defaultCount)");
assert!(got.iter().any(|(text, kind)| text == "name" && *kind == SyntaxHighlightKind::Property));
assert!(got.iter().any(|(text, kind)| text == "count" && *kind == SyntaxHighlightKind::Parameter));
for reference in ["fallback", "defaultCount"] {
assert!(!got.iter().any(|(text, _)| text == reference), "{got:?}");
}
}
#[test]
fn highlights_elixir() {
use SyntaxHighlightKind as K;
+52
Переглянути файл
@@ -0,0 +1,52 @@
import { EditorView } from "@codemirror/view";
import { syntaxHighlighting } from "@codemirror/language";
import { schemeCss } from "../src/platform/colorScheme";
import { openideaHighlightStyle } from "../src/editor/core/cm/highlight";
import { findLanguageDescription, loadSupport } from "../src/editor/core/cm/languages";
import { syntaxHighlightField, setSyntaxHighlights } from "../src/editor/core/cm/syntaxHighlight";
import { commands } from "../src/platform/ipc";
import { setInvokeRoute } from "../src/platform/tauriCore";
import { lspDoc } from "../src/editor/lsp/lsp";
import { semanticExtension } from "../src/editor/lsp/semantic";
import { lspTheme } from "../src/editor/lsp/theme";
const semantic = new URLSearchParams(location.search).has("semantic");
setInvokeRoute(() => null);
commands.lspSemanticTokens = async () => ({
status: "ok",
data: {
tokenTypes: ["operator", "parameter"],
tokenModifiers: ["declaration", "modification"],
data: [0, 4, 5, 0, 1, 0, 10, 6, 1, 2],
},
});
const style = document.createElement("style");
style.textContent = schemeCss("dark", {
dark: {
"kotlin.functionDeclaration": { foreground: "#557799", background: "#ddeeff", bold: true },
"kotlin.parameter": { foreground: "#6688aa" },
},
});
document.head.append(style);
const host = document.createElement("section");
host.dataset.language = "Kotlin";
document.body.append(host);
const support = await loadSupport(findLanguageDescription("Kotlin", "Demo.kt")!);
const view = new EditorView({
parent: host,
doc: "fun greet(prefix: String) {}",
extensions: [
support!,
syntaxHighlightField,
syntaxHighlighting(openideaHighlightStyle),
...(semantic
? [
lspTheme,
lspDoc.of({ doc: 1, path: "Demo.kt", language: "kotlin", uiMode: "Kotlin", readOnly: false }),
semanticExtension(),
]
: []),
],
});
view.dispatch({ effects: setSyntaxHighlights.of({ kinds: ["function", "parameter"], data: [4, 9, 0, 10, 16, 1] }) });
+9
Переглянути файл
@@ -0,0 +1,9 @@
<!doctype html>
<html>
<head>
<meta charset="UTF-8" />
</head>
<body>
<script type="module" src="./kotlinHighlight.fixture.ts"></script>
</body>
</html>
+33
Переглянути файл
@@ -0,0 +1,33 @@
import { expect, test } from "@playwright/test";
test("Kotlin backend roles override lexer styles", async ({ page }) => {
await page.goto("/e2e/kotlinHighlight.html");
const greet = page
.locator(".cm-line span")
.filter({ hasText: /^greet$/ })
.last();
await expect(greet).toHaveCSS("color", "rgb(85, 119, 153)");
await expect(greet).toHaveCSS("background-color", "rgb(221, 238, 255)");
await expect(greet).toHaveCSS("font-weight", "700");
await expect(
page
.locator(".cm-line span")
.filter({ hasText: /^prefix$/ })
.last(),
).toHaveCSS("color", "rgb(102, 136, 170)");
});
test("Kotlin semantic roles stay inside backend and lexer marks", async ({ page }) => {
await page.goto("/e2e/kotlinHighlight.html?semantic");
const declaration = page.locator(".cm-sem-function.cm-sem-declaration");
await expect(declaration).toHaveText("greet");
await expect(declaration).toHaveCSS("color", "rgb(85, 119, 153)");
await expect(declaration).toHaveCSS("background-color", "rgb(221, 238, 255)");
await expect(declaration).toHaveCSS("font-weight", "700");
await expect(declaration.locator("span")).toHaveCount(0);
const parameter = page.locator(".cm-sem-parameter.cm-sem-mutable");
await expect(parameter).toHaveText("prefix");
await expect(parameter).toHaveCSS("color", "rgb(102, 136, 170)");
await expect(parameter).toHaveCSS("text-decoration-line", "underline");
await expect(parameter.locator("span")).toHaveCount(0);
});
+1 -1
Переглянути файл
@@ -7805,7 +7805,7 @@ export type SyntaxErrors = {
};
/** Highlight classes sent to the UI, which maps each onto a Lezer tag. */
export type SyntaxHighlightKind = "keyword" | "string" | "stringSpecial" | "escape" | "regexp" | "atom" | "number" | "constant" | "comment" | "docComment" | "function" | "functionCall" | "builtin" | "type" | "namespace" | "variable" | "selfVariable" | "property" | "attribute" | "macro" | "label" | "tag" | "operator" | "punctuation";
export type SyntaxHighlightKind = "keyword" | "string" | "stringSpecial" | "escape" | "regexp" | "atom" | "number" | "constant" | "comment" | "docComment" | "function" | "functionCall" | "builtin" | "type" | "namespace" | "variable" | "selfVariable" | "property" | "attribute" | "macro" | "label" | "tag" | "operator" | "punctuation" | "parameter";
/** Highlighted ranges of a whole document, packed like LSP semantic tokens. */
export type SyntaxHighlights = {
+2
Переглянути файл
@@ -8,6 +8,7 @@ import { Tag, tags as t } from "@lezer/highlight";
/** Log warnings: the UI warning colour, since no syntax token reads as one in every scheme. */
export const logWarning = Tag.define("logWarning");
export const parameter = Tag.define(t.variableName);
function token(
tag: Tag | Tag[],
@@ -72,6 +73,7 @@ export const openideaHighlightStyle = HighlightStyle.define([
token(t.labelName, "constant"),
token([t.variableName, t.definition(t.variableName), t.local(t.variableName)], "variable"),
token([t.propertyName, t.definition(t.propertyName)], "property"),
token(parameter, "param"),
token(t.attributeName, "attribute"),
token(t.special(t.brace), "decorator"),
token([t.tagName, t.angleBracket], "tag"),
+1 -1
Переглянути файл
@@ -362,8 +362,8 @@ export function baseExtensions(o: SetupOptions): Extension {
clipboardDocumentPath.of(o.ctx.path),
clipboardFilesExtension,
indentOnInput(),
syntaxHighlighting(openideaHighlightStyle),
syntaxHighlightField,
syntaxHighlighting(openideaHighlightStyle),
closeBrackets(),
caretRowHighlight,
fontZoom,
+29 -2
Переглянути файл
@@ -1,10 +1,15 @@
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it } from "vitest";
import { EditorState } from "@codemirror/state";
import { EditorView } from "@codemirror/view";
import { highlightingFor, syntaxHighlighting } from "@codemirror/language";
import { tags as t } from "@lezer/highlight";
import { openideaHighlightStyle } from "./highlight";
import { openideaHighlightStyle, parameter } from "./highlight";
import { findLanguageDescription, loadSupport } from "./languages";
import { setSyntaxHighlights, syntaxHighlightField } from "./syntaxHighlight";
const views: EditorView[] = [];
afterEach(() => views.splice(0).forEach((view) => view.destroy()));
function marks(state: EditorState) {
const out: { text: string; cls: string }[] = [];
state.field(syntaxHighlightField).between(0, state.doc.length, (from, to, d) => {
@@ -40,4 +45,26 @@ describe("syntaxHighlightField", () => {
state = state.update({ changes: { from: 0, insert: " " } }).state;
expect(marks(state).map((m) => m.text)).toEqual(["def", "hi", ":ok"]);
});
it("lets Kotlin declaration roles override the stream lexer's generic definition style", async () => {
const support = await loadSupport(findLanguageDescription("Kotlin", "demo.kt")!);
const view = new EditorView({
parent: document.body,
doc: "fun greet(prefix: String) {}",
extensions: [support!, syntaxHighlightField, syntaxHighlighting(openideaHighlightStyle)],
});
views.push(view);
view.dispatch({
effects: setSyntaxHighlights.of({ kinds: ["function", "parameter"], data: [4, 9, 0, 10, 16, 1] }),
});
for (const [text, tag] of [
["greet", t.function(t.definition(t.variableName))],
["prefix", parameter],
] as const) {
const cls = highlightingFor(view.state, [tag])!;
const span = view.contentDOM.querySelector(`.${cls}`)!;
expect(span.textContent).toBe(text);
expect(span.children.length).toBe(0);
}
});
});
+5 -3
Переглянути файл
@@ -1,14 +1,15 @@
/**
* Tree-sitter highlighting (`doc_highlights`) for languages without a
* CodeMirror mode (Elixir, ...): backend kinds → Lezer tags, so the colours
* CodeMirror mode (Elixir, ...) or a limited stream mode (Kotlin): backend kinds → Lezer tags, so the colours
* come from the same highlight style and Color Scheme settings as Lezer
* highlighting. Refreshed with folding; in between, marks are mapped through
* the changes.
*/
import { StateEffect, StateField, type EditorState } from "@codemirror/state";
import { Prec, StateEffect, StateField, type EditorState } from "@codemirror/state";
import { Decoration, EditorView, type DecorationSet } from "@codemirror/view";
import { highlightingFor } from "@codemirror/language";
import { tags as t, type Tag } from "@lezer/highlight";
import { parameter } from "./highlight";
import type { SyntaxHighlightKind, SyntaxHighlights } from "@/platform/ipc";
const TAGS: Record<SyntaxHighlightKind, Tag> = {
@@ -27,6 +28,7 @@ const TAGS: Record<SyntaxHighlightKind, Tag> = {
builtin: t.standard(t.variableName),
type: t.typeName,
namespace: t.namespace,
parameter,
variable: t.variableName,
selfVariable: t.self,
property: t.propertyName,
@@ -62,5 +64,5 @@ export const syntaxHighlightField = StateField.define<DecorationSet>({
for (const e of tr.effects) if (e.is(setSyntaxHighlights)) return decorations(tr.state, e.value);
return tr.docChanged ? deco.map(tr.changes) : deco;
},
provide: (f) => EditorView.decorations.from(f),
provide: (f) => Prec.high(EditorView.decorations.from(f)),
});
+36 -1
Переглянути файл
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { EditorState } from "@codemirror/state";
import { indentUnit, language as languageFacet } from "@codemirror/language";
import { registerSaveParticipant, remoteChange } from "@/editor/api";
import { HEAVY_LIMIT, registerSaveParticipant, remoteChange } from "@/editor/api";
import { createMemoryBackend, type MemoryBackend } from "../backend";
import { _resetLanguages } from "../cm/languages";
import { DocumentManager, ModifiedRanges, saveCleanupChanges, useDocMeta } from "./document";
@@ -38,6 +38,41 @@ function setup(files: Record<string, string>, settings = {}) {
afterEach(() => useDocMeta.setState({}, true));
describe("DocumentManager + slots", () => {
it("requests Kotlin declaration highlights while retaining its CodeMirror language", async () => {
const { backend, manager, frame } = setup({ "/p/Palette.kt": "class Greeter { fun greet() {} }" });
const highlights = vi.spyOn(backend, "highlights").mockResolvedValue({ kinds: ["function"], data: [20, 25, 0] });
const model = await manager.acquire("/p/Palette.kt");
await frame();
expect(model.language).not.toBeNull();
expect(model.getMeta().language).toBe("kotlin");
await vi.waitFor(() => expect(highlights).toHaveBeenCalledWith(model.doc));
manager.stop();
});
it.each([
["Palette.kt", "fun greet() {}", 1],
["demo.py", "def greet(): pass", 0],
["demo.ex", "defmodule Demo do\nend", 1],
["Large.kt", " ".repeat(HEAVY_LIMIT + 1), 1],
])("debounces backend highlights after an edit burst in %s", async (name, text, requests) => {
const { backend, manager, frame } = setup({ [`/p/${name}`]: text });
const highlights = vi.spyOn(backend, "highlights");
try {
const model = await manager.acquire(`/p/${name}`);
const slot = new EditorSlot(model, name, host);
await frame();
expect(highlights).toHaveBeenCalledTimes(requests);
highlights.mockClear();
for (let i = 0; i < 5; i++) slot.dispatchLocal({ changes: { from: 0, insert: " " } });
await frame();
expect(highlights).not.toHaveBeenCalled();
await new Promise((resolve) => setTimeout(resolve, 550));
expect(highlights).toHaveBeenCalledTimes(requests);
} finally {
manager.stop();
}
});
it("gives a new slot the Code Style indent for Reformat Code, not the detected one", async () => {
const settings = {
editor: {
+2 -2
Переглянути файл
@@ -543,8 +543,8 @@ export class DocModel {
const [folds, outline, highlights] = await Promise.allSettled([
backend.folding(this.doc),
backend.outline(this.doc),
// Without a CodeMirror mode, colours come from the grammar's tree-sitter query.
this.desc ? null : backend.highlights(this.doc),
// Kotlin's stream mode lacks declaration categories; the grammar supplements it.
!this.desc || this.meta.language === "kotlin" ? backend.highlights(this.doc) : null,
]);
if (this.closed) return;
if (counter !== this.changeCounter) {
+13
Переглянути файл
@@ -1,6 +1,9 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { EditorState, Prec, type Extension } from "@codemirror/state";
import { Decoration, EditorView } from "@codemirror/view";
import { syntaxHighlighting } from "@codemirror/language";
import { openideaHighlightStyle } from "../../core/cm/highlight";
import { setSyntaxHighlights, syntaxHighlightField } from "../../core/cm/syntaxHighlight";
// One `property` token over the whole word, as jdtls reports an instance field.
const lspSemanticTokens = vi.fn(async () => ({
@@ -44,6 +47,16 @@ async function editor(extensions: Extension[]) {
}
describe("semantic decorations vs the syntax highlighter", () => {
it("keeps semantic roles inside supplemental backend syntax marks", async () => {
vi.useFakeTimers();
const view = await editor([syntaxHighlightField, syntaxHighlighting(openideaHighlightStyle), semanticExtension()]);
view.dispatch({ effects: setSyntaxHighlights.of({ kinds: ["variable"], data: [0, 13, 0] }) });
const sem = view.contentDOM.querySelector(".cm-sem-property")!;
expect(sem).not.toBeNull();
expect(sem.children.length).toBe(0);
expect(sem.parentElement!.textContent).toBe("nasProperties");
});
it("puts the semantic mark inside the syntax span, whichever is added last", async () => {
vi.useFakeTimers();
for (const extensions of [
+1
Переглянути файл
@@ -26,6 +26,7 @@ import { languageSemantics, type SemanticRange } from "./languages";
// elsewhere go through the re-exported `languageSemantics`.
export { languageSemantics } from "./languages";
// Registers each language's semantics (highlighting hooks, construct docs).
import "./kotlin";
import "./python";
import "./php";
+45
Переглянути файл
@@ -0,0 +1,45 @@
import { describe, expect, it } from "vitest";
import { Text } from "@codemirror/state";
import { semanticRanges } from "../index";
describe("Kotlin semantic tokens", () => {
it.each([
["variable", "modification", ["cm-sem-mutable"]],
["property", "modification", ["cm-sem-property", "cm-sem-mutable"]],
["parameter", "modification", ["cm-sem-parameter", "cm-sem-mutable"]],
["operator", "declaration", ["cm-sem-declaration", "cm-sem-function"]],
["label", "", ["cm-sem-constant"]],
])("recognizes %s with %s", (type, modifier, expected) => {
const tokens = { tokenTypes: [type], tokenModifiers: [modifier], data: [0, 0, 4, 0, 1] };
const result = semanticRanges(Text.of(["name"]), tokens, false, "kotlin");
expect(result[0]?.className.split(" ")).toEqual(expected);
});
it("keeps modification language-specific and leaves val properties as properties", () => {
const doc = Text.of(["name"]);
expect(
semanticRanges(
doc,
{
tokenTypes: ["variable"],
tokenModifiers: ["modification"],
data: [0, 0, 4, 0, 1],
},
false,
"typescript",
),
).toEqual([]);
expect(
semanticRanges(
doc,
{
tokenTypes: ["property"],
tokenModifiers: ["readonly"],
data: [0, 0, 4, 0, 1],
},
false,
"kotlin",
)[0]?.className,
).toBe("cm-sem-property");
});
});
+13
Переглянути файл
@@ -0,0 +1,13 @@
import { registerLanguageSemantics, type LanguageSemantics } from "../languages";
const kotlinSemantics: LanguageSemantics = {
tokenClasses({ type, modifiers, classes }) {
if (type === "operator" && modifiers.includes("declaration")) classes.push("cm-sem-function");
// JetBrains Kotlin LSP uses `modification` for mutable declarations and references.
if ((type === "variable" || type === "property" || type === "parameter") && modifiers.includes("modification"))
classes.push("cm-sem-mutable");
if (type === "label") classes.push("cm-sem-constant");
},
};
registerLanguageSemantics("kotlin", kotlinSemantics);