diff --git a/package.json b/package.json index ff67a6760..d6d630c8b 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@codemirror/lang-javascript": "^6.2.5", "@codemirror/language": "^6.13.1", "@codemirror/language-data": "^6.5.2", + "@codemirror/legacy-modes": "^6.5.5", "@codemirror/lint": "^6.9.7", "@codemirror/merge": "^6.12.2", "@codemirror/search": "^6.7.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 80c5ac0e9..6e2995a32 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: '@codemirror/language-data': specifier: ^6.5.2 version: 6.5.2 + '@codemirror/legacy-modes': + specifier: ^6.5.5 + version: 6.5.5 '@codemirror/lint': specifier: ^6.9.7 version: 6.9.7 diff --git a/src/editor/core/cm/interpolation.test.ts b/src/editor/core/cm/interpolation.test.ts new file mode 100644 index 000000000..817719d8b --- /dev/null +++ b/src/editor/core/cm/interpolation.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { EditorState } from "@codemirror/state"; +import { ensureSyntaxTree, syntaxTree } from "@codemirror/language"; +import { classHighlighter, highlightTree } from "@lezer/highlight"; +import { findLanguageDescription, loadSupport, loadedSupport } from "./languages"; + +async function tokens(mode: string, source: string) { + const desc = findLanguageDescription(mode, "")!; + const support = await loadSupport(desc); + expect(loadedSupport(desc)).toBe(support); + const state = EditorState.create({ doc: source, extensions: [support!] }); + ensureSyntaxTree(state, source.length, 1000); + const result: { text: string; style: string; from: number; to: number }[] = []; + highlightTree(syntaxTree(state), classHighlighter, (from, to, style) => + result.push({ text: source.slice(from, to), style, from, to }), + ); + return result; +} + +describe("frontend string interpolation", () => { + it.each([ + ["Kotlin", 'println("$i + 1 = ${i + 1}")'], + ["Scala", 'println(s"$i + 1 = ${i + 1}")'], + ["Dart", 'print("$i + 1 = ${i + 1}")'], + ["Groovy", 'println("$i + 1 = ${i + 1}")'], + ["C#", 'Console.WriteLine($"{i} + 1 = {i + 1}")'], + ])("%s separates delimiters, names, operators and numbers", async (mode, source) => { + const out = await tokens(mode, source); + expect(out.filter((t) => t.text === "i")).toHaveLength(2); + expect(out.filter((t) => t.text === "i").every((t) => t.style.includes("tok-variableName"))).toBe(true); + expect(out).toContainEqual(expect.objectContaining({ text: "+", style: "tok-operator" })); + expect(out).toContainEqual(expect.objectContaining({ text: "1", style: "tok-number" })); + expect(out.some((t) => t.text === (mode === "C#" ? "{" : "${") && !t.style.includes("tok-string"))).toBe(true); + expect(out.some((t) => t.text === "}" && !t.style.includes("tok-string"))).toBe(true); + }); + + it.each(["Kotlin", "Scala", "Dart", "Groovy", "C#"])( + "%s keeps nested braces, strings and comments inside expressions", + async (mode) => { + const source = + mode === "C#" + ? '$"{call(new Foo { n = 2 }, "}") /* } */ + 1} tail"' + : `${mode === "Scala" ? "s" : ""}"\${call({ n -> 2 }, "}") /* } */ + 1} tail"`; + const out = await tokens(mode, source); + expect(out).toContainEqual(expect.objectContaining({ text: "1", style: "tok-number" })); + expect(out.some((t) => t.text.includes("tail") && t.style === "tok-string")).toBe(true); + expect(out.some((t) => t.text.includes("/* } */") && t.style === "tok-comment")).toBe(true); + }, + ); + + it("handles Kotlin multiline raw templates and nested templates", async () => { + const out = await tokens("Kotlin", 'val s = """raw \\$name\n${list.map { "${it + 1}" }} tail"""\nval after = 3'); + for (const text of ["name", "it", "after"]) + expect(out.some((t) => t.text === text && t.style.includes("variableName"))).toBe(true); + expect(out.filter((t) => t.style === "tok-number").map((t) => t.text)).toEqual(["1", "3"]); + }); + + it.each([ + ["Kotlin", '"\\$name \\${value}"'], + ["Scala", '"$name ${value}"'], + ["Scala", 's"$$name"'], + ["Dart", 'r"$name ${value}"'], + ["Groovy", "'$name ${value}'"], + ["C#", '$"{{value}}"'], + ])("%s preserves literal dollars and braces in %s", async (mode, source) => { + const out = await tokens(mode, source); + expect(out.filter((t) => t.style !== "tok-string" && t.text !== "r")).toEqual( + mode === "Scala" && source.startsWith("s") ? [expect.objectContaining({ text: "s" })] : [], + ); + }); + + it("handles C# verbatim interpolation and doubled quotes", async () => { + const out = await tokens("C#", '$@"say ""hello"" {i + 1}\nend"'); + expect(out).toContainEqual(expect.objectContaining({ text: "1", style: "tok-number" })); + expect(out.some((t) => t.text.includes('say ""hello""') && t.style === "tok-string")).toBe(true); + }); + it("handles C# raw interpolation with multiple-dollar delimiters", async () => { + const out = await tokens("C#", '$$"""literal {brace} {{i + 1}} tail"""'); + expect(out).toContainEqual(expect.objectContaining({ text: "1", style: "tok-number" })); + expect(out.some((t) => t.text === "{{" && !t.style.includes("string"))).toBe(true); + expect(out.some((t) => t.text === "}}" && !t.style.includes("string"))).toBe(true); + expect(out.some((t) => t.text.includes("{brace}") && t.style === "tok-string")).toBe(true); + }); + + it("keeps Groovy dotted interpolation in code", async () => { + const out = await tokens("Groovy", '"$person.name tail"'); + expect(out).toContainEqual(expect.objectContaining({ text: "name", style: "tok-variableName" })); + expect(out.some((t) => t.text.includes("tail") && t.style === "tok-string")).toBe(true); + }); + it.each([ + ["Kotlin", "a.kt"], + ["Scala", "a.scala"], + ["Dart", "a.dart"], + ["Groovy", "a.groovy"], + ["C#", "a.cs"], + ["Shell", "a.sh"], + ])("%s uses the same patched mode by name, alias and filename", (mode, path) => { + const description = findLanguageDescription(mode, path); + expect(findLanguageDescription(null, path)).toBe(description); + expect(findLanguageDescription(mode.toLowerCase(), path)).toBe(description); + }); +}); diff --git a/src/editor/core/cm/interpolation.ts b/src/editor/core/cm/interpolation.ts new file mode 100644 index 000000000..1615957f4 --- /dev/null +++ b/src/editor/core/cm/interpolation.ts @@ -0,0 +1,135 @@ +import { StringStream, type StreamParser } from "@codemirror/language"; +import { tags } from "@lezer/highlight"; + +export type InterpolationDialect = "Kotlin" | "Scala" | "Dart" | "Groovy" | "C#"; +type Literal = { + quote: string; + interpolate: boolean; + escape: boolean; + verbatim: boolean; + multiline: boolean; + braces: number; +}; +type Frame = { code: S; literal?: Literal; depth: number; name?: boolean; close?: string }; +type State = { frames: Frame[]; indentUnit: number }; + +/** Preserve the stock code tokenizer while suspending strings around embedded code. */ +export function interpolationParser(base: StreamParser, dialect: InterpolationDialect): StreamParser> { + const start = (unit: number) => base.startState!(unit); + const copy = (state: S): S => + base.copyState + ? base.copyState(state) + : (Object.fromEntries( + Object.entries(state as object).map(([key, value]) => [key, Array.isArray(value) ? value.slice() : value]), + ) as S); + return { + name: base.name, + languageData: base.languageData, + tokenTable: { ...base.tokenTable, templateDelimiter: tags.special(tags.brace) }, + startState: (indentUnit) => ({ frames: [{ code: start(indentUnit), depth: 0 }], indentUnit }), + copyState: (state) => ({ + ...state, + frames: state.frames.map((frame) => ({ + ...frame, + code: copy(frame.code), + literal: frame.literal && { ...frame.literal }, + })), + }), + indent: (state, text, cx) => base.indent?.(state.frames.at(-1)!.code, text, cx) ?? null, + token(stream, state) { + const frame = state.frames.at(-1)!; + if (frame.name) { + if (dialect === "Groovy" && stream.eat(".")) return "punctuation"; + stream.match(/^[\p{L}_][\p{L}\p{N}_]*/u); + if (dialect !== "Groovy" || !stream.match(/^\.[\p{L}_]/u, false)) state.frames.pop(); + return "variableName"; + } + const literal = frame.literal; + if (literal) { + const marker = () => + literal.interpolate && + (dialect === "C#" + ? literal.quote.length >= 3 + ? stream.match("{".repeat(literal.braces), false) + : stream.match(/^\{(?!\{)/, false) + : stream.match(/^\$(?:\{|[\p{L}_])/u, false)); + if (marker()) { + stream.pos += dialect === "C#" ? literal.braces : 1; + const expression = dialect === "C#" || !!stream.eat("{"); + state.frames.push({ + code: start(state.indentUnit), + depth: 0, + name: !expression, + close: "}".repeat(literal.braces), + }); + return "templateDelimiter"; + } + while (!stream.eol()) { + if (literal.verbatim && stream.match('""')) continue; + if (stream.match(literal.quote)) { + frame.literal = undefined; + break; + } + if (dialect === "C#" && literal.quote.length === 1 && (stream.match("{{") || stream.match("}}"))) continue; + if (dialect === "Scala" && literal.interpolate && stream.match("$$")) continue; + if (marker()) break; + if (stream.next() === "\\" && literal.escape) stream.next(); + } + if (stream.eol() && !literal.multiline) frame.literal = undefined; + return "string"; + } + // Probe with a copy so quotes/braces inside comments or characters stay opaque. + const from = stream.pos; + const probe = copy(frame.code); + const style = base.token(stream, probe); + const end = stream.pos; + stream.pos = from; + const text = stream.string.slice(from); + let opening: string | undefined; + let quote = '"'; + let interpolate = true; + let verbatim = false; + let braces = 1; + if (dialect === "C#") { + const raw = text.match(/^(\$+)("{3,})/); + opening = raw?.[0] ?? text.match(/^(?:\$@|@\$|\$)"/)?.[0]; + if (raw) { + quote = raw[2]!; + braces = raw[1]!.length; + } + verbatim = !!opening?.includes("@"); + } else if (style === "string" && (text[0] === '"' || (dialect === "Dart" && text[0] === "'"))) { + quote = text.startsWith(text[0]!.repeat(3)) ? text[0]!.repeat(3) : text[0]!; + opening = quote; + const prefix = stream.string.slice(0, from).match(/[\p{L}_][\p{L}\p{N}_]*$/u)?.[0]; + interpolate = dialect === "Scala" ? !!prefix : !(dialect === "Dart" && prefix === "r"); + } + if (opening && style !== "comment" && style !== "string.special") { + stream.pos = from + opening.length; + base.token(new StringStream('""', 4, state.indentUnit), frame.code); + frame.literal = { + quote, + braces, + interpolate, + verbatim, + escape: + !verbatim && + !(["Kotlin", "Scala", "C#"].includes(dialect) && quote.length >= 3) && + !(dialect === "Scala" && stream.string.slice(0, from).endsWith("raw")), + multiline: quote.length >= 3 || verbatim, + }; + return "string"; + } + if (state.frames.length > 1 && style !== "comment" && !style?.startsWith("string") && end === from + 1) { + if (frame.depth === 0 && stream.match(frame.close ?? "}")) { + state.frames.pop(); + return "templateDelimiter"; + } + if (text[0] === "{") frame.depth++; + if (text[0] === "}") frame.depth--; + } + stream.pos = from; + return base.token(stream, frame.code); + }, + }; +} diff --git a/src/editor/core/cm/interpolationIncremental.test.ts b/src/editor/core/cm/interpolationIncremental.test.ts new file mode 100644 index 000000000..0d2dc0cde --- /dev/null +++ b/src/editor/core/cm/interpolationIncremental.test.ts @@ -0,0 +1,43 @@ +import { expect, it } from "vitest"; +import { EditorState } from "@codemirror/state"; +import { ensureSyntaxTree, syntaxTree } from "@codemirror/language"; +import { classHighlighter, highlightTree } from "@lezer/highlight"; +import { findLanguageDescription, loadSupport } from "./languages"; + +function highlighted(state: EditorState) { + ensureSyntaxTree(state, state.doc.length, 1000); + const result: { text: string; style: string }[] = []; + highlightTree(syntaxTree(state), classHighlighter, (from, to, style) => { + result.push({ text: state.doc.sliceString(from, to), style }); + }); + return result; +} + +it.each([ + ["Kotlin", 'val text = """${call({ n ->\n n + 1\n})} tail"""\nval after = 3'], + ["Scala", 'val text = s"""${call({ n =>\n n + 1\n})} tail"""\nval after = 3'], + ["Dart", 'var text = """${call(() {\n return 1;\n})} tail""";\nvar after = 3;'], + ["Groovy", 'def text = """${call({ n ->\n n + 1\n})} tail"""\ndef after = 3'], + ["C#", 'var text = $@"{call(() => {\n return 1;\n})} tail";\nvar after = 3;'], + ["Shell", 'echo "$(echo\n $((n + 1))\n) tail"\necho 3'], +])("%s incremental highlighting agrees with a fresh parse", async (mode, doc) => { + const support = await loadSupport(findLanguageDescription(mode, "")!); + expect(support).toBeTruthy(); + let state = EditorState.create({ doc, extensions: [support!] }); + const original = highlighted(state); + const position = doc.indexOf("1"); + state = state.update({ changes: { from: position, to: position + 1, insert: "2 + 4" } }).state; + expect(highlighted(state)).toEqual(highlighted(EditorState.create({ doc: state.doc, extensions: [support!] }))); + expect(highlighted(state)).toContainEqual({ text: "4", style: "tok-number" }); + state = state.update({ changes: { from: position, to: position + 5, insert: "1" } }).state; + expect(highlighted(state)).toEqual(original); +}); + +it("loads shell interpolation through the editor's language registry", async () => { + const support = await loadSupport(findLanguageDescription("Shell", "example.sh")!); + const out = highlighted(EditorState.create({ doc: 'echo "$name $(echo $((n + 1)))"', extensions: [support!] })); + expect(out).toContainEqual({ text: "name", style: "tok-variableName" }); + expect(out).toContainEqual({ text: "1", style: "tok-number" }); + expect(out).toContainEqual({ text: "+", style: "tok-operator" }); + expect(out.some((token) => token.text === "$(" && !token.style.includes("string"))).toBe(true); +}); diff --git a/src/editor/core/cm/languages.ts b/src/editor/core/cm/languages.ts index 396966d6f..51e78e1fe 100644 --- a/src/editor/core/cm/languages.ts +++ b/src/editor/core/cm/languages.ts @@ -60,8 +60,30 @@ export function uiModeForPath(path: string): string | null | undefined { const supports = new Map>(); -/** Modes replacing `@codemirror/language-data`'s: Angular templates with control-flow blocks. */ +/** Frontend modes with template syntax missing from the bundled parsers. */ const OWN_MODES = [ + LanguageDescription.of({ + name: "Shell", + load: async () => { + const { shell } = await import("@codemirror/legacy-modes/mode/shell"); + const { shellInterpolation } = await import("./shellInterpolation"); + return new LanguageSupport(StreamLanguage.define(shellInterpolation(shell))); + }, + }), + ...(["Kotlin", "Scala", "Dart", "Groovy", "C#"] as const).map((name) => + LanguageDescription.of({ + name, + load: async () => { + const { interpolationParser } = await import("./interpolation"); + const modes = await import("@codemirror/legacy-modes/mode/clike"); + const base = + name === "Groovy" + ? (await import("@codemirror/legacy-modes/mode/groovy")).groovy + : modes[name === "C#" ? "csharp" : (name.toLowerCase() as "kotlin" | "scala" | "dart")]; + return new LanguageSupport(StreamLanguage.define(interpolationParser(base, name))); + }, + }), + ), LanguageDescription.of({ name: "Angular Template", load: () => import("./angular").then((m) => m.angular()) }), ]; @@ -144,11 +166,11 @@ export function findLanguageDescription(uiMode: string | null, path: string): La const exact = BUILTIN_MODES.find((l) => l.name === uiMode); if (exact) return exact; const loose = LanguageDescription.matchLanguageName(BUILTIN_MODES, uiMode, false); - if (loose) return loose; + if (loose) return OWN_MODES.find((l) => l.name === loose.name) ?? loose; } - return uiMode === null && infos - ? null - : LanguageDescription.matchFilename(BUILTIN_MODES, path.slice(path.lastIndexOf("/") + 1)); + if (uiMode === null && infos) return null; + const matched = LanguageDescription.matchFilename(BUILTIN_MODES, path.slice(path.lastIndexOf("/") + 1)); + return OWN_MODES.find((l) => l.name === matched?.name) ?? matched; } /** Loads (once) the CodeMirror language for `desc`. */ diff --git a/src/editor/core/cm/shellInterpolation.test.ts b/src/editor/core/cm/shellInterpolation.test.ts new file mode 100644 index 000000000..6a8cd373f --- /dev/null +++ b/src/editor/core/cm/shellInterpolation.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { StreamLanguage } from "@codemirror/language"; +import { shell } from "@codemirror/legacy-modes/mode/shell"; +import { highlightTree, tags } from "@lezer/highlight"; +import { shellInterpolation } from "./shellInterpolation"; +import { openideaHighlightStyle } from "./highlight"; + +const language = StreamLanguage.define(shellInterpolation(shell)); +function tokens(doc: string) { + const result: { text: string; cls: string }[] = []; + highlightTree(language.parser.parse(doc), openideaHighlightStyle, (from, to, cls) => { + result.push({ text: doc.slice(from, to), cls }); + }); + return result; +} +const token = (text: string, tag: typeof tags.string) => ({ text, cls: openideaHighlightStyle.style([tag]) }); + +describe("shell interpolation", () => { + it("separates variables, command substitutions and arithmetic inside strings", () => { + const result = tokens('echo "$name ${count:-1} $(echo hi) $((count + 1))"'); + for (const text of ["$", "${", "}", "$(", ")", "$((", "))"]) { + expect(result).toContainEqual(token(text, tags.special(tags.brace))); + } + for (const text of ["name", "count"]) expect(result).toContainEqual(token(text, tags.variableName)); + expect(result).toContainEqual(token("1", tags.number)); + expect(result).toContainEqual(token("+", tags.operator)); + expect(result).toContainEqual(token("echo", tags.standard(tags.variableName))); + }); + + it("highlights parameter operators without treating length expansion as a comment", () => { + const result = tokens('echo "${#name} ${name:-1}"'); + expect(result).toContainEqual(token("#", tags.operator)); + expect(result).toContainEqual(token("name", tags.variableName)); + expect(result).toContainEqual(token(":-", tags.operator)); + expect(result.some((item) => item.cls === openideaHighlightStyle.style([tags.comment]))).toBe(false); + }); + + it("keeps nested command substitutions and quoted strings independent", () => { + const result = tokens('echo "$(echo "$(echo hi)") after"'); + expect(result.filter((item) => item.text === "$(")).toHaveLength(2); + expect(result.filter((item) => item.text === ")")).toHaveLength(2); + expect(result).toContainEqual(token(' after"', tags.string)); + }); + + it("leaves single quotes, escaped dollars, comments and heredocs intact", () => { + for (const doc of ["echo '$name ${count}'", 'echo "\\$name"', '# "${count}"', "cat <<'EOF'\n$name ${count}\nEOF"]) { + expect(tokens(doc).some((item) => item.cls === openideaHighlightStyle.style([tags.special(tags.brace)]))).toBe( + false, + ); + } + }); +}); diff --git a/src/editor/core/cm/shellInterpolation.ts b/src/editor/core/cm/shellInterpolation.ts new file mode 100644 index 000000000..c3ca47d23 --- /dev/null +++ b/src/editor/core/cm/shellInterpolation.ts @@ -0,0 +1,124 @@ +import { type StreamParser } from "@codemirror/language"; +import { tags } from "@lezer/highlight"; + +type Frame = + | { kind: "string"; quote: string } + | { kind: "code"; close: string; depth: number; base: S; parameter: boolean }; + +type State = { frames: Frame[]; identifier: boolean; indentUnit: number }; + +/** Keep shell expansions out of the enclosing string's token. */ +export function shellInterpolation(base: StreamParser): StreamParser> { + const fresh = (unit: number) => base.startState!(unit); + const copy = (state: S): S => { + if (base.copyState) return base.copyState(state); + const result = { ...state }; + for (const key in result) { + if (Array.isArray(result[key])) result[key] = result[key].slice() as S[typeof key]; + } + return result; + }; + return { + name: base.name, + languageData: base.languageData, + tokenTable: { ...base.tokenTable, templateDelimiter: tags.special(tags.brace) }, + startState: (unit) => ({ + frames: [{ kind: "code", close: "", depth: 0, base: fresh(unit), parameter: false }], + identifier: false, + indentUnit: unit, + }), + copyState: (state) => ({ + ...state, + frames: state.frames.map((frame) => + frame.kind === "code" ? { ...frame, base: copy(frame.base) } : { ...frame }, + ), + }), + token(stream, state) { + const frame = state.frames[state.frames.length - 1]!; + if (state.identifier) { + state.identifier = false; + stream.match(/^(?:[A-Za-z_][\w]*|[0-9@*#?$!_-])/); + return "variableName"; + } + const expansion = () => { + const match = stream.match(/^\$(\(\(|\(|\{)/); + if (match) { + const open = (match as RegExpMatchArray)[1]!; + state.frames.push({ + kind: "code", + close: open === "{" ? "}" : open === "((" ? "))" : ")", + depth: 0, + base: fresh(state.indentUnit), + parameter: open === "{", + }); + return true; + } + if (stream.match(/^\$(?=[A-Za-z_0-9@*#?$!_-])/)) { + state.identifier = true; + return true; + } + return false; + }; + if (frame.kind === "string") { + if (frame.quote === '"' && expansion()) return "templateDelimiter"; + if (frame.quote === '"' && stream.eat("`")) { + state.frames.push({ kind: "code", close: "`", depth: 0, base: fresh(state.indentUnit), parameter: false }); + return "templateDelimiter"; + } + while (!stream.eol()) { + const ch = stream.next(); + if (ch === frame.quote) { + state.frames.pop(); + break; + } + if (ch === "\\" && frame.quote !== "'") { + stream.next(); + continue; + } + if ( + frame.quote === '"' && + (stream.peek() === "`" || /^\$(?:[({A-Za-z_0-9@*#?$!_-])/.test(stream.string.slice(stream.pos))) + ) + break; + } + return "string"; + } + // Let the stock mode retain ownership of heredoc bodies and comments. + const position = stream.pos; + const style = base.token(stream, copy(frame.base)); + stream.pos = position; + if ((style === "comment" && !frame.parameter && frame.close !== "))") || style === "string.special") + return base.token(stream, frame.base); + if (frame.close && frame.depth === 0 && stream.match(frame.close)) { + state.frames.pop(); + return "templateDelimiter"; + } + if (expansion()) return "templateDelimiter"; + if (stream.match(/^["']/)) { + state.frames.push({ kind: "string", quote: stream.current() }); + return "string"; + } + if (stream.eat("`")) { + state.frames.push({ kind: "code", close: "`", depth: 0, base: fresh(state.indentUnit), parameter: false }); + return "templateDelimiter"; + } + if (frame.close && frame.close !== "`") { + const opening = frame.close === "}" ? "{" : "("; + if (stream.eat(opening)) { + frame.depth++; + return "punctuation"; + } + if (frame.depth > 0 && stream.eat(frame.close[0]!)) { + frame.depth--; + return "punctuation"; + } + } + if (frame.close === "))" || frame.parameter) { + if (stream.match(/^\d+/)) return "number"; + if (stream.match(/^[A-Za-z_]\w*/)) return "variableName"; + if (stream.eat("/") || stream.match(/^[#+*%=!:?~^&|<>-]+/)) return "operator"; + } + return base.token(stream, frame.base); + }, + }; +}