diff --git a/crates/ide-language/src/syntax/cache.rs b/crates/ide-language/src/syntax/cache.rs index 0969fe7df..038ace1cb 100644 --- a/crates/ide-language/src/syntax/cache.rs +++ b/crates/ide-language/src/syntax/cache.rs @@ -209,8 +209,9 @@ impl SyntaxCache { |m| &mut m.folding, || { self.with_tree(doc, lang, version, text, |g, tree, text| { - Some(folding::folding(g.spec, g.kinds(), tree, text)) + Some(folding::folding(g.spec, g.kinds(), tree, text, self.syntax.comment_tokens(lang))) }) + .or_else(|| Some(self.syntax.custom_folding(lang, text))) }, ) } diff --git a/crates/ide-language/src/syntax/folding.rs b/crates/ide-language/src/syntax/folding.rs index 9d17298b9..8acfebf48 100644 --- a/crates/ide-language/src/syntax/folding.rs +++ b/crates/ide-language/src/syntax/folding.rs @@ -2,7 +2,7 @@ //! comment runs, import runs and `region` / `endregion` marker comments. use super::grammars::Spec; -use crate::{FoldKind, FoldRange}; +use crate::{FoldKind, FoldRange, LanguageConfig}; use tree_sitter::{Language, Node, Tree, TreeCursor}; const PAIRS: &[(&str, &str)] = &[("{", "}"), ("[", "]"), ("(", ")"), ("#{", "}"), ("${", "}"), ("@{", "}")]; @@ -85,7 +85,25 @@ impl KindTable { } } -pub(crate) fn folding(spec: &Spec, kinds: &KindTable, tree: &Tree, text: &str) -> Vec { +#[derive(Default)] +pub(super) struct CommentTokens { + line: Option, + block: Option<(String, String)>, +} + +impl From<&LanguageConfig> for CommentTokens { + fn from(config: &LanguageConfig) -> Self { + Self { line: config.line_comment.clone(), block: config.block_comment.clone() } + } +} + +pub(super) fn folding( + spec: &Spec, + kinds: &KindTable, + tree: &Tree, + text: &str, + tokens: &CommentTokens, +) -> Vec { let mut out: Vec = Vec::new(); let mut comments: Vec = Vec::new(); let mut cursor = tree.walk(); @@ -96,7 +114,7 @@ pub(crate) fn folding(spec: &Spec, kinds: &KindTable, tree: &Tree, text: &str) - start_row: start, end_row: last_content_row(&node, text), own_line: own_line(text, node.start_byte(), node.start_position().column), - marker: region_marker(text.get(node.start_byte()..node.end_byte()).unwrap_or_default()), + marker: region_marker(text.get(node.start_byte()..node.end_byte()).unwrap_or_default(), tokens), }); return true; } @@ -185,7 +203,7 @@ fn visit(cursor: &mut TreeCursor, mut f: impl FnMut(Node) -> bool) { } pub(crate) fn is_comment(kind: &str) -> bool { - kind.contains("comment") + kind.contains("comment") || kind == "Comment" } /// Row of the closing token when `node` is delimited by a matching bracket pair. @@ -220,21 +238,64 @@ fn own_line(text: &str, byte: usize, column: usize) -> bool { } /// `Some(true)` for a region start marker, `Some(false)` for an end marker. -fn region_marker(comment: &str) -> Option { - let body = comment - .trim_start_matches(['/', '*', '#', '-', ';', '%', '!', '<', ' ', '\t']) - .trim_start_matches("--") - .trim_start(); - let lower = body.get(..body.len().min(32))?.to_ascii_lowercase(); - if lower.starts_with("endregion") || lower.starts_with("#endregion") || lower.starts_with(" Option { + let comment = comment.trim(); + let body = if let Some((open, close)) = &tokens.block + && let Some(body) = comment.strip_prefix(open.as_str()) + { + let body = body.trim_end().strip_suffix(close.as_str())?; + if open == "/*" { body.trim_start_matches('*') } else { body } + } else if let Some(open) = &tokens.line + && let Some(body) = comment.strip_prefix(open.as_str()) + { + match open.as_str() { + "//" => body.trim_start_matches('/'), + ";" => body.trim_start_matches(';'), + _ => body, + } + } else { + return None; + }; + let body = body.trim(); + if body.contains(['\n', '\r']) { + return None; + } + let lower = body.to_ascii_lowercase(); + let region = lower.strip_prefix('#').unwrap_or(&lower); + let boundary = |tail: &str| !tail.starts_with(|c: char| c.is_ascii_alphanumeric() || c == '_'); + if region.strip_prefix("endregion").is_some_and(boundary) + || lower + .strip_prefix("')) + .is_some_and(|tail| tail.trim().is_empty()) + { Some(false) - } else if lower.starts_with("region") || lower.starts_with("#region") || lower.starts_with("')) + { Some(true) } else { None } } +/// Languages without a bundled grammar still support own-line custom markers. +pub(super) fn custom_folding(text: &str, tokens: &CommentTokens) -> Vec { + let comments: Vec<_> = text + .lines() + .enumerate() + .filter_map(|(row, line)| { + let marker = region_marker(line, tokens)?; + Some(Comment { start_row: row, end_row: row, own_line: true, marker: Some(marker) }) + }) + .collect(); + let mut out = Vec::new(); + comment_folds(&comments, &mut out); + out.sort_by_key(|f| f.start_line); + out +} + fn comment_folds(comments: &[Comment], out: &mut Vec) { let mut regions: Vec = Vec::new(); let mut run: Option<(usize, usize)> = None; diff --git a/crates/ide-language/src/syntax/mod.rs b/crates/ide-language/src/syntax/mod.rs index 776acdd4e..5e22c169f 100644 --- a/crates/ide-language/src/syntax/mod.rs +++ b/crates/ide-language/src/syntax/mod.rs @@ -32,6 +32,7 @@ pub struct Syntax { grammars: Vec, /// Language id (and grammar name) -> index into `grammars`. by_lang: HashMap, + comments: HashMap, } pub(crate) struct Grammar { @@ -107,7 +108,14 @@ impl Syntax { } } } - Self { grammars, by_lang } + let mut comments = HashMap::new(); + for cfg in languages { + comments.insert(cfg.id.as_str().to_owned(), folding::CommentTokens::from(cfg)); + if let Some(name) = &cfg.tree_sitter { + comments.entry(name.clone()).or_insert_with(|| folding::CommentTokens::from(cfg)); + } + } + Self { grammars, by_lang, comments } } /// Names of the bundled grammars (valid values for `tree-sitter = "..."`). @@ -161,8 +169,21 @@ impl Syntax { } pub fn folding(&self, lang: &LanguageId, text: &str) -> Vec { - self.analyze(lang, text, |g, tree, text| Some(folding::folding(g.spec, g.kinds(), tree, text))) - .unwrap_or_default() + let tokens = self.comment_tokens(lang); + self.analyze(lang, text, |g, tree, text| Some(folding::folding(g.spec, g.kinds(), tree, text, tokens))) + .unwrap_or_else(|| self.custom_folding(lang, text)) + } + + fn comment_tokens(&self, lang: &LanguageId) -> &folding::CommentTokens { + static EMPTY: OnceLock = OnceLock::new(); + self.comments.get(lang.as_str()).unwrap_or_else(|| EMPTY.get_or_init(Default::default)) + } + + fn custom_folding(&self, lang: &LanguageId, text: &str) -> Vec { + if text.len() > MAX_PARSE_BYTES { + return Vec::new(); + } + folding::custom_folding(text, self.comment_tokens(lang)) } pub fn tags(&self, lang: &LanguageId, text: &str) -> Vec { diff --git a/crates/ide-language/src/syntax/tests.rs b/crates/ide-language/src/syntax/tests.rs index 02c0ff6de..fde000c1b 100644 --- a/crates/ide-language/src/syntax/tests.rs +++ b/crates/ide-language/src/syntax/tests.rs @@ -969,6 +969,85 @@ fn g() {} ); } +#[test] +fn folding_xml_custom_regions() { + for (start, end) in [ + ("", ""), + ("", ""), + ("", ""), + ] { + let text = format!("\n {start}\n demo\n {end}\n\n"); + assert!(folds("xml", &text).contains(&(1, 3, FoldKind::Region)), "{text}"); + } +} + +#[test] +fn folding_custom_regions_use_configured_comment_tokens() { + let fixture: serde_json::Value = + serde_json::from_str(include_str!("../../../../fixtures/folding/regions.json")).unwrap(); + let registry = LanguageRegistry::load(None).unwrap(); + let cache = SyntaxCache::new(std::sync::Arc::new(Syntax::new())); + for language in fixture["languages"].as_array().unwrap() { + let id = language.as_str().unwrap(); + let cfg = registry.get(&lang(id)).unwrap(); + let mut tokens = Vec::new(); + if let Some(line) = &cfg.line_comment { + tokens.push((line.as_str(), "")); + } + if let Some((open, close)) = &cfg.block_comment { + tokens.push((open.as_str(), close.as_str())); + } + for (open, close) in tokens { + for space in ["", " ", " "] { + // Word delimiters require a separator in the language itself. + if space.is_empty() && open.ends_with(|c: char| c.is_ascii_alphanumeric()) { + continue; + } + let wrap = |body: &str| format!(" {open}{space}{body}{space}{close}"); + for marker in fixture["markers"].as_array().unwrap() { + let start = wrap(marker["start"].as_str().unwrap()); + let end = wrap(marker["end"].as_str().unwrap()); + let tokens = folding::CommentTokens::from(&cfg); + assert_eq!(folding::region_marker(&start, &tokens), Some(true), "{id}: {start}"); + assert_eq!(folding::region_marker(&end, &tokens), Some(false), "{id}: {end}"); + // Ruby block delimiters must occupy separate lines; Haskell + // interprets `--` followed immediately by a symbol as an operator. + if (id == "ruby" && open == "=begin") || (id == "haskell" && space.is_empty()) { + continue; + } + let text = format!("{start}\n\n{end}\n"); + let expected = vec![(0, 2, FoldKind::Region)]; + assert_eq!(folds(id, &text), expected, "{id}: {text}"); + let doc = ide_core::DocId::next(); + cache.open(doc, &lang(id), 0); + let actual = cache.folding(doc, &lang(id), 0, &text); + assert_eq!(cache.folding(doc, &lang(id), 0, &text), actual); + cache.close(doc); + assert_eq!( + actual.iter().map(|f| (f.start_line, f.end_line, f.kind)).collect::>(), + expected, + "cached {id}: {text}" + ); + let nested = format!("{end}\n\n{start}\n{start}\n\n{end}\n{end}\n{start}\n"); + assert_eq!( + folds(id, &nested), + vec![(2, 6, FoldKind::Region), (3, 5, FoldKind::Region)], + "nested {id}: {nested}" + ); + } + for invalid in fixture["invalid"].as_array().unwrap() { + let start = wrap(invalid.as_str().unwrap()); + assert_eq!( + folding::region_marker(&start, &folding::CommentTokens::from(&cfg)), + None, + "invalid {id}: {start}" + ); + } + } + } + } +} + #[test] fn folding_python_bodies_start_at_header() { let src = "import os\nimport sys\n\nclass A:\n def f(self):\n return [\n 1,\n ]\n\n\ndef g():\n pass\n"; diff --git a/fixtures/folding/regions.json b/fixtures/folding/regions.json new file mode 100644 index 000000000..c5dfa25b8 --- /dev/null +++ b/fixtures/folding/regions.json @@ -0,0 +1,74 @@ +{ + "languages": [ + "xml", + "html", + "vue", + "svelte", + "css", + "java", + "rust", + "typescript", + "python", + "csharp", + "sql", + "clojure", + "erlang", + "bat", + "ocaml", + "fsharp", + "haskell", + "lua", + "cmake", + "powershell", + "julia", + "ruby" + ], + "markers": [ + { + "start": "region Name", + "end": "endregion", + "name": "Name" + }, + { + "start": "region要折叠的内容", + "end": "endregion", + "name": "要折叠的内容" + }, + { + "start": "#region \"Quoted name\"", + "end": "#endregion", + "name": "Quoted name" + }, + { + "start": "REGION Mixed case", + "end": "ENDREGION", + "name": "Mixed case" + }, + { + "start": "region", + "end": "endregion", + "name": "" + }, + { + "start": "", + "end": "", + "name": "Head 内容" + }, + { + "start": "", + "end": "", + "name": "" + } + ], + "invalid": [ + "regional Name", + "regions.push(x)", + "endregionFoo", + "", + "", + "the region below", + "region_name", + " = { cmd: "bat", sql: "sql", xml: "xml", + svg: "xml", + vue: "vue", }; const MEM_LANGUAGES: LanguageInfo[] = [ @@ -240,6 +245,15 @@ const MEM_LANGUAGES: LanguageInfo[] = [ blockComment: null, indent: { tabSize: 4, insertSpaces: true }, }, + ...["xml", "html", "vue", "css"].map((id): LanguageInfo => ({ + id, + name: id.toUpperCase(), + extensions: id === "xml" ? ["xml", "svg"] : [id], + uiMode: id === "xml" ? "XML" : id === "css" ? "CSS" : id === "vue" ? "Vue" : "HTML", + lineComment: null, + blockComment: id === "css" ? ["/*", "*/"] : [""], + indent: { tabSize: 2, insertSpaces: true }, + })), { id: "plaintext", name: "Plain Text", @@ -507,8 +521,24 @@ export function createMemoryBackend(files: Record = {}): MemoryB }, async folding(doc) { const d = get(doc); - const out: FoldRange[] = []; - const seen = new Set(); + const info = MEM_LANGUAGES.find((language) => language.id === d.language); + const state = EditorState.create({ + doc: d.text, + extensions: EditorState.languageData.of(() => [ + { + commentTokens: { + line: info?.lineComment ?? undefined, + block: info?.blockComment && { open: info.blockComment[0], close: info.blockComment[1] }, + }, + }, + ]), + }); + const out: FoldRange[] = customRegions(state).map((region) => ({ + start_line: region.startLine - 1, + end_line: region.endLine - 1, + kind: "region", + })); + const seen = new Set(out.map((fold) => fold.start_line)); const stack: number[] = []; d.text.split("\n").forEach((line, i) => { for (const ch of line) { diff --git a/src/editor/core/cm/folding.test.ts b/src/editor/core/cm/folding.test.ts index 2166f0177..3db2474ca 100644 --- a/src/editor/core/cm/folding.test.ts +++ b/src/editor/core/cm/folding.test.ts @@ -78,7 +78,10 @@ describe("foldsFromRanges", () => { }); it("folds a custom region from its start marker through its end marker", () => { - const s = EditorState.create({ doc: "class A {\n //region Fields\n int a;\n //endregion\n}\n" }); + const s = EditorState.create({ + doc: "class A {\n //region Fields\n int a;\n //endregion\n}\n", + extensions: EditorState.languageData.of(() => [{ commentTokens: { line: "//" } }]), + }); const [f] = foldsFromRanges(s, [fold(1, 3)]); expect(f).toMatchObject({ from: s.doc.line(2).from + 2, to: s.doc.line(4).to, custom: true }); }); diff --git a/src/editor/core/cm/folding.ts b/src/editor/core/cm/folding.ts index f916b874e..593ad5858 100644 --- a/src/editor/core/cm/folding.ts +++ b/src/editor/core/cm/folding.ts @@ -24,7 +24,7 @@ import { } from "@codemirror/language"; import type { FoldRange } from "@/platform/ipc"; import { enclosingCodeBlock } from "./caretCommands"; -import { regionMarker, regionNameAt } from "./regions"; +import { regionMarkerAt, regionNameAt } from "./regions"; export interface BackendFold { /** Offset where the fold starts (end of the header line). */ @@ -58,7 +58,7 @@ export function foldsFromRanges(state: EditorState, ranges: readonly FoldRange[] if (r.start_line + 1 > lines || r.end_line + 1 > lines || r.end_line <= r.start_line) continue; const first = state.doc.line(r.start_line + 1); const end = state.doc.line(r.end_line + 1); - if (regionMarker(first.text)?.start) { + if (regionMarkerAt(state, first)?.start) { const from = first.from + /^\s*/.exec(first.text)![0].length; if (end.to > from) out.push({ from, to: end.to, kind: r.kind, custom: true }); continue; diff --git a/src/editor/core/cm/regions.test.ts b/src/editor/core/cm/regions.test.ts index dd15733d1..241ea966a 100644 --- a/src/editor/core/cm/regions.test.ts +++ b/src/editor/core/cm/regions.test.ts @@ -1,48 +1,107 @@ import { describe, expect, it } from "vitest"; -import { EditorState, Text } from "@codemirror/state"; -import { customRegions, regionMarker, regionNameAt } from "./regions"; +import { EditorState } from "@codemirror/state"; +import fixture from "../../../../fixtures/folding/regions.json"; +import { createMemoryBackend } from "../backend"; +import { commentTokens } from "./languages"; +import { foldsFromRanges } from "./folding"; +import { customRegions, regionMarker, regionNameAt, type RegionCommentTokens } from "./regions"; + +const definitions = import.meta.glob("../../../../languages/*.toml", { + query: "?raw", + import: "default", + eager: true, +}); + +function language(id: string) { + const source = definitions[`../../../../languages/${id}.toml`]!; + const lineComment = /^line-comment = (".*")$/m.exec(source)?.[1]; + const blockComment = /^block-comment = (\[.*\])$/m.exec(source)?.[1]; + return { + id, + name: id, + extensions: [], + uiMode: null, + indent: null, + lineComment: lineComment ? (JSON.parse(lineComment) as string) : null, + blockComment: blockComment ? (JSON.parse(blockComment) as [string, string]) : null, + }; +} + +const make = (doc: string, id = "java") => EditorState.create({ doc, extensions: commentTokens(language(id)) }); + +// The native folding suite consumes this same marker contract and the TOML tokens. +describe.each(fixture.languages)("custom regions in %s", (id) => { + const info = language(id); + const tokens: RegionCommentTokens = { + line: info.lineComment ?? undefined, + block: info.blockComment ? { open: info.blockComment[0], close: info.blockComment[1] } : undefined, + }; + const delimiters: [string, string][] = []; + if (tokens.line) delimiters.push([tokens.line, ""]); + if (tokens.block) delimiters.push([tokens.block.open, tokens.block.close]); + + it.each(delimiters)("uses %s … %s with optional spaces", (open, close) => { + for (const space of ["", " ", " "]) { + if (!space && /[a-z0-9]$/i.test(open)) continue; + const wrap = (body: string) => ` ${open}${space}${body}${space}${close}`; + for (const marker of fixture.markers) { + const start = wrap(marker.start); + const end = wrap(marker.end); + expect(regionMarker(start, tokens)).toEqual({ start: true, name: marker.name }); + expect(regionMarker(end, tokens)).toEqual({ start: false }); + const state = make(`${start}\n\n${end}\n`, id); + expect(customRegions(state)).toEqual([{ name: marker.name, startLine: 1, endLine: 3 }]); + expect(regionNameAt(state, 2)).toBe(marker.name || null); + expect(regionNameAt(state, 5)).toBeNull(); + const [fold] = foldsFromRanges(state, [{ start_line: 0, end_line: 2, kind: "region" }]); + expect(fold).toMatchObject({ from: 2, to: state.doc.line(3).to, custom: true }); + const nested = make(`${end}\n\n${start}\n${start}\n\n${end}\n${end}\n${start}\n`, id); + expect(customRegions(nested)).toEqual([ + { name: marker.name, startLine: 3, endLine: 7 }, + { name: marker.name, startLine: 4, endLine: 6 }, + ]); + } + for (const invalid of fixture.invalid) expect(regionMarker(wrap(invalid), tokens)).toBeNull(); + } + }); +}); describe("regionMarker", () => { - it.each([ - ["//region Accessors", "Accessors"], - [" // region Accessors", "Accessors"], - ["#region Setup", "Setup"], - ["# region Setup", "Setup"], - ["//#region Setup", "Setup"], - ['// ', "Getters"], - ['', "Head"], - ["/* region Block */", "Block"], - ["//region", ""], - ])("start %s", (line, name) => { - expect(regionMarker(line)).toEqual({ start: true, name }); + it.each(["///region Docs", "/**region Docs*/", "// #region Docs"])("preserves %s", (text) => { + expect(regionMarker(text, { line: "//", block: { open: "/*", close: "*/" } })).toEqual({ + start: true, + name: "Docs", + }); }); - it.each(["//endregion", "# endregion", "#endregion", "// ", ""])( - "end %s", - (line) => { - expect(regionMarker(line)).toEqual({ start: false }); + it.each(["const region = 1;", "region Name", "//region Wrong language", "" } })).toBeNull(); }, ); - it.each(["const region = 1;", "// the region below", "regions.push(x)"])("not a marker: %s", (line) => { - expect(regionMarker(line)).toBeNull(); + it("preserves C# region directives and their names", () => { + const state = make('#region "Fields"\nint a;\n#endregion', "csharp"); + expect(customRegions(state)).toEqual([{ name: "Fields", startLine: 1, endLine: 3 }]); + expect(regionNameAt(state, 0)).toBe("Fields"); + expect(foldsFromRanges(state, [{ start_line: 0, end_line: 2, kind: "region" }])[0]?.custom).toBe(true); + }); + + it("does not list plain text as a region without comment tokens", () => { + expect(customRegions(EditorState.create({ doc: "region Name\n\nendregion" }))).toEqual([]); }); }); -describe("customRegions", () => { - it("matches nested markers and drops unclosed ones", () => { - const doc = Text.of(["//region A", "//region B", "x", "//endregion", "//endregion", "//region C"]); - expect(customRegions(doc)).toEqual([ - { name: "A", startLine: 1, endLine: 5 }, - { name: "B", startLine: 2, endLine: 4 }, - ]); - }); -}); - -describe("regionNameAt", () => { - it("names a fold starting at a start marker", () => { - const state = EditorState.create({ doc: " //region Fields\nint a;\n//endregion" }); - expect(regionNameAt(state, 2)).toBe("Fields"); - expect(regionNameAt(state, 5)).toBeNull(); +describe("memory backend custom folding", () => { + it.each(["xml", "svg", "html", "vue", "css", "java", "py", "ts"])("folds %s markers", async (extension) => { + const path = `/project/file.${extension}`; + const css = extension === "css"; + const line = extension === "py" ? "#" : ["java", "ts"].includes(extension) ? "//" : null; + const open = line ?? (css ? "/*" : ""; + const backend = createMemoryBackend({ [path]: `${open}region Name${close}\ncontent\n${open}endregion${close}\n` }); + const doc = await backend.open(path); + expect(await backend.folding(doc.id)).toEqual([{ start_line: 0, end_line: 2, kind: "region" }]); }); }); diff --git a/src/editor/core/cm/regions.ts b/src/editor/core/cm/regions.ts index 43a0aaa29..cf4970190 100644 --- a/src/editor/core/cm/regions.ts +++ b/src/editor/core/cm/regions.ts @@ -1,30 +1,51 @@ /** * Custom folding regions: `//region Name` … `//endregion`, `#region` (C#, Python…), - * `` … ``, in any line comment style. + * `` … ``, in the language’s configured line or block comments. * The backend folds them; this names their placeholders and lists them for * Navigate › Custom Folding (⌥⌘.). */ -import type { EditorState, Text } from "@codemirror/state"; +import type { EditorState, Line } from "@codemirror/state"; export type RegionMarker = { start: true; name: string } | { start: false }; -const COMMENT_START = /^\s*(?:\/\/+|\/\*+|#|--|;+|%|)\s*$/, "") + .trim() .replace(/^"(.*)"$/, "$1") .trim(); return { start: true, name }; @@ -37,12 +58,19 @@ export interface CustomRegion { endLine: number; } -/** Every matched region of `doc`, in document order of their start markers. */ -export function customRegions(doc: Text): CustomRegion[] { +/** The marker using the configured comment syntax at this line. */ +export function regionMarkerAt(state: EditorState, line: Line): RegionMarker | null { + const tokens = state.languageDataAt("commentTokens", line.from)[0] ?? {}; + return regionMarker(line.text, tokens); +} + +/** Every matched region, in document order of its start marker. */ +export function customRegions(state: EditorState): CustomRegion[] { + const doc = state.doc; const out: CustomRegion[] = []; const open: { name: string; line: number; index: number }[] = []; for (let n = 1; n <= doc.lines; n++) { - const m = regionMarker(doc.line(n).text); + const m = regionMarkerAt(state, doc.line(n)); if (!m) continue; if (m.start) { open.push({ name: m.name, line: n, index: out.length }); @@ -59,6 +87,6 @@ export function customRegions(doc: Text): CustomRegion[] { export function regionNameAt(state: EditorState, pos: number): string | null { const line = state.doc.lineAt(pos); if (state.sliceDoc(line.from, pos).trim()) return null; - const m = regionMarker(line.text); + const m = regionMarkerAt(state, line); return m?.start && m.name ? m.name : null; }