feat(astro): add Astro language support and editor mode (#712)

Astro files get their own language: TypeScript frontmatter, HTML template with TSX expressions in text and tags, TypeScript scripts, astro-ls and oxlint servers, and file-type icons.

src/editor/core/cm/angular/index.ts

@@ -11,10 +11,10 @@ import { html } from "@codemirror/lang-html";
 import { javascriptLanguage } from "@codemirror/lang-javascript";
 import { styleTags, tags as t } from "@lezer/highlight";
 import { IterMode, parseMixed, type Input, type NestedParse, type SyntaxNode, type SyntaxNodeRef } from "@lezer/common";
+import { JS_TAGS } from "../jsTags";
 import { parser } from "./parser";
 
-// @lezer/javascript styles `.` as a deref operator, but not `?.`.
-const jsParser = javascriptLanguage.parser.configure({ props: [styleTags({ "?.": t.derefOperator })] });
+const jsParser = javascriptLanguage.parser.configure({ props: [styleTags(JS_TAGS)] });
 const exprParser = jsParser.configure({ top: "SingleExpression" });
 
 const baseParser = parser.configure({

src/editor/core/cm/astro/astro.grammar

@@ -0,0 +1,10 @@
+// Astro components: the frontmatter, a TypeScript script between `---` fences,
+// then the template. `index.ts` parses the script as TypeScript and the
+// template as HTML with `{…}` expressions. Regenerate with
+// `pnpm gen:astro-grammar`.
+
+@top Document { Frontmatter? Template? }
+
+Frontmatter { FrontmatterFence FrontmatterScript? FrontmatterFence? }
+
+@external tokens component from "./tokens" { FrontmatterFence, FrontmatterScript, Template }

src/editor/core/cm/astro/astro.test.ts

@@ -0,0 +1,164 @@
+import { describe, expect, it, vi } from "vitest";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { EditorState } from "@codemirror/state";
+import { indentUnit } from "@codemirror/language";
+import { buildParserFile } from "@lezer/generator";
+import { highlightTree, tagHighlighter, tags as t } from "@lezer/highlight";
+import { findLanguageDescription, loadSupport } from "../languages";
+import { pasteReindentFilter } from "../prefs";
+import { astro, astroLanguage } from "./index";
+
+const highlighter = tagHighlighter([
+  { tag: t.processingInstruction, class: "fence" },
+  { tag: t.definitionKeyword, class: "def-kw" },
+  { tag: t.keyword, class: "keyword" },
+  { tag: t.typeName, class: "type" },
+  { tag: t.function(t.variableName), class: "call" },
+  { tag: t.variableName, class: "var" },
+  { tag: t.propertyName, class: "prop" },
+  { tag: t.attributeName, class: "attr" },
+  { tag: t.attributeValue, class: "value" },
+  { tag: t.string, class: "str" },
+  { tag: t.operator, class: "op" },
+  { tag: t.brace, class: "brace" },
+  { tag: t.tagName, class: "tag" },
+  { tag: t.comment, class: "comment" },
+  { tag: t.invalid, class: "invalid" },
+]);
+
+/** `text:class` of the styled tokens of `doc`. */
+function tokens(doc: string): string[] {
+  const out: string[] = [];
+  highlightTree(astroLanguage.parser.parse(doc), highlighter, (from, to, cls) =>
+    out.push(`${doc.slice(from, to)}:${cls}`),
+  );
+  return out;
+}
+
+/** A layout with what made HTML mode misread Astro components. */
+const LAYOUT = `---
+import '@/styles/global.css'
+import Header from '@/components/Header.astro'
+
+interface Props {
+  title: string
+  visibility?: 'public' | 'private'
+}
+
+const { title, visibility = 'private' } = Astro.props
+// The canonical URL drops the trailing slash, as <link rel="canonical"> expects.
+const pageUrl = new URL(Astro.url.pathname.replace(/(.)\\/$/u, '$1'), Astro.url).href
+const image = { path: '/og.png', width: 1200 }
+---
+
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    {visibility === 'private' && <meta name="robots" content="noindex" />}
+    {visibility === 'public' && (
+      <>
+        <link rel="canonical" href={pageUrl} />
+        <meta property="og:image" content={new URL(image.path, Astro.url).href} />
+      </>
+    )}
+    <title>{title}</title>
+  </head>
+  <body class="page">
+    <Header {title} {...Astro.props} />
+    <slot />
+  </body>
+</html>
+
+<style is:global>
+  ::view-transition-old(root) {
+    display: none;
+  }
+</style>
+
+<script>
+  document.addEventListener('astro:after-swap', () => {
+    const style = document.head.querySelector('style[data-emotion]')
+    if (!style) {
+      return
+    }
+    style.remove()
+  })
+</script>
+`;
+
+describe("Astro components", () => {
+  it("parses the frontmatter as TypeScript, markup in its comments included", () => {
+    const doc = "---\ninterface P { title: string }\nconst a = 'x' // on <html>\n---\n<p>{a}</p>";
+    const out = tokens(doc);
+    expect(out).toEqual(
+      expect.arrayContaining([
+        "---:fence",
+        "interface:def-kw",
+        "string:type",
+        "const:def-kw",
+        "'x':str",
+        "// on <html>:comment",
+        "p:tag",
+        "a:var",
+      ]),
+    );
+    expect(out).not.toContain("html:tag");
+  });
+
+  it("parses expressions in text, fragments and markup inside them included", () => {
+    const out = tokens(
+      "<head>\n  {v === 'p' && <meta name=\"robots\" />}\n  {ok && (\n    <>\n      <b>{n}</b>\n    </>\n  )}\n</head>",
+    );
+    expect(out).toEqual(
+      expect.arrayContaining(["v:var", "===:op", "'p':str", "&&:op", "meta:tag", "name:attr", "n:var"]),
+    );
+    expect(out.filter((s) => s.endsWith(":invalid"))).toEqual([]);
+  });
+
+  it("parses expressions in tags, with spaces, shorthands and spreads", () => {
+    const out = tokens(
+      '<meta content={new URL(img.path, Astro.url).href} title="{literal}" />\n<C {title} {...rest} />',
+    );
+    expect(out).toEqual(expect.arrayContaining(["content:attr", "new:keyword", "img:var", "title:var", "rest:var"]));
+    expect(out).not.toContain("URL(img.path,:attr");
+    expect(out).toContain('"{literal}":value');
+  });
+
+  it("parses scripts as TypeScript and leaves styles to CSS", () => {
+    const out = tokens("<script>\n  const n: number = 1\n</script>\n<style>\n  p { color: red; }\n</style>");
+    expect(out).toEqual(expect.arrayContaining(["const:def-kw", "number:type", "script:tag", "style:tag"]));
+  });
+
+  it("keeps a well-indented component unchanged when it is pasted", () => {
+    // CodeMirror parses for at most 20 ms per transaction and indents with the partial tree.
+    // A frozen clock lets a slow CI runner parse the whole paste.
+    vi.useFakeTimers({ toFake: ["Date"] });
+    try {
+      const state = EditorState.create({
+        extensions: [astro(), indentUnit.of("  "), EditorState.tabSize.of(2), pasteReindentFilter("indentEachLine")],
+      });
+      const pasted = state.update({ changes: { from: 0, insert: LAYOUT }, userEvent: "input.paste" }).state;
+      expect(pasted.doc.toString()).toBe(LAYOUT);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it("is generated from astro.grammar", () => {
+    const dir = import.meta.dirname;
+    const { parser, terms } = buildParserFile(readFileSync(resolve(dir, "astro.grammar"), "utf8"), {
+      fileName: "src/editor/core/cm/astro/astro.grammar",
+      moduleStyle: "es",
+      typeScript: true,
+    });
+    expect(parser).toBe(readFileSync(resolve(dir, "parser.ts"), "utf8"));
+    expect(terms).toBe(readFileSync(resolve(dir, "parser.terms.ts"), "utf8"));
+  });
+
+  it("is the Astro mode", async () => {
+    const desc = findLanguageDescription("Astro", "/site/src/pages/index.astro");
+    expect((await loadSupport(desc!))?.language).toBe(astroLanguage);
+  });
+});

src/editor/core/cm/astro/index.ts

@@ -0,0 +1,134 @@
+/**
+ * Astro components: a TypeScript frontmatter between `---` fences
+ * (`astro.grammar`), then an HTML template whose `{…}` expressions, in text
+ * and in tags, are TSX. `<script>` is TypeScript, as Astro compiles it.
+ */
+import { LRLanguage, LanguageSupport } from "@codemirror/language";
+import { html } from "@codemirror/lang-html";
+import { tsxLanguage, typescriptLanguage } from "@codemirror/lang-javascript";
+import { styleTags, tags as t } from "@lezer/highlight";
+import { IterMode, parseMixed, type Input, type NestedParse, type SyntaxNodeRef } from "@lezer/common";
+import { JS_TAGS } from "../jsTags";
+import { parser } from "./parser";
+
+const tsParser = typescriptLanguage.parser.configure({ props: [styleTags(JS_TAGS)] });
+const tsxParser = tsxLanguage.parser.configure({ props: [styleTags(JS_TAGS)] });
+
+// To reuse old inner trees, parseMixed takes those that the node at a mount's
+// start and its ancestors had with the same parser. So each kind of mount has
+// its own instance (see angular/index.ts).
+const documentTextParser = tsxParser.configure({}),
+  elementTextParser = tsxParser.configure({}),
+  tagParser = tsxParser.configure({});
+
+const templateHTML = html({
+  selfClosingTags: true,
+  // Other script types (`application/ld+json`) keep HTML's own nesting.
+  nestedLanguages: [{ tag: "script", attrs: (a) => !a.type || a.type === "module", parser: tsParser }],
+});
+
+const BRACE_L = 123,
+  BRACE_R = 125,
+  BACKSLASH = 92,
+  QUOTES = new Set([34, 39, 96]);
+
+type Range = { from: number; to: number };
+
+/** Where a scan of expression braces stands between two pieces of text. */
+type Scan = { depth: number; quote: number };
+
+function pushRange(out: Range[], from: number, to: number) {
+  const last = out[out.length - 1];
+  if (last && last.to === from) last.to = to;
+  else out.push({ from, to });
+}
+
+/**
+ * Appends to `out` the parts of `text`, which starts at `from`, inside `{…}`,
+ * braces included. Strings count only inside braces, so prose apostrophes
+ * stay text; in a tag (`attrQuotes`), quotes outside braces delimit attribute
+ * values, whose braces are literal.
+ */
+function scanBraces(text: string, from: number, scan: Scan, out: Range[], attrQuotes = false) {
+  let start = scan.depth > 0 ? 0 : -1;
+  for (let i = 0; i < text.length; i++) {
+    const ch = text.charCodeAt(i);
+    if (scan.quote) {
+      if (ch === BACKSLASH && scan.depth > 0) i++;
+      else if (ch === scan.quote) scan.quote = 0;
+    } else if (QUOTES.has(ch) && (scan.depth > 0 || attrQuotes)) {
+      scan.quote = ch;
+    } else if (ch === BRACE_L) {
+      if (scan.depth++ === 0) start = i;
+    } else if (ch === BRACE_R && scan.depth > 0 && --scan.depth === 0) {
+      pushRange(out, from + start, from + i + 1);
+      start = -1;
+    }
+  }
+  if (scan.depth > 0 && start >= 0) pushRange(out, from + start, from + text.length);
+}
+
+/**
+ * Nodes of an element's own text. Markup inside an expression splits it into
+ * several (`{ok && <b>yes</b>}`), and HTML reads `&&` as entities and the
+ * fragment brackets `<>`, `</>` as incomplete tags.
+ */
+const TEXT_NODES = new Set([
+  "Text",
+  "EntityReference",
+  "CharacterReference",
+  "InvalidEntity",
+  "IncompleteTag",
+  "IncompleteCloseTag",
+]);
+
+/** The expressions in the text directly in `node`, `null` without any. */
+function textExpressions(node: SyntaxNodeRef, input: Input): Range[] | null {
+  const out: Range[] = [];
+  const scan: Scan = { depth: 0, quote: 0 };
+  const child = node.node.cursor(IterMode.IgnoreMounts);
+  if (child.firstChild()) {
+    do {
+      if (TEXT_NODES.has(child.name)) scanBraces(input.read(child.from, child.to), child.from, scan, out);
+    } while (child.nextSibling());
+  }
+  return out.length ? out : null;
+}
+
+/** The expressions in a tag: values, shorthands and spreads, also with spaces, which split HTML attributes. */
+function tagExpressions(node: SyntaxNodeRef, input: Input): Range[] | null {
+  const out: Range[] = [];
+  scanBraces(input.read(node.from, node.to), node.from, { depth: 0, quote: 0 }, out, true);
+  return out.length ? out : null;
+}
+
+const TAGS = new Set(["OpenTag", "SelfClosingTag", "OpenScriptTag", "OpenStyleTag", "OpenTextareaTag"]);
+
+function mixExpressions(node: SyntaxNodeRef, input: Input): NestedParse | null {
+  if (TAGS.has(node.name)) {
+    const overlay = tagExpressions(node, input);
+    return overlay && { parser: tagParser, overlay };
+  }
+  if (node.name === "Document" || node.name === "Element") {
+    const overlay = textExpressions(node, input);
+    return overlay && { parser: node.name === "Document" ? documentTextParser : elementTextParser, overlay };
+  }
+  return null;
+}
+
+const templateParser = (templateHTML.language as LRLanguage).parser.configure({ wrap: parseMixed(mixExpressions) });
+
+const componentParser = parser.configure({
+  props: [styleTags({ FrontmatterFence: t.processingInstruction })],
+  wrap: parseMixed((node) => {
+    if (node.name === "FrontmatterScript") return { parser: tsParser };
+    if (node.name === "Template") return { parser: templateParser };
+    return null;
+  }),
+});
+
+export const astroLanguage = LRLanguage.define({ name: "astro", parser: componentParser });
+
+export function astro(): LanguageSupport {
+  return new LanguageSupport(astroLanguage, [templateHTML.support]);
+}

src/editor/core/cm/astro/parser.terms.ts

@@ -0,0 +1,7 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+export const
+  FrontmatterFence = 1,
+  FrontmatterScript = 2,
+  Template = 3,
+  Document = 4,
+  Frontmatter = 5

src/editor/core/cm/astro/parser.ts

@@ -0,0 +1,17 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+import {LRParser} from "@lezer/lr"
+import {component} from "./tokens"
+export const parser = LRParser.deserialize({
+  version: 14,
+  states: "zQQOQOOOYORO'#CaQOOOOOQTOQOOOOOQ,58{,58{OhORO,58{OOOQ1G.g1G.g",
+  stateData: "s~OPPORQO~OPSOQTORTXUTX~OPUORTaUTa~O",
+  goto: "YUPPPPPVRRO",
+  nodeNames: "⚠ FrontmatterFence FrontmatterScript Template Document Frontmatter",
+  maxTerm: 6,
+  skippedNodes: [0],
+  repeatNodeCount: 0,
+  tokenData: "RORO",
+  tokenizers: [0, component],
+  topRules: {"Document":[0,4]},
+  tokenPrec: 0
+})

src/editor/core/cm/astro/tokens.ts

@@ -0,0 +1,53 @@
+/** External tokens of `astro.grammar`: the frontmatter fences, its script and the template. */
+import { ExternalTokenizer, type InputStream } from "@lezer/lr";
+import { FrontmatterFence, FrontmatterScript, Template } from "./parser.terms";
+
+const NEWLINE = 10,
+  CARRIAGE_RETURN = 13,
+  SPACE = 32,
+  TAB = 9,
+  DASH = 45;
+
+const isBlank = (ch: number) => ch === SPACE || ch === TAB;
+const isSpace = (ch: number) => isBlank(ch) || ch === NEWLINE || ch === CARRIAGE_RETURN;
+
+/**
+ * The end of a fence line starting at `offset` (a line holding `---` and
+ * blanks, the Astro compiler allows both around it), or -1 when the line is
+ * something else.
+ */
+function fenceEnd(input: InputStream, offset: number): number {
+  let i = offset;
+  while (isBlank(input.peek(i))) i++;
+  if (input.peek(i) !== DASH || input.peek(i + 1) !== DASH || input.peek(i + 2) !== DASH) return -1;
+  i += 3;
+  while (isBlank(input.peek(i))) i++;
+  const ch = input.peek(i);
+  return ch === NEWLINE || ch === CARRIAGE_RETURN || ch === -1 ? i : -1;
+}
+
+export const component = new ExternalTokenizer((input, stack) => {
+  if (input.next === -1) return;
+  const inFrontmatter = stack.canShift(FrontmatterScript);
+  if (inFrontmatter || stack.canShift(FrontmatterFence)) {
+    // The opening fence may follow blank lines at the start of the file.
+    let start = 0;
+    if (!inFrontmatter) while (isSpace(input.peek(start))) start++;
+    const end = fenceEnd(input, start);
+    if (end >= 0) {
+      input.acceptToken(FrontmatterFence, end);
+      return;
+    }
+  }
+  if (inFrontmatter) {
+    // The script runs to the start of the closing fence's line, or to the end.
+    for (let i = 0; ; i++) {
+      const ch = input.peek(i);
+      if (ch === -1) return input.acceptToken(FrontmatterScript, i);
+      if (ch === NEWLINE && fenceEnd(input, i + 1) >= 0) return input.acceptToken(FrontmatterScript, i + 1);
+    }
+  }
+  let length = 0;
+  while (input.peek(length) !== -1) length++;
+  input.acceptToken(Template, length);
+});

src/editor/core/cm/jsTags.ts

@@ -0,0 +1,4 @@
+import { tags as t, type styleTags } from "@lezer/highlight";
+
+/** @lezer/javascript styles `.` as a deref operator, but not `?.`. */
+export const JS_TAGS: Parameters<typeof styleTags>[0] = { "?.": t.derefOperator };

src/editor/core/cm/languages.ts

@@ -19,6 +19,7 @@ import { styleTags, tags as t } from "@lezer/highlight";
 import { languages as languageData } from "@codemirror/language-data";
 import type { DocBackend, LanguageInfo } from "../backend";
 import { _languageMode } from "@/editor/api";
+import { JS_TAGS } from "./jsTags";
 
 let infos: Map<string, LanguageInfo> | null = null;
 let infosPromise: Promise<Map<string, LanguageInfo>> | null = null;
@@ -120,6 +121,11 @@ const OWN_MODES = [
     load: () => import("./erb").then((m) => m.erb()),
   }),
   LanguageDescription.of({ name: "JSP", load: () => import("./jsp").then((m) => m.jsp()) }),
+  LanguageDescription.of({
+    name: "Astro",
+    extensions: ["astro"],
+    load: () => import("./astro").then((m) => m.astro()),
+  }),
   LanguageDescription.of({
     name: "YAML",
     alias: ["yml"],
@@ -252,9 +258,6 @@ const CSS_TAGS: Parameters<typeof styleTags>[0] = {
   "ClassName IdName PseudoClassName ColorLiteral CallTag": t.tagName,
 };
 
-// @lezer/javascript styles `.` as a deref operator, but not `?.`.
-const JS_TAGS: Parameters<typeof styleTags>[0] = { "?.": t.derefOperator };
-
 // @lezer/java tags annotations as plain identifiers.
 export const JAVA_TAGS: Parameters<typeof styleTags>[0] = {
   "MarkerAnnotation/Identifier Annotation/Identifier MarkerAnnotation Annotation": t.annotation,