fix(vim): keep IME compositions out of the document in Normal mode (#1055)

codemirror-vim only discards single-character IME commits in Normal mode, so multi-character commits (Chinese pinyin, Japanese conversions) were written into the document and could replace the selection. In Normal/Visual mode the composition now renders as usual and its whole span is restored when it ends; an ASCII single-character commit (dead keys) is replayed as a Vim key. Composition changes are marked transientChange: they stay out of undo history, the document model, the backend and other split panes. Insert/Replace mode is unchanged.

Closes #1056

Co-authored-by: xtr-hub <[email protected]>

src/editor/api.ts

@@ -191,6 +191,13 @@ export const remoteChange = Annotation.define<boolean>();
  */
 export const mirroredChange = Annotation.define<boolean>();
 
+/**
+ * A temporary change that must not reach the document model or the backend
+ * (an IME composition's preedit, which the feature removes again when the
+ * composition ends).
+ */
+export const transientChange = Annotation.define<boolean>();
+
 /**
  * Provided by editor/core per editor: resolves once the editor's local edits
  * reached the backend document. Language-server requests read the backend's

src/editor/core/model/attachment.ts

@@ -11,6 +11,7 @@ import {
   documentSynced,
   isExternalChange,
   remoteChange,
+  transientChange,
   type DocumentAttachment,
   type TextEditorHandle,
 } from "@/editor/api";
@@ -74,7 +75,9 @@ export async function attachDocument(manager: DocumentManager, path: string): Pr
       documentSynced.of(() => model.sync.whenIdle()),
       EditorView.updateListener.of((u) => {
         if (released || model.closed) return;
-        for (const tr of u.transactions) if (tr.docChanged && !isExternalChange(tr)) model.onLocal(tr.changes, target);
+        for (const tr of u.transactions)
+          if (tr.docChanged && !isExternalChange(tr) && !tr.annotation(transientChange))
+            model.onLocal(tr.changes, target);
       }),
     ],
     save: (options) => model.save(options?.exact ? "exact" : "explicit"),

src/editor/core/model/slot.ts

@@ -23,6 +23,7 @@ import {
   _removeEditor,
   documentSynced,
   isExternalChange,
+  transientChange,
   type EditorDocContext,
   type TextEditorHandle,
 } from "@/editor/api";
@@ -207,7 +208,7 @@ export class EditorSlot {
         const line = tr.startState.doc.lineAt(head);
         navigatedFrom = { line: line.number - 1, col: head - line.from };
       }
-      if (tr.docChanged && !isExternalChange(tr)) {
+      if (tr.docChanged && !isExternalChange(tr) && !tr.annotation(transientChange)) {
         this.model.onLocal(tr.changes, this);
         if (
           tr.isUserEvent("input") ||

src/features/vim/__tests__/vim.test.ts

@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
 import { EditorState, Prec } from "@codemirror/state";
 import { EditorView, keymap } from "@codemirror/view";
 import { getCM, Vim } from "@replit/codemirror-vim";
-import { embeddedEditorExtensions, useEditorRegistry } from "@/editor/api";
+import { embeddedEditorExtensions, transientChange, useEditorRegistry } from "@/editor/api";
 import { executeAction, registerAction } from "@/platform/actions";
 import { getContext } from "@/platform/context";
 import { _handleEx, applyRcCommands, vimExtension } from "../engine";
@@ -114,6 +114,82 @@ describe("Vim engine", () => {
     view.destroy();
   });
 
+  it("keeps an IME composition out of the document in Normal mode", async () => {
+    const transient: boolean[] = [];
+    const view = makeView([
+      vimExtension(),
+      EditorView.updateListener.of((u) => {
+        for (const tr of u.transactions) if (tr.docChanged) transient.push(!!tr.annotation(transientChange));
+      }),
+    ]);
+    const content = view.contentDOM;
+    const composition = (type: string, data: string) =>
+      content.dispatchEvent(new CompositionEvent(type, { data, bubbles: true }));
+
+    // A composition over a selection replaces it, then restores it.
+    view.dispatch({ selection: { anchor: 0, head: 3 } });
+    composition("compositionstart", "");
+    // Preedit, then the committed character (as CodeMirror reports them).
+    view.dispatch({ changes: { from: 0, to: 3, insert: "ni" }, userEvent: "input.type.compose.start" });
+    view.dispatch({ changes: { from: 0, to: 2, insert: "你" }, userEvent: "input.type.compose" });
+    expect(view.state.doc.toString()).toBe("你 two\nthree");
+    composition("compositionend", "你");
+    await tick();
+
+    expect(view.state.doc.toString()).toBe("one two\nthree");
+    expect(view.state.selection.main.from).toBe(0);
+    expect(view.state.selection.main.to).toBe(3);
+    // Preedit, commit and the cleanup itself never reach the document model.
+    expect(transient).toEqual([true, true, true]);
+    view.destroy();
+  });
+
+  it("removes a whole composition whose commit shares text with the preedit", async () => {
+    const view = makeView([vimExtension()]);
+    const composition = (type: string, data: string) =>
+      view.contentDOM.dispatchEvent(new CompositionEvent(type, { data, bubbles: true }));
+
+    view.dispatch({ selection: { anchor: 4 } });
+    composition("compositionstart", "");
+    view.dispatch({ changes: { from: 4, insert: "かんじを" }, userEvent: "input.type.compose.start" });
+    // CodeMirror reports the conversion as a minimal diff: the shared `を` is not part of it.
+    view.dispatch({ changes: { from: 4, to: 7, insert: "漢字" }, userEvent: "input.type.compose" });
+    expect(view.state.doc.toString()).toBe("one 漢字をtwo\nthree");
+    composition("compositionend", "漢字を");
+    await tick();
+
+    expect(view.state.doc.toString()).toBe("one two\nthree");
+    expect(view.state.selection.main.head).toBe(4);
+    view.destroy();
+  });
+
+  it("replays a dead-key composition as a Vim command and keeps insert-mode IME input", async () => {
+    const view = makeView();
+    const content = view.contentDOM;
+    const cm = getCM(view)!;
+    const composition = (type: string, data: string) =>
+      content.dispatchEvent(new CompositionEvent(type, { data, bubbles: true }));
+
+    // `^` (first non-blank) at column 5.
+    view.dispatch({ selection: { anchor: 5 } });
+    composition("compositionstart", "");
+    view.dispatch({ changes: { from: 5, insert: "^" }, userEvent: "input.type.compose.start" });
+    composition("compositionend", "^");
+    await tick();
+    expect(view.state.doc.toString()).toBe("one two\nthree");
+    expect(view.state.selection.main.head).toBe(0);
+
+    // IME input in Insert mode is ordinary text.
+    Vim.handleKey(cm as never, "i", "user");
+    await tick();
+    composition("compositionstart", "");
+    view.dispatch({ changes: { from: 0, insert: "你" }, userEvent: "input.type.compose.start" });
+    composition("compositionend", "你");
+    await tick();
+    expect(view.state.doc.toString()).toBe("你one two\nthree");
+    view.destroy();
+  });
+
   it("publishes the mode and lets the editor's Escape bindings (popups) go first", () => {
     let popupOpen = true;
     const closePopup = Prec.highest(

src/features/vim/engine.ts

@@ -3,15 +3,162 @@
  * is turned on: the per-editor extension, OpenIDEA's Ex commands (`:w`, `:q`,
  * `:action`) and applying the rc file.
  */
-import { Prec, type Extension } from "@codemirror/state";
+import { EditorState, Prec, StateEffect, StateField, Transaction, type Extension } from "@codemirror/state";
 import { EditorView, ViewPlugin, runScopeHandlers } from "@codemirror/view";
 import { Vim, getCM, vim } from "@replit/codemirror-vim";
+import { transientChange } from "@/editor/api";
 import { executeAction, getAction } from "@/platform/actions";
 import { showStatusInfo } from "@/workbench/contributions";
 import { VIM_EDITOR_ATTR } from "./conflicts";
 import { useVimMode, type VimModeState } from "./mode";
 import { resolveActionId, type RcCommand } from "./rc";
 
+/** Insert/Replace state as seen by transactions (mode changes are queued). */
+const setVimInsert = StateEffect.define<boolean>();
+
+const vimInsert = StateField.define<boolean>({
+  // Vim starts in Normal mode.
+  create: () => false,
+  update(value, tr) {
+    for (const e of tr.effects) if (e.is(setVimInsert)) value = e.value;
+    return value;
+  },
+});
+
+interface ImeComposition {
+  /** The span the composition occupies now: the replaced selection, mapped through its changes. */
+  from: number;
+  to: number;
+  /** Selection contents replaced by the composition. */
+  before: string;
+  /** Selection before the composition, to restore it afterwards. */
+  selection: { anchor: number; head: number };
+  /** The text the IME committed. */
+  ended: string | null;
+  timer: ReturnType<typeof setTimeout> | undefined;
+}
+
+const imeCompositions = new WeakMap<EditorView, ImeComposition>();
+
+/**
+ * codemirror-vim only discards single-character IME commits in Normal mode, so
+ * Chinese pinyin (often several characters) is written to the document. Let the
+ * IME and CodeMirror own the composition normally, then drop its net change:
+ * the preedit is temporary, an ASCII commit (dead keys) is replayed as a Vim
+ * key, and text commits (CJK, words) are discarded. The changes are marked
+ * transient, keeping them out of undo and the document model.
+ */
+function finishImeComposition(view: EditorView) {
+  const composition = imeCompositions.get(view);
+  if (!composition) return;
+  imeCompositions.delete(view);
+  clearTimeout(composition.timer);
+  const cm = getCM(view);
+  if (!cm) return;
+  const insert = !!cm.state.vim?.insertMode;
+  const { from, to, before, selection, ended } = composition;
+  // The whole span, not the last change CodeMirror read: its changes are minimal diffs, so a
+  // commit sharing a prefix or suffix with the preedit (`かんじを` → `漢字を`) touches only part of it.
+  if (view.state.sliceDoc(from, to) !== before) {
+    view.dispatch({
+      changes: { from, to, insert: before },
+      selection,
+      annotations: [Transaction.addToHistory.of(false), transientChange.of(true)],
+    });
+  } else {
+    const main = view.state.selection.main;
+    if (main.anchor !== selection.anchor || main.head !== selection.head) {
+      view.dispatch({ selection, annotations: transientChange.of(true) });
+    }
+  }
+  // Dead keys and ASCII compositions are Vim commands (`^`, `` ` ``, letters);
+  // CJK commits are text and must not move the cursor or consume a pending operator.
+  if (!insert && ended?.length === 1 && ended.charCodeAt(0) < 0x80) Vim.handleKey(cm, ended, "user");
+}
+
+const imeHandlers = EditorView.domEventHandlers({
+  compositionstart(e, view) {
+    if (typeof e.data !== "string") return false;
+    const cm = getCM(view);
+    if (!cm?.state.vim || cm.state.vim.insertMode) return false;
+    const previous = imeCompositions.get(view);
+    if (previous) clearTimeout(previous.timer);
+    const range = view.state.selection.main;
+    imeCompositions.set(view, {
+      from: range.from,
+      to: range.to,
+      before: view.state.sliceDoc(range.from, range.to),
+      selection: { anchor: range.anchor, head: range.head },
+      ended: null,
+      timer: undefined,
+    });
+    return false;
+  },
+  compositionupdate(_e, view) {
+    if (!imeCompositions.has(view)) return false;
+    const cm = getCM(view);
+    if (cm?.state.vim && !cm.state.vim.insertMode && cm.state.vimPlugin) {
+      // Keep codemirror-vim's single-character path from calling its
+      // forceEndComposition() while we let the browser compose normally.
+      cm.state.vimPlugin.useNextTextInput = false;
+    }
+    return false;
+  },
+  compositionend(e, view) {
+    const composition = imeCompositions.get(view);
+    // Ignore the untrusted, data-less event codemirror-vim synthesises when it
+    // force-ends a composition; real ends (even cancelled ones) are trusted.
+    if (!composition || (typeof e.data !== "string" && !e.isTrusted)) return false;
+    composition.ended = typeof e.data === "string" ? e.data : "";
+    clearTimeout(composition.timer);
+    composition.timer = setTimeout(() => finishImeComposition(view), 0);
+    return false;
+  },
+});
+
+/** Let CodeMirror apply the composing text; it is removed when the IME ends. */
+const imeInput = EditorView.inputHandler.of((view) => {
+  const composition = imeCompositions.get(view);
+  if (!composition) return false;
+  const cm = getCM(view);
+  if (cm?.state.vim && !cm.state.vim.insertMode && cm.state.vimPlugin) {
+    cm.state.vimPlugin.useNextTextInput = false;
+  }
+  return false;
+});
+
+/** Keeps the composition's span current, including commits CodeMirror reports after
+ * `compositionend`: composing text at its edges belongs to it, other edits there don't. */
+const imeTracker = ViewPlugin.define((view) => ({
+  update(update) {
+    const composition = imeCompositions.get(view);
+    if (!composition) return;
+    for (const tr of update.transactions) {
+      if (!tr.docChanged) continue;
+      const compose = tr.isUserEvent("input.type.compose");
+      composition.from = tr.changes.mapPos(composition.from, compose ? -1 : 1);
+      composition.to = Math.max(composition.from, tr.changes.mapPos(composition.to, compose ? 1 : -1));
+    }
+  },
+}));
+
+/** Composition changes are temporary: no undo entry, no document/backend sync. */
+const imeTransactions = EditorState.transactionExtender.of((tr) =>
+  !tr.startState.field(vimInsert) && tr.isUserEvent("input.type.compose")
+    ? { annotations: [Transaction.addToHistory.of(false), transientChange.of(true)] }
+    : null,
+);
+
+const imeCleanup = ViewPlugin.define((view) => ({
+  destroy() {
+    const composition = imeCompositions.get(view);
+    if (composition) {
+      clearTimeout(composition.timer);
+      imeCompositions.delete(view);
+    }
+  },
+}));
+
 /**
  * IdeaVim's Escape: an open popup, lookup, find bar or template closes first
  * (the editor's Escape key bindings); Vim only gets Escape that nothing else
@@ -33,6 +180,7 @@ const modeTracker = ViewPlugin.define((view) => {
   const focused = () => view.root.activeElement === view.contentDOM;
   const onModeChange = (e: { mode: string; subMode?: string }) => {
     state = { mode: e.mode, sub: e.subMode === "linewise" || e.subMode === "blockwise" ? e.subMode : null };
+    queueMicrotask(() => view.dispatch({ effects: setVimInsert.of(e.mode === "insert" || e.mode === "replace") }));
     if (focused()) publish();
   };
   cm?.on("vim-mode-change", onModeChange);
@@ -74,7 +222,17 @@ const vimTheme = EditorView.theme({
 export function vimExtension(): Extension {
   defineExCommands();
   return [
-    Prec.highest([escapeClosesPopupsFirst, vim(), modeTracker]),
+    Prec.highest([
+      vimInsert,
+      imeHandlers,
+      imeInput,
+      imeTracker,
+      imeTransactions,
+      escapeClosesPopupsFirst,
+      vim(),
+      modeTracker,
+    ]),
+    imeCleanup,
     vimTheme,
     EditorView.editorAttributes.of({ [VIM_EDITOR_ATTR]: "true" }),
   ];