feat(plugins): let plugins colour tokens in files of other languages (#1882)

Plugin API 1.3 adds an optional semantic-tokens-provider export (world plugin-semantic-tokens). A plugin declares the semantic-tokens kind under [[contributes.providers]] for any language, including ones it does not contribute, and returns single-line tokens with LSP token kinds and modifiers. The host asks it with the other editor providers (50 ms / 500 ms) when the editor requests semantic tokens and joins the result with the language server's: plugin tokens replace the server tokens they overlap, so the colour scheme styles them like any semantic token. Declaring the kind needs engines.openidea ^1.3.

API 1.2 shipped with 0.1.38, so wit/ and the SDK's copy, ENGINE_VERSION and both API_VERSION constants move to 1.3 and every shipped component is rebuilt. The SDK gains openidea_plugin::semantic_tokens, and register_plugin! accepts any combination of optional exports (providers, views, indexing, syntax, semantic_tokens) instead of a fixed list of pairs.

crates/ide-extension/src/manifest.rs

@@ -32,7 +32,7 @@ use std::path::{Component, Path, PathBuf};
 
 pub const MANIFEST_FILE: &str = "plugin.toml";
 /// The plugin API (engine) version this host implements.
-pub const ENGINE_VERSION: &str = "1.2.0";
+pub const ENGINE_VERSION: &str = "1.3.0";
 /// The 0.1 API, still accepted for contribution-only packages and 0.1 components.
 pub const LEGACY_API: &str = "0.1";
 /// Marketplace categories, in the order they are shown.

crates/ide-host/src/ops/lsp.rs

@@ -3,6 +3,7 @@ use crate::Host;
 use crate::apply_workspace_edit as apply_edit;
 use crate::plugins::{ProviderKind, editor_providers_for as editor_providers};
 use crate::rpc::Dispatcher;
+use ide_core::error::ErrorKind;
 use ide_core::{DocId, IdeResult, Location, Pos, Range, RequestId, ServerId, TextEdit, WorkspaceEdit};
 use ide_lsp::types::*;
 use ide_plugin::ProviderDocument;
@@ -614,7 +615,22 @@ pub async fn lsp_inlay_hints(host: &Host, window: &str, doc: DocId, range: Range
 }
 
 pub async fn lsp_semantic_tokens(host: &Host, window: &str, doc: DocId) -> IdeResult<Option<SemanticTokens>> {
-    host.project(window)?.lsp.semantic_tokens(doc).await
+    let lsp = host.project(window)?.lsp.clone();
+    let Some((plugins, d)) = plugin_providers(host, window, doc, ProviderKind::SemanticTokens) else {
+        return lsp.semantic_tokens(doc).await;
+    };
+    let (result, tokens) =
+        tokio::join!(lsp.semantic_tokens(doc), plugin_call(move || plugins.provide_semantic_tokens(&d)));
+    let tokens = tokens.unwrap_or_default();
+    if tokens.is_empty() {
+        return result;
+    }
+    let server = match result {
+        Ok(server) => server,
+        Err(e) if matches!(e.kind, ErrorKind::NotFound | ErrorKind::Unavailable) => None,
+        Err(e) => return Err(e),
+    };
+    Ok(Some(crate::plugins::merge_semantic_tokens(server, tokens)))
 }
 
 pub async fn lsp_code_lens(host: &Host, window: &str, doc: DocId) -> IdeResult<Vec<CodeLens>> {

crates/ide-host/src/plugins/mod.rs

@@ -30,6 +30,7 @@ mod providers;
 pub mod remote;
 pub mod remote_host;
 mod run_detectors;
+mod semantic_tokens;
 mod theme;
 mod types;
 pub mod ui_points;
@@ -47,6 +48,7 @@ pub use market::{
 pub use points::{COMMANDS, CommandDecl, register_builtin_points};
 pub use providers::{ProviderKind, set_editor_providers};
 pub(crate) use providers::{editor_plugin_host, editor_providers as editor_providers_for};
+pub(crate) use semantic_tokens::merge as merge_semantic_tokens;
 pub use theme::{ThemeDef, ThemeKind};
 pub use types::*;
 

crates/ide-host/src/plugins/providers.rs

@@ -37,6 +37,7 @@ pub enum ProviderKind {
     CodeActions,
     Definition,
     LineMarkers,
+    SemanticTokens,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -112,6 +113,9 @@ impl ExtensionPoint for ProvidersPoint {
         if d.kinds.is_empty() || d.languages.is_empty() || d.languages.len() > 64 {
             return Err("contributes.providers: give `kinds` and 1–64 `languages`".into());
         }
+        if d.kinds.contains(&ProviderKind::SemanticTokens) {
+            pkg.manifest.needs_api("contributes.providers: semantic-tokens", 3)?;
+        }
         for l in &d.languages {
             if l != "*"
                 && (l.is_empty()
@@ -161,7 +165,7 @@ use super::{LoadedPlugin, PluginHost, PluginStatus};
 use ide_core::CancellationToken;
 use ide_plugin::{
     ActorError, CallError, CodeAction, CompletionItem, InlineSuggestion, LineMarker, Location, Pending,
-    ProviderDocument, Runtime,
+    ProviderDocument, Runtime, SemanticToken,
 };
 use std::sync::Weak;
 use std::time::{Duration, Instant};
@@ -331,6 +335,16 @@ impl PluginHost {
         .collect()
     }
 
+    pub fn provide_semantic_tokens(&self, doc: &ProviderDocument) -> Vec<SemanticToken> {
+        let d = Arc::new(doc.clone());
+        self.edit_calls(ProviderKind::SemanticTokens, &doc.language, move |rt, timeout| {
+            rt.provide_semantic_tokens(&d, timeout)
+        })
+        .into_iter()
+        .flat_map(|(_, tokens)| tokens)
+        .collect()
+    }
+
     /// Ghost text from the first plugin that has some. Not bound by the
     /// edit-path soft deadline (it may need the network) but by `cancel`:
     /// the editor cancels a request as soon as the user types on.

crates/ide-host/src/plugins/semantic_tokens.rs

@@ -0,0 +1,131 @@
+use ide_lsp::types::SemanticTokens;
+use ide_plugin::SemanticToken;
+use std::collections::HashMap;
+
+pub(crate) fn merge(server: Option<SemanticTokens>, plugin: Vec<SemanticToken>) -> SemanticTokens {
+    let mut out = server.unwrap_or_else(|| SemanticTokens {
+        data: Vec::new(),
+        token_types: Vec::new(),
+        token_modifiers: Vec::new(),
+    });
+    let mut added: Vec<[u32; 5]> = Vec::with_capacity(plugin.len());
+    let mut lines: HashMap<u32, Vec<(u32, u32)>> = HashMap::new();
+    for token in plugin {
+        let (start, end) = (token.range.start, token.range.end);
+        if start.line != end.line || end.col <= start.col {
+            continue;
+        }
+        let kind = camel_case(&token.kind);
+        let kind = match out.token_types.iter().position(|t| *t == kind) {
+            Some(i) => i,
+            None => {
+                out.token_types.push(kind);
+                out.token_types.len() - 1
+            }
+        };
+        let mut bits = 0u32;
+        for modifier in &token.modifiers {
+            let modifier = camel_case(modifier);
+            let i = match out.token_modifiers.iter().position(|m| *m == modifier) {
+                Some(i) => i,
+                None if out.token_modifiers.len() < 32 => {
+                    out.token_modifiers.push(modifier);
+                    out.token_modifiers.len() - 1
+                }
+                None => continue,
+            };
+            bits |= 1 << i;
+        }
+        lines.entry(start.line).or_default().push((start.col, end.col));
+        added.push([start.line, start.col, end.col - start.col, kind as u32, bits]);
+    }
+    let covered = |t: &[u32; 5]| {
+        lines.get(&t[0]).is_some_and(|ranges| ranges.iter().any(|&(from, to)| t[1] < to && from < t[1] + t[2]))
+    };
+    let mut data: Vec<[u32; 5]> = out.data.as_chunks::<5>().0.iter().filter(|t| !covered(t)).copied().collect();
+    data.extend(added);
+    data.sort_by_key(|t| (t[0], t[1]));
+    out.data = data.into_iter().flatten().collect();
+    out
+}
+
+fn camel_case(name: &str) -> String {
+    let mut out = String::with_capacity(name.len());
+    let mut upper = false;
+    for c in name.chars() {
+        if c == '-' {
+            upper = true;
+        } else if upper {
+            out.extend(c.to_uppercase());
+            upper = false;
+        } else {
+            out.push(c);
+        }
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ide_core::{Pos, Range};
+
+    fn token(line: u32, from: u32, to: u32, kind: &str, modifiers: &[&str]) -> SemanticToken {
+        SemanticToken {
+            range: Range::new(Pos::new(line, from), Pos::new(line, to)),
+            kind: kind.into(),
+            modifiers: modifiers.iter().map(|m| (*m).into()).collect(),
+        }
+    }
+
+    fn server() -> SemanticTokens {
+        SemanticTokens {
+            data: vec![0, 0, 6, 0, 0, 1, 4, 5, 1, 1, 1, 12, 3, 1, 0],
+            token_types: vec!["variable".into(), "property".into()],
+            token_modifiers: vec!["readonly".into()],
+        }
+    }
+
+    #[test]
+    fn plugin_tokens_replace_the_server_tokens_they_overlap() {
+        let merged = merge(Some(server()), vec![token(1, 6, 10, "method", &[])]);
+        assert_eq!(merged.token_types, ["variable", "property", "method"]);
+        assert_eq!(merged.data, [0, 0, 6, 0, 0, 1, 6, 4, 2, 0, 1, 12, 3, 1, 0]);
+    }
+
+    #[test]
+    fn kinds_and_modifiers_use_the_lsp_names() {
+        let merged = merge(None, vec![token(2, 0, 3, "type-parameter", &["default-library", "readonly"])]);
+        assert_eq!(merged.token_types, ["typeParameter"]);
+        assert_eq!(merged.token_modifiers, ["defaultLibrary", "readonly"]);
+        assert_eq!(merged.data, [2, 0, 3, 0, 0b11]);
+    }
+
+    #[test]
+    fn known_names_reuse_the_server_legend() {
+        let merged = merge(Some(server()), vec![token(3, 1, 2, "property", &["readonly"])]);
+        assert_eq!(merged.token_types.len(), 2);
+        assert_eq!(merged.token_modifiers.len(), 1);
+        assert_eq!(merged.data[15..], [3, 1, 1, 1, 1]);
+    }
+
+    #[test]
+    fn tokens_spanning_lines_or_empty_are_ignored() {
+        let multi = SemanticToken {
+            range: Range::new(Pos::new(0, 4), Pos::new(1, 2)),
+            kind: "method".into(),
+            modifiers: Vec::new(),
+        };
+        let merged = merge(Some(server()), vec![multi, token(0, 3, 3, "method", &[])]);
+        assert_eq!(merged.data, server().data);
+    }
+
+    #[test]
+    fn modifiers_past_thirty_two_are_dropped() {
+        let names: Vec<String> = (0..40).map(|i| format!("m{i}")).collect();
+        let refs: Vec<&str> = names.iter().map(String::as_str).collect();
+        let merged = merge(None, vec![token(0, 0, 1, "type", &refs)]);
+        assert_eq!(merged.token_modifiers.len(), 32);
+        assert_eq!(merged.data[4], u32::MAX);
+    }
+}

crates/ide-host/tests/plugin_semantic_tokens.rs

@@ -0,0 +1,56 @@
+use ide_host::plugins::{Consent, PluginHost};
+use ide_plugin::{Limits, ProviderDocument};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::Duration;
+
+fn fixture_dir() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("../ide-plugin/tests/fixtures/semantic-tokens")
+}
+
+fn host(dir: &Path) -> PluginHost {
+    let limits = Limits { edit_soft: Duration::from_secs(5), edit_hard: Duration::from_secs(10), ..Limits::default() };
+    PluginHost::new(dir.join("plugins"), Arc::new(|_: Option<&str>, _| {}), limits).unwrap()
+}
+
+fn doc(dir: &Path, language: &str, text: &str) -> ProviderDocument {
+    ProviderDocument { path: dir.join("Order.php"), language: language.into(), version: 1, text: text.into() }
+}
+
+const ORDER: &str =
+    "<?php\nclass Order\n{\n    public function user() {}\n}\n$order->user;\n$order->total;\n$order->user();\n";
+
+#[test]
+fn relation_accesses_and_columns_get_their_own_kinds() {
+    let tmp = tempfile::tempdir().unwrap();
+    let host = host(tmp.path());
+    host.install_from_dir(&fixture_dir(), &Consent::all()).unwrap();
+    let tokens = host.provide_semantic_tokens(&doc(tmp.path(), "php", ORDER));
+    let found: Vec<_> = tokens
+        .iter()
+        .map(|t| (t.range.start.line, t.range.start.col, t.range.end.line, t.range.end.col, t.kind.as_str()))
+        .collect();
+    assert_eq!(found, [(5, 8, 5, 12, "method"), (6, 8, 6, 13, "property")]);
+    assert!(tokens.iter().all(|t| t.modifiers.is_empty()));
+}
+
+#[test]
+fn only_declared_languages_are_asked() {
+    let tmp = tempfile::tempdir().unwrap();
+    let host = host(tmp.path());
+    host.install_from_dir(&fixture_dir(), &Consent::all()).unwrap();
+    assert!(host.provide_semantic_tokens(&doc(tmp.path(), "javascript", "a->b")).is_empty());
+}
+
+#[test]
+fn the_kind_needs_plugin_api_1_3() {
+    let tmp = tempfile::tempdir().unwrap();
+    let host = host(tmp.path());
+    let pkg = tmp.path().join("old");
+    std::fs::create_dir_all(pkg.join("dist")).unwrap();
+    std::fs::copy(fixture_dir().join("dist/plugin.wasm"), pkg.join("dist/plugin.wasm")).unwrap();
+    let manifest = std::fs::read_to_string(fixture_dir().join("plugin.toml")).unwrap();
+    std::fs::write(pkg.join("plugin.toml"), manifest.replace("\"^1.3\"", "\"^1.0\"")).unwrap();
+    let err = host.install_from_dir(&pkg, &Consent::all()).unwrap_err();
+    assert!(err.message.contains("new in plugin API 1.3"), "{}", err.message);
+}

crates/ide-plugin/src/runtime.rs

@@ -3,7 +3,9 @@
 //! sandbox goes through [`HostApi`], implemented by the host; the imports
 //! themselves are in `wit_host`.
 
-use crate::types::{CodeAction, CompletionItem, InlineSuggestion, LineMarker, Location, ProviderDocument};
+use crate::types::{
+    CodeAction, CompletionItem, InlineSuggestion, LineMarker, Location, ProviderDocument, SemanticToken,
+};
 use crate::types::{
     EventKind, HostError, HttpRequest, HttpResponse, Invocation, Limits, NotificationLevel, ProcessOutput, Prompt,
     RepositoryStatus, StatusItem, SyntaxFold, SyntaxSymbol, SyntaxToken, WorkbenchEvent,
@@ -11,7 +13,9 @@ use crate::types::{
 use crate::types::{
     FieldValue, View, ViewBody, ViewButton, ViewEvent, ViewField, ViewListItem, ViewTable, ViewTreeItem, Viewport,
 };
-use crate::wit_host::{bindings, indexing_bindings, providers_bindings, syntax_bindings, views_bindings};
+use crate::wit_host::{
+    bindings, indexing_bindings, providers_bindings, semantic_bindings, syntax_bindings, views_bindings,
+};
 use ide_core::CancellationToken;
 use ide_core::ipc_path::to_ipc;
 use ide_extension::Capability;
@@ -29,12 +33,13 @@ use bindings::openidea::plugin::events as wit_events;
 use bindings::openidea::plugin::types as wit_types;
 use indexing_bindings::exports::openidea::plugin::document_indexer as wit_index;
 use providers_bindings::exports::openidea::plugin::providers as wit_prov;
+use semantic_bindings::exports::openidea::plugin::semantic_tokens_provider as wit_sem;
 use syntax_bindings::exports::openidea::plugin::syntax_provider as wit_syntax;
 use views_bindings::exports::openidea::plugin::view_provider as wit_view;
 use views_bindings::openidea::plugin::view_types as wit_vt;
 
 /// The API version of the world above, `major.minor` of [`host_api`].
-pub const API_VERSION: &str = "1.2";
+pub const API_VERSION: &str = "1.3";
 const EPOCH_TICK: Duration = Duration::from_millis(10);
 pub(crate) const MAX_TEXT: usize = 1 << 20;
 pub(crate) const MAX_TITLE: usize = 256;
@@ -513,6 +518,7 @@ struct Exports {
     providers: Option<wit_prov::Guest>,
     indexer: Option<wit_index::Guest>,
     views: Option<wit_view::Guest>,
+    semantic: Option<wit_sem::Guest>,
 }
 
 /// Most results one syntax call may return (more are dropped).
@@ -592,6 +598,7 @@ impl Runtime {
         let index_indices = wit_index::GuestIndices::new(&pre).ok();
         let provider_indices = wit_prov::GuestIndices::new(&pre).ok();
         let view_indices = wit_view::GuestIndices::new(&pre).ok();
+        let semantic_indices = wit_sem::GuestIndices::new(&pre).ok();
         let instance = pre.instantiate(&mut store).map_err(api_hint)?;
         let plugin = plugin_indices.load(&mut store, &instance).map_err(api_hint)?;
         let syntax = match syntax_indices {
@@ -610,9 +617,13 @@ impl Runtime {
             Some(indices) => Some(indices.load(&mut store, &instance).map_err(api_hint)?),
             None => None,
         };
+        let semantic = match semantic_indices {
+            Some(indices) => Some(indices.load(&mut store, &instance).map_err(api_hint)?),
+            None => None,
+        };
         Ok(Self {
             store,
-            exports: Exports { plugin, syntax, providers, views, indexer },
+            exports: Exports { plugin, syntax, providers, views, indexer, semantic },
             limits: setup.limits,
             version: setup.version,
         })
@@ -842,6 +853,34 @@ impl Runtime {
             .collect())
     }
 
+    pub fn provide_semantic_tokens(
+        &mut self,
+        doc: &ProviderDocument,
+        timeout: Duration,
+    ) -> Result<Vec<SemanticToken>, CallError> {
+        if self.exports.semantic.is_none() {
+            return Err(CallError::missing_export("semantic-tokens-provider"));
+        }
+        let d = wit_sem::Document {
+            path: to_ipc(&doc.path),
+            language: doc.language.clone(),
+            version: doc.version,
+            text: doc.text.clone(),
+        };
+        let tokens = self.call(Invocation::default(), timeout, |e, store| {
+            e.semantic.as_ref().expect("checked").call_semantic_tokens(store, &d)
+        })?;
+        Ok(tokens
+            .into_iter()
+            .take(MAX_SEMANTIC_TOKENS)
+            .map(|t| SemanticToken {
+                range: range_from_wit(t.range),
+                kind: clip(t.kind, 32),
+                modifiers: t.modifiers.into_iter().take(MAX_TOKEN_MODIFIERS).map(|m| clip(m, 32)).collect(),
+            })
+            .collect())
+    }
+
     pub fn activate(&mut self) -> Result<(), CallError> {
         let data = self.store.data();
         let ctx = wit_types::Context {
@@ -994,6 +1033,8 @@ impl Runtime {
 
 /// Most items one provider call may return (more are dropped).
 const MAX_PROVIDER_ITEMS: usize = 10_000;
+const MAX_SEMANTIC_TOKENS: usize = 1 << 16;
+const MAX_TOKEN_MODIFIERS: usize = 8;
 
 fn pos_to_wit(p: ide_core::Pos) -> wit_types::Position {
     wit_types::Position { line: p.line, character: p.col }

crates/ide-plugin/src/types.rs

@@ -419,6 +419,13 @@ pub struct LineMarker {
     pub target: Option<Location>,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SemanticToken {
+    pub range: ide_core::Range,
+    pub kind: String,
+    pub modifiers: Vec<String>,
+}
+
 // ---- declarative views (wit/views.wit) ---------------------------------------
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)]

crates/ide-plugin/src/wit_host.rs

@@ -814,3 +814,10 @@ pub(crate) mod indexing_bindings {
         with: { "openidea:plugin/types": super::bindings::openidea::plugin::types },
     });
 }
+
+pub(crate) mod semantic_bindings {
+    wasmtime::component::bindgen!({
+        world: "semantic-tokens", path: "../../wit",
+        with: { "openidea:plugin/types": super::bindings::openidea::plugin::types },
+    });
+}

crates/ide-plugin/tests/api_compat.rs

@@ -28,6 +28,7 @@ fn components() -> Vec<PathBuf> {
     let mut out = vec![
         repo().join("crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm"),
         repo().join("crates/ide-plugin/tests/fixtures/indexer/dist/plugin.wasm"),
+        repo().join("crates/ide-plugin/tests/fixtures/semantic-tokens/dist/plugin.wasm"),
     ];
     let official = repo().join("plugins/official");
     let mut dirs: Vec<_> = std::fs::read_dir(&official).unwrap().map(|e| e.unwrap().path()).collect();

crates/ide-plugin/tests/fixtures/hello/dist/plugin.wasm.sources

@@ -1 +1 @@
-20ce53ecb41ad94e9c7d59cd4ac51f63081c9f150832fd7febe07887fa3c07ab
+31c58d5da122b9ec92bc18c2ea0aa5a153075125f9df995eb458ca854d4693ba

crates/ide-plugin/tests/fixtures/indexer/dist/plugin.wasm.sources

@@ -1 +1 @@
-2e7d8932329d9383ecd4938225a111851f8a9805c6da558431dea84b7cecbe1c
+1a8d4b16e31b4904b6a82c5d37ba71731bda3ff8a5ddb31846960fdb117f3e3b

crates/ide-plugin/tests/fixtures/jobs/dist/plugin.wasm.sources

@@ -1 +1 @@
-7f58bace9b60d8b6b87c97e78dd8cca4955bd13d5f0737efe9d9a362cd1552bf
+c3e25003bb5cae1ba412becc31b9c9e1ee542de42898c425fa24899b0572822a

crates/ide-plugin/tests/fixtures/misbehave/dist/plugin.wasm.sources

@@ -1 +1 @@
-ee5ba9209eaf963ccc423828ca2e8ba221345c9c246635283214562668a872ad
+9c513786838e20dc5b7ae53dc57536ae24364b19e5bbad2126ba769e15f05aab

crates/ide-plugin/tests/fixtures/misbehave/src/lib.rs

@@ -5,8 +5,8 @@ wit_bindgen::generate!({
     inline: "
         package fixture:misbehave;
         world misbehave {
-            include openidea:plugin/plugin-syntax@1.2.0;
-            export openidea:plugin/providers@1.2.0;
+            include openidea:plugin/plugin-syntax@1.3.0;
+            export openidea:plugin/providers@1.3.0;
         }
     ",
     path: "../../../../../wit",

crates/ide-plugin/tests/fixtures/semantic-tokens/Cargo.lock

@@ -0,0 +1,333 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "anyhow"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
+
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
+[[package]]
+name = "equivalent"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
+
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "foldhash",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "id-arena"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
+
+[[package]]
+name = "indexmap"
+version = "2.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "leb128fmt"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
+
+[[package]]
+name = "log"
+version = "0.4.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
+
+[[package]]
+name = "macro-string"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e7b3a5bbb2f63aaa223a671ea1bd0e7bc16bd30ce387324cf63995deaddbbd2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "openidea-plugin"
+version = "0.1.0"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "openidea-test-semantic-tokens"
+version = "0.1.0"
+dependencies = [
+ "openidea-plugin",
+]
+
+[[package]]
+name = "prettyplease"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
+dependencies = [
+ "proc-macro2",
+ "syn",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "semver"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.154"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+
+[[package]]
+name = "wasm-encoder"
+version = "0.262.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4495a6f85dd94c411c749303efcbc5defade0b0cbdbc0541495241ab917a9b5a"
+dependencies = [
+ "leb128fmt",
+ "wasmparser",
+]
+
+[[package]]
+name = "wasm-metadata"
+version = "0.262.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c86ff6668190d9fb70372b306b4f9fe8930f0e2f57ce72e4922bf2f117900949"
+dependencies = [
+ "anyhow",
+ "indexmap",
+ "wasm-encoder",
+ "wasmparser",
+]
+
+[[package]]
+name = "wasmparser"
+version = "0.262.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "974d643b125b272607e2fff8012ea14a8328acd7621e4bafa7afb32500052b1b"
+dependencies = [
+ "bitflags",
+ "hashbrown",
+ "indexmap",
+ "semver",
+]
+
+[[package]]
+name = "wit-bindgen"
+version = "0.63.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b85a987ab61d365382d25c7d47e26edd48941b3bf6e3fbf76eff918bc71dab06"
+dependencies = [
+ "bitflags",
+ "wit-bindgen-rust-macro",
+]
+
+[[package]]
+name = "wit-bindgen-core"
+version = "0.63.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "775e122e390603abfacd548027016cd0113a6e2bf2f8ce6bdd3658b013456ea8"
+dependencies = [
+ "anyhow",
+ "heck",
+ "wit-parser",
+]
+
+[[package]]
+name = "wit-bindgen-rust"
+version = "0.63.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84e26e13b9d972026778d2ffb7985736de116026fdc8d9464f966e12df2eb240"
+dependencies = [
+ "anyhow",
+ "heck",
+ "indexmap",
+ "prettyplease",
+ "syn",
+ "wasm-metadata",
+ "wit-bindgen-core",
+ "wit-component",
+]
+
+[[package]]
+name = "wit-bindgen-rust-macro"
+version = "0.63.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16a008a0d6b4840677326beb4edbeb210a2f1b7d0183fb7cc33f466b90fd7fb6"
+dependencies = [
+ "anyhow",
+ "macro-string",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wit-bindgen-core",
+ "wit-bindgen-rust",
+]
+
+[[package]]
+name = "wit-component"
+version = "0.262.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "078def49fdb49cab405c3006e05793b3fdaf7069126ba6b9c97d01ec8c98fe4c"
+dependencies = [
+ "anyhow",
+ "bitflags",
+ "indexmap",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "wasm-encoder",
+ "wasm-metadata",
+ "wasmparser",
+ "wit-parser",
+]
+
+[[package]]
+name = "wit-parser"
+version = "0.262.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00f0dc051466ffe46787721fca3d088536f158ae400be044f1b1c386ba9397ad"
+dependencies = [
+ "anyhow",
+ "hashbrown",
+ "id-arena",
+ "indexmap",
+ "log",
+ "semver",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "unicode-ident",
+ "wasmparser",
+]
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

crates/ide-plugin/tests/fixtures/semantic-tokens/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "openidea-test-semantic-tokens"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+[workspace]
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+openidea-plugin = { path = "../../../../../sdk/rust/openidea-plugin" }
+
+[profile.release]
+opt-level = "s"
+lto = true
+strip = true
+codegen-units = 1
+panic = "abort"

crates/ide-plugin/tests/fixtures/semantic-tokens/README.md

@@ -0,0 +1,7 @@
+# Semantic tokens test fixture
+
+Colours the name after every `->` in PHP documents that is not called: `method`
+when the document declares `function <name>(`, `property` otherwise.
+
+Used by `cargo test -p ide-host --test plugin_semantic_tokens` to exercise the
+optional `semantic-tokens-provider` export. It is not a marketplace plugin.

crates/ide-plugin/tests/fixtures/semantic-tokens/dist/plugin.wasm.sources

@@ -0,0 +1 @@
+c1ea0651bd0127a01dd54884aad76a9c910ad0f56fd442dd7d50883da85242dd

crates/ide-plugin/tests/fixtures/semantic-tokens/plugin.toml

@@ -0,0 +1,9 @@
+id = "test.semantic-tokens"
+name = "Semantic tokens fixture"
+version = "0.1.0"
+engines.openidea = "^1.3"
+wasm = "dist/plugin.wasm"
+
+[[contributes.providers]]
+kinds = ["semantic-tokens"]
+languages = ["php"]

crates/ide-plugin/tests/fixtures/semantic-tokens/src/lib.rs

@@ -0,0 +1,37 @@
+use openidea_plugin::semantic_tokens::{Document, SemanticToken, SemanticTokens, TokenKind};
+use openidea_plugin::{Error, Invocation, Plugin, Range, register_plugin, text};
+
+#[derive(Default)]
+struct Fixture;
+
+impl Plugin for Fixture {
+    fn run_command(&mut self, id: &str, _: &Invocation, _: Option<&str>) -> Result<Option<String>, Error> {
+        Err(Error::NotFound(id.into()))
+    }
+}
+
+impl SemanticTokens for Fixture {
+    fn semantic_tokens(&mut self, doc: &Document) -> Vec<SemanticToken> {
+        let text = doc.text.as_str();
+        let mut out = Vec::new();
+        let mut rest = 0;
+        while let Some(i) = text[rest..].find("->") {
+            let start = rest + i + 2;
+            let end = text[start..]
+                .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
+                .map_or(text.len(), |n| start + n);
+            rest = end;
+            if end == start || text[end..].starts_with('(') {
+                continue;
+            }
+            let name = &text[start..end];
+            let kind =
+                if text.contains(&format!("function {name}(")) { TokenKind::Method } else { TokenKind::Property };
+            let range = Range { start: text::position(text, start), end: text::position(text, end) };
+            out.push(SemanticToken::new(range, kind));
+        }
+        out
+    }
+}
+
+register_plugin!(Fixture, semantic_tokens);

plugins/README.md

@@ -34,10 +34,10 @@ plugins/
    ├─ regex-tester/           # html view talking to its code, settings, the editor's selection
    └─ license-header/         # file-opened events, quick fix, project scan in a task
 sdk/rust/openidea-plugin/     # the Rust SDK (host API, Plugin trait, mock host)
-crates/ide-plugin/tests/fixtures/  # test plugins: hello, jobs, language-toy, misbehave
+crates/ide-plugin/tests/fixtures/  # test plugins: hello, jobs, language-toy, misbehave, semantic-tokens
 ```
 
-The plugin API is defined in [`wit/`](../wit/) (`openidea:plugin@1.2.0`, world
+The plugin API is defined in [`wit/`](../wit/) (`openidea:plugin@1.3.0`, world
 `plugin`).
 
 ## 1. Quick start
@@ -402,7 +402,7 @@ declaration matches the file's language are called (and loaded):
 
 ```toml
 [[contributes.providers]]
-kinds = ["definition", "line-markers", "completion"]  # also "inline-completion", "code-actions"
+kinds = ["definition", "line-markers", "completion"]  # also "inline-completion", "code-actions", "semantic-tokens"
 languages = ["java", "xml"]                           # editor language ids; "*" for any
 ```
 
@@ -432,6 +432,31 @@ dropped at 50 ms like a slow one. Inline completion is exempt from the 50 ms (it
 need the network) but is cancelled as soon as the user types on: check
 `Task::is_cancelled` or just let the host interrupt the call.
 
+#### Semantic highlighting
+
+The `semantic-tokens` kind (API 1.3: `engines.openidea = "^1.3"`) colours
+tokens in documents of the declared languages, including languages the plugin
+doesn't contribute, such as `php`. Register with
+`register_plugin!(MyPlugin, semantic_tokens)` (any combination works:
+`(MyPlugin, providers, views, semantic_tokens)`) and implement
+`openidea_plugin::semantic_tokens::SemanticTokens`:
+
+```rust
+impl SemanticTokens for MyPlugin {
+    fn semantic_tokens(&mut self, doc: &Document) -> Vec<SemanticToken> {
+        let (start, end) = (text::position(&doc.text, 10), text::position(&doc.text, 14));
+        vec![SemanticToken::new(Range { start, end }, TokenKind::Method)]
+    }
+}
+```
+
+A token covers part of one line and carries an LSP token kind (`TokenKind`:
+`method`, `property`, `type`, …) and modifiers (`TokenModifier`: `static`,
+`deprecated`, …); the WIT takes them as kebab-case strings. The tokens join
+the language server's semantic tokens and win where they overlap one, so the
+editor's colour scheme styles them like the server's. They are asked again
+after each edit, under the edit-path budget; at most 65,536 per document.
+
 ### Network and secrets
 
 `host::fetch` makes an HTTPS request (plain HTTP only to `localhost`) to a host
@@ -972,8 +997,8 @@ the API itself follows _Evolving the plugin API_ below.
 
 ## 10. Evolving the plugin API
 
-Released versions are frozen: 1.0.0 shipped with OpenIDEA 0.1.36 and 1.1.0
-with 0.1.37. wasmtime decides what stays compatible: it resolves interface names
+Released versions are frozen: 1.0.0 shipped with OpenIDEA 0.1.36, 1.1.0
+with 0.1.37 and 1.2.0 with 0.1.38. wasmtime decides what stays compatible: it resolves interface names
 by semver (a component importing `openidea:plugin/[email protected]` links against a
 host defining `@1.1.0`, and the host finds its exports the same way), but
 function types must match exactly — a record with another field count, or a

plugins/official/ai-completion/dist/plugin.wasm.sources

@@ -1 +1 @@
-fd5d8c0b9c6828bbeb0a93e394a0468abe83f051d1c20e50a7ac91fe12d5ab68
+cbc68107db26fa7d6fa77c8983f469536defe541d9bcd037d490030669ff999d

plugins/official/checkstyle/dist/plugin.wasm.sources

@@ -1 +1 @@
-eb4d8c5d84dd2ea0171297f756cdf56bf72d3db88b8f4736bc2b3ab8346bd5eb
+0715a5670087df9e7f653f0efcd4087988a995c1ffa3eb165d119cf2737438c7

plugins/official/csv-editor/dist/plugin.wasm.sources

@@ -1 +1 @@
-5db8c9213f6bef54f92c39c994717c8dfd4623a46b615293fe10b7a2990014a9
+b015bde71d2e116255e27190829c08bd6768e1077d0ee61321a8660d7008b229

plugins/official/git-toolbox/dist/plugin.wasm.sources

@@ -1 +1 @@
-b95b4b5c2e4451e670d1b013ad03662469456de03fabdbd8a3c62eccd93fb0f0
+6c93f780430507806394a152041e20062b69bc2ae2642ec24eaffc93e189bac5

plugins/official/graphviz/dist/plugin.wasm.sources

@@ -1 +1 @@
-6b800f268041ebb169117dfa31eead60315f51ebd92e409751f2670c92eac473
+0b0b37abc7412a1517c0c1bd7d678279acc1e1afbdb6cedcded1582779a068a1

plugins/official/key-promoter/dist/plugin.wasm.sources

@@ -1 +1 @@
-6a2c0c1f354b9c8defbcc9ec4a098fd956991dfb656ef2a4df92a219e60102d7
+629c5aed682a5be77a5e1a4e9a3f3483533e28679c07bfc88b21021a0dee28d8

plugins/official/license-header/dist/plugin.wasm.sources

@@ -1 +1 @@
-341466108d3a98c295a33d56c6f2a11736ced33dc0c4358d395cb376c24be663
+bcfd9e933a2c5cc575fc9d2f22b8b5a151fd1f366a31235b44dcc3ef8bd3422d

plugins/official/mybatis/dist/plugin.wasm.sources

@@ -1 +1 @@
-a6d1331afb12c8b215d8dc4450b8c4a424a6951b359dadb925da5ef0c52ecedc
+a69659e480b2f88b818d33b3b28ef8fbe29f7564800ac6d57e6c2e013c85a9d9

plugins/official/regex-tester/dist/plugin.wasm.sources

@@ -1 +1 @@
-6e1490c35f7094221a856cf23ba2b6dab92bf8e8c130b156aad1a0ef68a52086
+55f4e790142e1de70853080ced47dc80415384f7a0ec5645b5376b89f31d8354

plugins/official/solidity/dist/plugin.wasm.sources

@@ -1 +1 @@
-ce28fb9dfa83dfc46bc6215f9517e2b339b34b40abb85fe12b9534fd63a76899
+a89ac3590b8f7474a1d503ced271d2c60e3c99bbd5c28ae9e4c04256b1a794a2

plugins/official/string-tools/dist/plugin.wasm.sources

@@ -1 +1 @@
-d2b50d6ee2fe2a0991b7a0006c4899930cfb884f32db231f99c3144ef7f7d0d0
+adae51c270a7ad0dd05082f25deb0b0ff5012225b2dc7e7efda923dad3904869

sdk/rust/openidea-plugin/src/lib.rs

@@ -44,6 +44,7 @@ pub mod indexing;
 #[cfg(not(target_arch = "wasm32"))]
 pub mod mock;
 pub mod providers;
+pub mod semantic_tokens;
 pub mod syntax;
 pub mod text;
 mod types;
@@ -90,7 +91,7 @@ pub mod providers_bindings {
         export_macro_name: "__export_editor_providers_world",
         default_bindings_module: "openidea_plugin::providers_bindings",
         with: {
-            "openidea:plugin/types@1.2.0": crate::bindings::openidea::plugin::types,
+            "openidea:plugin/types@1.3.0": crate::bindings::openidea::plugin::types,
         },
     });
 }
@@ -107,10 +108,10 @@ pub mod views_bindings {
         export_macro_name: "__export_views_world",
         default_bindings_module: "openidea_plugin::views_bindings",
         with: {
-            "openidea:plugin/types@1.2.0": crate::bindings::openidea::plugin::types,
-            "openidea:plugin/workspace@1.2.0": crate::bindings::openidea::plugin::workspace,
-            "openidea:plugin/editor@1.2.0": crate::bindings::openidea::plugin::editor,
-            "openidea:plugin/invocation@1.2.0": crate::bindings::openidea::plugin::invocation,
+            "openidea:plugin/types@1.3.0": crate::bindings::openidea::plugin::types,
+            "openidea:plugin/workspace@1.3.0": crate::bindings::openidea::plugin::workspace,
+            "openidea:plugin/editor@1.3.0": crate::bindings::openidea::plugin::editor,
+            "openidea:plugin/invocation@1.3.0": crate::bindings::openidea::plugin::invocation,
         },
     });
 }
@@ -122,12 +123,23 @@ pub mod indexing_bindings {
         world: "indexing", path: "wit", pub_export_macro: true,
         export_macro_name: "__export_indexing_world",
         default_bindings_module: "openidea_plugin::indexing_bindings",
-        with: { "openidea:plugin/types@1.2.0": crate::bindings::openidea::plugin::types },
+        with: { "openidea:plugin/types@1.3.0": crate::bindings::openidea::plugin::types },
+    });
+}
+
+#[cfg(target_arch = "wasm32")]
+#[doc(hidden)]
+pub mod semantic_bindings {
+    wit_bindgen::generate!({
+        world: "semantic-tokens", path: "wit", pub_export_macro: true,
+        export_macro_name: "__export_semantic_tokens_world",
+        default_bindings_module: "openidea_plugin::semantic_bindings",
+        with: { "openidea:plugin/[email protected]": crate::bindings::openidea::plugin::types },
     });
 }
 
 /// The plugin API version this SDK targets.
-pub const API_VERSION: &str = "1.2";
+pub const API_VERSION: &str = "1.3";
 
 /// A plugin. One instance lives as long as the plugin's Wasm instance (until
 /// it is disabled, crashes or is reclaimed when idle).
@@ -154,7 +166,7 @@ pub trait Plugin: 'static {
 /// `syntax`, `T` is also a [`syntax::SyntaxProvider`] for its languages.
 #[macro_export]
 macro_rules! register_plugin {
-    ($plugin:ty) => {
+    ($plugin:ty $(, $export:ident)* $(,)?) => {
         // Natively (tests) nothing is exported; keep the type "used".
         #[cfg(not(target_arch = "wasm32"))]
         const _: fn() -> $plugin = <$plugin as ::core::default::Default>::default;
@@ -162,103 +174,55 @@ macro_rules! register_plugin {
         const _: () = {
             struct __OpenideaComponent;
             $crate::__plugin_exports!($plugin, __OpenideaComponent, $crate::bindings);
+            $($crate::__optional_exports!($export, $plugin, __OpenideaComponent);)*
     $crate::bindings::__export_plugin_world!(__OpenideaComponent with_types_in $crate::bindings);
         };
     };
-    ($plugin:ty, providers) => {
-        #[cfg(not(target_arch = "wasm32"))]
-        const _: fn() -> $plugin = <$plugin as ::core::default::Default>::default;
-        #[cfg(target_arch = "wasm32")]
-        const _: () = {
-            struct __OpenideaComponent;
-            $crate::__plugin_exports!($plugin, __OpenideaComponent, $crate::bindings);
-            $crate::__providers_exports!($plugin, __OpenideaComponent);
-    $crate::bindings::__export_plugin_world!(__OpenideaComponent with_types_in $crate::bindings);
-        };
-    };
-    ($plugin:ty, indexing) => {
-        #[cfg(not(target_arch = "wasm32"))]
-        const _: fn() -> $plugin = <$plugin as ::core::default::Default>::default;
-        #[cfg(target_arch = "wasm32")]
-        const _: () = {
-            struct __OpenideaComponent;
-            $crate::__plugin_exports!($plugin, __OpenideaComponent, $crate::bindings);
-            $crate::__indexing_exports!($plugin, __OpenideaComponent);
-    $crate::bindings::__export_plugin_world!(__OpenideaComponent with_types_in $crate::bindings);
-        };
-    };
-    ($plugin:ty, providers, indexing) => {
-        #[cfg(not(target_arch = "wasm32"))]
-        const _: fn() -> $plugin = <$plugin as ::core::default::Default>::default;
-        #[cfg(target_arch = "wasm32")]
-        const _: () = {
-            struct __OpenideaComponent;
-            $crate::__plugin_exports!($plugin, __OpenideaComponent, $crate::bindings);
-            $crate::__providers_exports!($plugin, __OpenideaComponent);
-            $crate::__indexing_exports!($plugin, __OpenideaComponent);
-    $crate::bindings::__export_plugin_world!(__OpenideaComponent with_types_in $crate::bindings);
-        };
-    };
-    ($plugin:ty, views) => {
-        #[cfg(not(target_arch = "wasm32"))]
-        const _: fn() -> $plugin = <$plugin as ::core::default::Default>::default;
-        #[cfg(target_arch = "wasm32")]
-        const _: () = {
-            struct __OpenideaComponent;
-            $crate::__plugin_exports!($plugin, __OpenideaComponent, $crate::bindings);
-            $crate::__views_exports!($plugin, __OpenideaComponent);
-    $crate::bindings::__export_plugin_world!(__OpenideaComponent with_types_in $crate::bindings);
-        };
-    };
-    ($plugin:ty, providers, views) => {
-        #[cfg(not(target_arch = "wasm32"))]
-        const _: fn() -> $plugin = <$plugin as ::core::default::Default>::default;
-        #[cfg(target_arch = "wasm32")]
-        const _: () = {
-            struct __OpenideaComponent;
-            $crate::__plugin_exports!($plugin, __OpenideaComponent, $crate::bindings);
-            $crate::__providers_exports!($plugin, __OpenideaComponent);
-            $crate::__views_exports!($plugin, __OpenideaComponent);
-    $crate::bindings::__export_plugin_world!(__OpenideaComponent with_types_in $crate::bindings);
-        };
-    };
-    ($plugin:ty, syntax) => {
-        #[cfg(not(target_arch = "wasm32"))]
-        const _: fn() -> $plugin = <$plugin as ::core::default::Default>::default;
-        #[cfg(target_arch = "wasm32")]
-        const _: () = {
-            struct __OpenideaComponent;
-            $crate::__plugin_exports!($plugin, __OpenideaComponent, $crate::bindings);
+}
 
-            impl $crate::syntax_bindings::exports::openidea::plugin::syntax_provider::Guest for __OpenideaComponent {
-                fn highlights(
-                    language: ::std::string::String,
-                    text: ::std::string::String,
-                ) -> ::std::vec::Vec<$crate::syntax_bindings::exports::openidea::plugin::syntax_provider::Token>
-                {
-                    $crate::__rt::highlights::<$plugin>(&language, &text)
-                }
-
-                fn outline(
-                    language: ::std::string::String,
-                    text: ::std::string::String,
-                ) -> ::std::vec::Vec<$crate::syntax_bindings::exports::openidea::plugin::syntax_provider::Symbol>
-                {
-                    $crate::__rt::outline::<$plugin>(&language, &text)
-                }
-
-                fn folding(
-                    language: ::std::string::String,
-                    text: ::std::string::String,
-                ) -> ::std::vec::Vec<$crate::syntax_bindings::exports::openidea::plugin::syntax_provider::Fold>
-                {
-                    $crate::__rt::folding::<$plugin>(&language, &text)
-                }
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __optional_exports {
+    (providers, $plugin:ty, $component:ident) => {
+        $crate::__providers_exports!($plugin, $component);
+    };
+    (views, $plugin:ty, $component:ident) => {
+        $crate::__views_exports!($plugin, $component);
+    };
+    (indexing, $plugin:ty, $component:ident) => {
+        $crate::__indexing_exports!($plugin, $component);
+    };
+    (semantic_tokens, $plugin:ty, $component:ident) => {
+        $crate::__semantic_tokens_exports!($plugin, $component);
+    };
+    (syntax, $plugin:ty, $component:ident) => {
+        impl $crate::syntax_bindings::exports::openidea::plugin::syntax_provider::Guest for $component {
+            fn highlights(
+                language: ::std::string::String,
+                text: ::std::string::String,
+            ) -> ::std::vec::Vec<$crate::syntax_bindings::exports::openidea::plugin::syntax_provider::Token>
+            {
+                $crate::__rt::highlights::<$plugin>(&language, &text)
             }
 
-    $crate::bindings::__export_plugin_world!(__OpenideaComponent with_types_in $crate::bindings);
-    $crate::syntax_bindings::__export_syntax_world!(__OpenideaComponent with_types_in $crate::syntax_bindings);
-        };
+            fn outline(
+                language: ::std::string::String,
+                text: ::std::string::String,
+            ) -> ::std::vec::Vec<$crate::syntax_bindings::exports::openidea::plugin::syntax_provider::Symbol>
+            {
+                $crate::__rt::outline::<$plugin>(&language, &text)
+            }
+
+            fn folding(
+                language: ::std::string::String,
+                text: ::std::string::String,
+            ) -> ::std::vec::Vec<$crate::syntax_bindings::exports::openidea::plugin::syntax_provider::Fold>
+            {
+                $crate::__rt::folding::<$plugin>(&language, &text)
+            }
+        }
+
+    $crate::syntax_bindings::__export_syntax_world!($component with_types_in $crate::syntax_bindings);
     };
 }
 
@@ -362,6 +326,23 @@ macro_rules! __indexing_exports {
     };
 }
 
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __semantic_tokens_exports {
+    ($plugin:ty, $component:ident) => {
+        impl $crate::semantic_bindings::exports::openidea::plugin::semantic_tokens_provider::Guest for $component {
+            fn semantic_tokens(
+                doc: $crate::semantic_bindings::exports::openidea::plugin::semantic_tokens_provider::Document,
+            ) -> ::std::vec::Vec<
+                $crate::semantic_bindings::exports::openidea::plugin::semantic_tokens_provider::SemanticToken,
+            > {
+                $crate::__rt::semantic_tokens::<$plugin>(doc)
+            }
+        }
+        $crate::semantic_bindings::__export_semantic_tokens_world!($component with_types_in $crate::semantic_bindings);
+    };
+}
+
 #[doc(hidden)]
 #[macro_export]
 macro_rules! __plugin_exports {
@@ -602,6 +583,21 @@ pub mod __rt {
         }
     }
 
+    pub fn semantic_tokens<T: Plugin + crate::semantic_tokens::SemanticTokens + Default>(
+        d: crate::semantic_bindings::exports::openidea::plugin::semantic_tokens_provider::Document,
+    ) -> Vec<crate::semantic_bindings::exports::openidea::plugin::semantic_tokens_provider::SemanticToken> {
+        use crate::semantic_bindings::exports::openidea::plugin::semantic_tokens_provider as w;
+        let doc = crate::providers::Document { path: d.path, language: d.language, version: d.version, text: d.text };
+        with::<T, _>(|p| p.semantic_tokens(&doc))
+            .into_iter()
+            .map(|t| w::SemanticToken {
+                range: crate::host::__range_to(t.range),
+                kind: t.kind.as_str().into(),
+                modifiers: t.modifiers.into_iter().map(|m| m.as_str().into()).collect(),
+            })
+            .collect()
+    }
+
     // ---- views ----
 
     use crate::views as v;

sdk/rust/openidea-plugin/src/semantic_tokens.rs

@@ -0,0 +1,137 @@
+use crate::types::Range;
+
+pub use crate::providers::Document;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[non_exhaustive]
+pub enum TokenKind {
+    Namespace,
+    Type,
+    Class,
+    Enum,
+    Interface,
+    Struct,
+    TypeParameter,
+    Parameter,
+    Variable,
+    Property,
+    EnumMember,
+    Event,
+    Function,
+    Method,
+    Macro,
+    Keyword,
+    Modifier,
+    Comment,
+    String,
+    Number,
+    Regexp,
+    Operator,
+    Decorator,
+}
+
+impl TokenKind {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Namespace => "namespace",
+            Self::Type => "type",
+            Self::Class => "class",
+            Self::Enum => "enum",
+            Self::Interface => "interface",
+            Self::Struct => "struct",
+            Self::TypeParameter => "type-parameter",
+            Self::Parameter => "parameter",
+            Self::Variable => "variable",
+            Self::Property => "property",
+            Self::EnumMember => "enum-member",
+            Self::Event => "event",
+            Self::Function => "function",
+            Self::Method => "method",
+            Self::Macro => "macro",
+            Self::Keyword => "keyword",
+            Self::Modifier => "modifier",
+            Self::Comment => "comment",
+            Self::String => "string",
+            Self::Number => "number",
+            Self::Regexp => "regexp",
+            Self::Operator => "operator",
+            Self::Decorator => "decorator",
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[non_exhaustive]
+pub enum TokenModifier {
+    Declaration,
+    Definition,
+    Readonly,
+    Static,
+    Deprecated,
+    Abstract,
+    Async,
+    Modification,
+    Documentation,
+    DefaultLibrary,
+}
+
+impl TokenModifier {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Declaration => "declaration",
+            Self::Definition => "definition",
+            Self::Readonly => "readonly",
+            Self::Static => "static",
+            Self::Deprecated => "deprecated",
+            Self::Abstract => "abstract",
+            Self::Async => "async",
+            Self::Modification => "modification",
+            Self::Documentation => "documentation",
+            Self::DefaultLibrary => "default-library",
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SemanticToken {
+    pub range: Range,
+    pub kind: TokenKind,
+    pub modifiers: Vec<TokenModifier>,
+}
+
+impl SemanticToken {
+    pub fn new(range: Range, kind: TokenKind) -> Self {
+        Self { range, kind, modifiers: Vec::new() }
+    }
+
+    pub fn modifier(mut self, modifier: TokenModifier) -> Self {
+        self.modifiers.push(modifier);
+        self
+    }
+}
+
+pub trait SemanticTokens {
+    fn semantic_tokens(&mut self, doc: &Document) -> Vec<SemanticToken>;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::Position;
+
+    #[test]
+    fn names_follow_the_wit_vocabulary() {
+        assert_eq!(TokenKind::TypeParameter.as_str(), "type-parameter");
+        assert_eq!(TokenKind::EnumMember.as_str(), "enum-member");
+        assert_eq!(TokenModifier::DefaultLibrary.as_str(), "default-library");
+    }
+
+    #[test]
+    fn a_token_collects_its_modifiers() {
+        let range = Range { start: Position { line: 1, character: 2 }, end: Position { line: 1, character: 6 } };
+        let token = SemanticToken::new(range, TokenKind::Method)
+            .modifier(TokenModifier::Static)
+            .modifier(TokenModifier::Deprecated);
+        assert_eq!(token.modifiers, [TokenModifier::Static, TokenModifier::Deprecated]);
+    }
+}

sdk/rust/openidea-plugin/tests/wit_coverage.rs

@@ -51,3 +51,32 @@ fn every_imported_function_has_an_sdk_wrapper() {
     assert!(checked > 30, "only {checked} functions found: is the world still called `plugin`?");
     assert!(missing.is_empty(), "plugin API functions without an SDK wrapper:\n  {}", missing.join("\n  "));
 }
+
+#[test]
+fn every_optional_export_can_be_registered() {
+    let mut resolve = Resolve::default();
+    let (pkg, _) = resolve.push_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("wit")).unwrap();
+    let source = sdk_source();
+    let base = &resolve.worlds[resolve.packages[pkg].worlds["plugin"]];
+    let mut missing = Vec::new();
+    let mut checked = 0;
+    for (name, &id) in &resolve.packages[pkg].worlds {
+        if !name.starts_with("plugin-") {
+            continue;
+        }
+        for (key, item) in &resolve.worlds[id].exports {
+            let WorldItem::Interface { id, .. } = item else { continue };
+            if base.exports.contains_key(key) {
+                continue;
+            }
+            let module = resolve.interfaces[*id].name.as_deref().unwrap().replace('-', "_");
+            let guest = format!("exports::openidea::plugin::{module}::Guest for");
+            checked += 1;
+            if !source.contains(&guest) {
+                missing.push(format!("{name}: {module}"));
+            }
+        }
+    }
+    assert!(checked >= 5, "only {checked} optional exports found");
+    assert!(missing.is_empty(), "optional exports register_plugin! can't export:\n  {}", missing.join("\n  "));
+}

sdk/rust/openidea-plugin/wit/events.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Workbench events, delivered in batches to `event-handler.handle`.
 interface events {

sdk/rust/openidea-plugin/wit/exports.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 interface lifecycle {
     use types.{context, error};

sdk/rust/openidea-plugin/wit/host.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Writes to the IDE log, tagged with the plugin id.
 interface log {

sdk/rust/openidea-plugin/wit/indexing.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Cross-file associations a plugin publishes for the files it declares under
 /// `[[contributes.documentIndexers]]`. The host calls `index` in cancellable

sdk/rust/openidea-plugin/wit/io.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Problems in files, shown in the editor and the Problems view.
 interface diagnostics {

sdk/rust/openidea-plugin/wit/plugin.wit

@@ -7,7 +7,7 @@
 /// or processes beyond WASI's clocks and randomness; everything else goes
 /// through the imports below, gated by the capabilities granted in
 /// `plugin.toml` (a missing one returns `error::denied`).
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 world plugin {
     import log;
@@ -56,6 +56,11 @@ world plugin-indexing {
     export document-indexer;
 }
 
+world plugin-semantic-tokens {
+    include plugin;
+    export semantic-tokens-provider;
+}
+
 /// Only the `syntax-provider` export: guests generate it next to `plugin`
 /// (the component tools merge both into one component).
 world syntax {
@@ -76,3 +81,7 @@ world views-provider {
 world indexing {
     export document-indexer;
 }
+
+world semantic-tokens {
+    export semantic-tokens-provider;
+}

sdk/rust/openidea-plugin/wit/providers.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Editor features a plugin provides for the languages it declares under
 /// `[[contributes.providers]]` (the host calls only declared kinds). They run

sdk/rust/openidea-plugin/wit/semantic.wit

@@ -0,0 +1,20 @@
+package openidea:[email protected];
+
+interface semantic-tokens-provider {
+    use types.{range};
+
+    record document {
+        path: string,
+        language: string,
+        version: u64,
+        text: string,
+    }
+
+    record semantic-token {
+        range: range,
+        kind: string,
+        modifiers: list<string>,
+    }
+
+    semantic-tokens: func(doc: document) -> list<semantic-token>;
+}

sdk/rust/openidea-plugin/wit/types.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Shared types. Positions are 0-based lines and UTF-16 code unit columns,
 /// as in the editor and LSP.

sdk/rust/openidea-plugin/wit/views.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Declarative views: tool windows declared
 /// under `[[contributes.views]]` and custom editors under

sdk/rust/openidea-plugin/wit/workspace.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 interface workspace {
     use types.{error};

wit/events.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Workbench events, delivered in batches to `event-handler.handle`.
 interface events {

wit/exports.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 interface lifecycle {
     use types.{context, error};

wit/host.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Writes to the IDE log, tagged with the plugin id.
 interface log {

wit/indexing.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Cross-file associations a plugin publishes for the files it declares under
 /// `[[contributes.documentIndexers]]`. The host calls `index` in cancellable

wit/io.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Problems in files, shown in the editor and the Problems view.
 interface diagnostics {

wit/plugin.wit

@@ -7,7 +7,7 @@
 /// or processes beyond WASI's clocks and randomness; everything else goes
 /// through the imports below, gated by the capabilities granted in
 /// `plugin.toml` (a missing one returns `error::denied`).
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 world plugin {
     import log;
@@ -56,6 +56,11 @@ world plugin-indexing {
     export document-indexer;
 }
 
+world plugin-semantic-tokens {
+    include plugin;
+    export semantic-tokens-provider;
+}
+
 /// Only the `syntax-provider` export: guests generate it next to `plugin`
 /// (the component tools merge both into one component).
 world syntax {
@@ -76,3 +81,7 @@ world views-provider {
 world indexing {
     export document-indexer;
 }
+
+world semantic-tokens {
+    export semantic-tokens-provider;
+}

wit/providers.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Editor features a plugin provides for the languages it declares under
 /// `[[contributes.providers]]` (the host calls only declared kinds). They run

wit/semantic.wit

@@ -0,0 +1,20 @@
+package openidea:[email protected];
+
+interface semantic-tokens-provider {
+    use types.{range};
+
+    record document {
+        path: string,
+        language: string,
+        version: u64,
+        text: string,
+    }
+
+    record semantic-token {
+        range: range,
+        kind: string,
+        modifiers: list<string>,
+    }
+
+    semantic-tokens: func(doc: document) -> list<semantic-token>;
+}

wit/types.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Shared types. Positions are 0-based lines and UTF-16 code unit columns,
 /// as in the editor and LSP.

wit/views.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 /// Declarative views: tool windows declared
 /// under `[[contributes.views]]` and custom editors under

wit/workspace.wit

@@ -1,4 +1,4 @@
-package openidea:plugin@1.2.0;
+package openidea:plugin@1.3.0;
 
 interface workspace {
     use types.{error};