дзеркало
https://github.com/l0ng-ai/openidea.git
синхронізовано 2026-11-21 00:50:47 +02:00
Порівняти коміти
33
Коміти
| Автор | SHA1 | Дата | |
|---|---|---|---|
|
|
29f8955b38 | ||
|
|
00871ab971 | ||
|
|
5aba5e0629 | ||
|
|
eec447872c | ||
|
|
98418d727b | ||
|
|
2265497b18 | ||
|
|
7f33640895 | ||
|
|
a57e12dbc5 | ||
|
|
b3f8cec272 | ||
|
|
e44e389571 | ||
|
|
0cf0e93d92 | ||
|
|
c7ad3459c4 | ||
|
|
af281341b1 | ||
|
|
313a5f3a14 | ||
|
|
18cd2ef63b | ||
|
|
4fc55f7520 | ||
|
|
795413c366 | ||
|
|
dd3c9c7b0a | ||
|
|
361ca01cc0 | ||
|
|
1b9c330613 | ||
|
|
2d7bb56a8d | ||
|
|
2c8a029939 | ||
|
|
6e2c19846c | ||
|
|
6c2ae13387 | ||
|
|
34cb4ad5e0 | ||
|
|
d080ca28f8 | ||
|
|
22a5e4c9e6 | ||
|
|
e93ac42489 | ||
|
|
ae604b7933 | ||
|
|
bd27eb5474 | ||
|
|
15fcf4f2c1 | ||
|
|
65415a03bd | ||
|
|
58585e1064 |
No files matched your search
@@ -256,6 +256,38 @@ impl AgentCatalogs {
|
||||
.cloned()
|
||||
}
|
||||
|
||||
pub(super) async fn load_levels(
|
||||
self: &Arc<Self>,
|
||||
shared: &Arc<Shared>,
|
||||
provider: &str,
|
||||
model: &str,
|
||||
) -> Result<Vec<AgentConfigOption>, String> {
|
||||
if let Some(levels) = self.levels(shared, provider, model) {
|
||||
return Ok(levels);
|
||||
}
|
||||
let unavailable = || "The agent did not report settings for the selected model".to_owned();
|
||||
let spec = shared.spec(provider).ok_or_else(unavailable)?;
|
||||
let (_, flight) = self.request(shared, spec, false, Some(model));
|
||||
if let Some(mut ready) = flight {
|
||||
while !*ready.borrow_and_update() {
|
||||
if ready.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.levels(shared, provider, model).ok_or_else(|| {
|
||||
let state = self.state.lock();
|
||||
state
|
||||
.entries
|
||||
.get(&Self::key(shared, provider))
|
||||
.and_then(|entry| {
|
||||
entry.model_failures.get(model).map(|failure| &failure.error).or(entry.error.as_ref())
|
||||
})
|
||||
.cloned()
|
||||
.unwrap_or_else(unavailable)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn evict_if_idle(&self, shared: &Shared, provider: &str) {
|
||||
let key = Self::key(shared, provider);
|
||||
let mut state = self.state.lock();
|
||||
|
||||
@@ -323,6 +323,36 @@ async fn discovered_model_can_be_selected_while_the_session_is_not_running() {
|
||||
switch_chat_model(ChatSwitch::Idle).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discovered_model_effort_is_discovered_persisted_and_applied_before_the_first_reply() {
|
||||
switch_chat_model(ChatSwitch::Uncached).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discovered_model_with_no_effort_can_be_selected() {
|
||||
switch_chat_model(ChatSwitch::NoEffort).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discovered_model_settings_failure_leaves_the_chat_unchanged() {
|
||||
switch_chat_model(ChatSwitch::DiscoveryFails).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discovered_model_withdrawn_effort_prevents_sending() {
|
||||
switch_chat_model(ChatSwitch::EffortUnavailable).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discovered_model_rejected_effort_prevents_sending() {
|
||||
switch_chat_model(ChatSwitch::EffortRejected).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discovered_model_lookup_does_not_overwrite_a_newer_selection() {
|
||||
switch_chat_model(ChatSwitch::Superseded).await;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum ChatSwitch {
|
||||
Live,
|
||||
@@ -330,6 +360,12 @@ enum ChatSwitch {
|
||||
ResumeFails,
|
||||
Unavailable,
|
||||
Idle,
|
||||
Uncached,
|
||||
NoEffort,
|
||||
DiscoveryFails,
|
||||
EffortUnavailable,
|
||||
EffortRejected,
|
||||
Superseded,
|
||||
}
|
||||
|
||||
async fn switch_chat_model(scenario: ChatSwitch) {
|
||||
@@ -376,7 +412,21 @@ async fn switch_chat_model(scenario: ChatSwitch) {
|
||||
}
|
||||
let before_entries = serde_json::to_value(manager.entries(&task.id, None).unwrap()).unwrap();
|
||||
let before_config = manager.task(&task.id).unwrap().state.lock().task.config.clone();
|
||||
if matches!(scenario, ChatSwitch::Idle) {
|
||||
let offline = matches!(scenario, ChatSwitch::Idle | ChatSwitch::Uncached | ChatSwitch::Superseded);
|
||||
let missing_settings = matches!(
|
||||
scenario,
|
||||
ChatSwitch::Uncached | ChatSwitch::NoEffort | ChatSwitch::DiscoveryFails | ChatSwitch::Superseded
|
||||
);
|
||||
let no_effort = matches!(scenario, ChatSwitch::NoEffort);
|
||||
let effort_error = match scenario {
|
||||
ChatSwitch::EffortUnavailable => Some("no longer offers effort value thorough"),
|
||||
ChatSwitch::EffortRejected => Some("effort rejected by the agent"),
|
||||
_ => None,
|
||||
};
|
||||
let choose_effort = matches!(scenario, ChatSwitch::Uncached) || effort_error.is_some();
|
||||
let (catalog_models, catalog_effort): (&[&str], &[&str]) =
|
||||
if missing_settings { (&["old", "new"], &["low", "high"]) } else { (&["new", "old"], &["quick", "thorough"]) };
|
||||
if offline {
|
||||
manager.shutdown();
|
||||
manager = make_manager(tmp.path(), Arc::default(), &harness);
|
||||
}
|
||||
@@ -385,41 +435,122 @@ async fn switch_chat_model(scenario: ChatSwitch) {
|
||||
assert!(manager.refresh_models(PROVIDER, false, None).unwrap().refreshing);
|
||||
let next = harness.next().await;
|
||||
assert_eq!((next.launch, next.method.as_str()), (1, "session/new"));
|
||||
next.reply.send(Ok(response(&["new", "old"], &["catalog-only"]))).unwrap();
|
||||
assert_eq!(model_values(&completed(&mut events).await), ["new", "old"]);
|
||||
next.reply.send(Ok(response(catalog_models, catalog_effort))).unwrap();
|
||||
let catalog = completed(&mut events).await;
|
||||
assert_eq!(model_values(&catalog), catalog_models);
|
||||
if missing_settings {
|
||||
assert!(!catalog.thought_levels.contains_key("new"));
|
||||
}
|
||||
assert_eq!(manager.task(&task.id).unwrap().state.lock().task.config, before_config);
|
||||
|
||||
let live = matches!(scenario, ChatSwitch::Live);
|
||||
let offline = matches!(scenario, ChatSwitch::Idle);
|
||||
let resumed_launch = if missing_settings { 3 } else { 2 };
|
||||
let load_fails = matches!(scenario, ChatSwitch::ResumeFails);
|
||||
let unavailable = matches!(scenario, ChatSwitch::Unavailable);
|
||||
let switching = manager.set_config(&task.id, "model", "new".into());
|
||||
tokio::pin!(switching);
|
||||
if !offline {
|
||||
let next = next_chat_request(&mut harness, switching.as_mut()).await;
|
||||
assert_eq!((next.launch, next.method.as_str()), (0, "session/set_config_option"));
|
||||
assert_eq!(next.params, json!({"sessionId": "saved-chat", "configId": "model", "value": "new"}));
|
||||
if live {
|
||||
next.reply.send(Ok(response(&["new", "old"], &["session-only"]))).unwrap();
|
||||
} else {
|
||||
next.reply.send(Err("not in this process's model list".into())).unwrap();
|
||||
{
|
||||
let switching = manager.set_config(&task.id, "model", "new".into());
|
||||
tokio::pin!(switching);
|
||||
if !offline {
|
||||
let next = next_chat_request(&mut harness, switching.as_mut()).await;
|
||||
assert_eq!((next.launch, next.method.as_str()), (0, "session/set_config_option"));
|
||||
assert_eq!(next.params, json!({"sessionId": "saved-chat", "configId": "model", "value": "new"}));
|
||||
if live {
|
||||
next.reply.send(Ok(response(&["new", "old"], &["session-only"]))).unwrap();
|
||||
} else {
|
||||
next.reply.send(Err("not in this process's model list".into())).unwrap();
|
||||
}
|
||||
}
|
||||
if missing_settings {
|
||||
let discovery = next_chat_request(&mut harness, switching.as_mut()).await;
|
||||
assert_eq!((discovery.launch, discovery.method.as_str()), (2, "session/new"));
|
||||
discovery.reply.send(Ok(response(catalog_models, catalog_effort))).unwrap();
|
||||
let set = next_chat_request(&mut harness, switching.as_mut()).await;
|
||||
assert_eq!(set.params, json!({"sessionId": "discovery-only", "configId": "model", "value": "new"}));
|
||||
assert_eq!(set.method, "session/set_config_option");
|
||||
assert!(
|
||||
manager
|
||||
.prompt(&task.id, AgentPrompt { text: "too early".into(), ..Default::default() }, false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
if matches!(scenario, ChatSwitch::DiscoveryFails) {
|
||||
set.reply.send(Err("Sign in required".into())).unwrap();
|
||||
let error = switching.await.unwrap_err().to_string();
|
||||
assert!(error.contains("Sign in required"), "{error}");
|
||||
assert_eq!(manager.subscribe(Arc::new(|_| {})).remove(0).config, before_config);
|
||||
assert!(harness.pending.try_recv().is_err());
|
||||
manager.shutdown();
|
||||
return;
|
||||
}
|
||||
if matches!(scenario, ChatSwitch::Superseded) {
|
||||
manager.set_config(&task.id, "model", "old".into()).await.unwrap();
|
||||
set.reply.send(Ok(response(&["new", "old"], &["quick", "thorough"]))).unwrap();
|
||||
switching.await.unwrap();
|
||||
let task = manager.subscribe(Arc::new(|_| {})).remove(0);
|
||||
assert_eq!(task.config.iter().find(|o| o.id == "model").unwrap().current.as_deref(), Some("old"));
|
||||
assert_eq!(task.config.iter().find(|o| o.id == "effort").unwrap().current.as_deref(), Some("low"));
|
||||
assert!(harness.pending.try_recv().is_err());
|
||||
manager.shutdown();
|
||||
return;
|
||||
}
|
||||
let levels: &[&str] = if no_effort { &[] } else { &["quick", "thorough"] };
|
||||
set.reply.send(Ok(response(&["new", "old"], levels))).unwrap();
|
||||
}
|
||||
tokio::select! {
|
||||
result = &mut switching => result.unwrap(),
|
||||
request = harness.next() => panic!("selection must not reopen the chat or send: {request:?}"),
|
||||
}
|
||||
}
|
||||
// A fallback selection settles without waiting for a new process.
|
||||
tokio::select! {
|
||||
result = &mut switching => result.unwrap(),
|
||||
request = harness.next() => panic!("selection must not open another session: {request:?}"),
|
||||
}
|
||||
assert_eq!(serde_json::to_value(manager.entries(&task.id, None).unwrap()).unwrap(), before_entries);
|
||||
if matches!(scenario, ChatSwitch::Uncached) {
|
||||
let update = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let AgentEvent::Task { task: update } = events.recv().await.unwrap()
|
||||
&& update.id == task.id
|
||||
{
|
||||
break update;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let effort = update.config.iter().find(|o| o.id == "effort").unwrap();
|
||||
assert_eq!(effort.values.iter().map(|v| v.value.as_str()).collect::<Vec<_>>(), ["quick", "thorough"]);
|
||||
}
|
||||
let snapshot = |manager: &AgentManager| manager.subscribe(Arc::new(|_| {})).remove(0);
|
||||
let current = snapshot(&manager);
|
||||
assert_eq!(current.config.iter().find(|o| o.id == "model").unwrap().current.as_deref(), Some("new"));
|
||||
assert_eq!(current.model.as_deref(), Some(if live { "new" } else { "old" }));
|
||||
let effort =
|
||||
|task: &crate::AgentTask| task.config.iter().find(|o| o.id == "effort").and_then(|o| o.current.clone());
|
||||
assert_eq!(
|
||||
effort(¤t).as_deref(),
|
||||
if live {
|
||||
Some("session-only")
|
||||
} else if no_effort {
|
||||
None
|
||||
} else {
|
||||
Some("quick")
|
||||
},
|
||||
"the new model's effort must be available before sending"
|
||||
);
|
||||
if choose_effort {
|
||||
assert!(manager.set_config(&task.id, "effort", "high".into()).await.is_err());
|
||||
manager.set_config(&task.id, "effort", "thorough".into()).await.unwrap();
|
||||
assert_eq!(effort(&snapshot(&manager)).as_deref(), Some("thorough"));
|
||||
if matches!(scenario, ChatSwitch::Uncached) {
|
||||
manager.shutdown();
|
||||
manager = make_manager(tmp.path(), Arc::default(), &harness);
|
||||
let restored = snapshot(&manager);
|
||||
assert_eq!(restored.id, task.id);
|
||||
assert_eq!(effort(&restored).as_deref(), Some("thorough"));
|
||||
}
|
||||
}
|
||||
|
||||
manager.prompt(&task.id, AgentPrompt { text: "next".into(), ..Default::default() }, false).await.unwrap();
|
||||
if !live {
|
||||
let load = harness.next().await;
|
||||
assert_eq!((load.launch, load.method.as_str()), (2, "session/load"));
|
||||
assert_eq!((load.launch, load.method.as_str()), (resumed_launch, "session/load"));
|
||||
assert_eq!(load.params["sessionId"], "saved-chat");
|
||||
load.notifications
|
||||
.send(json!({"jsonrpc": "2.0", "method": "session/update", "params": {
|
||||
@@ -447,6 +578,7 @@ async fn switch_chat_model(scenario: ChatSwitch) {
|
||||
let task = snapshot(&manager);
|
||||
assert_eq!(task.model.as_deref(), Some("old"));
|
||||
assert_eq!(task.config.iter().find(|o| o.id == "model").unwrap().current.as_deref(), Some("old"));
|
||||
assert_eq!(effort(&task).as_deref(), Some("low"));
|
||||
let expected = if load_fails { "cannot resume saved-chat" } else { "does not offer model new" };
|
||||
assert!(task.error.as_deref().unwrap().contains(expected), "{:?}", task.error);
|
||||
assert!(harness.pending.try_recv().is_err(), "must not send on the previous model or create a session");
|
||||
@@ -472,24 +604,60 @@ async fn switch_chat_model(scenario: ChatSwitch) {
|
||||
return;
|
||||
}
|
||||
let set = harness.next().await;
|
||||
assert_eq!((set.launch, set.method.as_str()), (2, "session/set_config_option"));
|
||||
assert_eq!((set.launch, set.method.as_str()), (resumed_launch, "session/set_config_option"));
|
||||
assert_eq!(set.params, json!({"sessionId": "saved-chat", "configId": "model", "value": "new"}));
|
||||
set.reply.send(Ok(response(&["new", "old"], &["session-only"]))).unwrap();
|
||||
let levels: &[&str] = match scenario {
|
||||
ChatSwitch::NoEffort => &[],
|
||||
ChatSwitch::EffortUnavailable => &["session-only", "quick"],
|
||||
_ if choose_effort => &["session-only", "thorough"],
|
||||
_ => &["session-only"],
|
||||
};
|
||||
set.reply.send(Ok(response(&["new", "old"], levels))).unwrap();
|
||||
let set = harness.next().await;
|
||||
assert_eq!((set.launch, set.method.as_str()), (2, "session/set_mode"));
|
||||
assert_eq!((set.launch, set.method.as_str()), (resumed_launch, "session/set_mode"));
|
||||
assert_eq!(set.params, json!({"sessionId": "saved-chat", "modeId": "ask"}));
|
||||
set.reply.send(Ok(json!({}))).unwrap();
|
||||
if choose_effort && !matches!(scenario, ChatSwitch::EffortUnavailable) {
|
||||
let set = harness.next().await;
|
||||
assert_eq!((set.launch, set.method.as_str()), (resumed_launch, "session/set_config_option"));
|
||||
assert_eq!(set.params, json!({"sessionId": "saved-chat", "configId": "effort", "value": "thorough"}));
|
||||
let reply = if effort_error.is_some() {
|
||||
Err("effort rejected by the agent".into())
|
||||
} else {
|
||||
Ok(response(&["new", "old"], &["thorough", "session-only"]))
|
||||
};
|
||||
set.reply.send(reply).unwrap();
|
||||
}
|
||||
if let Some(expected) = effort_error {
|
||||
wait_for_chat(&manager, &task.id, AgentStatus::Failed).await;
|
||||
let current = snapshot(&manager);
|
||||
assert_eq!(current.model.as_deref(), Some("old"));
|
||||
assert_eq!(effort(¤t).as_deref(), Some("low"));
|
||||
assert!(current.error.as_deref().unwrap().contains(expected), "{:?}", current.error);
|
||||
assert!(harness.pending.try_recv().is_err(), "must not send using a fallback effort");
|
||||
manager.shutdown();
|
||||
return;
|
||||
}
|
||||
}
|
||||
let next = harness.next().await;
|
||||
assert_eq!(
|
||||
(next.launch, next.method.as_str(), &next.params["sessionId"]),
|
||||
(if live { 0 } else { 2 }, "session/prompt", &json!("saved-chat"))
|
||||
(if live { 0 } else { resumed_launch }, "session/prompt", &json!("saved-chat"))
|
||||
);
|
||||
next.reply.send(Ok(json!({"stopReason": "end_turn"}))).unwrap();
|
||||
wait_for_chat(&manager, &task.id, AgentStatus::Ready).await;
|
||||
let current = snapshot(&manager);
|
||||
assert_eq!(current.model.as_deref(), Some("new"));
|
||||
assert_eq!(current.config.iter().find(|o| o.id == "effort").unwrap().current.as_deref(), Some("session-only"));
|
||||
assert_eq!(
|
||||
effort(¤t).as_deref(),
|
||||
if choose_effort {
|
||||
Some("thorough")
|
||||
} else if no_effort {
|
||||
None
|
||||
} else {
|
||||
Some("session-only")
|
||||
}
|
||||
);
|
||||
assert_eq!(current.config.iter().find(|o| o.id == "mode").unwrap().current.as_deref(), Some("ask"));
|
||||
manager.shutdown();
|
||||
}
|
||||
|
||||
+309
-112
@@ -357,9 +357,11 @@ struct TaskFile {
|
||||
/// Settings the user chose, re-applied to every new or reloaded session.
|
||||
#[serde(default)]
|
||||
wanted: Vec<AgentConfigChoice>,
|
||||
/// Option id of a discovered model in `wanted` that still needs a same-session reload.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
/// Pending model option id in task files saved before pending settings were stored separately.
|
||||
#[serde(default, skip_serializing)]
|
||||
pending_model: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pending_switch: Option<PendingModel>,
|
||||
/// In the project's own checkout: files already changed when the task started.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
before: BTreeMap<String, worktree::Stamp>,
|
||||
@@ -384,6 +386,26 @@ struct TaskFile {
|
||||
title_renamed: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
struct PendingModel {
|
||||
model: AgentConfigChoice,
|
||||
candidate: Option<AgentConfigValue>,
|
||||
options: Vec<AgentConfigOption>,
|
||||
choices: Vec<AgentConfigChoice>,
|
||||
}
|
||||
|
||||
impl PendingModel {
|
||||
fn apply(&self, wanted: &mut Vec<AgentConfigChoice>, confirmed: &[AgentConfigOption]) {
|
||||
wanted.retain(|choice| {
|
||||
choice.id != self.model.id
|
||||
&& !self.options.iter().any(|o| o.id == choice.id)
|
||||
&& !confirmed.iter().any(|o| o.id == choice.id && config::is_per_model_option(o))
|
||||
});
|
||||
wanted.push(self.model.clone());
|
||||
wanted.extend(self.choices.iter().cloned());
|
||||
}
|
||||
}
|
||||
|
||||
/// The changed files of a checkout, stamped.
|
||||
type Snapshot = BTreeMap<String, worktree::Stamp>;
|
||||
|
||||
@@ -392,8 +414,9 @@ struct TaskState {
|
||||
session_id: Option<String>,
|
||||
conversation: Conversation,
|
||||
wanted: Vec<AgentConfigChoice>,
|
||||
pending_model: Option<String>,
|
||||
pending_model: Option<PendingModel>,
|
||||
model_choice: u64,
|
||||
discovering_model: Option<u64>,
|
||||
before: Arc<BTreeMap<String, worktree::Stamp>>,
|
||||
saved: Option<String>,
|
||||
/// See `TaskFile::touched`.
|
||||
@@ -415,23 +438,43 @@ struct TaskState {
|
||||
}
|
||||
|
||||
impl TaskState {
|
||||
fn pending_model(&self) -> Option<&AgentConfigChoice> {
|
||||
let id = self.pending_model.as_deref()?;
|
||||
self.wanted.iter().find(|c| c.id == id)
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> AgentTask {
|
||||
let mut task = self.task.clone();
|
||||
if let Some(choice) = self.pending_model()
|
||||
&& let Some(model) = task.config.iter_mut().find(|o| o.id == choice.id)
|
||||
if let Some(pending) = &self.pending_model
|
||||
&& let Some(model) = task.config.iter_mut().find(|o| o.id == pending.model.id)
|
||||
{
|
||||
model.current = Some(choice.value.clone());
|
||||
model.current = Some(pending.model.value.clone());
|
||||
if let Some(candidate) = &pending.candidate
|
||||
&& !model.values.iter().any(|v| v.value == candidate.value)
|
||||
{
|
||||
model.values.push(candidate.clone());
|
||||
}
|
||||
task.config.retain(|o| !config::is_per_model_option(o));
|
||||
task.config.extend(pending.options.iter().cloned().map(|mut option| {
|
||||
if let Some(choice) = pending.choices.iter().find(|choice| choice.id == option.id) {
|
||||
option.current = Some(choice.value.clone());
|
||||
}
|
||||
option
|
||||
}));
|
||||
}
|
||||
task
|
||||
}
|
||||
}
|
||||
|
||||
struct DiscoveringModel<'a> {
|
||||
state: &'a Mutex<TaskState>,
|
||||
generation: u64,
|
||||
}
|
||||
|
||||
impl Drop for DiscoveringModel<'_> {
|
||||
fn drop(&mut self) {
|
||||
let mut s = self.state.lock();
|
||||
if s.discovering_model == Some(self.generation) {
|
||||
s.discovering_model = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A running agent and its session.
|
||||
#[derive(Clone)]
|
||||
struct Live {
|
||||
@@ -796,6 +839,7 @@ impl AgentManager {
|
||||
entries: Vec::new(),
|
||||
wanted: Vec::new(),
|
||||
pending_model: None,
|
||||
pending_switch: None,
|
||||
before: BTreeMap::new(),
|
||||
saved: None,
|
||||
touched: Some(BTreeSet::new()),
|
||||
@@ -912,6 +956,7 @@ impl AgentManager {
|
||||
entries: Vec::new(),
|
||||
wanted: req.config,
|
||||
pending_model: None,
|
||||
pending_switch: None,
|
||||
before,
|
||||
saved,
|
||||
touched: (req.isolation == Isolation::Local).then(BTreeSet::new),
|
||||
@@ -949,6 +994,9 @@ impl AgentManager {
|
||||
if s.task.status == AgentStatus::Archived {
|
||||
return Err(IdeError::invalid("the task is archived"));
|
||||
}
|
||||
if s.discovering_model.is_some() {
|
||||
return Err(IdeError::unavailable("Model settings are still loading"));
|
||||
}
|
||||
if s.busy {
|
||||
let item = AgentQueued { id: uuid::Uuid::new_v4().simple().to_string(), prompt };
|
||||
if now {
|
||||
@@ -1068,6 +1116,24 @@ impl AgentManager {
|
||||
let task = self.task(id)?;
|
||||
let (target, change) = {
|
||||
let mut s = task.state.lock();
|
||||
let busy = s.busy;
|
||||
let archived = s.task.status == AgentStatus::Archived;
|
||||
if let Some(pending) = &mut s.pending_model
|
||||
&& let Some(target) = pending.options.iter().find(|o| o.id == option)
|
||||
{
|
||||
if archived || busy {
|
||||
return Err(IdeError::invalid("The pending model settings cannot be changed now"));
|
||||
}
|
||||
if !target.values.iter().any(|v| v.value == value) {
|
||||
return Err(IdeError::invalid(format!("{} has no value {value}", target.name)));
|
||||
}
|
||||
pending.choices.retain(|c| c.id != option);
|
||||
pending.choices.push(AgentConfigChoice { id: option.to_owned(), value });
|
||||
drop(s);
|
||||
task.persist();
|
||||
task.emit_task();
|
||||
return Ok(());
|
||||
}
|
||||
let target = s
|
||||
.task
|
||||
.config
|
||||
@@ -1075,6 +1141,9 @@ impl AgentManager {
|
||||
.find(|o| o.id == option)
|
||||
.ok_or_else(|| IdeError::not_found("no such setting"))?
|
||||
.clone();
|
||||
if s.pending_model.is_some() && config::is_per_model_option(&target) {
|
||||
return Err(IdeError::invalid("This setting is not available on the selected model"));
|
||||
}
|
||||
if !target.values.iter().any(|v| v.value == value) {
|
||||
if target.category.as_deref() != Some("model") {
|
||||
return Err(IdeError::invalid(format!("{} has no value {value}", target.name)));
|
||||
@@ -1083,8 +1152,12 @@ impl AgentManager {
|
||||
} else {
|
||||
let switching_model = target.category.as_deref() == Some("model");
|
||||
if switching_model {
|
||||
if s.pending_model.is_some() && s.busy {
|
||||
return Err(IdeError::invalid("The pending model settings cannot be changed now"));
|
||||
}
|
||||
s.model_choice += 1;
|
||||
s.pending_model = None;
|
||||
s.discovering_model = None;
|
||||
}
|
||||
let previous_levels = (switching_model && target.current.as_deref() != Some(value.as_str()))
|
||||
.then(|| s.task.config.extract_if(.., |o| config::is_per_model_option(o)).collect::<Vec<_>>());
|
||||
@@ -1116,33 +1189,19 @@ impl AgentManager {
|
||||
&& (previous.as_deref() != Some(value.as_str()) || !model_options)
|
||||
{
|
||||
let provider = task.state.lock().task.provider.clone();
|
||||
let mut levels = self.shared.paths.catalogs.levels(&self.shared, &provider, &value);
|
||||
let mut error = None;
|
||||
if levels.is_none()
|
||||
&& let Some(spec) = self.shared.spec(&provider)
|
||||
{
|
||||
let (catalog, flight) = self.shared.paths.catalogs.request(&self.shared, spec, false, Some(&value));
|
||||
error = catalog.error;
|
||||
if let Some(mut ready) = flight {
|
||||
while !*ready.borrow_and_update() {
|
||||
if ready.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
levels = self.shared.paths.catalogs.levels(&self.shared, &provider, &value);
|
||||
}
|
||||
let levels = self.shared.paths.catalogs.load_levels(&self.shared, &provider, &value).await;
|
||||
// Another selection may have superseded this discovery while it was waiting.
|
||||
if task.state.lock().model_choice != model_choice {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(levels) = levels else {
|
||||
restore_config_choice(&task, &target, &value, previous.as_deref(), model_choice, previous_levels);
|
||||
task.persist();
|
||||
task.emit_task();
|
||||
return Err(IdeError::unavailable(
|
||||
error.unwrap_or_else(|| "The agent did not report settings for the selected model".into()),
|
||||
));
|
||||
let levels = match levels {
|
||||
Ok(levels) => levels,
|
||||
Err(error) => {
|
||||
restore_config_choice(&task, &target, &value, previous.as_deref(), model_choice, previous_levels);
|
||||
task.persist();
|
||||
task.emit_task();
|
||||
return Err(IdeError::unavailable(error));
|
||||
}
|
||||
};
|
||||
let mut s = task.state.lock();
|
||||
if s.model_choice != model_choice {
|
||||
@@ -1265,6 +1324,7 @@ impl AgentManager {
|
||||
entries,
|
||||
wanted: s.wanted.clone(),
|
||||
pending_model: None,
|
||||
pending_switch: None,
|
||||
before: (*s.before).clone(),
|
||||
saved: s.saved.clone(),
|
||||
touched: s.touched.as_deref().cloned(),
|
||||
@@ -1310,6 +1370,8 @@ impl AgentManager {
|
||||
};
|
||||
s.session_id = None;
|
||||
s.pending_model = None;
|
||||
s.model_choice += 1;
|
||||
s.discovering_model = None;
|
||||
s.conversation = Conversation::default();
|
||||
// The fresh session reports its own usage; nothing carries over.
|
||||
s.task.usage = None;
|
||||
@@ -1908,6 +1970,24 @@ impl Task {
|
||||
let mut task = file.task;
|
||||
config::add_client_options(&task.provider, &mut task.config);
|
||||
let conversation = Conversation::restore(file.entries, file.subagents);
|
||||
let mut wanted = file.wanted;
|
||||
let pending_model = file.pending_switch.or_else(|| {
|
||||
let id = file.pending_model?;
|
||||
let index = wanted.iter().position(|choice| choice.id == id)?;
|
||||
let model = wanted.remove(index);
|
||||
if let Some(value) = task.model.clone() {
|
||||
wanted.push(AgentConfigChoice { id, value });
|
||||
}
|
||||
let catalog = shared.paths.catalogs.cached(&shared, &task.provider);
|
||||
let candidate = catalog
|
||||
.config
|
||||
.iter()
|
||||
.find(|o| o.category.as_deref() == Some("model"))
|
||||
.and_then(|o| o.values.iter().find(|v| v.value == model.value))
|
||||
.cloned();
|
||||
let options = catalog.thought_levels.get(&model.value).cloned().unwrap_or_default();
|
||||
Some(PendingModel { model, candidate, options, choices: Vec::new() })
|
||||
});
|
||||
Arc::new(Self {
|
||||
id: task.id.clone(),
|
||||
shared,
|
||||
@@ -1915,9 +1995,10 @@ impl Task {
|
||||
task,
|
||||
session_id: file.session_id,
|
||||
conversation,
|
||||
wanted: file.wanted,
|
||||
pending_model: file.pending_model,
|
||||
wanted,
|
||||
pending_model,
|
||||
model_choice: 0,
|
||||
discovering_model: None,
|
||||
before: Arc::new(file.before),
|
||||
saved: file.saved,
|
||||
touched: file.touched.map(Arc::new),
|
||||
@@ -2111,7 +2192,8 @@ impl Task {
|
||||
session_id: s.session_id.clone(),
|
||||
entries: s.conversation.main.entries.clone(),
|
||||
wanted: s.wanted.clone(),
|
||||
pending_model: s.pending_model.clone(),
|
||||
pending_model: None,
|
||||
pending_switch: s.pending_model.clone(),
|
||||
before: (*s.before).clone(),
|
||||
saved: s.saved.clone(),
|
||||
touched: s.touched.as_deref().cloned(),
|
||||
@@ -2264,13 +2346,18 @@ impl Task {
|
||||
let candidate = self.shared.paths.catalogs.fresh_model(&self.shared, &spec, &value).ok_or_else(|| {
|
||||
IdeError::invalid(format!("{} has no value {value} in the current model catalog", target.name))
|
||||
})?;
|
||||
let model_choice = {
|
||||
let selection = {
|
||||
let mut s = self.state.lock();
|
||||
if s.task.status == AgentStatus::Archived {
|
||||
return Err(IdeError::invalid("the task is archived"));
|
||||
}
|
||||
s.model_choice += 1;
|
||||
s.model_choice
|
||||
s.discovering_model = Some(s.model_choice);
|
||||
DiscoveringModel { state: &self.state, generation: s.model_choice }
|
||||
};
|
||||
let model_choice = selection.generation;
|
||||
// A prompt starting during the switch must not grab the peer we're about to recycle.
|
||||
let _connecting = self.connecting.lock().await;
|
||||
let connecting = self.connecting.lock().await;
|
||||
{
|
||||
let s = self.state.lock();
|
||||
if s.model_choice != model_choice {
|
||||
@@ -2293,16 +2380,16 @@ impl Task {
|
||||
Ok(None)
|
||||
};
|
||||
|
||||
let accepted = {
|
||||
let mut s = self.state.lock();
|
||||
if s.model_choice != model_choice {
|
||||
return Ok(());
|
||||
}
|
||||
if s.task.status == AgentStatus::Archived {
|
||||
return Err(IdeError::invalid("the task is archived"));
|
||||
}
|
||||
let pending = match result {
|
||||
Ok(Some(reply)) => {
|
||||
let refused = match result {
|
||||
Ok(Some(reply)) => {
|
||||
let (config, model) = {
|
||||
let mut s = self.state.lock();
|
||||
if s.model_choice != model_choice {
|
||||
return Ok(());
|
||||
}
|
||||
if s.task.status == AgentStatus::Archived {
|
||||
return Err(IdeError::invalid("the task is archived"));
|
||||
}
|
||||
if let Some(options) = reply.get("configOptions") {
|
||||
config::merge_update(&mut s.task.config, options);
|
||||
} else if s.task.model.as_deref() != Some(value.as_str()) {
|
||||
@@ -2321,36 +2408,66 @@ impl Task {
|
||||
model.current = Some(value.clone());
|
||||
s.task.model = Some(value.clone());
|
||||
sync_usage_context_size(&mut s.task);
|
||||
false
|
||||
}
|
||||
refused => {
|
||||
if s.session_id.is_none()
|
||||
|| live.as_ref().is_some_and(|l| !l.resumable)
|
||||
|| self.start_fresh.load(Ordering::Acquire)
|
||||
{
|
||||
return Err(IdeError::unavailable(
|
||||
refused.err().unwrap_or_else(|| "The agent does not support resuming this session.".into()),
|
||||
));
|
||||
}
|
||||
if s.busy || !s.background_tasks.is_empty() || !self.permissions.lock().is_empty() {
|
||||
return Err(IdeError::invalid("wait until the agent is done"));
|
||||
}
|
||||
true
|
||||
}
|
||||
};
|
||||
s.wanted.retain(|c| c.id != target.id);
|
||||
s.pending_model = pending.then(|| target.id.clone());
|
||||
s.wanted.push(AgentConfigChoice { id: target.id, value });
|
||||
(!pending).then(|| {
|
||||
let model = s.task.model.clone().filter(|_| s.task.config.iter().any(config::is_per_model_option));
|
||||
(s.task.config.clone(), model)
|
||||
})
|
||||
s.pending_model = None;
|
||||
s.wanted.retain(|c| c.id != target.id);
|
||||
s.wanted.push(AgentConfigChoice { id: target.id, value });
|
||||
let model = s.task.model.clone().filter(|_| s.task.config.iter().any(config::is_per_model_option));
|
||||
(s.task.config.clone(), model)
|
||||
};
|
||||
self.shared.remember(&provider, Some(&config), None, model.as_deref());
|
||||
self.persist();
|
||||
self.emit_task();
|
||||
return Ok(());
|
||||
}
|
||||
refused => refused,
|
||||
};
|
||||
if let Some((config, model)) = accepted {
|
||||
self.shared.remember(&provider, Some(&config), None, model.as_deref());
|
||||
} else {
|
||||
self.recycle_resumable_session();
|
||||
{
|
||||
let s = self.state.lock();
|
||||
if s.model_choice != model_choice {
|
||||
return Ok(());
|
||||
}
|
||||
if s.session_id.is_none()
|
||||
|| live.as_ref().is_some_and(|l| !l.resumable)
|
||||
|| self.start_fresh.load(Ordering::Acquire)
|
||||
{
|
||||
return Err(IdeError::unavailable(
|
||||
refused.err().unwrap_or_else(|| "The agent does not support resuming this session.".into()),
|
||||
));
|
||||
}
|
||||
if s.busy || !s.background_tasks.is_empty() || !self.permissions.lock().is_empty() {
|
||||
return Err(IdeError::invalid("wait until the agent is done"));
|
||||
}
|
||||
}
|
||||
// Catalog discovery never owns the chat session. Let a newer model choice supersede it.
|
||||
drop(connecting);
|
||||
let options = self.shared.paths.catalogs.load_levels(&self.shared, &provider, &value).await;
|
||||
let _connecting = self.connecting.lock().await;
|
||||
{
|
||||
let mut s = self.state.lock();
|
||||
if s.model_choice != model_choice {
|
||||
return Ok(());
|
||||
}
|
||||
if s.task.status == AgentStatus::Archived {
|
||||
return Err(IdeError::invalid("the task is archived"));
|
||||
}
|
||||
if s.busy || !s.background_tasks.is_empty() || !self.permissions.lock().is_empty() {
|
||||
return Err(IdeError::invalid("wait until the agent is done"));
|
||||
}
|
||||
let options = options.map_err(IdeError::unavailable)?;
|
||||
let choices = s
|
||||
.pending_model
|
||||
.take()
|
||||
.filter(|pending| pending.model.value == value)
|
||||
.map(|pending| pending.choices)
|
||||
.unwrap_or_default();
|
||||
s.pending_model = Some(PendingModel {
|
||||
model: AgentConfigChoice { id: target.id, value },
|
||||
candidate: Some(candidate),
|
||||
options,
|
||||
choices,
|
||||
});
|
||||
}
|
||||
self.recycle_resumable_session();
|
||||
self.persist();
|
||||
self.emit_task();
|
||||
Ok(())
|
||||
@@ -2612,10 +2729,9 @@ impl Task {
|
||||
}
|
||||
let (provider_id, cwd, saved, pending) = {
|
||||
let s = self.state.lock();
|
||||
let pending = s.pending_model().map(|choice| (choice.clone(), s.task.model.clone()));
|
||||
(s.task.provider.clone(), s.task.cwd.0.clone(), s.session_id.clone(), pending)
|
||||
(s.task.provider.clone(), s.task.cwd.0.clone(), s.session_id.clone(), s.pending_model.clone())
|
||||
};
|
||||
let pending_model = pending.as_ref().map(|(choice, _)| choice);
|
||||
let pending_model = pending.as_ref();
|
||||
let raw_spec = self.shared.spec(&provider_id).ok_or_else(|| format!("unknown agent {provider_id}"))?;
|
||||
let can_fallback =
|
||||
raw_spec.driver == provider::Driver::Copilot && !self.shared.uses_copilot_acp_fallback(&raw_spec);
|
||||
@@ -2641,22 +2757,12 @@ impl Task {
|
||||
let live = match opened {
|
||||
Ok(live) => live,
|
||||
Err(err) => {
|
||||
if let Some((choice, previous)) = pending {
|
||||
let mut s = self.state.lock();
|
||||
if s.pending_model() == Some(&choice) {
|
||||
s.pending_model = None;
|
||||
s.wanted.retain(|c| c.id != choice.id);
|
||||
if let Some(model) = s.task.config.iter_mut().find(|o| o.id == choice.id) {
|
||||
model.current.clone_from(&previous);
|
||||
}
|
||||
s.task.model.clone_from(&previous);
|
||||
if let Some(value) = previous {
|
||||
s.wanted.push(AgentConfigChoice { id: choice.id, value });
|
||||
}
|
||||
drop(s);
|
||||
self.persist();
|
||||
self.emit_task();
|
||||
}
|
||||
let mut s = self.state.lock();
|
||||
if pending.is_some() && s.pending_model == pending {
|
||||
s.pending_model = None;
|
||||
drop(s);
|
||||
self.persist();
|
||||
self.emit_task();
|
||||
}
|
||||
return Err(err.message);
|
||||
}
|
||||
@@ -2671,7 +2777,7 @@ impl Task {
|
||||
spec: provider::ProviderSpec,
|
||||
cwd: &Path,
|
||||
saved: Option<String>,
|
||||
pending_model: Option<&AgentConfigChoice>,
|
||||
pending_model: Option<&PendingModel>,
|
||||
) -> Result<Live, SessionOpenError> {
|
||||
let launch_cwd = cwd.to_path_buf();
|
||||
let mut spec = tokio::task::spawn_blocking(move || spec.resolve_for_launch(&launch_cwd))
|
||||
@@ -2697,14 +2803,15 @@ impl Task {
|
||||
}
|
||||
});
|
||||
// Speaks ACP v1 or v2, whichever the agent chooses; the task sees v1 either way.
|
||||
let client = acp::Dialect::wrap(Arc::new(Client { task: Arc::downgrade(self) }));
|
||||
let client = Client::new(self, saved.as_deref().filter(|_| pending_model.is_some()));
|
||||
let handler = acp::Dialect::wrap(client.clone());
|
||||
let raw_peer =
|
||||
(self.shared.connector)(&spec, cwd, client.clone(), on_stderr).map_err(|e| SessionOpenError {
|
||||
(self.shared.connector)(&spec, cwd, handler.clone(), on_stderr).map_err(|e| SessionOpenError {
|
||||
message: format!("Couldn't start {} (`{}`): {e}", spec.name, spec.command),
|
||||
allow_acp_fallback: !native::copilot::is_missing_binary_error(&e),
|
||||
})?;
|
||||
let mut stop_on_err = KillOnDrop(Some(raw_peer.clone()));
|
||||
let peer = acp::Conn::new(raw_peer, client);
|
||||
let peer = acp::Conn::new(raw_peer, handler);
|
||||
|
||||
let init = peer.initialize(&client_info()).await.map_err(|e| {
|
||||
let is_auth = matches!(&e, RpcError::Server { code, message, .. } if acp::auth_required(*code, message));
|
||||
@@ -2803,7 +2910,13 @@ impl Task {
|
||||
|
||||
// The session's settings, then the user's choices on top.
|
||||
// A session that reports none keeps the last known ones, but what is set is unknown.
|
||||
let reported = config::options_of(&result);
|
||||
let mut reported = config::options_of(&result);
|
||||
if let Some(update) = client.take_opening_config()
|
||||
&& let Some(config) = update.get("configOptions")
|
||||
&& reported.iter().all(|o| o.category.as_deref() != Some("model"))
|
||||
{
|
||||
config::merge_update(&mut reported, config);
|
||||
}
|
||||
let known = !reported.is_empty();
|
||||
if known {
|
||||
self.shared.remember(provider_id, Some(&reported), None, model_of(&reported).as_deref());
|
||||
@@ -2811,19 +2924,33 @@ impl Task {
|
||||
let (mut options, mut wanted) = {
|
||||
let s = self.state.lock();
|
||||
let options = if known { reported } else { s.task.config.clone() };
|
||||
let wanted = s.wanted.clone();
|
||||
let mut wanted = s.wanted.clone();
|
||||
if let Some(pending) = pending_model {
|
||||
pending.apply(&mut wanted, &s.task.config);
|
||||
}
|
||||
(options, wanted)
|
||||
};
|
||||
config::add_client_options(provider_id, &mut options);
|
||||
// Changing model can replace the effort / context-tier choices; validate those only afterwards.
|
||||
wanted.sort_by_key(|c| match options.iter().find(|o| o.id == c.id) {
|
||||
Some(o) if o.category.as_deref() == Some("model") => 0,
|
||||
Some(o) if config::is_per_model_option(o) => 2,
|
||||
_ => 1,
|
||||
wanted.sort_by_key(|c| {
|
||||
if let Some(pending) = pending_model {
|
||||
if c == &pending.model {
|
||||
return 0;
|
||||
}
|
||||
if pending.choices.contains(c) {
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
match options.iter().find(|o| o.id == c.id) {
|
||||
Some(o) if o.category.as_deref() == Some("model") => 0,
|
||||
Some(o) if config::is_per_model_option(o) => 2,
|
||||
_ => 1,
|
||||
}
|
||||
});
|
||||
let mut model_options = known;
|
||||
for choice in &wanted {
|
||||
let pending = pending_model == Some(choice);
|
||||
let pending = pending_model.is_some_and(|p| &p.model == choice);
|
||||
let explicit = pending_model.is_some_and(|p| p.choices.contains(choice));
|
||||
if pending
|
||||
&& !options.iter().any(|o| {
|
||||
o.id == choice.id
|
||||
@@ -2836,6 +2963,19 @@ impl Task {
|
||||
allow_acp_fallback: false,
|
||||
});
|
||||
}
|
||||
if explicit
|
||||
&& (!model_options
|
||||
|| !options.iter().any(|o| {
|
||||
o.id == choice.id
|
||||
&& config::is_per_model_option(o)
|
||||
&& o.values.iter().any(|v| v.value == choice.value)
|
||||
}))
|
||||
{
|
||||
return Err(SessionOpenError {
|
||||
message: format!("The selected model no longer offers {} value {}.", choice.id, choice.value),
|
||||
allow_acp_fallback: false,
|
||||
});
|
||||
}
|
||||
// An option the session doesn't offer yet is set once it does (`apply_late_choices`).
|
||||
let Some(o) = options.iter().find(|o| o.id == choice.id).cloned() else { continue };
|
||||
if o.id == config::CURSOR_APPROVAL {
|
||||
@@ -2847,7 +2987,7 @@ impl Task {
|
||||
if config::is_per_model_option(&o) && !model_options {
|
||||
continue;
|
||||
}
|
||||
let already = known && o.current.as_deref() == Some(choice.value.as_str());
|
||||
let already = (known || explicit) && o.current.as_deref() == Some(choice.value.as_str());
|
||||
if already || !o.values.iter().any(|v| v.value == choice.value) {
|
||||
continue;
|
||||
}
|
||||
@@ -2856,14 +2996,30 @@ impl Task {
|
||||
params["sessionId"] = json!(session_id);
|
||||
match peer.request::<_, Value>(method, params, None).await {
|
||||
Ok(reply) => {
|
||||
if pending && !config::accepts_model(&reply, &o.id, &choice.value) {
|
||||
let observed = client.take_opening_config();
|
||||
let report =
|
||||
if reply.get("configOptions").is_some() { &reply } else { observed.as_ref().unwrap_or(&reply) };
|
||||
if pending && !config::accepts_model(report, &o.id, &choice.value) {
|
||||
return Err(SessionOpenError {
|
||||
message: format!("The reloaded session did not switch to model {}.", choice.value),
|
||||
allow_acp_fallback: false,
|
||||
});
|
||||
}
|
||||
if let Some(fresh) = reply.get("configOptions") {
|
||||
if let Some(fresh) = report.get("configOptions") {
|
||||
config::merge_update(&mut options, fresh);
|
||||
if explicit
|
||||
&& !options.iter().any(|option| {
|
||||
option.id == choice.id && option.current.as_deref() == Some(choice.value.as_str())
|
||||
})
|
||||
{
|
||||
return Err(SessionOpenError {
|
||||
message: format!(
|
||||
"The selected model did not accept {} value {}.",
|
||||
choice.id, choice.value
|
||||
),
|
||||
allow_acp_fallback: false,
|
||||
});
|
||||
}
|
||||
model_options = true;
|
||||
} else if o.category.as_deref() == Some("model") {
|
||||
if o.current.as_deref() != Some(choice.value.as_str()) {
|
||||
@@ -2875,7 +3031,7 @@ impl Task {
|
||||
o.current = Some(choice.value.clone());
|
||||
}
|
||||
}
|
||||
Err(e) if pending => {
|
||||
Err(e) if pending || explicit => {
|
||||
return Err(SessionOpenError { message: self.describe(e), allow_acp_fallback: false });
|
||||
}
|
||||
Err(e) => tracing::warn!("agent task {}: setting {} to {} failed: {e}", self.id, o.id, choice.value),
|
||||
@@ -2885,6 +3041,16 @@ impl Task {
|
||||
self.shared.remember(provider_id, Some(&options), None, model.as_deref());
|
||||
{
|
||||
let mut s = self.state.lock();
|
||||
if let Some(pending) = pending_model {
|
||||
if s.pending_model.as_ref() != Some(pending) {
|
||||
return Err(SessionOpenError {
|
||||
message: "The pending model settings changed while reopening the session.".into(),
|
||||
allow_acp_fallback: false,
|
||||
});
|
||||
}
|
||||
let TaskState { task, wanted, .. } = &mut *s;
|
||||
pending.apply(wanted, &task.config);
|
||||
}
|
||||
s.task.model = model_of(&options).or(s.task.model.take());
|
||||
s.task.config = options;
|
||||
s.pending_model = None;
|
||||
@@ -2894,6 +3060,7 @@ impl Task {
|
||||
// The (re)started process runs nothing yet: whatever it reports next is all of it.
|
||||
s.background_tasks = Vec::new();
|
||||
s.task.background.clear();
|
||||
*client.opening_config.lock() = None;
|
||||
}
|
||||
self.persist();
|
||||
self.emit_task();
|
||||
@@ -3361,6 +3528,28 @@ impl Task {
|
||||
/// Answers the agent's requests on behalf of one task.
|
||||
struct Client {
|
||||
task: Weak<Task>,
|
||||
opening_config: Mutex<Option<OpeningConfig>>,
|
||||
}
|
||||
|
||||
/// Config replays belong to the candidate connection until its model and settings are confirmed.
|
||||
struct OpeningConfig {
|
||||
session: String,
|
||||
options: Option<Value>,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
fn new(task: &Arc<Task>, opening_session: Option<&str>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
task: Arc::downgrade(task),
|
||||
opening_config: Mutex::new(
|
||||
opening_session.map(|id| OpeningConfig { session: id.to_owned(), options: None }),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
fn take_opening_config(&self) -> Option<Value> {
|
||||
self.opening_config.lock().as_mut()?.options.take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler for Client {
|
||||
@@ -3377,6 +3566,14 @@ impl Handler for Client {
|
||||
if method == "session/update"
|
||||
&& let (Some(task), Some(update)) = (self.task.upgrade(), params.get("update"))
|
||||
{
|
||||
if update.get("sessionUpdate").and_then(Value::as_str) == Some("config_option_update")
|
||||
&& let Some(opening) = self.opening_config.lock().as_mut()
|
||||
{
|
||||
if params.get("sessionId").and_then(Value::as_str) == Some(opening.session.as_str()) {
|
||||
opening.options = Some(update.clone());
|
||||
}
|
||||
return;
|
||||
}
|
||||
task.on_update(params.get("sessionId").and_then(Value::as_str), update);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1935,7 +1935,7 @@ async fn startup_warnings_survive_replay_and_restart_without_failing_the_task()
|
||||
let handle = mgr.task(&task.id).unwrap();
|
||||
handle.replaying.store(true, Ordering::Release);
|
||||
handle.turn_started.store(false, Ordering::Release);
|
||||
let client = Client { task: Arc::downgrade(&handle) };
|
||||
let client = Client::new(&handle, None);
|
||||
client.on_notification(WARNING_NOTIFICATION, json!({"message": "Project MCP config is disabled until trusted."}));
|
||||
client.on_notification(WARNING_NOTIFICATION, json!({"message": " "}));
|
||||
client.on_notification(WARNING_NOTIFICATION, json!({}));
|
||||
|
||||
@@ -1194,11 +1194,31 @@ pub struct ArduinoSettings {
|
||||
}
|
||||
|
||||
/// Languages & Frameworks › Markdown. Like IDEA's, it is application-level only.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Type)]
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct MarkdownSettings {
|
||||
/// Preview (and notebook Markdown cell) font size in px; unset follows the editor font size.
|
||||
pub preview_font_size: Option<u32>,
|
||||
/// "Detect commands that can be run right from Markdown files": gutter run icons on
|
||||
/// shell code blocks and on command lines in code blocks.
|
||||
pub run_commands: bool,
|
||||
/// Where those commands run; unset asks when the project and the file's folder differ.
|
||||
pub command_directory: Option<MarkdownCommandDirectory>,
|
||||
}
|
||||
|
||||
impl Default for MarkdownSettings {
|
||||
fn default() -> Self {
|
||||
Self { preview_font_size: None, run_commands: true, command_directory: None }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum MarkdownCommandDirectory {
|
||||
/// The project's root folder.
|
||||
Project,
|
||||
/// The Markdown file's parent folder.
|
||||
File,
|
||||
}
|
||||
|
||||
impl LanguagesSettings {
|
||||
|
||||
@@ -170,6 +170,12 @@ pub struct CommitSettings {
|
||||
pub double_click_opens_diff: Option<bool>,
|
||||
/// Team conventions for Generate Commit Message with Agent, added to its prompt.
|
||||
pub agent_instructions: String,
|
||||
/// The coding agent Generate Commit Message with Agent uses; empty follows the latest task's agent.
|
||||
pub agent: String,
|
||||
/// That agent's model for commit messages; empty keeps the model the agent would use.
|
||||
pub agent_model: String,
|
||||
/// That model's thinking level for commit messages; empty keeps the agent's.
|
||||
pub agent_effort: String,
|
||||
}
|
||||
|
||||
impl Default for CommitSettings {
|
||||
@@ -192,6 +198,9 @@ impl Default for CommitSettings {
|
||||
disabled_plugin_checks: Vec::new(),
|
||||
double_click_opens_diff: None,
|
||||
agent_instructions: String::new(),
|
||||
agent: String::new(),
|
||||
agent_model: String::new(),
|
||||
agent_effort: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -902,9 +902,11 @@ fn always_update_snapshots_adds_dash_u_once() {
|
||||
apply_to_run_configs(&mut own, tmp.path(), &s);
|
||||
assert_eq!(own[0].args, ["--update-snapshots", "package"]);
|
||||
|
||||
// Settings apply to the stored configuration at each launch, so turning it off drops -U.
|
||||
s.languages.java.maven.update_snapshots = false;
|
||||
apply_to_run_configs(&mut configs, tmp.path(), &s);
|
||||
assert_eq!(configs[0].args, ["package"]);
|
||||
let mut stored = vec![run("maven", "./mvnw", &["package"])];
|
||||
apply_to_run_configs(&mut stored, tmp.path(), &s);
|
||||
assert_eq!(stored[0].args, ["package"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -847,6 +847,9 @@ mod tests {
|
||||
"disabledPluginChecks": [],
|
||||
"doubleClickOpensDiff": null,
|
||||
"agentInstructions": "",
|
||||
"agent": "",
|
||||
"agentModel": "",
|
||||
"agentEffort": "",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -29,7 +29,13 @@ pnpm perf -- --only typing --profile --unminified # CPU profile and top functi
|
||||
|
||||
Other options: `--warmup <n>` (default 1), `--headed`, `--no-build` (reuse the last build), `--channel <browser>` (default `chromium`, or `OPENIDEA_PERF_CHANNEL`), `--fail-on-regression`.
|
||||
|
||||
Scenarios (`perf/scenarios.ts`): `typing`, `typingLarge`, `typingAngular`, `completion`, `consoleOutput`, `terminalTui`, `keyRepeat`, `scroll`, `gitLog`, `gitLogScrollbar`, `resizePanels`, `startup`, `tabs`, `agentTabs`, `popups`, `toolWindows`, `projectTree`, `leaks`. Bundle size (`bundle.initialJsKb`, `bundle.initialCssKb`) is checked on every run.
|
||||
`--runs` must be a positive integer, `--warmup` a nonnegative integer, and `--cpu` a finite number of at least 1. Empty or unknown `--only` entries fail before building or launching a browser. Every declared selected scenario must return exactly one result; missing budgeted metrics and nonfinite metrics fail even when CPU throttling disables timing-budget enforcement. Unminified profiling builds intentionally omit bundle-size checks.
|
||||
|
||||
Every attempted repetition, including warmups, is retained in `target/perf/report.json` with status, metrics and failure reason. Markdown shows valid/planned and attempted counts. Missing or nonfinite repetitions are not filtered out of medians; invalid warmups also fail the scenario. Build, browser and cleanup failures replace the previous report with a fresh failure.
|
||||
|
||||
UI builds and individual trials have five-minute wall-clock deadlines independent of renderer animation frames. Browser startup is bounded to 30 seconds; context/HTTP cleanup to five seconds, with bounded forced browser termination. The owned Playwright browser server adds a local WebSocket command hop; in-page timing clocks are unchanged.
|
||||
|
||||
Scenarios (`perf/scenarios.ts`): `typing`, `typingLarge`, `typingAngular`, `completion`, `consoleOutput`, `terminalTui`, `keyRepeat`, `scroll`, `gitLog`, `gitLogScrollbar`, `resizePanels`, `startup`, `tabs`, `agentTabs`, `popups`, `toolWindows`, `projectTree`, `leaks`. Bundle size (`bundle.initialJsKb`, `bundle.initialCssKb`) is checked except with `--unminified`.
|
||||
|
||||
Latencies are measured from input to the start of the first frame that shows the result, so about 16.7 ms means "next frame".
|
||||
|
||||
@@ -42,9 +48,17 @@ Latencies are measured from input to the start of the first frame that shows the
|
||||
|
||||
A scenario in `perf/scenarios.ts` has an `id`, a `title` and a `run(h)` that drives the page through the harness (`perf/lib/harness.ts`: open files, set the cursor, type and measure, read editor state) and returns metrics. Add a budget for each metric you want enforced as `"<scenario>.<metric>"` in `budgets.json`.
|
||||
|
||||
### Current-operation checks
|
||||
|
||||
An already-visible result is not evidence that the measured action worked. Interaction waits require a current input and the requested state transition; file opening checks the active full path, typing checks the resulting text/caret and key-sample count, and scroll/completion navigation checks its destination.
|
||||
|
||||
Code-intelligence evals also wait for successful action completion. Rename correctness uses the current matching request only: multi-file previews validate proposed edits, while applied local renames validate the edited document. Incomplete answers are degraded results. Clean-file diagnostics require a fresh backend publication for that document, including an explicitly empty publication; a quiet empty UI alone cannot pass.
|
||||
|
||||
Diagnostics acknowledgements match the path, backend source and local opening token. Backend publications carry no document version, so a delayed same-path publication after reopening cannot be distinguished from a newly computed one.
|
||||
|
||||
## Backend: `pnpm perf:rust`
|
||||
|
||||
`perf/rust.ts` runs `cargo bench` for `ide-text`, `ide-language`, `ide-index`, `ide-search` and `ide-host`, reads the Criterion results from `target/criterion`, and checks each bench's mean (in µs).
|
||||
`perf/rust.ts` runs `cargo bench` for `ide-text`, `ide-language`, `ide-index`, `ide-search`, `ide-host` and `ide-heap`, reads the Criterion results, and checks each bench's mean (in µs).
|
||||
|
||||
```sh
|
||||
pnpm perf:rust # run the benches, then check
|
||||
@@ -62,6 +76,22 @@ The benches live in `crates/<crate>/benches/`:
|
||||
| `search_5k/…` | `ide-search` | literal, regex and dense-match search over 5k files |
|
||||
| `plugins/…` | `ide-host` | loading 50 plugins, cold activation, syntax highlights of a 2k-line file |
|
||||
|
||||
Expected IDs and package ownership are declared independently of collected artifacts in `perf/lib/rust.ts`. A selected package must produce every expected ID, including observe-only measurements. `--package` scopes both collection and required budgets, also with `--no-run`; unknown packages fail before Cargo starts.
|
||||
|
||||
Six IDs are observe-only (required finite measurements, without an absolute budget): `syntax_2mb/outline_uncached`, `syntax_2mb/folding_uncached`, `index_100k/find_files/zzzz`, `index_100k/find_symbols/renderView`, `startup_20k/reopen_wait_idle`, and `search_excluded_subtrees/5k_dependencies`. New IDs need an explicit budget or observe-only classification.
|
||||
|
||||
Normal runs write isolated `<Criterion root>/runs/run-*/criterion` artifacts, a `run.json` manifest and Cargo logs. `CRITERION_HOME` and `CARGO_TARGET_DIR` are passed consistently to Cargo. Both artifact files must fall within the recorded run window; future timestamps fail. `--no-run` inspects the newest recorded run covering the selected packages, retaining original artifact dates and any recorded failure; pre-manifest output is explicitly labelled legacy inspection, not current execution.
|
||||
|
||||
Cargo has a 60-minute deadline including cold compilation (`PERF_RUST_TIMEOUT_MS` overrides milliseconds), followed by bounded process-tree termination. Cargo, artifact and manifest failures write fresh `report-rust.json` and Markdown with partial evidence.
|
||||
|
||||
## Code intelligence: `pnpm eval`
|
||||
|
||||
Use `pnpm eval --list` for scenario IDs and supported conditions. `--only` accepts comma-separated IDs or dot-prefix groups; every entry must match executable work. `--condition` accepts only `warm` or `cold`, and a selection incompatible with that condition is rejected before building or launching. Repetition counts follow the UI runner's integer requirements.
|
||||
|
||||
The selected scenario/condition plan defines the required outcomes independently of collected results. Missing, duplicate or unexpected outcomes fail; an empty workload cannot pass. Required timing and correctness metrics must be present and finite, rather than silently skipped by the judge.
|
||||
|
||||
Raw attempts, warmups, results and partial timeout timings remain in `target/eval/report.json`; invalid trials cannot disappear into an otherwise valid median. Startup/build failures also produce a fresh failing report. Session startup, fixture setup and each trial have five-minute wall-clock bounds; renderer measurements additionally have a Node deadline one second beyond their configured timeout. Server initialization is limited to 30 seconds, individual stdio requests to five minutes, release compilation to 60 minutes. Cleanup errors fail the run.
|
||||
|
||||
## Budgets and baselines
|
||||
|
||||
[`perf/budgets.json`](https://github.com/l0ng-ai/openidea/blob/main/perf/budgets.json) holds upper bounds (lower is better):
|
||||
@@ -92,4 +122,6 @@ python perf/compare/bench.py ~/src/some-project --runs 5 --wipe-caches # cold
|
||||
|
||||
Each run launches the app on the project, waits until it is ready (the process tree's CPU stays low after the window and the log markers appear), idles, closes it, and records time to window and to ready, peak and idle memory summed over every process the app started (webview, language servers, JVMs), CPU time, exit time and cache size. Apps run interleaved after warmup runs; output goes to `results/<time>-<project>/` (`report.md`, `results.json`).
|
||||
|
||||
Reports retain failed trials and warmups, display per-cell valid/attempted counts, and distinguish unsupported metrics from missing required values. Startup timeouts, crashes, forced shutdowns and interruptions fail the command. Quit/cache commands have explicit deadlines; cleanup runs even after observation failures. See the comparison README for the unchanged CPU-settled readiness proxy and daemon exclusions.
|
||||
|
||||
Use release builds only; the script refuses to run while another instance of the same executable is running. See [`perf/compare/README.md`](https://github.com/l0ng-ai/openidea/blob/main/perf/compare/README.md) for fair-run rules and platform notes.
|
||||
@@ -78,7 +78,7 @@ The task tab shows the conversation: your messages, the agent's replies, its rea
|
||||
|
||||
On your own messages you can **Edit** (the messages after it are replaced; files already changed stay as they are) or **Retract Message**. On replies you can **Copy Markdown**, **Quote Reply**, **Regenerate Response** and **Branch Conversation from Here**, which starts a new task from that point.
|
||||
|
||||
In an existing chat, open **Model › Refresh models** to discover newly available models without sending a message or changing the selection. If the agent can't switch in place, the choice stays pending until your next message reopens the same session. If the session can't be reopened or doesn't accept the model, OpenIDEA restores the previous selection and reports the error without sending the message.
|
||||
In an existing chat, open **Model › Refresh models** to discover newly available models without sending a message or changing the selection. If the agent can't switch in place, the choice stays pending. You can choose an effort level as soon as the provider reports the new model's options. On your next message, OpenIDEA reopens the same session and applies the model and any effort you selected before sending. If the session can't be reopened or rejects either choice, OpenIDEA restores the previous selection and reports the error without sending the message.
|
||||
|
||||
**Focus Agent Input** defaults to `⌘L` on macOS and `Ctrl+L` on Windows / Linux in every built-in keymap. When focus is inside your writable task conversation, in either an editor tab or the tool window, it returns focus to the message input without changing your draft, caret position or selection—even when pressed repeatedly. It does not send a message or jump through message history.
|
||||
|
||||
|
||||
@@ -29,7 +29,13 @@ pnpm perf -- --only typing --profile --unminified # CPU profile 和耗时最
|
||||
|
||||
其他选项:`--warmup <n>`(默认 1)、`--headed`、`--no-build`(复用上次构建)、`--channel <browser>`(默认 `chromium`,或 `OPENIDEA_PERF_CHANNEL`)、`--fail-on-regression`。
|
||||
|
||||
场景(`perf/scenarios.ts`):`typing`、`typingLarge`、`typingAngular`、`completion`、`consoleOutput`、`terminalTui`、`keyRepeat`、`scroll`、`gitLog`、`gitLogScrollbar`、`resizePanels`、`startup`、`tabs`、`agentTabs`、`popups`、`toolWindows`、`projectTree`、`leaks`。每次运行都会检查包体积(`bundle.initialJsKb`、`bundle.initialCssKb`)。
|
||||
`--runs` 必须为正整数,`--warmup` 为非负整数,`--cpu` 为不小于 1 的有限数值。`--only` 中的空项或未知场景会在构建或启动浏览器前报错。每个选中场景必须恰好返回一份结果;预算指标缺失或指标非有限值都会失败,即使 CPU 降速关闭了耗时预算检查。未压缩的性能分析构建不检查包体积。
|
||||
|
||||
`target/perf/report.json` 保留每次尝试(包括预热)的状态、指标和失败原因,Markdown 显示有效/计划次数及实际尝试次数。缺失或非有限值不会从中位数中被过滤,预热无效也会导致场景失败。构建、浏览器和清理失败都会用新的失败报告替换旧报告。
|
||||
|
||||
UI 构建和每次尝试都有独立于渲染器动画帧的五分钟墙钟期限。浏览器启动限时 30 秒,上下文及 HTTP 清理限时五秒,必要时有界强制终止浏览器。独立管理的 Playwright 浏览器服务增加一次本地 WebSocket 命令跳转,页面内计时基准不变。
|
||||
|
||||
场景(`perf/scenarios.ts`):`typing`、`typingLarge`、`typingAngular`、`completion`、`consoleOutput`、`terminalTui`、`keyRepeat`、`scroll`、`gitLog`、`gitLogScrollbar`、`resizePanels`、`startup`、`tabs`、`agentTabs`、`popups`、`toolWindows`、`projectTree`、`leaks`。除 `--unminified` 外,每次运行都会检查包体积(`bundle.initialJsKb`、`bundle.initialCssKb`)。
|
||||
|
||||
延迟是从输入到显示结果的第一帧开始的时间,因此约 16.7 ms 就表示“下一帧”。
|
||||
|
||||
@@ -42,9 +48,17 @@ pnpm perf -- --only typing --profile --unminified # CPU profile 和耗时最
|
||||
|
||||
`perf/scenarios.ts` 中的场景包含 `id`、`title` 和 `run(h)`:通过 harness(`perf/lib/harness.ts`:打开文件、设置光标、输入并测量、读取编辑器状态)驱动页面并返回指标。需要强制检查的指标,在 `budgets.json` 中以 `"<scenario>.<metric>"` 加上预算。
|
||||
|
||||
### 当前操作的正确性检查
|
||||
|
||||
已显示的结果不能证明被测操作成功。交互等待必须观察到当前输入及目标状态变化;打开文件会核对活动编辑器的完整路径,输入会核对最终文本、光标和按键样本数,滚动及补全导航会核对目标位置。
|
||||
|
||||
代码智能 eval 也会等待动作成功结束。重命名只使用当前匹配请求的结果:跨文件预览验证拟议的修改,已应用的局部重命名验证实际文档。不完整的答复按降级结果处理。干净文件的诊断必须收到针对该文档的新后端发布,包括明确的空发布;仅有安静且为空的 UI 不能通过。
|
||||
|
||||
诊断确认匹配路径、后端来源和本地打开令牌。后端发布不携带文档版本,因此重新打开后延迟到达的同路径发布,无法与新计算的发布区分。
|
||||
|
||||
## 后端:`pnpm perf:rust`
|
||||
|
||||
`perf/rust.ts` 对 `ide-text`、`ide-language`、`ide-index`、`ide-search` 和 `ide-host` 运行 `cargo bench`,从 `target/criterion` 读取 Criterion 结果,检查每个基准的平均值(µs)。
|
||||
`perf/rust.ts` 对 `ide-text`、`ide-language`、`ide-index`、`ide-search`、`ide-host` 和 `ide-heap` 运行 `cargo bench`,读取 Criterion 结果,检查每个基准的平均值(µs)。
|
||||
|
||||
```sh
|
||||
pnpm perf:rust # 跑基准并检查
|
||||
@@ -62,6 +76,22 @@ pnpm perf:rust -- --package ide-text --save-baseline
|
||||
| `search_5k/…` | `ide-search` | 在 5 千个文件中做字面量、正则和密集匹配搜索 |
|
||||
| `plugins/…` | `ide-host` | 加载 50 个插件、冷激活、2 千行文件的语法高亮 |
|
||||
|
||||
预期 ID 及其所属包在 `perf/lib/rust.ts` 中独立于收集到的产物声明。选中的包必须提供全部预期 ID,包括仅观察的指标。`--package` 同时限定收集范围和必需预算,使用 `--no-run` 时也如此;未知包会在启动 Cargo 前报错。
|
||||
|
||||
六个 ID 仅观察(必须提供有限测量值,但没有绝对预算):`syntax_2mb/outline_uncached`、`syntax_2mb/folding_uncached`、`index_100k/find_files/zzzz`、`index_100k/find_symbols/renderView`、`startup_20k/reopen_wait_idle`、`search_excluded_subtrees/5k_dependencies`。新增 ID 必须明确设置预算或归类为仅观察。
|
||||
|
||||
正常执行将产物隔离到 `<Criterion 根目录>/runs/run-*/criterion`,并保留 `run.json` 清单和 Cargo 日志。`CRITERION_HOME`、`CARGO_TARGET_DIR` 与 Cargo 保持一致。两个产物文件都必须处于记录的运行时间窗口内,未来时间戳会失败。`--no-run` 检查覆盖所选包的最新记录,保留原始产物时间及已有失败;清单引入前的产物明确标为旧产物检查,而非本次执行。
|
||||
|
||||
Cargo 包括冷编译在内默认限时 60 分钟(`PERF_RUST_TIMEOUT_MS` 可按毫秒覆盖),随后有界终止所属进程树。Cargo、产物和清单失败都会输出新的 `report-rust.json` 和 Markdown,并保留部分证据。
|
||||
|
||||
## 代码智能:`pnpm eval`
|
||||
|
||||
用 `pnpm eval --list` 查看场景 ID 及支持的条件。`--only` 接受逗号分隔的 ID 或点号前缀分组;每项都必须匹配可执行的工作。`--condition` 只接受 `warm` 或 `cold`,与该条件不兼容的选择会在构建或启动前被拒绝。重复次数遵循 UI 运行器的整数要求。
|
||||
|
||||
选中的场景/条件计划独立于收集结果定义必需的输出。缺失、重复或非预期输出都会失败,空工作集不能通过。必需的耗时和正确性指标必须存在且为有限数值,判定器不再静默跳过它们。
|
||||
|
||||
`target/eval/report.json` 保留原始尝试、预热、结果及超时前的部分计时,无效尝试不能消失在看似有效的中位数中。启动或构建失败也会生成新的失败报告。会话启动、夹具准备和每次尝试均限时五分钟;渲染器测量另有 Node 期限,比场景配置超时多一秒。服务初始化限时 30 秒,单个 stdio 请求限时五分钟,release 编译限时 60 分钟。清理错误会导致整次运行失败。
|
||||
|
||||
## 预算与基线
|
||||
|
||||
[`perf/budgets.json`](https://github.com/l0ng-ai/openidea/blob/main/perf/budgets.json) 保存上限(越低越好):
|
||||
@@ -92,4 +122,6 @@ python perf/compare/bench.py ~/src/some-project --runs 5 --wipe-caches # 冷
|
||||
|
||||
每次运行都会用该应用打开项目,等待就绪(窗口出现、日志标记出现后,进程树的 CPU 保持在低位),空闲一段时间后关闭,并记录:到窗口出现和到就绪的时间、应用启动的所有进程(webview、语言服务器、JVM)合计的峰值与空闲内存、CPU 时间、退出耗时和缓存大小。各应用在预热后交替运行,结果输出到 `results/<time>-<project>/`(`report.md`、`results.json`)。
|
||||
|
||||
报告保留失败尝试和预热,逐单元格显示有效/尝试次数,并区分不支持的指标与缺失的必需指标。启动超时、崩溃、强制关闭和中断都会使命令失败。退出及清缓存命令有明确期限,观察失败后仍会清理。CPU 平静就绪代理和守护进程排除策略保持不变,详见对比工具 README。
|
||||
|
||||
只使用 release 构建;同一可执行文件已有实例在运行时脚本会拒绝执行。公平测量的规则和各平台说明见 [`perf/compare/README.md`](https://github.com/l0ng-ai/openidea/blob/main/perf/compare/README.md)。
|
||||
@@ -78,7 +78,7 @@ Codex 有自己的项目信任设置。如果它禁用了当前项目的 `.codex
|
||||
|
||||
在你自己的消息上可以 **编辑**(其后的消息会被替换,已经改动的文件保持不变)或 **撤回消息**。在回复上可以 **复制 Markdown**、**引用回复**、**重新生成回复**,以及 **从此处创建分支对话**,从这一点开始一个新任务。
|
||||
|
||||
在已有对话中,打开**模型 › 刷新模型**即可查找新上线的模型,不会发送消息或更改当前选择。如果 agent 无法直接切换,所选模型会在下一条消息重新打开原会话时才应用。如果无法恢复会话,或原会话不接受所选模型,OpenIDEA 会还原之前的选择并提示错误,不会发送该消息。
|
||||
在已有对话中,打开**模型 › 刷新模型**即可查找新上线的模型,不会发送消息或更改当前选择。如果 agent 无法直接切换,所选模型会保持待应用状态。提供商返回新模型支持的思考强度后,你就可以选择强度。发送下一条消息时,OpenIDEA 会先恢复原会话并应用所选模型和思考强度,再发送消息。如果无法恢复会话,或原会话不接受其中任何一项,OpenIDEA 会还原之前的选择并提示错误,不会发送该消息。
|
||||
|
||||
所有内置按键映射中,**聚焦 Agent 输入框** 的默认快捷键都是 macOS 上的 `⌘L`、Windows / Linux 上的 `Ctrl+L`。焦点位于你自己的可输入任务对话中时,无论对话显示在编辑器标签页还是工具窗口中,它都会将焦点移回消息输入框,保留草稿、光标位置和选区;重复按下也不会改变这些内容。它不会发送消息,也不会跳转消息历史。
|
||||
|
||||
|
||||
@@ -108,15 +108,23 @@ async function seed(
|
||||
surface: "new" | "task" = "task",
|
||||
model = "atlas",
|
||||
status: AgentTask["status"] = "ready",
|
||||
catalogOnly = false,
|
||||
) {
|
||||
await stubDesktop(page);
|
||||
await page.evaluate(
|
||||
async ({ placement, surface, provider, firstTask, secondTask, subagent, model, status, modelId }) => {
|
||||
async ({ placement, surface, provider, firstTask, secondTask, subagent, model, status, modelId, catalogOnly }) => {
|
||||
// In-memory sample tasks belong to Vite's browser realm. Static Node imports create a different store and cannot seed the workbench.
|
||||
const storePath = "/src/features/agents/store.ts";
|
||||
const store = await import(storePath);
|
||||
const initialCatalog = {
|
||||
...provider,
|
||||
config: provider.config.map((entry) => ({
|
||||
...entry,
|
||||
values: entry.values.filter((value) => value.value !== "swift"),
|
||||
})),
|
||||
};
|
||||
const configFor = (model: string) => [
|
||||
{ ...provider.config[0]!, current: model },
|
||||
{ ...(catalogOnly && model !== "swift" ? initialCatalog : provider).config[0]!, current: model },
|
||||
...(provider.thoughtLevels?.[model] ?? []),
|
||||
];
|
||||
const tasks = new Map(
|
||||
@@ -150,10 +158,10 @@ async function seed(
|
||||
channel = onEvent;
|
||||
return [...tasks.values()];
|
||||
};
|
||||
handlers.agent_providers = () => [provider];
|
||||
handlers.agent_refresh_models = () => ({
|
||||
handlers.agent_providers = () => [catalogOnly ? initialCatalog : provider];
|
||||
handlers.agent_refresh_models = ({ force }) => ({
|
||||
provider: provider.id,
|
||||
config: provider.config,
|
||||
config: (catalogOnly && !force ? initialCatalog : provider).config,
|
||||
thoughtLevels: provider.thoughtLevels,
|
||||
refreshing: false,
|
||||
refreshedAtMs: Date.now(),
|
||||
@@ -170,9 +178,15 @@ async function seed(
|
||||
handlers.vcs_branches = () => [];
|
||||
handlers.agent_set_config = async ({ task: id, option, value }) => {
|
||||
const previous = tasks.get(String(id))!;
|
||||
const pendingModel = catalogOnly && option === modelId && value === "swift";
|
||||
if (pendingModel) {
|
||||
const { promise, resolve } = Promise.withResolvers<void>();
|
||||
document.addEventListener("test:chat-model-settings-ready", () => resolve(), { once: true });
|
||||
await promise;
|
||||
}
|
||||
const updated =
|
||||
option === modelId
|
||||
? { ...previous, model: String(value), config: configFor(String(value)) }
|
||||
? { ...previous, model: pendingModel ? previous.model : String(value), config: configFor(String(value)) }
|
||||
: {
|
||||
...previous,
|
||||
config: previous.config.map((entry) =>
|
||||
@@ -204,7 +218,7 @@ async function seed(
|
||||
if (surface === "new") store.openHome();
|
||||
else store.openTask(firstTask.id);
|
||||
},
|
||||
{ placement, surface, provider, firstTask, secondTask, subagent, model, status, modelId },
|
||||
{ placement, surface, provider, firstTask, secondTask, subagent, model, status, modelId, catalogOnly },
|
||||
);
|
||||
await expect(box(page)).toBeVisible();
|
||||
await page.evaluate(() => document.fonts.ready);
|
||||
@@ -602,3 +616,33 @@ test("New Task discovers uncached model effort before the first prompt", async (
|
||||
await page.screenshot({ path: testInfo.outputPath("uncached-model-effort.png") });
|
||||
}
|
||||
});
|
||||
|
||||
test("an existing chat can choose effort for a refreshed model before sending", async ({ workbench: page }) => {
|
||||
await seed(page, "editor", "task", "atlas", "ready", true);
|
||||
await box(page).fill(draft);
|
||||
await expectEffort(page, "Gentle");
|
||||
await modelButton(page).click();
|
||||
const swiftModel = page.getByRole("option", { name: /^Swift\b/ });
|
||||
await expect(swiftModel).toHaveCount(0);
|
||||
await page.getByRole("button", { name: "Refresh models", exact: true }).click();
|
||||
await swiftModel.click();
|
||||
|
||||
const loading = page.getByRole("status").filter({ hasText: "Loading model settings…" });
|
||||
const send = page.getByRole("button", { name: "Send", exact: true });
|
||||
await expect(loading).toBeVisible();
|
||||
await expect(send).toBeDisabled();
|
||||
await expect(modelButton(page)).toHaveAttribute("title", /^Model: Atlas\b/);
|
||||
await page.evaluate(() => document.dispatchEvent(new Event("test:chat-model-settings-ready")));
|
||||
await expect(loading).toHaveCount(0);
|
||||
await expect(modelButton(page)).toHaveAttribute("title", /^Model: Swift\b/);
|
||||
await expectEffort(page, "Quick");
|
||||
await modelButton(page).click();
|
||||
const efforts = page.getByRole("radiogroup", { name: "Effort Level", exact: true });
|
||||
await expect(efforts.getByRole("radio", { name: "Gentle", exact: true })).toHaveCount(0);
|
||||
await efforts.getByRole("radio", { name: "Thorough", exact: true }).click();
|
||||
await page.keyboard.press("Escape");
|
||||
await expectEffort(page, "Thorough");
|
||||
await expect(send).toBeEnabled();
|
||||
await expect(box(page)).toHaveText(draft);
|
||||
await expectNoSubmission(page);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import type { Page } from "@playwright/test";
|
||||
import { expect, test } from "./fixtures";
|
||||
|
||||
async function showBuild(page: Page) {
|
||||
await page.evaluate(async () => {
|
||||
const storePath = "/src/features/build/store.ts";
|
||||
const layoutPath = "/src/workbench/toolwindows/store.ts";
|
||||
const { useBuildRun } = await import(/* @vite-ignore */ storePath);
|
||||
const { showToolWindow } = await import(/* @vite-ignore */ layoutPath);
|
||||
useBuildRun.setState({
|
||||
title: "Build Project",
|
||||
system: "typescript",
|
||||
status: "failed",
|
||||
durationMs: 1200,
|
||||
selected: null,
|
||||
lines: [
|
||||
{ kind: "out", text: "Compiling TypeScript" },
|
||||
{ kind: "err", text: "Expected a number" },
|
||||
],
|
||||
messages: [{ severity: "error", text: "Expected a number", path: null, range: null, code: null }],
|
||||
});
|
||||
showToolWindow("build", true);
|
||||
});
|
||||
const build = page.getByRole("region", { name: "Build", exact: true });
|
||||
await expect(build.getByRole("log", { name: "Build output" })).toBeVisible();
|
||||
return build;
|
||||
}
|
||||
|
||||
test("Build gives about two thirds of the initial split to output", async ({ workbench: page }) => {
|
||||
const build = await showBuild(page);
|
||||
const tree = build.locator("[data-panel][id=build-tree]");
|
||||
const console = build.locator("[data-panel][id=build-console]");
|
||||
const treeWidth = (await tree.boundingBox())!.width;
|
||||
const consoleWidth = (await console.boundingBox())!.width;
|
||||
expect(treeWidth / (treeWidth + consoleWidth)).toBeCloseTo(0.33, 2);
|
||||
});
|
||||
|
||||
test("Build output and diagnostics respond to a console font change", async ({ workbench: page }) => {
|
||||
const build = await showBuild(page);
|
||||
const log = build.getByRole("log", { name: "Build output" });
|
||||
const rowGap = async () => {
|
||||
const first = (await log.getByText("Compiling TypeScript", { exact: true }).boundingBox())!;
|
||||
const second = (await log.getByText("Expected a number", { exact: true }).boundingBox())!;
|
||||
return second.y - first.y;
|
||||
};
|
||||
await expect.poll(rowGap).toBe(20);
|
||||
await page.evaluate(async () => {
|
||||
const path = "/src/platform/project.ts";
|
||||
const { useProject } = await import(/* @vite-ignore */ path);
|
||||
const { settings } = useProject.getState();
|
||||
useProject.setState({
|
||||
settings: {
|
||||
...settings,
|
||||
terminal: {
|
||||
...settings?.terminal,
|
||||
useEditorFont: false,
|
||||
fontFamily: "Fira Code",
|
||||
fallbackFontFamily: "Maple Mono NF CN",
|
||||
fontSize: 18,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
await expect(log).toHaveCSS("font-size", "18px");
|
||||
await expect(log).toHaveCSS("font-family", /"Fira Code".*"Maple Mono NF CN"/);
|
||||
await expect.poll(rowGap).toBe(27);
|
||||
await build.getByRole("treeitem", { name: "Expected a number", exact: true }).click();
|
||||
const diagnostic = build.getByRole("document", { name: "Build message" });
|
||||
await expect(diagnostic).toContainText("Expected a number");
|
||||
await expect(diagnostic).toHaveCSS("font-size", "18px");
|
||||
await expect(diagnostic).toHaveCSS("line-height", "27px");
|
||||
});
|
||||
@@ -98,3 +98,59 @@ test("commit message history opens above the message without an empty icon colum
|
||||
expect(Math.abs(menuBox!.x - messageBox!.x)).toBeLessThan(2);
|
||||
await expect(row.locator("span")).toHaveCount(1);
|
||||
});
|
||||
|
||||
test("the Commit changes toolbar fits a narrow side pane", async ({ workbench: page }) => {
|
||||
await stubRepository(page);
|
||||
await page.keyboard.press("Meta+k");
|
||||
await page.evaluate(async () => {
|
||||
const path = "/src/workbench/toolwindows/store.ts";
|
||||
const layout = await import(/* @vite-ignore */ path);
|
||||
layout.moveToolWindow("commit", "left");
|
||||
layout.setToolWindowSize("left", 164);
|
||||
});
|
||||
const panel = page.getByRole("region", { name: "Commit", exact: true });
|
||||
const toolbar = panel.getByRole("toolbar", { name: "Changes" });
|
||||
await expect(toolbar).toBeVisible();
|
||||
const clipped = await toolbar.evaluate((bar) => {
|
||||
const bounds = bar.getBoundingClientRect();
|
||||
return [...bar.querySelectorAll("button")].flatMap((button) => {
|
||||
const box = button.getBoundingClientRect();
|
||||
return box.right > bounds.right + 1 || box.bottom > bounds.bottom + 1 ? [button.ariaLabel] : [];
|
||||
});
|
||||
});
|
||||
expect(clipped).toEqual([]);
|
||||
const push = panel.getByRole("button", { name: "Commit and Push…" });
|
||||
const overflow = await push.locator("span").evaluate((label) => label.scrollWidth - label.clientWidth);
|
||||
expect(overflow).toBeLessThanOrEqual(1);
|
||||
await toolbar.getByRole("button", { name: "Collapse All" }).click();
|
||||
await expect(panel.getByRole("checkbox", { name: "Include build.gradle.kts in commit" })).toBeHidden();
|
||||
});
|
||||
|
||||
test("the commit message uses the configured editor typography", async ({ workbench: page }) => {
|
||||
await stubRepository(page);
|
||||
await page.keyboard.press("Meta+k");
|
||||
const message = page.getByRole("textbox", { name: "Commit message" });
|
||||
await expect(message).toBeFocused();
|
||||
await page.evaluate(async () => {
|
||||
const path = "/src/platform/project.ts";
|
||||
const { useProject } = await import(/* @vite-ignore */ path);
|
||||
const settings = useProject.getState().settings;
|
||||
useProject.setState({
|
||||
settings: {
|
||||
...settings,
|
||||
editor: {
|
||||
...settings?.editor,
|
||||
fontFamily: "Fira Code",
|
||||
fallbackFont: "Maple Mono NF CN",
|
||||
fontSize: 17,
|
||||
lineHeight: 28,
|
||||
fontWeight: 500,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
await expect(message).toHaveCSS("font-size", "17px");
|
||||
await expect(message).toHaveCSS("line-height", "28px");
|
||||
await expect(message).toHaveCSS("font-weight", "500");
|
||||
await expect(message).toHaveCSS("font-family", /"Fira Code".*"Maple Mono NF CN"/);
|
||||
});
|
||||
@@ -30,7 +30,7 @@ for (const kind of ["run", "debug", "services-run", "services-debug", "stacktrac
|
||||
component = SessionPanel;
|
||||
props = {
|
||||
session: run.useRun.getState().sessions.find((s: { id: string }) => s.id === id),
|
||||
...(kind === "services-run" ? { onRerun: null, onDebug: null } : {}),
|
||||
showLifecycle: kind === "run",
|
||||
};
|
||||
} else if (kind === "debug" || kind === "services-debug") {
|
||||
const statePath = "/src/features/debugger/state.ts";
|
||||
|
||||
@@ -1,5 +1,29 @@
|
||||
import { expect, test } from "./fixtures";
|
||||
|
||||
test("data source tabs share a stable layout and scroll within a short window", async ({ workbench: page }) => {
|
||||
await page.setViewportSize({ width: 560, height: 480 });
|
||||
await page.evaluate(async () => {
|
||||
const path = "/src/features/database/store.ts";
|
||||
const { openSourceDialog } = await import(/* @vite-ignore */ path);
|
||||
openSourceDialog(null, { name: "Local development", driver: "postgres", host: "localhost" });
|
||||
});
|
||||
|
||||
const dialog = page.getByRole("dialog", { name: "New Data Source" });
|
||||
const panel = dialog.getByRole("tabpanel");
|
||||
const dialogSize = () => dialog.evaluate((element) => [element.offsetWidth, element.offsetHeight]);
|
||||
await expect(panel.getByRole("textbox", { name: "Host:" })).toBeVisible();
|
||||
const size = await dialogSize();
|
||||
await dialog.getByRole("tab", { name: "SSH/SSL" }).click();
|
||||
await expect(dialog.getByRole("tab", { name: "SSH/SSL" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(await dialogSize()).toEqual(size);
|
||||
|
||||
await dialog.getByRole("tab", { name: "General" }).click();
|
||||
const canScroll = await panel.evaluate((element) => element.scrollHeight > element.clientHeight);
|
||||
expect(canScroll).toBe(true);
|
||||
await panel.getByRole("textbox", { name: "URL:" }).scrollIntoViewIfNeeded();
|
||||
await expect(dialog.getByRole("button", { name: "Cancel" })).toBeInViewport();
|
||||
});
|
||||
|
||||
for (const kind of ["password", "sshSecret", "params"] as const) {
|
||||
test(`database ${kind} dialog keeps its fields inside the window`, async ({ workbench: page }) => {
|
||||
await page.evaluate(async (kind) => {
|
||||
|
||||
@@ -1,6 +1,40 @@
|
||||
import type { Page } from "@playwright/test";
|
||||
import { expect, openFromTree, test } from "./fixtures";
|
||||
|
||||
test("managed console tabs identify their data source and ordinary SQL tabs keep the file name", async ({
|
||||
workbench: page,
|
||||
}) => {
|
||||
await page.evaluate(async () => {
|
||||
const database = "/src/features/database/store.ts";
|
||||
const workbench = "/src/workbench/contributions.ts";
|
||||
const { useDatabase, bindConsole } = await import(database);
|
||||
const { editorService } = await import(workbench);
|
||||
const console = "/app-data/consoles/db/demo/console.sql";
|
||||
useDatabase.setState({ loaded: true, sources: [{ id: "demo", name: "Shop Demo", driver: "sqlite" }] });
|
||||
bindConsole(console, "demo");
|
||||
bindConsole("/acme-api/console.sql", "demo");
|
||||
await editorService().openFile("/acme-api/console.sql", { focus: true });
|
||||
await editorService().openFile(console, { focus: true });
|
||||
});
|
||||
|
||||
const consoleTab = page.getByRole("tab", { name: "console [Shop Demo]", exact: true });
|
||||
await expect(consoleTab).toHaveAttribute("aria-selected", "true");
|
||||
await expect(page.getByRole("tab", { name: "console.sql", exact: true })).toBeVisible();
|
||||
await expect(page.getByRole("toolbar", { name: "Query Console", exact: true })).toContainText("main");
|
||||
|
||||
await page.evaluate(async () => {
|
||||
const path = "/src/features/database/store.ts";
|
||||
const { useDatabase } = await import(path);
|
||||
useDatabase.setState((state: { sources: Array<{ name: string }> }) => ({
|
||||
sources: state.sources.map((source) => ({ ...source, name: "Shop QA" })),
|
||||
}));
|
||||
});
|
||||
await expect(page.getByRole("tab", { name: "console [Shop QA]", exact: true })).toHaveAttribute(
|
||||
"aria-selected",
|
||||
"true",
|
||||
);
|
||||
});
|
||||
|
||||
const edgeTitle = "Result 3 — a long query result at the edge of the Services pane";
|
||||
const bindings = { "alpha-console.sql": "alpha", "beta-console.sql": "beta" };
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { expect, test } from "./fixtures";
|
||||
|
||||
test("keeps debug actions and views accessible when the Debug window is docked at the side", async ({
|
||||
workbench: page,
|
||||
}) => {
|
||||
await page.setViewportSize({ width: 900, height: 760 });
|
||||
await page.evaluate(async () => {
|
||||
const debugModule = "/src/features/debugger/state.ts";
|
||||
const layoutModule = "/src/workbench/toolwindows/store.ts";
|
||||
const servicesModule = "/src/workbench/contributions.ts";
|
||||
const { adoptSession } = await import(/* @vite-ignore */ debugModule);
|
||||
const { moveToolWindow, setToolWindowSize } = await import(/* @vite-ignore */ layoutModule);
|
||||
const { toolWindowService } = await import(/* @vite-ignore */ servicesModule);
|
||||
adoptSession(91, { name: "Checkout service", adapter: "java-debug", status: "starting" });
|
||||
moveToolWindow("debug", "right");
|
||||
setToolWindowSize("right", 300);
|
||||
toolWindowService().show("debug", true);
|
||||
});
|
||||
|
||||
const pane = page.getByRole("region", { name: "Debug", exact: true });
|
||||
const toolbar = pane.getByRole("toolbar", { name: "Debugger actions" });
|
||||
const views = pane.getByRole("tablist", { name: "Debug views" });
|
||||
await expect(toolbar.getByRole("button", { name: "More", exact: true })).toBeInViewport();
|
||||
await expect(views.getByRole("tab", { name: "Console", exact: true })).toBeInViewport();
|
||||
await views.getByRole("tab", { name: "Memory", exact: true }).click();
|
||||
await expect(views.getByRole("tab", { name: "Memory", exact: true })).toHaveAttribute("aria-selected", "true");
|
||||
});
|
||||
@@ -19,6 +19,33 @@ async function openChain(page: Page) {
|
||||
return toolbar;
|
||||
}
|
||||
|
||||
test("a narrow diff keeps navigation, settings and the difference count visible", async ({ workbench: page }) => {
|
||||
const toolbar = await openChain(page);
|
||||
await page.setViewportSize({ width: 400, height: 600 });
|
||||
const clipped = await toolbar.evaluate((bar) => {
|
||||
const bounds = bar.getBoundingClientRect();
|
||||
const items = [...bar.querySelectorAll("button, span")].filter(
|
||||
(el) => el.matches("button") || el.textContent === "1 difference",
|
||||
);
|
||||
return items.flatMap((el) => {
|
||||
const box = el.getBoundingClientRect();
|
||||
if (el.matches("button") && !box.width) return [];
|
||||
return !box.width ||
|
||||
box.right > bounds.right + 1 ||
|
||||
box.bottom > bounds.bottom + 1 ||
|
||||
el.scrollWidth > el.clientWidth + 1
|
||||
? [el.getAttribute("aria-label") || el.textContent]
|
||||
: [];
|
||||
});
|
||||
});
|
||||
expect(clipped).toEqual([]);
|
||||
await toolbar.getByRole("button", { name: "Settings", exact: true }).click();
|
||||
await page.getByRole("menuitemcheckbox", { name: "Unified Viewer", exact: true }).click();
|
||||
await expect(toolbar.locator("..").locator(".cm-editor")).toHaveCount(1);
|
||||
await toolbar.getByRole("button", { name: "Next Difference", exact: true }).click();
|
||||
await expect(toolbar).toContainText("1 of 1 differences");
|
||||
});
|
||||
|
||||
test("maximizes a diff preview tab and restores its previous layout", async ({ workbench: page }) => {
|
||||
const toolbar = await openChain(page);
|
||||
const viewer = toolbar.locator("..");
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { expect, openFromTree, test } from "./fixtures";
|
||||
|
||||
test("fits a long wrapped tab and its close button inside a narrow editor group", async ({ workbench: page }) => {
|
||||
const filename = "QuarterlyReconciliationAndCheckoutAllocation.test.ts";
|
||||
await page.setViewportSize({ width: 720, height: 760 });
|
||||
await openFromTree(page, "README.md");
|
||||
await page.evaluate(async (filename) => {
|
||||
const projectModule = "/src/platform/project.ts";
|
||||
const projectFilesModule = "/src/features/project-tree/fs.ts";
|
||||
const editorModule = "/src/editor/core/controller.ts";
|
||||
const { useProject } = await import(/* @vite-ignore */ projectModule);
|
||||
const { projectFs } = await import(/* @vite-ignore */ projectFilesModule);
|
||||
const { editorController, useTabs } = await import(/* @vite-ignore */ editorModule);
|
||||
const { project, settings } = useProject.getState();
|
||||
useProject.setState({
|
||||
settings: {
|
||||
...settings,
|
||||
editor: { ...settings?.editor, tabs: { ...settings?.editor?.tabs, multipleRows: true } },
|
||||
},
|
||||
});
|
||||
const path = `${project.root}/${filename}`;
|
||||
await projectFs(project.root).createFile(path);
|
||||
await editorController().openFilesIn([path], useTabs.getState().activePane, "right");
|
||||
}, filename);
|
||||
|
||||
const tab = page.getByRole("tab", { name: filename, exact: true });
|
||||
await expect(tab).toHaveAttribute("aria-selected", "true");
|
||||
await expect
|
||||
.poll(() =>
|
||||
tab.evaluate((element) => {
|
||||
const close = element.parentElement?.querySelector("[data-tab-close]");
|
||||
const strip = element.closest('[role="tablist"]');
|
||||
return !!close && !!strip && close.getBoundingClientRect().right <= strip.getBoundingClientRect().right;
|
||||
}),
|
||||
)
|
||||
.toBe(true);
|
||||
await tab.locator("..").getByRole("button", { name: "Close Tab", exact: true }).click();
|
||||
await expect(tab).toBeHidden();
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { expect, openFromTree, stubDesktop, test } from "./fixtures";
|
||||
|
||||
test("File Structure keeps symbol filters readable at desktop and minimum window widths", async ({
|
||||
workbench: page,
|
||||
}) => {
|
||||
await openFromTree(page, "build.gradle.kts");
|
||||
await stubDesktop(page);
|
||||
await page.evaluate(async () => {
|
||||
const range = { start: { line: 0, col: 0 }, end: { line: 1, col: 1 } };
|
||||
const field = {
|
||||
name: "orderCount",
|
||||
kind: "field",
|
||||
visibility: "private",
|
||||
range,
|
||||
selection_range: range,
|
||||
children: [],
|
||||
};
|
||||
window.__ipc.handlers.doc_outline = () => [
|
||||
{ name: "OrderController", kind: "class", range, selection_range: range, children: [field] },
|
||||
];
|
||||
window.__ipc.handlers.lsp_type_hierarchy = () => [];
|
||||
const path = "/src/platform/actions.ts";
|
||||
const { executeAction } = await import(/* @vite-ignore */ path);
|
||||
await executeAction("navigate.fileStructure");
|
||||
});
|
||||
|
||||
const dialog = page.getByRole("dialog", { name: "File Structure" });
|
||||
const fields = dialog.getByRole("checkbox", { name: "Show fields" });
|
||||
await expect(dialog.getByRole("checkbox", { name: "Show inherited members" })).toBeVisible();
|
||||
for (const width of [400, 1000]) {
|
||||
await page.setViewportSize({ width, height: 600 });
|
||||
const clipped = await dialog.evaluate((element) => {
|
||||
const labels = [...element.querySelectorAll("label")].filter((label) =>
|
||||
label.querySelector('input[type="checkbox"]'),
|
||||
);
|
||||
const row = labels[0]!.parentElement!.getBoundingClientRect();
|
||||
return labels.flatMap((label) => {
|
||||
const range = document.createRange();
|
||||
range.selectNodeContents(label);
|
||||
const box = range.getBoundingClientRect();
|
||||
return box.top < row.top - 1 || box.right > row.right + 1 || box.bottom > row.bottom + 1
|
||||
? [label.textContent]
|
||||
: [];
|
||||
});
|
||||
});
|
||||
expect(clipped).toEqual([]);
|
||||
}
|
||||
// The dialog scales in when it opens: wait for its settled width.
|
||||
await expect.poll(async () => Math.round((await dialog.boundingBox())!.width)).toBe(540);
|
||||
const field = dialog.getByRole("treeitem", { name: "orderCount" });
|
||||
await expect(field).toBeVisible();
|
||||
await fields.uncheck();
|
||||
await expect(field).toBeHidden();
|
||||
});
|
||||
@@ -76,6 +76,47 @@ test.beforeEach(async ({ workbench: page }) => {
|
||||
});
|
||||
});
|
||||
|
||||
test("fits a wide graph and saved columns without horizontal scrolling", async ({ workbench: page }) => {
|
||||
await page.evaluate(async () => {
|
||||
const module = "/src/features/vcs/logOptions.ts";
|
||||
const { useLogOptions } = await import(/* @vite-ignore */ module);
|
||||
useLogOptions.setState({
|
||||
showColumnHeaders: true,
|
||||
showHash: true,
|
||||
showSignature: true,
|
||||
columnWidths: { author: 400, date: 240 },
|
||||
});
|
||||
});
|
||||
const list = page.getByRole("listbox", { name: "Commits" });
|
||||
await expect(list.getByRole("separator", { name: "Resize Date column" })).toBeVisible();
|
||||
|
||||
for (const width of [1400, 620, 400]) {
|
||||
await page.setViewportSize({ width, height: 900 });
|
||||
await expect
|
||||
.poll(() =>
|
||||
list.evaluate((element) => {
|
||||
const header = [...element.querySelectorAll("[data-log-column]")];
|
||||
const row = [...element.querySelector('[role="option"]')!.children];
|
||||
const right = element.getBoundingClientRect().right;
|
||||
return {
|
||||
fits: element.scrollWidth === element.clientWidth,
|
||||
aligned: header.every((cell, i) => {
|
||||
const heading = cell.getBoundingClientRect();
|
||||
const data = row[i]!.getBoundingClientRect();
|
||||
return heading.width > 0 && heading.right <= right && Math.abs(heading.right - data.right) < 1;
|
||||
}),
|
||||
};
|
||||
}),
|
||||
)
|
||||
.toEqual({ fits: true, aligned: true });
|
||||
}
|
||||
|
||||
await list.hover();
|
||||
await page.mouse.wheel(450, 450);
|
||||
await expect.poll(() => list.evaluate((element) => element.scrollTop)).toBeGreaterThan(0);
|
||||
expect(await list.evaluate((element) => element.scrollLeft)).toBe(0);
|
||||
});
|
||||
|
||||
test("keeps a wide Git graph filled during scrollbar drags and its edges interactive afterwards", async ({
|
||||
workbench: page,
|
||||
}) => {
|
||||
|
||||
@@ -94,3 +94,30 @@ test("a second click on a Git Log toolbar button closes its popup", async ({ wor
|
||||
await expect(row).toBeHidden();
|
||||
}
|
||||
});
|
||||
|
||||
test("Git Log keeps filters and actions inside the commit pane at the minimum window width", async ({
|
||||
workbench: page,
|
||||
}) => {
|
||||
await page.setViewportSize({ width: 400, height: 500 });
|
||||
const toolbar = page.getByRole("toolbar", { name: "Log filters" });
|
||||
const clippedButtons = () =>
|
||||
toolbar.evaluate((el) => {
|
||||
const bar = el.getBoundingClientRect();
|
||||
return Array.from(el.querySelectorAll("button"))
|
||||
.filter((button) => {
|
||||
if (getComputedStyle(button).visibility === "hidden") return false;
|
||||
const box = button.getBoundingClientRect();
|
||||
return box.width > 0 && (box.right > bar.right + 1 || box.bottom > bar.bottom + 1);
|
||||
})
|
||||
.map((button) => button.getAttribute("aria-label"));
|
||||
});
|
||||
await expect.poll(clippedButtons).toEqual([]);
|
||||
const search = toolbar.getByRole("textbox", { name: "Filter by message" });
|
||||
await expect(search).toBeVisible();
|
||||
expect((await search.boundingBox())!.width).toBeGreaterThan(12);
|
||||
await toolbar.getByRole("button", { name: "View Options" }).click();
|
||||
await expect(page.getByRole("menuitemcheckbox", { name: "Tag Names" })).toBeVisible();
|
||||
await page.keyboard.press("Escape");
|
||||
await toolbar.getByRole("button", { name: "More Filters" }).click();
|
||||
await expect(page.getByRole("menuitem", { name: "Paths" })).toBeVisible();
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import { expect, test } from "./fixtures";
|
||||
|
||||
test("keeps diagnostic text and file counts visible in a narrow Problems pane", async ({ workbench: page }) => {
|
||||
await page.setViewportSize({ width: 720, height: 760 });
|
||||
await page.evaluate(async () => {
|
||||
const projectModule = "/src/platform/project.ts";
|
||||
const problemsModule = "/src/features/code-insight/problemsStore.ts";
|
||||
const { useProject } = await import(/* @vite-ignore */ projectModule);
|
||||
const { applyDiagnosticsEvent } = await import(/* @vite-ignore */ problemsModule);
|
||||
const root = useProject.getState().project.root;
|
||||
const path = `${root}/src/features/checkout/shared/services/with-a-long-directory-name/for-order-processing/orders.ts`;
|
||||
const source = "typescript-eslint-application-rules-with-a-very-long-source-label";
|
||||
applyDiagnosticsEvent(path, source, [
|
||||
{
|
||||
range: { start: { line: 2, col: 2 }, end: { line: 2, col: 9 } },
|
||||
severity: "error",
|
||||
source,
|
||||
message: "Invalid order total",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
await page.getByRole("button", { name: "Problems", exact: true }).click();
|
||||
const pane = page.locator('[data-toolwindow="problems"]');
|
||||
await pane.getByRole("tab", { name: /Project Errors/ }).click();
|
||||
await expect(pane.getByRole("toolbar", { name: "Problems actions" })).toHaveAttribute("aria-orientation", "vertical");
|
||||
const tree = pane.getByRole("tree", { name: "Problems" });
|
||||
const count = tree.getByText("1 error", { exact: true });
|
||||
const message = tree.getByText("Invalid order total", { exact: true });
|
||||
await expect(count).toBeVisible();
|
||||
await expect(message).toBeVisible();
|
||||
await expect.poll(() => message.evaluate((element) => element.scrollWidth <= element.clientWidth)).toBe(true);
|
||||
await expect.poll(() => tree.evaluate((element) => element.scrollWidth <= element.clientWidth)).toBe(true);
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { expect, test } from "./fixtures";
|
||||
|
||||
test("gives the project name room before its path in a narrow tree", async ({ workbench: page }) => {
|
||||
await page.evaluate(async () => {
|
||||
const storeModule = "/src/features/project-tree/store.ts";
|
||||
const { useProjectTree } = await import(/* @vite-ignore */ storeModule);
|
||||
useProjectTree.setState({ name: "checkout-order-processing" });
|
||||
});
|
||||
|
||||
const pane = page.getByRole("region", { name: "Project", exact: true });
|
||||
await pane.getByRole("button", { name: "Options", exact: true }).click();
|
||||
await page.getByRole("menuitemcheckbox", { name: "Horizontal Scrolling" }).click();
|
||||
|
||||
const tree = pane.getByRole("tree", { name: "Project" });
|
||||
const name = tree.getByText("checkout-order-processing", { exact: true });
|
||||
await expect(name).toBeVisible();
|
||||
await expect.poll(() => name.evaluate((element) => element.scrollWidth <= element.clientWidth)).toBe(true);
|
||||
await expect.poll(() => tree.evaluate((element) => element.scrollWidth <= element.clientWidth)).toBe(true);
|
||||
});
|
||||
@@ -81,6 +81,39 @@ test("⌘F2 stops a running process", async ({ workbench: page }) => {
|
||||
await expect(run.getByRole("tab", { name: "dev" })).toHaveAttribute("data-tooltip", /exit code 143$/);
|
||||
});
|
||||
|
||||
test("a short Run pane has room for the full toolbar above the output", async ({ workbench: page }) => {
|
||||
await page.setViewportSize({ width: 400, height: 500 });
|
||||
const output = "Watching for changes…";
|
||||
await stubRun(page, [{ kind: "stdout", text: `${output}\n` }], null);
|
||||
await page.keyboard.press("Control+r");
|
||||
const run = page.getByRole("region", { name: "Run", exact: true });
|
||||
await expect(run.getByText(output)).toBeVisible();
|
||||
await page.evaluate(async () => {
|
||||
const path = "/src/workbench/toolwindows/store.ts";
|
||||
const { setToolWindowSize } = await import(/* @vite-ignore */ path);
|
||||
setToolWindowSize("bottom", 160);
|
||||
});
|
||||
const toolbar = run.getByRole("toolbar", { name: "Run actions" });
|
||||
const clipped = await toolbar.evaluate((el) => {
|
||||
const bar = el.getBoundingClientRect();
|
||||
return Array.from(el.querySelectorAll("button")).flatMap((button) => {
|
||||
const box = button.getBoundingClientRect();
|
||||
return box.left < bar.left - 1 || box.right > bar.right + 1 || box.bottom > bar.bottom + 1
|
||||
? [button.getAttribute("aria-label")]
|
||||
: [];
|
||||
});
|
||||
});
|
||||
expect(clipped).toEqual([]);
|
||||
await expect(toolbar).toHaveAttribute("aria-orientation", "horizontal");
|
||||
const toolbarBox = (await toolbar.boundingBox())!;
|
||||
const consoleBox = (await run.getByRole("log").boundingBox())!;
|
||||
expect(consoleBox.y).toBeGreaterThanOrEqual(toolbarBox.y + toolbarBox.height - 1);
|
||||
await toolbar.getByRole("button", { name: "Clear All" }).click();
|
||||
await expect(run.getByText(output)).toBeHidden();
|
||||
await toolbar.getByRole("button", { name: "Stop" }).click();
|
||||
await expect.poll(() => ipcCalls(page, "run_stop")).toEqual([{ id: RUN_ID }]);
|
||||
});
|
||||
|
||||
test("Go reruns keep each package's exact failures and the original configuration", async ({ workbench: page }) => {
|
||||
await stubDesktop(page);
|
||||
await page.evaluate(async () => {
|
||||
|
||||
@@ -13,6 +13,32 @@ async function run(page: Page, id: string) {
|
||||
}, id);
|
||||
}
|
||||
|
||||
async function stubFindMatches(page: Page) {
|
||||
await stubDesktop(page);
|
||||
await page.evaluate(() => {
|
||||
window.__ipc.handlers.search_validate = () => null;
|
||||
window.__ipc.handlers.search_start = (args) => {
|
||||
const send = (event: unknown) => window.__ipc.send(args.onEvent, event);
|
||||
const text = "base=http://localhost:8080";
|
||||
setTimeout(() => {
|
||||
for (let i = 0; i < 12; i++) {
|
||||
const relPath = `config/endpoint-${i}.txt`;
|
||||
const matches = [{ line: 0, text, ranges: [[5, text.length]], textOffset: 0 }];
|
||||
send({
|
||||
type: "file",
|
||||
file: { path: `/dev/acme-api/${relPath}`, relPath, matches },
|
||||
});
|
||||
}
|
||||
send({
|
||||
type: "done",
|
||||
summary: { files: 12, matches: 12, limitHit: false, cancelled: false, elapsedMs: 4 },
|
||||
});
|
||||
}, 20);
|
||||
return null;
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function expectControlsFit(page: Page) {
|
||||
const dialog = page.getByRole("dialog");
|
||||
await expect(dialog).toBeVisible();
|
||||
@@ -42,35 +68,7 @@ test.describe("search popups at the minimum window width", () => {
|
||||
|
||||
for (const id of ["find.inPath", "find.replaceInPath"]) {
|
||||
test(`${id} keeps its title, count and buttons visible`, async ({ workbench: page }) => {
|
||||
await stubDesktop(page);
|
||||
await page.evaluate(() => {
|
||||
window.__ipc.handlers.search_validate = () => null;
|
||||
window.__ipc.handlers.search_start = (args) => {
|
||||
const send = (event: unknown) => window.__ipc.send(args.onEvent, event);
|
||||
const text = "base=http://localhost:8080";
|
||||
setTimeout(() => {
|
||||
for (let i = 0; i < 12; i++) {
|
||||
const relPath = `config/endpoint-${i}.txt`;
|
||||
const matches = [{ line: 0, text, ranges: [[5, text.length]], textOffset: 0 }];
|
||||
send({
|
||||
type: "file",
|
||||
file: { path: `/dev/acme-api/${relPath}`, relPath, matches },
|
||||
});
|
||||
}
|
||||
send({
|
||||
type: "done",
|
||||
summary: {
|
||||
files: 12,
|
||||
matches: 12,
|
||||
limitHit: false,
|
||||
cancelled: false,
|
||||
elapsedMs: 4,
|
||||
},
|
||||
});
|
||||
}, 20);
|
||||
return null;
|
||||
};
|
||||
});
|
||||
await stubFindMatches(page);
|
||||
await run(page, id);
|
||||
await page.getByRole("dialog").getByPlaceholder("Search").fill("http://localhost:8080");
|
||||
await expect(page.getByRole("dialog").locator("header [aria-live]")).toContainText(/match/);
|
||||
@@ -95,6 +93,19 @@ test.describe("search popups at the minimum window width", () => {
|
||||
});
|
||||
}
|
||||
|
||||
test("directory scope gives the path field room at the minimum width", async ({ workbench: page }) => {
|
||||
await stubDesktop(page);
|
||||
await run(page, "find.inPath");
|
||||
const dialog = page.getByRole("dialog", { name: "Find in Files" });
|
||||
await dialog.getByRole("tab", { name: "Directory", exact: true }).click();
|
||||
const input = dialog.getByRole("textbox", { name: "Directory", exact: true });
|
||||
await expect(input).toBeVisible();
|
||||
const { width } = (await input.boundingBox())!;
|
||||
expect(width).toBeGreaterThan(200);
|
||||
await expectControlsFit(page);
|
||||
await dialog.getByRole("button", { name: /Browse/ }).click({ trial: true });
|
||||
});
|
||||
|
||||
for (const id of [
|
||||
"navigate.searchEverywhere",
|
||||
"navigate.searchText",
|
||||
@@ -111,3 +122,31 @@ test.describe("search popups at the minimum window width", () => {
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("Find in Files prioritizes the preview and remembers an adjusted split", async ({ workbench: page }) => {
|
||||
await stubFindMatches(page);
|
||||
await run(page, "find.inPath");
|
||||
const dialog = page.getByRole("dialog", { name: "Find in Files" });
|
||||
await dialog.getByPlaceholder("Search").fill("http://localhost:8080");
|
||||
const results = dialog.locator("[data-panel][id=results]");
|
||||
const preview = dialog.locator("[data-panel][id=preview]");
|
||||
const handle = dialog.getByTestId("find-popup-split").getByRole("separator");
|
||||
const resultsShare = async () => {
|
||||
const resultHeight = (await results.boundingBox())!.height;
|
||||
const previewHeight = (await preview.boundingBox())!.height;
|
||||
return resultHeight / (resultHeight + previewHeight);
|
||||
};
|
||||
await expect(handle).toBeAttached();
|
||||
await expect.poll(resultsShare).toBeCloseTo(0.33, 2);
|
||||
await handle.press("ArrowUp");
|
||||
await expect.poll(resultsShare).toBeLessThan(0.31);
|
||||
const adjusted = await resultsShare();
|
||||
await page.setViewportSize({ width: 400, height: 500 });
|
||||
await expect.poll(resultsShare).toBeGreaterThan(adjusted + 0.02);
|
||||
await dialog.getByPlaceholder("Search").press("Escape");
|
||||
await expect(dialog).toBeHidden();
|
||||
await page.setViewportSize({ width: 1400, height: 900 });
|
||||
await run(page, "find.inPath");
|
||||
await expect(handle).toBeAttached();
|
||||
await expect.poll(resultsShare).toBeCloseTo(adjusted, 2);
|
||||
});
|
||||
@@ -48,6 +48,51 @@ async function move(page: Page, source: string, target: string) {
|
||||
await page.mouse.up();
|
||||
}
|
||||
|
||||
test("Services favors details and stacks the panes when docked to the side", async ({ workbench: page }) => {
|
||||
await configure(page);
|
||||
const services = page.getByRole("region", { name: "Services", exact: true });
|
||||
const items = services.locator("[data-panel][id=services-items]");
|
||||
const detail = services.locator("[data-panel][id=services-detail]");
|
||||
const handle = services.locator("[data-separator]");
|
||||
const share = async (axis: "width" | "height") => {
|
||||
const treeSize = (await items.boundingBox())![axis];
|
||||
return treeSize / (treeSize + (await detail.boundingBox())![axis]);
|
||||
};
|
||||
const dock = (anchor: "bottom" | "right" | "left") =>
|
||||
page.evaluate(async (anchor) => {
|
||||
const path = "/src/workbench/toolwindows/store.ts";
|
||||
const { moveToolWindow, setToolWindowSize } = await import(/* @vite-ignore */ path);
|
||||
moveToolWindow("services", anchor);
|
||||
if (anchor !== "bottom") setToolWindowSize(anchor, 280);
|
||||
}, anchor);
|
||||
|
||||
await expect(handle).toHaveAttribute("aria-orientation", "vertical");
|
||||
await expect.poll(() => share("width")).toBeCloseTo(0.3, 2);
|
||||
await handle.press("ArrowRight");
|
||||
await expect.poll(() => share("width")).toBeGreaterThan(0.31);
|
||||
const bottom = await share("width");
|
||||
|
||||
for (const anchor of ["right", "left"] as const) {
|
||||
await dock(anchor);
|
||||
await expect(handle).toHaveAttribute("aria-orientation", "horizontal");
|
||||
const treeBox = (await items.boundingBox())!;
|
||||
const detailBox = (await detail.boundingBox())!;
|
||||
expect(detailBox.width).toBeCloseTo(treeBox.width, 0);
|
||||
expect(detailBox.y).toBeGreaterThanOrEqual(treeBox.y + treeBox.height - 1);
|
||||
await expect.poll(() => share("height")).toBeCloseTo(0.3, 2);
|
||||
}
|
||||
await handle.press("ArrowDown");
|
||||
await expect.poll(() => share("height")).toBeGreaterThan(0.31);
|
||||
const side = await share("height");
|
||||
|
||||
await dock("bottom");
|
||||
await expect(handle).toHaveAttribute("aria-orientation", "vertical");
|
||||
await expect.poll(() => share("width")).toBeCloseTo(bottom, 2);
|
||||
await dock("right");
|
||||
await expect(handle).toHaveAttribute("aria-orientation", "horizontal");
|
||||
await expect.poll(() => share("height")).toBeCloseTo(side, 2);
|
||||
});
|
||||
|
||||
test("service pointer ordering works for Java and npm and survives reload", async ({ workbench: page }) => {
|
||||
await configure(page);
|
||||
const tree = page.getByRole("tree", { name: "Services", exact: true });
|
||||
|
||||
@@ -72,6 +72,7 @@ test("settings sidebar resizes with mouse and keyboard and remembers its width",
|
||||
await expect(handle).toBeVisible();
|
||||
const sidebar = dialog.locator("#navigation");
|
||||
const initial = (await sidebar.boundingBox())!.width;
|
||||
expect(initial / (await dialog.getByTestId("settings").boundingBox())!.width).toBeCloseTo(0.225, 2);
|
||||
const box = (await handle.boundingBox())!;
|
||||
await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2);
|
||||
await page.mouse.down();
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { expect, test } from "./fixtures";
|
||||
|
||||
test("keeps the file count visible beside a long directory in the TODO pane", async ({ workbench: page }) => {
|
||||
await page.setViewportSize({ width: 900, height: 760 });
|
||||
await page.evaluate(async () => {
|
||||
const projectModule = "/src/platform/project.ts";
|
||||
const todoModule = "/src/features/todo/store.ts";
|
||||
const { useProject } = await import(/* @vite-ignore */ projectModule);
|
||||
const { useTodos } = await import(/* @vite-ignore */ todoModule);
|
||||
const directory = "src/features/checkout/shared/services/with-a-long-directory-name/for-order-processing";
|
||||
const path = `${useProject.getState().project.root}/${directory}/orders.ts`;
|
||||
useTodos.setState({
|
||||
scannedOnce: true,
|
||||
error: null,
|
||||
scanned: new Map([[path, [{ path, line: 8, col: 3, keyword: "TODO", text: "TODO: validate orders" }]]]),
|
||||
});
|
||||
});
|
||||
|
||||
await page.getByRole("button", { name: "TODO" }).click();
|
||||
const pane = page.locator('[data-toolwindow="todo"]');
|
||||
const toolbar = pane.getByRole("toolbar", { name: "TODO actions" });
|
||||
await expect(toolbar).toHaveAttribute("aria-orientation", "vertical");
|
||||
const tree = pane.getByRole("tree", { name: "TODO items" });
|
||||
const fileName = tree.getByText("orders.ts", { exact: true });
|
||||
await expect(fileName).toBeVisible();
|
||||
await expect.poll(() => fileName.evaluate((element) => element.scrollWidth <= element.clientWidth)).toBe(true);
|
||||
const count = tree.getByText("1 item", { exact: true });
|
||||
await expect(count).toBeVisible();
|
||||
await expect
|
||||
.poll(() =>
|
||||
count.evaluate(
|
||||
(element) =>
|
||||
element.getBoundingClientRect().right <= element.closest('[role="tree"]')!.getBoundingClientRect().right,
|
||||
),
|
||||
)
|
||||
.toBe(true);
|
||||
});
|
||||
@@ -7,6 +7,8 @@ import { join } from "node:path";
|
||||
import { track } from "../lib/measure.ts";
|
||||
import { normalizePath, type EvalScenario, type Measured } from "../lib/scenario.ts";
|
||||
import type { Session } from "../lib/session.ts";
|
||||
import { renameFinished, renameResult, type RenameStep } from "../lib/renameObservation.ts";
|
||||
import type { Renamed } from "../../src/bindings.ts";
|
||||
|
||||
const SRC = "src/main/java/com/example/orders";
|
||||
const TEST = "src/test/java/com/example/orders";
|
||||
@@ -17,7 +19,7 @@ const LONG_TIMEOUT = 60_000;
|
||||
|
||||
interface ProbeWindow {
|
||||
__openideaPerf: {
|
||||
editorState(): { path: string; head: number; line: number; word: string } | null;
|
||||
editorState(): { doc: number; path: string; head: number; line: number; col: number; word: string } | null;
|
||||
editorText(): string | null;
|
||||
setCursor(offset: number): void;
|
||||
insertAfter(after: string, text: string): number;
|
||||
@@ -25,6 +27,8 @@ interface ProbeWindow {
|
||||
completionPending(): boolean;
|
||||
diagnostics(): Array<{ line: number; severity: string; message: string }>;
|
||||
lineAtY(y: number): number;
|
||||
openFileForDiagnostics(path: string, source: string): Promise<number>;
|
||||
diagnosticsPublished(generation: number): boolean;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -373,10 +377,21 @@ function diagnosticsOnOpen(project: string, id: string, title: string, file: str
|
||||
async setup() {},
|
||||
async step(s): Promise<Measured> {
|
||||
const timing = await track(s.page, {
|
||||
openFile: join(s.root, file),
|
||||
arg: null,
|
||||
first: () => !!document.querySelector(".cm-content .cm-line"),
|
||||
done: () => true,
|
||||
run: async (arg) => {
|
||||
const w = window as unknown as ProbeWindow;
|
||||
arg.generation = await w.__openideaPerf.openFileForDiagnostics(arg.path, "jdtls");
|
||||
},
|
||||
arg: { path: join(s.root, file), generation: 0 },
|
||||
first: (arg) => {
|
||||
const w = window as unknown as ProbeWindow;
|
||||
return (
|
||||
w.__openideaPerf.diagnosticsPublished(arg.generation) && !!document.querySelector(".cm-content .cm-line")
|
||||
);
|
||||
},
|
||||
done: (arg) => {
|
||||
const w = window as unknown as ProbeWindow;
|
||||
return w.__openideaPerf.diagnosticsPublished(arg.generation);
|
||||
},
|
||||
signature: problems,
|
||||
settleMs: 3000,
|
||||
timeoutMs: TIMEOUT,
|
||||
@@ -514,12 +529,19 @@ function quickDoc(project: string, id: string, title: string, at: Place): EvalSc
|
||||
};
|
||||
}
|
||||
|
||||
interface RenameAnswer {
|
||||
edit: { files: Array<{ path?: string; edits?: Array<{ range: { start: { line: number } }; new_text: string }> }> };
|
||||
}
|
||||
|
||||
/** Rename in place to `newName`, then ↵: every edit it makes (a preview is left unapplied). */
|
||||
function rename(project: string, id: string, title: string, at: Place, newName: string): EvalScenario {
|
||||
function rename(
|
||||
project: string,
|
||||
id: string,
|
||||
title: string,
|
||||
at: Place,
|
||||
newName: string,
|
||||
mode: RenameStep["mode"],
|
||||
): EvalScenario {
|
||||
const prepared = new WeakMap<
|
||||
Session,
|
||||
{ doc: number; path: string; pos: { line: number; col: number }; original: string }
|
||||
>();
|
||||
return {
|
||||
id,
|
||||
title,
|
||||
@@ -528,23 +550,39 @@ function rename(project: string, id: string, title: string, at: Place, newName:
|
||||
conditions: ["warm"],
|
||||
async setup(s) {
|
||||
await reach(s, at);
|
||||
const before = await s.page.evaluate(() => {
|
||||
const w = window as unknown as ProbeWindow;
|
||||
const state = w.__openideaPerf.editorState();
|
||||
const original = w.__openideaPerf.editorText();
|
||||
if (!state || original === null) throw new Error("no active editor before rename");
|
||||
return { doc: state.doc, path: state.path, pos: { line: state.line, col: state.col }, original };
|
||||
});
|
||||
prepared.set(s, before);
|
||||
await s.keepText();
|
||||
await s.runAction("refactor.rename");
|
||||
await s.page.waitForSelector(".cm-lsp-inplace-primary", { timeout: TIMEOUT });
|
||||
await s.page.keyboard.type(newName);
|
||||
},
|
||||
async step(s): Promise<Measured> {
|
||||
const answered = (newName: string) => {
|
||||
if (document.querySelector('[data-testid="refactor-preview"]')) return true;
|
||||
if (document.querySelector(".cm-lsp-hint:not([data-pending]):not(.cm-lsp-inplace-hint)")) return true;
|
||||
return (
|
||||
!document.querySelector(".cm-lsp-inplace") &&
|
||||
!!(window as unknown as ProbeWindow).__openideaPerf.editorText()?.includes(newName)
|
||||
);
|
||||
const before = prepared.get(s);
|
||||
if (!before) throw new Error("rename was not prepared");
|
||||
const scope = s.renameObservations.begin({ doc: before.doc, pos: before.pos, newName });
|
||||
const arg: RenameStep = {
|
||||
scope,
|
||||
root: s.root,
|
||||
path: before.path,
|
||||
original: before.original,
|
||||
oldName: at.word,
|
||||
newName,
|
||||
mode,
|
||||
answer: null,
|
||||
};
|
||||
const timing = await track(s.page, {
|
||||
run: () =>
|
||||
document.querySelector(".cm-content")?.dispatchEvent(
|
||||
run: async (arg) => {
|
||||
const w = window as unknown as { __openideaEvalRenameAnswer(scope: number): Promise<Renamed> };
|
||||
const content = document.querySelector(".cm-content");
|
||||
if (!content) throw new Error("no editor for rename");
|
||||
content.dispatchEvent(
|
||||
new KeyboardEvent("keydown", {
|
||||
key: "Enter",
|
||||
code: "Enter",
|
||||
@@ -552,25 +590,17 @@ function rename(project: string, id: string, title: string, at: Place, newName:
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
}),
|
||||
),
|
||||
arg: newName,
|
||||
first: answered,
|
||||
done: answered,
|
||||
);
|
||||
arg.answer = await w.__openideaEvalRenameAnswer(arg.scope);
|
||||
},
|
||||
arg,
|
||||
first: renameFinished,
|
||||
done: renameFinished,
|
||||
timeoutMs: TIMEOUT,
|
||||
});
|
||||
// The preview only counts per file: where the name lands is in the answer it shows. A server
|
||||
// may merge nearby occurrences into one edit, so every line of its new text counts.
|
||||
const answer = s.lastAnswer.get("lsp_rename") as RenameAnswer | undefined;
|
||||
const changes = (answer?.edit.files ?? []).flatMap((f) =>
|
||||
(f.edits ?? []).flatMap((e) =>
|
||||
e.new_text
|
||||
.split("\n")
|
||||
.flatMap((l, i) =>
|
||||
l.includes(newName) ? [`${normalizePath(s.root, f.path!)}:${e.range.start.line + i + 1}`] : [],
|
||||
),
|
||||
),
|
||||
);
|
||||
return { timing, results: { changes } };
|
||||
const answer = s.renameObservations.completed(scope) as Renamed | null;
|
||||
if (!answer) throw new Error("rename did not receive a current successful response");
|
||||
return { timing, ...renameResult(s.root, answer, newName) };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -748,6 +778,7 @@ export const JAVA_SCENARIOS: EvalScenario[] = [
|
||||
"Rename: method used across files",
|
||||
{ file: `${SRC}/Order.java`, line: "public long totalCents() {", word: "totalCents" },
|
||||
"orderTotalCents",
|
||||
"preview",
|
||||
),
|
||||
rename(
|
||||
APP,
|
||||
@@ -755,6 +786,7 @@ export const JAVA_SCENARIOS: EvalScenario[] = [
|
||||
"Rename: local variable",
|
||||
{ file: BOOK, line: "Order order = new Order(customer, product, quantity, unitPriceCents);", word: "order" },
|
||||
"placedOrder",
|
||||
"applied",
|
||||
),
|
||||
|
||||
declaration(
|
||||
@@ -837,5 +869,6 @@ export const JAVA_SCENARIOS: EvalScenario[] = [
|
||||
"Rename: entity method used across controllers and tests",
|
||||
{ file: `${PET}/owner/Owner.java`, line: "public void addVisit(Integer petId, Visit visit) {", word: "addVisit" },
|
||||
"recordVisit",
|
||||
"preview",
|
||||
),
|
||||
];
|
||||
@@ -0,0 +1,29 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
const runner = resolve(import.meta.dirname, "../run.ts");
|
||||
|
||||
describe("eval CLI preflight (no app builds)", () => {
|
||||
test.each([
|
||||
[["--condition", "hot"], /--condition must be warm or cold/],
|
||||
[["--only", "java.app,unknown"], /no scenario matches unknown/],
|
||||
[["--only", "java.app,"], /empty selector/],
|
||||
[["--only", "java.app.usages", "--condition", "cold"], /no compatible scenario/],
|
||||
[["--runs", "0"], /--runs/],
|
||||
[["--runs", "Infinity"], /--runs/],
|
||||
[["--runs", "1.5"], /--runs/],
|
||||
[["--warmup", "NaN"], /--warmup/],
|
||||
[["--warmup=-1"], /--warmup/],
|
||||
] as const)("rejects %j before build/start", (args, message) => {
|
||||
const result = spawnSync(process.execPath, [runner, ...args], {
|
||||
encoding: "utf8",
|
||||
timeout: 15_000,
|
||||
env: { ...process.env, PATH: "", OPENIDEA_SERVER: "/nonexistent/eval-preflight-server" },
|
||||
});
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toMatch(message);
|
||||
expect(result.stdout).not.toContain("Building openidea-server");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { RenameObservations } from "./commandObservation.ts";
|
||||
|
||||
const target = { doc: 7, pos: { line: 2, col: 8 }, newName: "renamed" };
|
||||
const answer = { edit: { files: [] }, incomplete: null };
|
||||
|
||||
function deferred() {
|
||||
let resolve!: (value: unknown) => void;
|
||||
// The harness TypeScript project targets ES2023.
|
||||
const promise = new Promise<unknown>((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe("rename command observations", () => {
|
||||
test("consumes only a matching command in the current operation", async () => {
|
||||
const capture = new RenameObservations();
|
||||
const scope = capture.begin(target);
|
||||
await capture.invoke("lsp_rename", { ...target, doc: 8, request: 1 }, async () => answer);
|
||||
await capture.invoke("lsp_rename", { ...target, newName: "other", request: 2 }, async () => answer);
|
||||
await capture.invoke("lsp_rename", { ...target, pos: { line: 3, col: 8 }, request: 3 }, async () => answer);
|
||||
expect(capture.completed(scope)).toBeNull();
|
||||
await capture.invoke("lsp_rename", { ...target, request: 4 }, async () => answer);
|
||||
expect(await capture.wait(scope)).toEqual(answer);
|
||||
});
|
||||
|
||||
test("reset and failure cannot supply a preceding success", async () => {
|
||||
const capture = new RenameObservations();
|
||||
const previous = capture.begin(target);
|
||||
await capture.invoke("lsp_rename", { ...target, request: 1 }, async () => answer);
|
||||
capture.reset();
|
||||
expect(capture.completed(previous)).toBeNull();
|
||||
await expect(capture.wait(previous)).rejects.toThrow("operation expired");
|
||||
const current = capture.begin(target);
|
||||
const waiting = capture.wait(current);
|
||||
const rejected = expect(waiting).rejects.toThrow("rename failed");
|
||||
await expect(
|
||||
capture.invoke("lsp_rename", { ...target, request: 2 }, async () => {
|
||||
throw new Error("rename failed");
|
||||
}),
|
||||
).rejects.toThrow("rename failed");
|
||||
await rejected;
|
||||
expect(capture.completed(current)).toBeNull();
|
||||
});
|
||||
|
||||
test("a failed retry clears a success in the same operation", async () => {
|
||||
const capture = new RenameObservations();
|
||||
const scope = capture.begin(target);
|
||||
await capture.invoke("lsp_rename", { ...target, request: 1 }, async () => answer);
|
||||
expect(capture.completed(scope)).toEqual(answer);
|
||||
await expect(
|
||||
capture.invoke("lsp_rename", { ...target, request: 2 }, async () => {
|
||||
throw new Error("retry failed");
|
||||
}),
|
||||
).rejects.toThrow("retry failed");
|
||||
expect(capture.completed(scope)).toBeNull();
|
||||
await expect(capture.wait(scope)).rejects.toThrow("retry failed");
|
||||
});
|
||||
|
||||
test("a delayed result from before reset cannot win over a new request", async () => {
|
||||
const capture = new RenameObservations();
|
||||
capture.begin(target);
|
||||
const old = deferred();
|
||||
const sending = capture.invoke("lsp_rename", { ...target, request: 1 }, () => old.promise);
|
||||
capture.reset();
|
||||
const current = capture.begin(target);
|
||||
old.resolve(answer);
|
||||
await sending;
|
||||
expect(capture.completed(current)).toBeNull();
|
||||
const fresh = { ...answer, preview: true };
|
||||
await capture.invoke("lsp_rename", { ...target, request: 2 }, async () => fresh);
|
||||
expect(await capture.wait(current)).toEqual(fresh);
|
||||
});
|
||||
|
||||
test("a superseded in-flight request cannot overwrite the latest failure", async () => {
|
||||
const capture = new RenameObservations();
|
||||
const scope = capture.begin(target);
|
||||
const old = deferred();
|
||||
const sending = capture.invoke("lsp_rename", { ...target, request: 1 }, () => old.promise);
|
||||
await expect(
|
||||
capture.invoke("lsp_rename", { ...target, request: 2 }, async () => {
|
||||
throw new Error("latest failed");
|
||||
}),
|
||||
).rejects.toThrow("latest failed");
|
||||
old.resolve(answer);
|
||||
await sending;
|
||||
expect(capture.completed(scope)).toBeNull();
|
||||
await expect(capture.wait(scope)).rejects.toThrow("latest failed");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
/** Command capture shared by the Session relay and rename's measured operation. */
|
||||
export interface RenameTarget {
|
||||
doc: number;
|
||||
pos: { line: number; col: number };
|
||||
newName: string;
|
||||
}
|
||||
|
||||
interface Operation {
|
||||
scope: number;
|
||||
target: RenameTarget;
|
||||
request: number;
|
||||
result: { ok: true; value: unknown } | { ok: false; error: unknown } | null;
|
||||
listeners: Array<{ resolve(value: unknown): void; reject(error: unknown): void }>;
|
||||
}
|
||||
|
||||
export class RenameObservations {
|
||||
private generation = 0;
|
||||
private operation: Operation | null = null;
|
||||
|
||||
begin(target: RenameTarget): number {
|
||||
this.reset();
|
||||
this.operation = { scope: this.generation, target, request: 0, result: null, listeners: [] };
|
||||
return this.generation;
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.generation++;
|
||||
for (const listener of this.operation?.listeners ?? []) listener.reject(new Error("rename operation expired"));
|
||||
this.operation = null;
|
||||
}
|
||||
|
||||
/** Captures the invocation, not just its answer: old requests retain their original scope. */
|
||||
async invoke<T>(cmd: string, args: unknown, send: () => Promise<T>): Promise<T> {
|
||||
const op = this.operation;
|
||||
const actual = args as Partial<RenameTarget> | null;
|
||||
const matching =
|
||||
cmd === "lsp_rename" &&
|
||||
op &&
|
||||
actual &&
|
||||
actual.doc === op.target.doc &&
|
||||
actual.newName === op.target.newName &&
|
||||
actual.pos?.line === op.target.pos.line &&
|
||||
actual.pos?.col === op.target.pos.col;
|
||||
const request = matching ? ++op.request : 0;
|
||||
if (matching) op.result = null;
|
||||
try {
|
||||
const value = await send();
|
||||
if (matching && this.operation === op && op.request === request) {
|
||||
op.result = { ok: true, value };
|
||||
for (const listener of op.listeners.splice(0)) listener.resolve(value);
|
||||
}
|
||||
return value;
|
||||
} catch (error) {
|
||||
if (matching && this.operation === op && op.request === request) {
|
||||
op.result = { ok: false, error };
|
||||
for (const listener of op.listeners.splice(0)) listener.reject(error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
completed(scope: number): unknown | null {
|
||||
const op = this.operation;
|
||||
return op?.scope === scope && op.result?.ok ? op.result.value : null;
|
||||
}
|
||||
|
||||
wait(scope: number): Promise<unknown> {
|
||||
const op = this.operation;
|
||||
if (!op || op.scope !== scope) return Promise.reject(new Error("rename operation expired"));
|
||||
if (op.result) return op.result.ok ? Promise.resolve(op.result.value) : Promise.reject(op.result.error);
|
||||
// ES2023 harnesses cannot use Promise.withResolvers().
|
||||
return new Promise((resolve, reject) => op.listeners.push({ resolve, reject }));
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,11 @@ export function pinnedTools(file: string): Record<string, string> {
|
||||
|
||||
function which(cmd: string): string | null {
|
||||
try {
|
||||
const found = execFileSync(process.platform === "win32" ? "where" : "which", [cmd], { encoding: "utf8" })
|
||||
const found = execFileSync(process.platform === "win32" ? "where" : "which", [cmd], {
|
||||
encoding: "utf8",
|
||||
timeout: 10_000,
|
||||
killSignal: "SIGKILL",
|
||||
})
|
||||
.split("\n")[0]!
|
||||
.trim();
|
||||
return found ? realpathSync(found) : null;
|
||||
@@ -69,7 +73,11 @@ function jdtlsVersion(): string | null {
|
||||
function javaVersion(): string | null {
|
||||
const home = process.env.JAVA_HOME;
|
||||
const java = home ? join(home, "bin", process.platform === "win32" ? "java.exe" : "java") : "java";
|
||||
const r = spawnSync(java, ["-XshowSettings:properties", "-version"], { encoding: "utf8" });
|
||||
const r = spawnSync(java, ["-XshowSettings:properties", "-version"], {
|
||||
encoding: "utf8",
|
||||
timeout: 10_000,
|
||||
killSignal: "SIGKILL",
|
||||
});
|
||||
const props = `${r.stdout ?? ""}${r.stderr ?? ""}`;
|
||||
const version = /java\.runtime\.version = (\S+)/.exec(props)?.[1];
|
||||
const vendor = /java\.vendor = (.+)/.exec(props)?.[1]?.trim();
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// @vitest-environment node
|
||||
import { chromium, type Browser, type Page } from "@playwright/test";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { track } from "./measure.ts";
|
||||
|
||||
let browser: Browser;
|
||||
let page: Page;
|
||||
beforeAll(async () => {
|
||||
browser = await chromium.launch({ headless: true });
|
||||
});
|
||||
beforeEach(async () => {
|
||||
page = await browser.newPage();
|
||||
await page.setContent('<div role="dialog">Previous dialog</div><div id="result">Previous result</div>');
|
||||
await page.evaluate(() => {
|
||||
Object.assign(window, {
|
||||
__openideaPerf: {
|
||||
editorState: () => null,
|
||||
runAction: async () => false,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
afterEach(async () => {
|
||||
await page.close();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await browser.close();
|
||||
});
|
||||
|
||||
const existingResult = () => document.querySelector("#result") !== null;
|
||||
|
||||
describe("tracking the current action", () => {
|
||||
it("rejects an action that declines to run even with an old result visible", async () => {
|
||||
await expect(
|
||||
track(page, { action: "unavailable", arg: null, first: existingResult, done: existingResult, timeoutMs: 1000 }),
|
||||
).rejects.toThrow(/did not run/);
|
||||
});
|
||||
|
||||
it("cannot report success before an asynchronous action later rejects", async () => {
|
||||
await expect(
|
||||
track(page, {
|
||||
// Real browser frames exercise ordering against track's rAF observer; no elapsed-time guess.
|
||||
run: () =>
|
||||
new Promise((_, reject) => {
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => reject(new Error("operation failed"))));
|
||||
}),
|
||||
arg: null,
|
||||
first: existingResult,
|
||||
done: existingResult,
|
||||
timeoutMs: 1000,
|
||||
}),
|
||||
).rejects.toThrow("operation failed");
|
||||
});
|
||||
|
||||
it("waits for successful action completion and observes its current result", async () => {
|
||||
const timing = await track(page, {
|
||||
run: () =>
|
||||
new Promise<void>((resolve) => {
|
||||
requestAnimationFrame(() => {
|
||||
document.querySelector("#result")!.textContent = "Current result";
|
||||
requestAnimationFrame(() => {
|
||||
document.querySelector("#result")!.setAttribute("data-completed", "true");
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}),
|
||||
arg: null,
|
||||
first: () => document.querySelector("#result")?.textContent === "Current result",
|
||||
done: () => document.querySelector("#result")?.textContent === "Current result",
|
||||
timeoutMs: 1000,
|
||||
});
|
||||
expect(await page.locator("#result").textContent()).toBe("Current result");
|
||||
expect(timing.completeMs).toBeGreaterThanOrEqual(timing.firstResultMs);
|
||||
expect(timing.feedbackMs).toBeGreaterThanOrEqual(timing.firstResultMs);
|
||||
expect(await page.locator("#result").getAttribute("data-completed")).toBe("true");
|
||||
});
|
||||
});
|
||||
+46
-14
@@ -3,6 +3,7 @@
|
||||
* what the user would see. A condition true at the start of a frame is painted by it.
|
||||
*/
|
||||
import type { Page } from "@playwright/test";
|
||||
import { withDeadline } from "../../perf/lib/lifecycle.ts";
|
||||
|
||||
/**
|
||||
* A check of the page, run in it every frame: self-contained (it is re-created from its
|
||||
@@ -55,7 +56,7 @@ export class TimeoutError extends Error {
|
||||
}
|
||||
|
||||
export async function track<A>(page: Page, spec: TrackSpec<A>): Promise<Tracked> {
|
||||
const r = await page.evaluate(
|
||||
const operation = page.evaluate(
|
||||
({ action, actionArg, openFile, run, arg, first, done, feedback, signature, settleMs, timeoutMs }) => {
|
||||
const w = window as unknown as {
|
||||
__openideaPerf: {
|
||||
@@ -75,26 +76,57 @@ export async function track<A>(page: Page, spec: TrackSpec<A>): Promise<Tracked>
|
||||
let lastSig: string | null = null;
|
||||
let lastChange = 0;
|
||||
const before = w.__openideaPerf.editorState();
|
||||
const visible = (sel: string) =>
|
||||
[...document.querySelectorAll(sel)].some((el) => (el as HTMLElement).getClientRects().length > 0);
|
||||
const feedbackSelector = '.cm-lsp-hint, [data-testid="usages-popup"], [role="dialog"], [role="listbox"]';
|
||||
const beforeFeedback = new Map<Element, { text: string | null; loading: string | null; busy: string | null }>();
|
||||
for (const el of document.querySelectorAll(feedbackSelector)) {
|
||||
if (el.getClientRects().length) {
|
||||
beforeFeedback.set(el, {
|
||||
text: el.textContent,
|
||||
loading: el.getAttribute("data-loading"),
|
||||
busy: el.getAttribute("aria-busy"),
|
||||
});
|
||||
}
|
||||
}
|
||||
const anyFeedback = () => {
|
||||
if (visible('.cm-lsp-hint, [data-testid="usages-popup"], [role="dialog"], [role="listbox"]')) return true;
|
||||
for (const el of document.querySelectorAll(feedbackSelector)) {
|
||||
if (!el.getClientRects().length) continue;
|
||||
const previous = beforeFeedback.get(el);
|
||||
if (
|
||||
!previous ||
|
||||
previous.text !== el.textContent ||
|
||||
previous.loading !== el.getAttribute("data-loading") ||
|
||||
previous.busy !== el.getAttribute("aria-busy")
|
||||
)
|
||||
return true;
|
||||
}
|
||||
const now = w.__openideaPerf.editorState();
|
||||
if ((now?.path ?? null) !== (before?.path ?? null) || (now && before && now.head !== before.head)) return true;
|
||||
return checks.feedback?.(arg, before) ?? false;
|
||||
};
|
||||
const times: { feedbackMs?: number; firstResultMs?: number; completeMs?: number } = {};
|
||||
const t0 = performance.now();
|
||||
let failed: unknown = null;
|
||||
const started = run
|
||||
? new Promise((resolve) => resolve(make(run)!(arg, before)))
|
||||
: openFile
|
||||
? w.__openideaPerf.openFile(openFile)
|
||||
: w.__openideaPerf.runAction(action!, actionArg);
|
||||
void started.catch((e: unknown) => (failed = e));
|
||||
let completed = false;
|
||||
let failed: string | null = null;
|
||||
const started = Promise.resolve().then<unknown>(() =>
|
||||
run
|
||||
? make(run)!(arg, before)
|
||||
: openFile
|
||||
? w.__openideaPerf.openFile(openFile)
|
||||
: w.__openideaPerf.runAction(action!, actionArg),
|
||||
);
|
||||
void started.then(
|
||||
(value) => {
|
||||
if (!run && !openFile && value !== true) failed = "action did not run";
|
||||
else completed = true;
|
||||
},
|
||||
(error: unknown) => {
|
||||
failed = error instanceof Error ? error.message : String(error);
|
||||
},
|
||||
);
|
||||
return new Promise<{ times: typeof times; timedOut: boolean; error: string | null }>((resolve) => {
|
||||
const tick = () => {
|
||||
const t = performance.now() - t0;
|
||||
if (failed !== null) return resolve({ times, timedOut: false, error: failed });
|
||||
try {
|
||||
if (times.feedbackMs === undefined && anyFeedback()) times.feedbackMs = t;
|
||||
if (times.firstResultMs === undefined && checks.first(arg, before)) times.firstResultMs = t;
|
||||
@@ -106,7 +138,7 @@ export async function track<A>(page: Page, spec: TrackSpec<A>): Promise<Tracked>
|
||||
}
|
||||
}
|
||||
const settled = !sign || t - lastChange >= (settleMs ?? 0);
|
||||
if (times.firstResultMs !== undefined && settled && checks.done(arg, before)) {
|
||||
if (completed && times.firstResultMs !== undefined && settled && checks.done(arg, before)) {
|
||||
// Session.reset waits out the time in which invoking it again widens it.
|
||||
if (action === "find.showUsages")
|
||||
(window as unknown as { __openideaEvalShowUsagesAt?: number }).__openideaEvalShowUsagesAt =
|
||||
@@ -118,7 +150,6 @@ export async function track<A>(page: Page, spec: TrackSpec<A>): Promise<Tracked>
|
||||
} catch (e) {
|
||||
return resolve({ times, timedOut: false, error: String(e) });
|
||||
}
|
||||
if (failed) return resolve({ times, timedOut: false, error: String((failed as Error).message ?? failed) });
|
||||
if (t > timeoutMs) return resolve({ times, timedOut: true, error: null });
|
||||
requestAnimationFrame(tick);
|
||||
};
|
||||
@@ -139,8 +170,9 @@ export async function track<A>(page: Page, spec: TrackSpec<A>): Promise<Tracked>
|
||||
timeoutMs: spec.timeoutMs,
|
||||
},
|
||||
);
|
||||
const r = await withDeadline(operation, spec.timeoutMs + 1000, "page measurement");
|
||||
const what = spec.action ?? (spec.openFile ? `open ${spec.openFile}` : "step");
|
||||
if (r.error) throw new Error(`${what}: ${r.error}`);
|
||||
if (r.error !== null) throw new Error(`${what}: ${r.error}`);
|
||||
if (r.timedOut) {
|
||||
const seen =
|
||||
r.times.firstResultMs !== undefined
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
// @vitest-environment node
|
||||
import { expect, test } from "vitest";
|
||||
import { launchBrowser } from "../../perf/lib/lifecycle.ts";
|
||||
import { track } from "./measure.ts";
|
||||
|
||||
test("track times out on Node's clock when animation frames stop", { timeout: 20_000 }, async () => {
|
||||
const runtime = await launchBrowser({ headless: true });
|
||||
try {
|
||||
const page = await runtime.browser.newPage();
|
||||
await page.evaluate(() => {
|
||||
// Install the otherwise undeclared probe consumed by the serialized measurement.
|
||||
const w = window as unknown as { __openideaPerf: unknown };
|
||||
w.__openideaPerf = {
|
||||
editorState: () => null,
|
||||
runAction: async () => true,
|
||||
};
|
||||
window.requestAnimationFrame = () => 0;
|
||||
});
|
||||
// The platform-clock boundary is the regression; a fake Node clock cannot drive Chromium.
|
||||
await expect(
|
||||
track(page, { action: "fixture.action", arg: null, first: () => false, done: () => false, timeoutMs: 20 }),
|
||||
).rejects.toThrow("page measurement timed out");
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,104 @@
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import { renameFinished, renameResult, type RenameStep } from "./renameObservation.ts";
|
||||
import type { Renamed } from "../../src/bindings.ts";
|
||||
|
||||
const original = "int order = 1;\nuse(order);";
|
||||
const answer: Renamed = {
|
||||
edit: {
|
||||
files: [
|
||||
{
|
||||
kind: "edit",
|
||||
path: "/p/A.java",
|
||||
edits: [
|
||||
{ range: { start: { line: 0, col: 4 }, end: { line: 0, col: 9 } }, new_text: "placedOrder" },
|
||||
{ range: { start: { line: 1, col: 4 }, end: { line: 1, col: 9 } }, new_text: "placedOrder" },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
incomplete: null,
|
||||
};
|
||||
const step: RenameStep = {
|
||||
scope: 1,
|
||||
path: "/p/A.java",
|
||||
root: "/p",
|
||||
oldName: "order",
|
||||
newName: "placedOrder",
|
||||
original,
|
||||
mode: "applied",
|
||||
answer,
|
||||
};
|
||||
let text = original;
|
||||
let state = { path: step.path, word: "order", head: 9, line: 0 };
|
||||
const w = window as unknown as { __openideaPerf?: unknown };
|
||||
function fixture() {
|
||||
w.__openideaPerf = { editorText: () => text, editorState: () => state };
|
||||
document.body.innerHTML = '<div class="cm-content"><div class="cm-line"></div></div>';
|
||||
}
|
||||
afterEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
delete w.__openideaPerf;
|
||||
text = original;
|
||||
state = { path: step.path, word: "order", head: 9, line: 0 };
|
||||
});
|
||||
|
||||
test("a successful response and typed new name are not an applied rename", () => {
|
||||
fixture();
|
||||
document.body.insertAdjacentHTML("beforeend", '<span class="cm-lsp-hint">Rename failed</span>');
|
||||
text = original.replace("order", "placedOrder");
|
||||
expect(renameFinished(step)).toBe(false);
|
||||
text = original.replaceAll("order", "placedOrder");
|
||||
state = { ...state, word: "placedOrder", head: 14 };
|
||||
expect(renameFinished(step)).toBe(false);
|
||||
document.querySelector(".cm-lsp-hint")!.remove();
|
||||
expect(renameFinished(step)).toBe(true);
|
||||
expect(renameFinished({ ...step, answer: null })).toBe(false);
|
||||
state = { ...state, path: "/p/other.java" };
|
||||
expect(renameFinished(step)).toBe(false);
|
||||
});
|
||||
|
||||
test("preview requires the matching title, files and counts without applying text", () => {
|
||||
fixture();
|
||||
const preview: RenameStep = { ...step, mode: "preview" };
|
||||
document.body.insertAdjacentHTML(
|
||||
"beforeend",
|
||||
'<div role="dialog"><h2>Rename \'order\' to \'placedOrder\'</h2><ul data-testid="refactor-preview"><li><span title="A.java">A.java</span><span>2 occurrences</span></li></ul></div>',
|
||||
);
|
||||
expect(renameFinished(preview)).toBe(true);
|
||||
expect(renameFinished(step)).toBe(false);
|
||||
text = original.replaceAll("order", "placedOrder");
|
||||
expect(renameFinished(preview)).toBe(false);
|
||||
text = original;
|
||||
document.querySelector("[title]")!.setAttribute("title", "other.java");
|
||||
expect(renameFinished(preview)).toBe(false);
|
||||
});
|
||||
|
||||
test("incomplete responses retain warning identity instead of claiming local application", () => {
|
||||
fixture();
|
||||
text = original.replaceAll("order", "placedOrder");
|
||||
state = { ...state, word: "placedOrder", head: 14 };
|
||||
expect(renameFinished({ ...step, answer: { ...answer, incomplete: "indexing" } })).toBe(false);
|
||||
const incomplete: Renamed = { ...answer, incomplete: "indexing" };
|
||||
expect(renameResult("/p", incomplete, "placedOrder")).toEqual({
|
||||
results: { changes: ["A.java:1", "A.java:2"] },
|
||||
degraded: "Rename may be incomplete: indexing",
|
||||
});
|
||||
text = original;
|
||||
document.body.insertAdjacentHTML(
|
||||
"beforeend",
|
||||
'<div role="dialog"><h2>Rename \'order\' to \'placedOrder\'</h2><div role="note">May be incomplete: the language server is still indexing</div><ul data-testid="refactor-preview"><li><span title="A.java">A.java</span><span>2 occurrences</span></li></ul></div>',
|
||||
);
|
||||
expect(renameFinished({ ...step, answer: incomplete })).toBe(true);
|
||||
expect(renameResult("/p", answer, "placedOrder").degraded).toBeNull();
|
||||
});
|
||||
|
||||
test("an incomplete empty answer reports degradation only with its warning, never a terminal error", () => {
|
||||
fixture();
|
||||
const incomplete: Renamed = { edit: { files: [] }, incomplete: "indexing" };
|
||||
document.body.insertAdjacentHTML("beforeend", '<span class="cm-lsp-hint">Rename failed</span>');
|
||||
expect(renameFinished({ ...step, answer: incomplete })).toBe(false);
|
||||
document.querySelector(".cm-lsp-hint")!.textContent = "May be incomplete: the language server is still indexing";
|
||||
expect(renameFinished({ ...step, answer: incomplete })).toBe(true);
|
||||
expect(renameResult("/p", incomplete, "placedOrder").degraded).toBe("Rename may be incomplete: indexing");
|
||||
expect(renameFinished({ ...step, answer: { edit: { files: [] }, incomplete: null } })).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
import { normalizePath } from "./scenario.ts";
|
||||
import type { Renamed } from "../../src/bindings.ts";
|
||||
|
||||
export interface RenameStep {
|
||||
scope: number;
|
||||
root: string;
|
||||
path: string;
|
||||
original: string;
|
||||
oldName: string;
|
||||
newName: string;
|
||||
mode: "preview" | "applied";
|
||||
answer: Renamed | null;
|
||||
}
|
||||
|
||||
/** Self-contained: track serializes this check into the measured page. */
|
||||
export function renameFinished(arg: RenameStep): boolean {
|
||||
const answer = arg.answer;
|
||||
if (!answer || document.querySelector(".cm-lsp-inplace")) return false;
|
||||
type ProbeWindow = {
|
||||
__openideaPerf: {
|
||||
editorText(): string | null;
|
||||
editorState(): { path: string; head: number; word: string } | null;
|
||||
};
|
||||
};
|
||||
const w = window as unknown as ProbeWindow;
|
||||
const probe = w.__openideaPerf;
|
||||
const state = probe.editorState();
|
||||
const text = probe.editorText();
|
||||
if (state?.path !== arg.path || text === null) return false;
|
||||
if (!answer.edit.files.length) {
|
||||
return (
|
||||
text === arg.original &&
|
||||
!!answer.incomplete &&
|
||||
/may be incomplete/i.test(document.querySelector(".cm-lsp-hint:not([data-pending])")?.textContent ?? "")
|
||||
);
|
||||
}
|
||||
if (document.querySelector(".cm-lsp-hint:not([data-pending]):not(.cm-lsp-inplace-hint)")) return false;
|
||||
const preview = document.querySelector('[data-testid="refactor-preview"]');
|
||||
if (arg.mode === "preview" || answer.incomplete) {
|
||||
if (!preview || text !== arg.original) return false;
|
||||
const dialog = preview.closest('[role="dialog"]');
|
||||
const title = dialog?.querySelector('h1, h2, [data-slot="dialog-title"]')?.textContent ?? dialog?.textContent ?? "";
|
||||
if (!title.includes(`'${arg.oldName}'`) || !title.includes(`'${arg.newName}'`)) return false;
|
||||
if (answer.incomplete && !dialog?.querySelector('[role="note"]')) return false;
|
||||
const entries = new Map<string, { edits: number; operation?: string; to?: string }>();
|
||||
for (const file of answer.edit.files) {
|
||||
const path = file.kind === "rename" ? file.from : file.path;
|
||||
let entry = entries.get(path);
|
||||
if (!entry) entries.set(path, (entry = { edits: 0 }));
|
||||
if (file.kind === "edit") entry.edits += file.edits.length;
|
||||
else {
|
||||
entry.operation = file.kind;
|
||||
if (file.kind === "rename") entry.to = file.to;
|
||||
}
|
||||
}
|
||||
const expected = [...entries].filter(([, entry]) => entry.edits || entry.operation);
|
||||
const rows = [...preview.querySelectorAll("li")];
|
||||
return (
|
||||
rows.length === expected.length &&
|
||||
expected.every(([path, entry]) => {
|
||||
const relative = path.startsWith(arg.root + "/") ? path.slice(arg.root.length + 1) : path;
|
||||
const row = rows.find((row) =>
|
||||
[...row.querySelectorAll("[title]")].some((el) => el.getAttribute("title") === relative),
|
||||
);
|
||||
if (!row) return false;
|
||||
const count = `${entry.edits} ${entry.edits === 1 ? "occurrence" : "occurrences"}`;
|
||||
if (entry.edits && ![...row.querySelectorAll("span")].some((el) => el.textContent === count)) return false;
|
||||
if (entry.operation === "rename")
|
||||
return !!entry.to && !!row.textContent?.includes(`→ ${entry.to.slice(entry.to.lastIndexOf("/") + 1)}`);
|
||||
return !entry.operation || !!row.textContent?.includes(entry.operation);
|
||||
})
|
||||
);
|
||||
}
|
||||
if (preview || answer.preview || answer.edit.files.some((file) => file.kind !== "edit" || file.path !== arg.path))
|
||||
return false;
|
||||
const edits = answer.edit.files.flatMap((file) => (file.kind === "edit" ? file.edits : []));
|
||||
if (!edits.length) return false;
|
||||
const lines = arg.original.split("\n");
|
||||
const offset = (pos: { line: number; col: number }) => {
|
||||
if (pos.line < 0 || pos.line >= lines.length || pos.col < 0 || pos.col > lines[pos.line]!.length) return -1;
|
||||
let at = pos.col;
|
||||
for (let i = 0; i < pos.line; i++) at += lines[i]!.length + 1;
|
||||
return at;
|
||||
};
|
||||
const changes = edits
|
||||
.map((edit) => ({ from: offset(edit.range.start), to: offset(edit.range.end), text: edit.new_text }))
|
||||
.sort((a, b) => b.from - a.from);
|
||||
let expected = arg.original;
|
||||
let previous = expected.length;
|
||||
for (const change of changes) {
|
||||
if (change.from < 0 || change.to < change.from || change.to > previous) return false;
|
||||
expected = expected.slice(0, change.from) + change.text + expected.slice(change.to);
|
||||
previous = change.from;
|
||||
}
|
||||
// Exact edit application excludes the temporary, optimistically typed in-place name.
|
||||
return (
|
||||
expected !== arg.original &&
|
||||
text === expected &&
|
||||
state.word === arg.newName &&
|
||||
state.head >= 0 &&
|
||||
state.head <= text.length
|
||||
);
|
||||
}
|
||||
|
||||
export function renameResult(root: string, answer: Renamed, newName: string) {
|
||||
const changes = answer.edit.files.flatMap((file) =>
|
||||
file.kind === "edit"
|
||||
? file.edits.flatMap((edit) =>
|
||||
edit.new_text
|
||||
.split("\n")
|
||||
.flatMap((line, i) =>
|
||||
line.includes(newName) ? [`${normalizePath(root, file.path)}:${edit.range.start.line + i + 1}`] : [],
|
||||
),
|
||||
)
|
||||
: [],
|
||||
);
|
||||
return {
|
||||
results: { changes },
|
||||
degraded: answer.incomplete ? `Rename may be incomplete: ${answer.incomplete}` : null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { planSelection } from "./selection.ts";
|
||||
|
||||
const scenarios = [
|
||||
{ id: "java.a.one", title: "One", conditions: ["warm", "cold"] as const },
|
||||
{ id: "java.b.two", title: "Two", conditions: ["warm"] as const },
|
||||
];
|
||||
|
||||
describe("eval workload planning", () => {
|
||||
test("full and deduplicated subset plans are independent of collected outcomes", () => {
|
||||
expect(planSelection(scenarios, {}).expected).toHaveLength(3);
|
||||
const plan = planSelection(scenarios, { only: " java.a, java.a.one ", condition: "cold" });
|
||||
expect(plan.selected).toEqual([scenarios[0]]);
|
||||
expect(plan.expected).toEqual([{ scenario: "java.a.one", title: "One", condition: "cold" }]);
|
||||
});
|
||||
test.each([
|
||||
{ condition: "hot" },
|
||||
{ only: "java.a,unknown" },
|
||||
{ only: "" },
|
||||
{ only: "java.a," },
|
||||
{ only: "java.b", condition: "cold" },
|
||||
{ only: "java.a,java.b", condition: "cold" },
|
||||
{ runs: "0" },
|
||||
{ runs: "NaN" },
|
||||
{ runs: "Infinity" },
|
||||
{ runs: "1.5" },
|
||||
{ runs: "9007199254740992" },
|
||||
{ warmup: "-1" },
|
||||
{ warmup: "Infinity" },
|
||||
{ warmup: "0.5" },
|
||||
])("rejects invalid selection/counts %j", (args) => {
|
||||
expect(() => planSelection(scenarios, args)).toThrow();
|
||||
});
|
||||
test("rejects zero work", () => {
|
||||
expect(() => planSelection([], {})).toThrow(/no compatible/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
import { countOption } from "../../perf/lib/options.ts";
|
||||
import type { Condition } from "./verdict.ts";
|
||||
|
||||
interface Scenario {
|
||||
id: string;
|
||||
title: string;
|
||||
conditions: readonly Condition[];
|
||||
}
|
||||
|
||||
export interface ExpectedOutcome {
|
||||
scenario: string;
|
||||
title: string;
|
||||
condition: Condition;
|
||||
}
|
||||
|
||||
export function planSelection<T extends Scenario>(
|
||||
scenarios: readonly T[],
|
||||
args: { only?: string; condition?: string; runs?: string; warmup?: string },
|
||||
) {
|
||||
const runs = countOption("runs", args.runs ?? "3", 1);
|
||||
const warmup = countOption("warmup", args.warmup ?? "1", 0);
|
||||
if (args.condition !== undefined && args.condition !== "warm" && args.condition !== "cold")
|
||||
throw new Error("--condition must be warm or cold");
|
||||
const condition = args.condition as Condition | undefined;
|
||||
const selectors = args.only?.split(",").map((s) => s.trim());
|
||||
const matches = (s: T, selector: string) => s.id === selector || s.id.startsWith(`${selector}.`);
|
||||
for (const selector of selectors ?? []) {
|
||||
if (!selector) throw new Error("--only contains an empty selector");
|
||||
const matching = scenarios.filter((s) => matches(s, selector));
|
||||
if (!matching.length) throw new Error(`no scenario matches ${selector} (see --list)`);
|
||||
if (!matching.some((s) => s.conditions.some((c) => !condition || c === condition)))
|
||||
throw new Error(`no compatible scenario for selector ${selector} under ${condition}`);
|
||||
}
|
||||
const selected = scenarios.filter((s) => !selectors || selectors.some((selector) => matches(s, selector)));
|
||||
const expected: ExpectedOutcome[] = selected.flatMap((s) =>
|
||||
s.conditions
|
||||
.filter((c) => !condition || c === condition)
|
||||
.map((c) => ({ scenario: s.id, title: s.title, condition: c })),
|
||||
);
|
||||
if (!expected.length) throw new Error("no compatible scenarios selected");
|
||||
return { selected, expected, runs, warmup, condition };
|
||||
}
|
||||
+71
-51
@@ -8,9 +8,12 @@ import { homedir, tmpdir } from "node:os";
|
||||
import { join, sep } from "node:path";
|
||||
import type { Browser, BrowserContext, Page } from "@playwright/test";
|
||||
import { installInPage } from "../../perf/lib/inpage.ts";
|
||||
import { withDeadline } from "../../perf/lib/lifecycle.ts";
|
||||
import { Server } from "../../smoke/server.ts";
|
||||
import { copyProject } from "./projects.ts";
|
||||
import { installTauriShim, type ShimWindow } from "../../smoke/tauriShim.ts";
|
||||
import { RenameObservations } from "./commandObservation.ts";
|
||||
import { openFileMeasured } from "../../perf/lib/editing.ts";
|
||||
|
||||
/** How long after its answer Show Usages invoked again widens it (`HINT_MS` in code-insight/usages.ts), plus a margin. */
|
||||
const SHOW_USAGES_HINT_MS = 4200;
|
||||
@@ -125,11 +128,11 @@ export class Session {
|
||||
private server!: Server;
|
||||
private relayQueue: Promise<unknown> = Promise.resolve();
|
||||
private pendingNotes: Array<[kind: "channel" | "event", a: unknown, b: unknown]> = [];
|
||||
/** The last answer to each command that succeeded, for what the UI only summarises. */
|
||||
readonly lastAnswer = new Map<string, unknown>();
|
||||
readonly renameObservations = new RenameObservations();
|
||||
/** Editors changed by a scenario: path → the text to put back. */
|
||||
private readonly edited = new Map<string, string>();
|
||||
readonly pageErrors: string[] = [];
|
||||
private closed = false;
|
||||
|
||||
private constructor(opts: SessionOptions) {
|
||||
this.root = join(opts.workDir, "projects", opts.fixture);
|
||||
@@ -138,46 +141,71 @@ export class Session {
|
||||
|
||||
static async start(browser: Browser, url: string, repoRoot: string, opts: SessionOptions): Promise<Session> {
|
||||
const s = new Session(opts);
|
||||
copyProject(repoRoot, opts.fixture, s.root);
|
||||
if (!opts.keepHome) rmSync(s.home, { recursive: true, force: true, maxRetries: 5, retryDelay: 500 });
|
||||
mkdirSync(s.home, { recursive: true });
|
||||
try {
|
||||
await withDeadline(s.initialize(browser, url, repoRoot, opts), 5 * 60_000, "session startup");
|
||||
return s;
|
||||
} catch (error) {
|
||||
try {
|
||||
await s.close();
|
||||
} catch (cleanup) {
|
||||
throw new AggregateError([error, cleanup], "Session startup and cleanup failed", { cause: cleanup });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
s.server = await Server.start(s.home, opts.log ? { OPENIDEA_LOG: opts.log } : {});
|
||||
s.server.onEvent = (target, event) => {
|
||||
private async initialize(browser: Browser, url: string, repoRoot: string, opts: SessionOptions) {
|
||||
copyProject(repoRoot, opts.fixture, this.root);
|
||||
if (!opts.keepHome) rmSync(this.home, { recursive: true, force: true, maxRetries: 5, retryDelay: 500 });
|
||||
mkdirSync(this.home, { recursive: true });
|
||||
|
||||
this.server = await Server.start(this.home, opts.log ? { OPENIDEA_LOG: opts.log } : {});
|
||||
if (this.closed) {
|
||||
await this.server.close();
|
||||
throw new Error("session already closed");
|
||||
}
|
||||
this.server.onEvent = (target, event) => {
|
||||
if (target !== null && target !== "main") return;
|
||||
s.timeline.onEvent(event as { type: string });
|
||||
s.notify("event", target, event);
|
||||
this.timeline.onEvent(event as { type: string });
|
||||
this.notify("event", target, event);
|
||||
};
|
||||
s.server.onChannel = (id, data) => s.notify("channel", id, data);
|
||||
await s.server.invoke("project_trust_set", { path: s.root, trusted: true, folder: false });
|
||||
this.server.onChannel = (id, data) => this.notify("channel", id, data);
|
||||
await this.server.invoke("project_trust_set", { path: this.root, trusted: true, folder: false });
|
||||
|
||||
s.context = await browser.newContext({
|
||||
this.context = await browser.newContext({
|
||||
baseURL: url,
|
||||
viewport: { width: 1400, height: 900 },
|
||||
deviceScaleFactor: 1,
|
||||
});
|
||||
await s.context.exposeFunction("__openideaRelay", async (cmd: string, args: unknown) => {
|
||||
const start = s.timeline.now();
|
||||
s.timeline.inFlight++;
|
||||
const answered = s.timeline.sent(cmd, start);
|
||||
if (this.closed) {
|
||||
await withDeadline(this.context.close(), 5000, "late session context close");
|
||||
throw new Error("session already closed");
|
||||
}
|
||||
await this.context.exposeFunction("__openideaEvalRenameAnswer", (scope: number) =>
|
||||
this.renameObservations.wait(scope),
|
||||
);
|
||||
await this.context.exposeFunction("__openideaRelay", async (cmd: string, args: unknown) => {
|
||||
const start = this.timeline.now();
|
||||
this.timeline.inFlight++;
|
||||
const answered = this.timeline.sent(cmd, start);
|
||||
try {
|
||||
const value = await s.server.invoke(cmd, args);
|
||||
s.record(cmd, start, true, value);
|
||||
const value = await this.renameObservations.invoke(cmd, args, () => this.server.invoke(cmd, args));
|
||||
this.record(cmd, start, true, value);
|
||||
return { ok: value ?? null };
|
||||
} catch (err) {
|
||||
s.record(cmd, start, false, err);
|
||||
this.record(cmd, start, false, err);
|
||||
return { err };
|
||||
} finally {
|
||||
answered();
|
||||
s.timeline.inFlight--;
|
||||
s.timeline.lastActivity = s.timeline.now();
|
||||
this.timeline.inFlight--;
|
||||
this.timeline.lastActivity = this.timeline.now();
|
||||
}
|
||||
});
|
||||
await s.context.addInitScript(installTauriShim, {
|
||||
await this.context.addInitScript(installTauriShim, {
|
||||
directories: { 12: tmpdir(), 21: homedir() },
|
||||
sep: sep as "/" | "\\",
|
||||
});
|
||||
await s.context.addInitScript(() => {
|
||||
await this.context.addInitScript(() => {
|
||||
const w = window as unknown as ShimWindow & {
|
||||
__openideaRelay(cmd: string, args: unknown): Promise<{ ok: unknown } | { err: unknown }>;
|
||||
};
|
||||
@@ -187,25 +215,24 @@ export class Session {
|
||||
return r.ok;
|
||||
};
|
||||
});
|
||||
await s.context.addInitScript(installInPage);
|
||||
s.page = await s.context.newPage();
|
||||
s.page.on("pageerror", (e) => s.pageErrors.push(e.message));
|
||||
await this.context.addInitScript(installInPage);
|
||||
this.page = await this.context.newPage();
|
||||
this.page.on("pageerror", (e) => this.pageErrors.push(e.message));
|
||||
if (process.env.OPENIDEA_EVAL_DEBUG) {
|
||||
s.page.on("console", (m) => console.log("[page]", m.type(), m.text().slice(0, 300)));
|
||||
s.page.on("pageerror", (e) => console.log("[pageerror]", e.message));
|
||||
this.page.on("console", (m) => console.log("[page]", m.type(), m.text().slice(0, 300)));
|
||||
this.page.on("pageerror", (e) => console.log("[pageerror]", e.message));
|
||||
}
|
||||
|
||||
await s.page.goto("/");
|
||||
await s.page.waitForFunction(() => "__openideaPerf" in window && "__openideaShim" in window);
|
||||
await s.page.evaluate(
|
||||
await this.page.goto("/");
|
||||
await this.page.waitForFunction(() => "__openideaPerf" in window && "__openideaShim" in window);
|
||||
await this.page.evaluate(
|
||||
(root) =>
|
||||
(window as unknown as { __openideaPerf: { openProject(p: string): Promise<void> } }).__openideaPerf.openProject(
|
||||
root,
|
||||
),
|
||||
s.root,
|
||||
this.root,
|
||||
);
|
||||
await s.page.waitForSelector('[role="tree"][aria-label="Project"] [role="treeitem"]', { timeout: 60_000 });
|
||||
return s;
|
||||
await this.page.waitForSelector('[role="tree"][aria-label="Project"] [role="treeitem"]', { timeout: 60_000 });
|
||||
}
|
||||
|
||||
private record(cmd: string, start: number, ok: boolean, value: unknown) {
|
||||
@@ -221,7 +248,6 @@ export class Session {
|
||||
? (value as { locations: unknown[] }).locations.length
|
||||
: undefined;
|
||||
this.timeline.calls.push({ cmd, start, ms: this.timeline.now() - start, ok, bytes, items });
|
||||
if (ok) this.lastAnswer.set(cmd, value);
|
||||
}
|
||||
|
||||
/** Delivers server notifications in order, batched per turn of the event loop. */
|
||||
@@ -263,20 +289,7 @@ export class Session {
|
||||
/** Opens `rel` (project-relative, or a URI) in an editor and waits until its text is shown. */
|
||||
async openFile(rel: string) {
|
||||
const path = /^[a-z]+:\/\//.test(rel) ? rel : join(this.root, rel);
|
||||
const bare = rel.replace(/[?#].*$/, "");
|
||||
const name = bare.slice(bare.lastIndexOf("/") + 1);
|
||||
await this.page.evaluate(
|
||||
async ({ path, name }) => {
|
||||
const w = window as unknown as {
|
||||
__perf: { waitShown(s: string, t: string | null, ms: number): Promise<number> };
|
||||
__openideaPerf: { openFile(p: string): Promise<void> };
|
||||
};
|
||||
await w.__openideaPerf.openFile(path);
|
||||
await w.__perf.waitShown('[role="tab"][aria-selected="true"]', name, 30_000);
|
||||
await w.__perf.waitShown(".cm-content .cm-line", null, 30_000);
|
||||
},
|
||||
{ path, name },
|
||||
);
|
||||
await this.page.evaluate(openFileMeasured, { path, timeoutMs: 30_000 });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -399,6 +412,7 @@ export class Session {
|
||||
|
||||
/** Back to an idle workbench: popups dismissed, edits undone, every editor closed. */
|
||||
async reset() {
|
||||
this.renameObservations.reset();
|
||||
// Show Usages invoked again within 4 s of its hint widens the previous search: wait it out.
|
||||
const since = await this.page.evaluate(() => {
|
||||
const at = (window as unknown as { __openideaEvalShowUsagesAt?: number }).__openideaEvalShowUsagesAt;
|
||||
@@ -419,7 +433,13 @@ export class Session {
|
||||
}
|
||||
|
||||
async close() {
|
||||
await this.context?.close().catch(() => {});
|
||||
await this.server?.close();
|
||||
this.closed = true;
|
||||
this.renameObservations.reset();
|
||||
const results = await Promise.allSettled([
|
||||
this.context ? withDeadline(this.context.close(), 5000, "session context close") : Promise.resolve(),
|
||||
this.server?.close(),
|
||||
]);
|
||||
const errors = results.flatMap((r) => (r.status === "rejected" ? [r.reason] : []));
|
||||
if (errors.length) throw new AggregateError(errors, "Session cleanup failed");
|
||||
}
|
||||
}
|
||||
@@ -20,10 +20,11 @@ const outcome = (o: Partial<Outcome>): Outcome => ({
|
||||
});
|
||||
|
||||
const repro = () => "pnpm eval --only usages";
|
||||
const expected = [{ scenario: "usages", title: "Show Usages", condition: "warm" as const }];
|
||||
|
||||
describe("judge", () => {
|
||||
test("within budgets and matching the golden passes", () => {
|
||||
const [v] = judge([outcome({})], budgets, null, repro);
|
||||
const [v] = judge([outcome({})], budgets, null, repro, expected);
|
||||
expect(v).toMatchObject({ id: "usages@warm", status: "pass", failures: [], repro: "pnpm eval --only usages" });
|
||||
});
|
||||
|
||||
@@ -39,6 +40,7 @@ describe("judge", () => {
|
||||
budgets,
|
||||
null,
|
||||
repro,
|
||||
expected,
|
||||
);
|
||||
expect(v!.status).toBe("fail");
|
||||
expect(v!.failures.map((f) => [f.kind, f.metric])).toEqual([
|
||||
@@ -52,21 +54,29 @@ describe("judge", () => {
|
||||
|
||||
test("condition budgets override scenario budgets", () => {
|
||||
const [v] = judge(
|
||||
[outcome({ condition: "cold", metrics: { feedbackMs: 40, completeMs: 5000, recall: 1, precision: 1 } })],
|
||||
[
|
||||
outcome({
|
||||
condition: "cold",
|
||||
metrics: { feedbackMs: 40, firstResultMs: 300, completeMs: 5000, recall: 1, precision: 1 },
|
||||
}),
|
||||
],
|
||||
budgets,
|
||||
null,
|
||||
repro,
|
||||
[{ ...expected[0]!, condition: "cold" }],
|
||||
);
|
||||
expect(v!.status).toBe("pass");
|
||||
});
|
||||
|
||||
test("timeouts and missing goldens fail", () => {
|
||||
const [timedOut, noGolden] = judge(
|
||||
[outcome({ error: "find.showUsages timed out after 30000 ms" }), outcome({ comparison: undefined })],
|
||||
const [timedOut] = judge(
|
||||
[outcome({ error: "find.showUsages timed out after 30000 ms" })],
|
||||
budgets,
|
||||
null,
|
||||
repro,
|
||||
expected,
|
||||
);
|
||||
const [noGolden] = judge([outcome({ comparison: undefined })], budgets, null, repro, expected);
|
||||
expect(timedOut!.failures[0]!.kind).toBe("timeout");
|
||||
expect(noGolden!.failures[0]!.kind).toBe("golden");
|
||||
});
|
||||
@@ -78,9 +88,41 @@ describe("judge", () => {
|
||||
budgets,
|
||||
baseline,
|
||||
repro,
|
||||
expected,
|
||||
);
|
||||
expect(v!.status).toBe("pass");
|
||||
expect(v!.regressions.map((r) => r.metric)).toEqual(["completeMs"]);
|
||||
expect(table([v!], false)).toContain("regressed: completeMs");
|
||||
});
|
||||
|
||||
test("missing, duplicate, unexpected and empty workloads fail", () => {
|
||||
for (const actual of [[], [outcome({}), outcome({})], [outcome({ scenario: "other" })]]) {
|
||||
expect(judge(actual, budgets, null, repro, expected).some((v) => v.status === "fail")).toBe(true);
|
||||
}
|
||||
expect(judge([], budgets, null, repro, []).some((v) => v.status === "fail")).toBe(true);
|
||||
});
|
||||
|
||||
test.each(["feedbackMs", "firstResultMs", "completeMs", "recall", "precision"])("requires finite %s", (metric) => {
|
||||
for (const value of [undefined, NaN, Infinity, -Infinity]) {
|
||||
const o = outcome({});
|
||||
if (value === undefined) delete o.metrics[metric];
|
||||
else o.metrics[metric] = value;
|
||||
const [v] = judge([o], budgets, null, repro, expected);
|
||||
expect(v!.status).toBe("fail");
|
||||
expect(v!.failures.some((f) => f.metric === metric)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test("requires scenario budget metrics and rejects nonfinite unbudgeted values", () => {
|
||||
const custom = { ...budgets, scenarios: { usages: { readyMs: 100 } } };
|
||||
expect(judge([outcome({})], custom, null, repro, expected)[0]!.status).toBe("fail");
|
||||
const o = outcome({});
|
||||
o.metrics.extraMs = NaN;
|
||||
expect(judge([o], budgets, null, repro, expected)[0]!.status).toBe("fail");
|
||||
});
|
||||
|
||||
test("a clean empty result set with a perfect comparison passes", () => {
|
||||
const o = outcome({ results: { problems: [] } });
|
||||
expect(judge([o], budgets, null, repro, expected)[0]!.status).toBe("pass");
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import type { Comparison, ResultSet } from "./golden.ts";
|
||||
import type { ExpectedOutcome } from "./selection.ts";
|
||||
|
||||
export type Condition = "warm" | "cold";
|
||||
|
||||
@@ -97,15 +98,42 @@ export function judge(
|
||||
budgets: Budgets,
|
||||
baseline: Outcome[] | null,
|
||||
repro: (o: Outcome) => string,
|
||||
expected: readonly ExpectedOutcome[],
|
||||
): Verdict[] {
|
||||
const base = new Map((baseline ?? []).map((o) => [idOf(o), o.metrics]));
|
||||
return outcomes.map((o) => {
|
||||
const expectedIds = new Set(expected.map(idOf));
|
||||
const counts = new Map<string, number>();
|
||||
for (const o of outcomes) counts.set(idOf(o), (counts.get(idOf(o)) ?? 0) + 1);
|
||||
const work: Outcome[] = [...outcomes];
|
||||
for (const e of expected) {
|
||||
if (!counts.has(idOf(e))) work.push({ ...e, metrics: {}, samples: {}, error: "missing expected outcome" });
|
||||
}
|
||||
if (!work.length) work.push({ scenario: "workload", title: "Workload", condition: "warm", metrics: {}, samples: {} });
|
||||
return work.map((o) => {
|
||||
const failures: Failure[] = [];
|
||||
if (!expected.length) failures.push({ kind: "error", detail: "expected workload is empty" });
|
||||
if (expectedIds.size !== expected.length) failures.push({ kind: "error", detail: "duplicate expected outcome" });
|
||||
if (!expectedIds.has(idOf(o))) failures.push({ kind: "error", detail: "unexpected outcome" });
|
||||
if ((counts.get(idOf(o)) ?? 0) > 1) failures.push({ kind: "error", detail: "duplicate outcome" });
|
||||
if (o.error) failures.push({ kind: /timed out/.test(o.error) ? "timeout" : "error", detail: o.error });
|
||||
if (o.degraded) failures.push({ kind: "degraded", detail: o.degraded });
|
||||
if (o.results && !o.comparison && !o.error)
|
||||
failures.push({ kind: "golden", detail: "no golden recorded: run with --update-golden and review it" });
|
||||
for (const [metric, limit] of Object.entries(budgetsFor(budgets, o))) {
|
||||
const limits = budgetsFor(budgets, o);
|
||||
const required = new Set(["feedbackMs", "firstResultMs", "completeMs", ...Object.keys(limits)]);
|
||||
if (o.results || o.comparison) {
|
||||
required.add("recall");
|
||||
required.add("precision");
|
||||
}
|
||||
for (const metric of new Set([...required, ...Object.keys(o.metrics)])) {
|
||||
if (!Number.isFinite(o.metrics[metric]))
|
||||
failures.push({
|
||||
kind: lowerBound(metric) ? "correctness" : "budget",
|
||||
metric,
|
||||
detail: `${metric} is missing or nonfinite`,
|
||||
});
|
||||
}
|
||||
for (const [metric, limit] of Object.entries(limits)) {
|
||||
const value = o.metrics[metric];
|
||||
if (value === undefined || !Number.isFinite(value)) continue;
|
||||
const over = lowerBound(metric) ? value < limit : value > limit;
|
||||
|
||||
+193
-84
@@ -13,18 +13,29 @@
|
||||
* and target/eval/report.md. Timings go over the remote-development route (one stdio hop
|
||||
* more than Tauri IPC): compare them before / after on one machine, not as absolute numbers.
|
||||
*/
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import type { Server as HttpServer } from "node:http";
|
||||
import { join, resolve } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { chromium, type Browser } from "@playwright/test";
|
||||
import type { Browser } from "@playwright/test";
|
||||
import { buildBundle, serveBundle } from "../perf/lib/bundle.ts";
|
||||
import { median } from "../perf/lib/stats.ts";
|
||||
import { runCommand } from "../perf/lib/process.ts";
|
||||
import { closeServer, DeadlineError, launchBrowser, withDeadline, type BrowserSession } from "../perf/lib/lifecycle.ts";
|
||||
import {
|
||||
errorMessage,
|
||||
evidenceJson,
|
||||
metricErrors,
|
||||
summarizeMetrics,
|
||||
summarizeTrials,
|
||||
type Trial,
|
||||
} from "../perf/lib/trials.ts";
|
||||
import { JAVA_SCENARIOS } from "./java/scenarios.ts";
|
||||
import { compare, readGolden, toolDrift, writeGolden, type ResultSet } from "./lib/golden.ts";
|
||||
import { TimeoutError } from "./lib/measure.ts";
|
||||
import { fingerprint, pinnedTools, sameMachine, toolVersions, type Fingerprint } from "./lib/machine.ts";
|
||||
import type { EvalScenario } from "./lib/scenario.ts";
|
||||
import { planSelection } from "./lib/selection.ts";
|
||||
import { Session } from "./lib/session.ts";
|
||||
import { idOf, judge, loadBudgets, table, type Condition, type Diagnostics, type Outcome } from "./lib/verdict.ts";
|
||||
|
||||
@@ -56,25 +67,22 @@ if (args.list) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const runs = Math.max(1, Number(args.runs));
|
||||
const warmup = Math.max(0, Number(args.warmup));
|
||||
const only = args.only?.split(",").map((s) => s.trim());
|
||||
const conditionFilter = args.condition as Condition | undefined;
|
||||
const selected = SCENARIOS.filter((s) => !only || only.some((o) => s.id === o || s.id.startsWith(`${o}.`)));
|
||||
if (only && !selected.length) throw new Error(`no scenario matches ${args.only} (see --list)`);
|
||||
const { runs, warmup, selected, expected, condition: conditionFilter } = planSelection(SCENARIOS, args);
|
||||
const trialEvidence: Record<string, Array<Trial & { results?: ResultSet; degraded?: string | null }>> = {};
|
||||
const evalBudgets = loadBudgets(join(ROOT, "evals", "budgets.json"));
|
||||
|
||||
const goldenFile = (id: string) => join(ROOT, "evals", id.slice(0, id.indexOf(".")), "golden", `${id}.json`);
|
||||
|
||||
function serverBinary(): string {
|
||||
async function serverBinary(): Promise<string> {
|
||||
const exe = process.platform === "win32" ? ".exe" : "";
|
||||
const bin = process.env.OPENIDEA_SERVER ?? join(ROOT, "target", "release", `openidea-server${exe}`);
|
||||
if (!args["no-build"]) {
|
||||
console.log("Building openidea-server (cargo build --release)…");
|
||||
const r = spawnSync("cargo", ["build", "--release", "-p", "ide-host", "--bin", "openidea-server"], {
|
||||
await runCommand("cargo", ["build", "--release", "-p", "ide-host", "--bin", "openidea-server"], {
|
||||
cwd: ROOT,
|
||||
stdio: ["ignore", "ignore", "inherit"],
|
||||
timeoutMs: 60 * 60_000,
|
||||
label: "cargo build",
|
||||
});
|
||||
if (r.status !== 0) throw new Error("cargo build failed");
|
||||
}
|
||||
if (!existsSync(bin)) throw new Error(`no server at ${bin}; run without --no-build`);
|
||||
return bin;
|
||||
@@ -82,23 +90,12 @@ function serverBinary(): string {
|
||||
|
||||
function git(...a: string[]): string {
|
||||
try {
|
||||
return execFileSync("git", a, { cwd: ROOT, encoding: "utf8" }).trim();
|
||||
return execFileSync("git", a, { cwd: ROOT, encoding: "utf8", timeout: 10_000, killSignal: "SIGKILL" }).trim();
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** Medians of the timings; correctness from the worst run. */
|
||||
function summarize(samples: Array<Record<string, number>>): Record<string, number> {
|
||||
const keys = [...new Set(samples.flatMap((s) => Object.keys(s)))];
|
||||
return Object.fromEntries(
|
||||
keys.map((k) => {
|
||||
const values = samples.map((s) => s[k]!).filter(Number.isFinite);
|
||||
return [k, k === "recall" || k === "precision" ? Math.min(...values) : median(values)];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
interface Run {
|
||||
metrics: Record<string, number>;
|
||||
results?: ResultSet;
|
||||
@@ -157,8 +154,10 @@ function outcomeOf(
|
||||
scenario: scenario.id,
|
||||
title: scenario.title,
|
||||
condition,
|
||||
metrics: { ...summarize(samples), ...extra },
|
||||
samples: Object.fromEntries(Object.keys(samples[0] ?? {}).map((k) => [k, samples.map((x) => x[k]!)])),
|
||||
metrics: { ...summarizeMetrics(samples, ["recall", "precision"]), ...extra },
|
||||
samples: Object.fromEntries(
|
||||
[...new Set(samples.flatMap((s) => Object.keys(s)))].map((k) => [k, samples.map((x) => x[k]!)]),
|
||||
),
|
||||
results: done.at(-1)?.results,
|
||||
comparison: worst,
|
||||
drift: golden ? toolDrift(golden, tools) : undefined,
|
||||
@@ -172,21 +171,59 @@ async function runScenario(
|
||||
condition: Condition,
|
||||
browser: Browser,
|
||||
tools: Record<string, string>,
|
||||
progress: () => void,
|
||||
extra: Record<string, number> = {},
|
||||
): Promise<Outcome> {
|
||||
const total = condition === "cold" ? 1 : warmup + runs;
|
||||
const done: Run[] = [];
|
||||
process.stdout.write(`${idOf({ scenario: scenario.id, condition }).padEnd(52)} `);
|
||||
const attempts: Array<Trial & { results?: ResultSet; degraded?: string | null }> = (trialEvidence[
|
||||
idOf({ scenario: scenario.id, condition })
|
||||
] = []);
|
||||
let phase: Trial["phase"] = condition === "warm" && warmup ? "warmup" : "measured";
|
||||
let index = 1;
|
||||
try {
|
||||
for (let i = 0; i < total; i++) {
|
||||
const last = i === total - 1;
|
||||
const trace = args.trace && last ? join(OUT, "traces", `${scenario.id}@${condition}.json`) : null;
|
||||
const r = await measureOnce(s, scenario, trace, browser);
|
||||
if (condition === "cold" || i >= warmup) done.push(r);
|
||||
process.stdout.write(condition === "warm" && i < warmup ? "w" : ".");
|
||||
phase = condition === "warm" && i < warmup ? "warmup" : "measured";
|
||||
index = phase === "warmup" ? i + 1 : condition === "cold" ? 1 : i - warmup + 1;
|
||||
const r = await withDeadline(measureOnce(s, scenario, trace, browser), 5 * 60_000, `${scenario.id} trial`);
|
||||
const golden = readGolden(goldenFile(scenario.id));
|
||||
const comparison = golden && r.results ? compare(r.results, golden.groups) : undefined;
|
||||
const metrics = comparison
|
||||
? { ...r.metrics, recall: comparison.recall, precision: comparison.precision }
|
||||
: r.metrics;
|
||||
const limits = {
|
||||
...evalBudgets.defaults,
|
||||
...evalBudgets.scenarios[scenario.id],
|
||||
...evalBudgets.scenarios[idOf({ scenario: scenario.id, condition })],
|
||||
};
|
||||
const required = [
|
||||
"feedbackMs",
|
||||
"firstResultMs",
|
||||
"completeMs",
|
||||
...Object.keys(limits).filter((k) => k !== "recall" && k !== "precision"),
|
||||
];
|
||||
const errors = metricErrors(metrics, required);
|
||||
if (r.degraded) errors.push(r.degraded);
|
||||
if (comparison && (comparison.recall < 1 || comparison.precision < 1)) errors.push("result differs from golden");
|
||||
attempts.push({
|
||||
phase,
|
||||
index,
|
||||
status: errors.length ? "invalid" : "ok",
|
||||
metrics,
|
||||
results: r.results,
|
||||
degraded: r.degraded,
|
||||
error: errors.join("; ") || undefined,
|
||||
});
|
||||
if (phase === "measured") done.push(r);
|
||||
progress();
|
||||
process.stdout.write(errors.length ? "!" : phase === "warmup" ? "w" : ".");
|
||||
}
|
||||
console.log();
|
||||
const o = outcomeOf(scenario, condition, done, tools, extra);
|
||||
o.error = summarizeTrials(attempts, condition === "cold" ? 1 : runs).error;
|
||||
const slowest = done.reduce((a, b) => ((b.metrics.completeMs ?? 0) > (a.metrics.completeMs ?? 0) ? b : a));
|
||||
o.diagnostics = diagnose(
|
||||
s,
|
||||
@@ -199,6 +236,15 @@ async function runScenario(
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
const partial = e instanceof TimeoutError ? (e.partial as Record<string, number>) : {};
|
||||
const now = s.timeline.now();
|
||||
if (!attempts.some((t) => t.phase === phase && t.index === index))
|
||||
attempts.push({
|
||||
phase,
|
||||
index,
|
||||
status: e instanceof DeadlineError || e instanceof TimeoutError ? "timeout" : "error",
|
||||
metrics: partial,
|
||||
error: message,
|
||||
});
|
||||
progress();
|
||||
return {
|
||||
...outcomeOf(scenario, condition, done, tools, extra),
|
||||
metrics: { ...partial, ...extra },
|
||||
@@ -210,69 +256,113 @@ async function runScenario(
|
||||
|
||||
async function main() {
|
||||
mkdirSync(join(OUT, "traces"), { recursive: true });
|
||||
process.env.OPENIDEA_SERVER = serverBinary();
|
||||
if (!args["no-build"]) buildBundle(ROOT, DIST);
|
||||
if (!existsSync(join(DIST, "index.html"))) throw new Error(`no build in ${DIST}; run without --no-build`);
|
||||
|
||||
const tools = toolVersions();
|
||||
const pins = pinnedTools(join(ROOT, ".github", "tool-versions.env"));
|
||||
const machine: Fingerprint = fingerprint();
|
||||
for (const [k, want] of Object.entries(pins)) {
|
||||
if (tools[k] !== undefined && !tools[k]!.startsWith(want))
|
||||
console.warn(`warning: ${k} ${tools[k]} is not the pinned ${want} (.github/tool-versions.env)`);
|
||||
}
|
||||
|
||||
const { server, url } = await serveBundle(DIST);
|
||||
const browser = await chromium.launch({
|
||||
channel: args.channel === "chromium" ? undefined : args.channel,
|
||||
headless: !args.headed,
|
||||
args: [
|
||||
"--disable-renderer-backgrounding",
|
||||
"--disable-background-timer-throttling",
|
||||
"--disable-backgrounding-occluded-windows",
|
||||
],
|
||||
});
|
||||
const outcomes: Outcome[] = [];
|
||||
const infrastructure: string[] = [];
|
||||
const machine: Fingerprint = fingerprint();
|
||||
const date = new Date().toISOString();
|
||||
let tools: Record<string, string> = {};
|
||||
let pins: Record<string, string> = {};
|
||||
let runtime: BrowserSession | undefined;
|
||||
let server: HttpServer | undefined;
|
||||
let browserVersion = "not started";
|
||||
const progress = () => {
|
||||
writeFileSync(
|
||||
join(OUT, "report.json"),
|
||||
evidenceJson({
|
||||
meta: { date, runs, warmup, machine, tools, pins },
|
||||
status: "running",
|
||||
pass: false,
|
||||
expected,
|
||||
outcomes,
|
||||
trials: trialEvidence,
|
||||
}),
|
||||
);
|
||||
};
|
||||
progress();
|
||||
writeFileSync(join(OUT, "report.md"), "# OpenIDEA eval report\n\nRun in progress; no passing verdict.\n");
|
||||
try {
|
||||
// Warm: one session per fixture, its project imported once, scenarios one after another.
|
||||
process.env.OPENIDEA_SERVER = await serverBinary();
|
||||
if (!args["no-build"]) await buildBundle(ROOT, DIST);
|
||||
if (!existsSync(join(DIST, "index.html"))) throw new Error(`no build in ${DIST}; run without --no-build`);
|
||||
tools = toolVersions();
|
||||
pins = pinnedTools(join(ROOT, ".github", "tool-versions.env"));
|
||||
for (const [k, want] of Object.entries(pins)) {
|
||||
if (tools[k] !== undefined && !tools[k]!.startsWith(want))
|
||||
console.warn(`warning: ${k} ${tools[k]} is not the pinned ${want} (.github/tool-versions.env)`);
|
||||
}
|
||||
const serving = await serveBundle(DIST);
|
||||
server = serving.server;
|
||||
runtime = await launchBrowser({
|
||||
channel: args.channel === "chromium" ? undefined : args.channel,
|
||||
headless: !args.headed,
|
||||
args: [
|
||||
"--disable-renderer-backgrounding",
|
||||
"--disable-background-timer-throttling",
|
||||
"--disable-backgrounding-occluded-windows",
|
||||
],
|
||||
});
|
||||
const browser = runtime.browser;
|
||||
browserVersion = browser.version();
|
||||
const warm = selected.filter(
|
||||
(s) => s.conditions.includes("warm") && (!conditionFilter || conditionFilter === "warm"),
|
||||
);
|
||||
for (const fixture of [...new Set(warm.map((s) => s.fixture))]) {
|
||||
const scenarios = warm.filter((s) => s.fixture === fixture);
|
||||
const s = await Session.start(browser, url, ROOT, { fixture, workDir: join(OUT, "warm"), keepHome: false });
|
||||
const s = await Session.start(browser, serving.url, ROOT, {
|
||||
fixture,
|
||||
workDir: join(OUT, "warm"),
|
||||
keepHome: false,
|
||||
});
|
||||
try {
|
||||
await scenarios[0]!.setup(s);
|
||||
await withDeadline(scenarios[0]!.setup(s), 5 * 60_000, "warm fixture setup");
|
||||
await s.settle(scenarios[0]!.server);
|
||||
for (const scenario of scenarios) outcomes.push(await runScenario(s, scenario, "warm", browser, tools));
|
||||
for (const scenario of scenarios) {
|
||||
const outcome = await runScenario(s, scenario, "warm", browser, tools, progress);
|
||||
outcomes.push(outcome);
|
||||
progress();
|
||||
if (outcome.error) break;
|
||||
}
|
||||
} finally {
|
||||
await s.close();
|
||||
}
|
||||
}
|
||||
// Cold: a fresh server and project import per scenario, measured once after the import settles.
|
||||
const cold = selected.filter(
|
||||
(s) => s.conditions.includes("cold") && (!conditionFilter || conditionFilter === "cold"),
|
||||
);
|
||||
for (const scenario of cold) {
|
||||
const started = Date.now();
|
||||
const s = await Session.start(browser, url, ROOT, {
|
||||
const s = await Session.start(browser, serving.url, ROOT, {
|
||||
fixture: scenario.fixture,
|
||||
workDir: join(OUT, "cold"),
|
||||
keepHome: false,
|
||||
});
|
||||
try {
|
||||
await scenario.setup(s);
|
||||
await withDeadline(scenario.setup(s), 5 * 60_000, "cold fixture setup");
|
||||
await s.settle(scenario.server);
|
||||
// Settling waits out the quiet period at its end.
|
||||
const readyMs = Date.now() - started - 2000;
|
||||
outcomes.push(await runScenario(s, scenario, "cold", browser, tools, { readyMs }));
|
||||
outcomes.push(await runScenario(s, scenario, "cold", browser, tools, progress, { readyMs }));
|
||||
progress();
|
||||
} finally {
|
||||
await s.close();
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
infrastructure.push(errorMessage(error));
|
||||
} finally {
|
||||
await browser.close();
|
||||
server.close();
|
||||
if (runtime) {
|
||||
try {
|
||||
await runtime.close();
|
||||
} catch (error) {
|
||||
infrastructure.push(`browser cleanup: ${errorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
if (server) {
|
||||
try {
|
||||
await closeServer(server);
|
||||
} catch (error) {
|
||||
infrastructure.push(`server cleanup: ${errorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (args["update-golden"]) {
|
||||
@@ -291,15 +381,19 @@ async function main() {
|
||||
const baselineFile = join(OUT, "baseline.json");
|
||||
let baseline: Outcome[] | null = null;
|
||||
if (existsSync(baselineFile)) {
|
||||
const saved = JSON.parse(readFileSync(baselineFile, "utf8")) as { machine: Fingerprint; outcomes: Outcome[] };
|
||||
if (sameMachine(saved.machine, machine)) baseline = saved.outcomes;
|
||||
else console.warn("warning: target/eval/baseline.json is from another machine or OS: not compared");
|
||||
try {
|
||||
const saved = JSON.parse(readFileSync(baselineFile, "utf8")) as { machine: Fingerprint; outcomes: Outcome[] };
|
||||
if (sameMachine(saved.machine, machine)) baseline = saved.outcomes;
|
||||
else console.warn("warning: target/eval/baseline.json is from another machine or OS: not compared");
|
||||
} catch (error) {
|
||||
infrastructure.push(`baseline: ${errorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
const repro = (o: Outcome) => `pnpm eval --only ${o.scenario} --condition ${o.condition}`;
|
||||
const verdicts = judge(outcomes, loadBudgets(join(ROOT, "evals", "budgets.json")), baseline, repro);
|
||||
const verdicts = judge(outcomes, evalBudgets, baseline, repro, expected);
|
||||
|
||||
const meta = {
|
||||
date: new Date().toISOString(),
|
||||
date,
|
||||
commit: git("rev-parse", "--short", "HEAD"),
|
||||
dirty: git("status", "--porcelain", "--untracked-files=no") !== "",
|
||||
runs,
|
||||
@@ -307,27 +401,32 @@ async function main() {
|
||||
machine,
|
||||
tools,
|
||||
pins,
|
||||
browser: browser.version(),
|
||||
browser: browserVersion,
|
||||
};
|
||||
const report = {
|
||||
meta,
|
||||
pass: verdicts.every((v) => v.status === "pass"),
|
||||
pass: infrastructure.length === 0 && verdicts.every((v) => v.status === "pass"),
|
||||
status: infrastructure.length || verdicts.some((v) => v.status === "fail") ? "failed" : "complete",
|
||||
expected,
|
||||
outcomes,
|
||||
trials: trialEvidence,
|
||||
infrastructure,
|
||||
verdicts: verdicts.map((v) => {
|
||||
const o = outcomes.find((x) => idOf(x) === v.id)!;
|
||||
const o = outcomes.find((x) => idOf(x) === v.id);
|
||||
return {
|
||||
...v,
|
||||
samples: o.samples,
|
||||
results: o.results,
|
||||
comparison: o.comparison,
|
||||
drift: o.drift,
|
||||
degraded: o.degraded,
|
||||
error: o.error,
|
||||
samples: o?.samples,
|
||||
results: o?.results,
|
||||
comparison: o?.comparison,
|
||||
drift: o?.drift,
|
||||
degraded: o?.degraded,
|
||||
error: o?.error,
|
||||
// Diagnostics only where something needs a look.
|
||||
diagnostics: v.status === "fail" || v.regressions.length ? o.diagnostics : undefined,
|
||||
diagnostics: v.status === "fail" || v.regressions.length ? o?.diagnostics : undefined,
|
||||
};
|
||||
}),
|
||||
};
|
||||
writeFileSync(join(OUT, "report.json"), JSON.stringify(report, null, 2));
|
||||
writeFileSync(join(OUT, "report.json"), evidenceJson(report));
|
||||
const failing = verdicts.filter((v) => v.status === "fail");
|
||||
const md = [
|
||||
"# OpenIDEA eval report",
|
||||
@@ -337,6 +436,16 @@ async function main() {
|
||||
table(verdicts, true),
|
||||
"",
|
||||
...failing.flatMap((v) => [`- **${v.id}**: ${v.failures.map((f) => f.detail).join("; ")} — \`${v.repro}\``]),
|
||||
...infrastructure.map((error) => `- **Infrastructure**: ${error}`),
|
||||
"",
|
||||
"## Trials",
|
||||
"",
|
||||
"| Scenario | Valid / planned measured | Attempted measured | Valid / attempted warmup |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
...Object.entries(trialEvidence).map(([id, attempts]) => {
|
||||
const c = summarizeTrials(attempts, id.endsWith("@cold") ? 1 : runs).counts;
|
||||
return `| ${id} | ${c.valid}/${c.planned} | ${c.attempted} | ${c.warmupValid}/${c.warmupAttempted} |`;
|
||||
}),
|
||||
].join("\n");
|
||||
writeFileSync(join(OUT, "report.md"), md + "\n");
|
||||
|
||||
@@ -345,9 +454,9 @@ async function main() {
|
||||
for (const v of failing) {
|
||||
console.log(`\n${v.id} FAILED (${v.repro})`);
|
||||
for (const f of v.failures) console.log(` ${f.kind}: ${f.detail}`);
|
||||
const o = outcomes.find((x) => idOf(x) === v.id)!;
|
||||
if (o.drift?.length) console.log(` golden recorded with other tools: ${o.drift.join(", ")}`);
|
||||
const slow = [...(o.diagnostics?.calls ?? [])].sort((a, b) => b.ms - a.ms).slice(0, 5);
|
||||
const o = outcomes.find((x) => idOf(x) === v.id);
|
||||
if (o?.drift?.length) console.log(` golden recorded with other tools: ${o.drift.join(", ")}`);
|
||||
const slow = [...(o?.diagnostics?.calls ?? [])].sort((a, b) => b.ms - a.ms).slice(0, 5);
|
||||
if (slow.length)
|
||||
console.log(
|
||||
` slowest commands: ${slow.map((c) => `${c.cmd} ${Math.round(c.ms)} ms${c.pending ? " (no answer)" : c.ok ? "" : " (failed)"}`).join(", ")}`,
|
||||
@@ -355,11 +464,11 @@ async function main() {
|
||||
}
|
||||
console.log(`\nReport: ${join(OUT, "report.json")}`);
|
||||
if (args["save-baseline"]) {
|
||||
writeFileSync(baselineFile, JSON.stringify({ machine, meta, outcomes }, null, 2));
|
||||
writeFileSync(baselineFile, evidenceJson({ machine, meta, outcomes }));
|
||||
console.log(`Baseline saved to ${baselineFile}`);
|
||||
}
|
||||
const regressed = verdicts.some((v) => v.regressions.length);
|
||||
process.exit(failing.length || (args["fail-on-regression"] && regressed) ? 1 : 0);
|
||||
process.exit(infrastructure.length || failing.length || (args["fail-on-regression"] && regressed) ? 1 : 0);
|
||||
}
|
||||
|
||||
await main();
|
||||
+1
-1
@@ -3,7 +3,7 @@ import type { KnipConfig } from "knip";
|
||||
const config: KnipConfig = {
|
||||
// Entries come from the Vite (index.html), Vitest, Playwright and package.json script plugins;
|
||||
// standalone browser fixtures are loaded by their own HTML pages.
|
||||
entry: ["e2e/*.fixture.{ts,tsx}"],
|
||||
entry: ["e2e/*.fixture.{ts,tsx}", "perf/**/*.fixture.{ts,tsx}"],
|
||||
project: [
|
||||
"src/**/*.{ts,tsx,css}",
|
||||
"!src/editor/core/cm/angular/parser.terms.ts",
|
||||
|
||||
@@ -23,9 +23,46 @@ closes it, and records:
|
||||
| `exit_s` | until every process has exited |
|
||||
| `cache_mb`, install size | `cache_dirs`, `install_dir` |
|
||||
|
||||
Apps run interleaved, rotating the order, after `--warmup` discarded runs.
|
||||
Output goes to `results/<time>-<project>/`: `report.md` (median, min–max),
|
||||
`results.json`, and each run's stdout.
|
||||
Apps run interleaved, rotating the order, after `--warmup` unaggregated runs.
|
||||
Every attempt, including warmups and failed launches, is retained in
|
||||
`results/<time>-<project>-<unique>/`: `report.md`, `results.json`, and each
|
||||
launched run's stdout. Reports are refreshed before and after each attempt
|
||||
and on errors or interruption; separate invocations never overwrite each other.
|
||||
|
||||
The table shows median (min–max) and **valid/attempted** measured-trial counts
|
||||
for every fixed metric, even when there are zero valid values. Only completely
|
||||
valid trials enter aggregates; failed trials retain partial measurements and
|
||||
explicit reasons in JSON and the attempts table. Warmups never enter medians
|
||||
but a failed warmup still fails the suite. An empty or incomplete suite is not
|
||||
successful.
|
||||
|
||||
Startup timeout, early exit/crash, observation/access errors, invalid or missing
|
||||
required metrics, quit-command errors, forced termination, and interruption
|
||||
produce a nonzero exit. JSON records `status`, `reasons`, `warmup`, and
|
||||
`metric_status` (`valid`, `missing`, `invalid`, or `unsupported`). Nonfinite
|
||||
numeric evidence is preserved as an `invalid_number` object, never JSON NaN.
|
||||
Unsupported window observation, non-Linux PSS, and unconfigured cache directories
|
||||
are explicit exclusions rather than missing required values. Unobserved optional
|
||||
log markers do not fail a trial; `ready_after` markers are required.
|
||||
|
||||
`--runs` must be positive; `--warmup` and `--cooldown` may be zero. Other
|
||||
durations and the CPU threshold must be finite and positive, and the quiet
|
||||
period cannot exceed the startup timeout.
|
||||
|
||||
Shutdown always runs after a launch, including observation errors and Ctrl-C.
|
||||
`--close-timeout` bounds the quit command plus graceful wait; each forced cleanup
|
||||
has at most two additional seconds to reap processes. `--command-timeout`
|
||||
(default 60 seconds) bounds `--drop-os-cache`, including interactive commands.
|
||||
Only processes attributed to the launch are terminated; preexisting instances
|
||||
are refused, never killed. Configured `daemons` remain explicitly excluded from
|
||||
shutdown and forced cleanup. POSIX launch-owned process groups retain orphaned
|
||||
children; on Windows discovery uses the recorded process ancestry.
|
||||
|
||||
Run the small process-fixture regressions without launching an installed IDE:
|
||||
|
||||
```sh
|
||||
python -m unittest discover -s perf/compare -p 'test_bench.py'
|
||||
```
|
||||
|
||||
## Fair runs
|
||||
|
||||
@@ -46,6 +83,7 @@ Output goes to `results/<time>-<project>/`: `report.md` (median, min–max),
|
||||
- Windows: windows via `user32`, closed with `WM_CLOSE`.
|
||||
- macOS: windows via Quartz (`pyobjc-framework-Quartz`; titles need Screen
|
||||
Recording permission). WebKit helpers are children of launchd and are
|
||||
attributed through `responsibility_get_pid_responsible_for_pid`.
|
||||
attributed through `responsibility_get_pid_responsible_for_pid`. Without that
|
||||
API, unrelated WebKit processes are not guessed by name/start time or killed.
|
||||
- Linux: windows via `wmctrl` (X11). Without it `window_s` is skipped and
|
||||
readiness counts from launch.
|
||||
+399
-188
@@ -14,6 +14,7 @@ import ctypes
|
||||
import datetime as dt
|
||||
import glob
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
@@ -24,6 +25,7 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
@@ -34,7 +36,7 @@ except ImportError:
|
||||
OS = {"Windows": "windows", "Darwin": "macos", "Linux": "linux"}[platform.system()]
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]")
|
||||
GONE = (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied)
|
||||
GONE = (psutil.NoSuchProcess, psutil.ZombieProcess)
|
||||
MB = 1024 * 1024
|
||||
POLL_S = 0.2
|
||||
|
||||
@@ -140,10 +142,7 @@ def _mac_windows(pids: set[int]) -> list[tuple[object, str]] | None:
|
||||
def _x11_windows(pids: set[int]) -> list[tuple[object, str]] | None:
|
||||
if not shutil.which("wmctrl"):
|
||||
return None
|
||||
try:
|
||||
out = subprocess.run(["wmctrl", "-lp"], capture_output=True, text=True, timeout=2).stdout
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
out = subprocess.run(["wmctrl", "-lp"], capture_output=True, text=True, timeout=2, check=True).stdout
|
||||
found = []
|
||||
for line in out.splitlines():
|
||||
parts = line.split(None, 4)
|
||||
@@ -174,11 +173,11 @@ if OS == "macos":
|
||||
|
||||
|
||||
class Tree:
|
||||
"""Every process the app started, including ones whose parent already exited.
|
||||
"""Processes owned by this launch, identified by process identity and ancestry.
|
||||
|
||||
On macOS, WKWebView's helper processes are children of launchd, so they are
|
||||
attributed through the responsible-pid API (or, failing that, by name and
|
||||
start time).
|
||||
POSIX launches have a private process group, retaining orphaned children.
|
||||
macOS launchd-owned WebKit helpers require responsible-pid attribution:
|
||||
name/start-time guesses cannot safely authorize terminating a process.
|
||||
"""
|
||||
|
||||
def __init__(self, root: psutil.Process, started: float):
|
||||
@@ -187,14 +186,22 @@ class Tree:
|
||||
self.procs: dict[tuple[int, float], psutil.Process] = {}
|
||||
self.cpu: dict[tuple[int, float], float] = {}
|
||||
self.cpu_exited = 0.0
|
||||
self.group = root.pid if os.name == "posix" else None
|
||||
|
||||
def _discover(self):
|
||||
found = [self.root]
|
||||
for p in list(self.procs.values()):
|
||||
for p in [self.root, *self.procs.values()]:
|
||||
try:
|
||||
found += p.children(recursive=True)
|
||||
except GONE:
|
||||
pass
|
||||
if self.group is not None:
|
||||
for p in psutil.process_iter():
|
||||
try:
|
||||
if os.getpgid(p.pid) == self.group:
|
||||
found.append(p)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
continue
|
||||
if OS == "macos":
|
||||
found += self._webkit()
|
||||
for p in found:
|
||||
@@ -211,8 +218,7 @@ class Tree:
|
||||
if _responsible:
|
||||
if _responsible(p.pid) == self.root.pid:
|
||||
out.append(p)
|
||||
elif (p.info["create_time"] or 0) >= self.started - 1:
|
||||
out.append(p)
|
||||
# Without attribution, leave unrelated WebKit processes untouched.
|
||||
return out
|
||||
|
||||
def sample(self) -> tuple[float, int]:
|
||||
@@ -234,7 +240,14 @@ class Tree:
|
||||
return {pid for pid, _ in self.procs}
|
||||
|
||||
def alive(self) -> list[psutil.Process]:
|
||||
return [p for p in self.procs.values() if p.is_running()]
|
||||
out = []
|
||||
for p in self.procs.values():
|
||||
try:
|
||||
if p.is_running() and p.status() != psutil.STATUS_ZOMBIE:
|
||||
out.append(p)
|
||||
except GONE:
|
||||
pass
|
||||
return out
|
||||
|
||||
def memory(self) -> dict:
|
||||
"""Summed memory of the live tree. RSS double-counts shared pages, USS leaves them out."""
|
||||
@@ -284,7 +297,7 @@ class LogTail:
|
||||
f.seek(self.offsets.get(path, 0))
|
||||
chunk = f.read()
|
||||
self.offsets[path] = f.tell()
|
||||
except OSError:
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if not chunk:
|
||||
continue
|
||||
@@ -310,151 +323,246 @@ def dir_size(path: Path) -> int:
|
||||
if path.is_file():
|
||||
return path.stat().st_size
|
||||
total = 0
|
||||
for root, _, files in os.walk(path):
|
||||
def walk_error(error):
|
||||
raise error
|
||||
|
||||
for root, _, files in os.walk(path, onerror=walk_error):
|
||||
for f in files:
|
||||
try:
|
||||
total += os.lstat(os.path.join(root, f)).st_size
|
||||
except OSError:
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return total
|
||||
|
||||
|
||||
def close(app: App, tree: Tree, proc: subprocess.Popen, timeout: float) -> tuple[float | None, list[str]]:
|
||||
"""Asks the app to quit, waits for the whole tree, kills what's left.
|
||||
def failure(result: dict, reason: str):
|
||||
result["reasons"].append(reason)
|
||||
if result["status"] != "interrupted":
|
||||
result["status"] = "failed"
|
||||
|
||||
Returns (seconds until every process exited, or None if it had to kill; names of killed leftovers).
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
if app.quit_command:
|
||||
subprocess.run(app.quit_command, check=False)
|
||||
elif OS == "windows":
|
||||
# Only the project windows: closing a framework's hidden message windows
|
||||
# (Tauri's single-instance and event-target ones) stalls the real shutdown.
|
||||
for hwnd, title in _win_windows(tree.pids()):
|
||||
if app.window_title.search(title):
|
||||
_user32.PostMessageW(hwnd, 0x0010, 0, 0) # WM_CLOSE
|
||||
else:
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
def remaining() -> list[psutil.Process]:
|
||||
out = []
|
||||
for p in tree.alive():
|
||||
try:
|
||||
if p.name().lower() not in app.daemons:
|
||||
out.append(p)
|
||||
except GONE:
|
||||
pass
|
||||
return out
|
||||
|
||||
while time.perf_counter() - t0 < timeout:
|
||||
tree.sample()
|
||||
if not remaining():
|
||||
return time.perf_counter() - t0, []
|
||||
time.sleep(POLL_S)
|
||||
leftovers = remaining()
|
||||
def force_cleanup(tree: Tree | None, proc: subprocess.Popen, daemons: set[str], result: dict) -> list[str]:
|
||||
"""Kill only launch-owned identities; allow at most two seconds to reap."""
|
||||
names = []
|
||||
for p in leftovers:
|
||||
targets = []
|
||||
if tree is not None:
|
||||
try:
|
||||
names.append(p.name())
|
||||
tree._discover()
|
||||
except Exception as exc:
|
||||
failure(result, f"cleanup discovery: {type(exc).__name__}: {exc}")
|
||||
targets = list(tree.procs.values())
|
||||
try:
|
||||
# A reaped PID may already belong to somebody else.
|
||||
if proc.poll() is None:
|
||||
root = psutil.Process(proc.pid)
|
||||
if all(p.pid != root.pid for p in targets):
|
||||
targets.append(root)
|
||||
except GONE:
|
||||
pass
|
||||
killed = []
|
||||
for p in targets:
|
||||
try:
|
||||
if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE:
|
||||
continue
|
||||
name = p.name()
|
||||
if name.lower() in daemons:
|
||||
result.setdefault("excluded_daemons", []).append({"pid": p.pid, "name": name})
|
||||
continue
|
||||
p.kill()
|
||||
names.append(name)
|
||||
killed.append(p)
|
||||
except GONE:
|
||||
pass
|
||||
psutil.wait_procs(leftovers, timeout=10)
|
||||
return None, names
|
||||
|
||||
|
||||
def run_once(app: App, args, out_dir: Path, label: str) -> dict:
|
||||
running = running_instances(app) if app.single_instance else []
|
||||
if running:
|
||||
pids = ", ".join(str(p.pid) for p in running)
|
||||
raise SystemExit(f"{app.name} is already running (pid {pids}); close it first, it would take over the launch.")
|
||||
if args.wipe_caches:
|
||||
for path in app.cache_paths():
|
||||
shutil.rmtree(path, ignore_errors=True) if path.is_dir() else path.unlink(missing_ok=True)
|
||||
if args.drop_os_cache:
|
||||
subprocess.run(args.drop_os_cache, shell=True, check=True)
|
||||
if args.cooldown:
|
||||
time.sleep(args.cooldown)
|
||||
|
||||
stdout_path = out_dir / f"{label}.stdout.log"
|
||||
tail = LogTail(app.logs, [stdout_path])
|
||||
started_wall = time.time()
|
||||
with open(stdout_path, "wb") as stdout:
|
||||
t0 = time.perf_counter()
|
||||
proc = subprocess.Popen(app.command, stdout=stdout, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL)
|
||||
tree = Tree(psutil.Process(proc.pid), started_wall)
|
||||
|
||||
result: dict = {"app": app.key, "label": label, "markers_first_s": {}, "markers_last_s": {}}
|
||||
window_supported = True
|
||||
window_s = None
|
||||
history: collections.deque[tuple[float, float]] = collections.deque()
|
||||
peak_rss = 0
|
||||
ready_s = None
|
||||
|
||||
while True:
|
||||
now = time.perf_counter() - t0
|
||||
cpu, rss = tree.sample()
|
||||
peak_rss = max(peak_rss, rss)
|
||||
history.append((now, cpu))
|
||||
while history and now - history[0][0] > args.quiet_secs:
|
||||
history.popleft()
|
||||
|
||||
for line in tail.lines():
|
||||
for name, rx in app.markers.items():
|
||||
if rx.search(line):
|
||||
result["markers_first_s"].setdefault(name, now)
|
||||
result["markers_last_s"][name] = now
|
||||
|
||||
if window_s is None and window_supported:
|
||||
windows = visible_windows(tree.pids())
|
||||
if windows is None:
|
||||
window_supported = False
|
||||
elif any(app.window_title.search(title) for _, title in windows):
|
||||
window_s = now
|
||||
|
||||
if not tree.alive():
|
||||
raise SystemExit(
|
||||
f"{app.name} exited during startup (code {proc.poll()}); an instance started elsewhere may have "
|
||||
f"taken the launch. Output: {stdout_path}"
|
||||
)
|
||||
|
||||
waited = (window_s is not None or not window_supported) and all(
|
||||
m in result["markers_first_s"] for m in app.ready_after
|
||||
)
|
||||
span = now - history[0][0]
|
||||
if waited and span >= args.quiet_secs * 0.95:
|
||||
busy = (cpu - history[0][1]) / span
|
||||
if busy < args.quiet_cores:
|
||||
# Never before what it waited for (the quiet span can start earlier).
|
||||
ready_s = max(now - span, window_s or 0, *(result["markers_first_s"][m] for m in app.ready_after))
|
||||
break
|
||||
if now > args.timeout:
|
||||
result["timed_out"] = True
|
||||
break
|
||||
time.sleep(POLL_S)
|
||||
|
||||
result["window_s"] = window_s
|
||||
result["ready_s"] = ready_s
|
||||
result["peak_rss_mb"] = peak_rss / MB
|
||||
|
||||
# Idle: what it costs to just sit there with the project open.
|
||||
idle_cpu0, _ = tree.sample()
|
||||
i0 = time.perf_counter()
|
||||
while time.perf_counter() - i0 < args.idle_secs:
|
||||
time.sleep(POLL_S * 5)
|
||||
_, rss = tree.sample()
|
||||
peak_rss = max(peak_rss, rss)
|
||||
idle_cpu1, _ = tree.sample()
|
||||
result["idle_cpu_pct"] = 100 * (idle_cpu1 - idle_cpu0) / (time.perf_counter() - i0)
|
||||
result["peak_rss_mb"] = peak_rss / MB
|
||||
result.update(tree.memory())
|
||||
result["total_cpu_s"] = idle_cpu0
|
||||
|
||||
exit_s, killed = close(app, tree, proc, args.close_timeout)
|
||||
result["exit_s"] = exit_s
|
||||
except Exception as exc:
|
||||
failure(result, f"cleanup pid {p.pid}: {type(exc).__name__}: {exc}")
|
||||
deadline = time.perf_counter() + 2
|
||||
if killed:
|
||||
result["killed_on_exit"] = killed
|
||||
result["cache_mb"] = sum(dir_size(p) for p in app.cache_paths()) / MB
|
||||
_, alive = psutil.wait_procs(killed, timeout=max(0, deadline - time.perf_counter()))
|
||||
if alive:
|
||||
failure(result, f"cleanup survivors: {[p.pid for p in alive]}")
|
||||
try:
|
||||
proc.wait(timeout=max(0, deadline - time.perf_counter()))
|
||||
except subprocess.TimeoutExpired:
|
||||
# The root can itself be an explicitly excluded daemon.
|
||||
if not any(p["pid"] == proc.pid for p in result.get("excluded_daemons", [])):
|
||||
failure(result, f"cleanup root {proc.pid} did not exit")
|
||||
return names
|
||||
|
||||
|
||||
def bounded_command(command, timeout: float, *, shell=False):
|
||||
"""External cache/quit commands have their own launch-owned cleanup."""
|
||||
proc = subprocess.Popen(command, shell=shell, stdin=subprocess.DEVNULL,
|
||||
start_new_session=os.name == "posix")
|
||||
tree = None
|
||||
evidence = {"status": "valid", "reasons": []}
|
||||
try:
|
||||
tree = Tree(psutil.Process(proc.pid), time.time())
|
||||
deadline = time.perf_counter() + timeout
|
||||
while proc.poll() is None:
|
||||
tree._discover()
|
||||
remaining = deadline - time.perf_counter()
|
||||
if remaining <= 0:
|
||||
raise subprocess.TimeoutExpired(command, timeout)
|
||||
time.sleep(min(POLL_S, remaining))
|
||||
if proc.returncode:
|
||||
raise subprocess.CalledProcessError(proc.returncode, command)
|
||||
finally:
|
||||
force_cleanup(tree, proc, set(), evidence)
|
||||
if evidence["reasons"]:
|
||||
raise RuntimeError("; ".join(evidence["reasons"]))
|
||||
|
||||
|
||||
def close(app: App, tree: Tree, proc: subprocess.Popen, timeout: float, result: dict) -> tuple[float | None, list[str]]:
|
||||
"""The quit command and graceful wait share one budget; forced reap is bounded."""
|
||||
t0 = time.perf_counter()
|
||||
deadline = t0 + timeout
|
||||
exit_s = None
|
||||
killed = []
|
||||
try:
|
||||
tree._discover()
|
||||
if app.quit_command:
|
||||
try:
|
||||
bounded_command(app.quit_command, max(0.001, deadline - time.perf_counter()))
|
||||
except Exception as exc:
|
||||
failure(result, f"quit command: {type(exc).__name__}: {exc}")
|
||||
elif OS == "windows":
|
||||
for hwnd, title in _win_windows(tree.pids()):
|
||||
if app.window_title.search(title):
|
||||
_user32.PostMessageW(hwnd, 0x0010, 0, 0)
|
||||
elif proc.poll() is None:
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
while time.perf_counter() < deadline:
|
||||
tree._discover()
|
||||
remaining = [p for p in tree.alive() if p.name().lower() not in app.daemons]
|
||||
if not remaining:
|
||||
exit_s = time.perf_counter() - t0
|
||||
break
|
||||
time.sleep(min(POLL_S, max(0, deadline - time.perf_counter())))
|
||||
if exit_s is None:
|
||||
failure(result, "graceful-close timeout")
|
||||
except KeyboardInterrupt:
|
||||
result["status"] = "interrupted"
|
||||
failure(result, "interrupted during shutdown")
|
||||
except Exception as exc:
|
||||
failure(result, f"shutdown observation: {type(exc).__name__}: {exc}")
|
||||
finally:
|
||||
killed = force_cleanup(tree, proc, app.daemons, result)
|
||||
if killed:
|
||||
failure(result, "forced cleanup: " + ", ".join(killed))
|
||||
exit_s = None
|
||||
return exit_s, killed
|
||||
|
||||
|
||||
def new_trial(app: App, label: str, warmup=False) -> dict:
|
||||
return {"app": app.key, "label": label, "warmup": warmup, "status": "running", "reasons": [],
|
||||
"markers_first_s": {}, "markers_last_s": {}, "metric_status": {},
|
||||
**{key: None for key, _ in METRICS}}
|
||||
|
||||
|
||||
def run_once(app: App, args, out_dir: Path, label: str, result: dict | None = None) -> dict:
|
||||
result = new_trial(app, label) if result is None else result
|
||||
proc = tree = None
|
||||
try:
|
||||
running = running_instances(app) if app.single_instance else []
|
||||
if running:
|
||||
raise RuntimeError(f"{app.name} already running (pid {', '.join(str(p.pid) for p in running)}); close it first")
|
||||
if args.wipe_caches:
|
||||
for path in app.cache_paths():
|
||||
shutil.rmtree(path) if path.is_dir() else path.unlink(missing_ok=True)
|
||||
if args.drop_os_cache:
|
||||
bounded_command(args.drop_os_cache, args.command_timeout, shell=True)
|
||||
if args.cooldown:
|
||||
time.sleep(args.cooldown)
|
||||
|
||||
stdout_path = out_dir / f"{label}.stdout.log"
|
||||
result["stdout"] = str(stdout_path)
|
||||
tail = LogTail(app.logs, [stdout_path])
|
||||
started_wall = time.time()
|
||||
with open(stdout_path, "wb") as stdout:
|
||||
t0 = time.perf_counter()
|
||||
proc = subprocess.Popen(app.command, stdout=stdout, stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.DEVNULL, start_new_session=os.name == "posix")
|
||||
result["pid"] = proc.pid
|
||||
tree = Tree(psutil.Process(proc.pid), started_wall)
|
||||
window_supported = True
|
||||
history: collections.deque[tuple[float, float]] = collections.deque()
|
||||
peak_rss = 0
|
||||
|
||||
while True:
|
||||
now = time.perf_counter() - t0
|
||||
cpu, rss = tree.sample()
|
||||
peak_rss = max(peak_rss, rss)
|
||||
result["peak_rss_mb"] = peak_rss / MB
|
||||
history.append((now, cpu))
|
||||
while history and now - history[0][0] > args.quiet_secs:
|
||||
history.popleft()
|
||||
|
||||
for line in tail.lines():
|
||||
for name, rx in app.markers.items():
|
||||
if rx.search(line):
|
||||
result["markers_first_s"].setdefault(name, now)
|
||||
result["markers_last_s"][name] = now
|
||||
if result["window_s"] is None and window_supported:
|
||||
windows = visible_windows(tree.pids())
|
||||
if windows is None:
|
||||
window_supported = False
|
||||
result["metric_status"]["window_s"] = "unsupported"
|
||||
elif any(app.window_title.search(title) for _, title in windows):
|
||||
result["window_s"] = now
|
||||
if proc.poll() is not None or not tree.alive():
|
||||
raise RuntimeError(f"{app.name} exited during startup (code {proc.poll()})")
|
||||
|
||||
waited = (result["window_s"] is not None or not window_supported) and all(
|
||||
m in result["markers_first_s"] for m in app.ready_after)
|
||||
span = now - history[0][0]
|
||||
if waited and span >= args.quiet_secs * 0.95:
|
||||
busy = (cpu - history[0][1]) / span
|
||||
if busy < args.quiet_cores:
|
||||
result["ready_s"] = max(now - span, result["window_s"] or 0,
|
||||
*(result["markers_first_s"][m] for m in app.ready_after))
|
||||
break
|
||||
if now > args.timeout:
|
||||
result["timed_out"] = True
|
||||
raise TimeoutError("startup timeout: CPU/window/required markers never became ready")
|
||||
time.sleep(POLL_S)
|
||||
|
||||
idle_cpu0, _ = tree.sample()
|
||||
result["total_cpu_s"] = idle_cpu0
|
||||
i0 = time.perf_counter()
|
||||
while time.perf_counter() - i0 < args.idle_secs:
|
||||
time.sleep(min(POLL_S * 5, max(0, args.idle_secs - (time.perf_counter() - i0))))
|
||||
_, rss = tree.sample()
|
||||
peak_rss = max(peak_rss, rss)
|
||||
result["peak_rss_mb"] = peak_rss / MB
|
||||
if proc.poll() is not None or not tree.alive():
|
||||
raise RuntimeError(f"{app.name} exited during idle (code {proc.poll()})")
|
||||
idle_cpu1, _ = tree.sample()
|
||||
result["idle_cpu_pct"] = 100 * (idle_cpu1 - idle_cpu0) / (time.perf_counter() - i0)
|
||||
result.update(tree.memory())
|
||||
result["cache_mb"] = sum(dir_size(p) for p in app.cache_paths()) / MB
|
||||
result["status"] = "valid"
|
||||
except KeyboardInterrupt:
|
||||
result["status"] = "interrupted"
|
||||
failure(result, "interrupted")
|
||||
except Exception as exc:
|
||||
failure(result, f"{type(exc).__name__}: {exc}")
|
||||
finally:
|
||||
if proc is not None:
|
||||
try:
|
||||
if tree is not None:
|
||||
result["exit_s"], result["killed_on_exit"] = close(app, tree, proc, args.close_timeout, result)
|
||||
else:
|
||||
result["killed_on_exit"] = force_cleanup(tree, proc, app.daemons, result)
|
||||
except KeyboardInterrupt:
|
||||
result["status"] = "interrupted"
|
||||
failure(result, "interrupted during cleanup")
|
||||
result["killed_on_exit"] = force_cleanup(tree, proc, app.daemons, result)
|
||||
except Exception as exc:
|
||||
failure(result, f"cleanup: {type(exc).__name__}: {exc}")
|
||||
result["killed_on_exit"] = force_cleanup(tree, proc, app.daemons, result)
|
||||
result["exit_code"] = proc.poll()
|
||||
validate_metrics(app, result)
|
||||
return result
|
||||
|
||||
|
||||
@@ -475,8 +583,67 @@ METRICS = [
|
||||
]
|
||||
|
||||
|
||||
def valid_number(value) -> bool:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
|
||||
|
||||
|
||||
def validate_metrics(app: App, result: dict):
|
||||
states = result["metric_status"]
|
||||
for key, _ in METRICS:
|
||||
if (key == "pss_mb" and OS != "linux") or (key == "cache_mb" and not app.cache_dirs):
|
||||
states[key] = "unsupported"
|
||||
continue
|
||||
if key == "window_s" and states.get(key) == "unsupported":
|
||||
continue
|
||||
value = result.get(key)
|
||||
good = valid_number(value) and (key != "procs" or (value > 0 and value == int(value)))
|
||||
states[key] = "valid" if good else "missing" if value is None else "invalid"
|
||||
if not good:
|
||||
reason = f"required metric {key}: {states[key]}"
|
||||
if reason not in result["reasons"]:
|
||||
failure(result, reason)
|
||||
for marker in app.ready_after:
|
||||
if not valid_number(result["markers_first_s"].get(marker)):
|
||||
reason = f"required marker {marker}: missing or invalid"
|
||||
if reason not in result["reasons"]:
|
||||
failure(result, reason)
|
||||
|
||||
|
||||
def json_safe(value):
|
||||
"""Strict JSON retains invalid numeric evidence without emitting NaN/Infinity."""
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
return {"invalid_number": str(value)}
|
||||
if isinstance(value, dict):
|
||||
return {key: json_safe(v) for key, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [json_safe(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def suite_status(apps: list[App], runs: list[dict], meta: dict) -> str:
|
||||
if meta.get("suite_errors") or not runs or any(r["status"] != "valid" for r in runs):
|
||||
return "failed"
|
||||
for app in apps:
|
||||
measured = [r for r in runs if r["app"] == app.key and not r["warmup"]]
|
||||
if len(measured) != meta["runs"]:
|
||||
return "failed"
|
||||
return "valid"
|
||||
|
||||
|
||||
def persist(apps: list[App], runs: list[dict], meta: dict, out_dir: Path):
|
||||
meta["status"] = suite_status(apps, runs, meta)
|
||||
# JSON first: report rendering must never discard the raw failure evidence.
|
||||
def replace(name, text):
|
||||
temporary = out_dir / (name + ".tmp")
|
||||
temporary.write_text(text, encoding="utf-8")
|
||||
temporary.replace(out_dir / name)
|
||||
|
||||
replace("results.json", json.dumps(json_safe({"meta": meta, "runs": runs}), indent=2, allow_nan=False))
|
||||
replace("report.md", report(apps, runs, meta))
|
||||
|
||||
|
||||
def fmt(value: float | None) -> str:
|
||||
return "–" if value is None else f"{value:.2f}"
|
||||
return "–" if not valid_number(value) else f"{value:.2f}"
|
||||
|
||||
|
||||
def summarize(values: list[float]) -> str:
|
||||
@@ -489,15 +656,25 @@ def summarize(values: list[float]) -> str:
|
||||
|
||||
|
||||
def report(apps: list[App], runs: list[dict], meta: dict) -> str:
|
||||
def column(app: App, getter) -> str:
|
||||
vals = [v for r in runs if r["app"] == app.key and (v := getter(r)) is not None]
|
||||
return summarize(vals)
|
||||
measured = [r for r in runs if not r.get("warmup", False)]
|
||||
|
||||
rows = [(title, lambda r, k=key: r.get(k)) for key, title in METRICS]
|
||||
def column(app: App, key, getter, owner=None) -> str:
|
||||
attempts = [r for r in measured if r["app"] == app.key]
|
||||
vals = [v for r in attempts if r["status"] == "valid"
|
||||
and (owner is None or owner == app.key)
|
||||
and r.get("metric_status", {}).get(key) != "unsupported"
|
||||
and valid_number(v := getter(r))]
|
||||
unsupported = sum(r.get("metric_status", {}).get(key) == "unsupported" for r in attempts)
|
||||
suffix = " unsupported" if unsupported or (owner is not None and owner != app.key) else ""
|
||||
return f"{summarize(vals)} [{len(vals)}/{len(attempts)}]{suffix}"
|
||||
|
||||
rows = [(key, title, lambda r, k=key: r.get(k), None) for key, title in METRICS]
|
||||
for app in apps:
|
||||
for m in app.markers:
|
||||
rows.append((f"{app.name}: log `{m}` first (s)", lambda r, m=m: r["markers_first_s"].get(m)))
|
||||
rows.append((f"{app.name}: log `{m}` last (s)", lambda r, m=m: r["markers_last_s"].get(m)))
|
||||
rows.append((None, f"{app.name}: log `{m}` first (s)",
|
||||
lambda r, m=m: r["markers_first_s"].get(m), app.key))
|
||||
rows.append((None, f"{app.name}: log `{m}` last (s)",
|
||||
lambda r, m=m: r["markers_last_s"].get(m), app.key))
|
||||
|
||||
lines = [
|
||||
f"# IDE benchmark: {meta['project']}",
|
||||
@@ -505,24 +682,34 @@ def report(apps: list[App], runs: list[dict], meta: dict) -> str:
|
||||
f"{meta['os']} · {meta['cpu']} · {meta['cores']} cores · {meta['ram_gb']:.0f} GB RAM · "
|
||||
f"mode **{meta['mode']}** · {meta['runs']} runs · {meta['date']}",
|
||||
"",
|
||||
"Median (min–max). Times are seconds from launch.",
|
||||
f"Status: **{meta.get('status', 'failed')}**. Median (min–max), [valid/attempted] measured trials.",
|
||||
"Only valid trials enter aggregates; warmups and partial evidence remain below and in JSON.",
|
||||
"Times are seconds from launch. Unsupported metrics are not required; missing required metrics fail a trial.",
|
||||
"",
|
||||
"| Metric | " + " | ".join(a.name for a in apps) + " |",
|
||||
"|---|" + "---|" * len(apps),
|
||||
]
|
||||
for title, getter in rows:
|
||||
cells = [column(a, getter) for a in apps]
|
||||
if any(c != "–" for c in cells):
|
||||
lines.append(f"| {title} | " + " | ".join(cells) + " |")
|
||||
for key, title, getter, owner in rows:
|
||||
cells = [column(a, key, getter, owner) for a in apps]
|
||||
lines.append(f"| {title} | " + " | ".join(cells) + " |")
|
||||
lines += ["", "## Attempts (including warmups)", "", "| Trial | Status | Reasons |",
|
||||
"|---|---|---|"]
|
||||
for r in runs:
|
||||
label = r["label"] + (" (warmup)" if r.get("warmup") else "")
|
||||
reasons = "; ".join(r["reasons"]).replace("|", "\\|").replace("\n", " ")
|
||||
lines.append(f"| {label} | {r['status']} | {reasons or '–'} |")
|
||||
for error in meta.get("suite_errors", []):
|
||||
lines.append(f"\nSuite error: {error}")
|
||||
if not runs:
|
||||
lines.append("\nNo trials completed: suite is not successful.")
|
||||
for app in apps:
|
||||
if app.install_dir and Path(app.install_dir).exists():
|
||||
lines.append(f"\n{app.name} install size: {dir_size(Path(app.install_dir)) / MB:.0f} MB")
|
||||
timeouts = [r["label"] for r in runs if r.get("timed_out")]
|
||||
if timeouts:
|
||||
lines.append(f"\n⚠ CPU never settled within the timeout: {', '.join(timeouts)}")
|
||||
killed = [f"{r['label']}: {', '.join(r['killed_on_exit'])}" for r in runs if r.get("killed_on_exit")]
|
||||
if killed:
|
||||
lines.append("\n⚠ Killed after the exit timeout: " + "; ".join(killed))
|
||||
if app.install_dir:
|
||||
try:
|
||||
path = Path(app.install_dir)
|
||||
if path.exists():
|
||||
lines.append(f"\n{app.name} install size: {dir_size(path) / MB:.0f} MB")
|
||||
except OSError as exc:
|
||||
lines.append(f"\n{app.name} install size unavailable: {exc}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
@@ -535,7 +722,7 @@ def main():
|
||||
ap.add_argument("--config", type=Path, default=HERE / "apps.toml")
|
||||
ap.add_argument("--apps", help="comma-separated keys from the config (default: all)")
|
||||
ap.add_argument("--runs", type=int, default=5, help="measured runs per app (default 5)")
|
||||
ap.add_argument("--warmup", type=int, default=1, help="discarded runs per app first (default 1)")
|
||||
ap.add_argument("--warmup", type=int, default=1, help="unaggregated warmup runs per app first (default 1)")
|
||||
ap.add_argument("--wipe-caches", action="store_true", help="delete each app's cache_dirs before every run (cold index)")
|
||||
ap.add_argument("--drop-os-cache", metavar="CMD", help="shell command run before every run to drop the OS file cache")
|
||||
ap.add_argument("--quiet-secs", type=float, default=5, help="CPU must stay low this long to count as ready")
|
||||
@@ -543,10 +730,19 @@ def main():
|
||||
ap.add_argument("--idle-secs", type=float, default=30, help="idle period measured after ready")
|
||||
ap.add_argument("--timeout", type=float, default=900, help="give up waiting for ready after this many seconds")
|
||||
ap.add_argument("--close-timeout", type=float, default=60)
|
||||
ap.add_argument("--command-timeout", type=float, default=60, help="OS cache command deadline in seconds")
|
||||
ap.add_argument("--cooldown", type=float, default=5, help="pause before each launch")
|
||||
ap.add_argument("--out", type=Path, default=HERE / "results")
|
||||
ap.add_argument("--yes", action="store_true", help="don't ask before wiping caches")
|
||||
args = ap.parse_args()
|
||||
if args.runs <= 0 or args.warmup < 0:
|
||||
ap.error("--runs must be positive and --warmup must be nonnegative")
|
||||
for name in ("quiet_secs", "quiet_cores", "idle_secs", "timeout", "close_timeout", "command_timeout", "cooldown"):
|
||||
value = getattr(args, name)
|
||||
if not math.isfinite(value) or value < 0 or (name != "cooldown" and value == 0):
|
||||
ap.error(f"--{name.replace('_', '-')} must be finite and {'nonnegative' if name == 'cooldown' else 'positive'}")
|
||||
if args.quiet_secs > args.timeout:
|
||||
ap.error("--quiet-secs must not exceed --timeout")
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
@@ -555,6 +751,8 @@ def main():
|
||||
raise SystemExit(f"not a directory: {project}")
|
||||
config = tomllib.loads(args.config.read_text(encoding="utf-8"))
|
||||
keys = args.apps.split(",") if args.apps else list(config["apps"])
|
||||
if not keys or any(not key or key not in config["apps"] for key in keys) or len(set(keys)) != len(keys):
|
||||
ap.error("--apps must select at least one known app without duplicates")
|
||||
apps = [App(k, config["apps"][k], project) for k in keys]
|
||||
|
||||
if args.wipe_caches:
|
||||
@@ -567,8 +765,8 @@ def main():
|
||||
return
|
||||
|
||||
stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
out_dir = args.out / f"{stamp}-{project.name}"
|
||||
out_dir.mkdir(parents=True)
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
out_dir = Path(tempfile.mkdtemp(prefix=f"{stamp}-{project.name}-", dir=args.out))
|
||||
mode = "cold index" if args.wipe_caches else "warm"
|
||||
if args.drop_os_cache:
|
||||
mode += " + cold OS cache"
|
||||
@@ -586,29 +784,42 @@ def main():
|
||||
}
|
||||
|
||||
runs: list[dict] = []
|
||||
total = args.warmup + args.runs
|
||||
for i in range(total):
|
||||
# Rotate the order so neither app always runs right after the other.
|
||||
order = apps[i % len(apps) :] + apps[: i % len(apps)]
|
||||
for app in order:
|
||||
warm = i < args.warmup
|
||||
label = f"{app.key}-{'warmup' if warm else 'run'}{i + 1 if warm else i - args.warmup + 1}"
|
||||
print(f"[{label}] launching…", flush=True)
|
||||
r = run_once(app, args, out_dir, label)
|
||||
print(
|
||||
f"[{label}] window {fmt(r['window_s'])} s · ready {fmt(r['ready_s'])} s · "
|
||||
f"USS {r['uss_mb']:.0f} MB · idle CPU {r['idle_cpu_pct']:.1f}%",
|
||||
flush=True,
|
||||
)
|
||||
if not warm:
|
||||
previous_term = signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
|
||||
try:
|
||||
persist(apps, runs, meta, out_dir)
|
||||
total = args.warmup + args.runs
|
||||
for i in range(total):
|
||||
# Rotate the order so neither app always runs right after the other.
|
||||
order = apps[i % len(apps) :] + apps[: i % len(apps)]
|
||||
for app in order:
|
||||
warm = i < args.warmup
|
||||
label = f"{app.key}-{'warmup' if warm else 'run'}{i + 1 if warm else i - args.warmup + 1}"
|
||||
r = new_trial(app, label, warm)
|
||||
runs.append(r)
|
||||
(out_dir / "results.json").write_text(json.dumps({"meta": meta, "runs": runs}, indent=2), encoding="utf-8")
|
||||
|
||||
text = report(apps, runs, meta)
|
||||
(out_dir / "report.md").write_text(text, encoding="utf-8")
|
||||
print("\n" + text)
|
||||
persist(apps, runs, meta, out_dir)
|
||||
print(f"[{label}] launching…", flush=True)
|
||||
run_once(app, args, out_dir, label, r)
|
||||
persist(apps, runs, meta, out_dir)
|
||||
print(f"[{label}] {r['status']} · window {fmt(r['window_s'])} s · ready {fmt(r['ready_s'])} s", flush=True)
|
||||
if r["status"] == "interrupted":
|
||||
raise KeyboardInterrupt
|
||||
except KeyboardInterrupt:
|
||||
meta.setdefault("suite_errors", []).append("interrupted")
|
||||
for r in runs:
|
||||
if r["status"] == "running":
|
||||
r["status"] = "interrupted"
|
||||
failure(r, "interrupted before launch completed")
|
||||
except Exception as exc:
|
||||
meta.setdefault("suite_errors", []).append(f"{type(exc).__name__}: {exc}")
|
||||
for r in runs:
|
||||
if r["status"] == "running":
|
||||
failure(r, "suite aborted before launch completed")
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, previous_term)
|
||||
persist(apps, runs, meta, out_dir)
|
||||
print(f"Raw results: {out_dir / 'results.json'}")
|
||||
return 0 if meta["status"] == "valid" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Small real-process regressions; no installed IDE or window server required."""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import psutil
|
||||
|
||||
spec = importlib.util.spec_from_file_location("compare_bench", Path(__file__).with_name("bench.py"))
|
||||
bench = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(bench)
|
||||
|
||||
|
||||
class TrialTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.out = Path(self.temp.name)
|
||||
self.args = argparse.Namespace(wipe_caches=False, drop_os_cache=None, cooldown=0,
|
||||
quiet_secs=0.03, quiet_cores=100, timeout=0.3, idle_secs=0.03, close_timeout=0.3,
|
||||
command_timeout=0.3)
|
||||
self.windows = patch.object(bench, "visible_windows", return_value=None)
|
||||
self.windows.start()
|
||||
self.addCleanup(self.windows.stop)
|
||||
poll = patch.object(bench, "POLL_S", 0.01)
|
||||
poll.start()
|
||||
self.addCleanup(poll.stop)
|
||||
|
||||
def app(self, source, **extra):
|
||||
return bench.App("fixture", {"command": sys.executable, "args": ["-c", source],
|
||||
"single_instance": False, **extra}, self.out)
|
||||
|
||||
def assert_failed(self, result, reason):
|
||||
self.assertNotEqual(result["status"], "valid")
|
||||
self.assertTrue(any(reason in r for r in result["reasons"]), result)
|
||||
|
||||
def test_crash_preserves_attempt(self):
|
||||
result = bench.run_once(self.app("import sys; print('before crash', flush=True); sys.exit(7)"),
|
||||
self.args, self.out, "crash")
|
||||
self.assert_failed(result, "exited")
|
||||
self.assertEqual(result["exit_code"], 7)
|
||||
self.assertIn("before crash", (self.out / "crash.stdout.log").read_text())
|
||||
|
||||
def test_startup_timeout(self):
|
||||
app = self.app("import time; time.sleep(30)", markers={"ready": "NEVER"}, ready_after=["ready"])
|
||||
result = bench.run_once(app, self.args, self.out, "timeout")
|
||||
self.assert_failed(result, "startup timeout")
|
||||
self.assertIsNone(result["ready_s"])
|
||||
self.assertEqual(result["metric_status"]["window_s"], "unsupported")
|
||||
|
||||
def test_blocked_quit_is_bounded(self):
|
||||
app = self.app("import time; time.sleep(30)",
|
||||
quit_command=[sys.executable, "-c", "import time; time.sleep(30)"])
|
||||
start = time.monotonic()
|
||||
result = bench.run_once(app, self.args, self.out, "blocked")
|
||||
self.assertLess(time.monotonic() - start, 4)
|
||||
self.assert_failed(result, "quit command")
|
||||
self.assertTrue(result["killed_on_exit"])
|
||||
|
||||
def test_observation_failure_and_interruption_clean_children(self):
|
||||
for error in (RuntimeError("observer failed"), psutil.AccessDenied(os.getpid()), KeyboardInterrupt()):
|
||||
with self.subTest(error=type(error).__name__):
|
||||
pidfile = self.out / "child.pid"
|
||||
pidfile.unlink(missing_ok=True)
|
||||
source = ("import subprocess,sys,time; "
|
||||
"p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)']); "
|
||||
f"open({str(pidfile)!r},'w').write(str(p.pid)); time.sleep(30)")
|
||||
original = bench.Tree.sample
|
||||
triggered = False
|
||||
|
||||
def observe(tree):
|
||||
nonlocal triggered
|
||||
sampled = original(tree)
|
||||
if pidfile.exists() and not triggered:
|
||||
triggered = True
|
||||
raise error
|
||||
return sampled
|
||||
|
||||
with patch.object(bench.Tree, "sample", observe):
|
||||
result = bench.run_once(self.app(source), self.args, self.out, "observe")
|
||||
self.assertTrue(triggered)
|
||||
self.assertNotEqual(result["status"], "valid")
|
||||
child = psutil.Process(int(pidfile.read_text())) if psutil.pid_exists(int(pidfile.read_text())) else None
|
||||
if child:
|
||||
self.assertEqual(child.status(), psutil.STATUS_ZOMBIE)
|
||||
|
||||
def test_existing_instance_is_not_terminated(self):
|
||||
proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])
|
||||
try:
|
||||
with patch.object(bench, "running_instances", return_value=[psutil.Process(proc.pid)]):
|
||||
app = self.app("raise AssertionError('must not launch')")
|
||||
app.single_instance = True
|
||||
result = bench.run_once(app, self.args, self.out, "preexisting")
|
||||
self.assert_failed(result, "already running")
|
||||
self.assertIsNone(proc.poll())
|
||||
finally:
|
||||
proc.kill()
|
||||
proc.wait(timeout=2)
|
||||
|
||||
def test_report_keeps_empty_rows_and_denominators(self):
|
||||
app = self.app("pass")
|
||||
runs = []
|
||||
for index, value in enumerate((1.0, None, math.inf)):
|
||||
result = bench.new_trial(app, str(index))
|
||||
result.update(status="valid", ready_s=value)
|
||||
bench.validate_metrics(app, result)
|
||||
runs.append(result)
|
||||
runs[0].update({key: 1.0 for key, _ in bench.METRICS})
|
||||
runs[0]["procs"] = 1
|
||||
runs[0]["status"] = "valid"
|
||||
runs[0]["reasons"] = []
|
||||
bench.validate_metrics(app, runs[0])
|
||||
text = bench.report([app], runs, {"project": "fixture", "os": "test", "cpu": "test", "cores": 1,
|
||||
"ram_gb": 1, "mode": "warm", "runs": 3, "date": "today", "status": "failed"})
|
||||
self.assertIn("1/3", text)
|
||||
self.assertIn("failed", text)
|
||||
self.assertIn("PSS after idle", text)
|
||||
self.assertIn("0/3", text)
|
||||
self.assertNotIn("inf", text)
|
||||
encoded = json.dumps(bench.json_safe(runs), allow_nan=False)
|
||||
self.assertIn("invalid_number", encoded)
|
||||
self.assertEqual(bench.suite_status([app], [], {"runs": 1}), "failed")
|
||||
|
||||
def test_cache_command_timeout_cleans_its_child(self):
|
||||
pidfile = self.out / "cache-child.pid"
|
||||
source = ("import subprocess,sys,time; "
|
||||
"p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)']); "
|
||||
f"open({str(pidfile)!r},'w').write(str(p.pid)); time.sleep(30)")
|
||||
with self.assertRaises(subprocess.TimeoutExpired):
|
||||
bench.bounded_command([sys.executable, "-c", source], 0.3)
|
||||
self.assertTrue(pidfile.exists())
|
||||
pid = int(pidfile.read_text())
|
||||
if psutil.pid_exists(pid):
|
||||
self.assertEqual(psutil.Process(pid).status(), psutil.STATUS_ZOMBIE)
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "CLI SIGINT delivery requires POSIX")
|
||||
def test_cli_interruption_preserves_reports_and_cleans_child(self):
|
||||
pidfile = self.out / "cli-child.pid"
|
||||
source = ("import subprocess,sys,time; "
|
||||
"p=subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)']); "
|
||||
f"open({str(pidfile)!r},'w').write(str(p.pid)); time.sleep(30)")
|
||||
config = self.out / "interrupt.toml"
|
||||
config.write_text("[apps.fixture]\ncommand = " + json.dumps(sys.executable)
|
||||
+ "\nargs = " + json.dumps(["-c", source]) + "\nsingle_instance = false\n")
|
||||
proc = subprocess.Popen([sys.executable, str(Path(bench.__file__)), str(self.out),
|
||||
"--config", str(config), "--runs", "1", "--warmup", "0", "--cooldown", "0",
|
||||
"--close-timeout", "0.3", "--out", str(self.out / "results")],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
try:
|
||||
deadline = time.monotonic() + 5
|
||||
while not pidfile.exists() and proc.poll() is None and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self.assertTrue(pidfile.exists(), "fixture did not launch")
|
||||
proc.send_signal(signal.SIGINT)
|
||||
proc.communicate(timeout=5)
|
||||
self.assertNotEqual(proc.returncode, 0)
|
||||
output = next((self.out / "results").iterdir())
|
||||
raw = json.loads((output / "results.json").read_text())
|
||||
self.assertEqual(raw["runs"][0]["status"], "interrupted")
|
||||
self.assertEqual(raw["meta"]["status"], "failed")
|
||||
self.assertIn("interrupted", (output / "report.md").read_text())
|
||||
pid = int(pidfile.read_text())
|
||||
if psutil.pid_exists(pid):
|
||||
self.assertEqual(psutil.Process(pid).status(), psutil.STATUS_ZOMBIE)
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
proc.communicate(timeout=3)
|
||||
# A regression must not leak the fixture even when an assertion fails.
|
||||
if pidfile.exists():
|
||||
pid = int(pidfile.read_text())
|
||||
try:
|
||||
psutil.Process(pid).kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
def test_cli_failed_launch_writes_both_reports_and_nonzero(self):
|
||||
config = self.out / "apps.toml"
|
||||
config.write_text('[apps.fixture]\ncommand = "/missing/benchmark-fixture"\nargs = []\nsingle_instance = false\n')
|
||||
result = subprocess.run([sys.executable, str(Path(bench.__file__)), str(self.out),
|
||||
"--config", str(config), "--runs", "1", "--warmup", "1", "--cooldown", "0",
|
||||
"--out", str(self.out / "results")], capture_output=True, text=True, timeout=5)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
output = next((self.out / "results").iterdir())
|
||||
raw = json.loads((output / "results.json").read_text())
|
||||
self.assertEqual(len(raw["runs"]), 2)
|
||||
self.assertTrue(raw["runs"][0]["warmup"])
|
||||
self.assertEqual(raw["meta"]["status"], "failed")
|
||||
self.assertIn("failed", (output / "report.md").read_text())
|
||||
|
||||
def test_invalid_numeric_arguments_do_not_launch(self):
|
||||
for flag, value in (("--runs", "0"), ("--warmup", "-1"), ("--idle-secs", "nan"),
|
||||
("--close-timeout", "inf"), ("--quiet-cores", "0")):
|
||||
result = subprocess.run([sys.executable, str(Path(bench.__file__)), str(self.out), flag, value,
|
||||
"--out", str(self.out / "results")],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse((self.out / "results").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,21 +1,16 @@
|
||||
/** The workbench's `--mode perf` production bundle: building it and serving it to Chromium. */
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { runCommand } from "./process.ts";
|
||||
import { createReadStream, existsSync, statSync } from "node:fs";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { extname, join } from "node:path";
|
||||
|
||||
export function buildBundle(root: string, outDir: string, flags: string[] = []) {
|
||||
export async function buildBundle(root: string, outDir: string, flags: string[] = []) {
|
||||
console.log("Building the workbench (vite build --mode perf)…");
|
||||
const r = spawnSync(
|
||||
"pnpm",
|
||||
await runCommand(
|
||||
process.platform === "win32" ? "pnpm.cmd" : "pnpm",
|
||||
["exec", "vite", "build", "--mode", "perf", ...flags, "--outDir", outDir, "--emptyOutDir"],
|
||||
{
|
||||
cwd: root,
|
||||
stdio: ["ignore", "ignore", "inherit"],
|
||||
shell: process.platform === "win32",
|
||||
},
|
||||
{ cwd: root, timeoutMs: 5 * 60_000, label: "vite build" },
|
||||
);
|
||||
if (r.status !== 0) throw new Error("vite build failed");
|
||||
}
|
||||
|
||||
const MIME: Record<string, string> = {
|
||||
@@ -38,10 +33,11 @@ export function serveBundle(dir: string): Promise<{ server: Server; url: string
|
||||
res.writeHead(200, { "content-type": MIME[extname(file)] ?? "application/octet-stream" });
|
||||
createReadStream(file).pipe(res);
|
||||
});
|
||||
return new Promise((ok) =>
|
||||
return new Promise((ok, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const addr = server.address();
|
||||
ok({ server, url: `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}` });
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { EditorSelection, EditorState, Prec } from "@codemirror/state";
|
||||
import { EditorView, keymap } from "@codemirror/view";
|
||||
import { defaultKeymap, indentWithTab } from "@codemirror/commands";
|
||||
import { autocompletion, closeBrackets, closeBracketsKeymap } from "@codemirror/autocomplete";
|
||||
import { javascript } from "@codemirror/lang-javascript";
|
||||
import { indentOnInput } from "@codemirror/language";
|
||||
import type { InPage } from "./inpage.ts";
|
||||
import type { EditorProbe } from "./editing.ts";
|
||||
|
||||
// The fixture owns both browser globals and exposes the same editor observation boundary as the app.
|
||||
const w = window as unknown as {
|
||||
__fixture: Record<string, boolean>;
|
||||
__openideaPerf: EditorProbe;
|
||||
__perf?: InPage;
|
||||
__releaseOpen?: () => void;
|
||||
};
|
||||
const flags: Record<string, boolean> = {};
|
||||
w.__fixture = flags;
|
||||
const files = new Map<string, string>();
|
||||
let path = "";
|
||||
let view: EditorView;
|
||||
const style = document.createElement("style");
|
||||
style.textContent = ".cm-editor { height: 240px; width: 700px; } .cm-scroller { overflow: auto; }";
|
||||
document.head.append(style);
|
||||
|
||||
// Registered before the measurement listener, so a lost recording is tested independently of editing.
|
||||
window.addEventListener(
|
||||
"keydown",
|
||||
() => {
|
||||
if (flags.missingSample && w.__perf) w.__perf.recordKeys = false;
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
||||
function mount(text: string) {
|
||||
view?.destroy();
|
||||
view = new EditorView({
|
||||
parent: document.querySelector("#editor")!,
|
||||
state: EditorState.create({
|
||||
doc: text,
|
||||
extensions: [
|
||||
javascript(),
|
||||
indentOnInput(),
|
||||
closeBrackets(),
|
||||
Prec.highest(
|
||||
keymap.of([
|
||||
{ key: "Mod-End", run: () => !!flags.disableJump },
|
||||
{ key: "ArrowDown", run: () => !!flags.disableArrows },
|
||||
{ key: "x", run: () => !!flags.swallowTyping },
|
||||
]),
|
||||
),
|
||||
keymap.of([...closeBracketsKeymap, ...defaultKeymap, indentWithTab]),
|
||||
autocompletion({
|
||||
// Test movement itself, without CodeMirror's accidental-key guard after opening.
|
||||
interactionDelay: 0,
|
||||
override: [
|
||||
(ctx) => {
|
||||
const word = ctx.matchBefore(/\w*/);
|
||||
return {
|
||||
from: word?.from ?? ctx.pos,
|
||||
options: Array.from({ length: 50 }, (_, i) => ({ label: `getOrder${String(i).padStart(2, "0")}` })),
|
||||
};
|
||||
},
|
||||
],
|
||||
}),
|
||||
EditorView.updateListener.of((update) => {
|
||||
if (flags.wrongCaret && update.docChanged)
|
||||
queueMicrotask(() => view.dispatch({ selection: EditorSelection.cursor(0) }));
|
||||
if (flags.mutateDistant && update.docChanged) {
|
||||
flags.mutateDistant = false;
|
||||
const end = view.state.doc.length;
|
||||
queueMicrotask(() => view.dispatch({ changes: { from: end - 1, to: end, insert: "z" } }));
|
||||
}
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
view.focus();
|
||||
}
|
||||
|
||||
w.__openideaPerf = {
|
||||
seed: (p, text) => {
|
||||
files.set(p, text);
|
||||
},
|
||||
async openFile(p) {
|
||||
if (flags.rejectOpen) throw new Error("open failed");
|
||||
if (flags.ignoreOpen) return;
|
||||
if (flags.delayOpen)
|
||||
await new Promise<void>((resolve) => {
|
||||
w.__releaseOpen = resolve;
|
||||
});
|
||||
path = p;
|
||||
mount(flags.wrongContents ? view.state.doc.toString() : (files.get(p) ?? ""));
|
||||
let tab = document.querySelector<HTMLElement>('[role="tab"]');
|
||||
if (!tab) {
|
||||
tab = document.createElement("div");
|
||||
tab.setAttribute("role", "tab");
|
||||
tab.setAttribute("aria-selected", "true");
|
||||
document.body.prepend(tab);
|
||||
}
|
||||
tab.textContent = p.slice(p.lastIndexOf("/") + 1);
|
||||
},
|
||||
editorText: (from = 0, to) => view?.state.doc.sliceString(from, to) ?? null,
|
||||
editorState() {
|
||||
if (!view) return null;
|
||||
const head = view.state.selection.main.head;
|
||||
return {
|
||||
path,
|
||||
head,
|
||||
length: view.state.doc.length,
|
||||
lines: view.state.doc.lines,
|
||||
line: view.state.doc.lineAt(head).number - 1,
|
||||
word: "",
|
||||
};
|
||||
},
|
||||
lineAtY(y) {
|
||||
const block = view.lineBlockAtHeight(y - view.documentTop);
|
||||
return view.state.doc.lineAt(block.from).number - 1;
|
||||
},
|
||||
setCursor(head) {
|
||||
view.dispatch({ selection: EditorSelection.cursor(head), scrollIntoView: true });
|
||||
view.focus();
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,215 @@
|
||||
// @vitest-environment node
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { chromium, type Browser, type BrowserContext, type Page } from "@playwright/test";
|
||||
import { createServer, type ViteDevServer } from "vite";
|
||||
import { Harness } from "./harness.ts";
|
||||
import { installInPage } from "./inpage.ts";
|
||||
|
||||
let server: ViteDevServer;
|
||||
let browser: Browser;
|
||||
let context: BrowserContext;
|
||||
let page: Page;
|
||||
let h: Harness;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await createServer({
|
||||
configFile: false,
|
||||
root: fileURLToPath(new URL("../../", import.meta.url)),
|
||||
server: { host: "127.0.0.1", port: 0 },
|
||||
optimizeDeps: { entries: ["perf/lib/causality.fixture.ts"] },
|
||||
plugins: [
|
||||
{
|
||||
name: "causality-fixture",
|
||||
configureServer(s) {
|
||||
s.middlewares.use((req, res, next) => {
|
||||
if (req.url !== "/") return next();
|
||||
res.setHeader("Content-Type", "text/html");
|
||||
res.end(
|
||||
'<div id="editor"></div><input id="other"><div id="result">ready</div><script type="module" src="/perf/lib/causality.fixture.ts"></script>',
|
||||
);
|
||||
});
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await server.listen();
|
||||
browser = await chromium.launch({ channel: process.env.OPENIDEA_PERF_CHANNEL ?? "chromium" });
|
||||
}, 30_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
context = await browser.newContext();
|
||||
page = await context.newPage();
|
||||
await page.goto(server.resolvedUrls!.local[0]!);
|
||||
await page.waitForFunction(() => "__openideaPerf" in window);
|
||||
await page.evaluate(installInPage);
|
||||
const cdp = await context.newCDPSession(page);
|
||||
h = new Harness(page, cdp);
|
||||
});
|
||||
afterEach(async () => {
|
||||
await context?.close();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await browser?.close();
|
||||
await server?.close();
|
||||
});
|
||||
|
||||
async function mode(name: string) {
|
||||
await page.evaluate((name) => {
|
||||
const w = window as unknown as { __fixture: Record<string, boolean> };
|
||||
w.__fixture[name] = true;
|
||||
}, name);
|
||||
}
|
||||
|
||||
async function fixtureFile(text = "start\nend") {
|
||||
await h.openFile("/one/Same.ts", text);
|
||||
await h.setCursor(0);
|
||||
}
|
||||
|
||||
describe("current-operation UI measurements", () => {
|
||||
it("observes startup without input, but rejects pre-existing DOM even after a no-op key", async () => {
|
||||
expect(await h.inPage((p) => p.waitShown("#result", "ready", 500))).toBeGreaterThanOrEqual(0);
|
||||
await expect(h.measure(async () => {}, "#result", "ready", 80)).rejects.toThrow("timed out");
|
||||
await expect(
|
||||
h.measure(
|
||||
() =>
|
||||
page.evaluate(() => {
|
||||
document.querySelector("#result")!.textContent = "ready before input";
|
||||
}),
|
||||
"#result",
|
||||
"ready",
|
||||
80,
|
||||
),
|
||||
).rejects.toThrow("timed out");
|
||||
await expect(
|
||||
h.measure(
|
||||
async () => {
|
||||
await page.evaluate(() => {
|
||||
document.querySelector("#result")!.textContent = "ready changed before input";
|
||||
});
|
||||
await page.keyboard.press("Shift");
|
||||
},
|
||||
"#result",
|
||||
"ready",
|
||||
80,
|
||||
),
|
||||
).rejects.toThrow("timed out");
|
||||
await expect(h.measure(() => page.keyboard.press("Shift"), "#result", "ready", 80)).rejects.toThrow("timed out");
|
||||
});
|
||||
|
||||
it("waits for input and the resulting target change, not an earlier frame", async () => {
|
||||
let changedAt = 0;
|
||||
const result = h.measure(
|
||||
async () => {
|
||||
await h.inPage((p) => p.afterNextPaint());
|
||||
await page.keyboard.press("a");
|
||||
await h.inPage((p) => p.afterNextPaint());
|
||||
changedAt = await page.evaluate(() => {
|
||||
document.querySelector("#result")!.textContent = "ready now";
|
||||
return performance.now();
|
||||
});
|
||||
},
|
||||
"#result",
|
||||
"ready",
|
||||
1_000,
|
||||
);
|
||||
const latency = await result;
|
||||
expect((await h.inPage((p) => p.lastInput)) + latency).toBeGreaterThanOrEqual(changedAt);
|
||||
});
|
||||
|
||||
it("propagates trigger failure and cancels its waiter without browser rejection", async () => {
|
||||
const errors: string[] = [];
|
||||
page.on("pageerror", (error) => errors.push(error.message));
|
||||
await expect(
|
||||
h.measure(
|
||||
async () => {
|
||||
throw new Error("trigger failed");
|
||||
},
|
||||
"#missing",
|
||||
null,
|
||||
80,
|
||||
),
|
||||
).rejects.toThrow("trigger failed");
|
||||
await expect(h.measure(async () => false, "#missing", null, 80)).rejects.toThrow("returned false");
|
||||
await h.inPage(async (p) => {
|
||||
await p.afterNextPaint();
|
||||
await p.afterNextPaint();
|
||||
});
|
||||
expect(errors).toEqual([]);
|
||||
expect(await h.inPage((p) => p.waitShown("#result", "ready", 80))).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it.each(["ignoreOpen", "wrongContents"])("requires the requested path and seeded contents with %s", async (fault) => {
|
||||
await fixtureFile();
|
||||
await mode(fault);
|
||||
await expect(h.openFile("/two/Same.ts", "different", 100)).rejects.toThrow("requested editor");
|
||||
});
|
||||
|
||||
it("does not finish before the open action succeeds, even with matching existing contents", async () => {
|
||||
await fixtureFile();
|
||||
await mode("delayOpen");
|
||||
let completed = false;
|
||||
const opening = h.openFile("/one/Same.ts", "start\nend").finally(() => {
|
||||
completed = true;
|
||||
});
|
||||
await page.waitForFunction(() => "__releaseOpen" in window);
|
||||
try {
|
||||
await h.inPage((p) => p.afterNextPaint());
|
||||
expect(completed).toBe(false);
|
||||
} finally {
|
||||
await page.evaluate(() => {
|
||||
if (!("__releaseOpen" in window) || typeof window.__releaseOpen !== "function")
|
||||
throw new Error("open action was not held");
|
||||
window.__releaseOpen();
|
||||
});
|
||||
await opening;
|
||||
}
|
||||
});
|
||||
|
||||
it("awaits the open action and rejects failed actions even with a matching editor", async () => {
|
||||
await fixtureFile();
|
||||
await mode("rejectOpen");
|
||||
await expect(h.openFile("/one/Same.ts")).rejects.toThrow("open failed");
|
||||
});
|
||||
|
||||
it("measures real typing, including automatic brackets, indentation and backspace", async () => {
|
||||
await fixtureFile("function f() {\n}\n");
|
||||
await h.setCursor(14);
|
||||
expect(await h.typeMeasured("const n = (1 + 2);\n\b", 0)).toHaveLength(20);
|
||||
});
|
||||
|
||||
it.each(["swallowTyping", "wrongCaret", "missingSample", "wrongFocus"])("rejects typing with %s", async (fault) => {
|
||||
await fixtureFile();
|
||||
if (fault === "wrongFocus") await page.locator("#other").click();
|
||||
else await mode(fault);
|
||||
await expect(h.typeMeasured("x", 0)).rejects.toThrow(/typing|caret|sample|focus/);
|
||||
});
|
||||
|
||||
it("checks the whole document after bounded per-key observations", async () => {
|
||||
await fixtureFile("start\n" + "unchanged\n".repeat(200) + "end");
|
||||
expect(await h.typeMeasured("ok", 0)).toHaveLength(2);
|
||||
await mode("mutateDistant");
|
||||
await expect(h.typeMeasured("x", 0)).rejects.toThrow("outside the intended insertion");
|
||||
});
|
||||
|
||||
it("requires both cursor destination and viewport displacement for the jump", async () => {
|
||||
await fixtureFile(Array.from({ length: 1_000 }, (_, i) => `export class Repository${i} {}`).join("\n"));
|
||||
await mode("disableJump");
|
||||
await expect(h.jumpToEnd(100)).rejects.toThrow("timed out");
|
||||
await page.evaluate(() => {
|
||||
const w = window as unknown as { __fixture: Record<string, boolean> };
|
||||
w.__fixture.disableJump = false;
|
||||
});
|
||||
expect(await h.jumpToEnd(1_000)).toBeGreaterThanOrEqual(0);
|
||||
expect((await h.editorState())!.head).toBe((await h.editorState())!.length);
|
||||
});
|
||||
|
||||
it("requires each completion arrow to advance exactly one selected item", async () => {
|
||||
await fixtureFile("");
|
||||
await h.measure(() => page.keyboard.press("g"), ".cm-tooltip-autocomplete li", "getOrder", 2_000);
|
||||
await page.waitForSelector('.cm-tooltip-autocomplete li[aria-selected="true"]');
|
||||
expect(await h.completionArrowsMeasured(3, 0)).toHaveLength(3);
|
||||
await mode("disableArrows");
|
||||
await expect(h.completionArrowsMeasured(1, 0)).rejects.toThrow("completion selection");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
export interface EditorProbe {
|
||||
seed(path: string, text: string): void;
|
||||
openFile(path: string): Promise<void>;
|
||||
editorState(): { path: string; length: number; lines: number; head: number; line: number; word: string } | null;
|
||||
editorText(from?: number, to?: number): string | null;
|
||||
setCursor(offset: number): void;
|
||||
lineAtY(y: number): number;
|
||||
}
|
||||
|
||||
export interface EditableSnapshot {
|
||||
kind: "editor" | "input";
|
||||
target: string;
|
||||
text: string;
|
||||
head: number;
|
||||
from: number;
|
||||
to: number;
|
||||
}
|
||||
|
||||
export interface EditableRange {
|
||||
from: number;
|
||||
to: number;
|
||||
length: number;
|
||||
}
|
||||
|
||||
/** Runs in the page: a key latency is not evidence that the intended editable received it. */
|
||||
export function readEditable(range?: EditableRange): EditableSnapshot {
|
||||
const active = document.activeElement;
|
||||
if (active instanceof HTMLInputElement || active instanceof HTMLTextAreaElement) {
|
||||
if (active.selectionStart === null || active.selectionEnd === null)
|
||||
throw new Error("typing focus is not a text input");
|
||||
return {
|
||||
kind: "input",
|
||||
target: active.id || active.getAttribute("aria-label") || active.tagName,
|
||||
text: active.value,
|
||||
head: active.selectionEnd,
|
||||
from: active.selectionStart,
|
||||
to: active.selectionEnd,
|
||||
};
|
||||
}
|
||||
if (!active?.closest(".cm-content")) throw new Error("typing focus is not in the editor");
|
||||
// The app and browser fixture expose this typed, in-process observation boundary.
|
||||
const w = window as unknown as { __openideaPerf: EditorProbe };
|
||||
const state = w.__openideaPerf.editorState();
|
||||
if (!state) throw new Error("typing editor is unavailable");
|
||||
const text = w.__openideaPerf.editorText(range?.from, range ? range.to + state.length - range.length : undefined);
|
||||
if (text === null) throw new Error("typing editor is unavailable");
|
||||
const head = state.head - (range?.from ?? 0);
|
||||
return { kind: "editor", target: state.path, text, head, from: head, to: head };
|
||||
}
|
||||
|
||||
export function assertTypedKey(before: EditableSnapshot, after: EditableSnapshot, key: string) {
|
||||
const fail = () => {
|
||||
throw new Error(`typing ${JSON.stringify(key)} did not produce the expected text and caret in ${before.target}`);
|
||||
};
|
||||
if (before.kind !== after.kind || before.target !== after.target) fail();
|
||||
const left = before.text.slice(0, before.from);
|
||||
const right = before.text.slice(before.to);
|
||||
if (before.kind === "input") {
|
||||
const from = key === "\b" && before.from === before.to ? Math.max(0, before.from - 1) : before.from;
|
||||
const inserted = key === "\b" ? "" : key;
|
||||
if (after.text !== before.text.slice(0, from) + inserted + right || after.head !== from + inserted.length) fail();
|
||||
return;
|
||||
}
|
||||
const afterLeft = after.text.slice(0, after.head);
|
||||
const afterRight = after.text.slice(after.head);
|
||||
if (key === "\b") {
|
||||
const removed = left.slice(afterLeft.length);
|
||||
const pair = ({ "(": ")", "[": "]", "{": "}", '"': '"', "'": "'" } as Record<string, string>)[removed];
|
||||
if (
|
||||
after.head >= before.head ||
|
||||
!left.startsWith(afterLeft) ||
|
||||
!removed ||
|
||||
([...removed].length !== 1 && !/^[\t ]+$/.test(removed)) ||
|
||||
(afterRight !== right && !(pair && right.startsWith(pair) && afterRight === right.slice(pair.length)))
|
||||
)
|
||||
fail();
|
||||
return;
|
||||
}
|
||||
// CodeMirror may adjust indentation, insert a matching bracket, or step over an auto-inserted closer.
|
||||
// All non-indentation text, the insertion point, and the remaining document must still match.
|
||||
const unindent = (text: string) => text.replace(/^[\t ]+/gm, "");
|
||||
if (after.head === before.head || unindent(afterLeft) !== unindent(left + key)) fail();
|
||||
const closers: Record<string, string> = { "(": ")", "[": "]", "{": "}", '"': '"', "'": "'", "`": "`" };
|
||||
const closer = closers[key];
|
||||
const suffix = unindent(afterRight);
|
||||
const previous = unindent(right);
|
||||
if (suffix === previous) return;
|
||||
if (closer && suffix === closer + previous) return;
|
||||
if (
|
||||
/[)\]}"'`]/.test(key) &&
|
||||
right.startsWith(key) &&
|
||||
unindent(after.text) === unindent(before.text) &&
|
||||
suffix === unindent(right.slice(key.length))
|
||||
)
|
||||
return;
|
||||
if (key === "\n" && suffix === "\n" + previous) return;
|
||||
fail();
|
||||
}
|
||||
|
||||
/** Runs in the page; shared by UI performance scenarios and code-intelligence evals. */
|
||||
export async function openFileMeasured({ path, text, timeoutMs }: { path: string; text?: string; timeoutMs: number }) {
|
||||
const w = window as unknown as { __openideaPerf: EditorProbe };
|
||||
const t0 = performance.now();
|
||||
let timer: number | undefined;
|
||||
const expired = new Promise<never>((_, reject) => {
|
||||
timer = window.setTimeout(() => reject(new Error(`open action timed out for ${path}`)), timeoutMs);
|
||||
});
|
||||
let result: unknown;
|
||||
try {
|
||||
result = await Promise.race([w.__openideaPerf.openFile(path), expired]);
|
||||
} finally {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
if (result === false) throw new Error(`open action failed for ${path}`);
|
||||
while (performance.now() - t0 <= timeoutMs) {
|
||||
await new Promise(requestAnimationFrame);
|
||||
const state = w.__openideaPerf.editorState();
|
||||
const content = w.__openideaPerf.editorText();
|
||||
const editor = document.activeElement?.closest(".cm-editor");
|
||||
const lines = [...(editor?.querySelectorAll<HTMLElement>(".cm-content .cm-line") ?? [])];
|
||||
if (
|
||||
state?.path === path &&
|
||||
content !== null &&
|
||||
state.length === content.length &&
|
||||
(text === undefined || content === text) &&
|
||||
lines.length &&
|
||||
lines.some((line) => line.getClientRects().length && content.includes(line.textContent ?? ""))
|
||||
) {
|
||||
return performance.now() - t0;
|
||||
}
|
||||
}
|
||||
throw new Error(`requested editor ${path} did not show its contents`);
|
||||
}
|
||||
+164
-38
@@ -2,6 +2,7 @@
|
||||
import type { CDPSession, Page } from "@playwright/test";
|
||||
import type { InPage } from "./inpage.ts";
|
||||
import { median } from "./stats.ts";
|
||||
import { assertTypedKey, openFileMeasured, readEditable, type EditableRange, type EditorProbe } from "./editing.ts";
|
||||
|
||||
export type Metrics = Record<string, number>;
|
||||
|
||||
@@ -69,45 +70,182 @@ export class Harness {
|
||||
for (const k of await this.keysFor(actionId)) await this.page.keyboard.press(k);
|
||||
}
|
||||
|
||||
/** Arms latency capture: the next input starts the clock, the paint showing `selector` stops it. */
|
||||
async measure(trigger: () => Promise<unknown>, selector: string, text: string | null = null, timeoutMs = 10_000) {
|
||||
await this.inPage((p) => p.arm());
|
||||
const done = this.inPage((p, a) => p.waitShown(a.selector, a.text, a.timeoutMs), { selector, text, timeoutMs });
|
||||
await trigger();
|
||||
return done;
|
||||
/** Measures a current input and a changed, matching target at frame start. */
|
||||
async measure<A = undefined>(
|
||||
trigger: () => Promise<unknown>,
|
||||
selector: string,
|
||||
text: string | null = null,
|
||||
timeoutMs = 10_000,
|
||||
check?: (arg: A) => boolean,
|
||||
arg?: A,
|
||||
) {
|
||||
const token = await this.inPage((p) => p.arm());
|
||||
// Attach both handlers immediately: a failed trigger must not strand a rejecting page evaluation.
|
||||
const done = this.inPage(
|
||||
(p, a) => {
|
||||
const condition = a.check ? new Function("arg", `return (${a.check})(arg)`) : null;
|
||||
return p.waitShown(a.selector, a.text, a.timeoutMs, a.token, condition ? () => condition(a.arg) : undefined);
|
||||
},
|
||||
{ selector, text, timeoutMs, token, check: check?.toString(), arg },
|
||||
).then(
|
||||
(value) => ({ ok: true as const, value }),
|
||||
(error: unknown) => ({ ok: false as const, error }),
|
||||
);
|
||||
try {
|
||||
if ((await trigger()) === false) throw new Error("measurement trigger returned false");
|
||||
const result = await done;
|
||||
if (!result.ok) throw result.error;
|
||||
return result.value;
|
||||
} finally {
|
||||
await this.inPage((p, token) => p.cancel(token), token);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Types `text` one key at a time, `intervalMs` apart, recording keydown → next
|
||||
* frame for each key. `\n` presses Enter, `\b` Backspace.
|
||||
*/
|
||||
async typeMeasured(text: string, intervalMs: number): Promise<number[]> {
|
||||
return this.pressMeasured(
|
||||
[...text].map((ch) => (ch === "\n" ? "Enter" : ch === "\b" ? "Backspace" : ch)),
|
||||
async typeMeasured(text: string, intervalMs: number, target: "editor" | "input" = "editor"): Promise<number[]> {
|
||||
const initial = await this.page.evaluate(readEditable, undefined);
|
||||
if (initial.kind !== target) throw new Error(`typing focus is not in the expected ${target}`);
|
||||
const characters = [...text];
|
||||
let range: EditableRange | undefined;
|
||||
let before = initial;
|
||||
if (initial.kind === "editor") {
|
||||
let from = initial.text.lastIndexOf("\n", initial.head - 1) + 1;
|
||||
// A Backspace can cross at most one existing line; indentation deletion stays within it.
|
||||
for (const character of characters) {
|
||||
if (character === "\b" && from > 0) from = initial.text.lastIndexOf("\n", from - 2) + 1;
|
||||
}
|
||||
const nextLine = initial.text.indexOf("\n", Math.min(initial.text.length, initial.head + characters.length + 1));
|
||||
const to = nextLine < 0 ? initial.text.length : nextLine + 1;
|
||||
range = { from, to, length: initial.text.length };
|
||||
const head = initial.head - from;
|
||||
before = { ...initial, text: initial.text.slice(from, to), head, from: head, to: head };
|
||||
}
|
||||
const latencies = await this.pressMeasured(
|
||||
characters.map((ch) => (ch === "\n" ? "Enter" : ch === "\b" ? "Backspace" : ch)),
|
||||
intervalMs,
|
||||
async (_, index) => {
|
||||
const after = await this.page.evaluate(readEditable, range);
|
||||
assertTypedKey(before, after, characters[index]!);
|
||||
before = after;
|
||||
},
|
||||
);
|
||||
const after = await this.page.evaluate(readEditable, range);
|
||||
if (
|
||||
after.kind !== before.kind ||
|
||||
after.target !== before.target ||
|
||||
after.text !== before.text ||
|
||||
after.head !== before.head
|
||||
) {
|
||||
throw new Error("typing text or caret changed after the measured keys");
|
||||
}
|
||||
if (range) {
|
||||
const final = await this.page.evaluate(readEditable, undefined);
|
||||
const to = range.to + final.text.length - initial.text.length;
|
||||
if (
|
||||
final.kind !== initial.kind ||
|
||||
final.target !== initial.target ||
|
||||
final.head !== before.head + range.from ||
|
||||
final.text.slice(0, range.from) !== initial.text.slice(0, range.from) ||
|
||||
final.text.slice(range.from, to) !== before.text ||
|
||||
final.text.slice(to) !== initial.text.slice(range.to)
|
||||
) {
|
||||
throw new Error("typing changed text outside the intended insertion or left an incorrect caret");
|
||||
}
|
||||
}
|
||||
return latencies;
|
||||
}
|
||||
|
||||
/** Presses `keys` `intervalMs` apart and returns keydown → next frame for each. */
|
||||
async pressMeasured(keys: string[], intervalMs: number): Promise<number[]> {
|
||||
async pressMeasured(
|
||||
keys: string[],
|
||||
intervalMs: number,
|
||||
afterKey?: (key: string, index: number) => Promise<void>,
|
||||
): Promise<number[]> {
|
||||
await this.inPage((p) => {
|
||||
p.keyLatencies = [];
|
||||
p.recordKeys = true;
|
||||
});
|
||||
for (const key of keys) {
|
||||
const started = Date.now();
|
||||
await this.page.keyboard.press(key);
|
||||
const wait = intervalMs - (Date.now() - started);
|
||||
if (wait > 0) await this.page.waitForTimeout(wait);
|
||||
try {
|
||||
for (const [index, key] of keys.entries()) {
|
||||
const started = Date.now();
|
||||
await this.page.keyboard.press(key);
|
||||
await afterKey?.(key, index);
|
||||
const wait = intervalMs - (Date.now() - started);
|
||||
if (wait > 0) await this.page.waitForTimeout(wait);
|
||||
}
|
||||
await this.inPage((p) => p.afterNextPaint().then(() => p.afterNextPaint()));
|
||||
const latencies = await this.inPage((p) => p.keyLatencies);
|
||||
if (latencies.length !== keys.length || latencies.some((latency) => !Number.isFinite(latency) || latency < 0)) {
|
||||
throw new Error(`key sample count mismatch: expected ${keys.length}, received ${latencies.length}`);
|
||||
}
|
||||
return latencies;
|
||||
} finally {
|
||||
await this.inPage((p) => {
|
||||
p.recordKeys = false;
|
||||
});
|
||||
}
|
||||
// Let the last frames land.
|
||||
await this.inPage((p) => p.afterNextPaint().then(() => p.afterNextPaint()));
|
||||
return this.inPage((p) => {
|
||||
p.recordKeys = false;
|
||||
return p.keyLatencies;
|
||||
}
|
||||
|
||||
/** Checks the actual selected option after every ArrowDown, including virtualized lists. */
|
||||
async completionArrowsMeasured(count: number, intervalMs: number) {
|
||||
const selection = () =>
|
||||
this.page.evaluate(() => {
|
||||
const selected = document.querySelector('.cm-tooltip-autocomplete li[aria-selected="true"]');
|
||||
const index = selected?.id.match(/-(\d+)$/)?.[1];
|
||||
if (!selected || index === undefined) throw new Error("completion selection is unavailable");
|
||||
return { id: selected.id, index: Number(index) };
|
||||
});
|
||||
let before = await selection();
|
||||
return this.pressMeasured(Array(count).fill("ArrowDown"), intervalMs, async () => {
|
||||
const after = await selection();
|
||||
if (after.id === before.id || after.index !== before.index + 1) {
|
||||
throw new Error(`completion selection did not advance from ${before.index} to ${before.index + 1}`);
|
||||
}
|
||||
before = after;
|
||||
});
|
||||
}
|
||||
|
||||
/** A far jump must move both the active caret and its viewport to the document end. */
|
||||
async jumpToEnd(timeoutMs = 5_000) {
|
||||
await this.inPage((p) => p.afterNextPaint());
|
||||
const start = await this.page.evaluate(() => {
|
||||
const w = window as unknown as { __openideaPerf: EditorProbe };
|
||||
const state = w.__openideaPerf.editorState();
|
||||
const scroller = document.activeElement?.closest(".cm-editor")?.querySelector(".cm-scroller");
|
||||
if (!state || !scroller || state.head === state.length || scroller.scrollHeight <= scroller.clientHeight) {
|
||||
throw new Error("jump start must be outside the destination in a scrollable editor");
|
||||
}
|
||||
return { path: state.path, head: state.head, scrollTop: scroller.scrollTop };
|
||||
});
|
||||
return this.measure(
|
||||
() => this.page.keyboard.press("ControlOrMeta+End"),
|
||||
".cm-content",
|
||||
null,
|
||||
timeoutMs,
|
||||
(start) => {
|
||||
const w = window as unknown as { __openideaPerf: EditorProbe };
|
||||
const state = w.__openideaPerf.editorState();
|
||||
const editor = document.activeElement?.closest(".cm-editor");
|
||||
const scroller = editor?.querySelector(".cm-scroller");
|
||||
const bottom = scroller?.getBoundingClientRect().bottom;
|
||||
return (
|
||||
!!state &&
|
||||
state.path === start.path &&
|
||||
state.head === state.length &&
|
||||
state.head !== start.head &&
|
||||
!!scroller &&
|
||||
scroller.scrollTop > start.scrollTop + scroller.clientHeight &&
|
||||
bottom !== undefined &&
|
||||
w.__openideaPerf.lineAtY(bottom - 4) === state.lines - 1
|
||||
);
|
||||
},
|
||||
start,
|
||||
);
|
||||
}
|
||||
|
||||
/** Display refresh interval measured over idle frames. */
|
||||
async refreshMs(): Promise<number> {
|
||||
await this.inPage((p) => p.startFrames());
|
||||
@@ -162,20 +300,9 @@ export class Harness {
|
||||
}
|
||||
|
||||
/** Seeds a file into the in-memory backend and opens it, returning open → painted ms. */
|
||||
async openFile(path: string, text?: string): Promise<number> {
|
||||
async openFile(path: string, text?: string, timeoutMs = 15_000): Promise<number> {
|
||||
if (text !== undefined) await this.seed(path, text);
|
||||
const name = path.slice(path.lastIndexOf("/") + 1);
|
||||
return this.page.evaluate(
|
||||
async ({ path, name }) => {
|
||||
const w = window as never as { __perf: InPage; __openideaPerf: { openFile(p: string): Promise<void> } };
|
||||
const t0 = performance.now();
|
||||
void w.__openideaPerf.openFile(path);
|
||||
await w.__perf.waitShown('[role="tab"][aria-selected="true"]', name, 15_000);
|
||||
await w.__perf.waitShown(".cm-content .cm-line", null, 15_000);
|
||||
return performance.now() - t0;
|
||||
},
|
||||
{ path, name },
|
||||
);
|
||||
return this.page.evaluate(openFileMeasured, { path, text, timeoutMs });
|
||||
}
|
||||
|
||||
/** Waits until the active editor's background parse has caught up (steady-state editing). */
|
||||
@@ -195,10 +322,9 @@ export class Harness {
|
||||
}
|
||||
|
||||
async editorState() {
|
||||
return this.page.evaluate(() =>
|
||||
(
|
||||
window as never as { __openideaPerf: { editorState(): { length: number; lines: number; head: number } | null } }
|
||||
).__openideaPerf.editorState(),
|
||||
);
|
||||
return this.page.evaluate(() => {
|
||||
const w = window as unknown as { __openideaPerf: EditorProbe };
|
||||
return w.__openideaPerf.editorState();
|
||||
});
|
||||
}
|
||||
}
|
||||
+78
-19
@@ -20,16 +20,23 @@ export interface InPage {
|
||||
afterNextPaint(): Promise<number>;
|
||||
startFrames(): void;
|
||||
stopFrames(): number[];
|
||||
/** Resolves with (frame start − first input after `arm`) once `selector` (and `text`) is rendered. */
|
||||
arm(): void;
|
||||
waitShown(selector: string, text: string | null, timeoutMs: number): Promise<number>;
|
||||
/** Arms a current-input transition; unarmed waits remain navigation observations. */
|
||||
arm(): number;
|
||||
cancel(token: number): void;
|
||||
waitShown(
|
||||
selector: string,
|
||||
text: string | null,
|
||||
timeoutMs: number,
|
||||
token?: number,
|
||||
check?: () => boolean,
|
||||
): Promise<number>;
|
||||
}
|
||||
|
||||
export function installInPage() {
|
||||
const w = window as unknown as { __perf: InPage };
|
||||
if (w.__perf) return;
|
||||
let armedAt = -1;
|
||||
let firstInputAfterArm = -1;
|
||||
let nextToken = 0;
|
||||
let operation: { token: number; input: number; capture?: () => void; cancel?: () => void } | null = null;
|
||||
|
||||
const afterNextPaint = () =>
|
||||
new Promise<number>((resolve) =>
|
||||
@@ -43,7 +50,10 @@ export function installInPage() {
|
||||
const onInput = (e: Event) => {
|
||||
const t = e.timeStamp;
|
||||
h.lastInput = t;
|
||||
if (armedAt >= 0 && firstInputAfterArm < 0) firstInputAfterArm = t;
|
||||
if (operation && operation.input < 0) {
|
||||
operation.input = t;
|
||||
operation.capture?.();
|
||||
}
|
||||
if (e.type === "keydown" && h.recordKeys) requestAnimationFrame(() => h.keyLatencies.push(performance.now() - t));
|
||||
};
|
||||
// `input` covers `keyboard.insertText`, which dispatches no keydown.
|
||||
@@ -76,25 +86,74 @@ export function installInPage() {
|
||||
return f;
|
||||
},
|
||||
arm() {
|
||||
armedAt = performance.now();
|
||||
firstInputAfterArm = -1;
|
||||
if (operation) throw new Error("a measurement is already armed");
|
||||
const token = ++nextToken;
|
||||
operation = { token, input: -1 };
|
||||
return token;
|
||||
},
|
||||
waitShown(selector, text, timeoutMs) {
|
||||
cancel(token) {
|
||||
if (operation?.token !== token) return;
|
||||
operation.cancel?.();
|
||||
operation = null;
|
||||
},
|
||||
waitShown(selector, text, timeoutMs, token, check) {
|
||||
const current = token === undefined ? null : operation;
|
||||
if (token !== undefined && current?.token !== token) return Promise.reject(new Error("measurement is not armed"));
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
const matches = () => {
|
||||
const el = document.querySelector(selector) as HTMLElement | null;
|
||||
if (!el || el.getClientRects().length === 0) return false;
|
||||
return text === null || (el.textContent ?? "").includes(text);
|
||||
const snapshot = () => {
|
||||
if (check && !check()) return [];
|
||||
return [...document.querySelectorAll<HTMLElement>(selector)]
|
||||
.filter(
|
||||
(el) =>
|
||||
el.getClientRects().length > 0 &&
|
||||
getComputedStyle(el).visibility === "visible" &&
|
||||
!el.closest('[aria-hidden="true"]') &&
|
||||
(text === null || (el.textContent ?? "").includes(text)),
|
||||
)
|
||||
.map((el) => ({
|
||||
el,
|
||||
text: el.textContent,
|
||||
value: el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement ? el.value : null,
|
||||
selected: el.getAttribute("aria-selected"),
|
||||
}));
|
||||
};
|
||||
let baseline = snapshot();
|
||||
if (current)
|
||||
current.capture = () => {
|
||||
baseline = snapshot();
|
||||
};
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
let stopped = false;
|
||||
const finish = () => {
|
||||
stopped = true;
|
||||
if (operation === current) operation = null;
|
||||
};
|
||||
if (current)
|
||||
current.cancel = () => {
|
||||
finish();
|
||||
reject(new Error("measurement cancelled"));
|
||||
};
|
||||
const tick = () => {
|
||||
// Checked at the start of a frame: a match here is painted by this frame.
|
||||
if (matches()) {
|
||||
const start = firstInputAfterArm >= 0 ? firstInputAfterArm : armedAt;
|
||||
armedAt = -1;
|
||||
resolve(performance.now() - start);
|
||||
if (stopped) return;
|
||||
const frameStart = performance.now();
|
||||
const shown = snapshot();
|
||||
const changed = shown.some(
|
||||
(now) =>
|
||||
!baseline.some(
|
||||
(old) =>
|
||||
old.el === now.el &&
|
||||
old.text === now.text &&
|
||||
old.value === now.value &&
|
||||
old.selected === now.selected,
|
||||
),
|
||||
);
|
||||
// Keep the endpoint at frame start, but only credit this input's target transition.
|
||||
if (shown.length && (!current || (current.input >= 0 && changed))) {
|
||||
const start = current?.input ?? 0;
|
||||
finish();
|
||||
resolve(frameStart - start);
|
||||
} else if (performance.now() > deadline) {
|
||||
armedAt = -1;
|
||||
finish();
|
||||
reject(new Error(`timed out waiting for ${selector}${text ? ` containing "${text}"` : ""}`));
|
||||
} else requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// @vitest-environment node
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { launchBrowser, withDeadline } from "./lifecycle.ts";
|
||||
|
||||
test("a deadline rejects an unanswered operation and releases its timer", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const pending = withDeadline(Promise.withResolvers<void>().promise, 20, "RPC");
|
||||
const assertion = expect(pending).rejects.toThrow("RPC timed out after 20 ms");
|
||||
await vi.advanceTimersByTimeAsync(20);
|
||||
await assertion;
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test.each(["blocked renderer", "missing animation frames"])(
|
||||
"Node deadline survives %s and browser cleanup terminates it",
|
||||
{ timeout: 20_000 },
|
||||
async (mode) => {
|
||||
const runtime = await launchBrowser({ headless: true });
|
||||
try {
|
||||
const page = await runtime.browser.newPage();
|
||||
// Real renderer/Node clocks are the boundary under test; fake timers cannot drive Chromium.
|
||||
const operation =
|
||||
mode === "blocked renderer"
|
||||
? page.evaluate(() => {
|
||||
for (;;) {
|
||||
/* Deliberately unresponsive renderer. */
|
||||
}
|
||||
})
|
||||
: page.evaluate(() => {
|
||||
window.requestAnimationFrame = () => 0;
|
||||
return new Promise<void>((resolve) => requestAnimationFrame(() => resolve()));
|
||||
});
|
||||
await expect(withDeadline(operation, 30, mode)).rejects.toThrow("wall clock");
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,59 @@
|
||||
import { chromium, type Browser, type LaunchOptions } from "@playwright/test";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
export class DeadlineError extends Error {
|
||||
constructor(label: string, timeoutMs: number) {
|
||||
super(`${label} timed out after ${timeoutMs} ms (wall clock)`);
|
||||
this.name = "DeadlineError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs on Node's clock even if the renderer, rAF, or an RPC never answers. */
|
||||
export async function withDeadline<T>(operation: Promise<T>, timeoutMs: number, label: string): Promise<T> {
|
||||
const { promise, reject } = Promise.withResolvers<never>();
|
||||
const timer = setTimeout(() => reject(new DeadlineError(label, timeoutMs)), timeoutMs);
|
||||
try {
|
||||
return await Promise.race([operation, promise]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export interface BrowserSession {
|
||||
browser: Browser;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export async function launchBrowser(options: LaunchOptions, cleanupMs = 5000): Promise<BrowserSession> {
|
||||
const server = await chromium.launchServer({ ...options, timeout: 30_000 });
|
||||
const close = async () => {
|
||||
try {
|
||||
await withDeadline(server.close(), cleanupMs, "browser close");
|
||||
} catch (error) {
|
||||
try {
|
||||
await withDeadline(server.kill(), cleanupMs, "browser kill");
|
||||
} catch (killError) {
|
||||
throw new AggregateError([error, killError], "Browser cleanup failed", { cause: killError });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
try {
|
||||
const browser = await chromium.connect(server.wsEndpoint(), { timeout: 30_000 });
|
||||
return { browser, close };
|
||||
} catch (error) {
|
||||
try {
|
||||
await close();
|
||||
} catch (cleanup) {
|
||||
throw new AggregateError([error, cleanup], "Browser startup failed", { cause: cleanup });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function closeServer(server: Server) {
|
||||
server.closeAllConnections();
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
await withDeadline(promise, 5000, "HTTP server close");
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { countOption } from "./options.ts";
|
||||
|
||||
const root = resolve(import.meta.dirname, "../..");
|
||||
|
||||
describe("benchmark CLI validation", () => {
|
||||
it.each([
|
||||
["runs", "NaN"],
|
||||
["runs", "Infinity"],
|
||||
["runs", "0"],
|
||||
["runs", "1.5"],
|
||||
["warmup", "-1"],
|
||||
["warmup", "NaN"],
|
||||
["warmup", "0.5"],
|
||||
["cpu", "NaN"],
|
||||
["cpu", "Infinity"],
|
||||
["cpu", "0"],
|
||||
])("rejects --%s %s before selecting or launching workloads", (option, value) => {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["perf/run.ts", "--no-build", "--only", "__unknown__", `--${option}=${value}`],
|
||||
{ cwd: root, encoding: "utf8", timeout: 10_000 },
|
||||
);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain(`--${option} must be`);
|
||||
expect(result.stdout).not.toContain("Building");
|
||||
});
|
||||
|
||||
it.each(["typing,__unknown__", "", "typing,"])("rejects invalid selection %j", (selection) => {
|
||||
const result = spawnSync(process.execPath, ["perf/run.ts", "--no-build", "--only", selection], {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
timeout: 10_000,
|
||||
});
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain("unknown scenario(s)");
|
||||
});
|
||||
|
||||
it("accepts integral counts and zero warmup without silently clamping", () => {
|
||||
expect(countOption("runs", "5", 1)).toBe(5);
|
||||
expect(countOption("warmup", "0", 0)).toBe(0);
|
||||
expect(() => countOption("warmup", "", 0)).toThrow();
|
||||
expect(() => countOption("runs", "9007199254740992", 1)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
/** CLI repetition counts must be validated before any build or process launch. */
|
||||
export function countOption(name: string, raw: string, minimum: 0 | 1): number {
|
||||
const value = Number(raw);
|
||||
if (!raw.trim() || !Number.isSafeInteger(value) || value < minimum) {
|
||||
throw new Error(`--${name} must be an integer >= ${minimum}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/// <reference lib="es2024.promise" />
|
||||
import { spawn, type ChildProcess, type StdioOptions } from "node:child_process";
|
||||
|
||||
/** Run one owned command and bound both execution and process-tree cleanup. */
|
||||
export async function runCommand(
|
||||
command: string,
|
||||
args: string[],
|
||||
options: {
|
||||
cwd: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
timeoutMs: number;
|
||||
label?: string;
|
||||
graceMs?: number;
|
||||
stdio?: StdioOptions;
|
||||
},
|
||||
): Promise<void> {
|
||||
const label = options.label ?? command;
|
||||
const graceMs = options.graceMs ?? 1000;
|
||||
if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0 || options.timeoutMs > 2_147_483_647) {
|
||||
throw new Error(`${label}: invalid execution deadline`);
|
||||
}
|
||||
if (!Number.isFinite(graceMs) || graceMs <= 0 || graceMs > 2_147_483_647) {
|
||||
throw new Error(`${label}: invalid cleanup deadline`);
|
||||
}
|
||||
const child = spawn(command, args, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
detached: process.platform !== "win32",
|
||||
shell: process.platform === "win32" && /\.(cmd|bat)$/i.test(command),
|
||||
stdio: options.stdio ?? "inherit",
|
||||
});
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const errors: string[] = [];
|
||||
let stopping = false;
|
||||
let finished = false;
|
||||
let cleanup: NodeJS.Timeout | undefined;
|
||||
let treeKiller: ChildProcess | undefined;
|
||||
const group = process.platform !== "win32";
|
||||
const alive = () => {
|
||||
if (!child.pid) return false;
|
||||
try {
|
||||
process.kill(group ? -child.pid : child.pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return (error as NodeJS.ErrnoException).code !== "ESRCH";
|
||||
}
|
||||
};
|
||||
const signal = (value: NodeJS.Signals) => {
|
||||
if (!child.pid) return;
|
||||
try {
|
||||
if (group) process.kill(-child.pid, value);
|
||||
else child.kill(value);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ESRCH") {
|
||||
errors.push(`cleanup: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
const finish = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(deadline);
|
||||
clearTimeout(cleanup);
|
||||
treeKiller?.kill("SIGKILL");
|
||||
treeKiller?.unref();
|
||||
process.removeListener("SIGINT", interrupt);
|
||||
process.removeListener("SIGTERM", terminate);
|
||||
// Never wait indefinitely for an uninterruptible process or inherited pipe.
|
||||
child.unref();
|
||||
child.stdout?.destroy();
|
||||
child.stderr?.destroy();
|
||||
if (errors.length) reject(new Error(`${label}: ${errors.join("; ")}`));
|
||||
else resolve();
|
||||
};
|
||||
const stop = (error?: string) => {
|
||||
if (error) errors.push(error);
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
clearTimeout(deadline);
|
||||
if (!alive()) return finish();
|
||||
if (!group && child.pid) {
|
||||
// Windows has no POSIX process groups. Bound this PID-scoped taskkill too.
|
||||
treeKiller = spawn("taskkill", ["/PID", String(child.pid), "/T", "/F"], { stdio: "ignore" });
|
||||
treeKiller.on("error", (failure) => errors.push(`tree cleanup: ${failure.message}`));
|
||||
} else signal("SIGTERM");
|
||||
cleanup = setTimeout(() => {
|
||||
signal("SIGKILL");
|
||||
cleanup = setTimeout(() => {
|
||||
if (alive()) errors.push("process group remained after SIGKILL");
|
||||
finish();
|
||||
}, graceMs);
|
||||
}, graceMs);
|
||||
};
|
||||
const interrupt = () => stop("interrupted by SIGINT");
|
||||
const terminate = () => stop("interrupted by SIGTERM");
|
||||
const deadline = setTimeout(() => stop(`timed out after ${options.timeoutMs} ms`), options.timeoutMs);
|
||||
process.on("SIGINT", interrupt);
|
||||
process.on("SIGTERM", terminate);
|
||||
child.once("error", (error) => stop(`spawn failed: ${error.message}`));
|
||||
child.once("exit", (code, exitSignal) => {
|
||||
if (code !== 0 && !errors.length) errors.push(`failed (exit ${code}, signal ${exitSignal})`);
|
||||
if (!stopping) stop();
|
||||
else if (!alive()) finish();
|
||||
});
|
||||
return promise;
|
||||
}
|
||||
@@ -32,6 +32,7 @@ describe("evaluate", () => {
|
||||
it("fails metrics over budget and budgeted metrics that are missing", () => {
|
||||
const r = evaluate(results, { "typing.maxMs": 33, "typing.missedFramePct": 2, "other.x": 1 }, null, {
|
||||
enforceBudgets: true,
|
||||
expected: ["typing"],
|
||||
});
|
||||
expect(r.rows.find((x) => x.key === "typing.maxMs")?.verdict).toBe("over-budget");
|
||||
expect(r.rows.find((x) => x.key === "typing.missedFramePct")?.verdict).toBe("error");
|
||||
@@ -41,18 +42,57 @@ describe("evaluate", () => {
|
||||
});
|
||||
|
||||
it("reports but does not fail budgets when not enforced", () => {
|
||||
const r = evaluate(results, { "typing.maxMs": 33 }, null, { enforceBudgets: false });
|
||||
const r = evaluate(results, { "typing.maxMs": 33 }, null, { enforceBudgets: false, expected: ["typing"] });
|
||||
expect(r.rows.find((x) => x.key === "typing.maxMs")?.verdict).toBe("over-budget-info");
|
||||
expect(r.failures).toBe(0);
|
||||
});
|
||||
|
||||
it("compares against the baseline with relative and absolute slack", () => {
|
||||
const baseline = [{ id: "typing", title: "", metrics: { p95Ms: 14, maxMs: 39.5, layoutsPerKey: 2 } }];
|
||||
const r = evaluate(results, {}, baseline, { enforceBudgets: true, tolerance: 0.2, floor: 1 });
|
||||
const r = evaluate(results, {}, baseline, {
|
||||
enforceBudgets: true,
|
||||
expected: ["typing"],
|
||||
tolerance: 0.2,
|
||||
floor: 1,
|
||||
});
|
||||
const verdict = (k: string) => r.rows.find((x) => x.key === k)?.verdict;
|
||||
expect(verdict("typing.p95Ms")).toBe("regressed"); // +28 %, +4 ms
|
||||
expect(verdict("typing.maxMs")).toBe("ok"); // within 20 %
|
||||
expect(verdict("typing.layoutsPerKey")).toBe("ok"); // +50 % but only +1 (floor)
|
||||
expect(r.regressions).toBe(1);
|
||||
});
|
||||
|
||||
it("fails when a declared scenario disappears from the results", () => {
|
||||
const report = evaluate(results, { "scroll.jumpMs": 50 }, null, {
|
||||
enforceBudgets: true,
|
||||
expected: ["typing", "scroll"],
|
||||
});
|
||||
expect(report.errors).toContainEqual({ id: "scroll", error: "missing scenario result" });
|
||||
expect(report.rows.find((r) => r.key === "scroll.jumpMs")?.verdict).toBe("error");
|
||||
expect(report.failures).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("rejects duplicate and unexpected scenario results", () => {
|
||||
const report = evaluate([...results, ...results, { id: "other", title: "", metrics: { ms: 1 } }], {}, null, {
|
||||
enforceBudgets: true,
|
||||
expected: ["typing"],
|
||||
});
|
||||
expect(report.errors).toEqual([
|
||||
{ id: "typing", error: "duplicate scenario result" },
|
||||
{ id: "other", error: "unexpected scenario result" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects an empty workload instead of returning a passing report", () => {
|
||||
expect(evaluate([], {}, null, { enforceBudgets: true, expected: [] }).failures).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("fails nonfinite metrics even without an absolute budget", () => {
|
||||
const report = evaluate([{ id: "typing", title: "", metrics: { ms: Infinity } }], {}, null, {
|
||||
enforceBudgets: false,
|
||||
expected: ["typing"],
|
||||
});
|
||||
expect(report.rows[0]?.verdict).toBe("error");
|
||||
expect(report.failures).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -52,7 +52,7 @@ export function evaluate(
|
||||
results: ScenarioResult[],
|
||||
budgets: Record<string, number>,
|
||||
baseline: ScenarioResult[] | null,
|
||||
opts: { enforceBudgets: boolean; tolerance?: number; floor?: number },
|
||||
opts: { expected: readonly string[]; enforceBudgets: boolean; tolerance?: number; floor?: number },
|
||||
): Report {
|
||||
const tol = opts.tolerance ?? 0.2;
|
||||
const floor = opts.floor ?? 1;
|
||||
@@ -62,7 +62,21 @@ export function evaluate(
|
||||
const rows: Row[] = [];
|
||||
const errors: Report["errors"] = [];
|
||||
const seen = new Set<string>();
|
||||
const expected = new Set(opts.expected);
|
||||
const produced = new Set<string>();
|
||||
if (!expected.size) errors.push({ id: "run", error: "empty workload" });
|
||||
if (expected.size !== opts.expected.length) errors.push({ id: "run", error: "duplicate expected scenario" });
|
||||
for (const r of results) {
|
||||
if (!expected.has(r.id)) {
|
||||
errors.push({ id: r.id, error: "unexpected scenario result" });
|
||||
continue;
|
||||
}
|
||||
if (produced.has(r.id)) {
|
||||
errors.push({ id: r.id, error: "duplicate scenario result" });
|
||||
continue;
|
||||
}
|
||||
produced.add(r.id);
|
||||
if (!r.error && !Object.keys(r.metrics).length) errors.push({ id: r.id, error: "no metrics produced" });
|
||||
if (r.error) errors.push({ id: r.id, error: r.error });
|
||||
for (const [m, value] of Object.entries(r.metrics)) {
|
||||
const key = `${r.id}.${m}`;
|
||||
@@ -70,7 +84,7 @@ export function evaluate(
|
||||
const budget = budgets[key];
|
||||
const before = base.get(key);
|
||||
let verdict: Verdict = "ok";
|
||||
if (!Number.isFinite(value)) verdict = budget === undefined ? "ok" : "error";
|
||||
if (!Number.isFinite(value)) verdict = "error";
|
||||
else if (budget !== undefined && value > budget)
|
||||
verdict = opts.enforceBudgets ? "over-budget" : "over-budget-info";
|
||||
else if (before !== undefined && value > before * (1 + tol) && value - before > floor) verdict = "regressed";
|
||||
@@ -78,10 +92,12 @@ export function evaluate(
|
||||
rows.push({ key, value, budget, baseline: before, verdict });
|
||||
}
|
||||
}
|
||||
// A budgeted metric that was not produced (scenario failed or renamed) fails too.
|
||||
const ran = new Set(results.map((r) => r.id));
|
||||
for (const id of expected) {
|
||||
if (!produced.has(id)) errors.push({ id, error: "missing scenario result" });
|
||||
}
|
||||
// Required metrics are scoped by the declared workload, never by returned data.
|
||||
for (const key of Object.keys(budgets)) {
|
||||
if (!seen.has(key) && ran.has(key.slice(0, key.indexOf(".")))) {
|
||||
if (!seen.has(key) && expected.has(key.slice(0, key.indexOf(".")))) {
|
||||
rows.push({ key, value: NaN, budget: budgets[key], verdict: "error" });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
// @vitest-environment node
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { expect, test } from "vitest";
|
||||
|
||||
const repo = resolve(import.meta.dirname, "../..");
|
||||
|
||||
test.each([
|
||||
{ family: "perf", args: ["--only", "typing", "--no-build"], kind: "missing bundle" },
|
||||
{ family: "perf", args: ["--only", "typing"], kind: "build failure" },
|
||||
{ family: "perf", args: ["--only", "typing", "--no-build", "--channel", "not-a-browser"], kind: "browser failure" },
|
||||
{ family: "evals", args: ["--only", "java.app", "--no-build"], kind: "missing server" },
|
||||
])("$kind replaces old passing reports with structured failure", { timeout: 30_000 }, ({ family, args, kind }) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "openidea-runner-failure-"));
|
||||
try {
|
||||
for (const dir of ["perf", "evals", "smoke"]) cpSync(join(repo, dir), join(root, dir), { recursive: true });
|
||||
cpSync(join(repo, "package.json"), join(root, "package.json"));
|
||||
symlinkSync(join(repo, "node_modules"), join(root, "node_modules"), "junction");
|
||||
const output = join(root, "target", family === "perf" ? "perf" : "eval");
|
||||
mkdirSync(output, { recursive: true });
|
||||
writeFileSync(join(output, "report.json"), '{"pass":true,"sentinel":"old success"}');
|
||||
writeFileSync(join(output, "report.md"), "old success");
|
||||
if (kind === "browser failure") {
|
||||
mkdirSync(join(output, "dist"));
|
||||
writeFileSync(join(output, "dist", "index.html"), "<!doctype html><title>launch boundary</title>");
|
||||
}
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[join(root, family, "run.ts"), ...args, "--runs", "1", "--warmup", "0"],
|
||||
{
|
||||
encoding: "utf8",
|
||||
timeout: 20_000,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: kind === "build failure" ? "" : process.env.PATH,
|
||||
OPENIDEA_SERVER: join(root, "nonexistent-server"),
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status, result.stderr).toBe(1);
|
||||
const report = JSON.parse(readFileSync(join(output, "report.json"), "utf8"));
|
||||
expect(report.status).toBe("failed");
|
||||
expect(report.infrastructure.length).toBeGreaterThan(0);
|
||||
expect(report.expected.length).toBeGreaterThan(0);
|
||||
expect(report.trials).toEqual({});
|
||||
expect(report.sentinel).toBeUndefined();
|
||||
if (family === "evals") expect(report.pass).toBe(false);
|
||||
else expect(report.failures).toBeGreaterThan(0);
|
||||
const markdown = readFileSync(join(output, "report.md"), "utf8");
|
||||
expect(markdown).not.toContain("old success");
|
||||
expect(markdown).toMatch(/Infrastructure|infrastructure/);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,200 @@
|
||||
import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
collectRustMetrics,
|
||||
inspectRustArtifacts,
|
||||
RUST_BENCHMARKS,
|
||||
rustBudgets,
|
||||
rustPaths,
|
||||
selectRustPackages,
|
||||
} from "./rust.ts";
|
||||
import { runRust } from "./rustRun.ts";
|
||||
|
||||
const TEXT_BENCHMARKS = RUST_BENCHMARKS["ide-text"]!.length;
|
||||
|
||||
const roots: string[] = [];
|
||||
function fixture() {
|
||||
const root = mkdtempSync(join(tmpdir(), "rust-validity-"));
|
||||
roots.push(root);
|
||||
return root;
|
||||
}
|
||||
function artifact(root: string, id: string, value: unknown = 1000, directory = id) {
|
||||
const dir = join(root, directory, "new");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, "benchmark.json"), JSON.stringify({ full_id: id }));
|
||||
writeFileSync(join(dir, "estimates.json"), JSON.stringify({ mean: { point_estimate: value } }));
|
||||
return dir;
|
||||
}
|
||||
function complete(root: string, packages: readonly string[]) {
|
||||
for (const p of packages) for (const id of RUST_BENCHMARKS[p]!) artifact(root, id);
|
||||
}
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("Rust workload contract", () => {
|
||||
it("owns 40 unique IDs and exactly six observe-only metrics", () => {
|
||||
const ids = Object.values(RUST_BENCHMARKS).flat();
|
||||
expect(ids).toHaveLength(40);
|
||||
expect(new Set(ids).size).toBe(40);
|
||||
const budgets = JSON.parse(readFileSync(resolve(import.meta.dirname, "../budgets.json"), "utf8")).rust;
|
||||
const selected = rustBudgets(selectRustPackages(), budgets);
|
||||
expect(Object.keys(selected)).toHaveLength(34);
|
||||
expect(ids.filter((id) => !(id in budgets))).toEqual([
|
||||
"syntax_2mb/outline_uncached",
|
||||
"syntax_2mb/folding_uncached",
|
||||
"index_100k/find_files/zzzz",
|
||||
"index_100k/find_symbols/renderView",
|
||||
"startup_20k/reopen_wait_idle",
|
||||
"search_excluded_subtrees/5k_dependencies",
|
||||
]);
|
||||
expect(Object.keys(rustBudgets(["ide-text"], budgets))).toHaveLength(9);
|
||||
const missing = { ...budgets };
|
||||
delete missing["text/open_1mb"];
|
||||
expect(() => rustBudgets(["ide-text"], missing)).toThrow(/classification/);
|
||||
expect(() => rustBudgets(["ide-text"], { ...budgets, "text/new": 1 })).toThrow(/unknown/);
|
||||
expect(() => rustBudgets(["ide-text"], { ...budgets, "syntax_2mb/outline_uncached": 1 })).toThrow(/observe-only/);
|
||||
});
|
||||
|
||||
it("rejects unknown or empty packages and deduplicates before cargo", () => {
|
||||
expect(() => selectRustPackages([])).toThrow(/empty/);
|
||||
expect(() => selectRustPackages([""])).toThrow(/unknown/);
|
||||
expect(() => selectRustPackages(["not-a-crate"])).toThrow(/unknown/);
|
||||
expect(selectRustPackages(["ide-text", "ide-text"])).toEqual(["ide-text"]);
|
||||
});
|
||||
|
||||
it("reports unknown packages without invoking cargo", async () => {
|
||||
const root = fixture();
|
||||
const marker = join(root, "cargo-invoked");
|
||||
writeFileSync(join(root, "cargo"), `#!/bin/sh\n: > '${marker}'\nexit 99\n`, { mode: 0o755 });
|
||||
const code = await runRust({
|
||||
root,
|
||||
packages: ["not-a-crate"],
|
||||
env: { ...process.env, PATH: root },
|
||||
});
|
||||
expect(code).toBe(1);
|
||||
const report = JSON.parse(readFileSync(join(root, "target/perf/report-rust.json"), "utf8"));
|
||||
expect(report.errors[0].error).toMatch(/unknown Rust package/);
|
||||
expect(existsSync(marker)).toBe(false);
|
||||
});
|
||||
|
||||
it("resolves target and Criterion overrides relative to cargo's cwd", () => {
|
||||
expect(rustPaths("/work", {})).toEqual({ target: "/work/target", criterion: "/work/target/criterion" });
|
||||
expect(rustPaths("/work", { CARGO_TARGET_DIR: "private" })).toEqual({
|
||||
target: "/work/private",
|
||||
criterion: "/work/private/criterion",
|
||||
});
|
||||
expect(rustPaths("/work", { CARGO_TARGET_DIR: "/private", CRITERION_HOME: "results" })).toEqual({
|
||||
target: "/private",
|
||||
criterion: "/work/results",
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts a complete full workload and converts nanoseconds to microseconds", () => {
|
||||
const root = fixture();
|
||||
complete(root, selectRustPackages());
|
||||
expect(Object.keys(collectRustMetrics(root, selectRustPackages()))).toHaveLength(40);
|
||||
expect(collectRustMetrics(root, ["ide-text"])["text/open_1mb"]).toBe(1);
|
||||
});
|
||||
|
||||
it("requires selected IDs independently of budgets, including observe-only IDs", () => {
|
||||
const root = fixture();
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow(/missing/);
|
||||
complete(root, ["ide-language"]);
|
||||
rmSync(join(root, "syntax_2mb/outline_uncached"), { recursive: true });
|
||||
expect(() => collectRustMetrics(root, ["ide-language"])).toThrow(/outline_uncached/);
|
||||
expect(() => collectRustMetrics(root, selectRustPackages())).toThrow(/missing/);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["ide-language", "syntax_2mb/outline_uncached"],
|
||||
["ide-language", "syntax_2mb/folding_uncached"],
|
||||
["ide-index", "index_100k/find_files/zzzz"],
|
||||
["ide-index", "index_100k/find_symbols/renderView"],
|
||||
["ide-index", "startup_20k/reopen_wait_idle"],
|
||||
["ide-search", "search_excluded_subtrees/5k_dependencies"],
|
||||
])("requires observe-only metric %s / %s", (p, id) => {
|
||||
const root = fixture();
|
||||
complete(root, [p]);
|
||||
rmSync(join(root, id), { recursive: true });
|
||||
expect(() => collectRustMetrics(root, [p])).toThrow(/missing selected/);
|
||||
});
|
||||
|
||||
it("ignores known unselected artifacts, including their malformed estimates", () => {
|
||||
const root = fixture();
|
||||
complete(root, ["ide-text"]);
|
||||
artifact(root, "heap/index_1m_objects", "bad");
|
||||
const unrelated = artifact(root, "heap/dominators_1m_random");
|
||||
writeFileSync(join(unrelated, "benchmark.json"), "{");
|
||||
expect(Object.keys(collectRustMetrics(root, ["ide-text"]))).toHaveLength(TEXT_BENCHMARKS);
|
||||
});
|
||||
|
||||
it("rejects duplicate IDs and unknown selected IDs even with complete expected results", () => {
|
||||
const root = fixture();
|
||||
complete(root, ["ide-text"]);
|
||||
artifact(root, "text/open_1mb", 1000, "duplicate");
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow(/duplicate/);
|
||||
rmSync(join(root, "duplicate"), { recursive: true });
|
||||
artifact(root, "text/renamed");
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow(/unknown/);
|
||||
});
|
||||
|
||||
it.each([null, "1000", -1, {}])("rejects invalid means: %j", (value) => {
|
||||
const root = fixture();
|
||||
complete(root, ["ide-text"]);
|
||||
artifact(root, "text/open_1mb", value);
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow(/estimate/);
|
||||
});
|
||||
|
||||
it("accepts zero and rejects malformed metadata, absent estimates, and invalid JSON", () => {
|
||||
const root = fixture();
|
||||
complete(root, ["ide-text"]);
|
||||
const dir = artifact(root, "text/open_1mb", 0);
|
||||
expect(collectRustMetrics(root, ["ide-text"])["text/open_1mb"]).toBe(0);
|
||||
writeFileSync(join(dir, "benchmark.json"), "{}");
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow(/full_id/);
|
||||
artifact(root, "text/open_1mb");
|
||||
rmSync(join(dir, "estimates.json"));
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow();
|
||||
writeFileSync(join(dir, "estimates.json"), "{");
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow();
|
||||
writeFileSync(join(dir, "estimates.json"), "{}");
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow(/estimate/);
|
||||
});
|
||||
|
||||
it.each(["benchmark.json", "estimates.json"])("requires fresh %s for a newly run workload", (file) => {
|
||||
const root = fixture();
|
||||
complete(root, ["ide-text"]);
|
||||
const dir = artifact(root, "text/open_1mb");
|
||||
utimesSync(join(dir, file), new Date(0), new Date(0));
|
||||
expect(() => collectRustMetrics(root, ["ide-text"], 1)).toThrow(/stale/);
|
||||
expect(Object.keys(collectRustMetrics(root, ["ide-text"]))).toHaveLength(TEXT_BENCHMARKS);
|
||||
});
|
||||
it("rejects overflowing JSON numeric estimates, including observe-only metrics", () => {
|
||||
const root = fixture();
|
||||
complete(root, ["ide-language"]);
|
||||
const dir = artifact(root, "syntax_2mb/outline_uncached");
|
||||
writeFileSync(join(dir, "estimates.json"), '{"mean":{"point_estimate":1e309}}');
|
||||
expect(() => collectRustMetrics(root, ["ide-language"])).toThrow(/estimate/);
|
||||
});
|
||||
|
||||
it.each(["benchmark.json", "estimates.json"])("rejects future %s even in inspection mode", (file) => {
|
||||
const root = fixture();
|
||||
complete(root, ["ide-text"]);
|
||||
const date = new Date(Date.now() + 60_000);
|
||||
utimesSync(join(root, "text/open_1mb/new", file), date, date);
|
||||
expect(() => collectRustMetrics(root, ["ide-text"])).toThrow(/future Criterion artifact/);
|
||||
});
|
||||
|
||||
it("retains partial valid metrics and dated artifacts after malformed selected output", () => {
|
||||
const root = fixture();
|
||||
complete(root, ["ide-text"]);
|
||||
writeFileSync(join(root, "text/open_1mb/new/estimates.json"), "{");
|
||||
const result = inspectRustArtifacts(root, ["ide-text"]);
|
||||
expect(Object.keys(result.metrics)).toHaveLength(TEXT_BENCHMARKS - 1);
|
||||
expect(result.errors.length).toBeGreaterThan(0);
|
||||
expect(result.artifacts).toContainEqual(expect.objectContaining({ path: "text/open_1mb/new/benchmark.json" }));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,190 @@
|
||||
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { join, relative, resolve, sep } from "node:path";
|
||||
|
||||
// Package-owned expectations, independent of collected artifacts and human-owned budgets.
|
||||
export const RUST_BENCHMARKS: Readonly<Record<string, readonly string[]>> = {
|
||||
"ide-text": [
|
||||
"text/open_1mb",
|
||||
"text/apply_keystroke_1mb",
|
||||
"text/apply_100_changes_1mb",
|
||||
"text/diff_two_edits_1mb",
|
||||
"text/diff_every_line_1mb",
|
||||
"text/offset_to_pos_round_trip",
|
||||
"text/pos_to_byte",
|
||||
"text/text_for_save_crlf_1mb",
|
||||
"text/diff_one_edit_5mb",
|
||||
],
|
||||
"ide-language": [
|
||||
"syntax_2mb/outline_uncached",
|
||||
"syntax_2mb/folding_uncached",
|
||||
"syntax_2mb/outline_cached_hit",
|
||||
"syntax_2mb/folding_cached_hit",
|
||||
"syntax_2mb/selection_cached_hit",
|
||||
"syntax_2mb/folding_after_keystroke_incremental",
|
||||
"syntax_2mb/outline_after_keystroke_incremental",
|
||||
],
|
||||
"ide-index": [
|
||||
"index_100k/find_files/ordcon",
|
||||
"index_100k/find_files/src/user",
|
||||
"index_100k/find_files/PaymentService",
|
||||
"index_100k/find_files/zzzz",
|
||||
"index_100k/find_files/a",
|
||||
"index_100k/find_dirs/util",
|
||||
"index_100k/find_dirs/src/ui",
|
||||
"index_100k/find_dirs/a",
|
||||
"index_100k/find_symbols/OCo",
|
||||
"index_100k/find_symbols/invoiceRepo",
|
||||
"index_100k/find_symbols/renderView",
|
||||
"index_100k/find_symbols/q",
|
||||
"index_100k/find_symbols_types/UsSe",
|
||||
"startup_20k/reopen_wait_idle",
|
||||
],
|
||||
"ide-search": [
|
||||
"search_5k/literal",
|
||||
"search_5k/regex",
|
||||
"search_5k/dense_limit",
|
||||
"search_dense_files/24x40k_limit",
|
||||
"search_excluded_subtrees/5k_dependencies",
|
||||
],
|
||||
"ide-host": ["plugins/load_50", "plugins/cold_activation", "plugins/syntax_highlights_2k_lines"],
|
||||
"ide-heap": ["heap/index_1m_objects", "heap/dominators_1m_random"],
|
||||
};
|
||||
|
||||
const OBSERVE_ONLY: Readonly<Record<string, true>> = {
|
||||
"syntax_2mb/outline_uncached": true,
|
||||
"syntax_2mb/folding_uncached": true,
|
||||
"index_100k/find_files/zzzz": true,
|
||||
"index_100k/find_symbols/renderView": true,
|
||||
"startup_20k/reopen_wait_idle": true,
|
||||
"search_excluded_subtrees/5k_dependencies": true,
|
||||
};
|
||||
|
||||
export function selectRustPackages(packages: readonly string[] = Object.keys(RUST_BENCHMARKS)): string[] {
|
||||
if (packages.length === 0) throw new Error("empty Rust package selection");
|
||||
for (const p of packages) {
|
||||
if (!Object.hasOwn(RUST_BENCHMARKS, p)) throw new Error(`unknown Rust package: ${JSON.stringify(p)}`);
|
||||
}
|
||||
return [...new Set(packages)];
|
||||
}
|
||||
|
||||
export function rustPaths(root: string, env: { CARGO_TARGET_DIR?: string; CRITERION_HOME?: string }) {
|
||||
const target = resolve(root, env.CARGO_TARGET_DIR ?? "target");
|
||||
return { target, criterion: resolve(root, env.CRITERION_HOME ?? join(target, "criterion")) };
|
||||
}
|
||||
|
||||
export function rustBudgets(packages: readonly string[], budgets: Record<string, number>): Record<string, number> {
|
||||
const selected = selectRustPackages(packages);
|
||||
const ids = new Set(Object.values(RUST_BENCHMARKS).flat());
|
||||
for (const id of Object.keys(OBSERVE_ONLY)) {
|
||||
if (!ids.has(id)) throw new Error(`unknown Rust observe-only classification: ${id}`);
|
||||
}
|
||||
for (const [id, value] of Object.entries(budgets)) {
|
||||
if (!ids.has(id)) throw new Error(`unknown Rust budget ID: ${id}`);
|
||||
if (Object.hasOwn(OBSERVE_ONLY, id)) throw new Error(`Rust observe-only ID has an absolute budget: ${id}`);
|
||||
if (!Number.isFinite(value) || value < 0) throw new Error(`invalid Rust budget: ${id}`);
|
||||
}
|
||||
for (const id of ids) {
|
||||
if (!Object.hasOwn(budgets, id) && !Object.hasOwn(OBSERVE_ONLY, id)) {
|
||||
throw new Error(`missing Rust budget classification: ${id}`);
|
||||
}
|
||||
}
|
||||
return Object.fromEntries(
|
||||
selected
|
||||
.flatMap((p) => RUST_BENCHMARKS[p]!)
|
||||
.filter((id) => !Object.hasOwn(OBSERVE_ONLY, id))
|
||||
.map((id) => [`rust.${id}`, budgets[id]!]),
|
||||
);
|
||||
}
|
||||
|
||||
export interface RustArtifacts {
|
||||
metrics: Record<string, number>;
|
||||
artifacts: Array<{ path: string; modifiedAt: string }>;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
/** Keep valid partial results and file dates even when another selected artifact is invalid. */
|
||||
export function inspectRustArtifacts(
|
||||
dir: string,
|
||||
packages: readonly string[],
|
||||
started?: number,
|
||||
now = Date.now(),
|
||||
): RustArtifacts {
|
||||
const selected = new Set(selectRustPackages(packages));
|
||||
const expected = new Set([...selected].flatMap((p) => RUST_BENCHMARKS[p]!));
|
||||
const owners = new Map<string, string>();
|
||||
const groups = new Map<string, string>();
|
||||
for (const [p, ids] of Object.entries(RUST_BENCHMARKS)) {
|
||||
for (const id of ids) {
|
||||
owners.set(id, p);
|
||||
groups.set(id.split("/")[0]!, p);
|
||||
}
|
||||
}
|
||||
const metrics: Record<string, number> = {};
|
||||
const artifacts: RustArtifacts["artifacts"] = [];
|
||||
const errors: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
function readNew(sub: string) {
|
||||
// Criterion sanitizes slashes in ungrouped function names to underscores.
|
||||
const top = relative(dir, sub).split(sep)[0]!;
|
||||
const pathOwner = [...groups].find(([group]) => top === group || top.startsWith(`${group}_`))?.[1];
|
||||
if (pathOwner && !selected.has(pathOwner)) return;
|
||||
const bench = join(sub, "new", "benchmark.json");
|
||||
const estimates = join(sub, "new", "estimates.json");
|
||||
if (!existsSync(bench)) {
|
||||
if (existsSync(estimates) && (!pathOwner || selected.has(pathOwner))) {
|
||||
throw new Error(`missing Criterion benchmark.json: ${sub}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const metadata: unknown = JSON.parse(readFileSync(bench, "utf8"));
|
||||
const id = metadata && typeof metadata === "object" && "full_id" in metadata ? metadata.full_id : undefined;
|
||||
if (typeof id !== "string" || !id) {
|
||||
throw new Error(`invalid Criterion full_id: ${bench}`);
|
||||
}
|
||||
const owner = owners.get(id) ?? groups.get(id.split("/")[0]!);
|
||||
if (owner && !selected.has(owner)) return;
|
||||
if (!expected.has(id)) throw new Error(`unknown selected Criterion ID: ${id}`);
|
||||
if (seen.has(id)) throw new Error(`duplicate Criterion ID: ${id}`);
|
||||
seen.add(id);
|
||||
for (const file of [bench, estimates]) {
|
||||
const modified = statSync(file).mtimeMs;
|
||||
artifacts.push({ path: relative(dir, file), modifiedAt: new Date(modified).toISOString() });
|
||||
// Allow filesystem timestamp rounding, not future-dated benchmark results.
|
||||
if (modified > now + 1000) throw new Error(`future Criterion artifact: ${file}`);
|
||||
if (started !== undefined && modified < started) throw new Error(`stale Criterion artifact: ${file}`);
|
||||
}
|
||||
const est: unknown = JSON.parse(readFileSync(estimates, "utf8"));
|
||||
const mean = est && typeof est === "object" && "mean" in est ? est.mean : undefined;
|
||||
const value = mean && typeof mean === "object" && "point_estimate" in mean ? mean.point_estimate : undefined;
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
||||
throw new Error(`invalid Criterion mean estimate: ${estimates}`);
|
||||
}
|
||||
metrics[id] = value / 1000;
|
||||
}
|
||||
function visit(path: string) {
|
||||
try {
|
||||
if (!existsSync(path)) return;
|
||||
try {
|
||||
readNew(path);
|
||||
} catch (error) {
|
||||
errors.push(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
for (const entry of readdirSync(path, { withFileTypes: true })) {
|
||||
if (entry.isDirectory() && !["new", "base", "report"].includes(entry.name)) visit(join(path, entry.name));
|
||||
}
|
||||
} catch (error) {
|
||||
errors.push(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
visit(dir);
|
||||
const missing = [...expected].filter((id) => !Object.hasOwn(metrics, id));
|
||||
if (missing.length) errors.push(`missing selected Criterion IDs: ${missing.join(", ")}`);
|
||||
return { metrics, artifacts, errors };
|
||||
}
|
||||
|
||||
/** Strict convenience interface for consumers that require the complete selected workload. */
|
||||
export function collectRustMetrics(dir: string, packages: readonly string[], started?: number): Record<string, number> {
|
||||
const result = inspectRustArtifacts(dir, packages, started);
|
||||
if (result.errors.length) throw new Error(result.errors.join("\n"));
|
||||
return result.metrics;
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RUST_BENCHMARKS } from "./rust.ts";
|
||||
import { runRust } from "./rustRun.ts";
|
||||
|
||||
const TEXT_BENCHMARKS = RUST_BENCHMARKS["ide-text"]!.length;
|
||||
|
||||
const roots: string[] = [];
|
||||
const ownedPids: number[] = [];
|
||||
beforeEach(() => {
|
||||
vi.stubEnv("CARGO_TARGET_DIR", "target");
|
||||
vi.stubEnv("CRITERION_HOME", "target/criterion");
|
||||
});
|
||||
function fixture() {
|
||||
const root = mkdtempSync(join(tmpdir(), "rust-run-"));
|
||||
roots.push(root);
|
||||
mkdirSync(join(root, "perf"));
|
||||
writeFileSync(join(root, "perf/budgets.json"), readFileSync(resolve(import.meta.dirname, "../budgets.json")));
|
||||
mkdirSync(join(root, "target/perf"), { recursive: true });
|
||||
writeFileSync(join(root, "target/perf/report-rust.json"), '{"sentinel":"old passing report"}');
|
||||
writeFileSync(join(root, "target/perf/report-rust.md"), "old passing report");
|
||||
return root;
|
||||
}
|
||||
function report(root: string) {
|
||||
return JSON.parse(readFileSync(join(root, "target/perf/report-rust.json"), "utf8"));
|
||||
}
|
||||
function artifacts(root: string, packages: readonly string[], date = new Date()) {
|
||||
for (const p of packages)
|
||||
for (const id of RUST_BENCHMARKS[p]!) {
|
||||
const directory = join(root, id, "new");
|
||||
mkdirSync(directory, { recursive: true });
|
||||
for (const [file, content] of [
|
||||
["benchmark.json", { full_id: id }],
|
||||
["estimates.json", { mean: { point_estimate: 1 } }],
|
||||
] as const) {
|
||||
const path = join(directory, file);
|
||||
writeFileSync(path, JSON.stringify(content));
|
||||
utimesSync(path, date, date);
|
||||
}
|
||||
}
|
||||
}
|
||||
function recorded(root: string, id: string, packages: string[], started: number, error?: string) {
|
||||
const runRoot = join(root, "target/criterion/runs", id);
|
||||
mkdirSync(runRoot, { recursive: true });
|
||||
writeFileSync(
|
||||
join(runRoot, "run.json"),
|
||||
JSON.stringify({
|
||||
id,
|
||||
packages,
|
||||
startedAt: new Date(started).toISOString(),
|
||||
finishedAt: new Date(started + 5000).toISOString(),
|
||||
...(error ? { error } : {}),
|
||||
}),
|
||||
);
|
||||
artifacts(join(runRoot, "criterion"), packages, new Date(started + 1000));
|
||||
return runRoot;
|
||||
}
|
||||
function childCommand(tail: string, ids = RUST_BENCHMARKS["ide-text"]!) {
|
||||
return {
|
||||
file: process.execPath,
|
||||
args: [
|
||||
"-e",
|
||||
`
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
for (const id of ${JSON.stringify(ids)}) {
|
||||
const directory = path.join(process.env.CRITERION_HOME, id, 'new');
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
fs.writeFileSync(path.join(directory, 'benchmark.json'), JSON.stringify({full_id:id}));
|
||||
fs.writeFileSync(path.join(directory, 'estimates.json'), JSON.stringify({mean:{point_estimate:1}}));
|
||||
}
|
||||
console.error('partial cargo evidence');
|
||||
${tail}
|
||||
`,
|
||||
],
|
||||
};
|
||||
}
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
for (const pid of ownedPids.splice(0)) {
|
||||
try {
|
||||
process.kill(pid, "SIGKILL");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
|
||||
}
|
||||
}
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("Rust trial lifecycle", () => {
|
||||
it.each(["nonzero", "timeout"])("replaces old reports and cleans descendants after %s", async (failure) => {
|
||||
const root = fixture();
|
||||
const pidFile = join(root, "descendant.pid");
|
||||
const heartbeat = join(root, "heartbeat");
|
||||
const command = childCommand(`
|
||||
const {spawn} = require('node:child_process');
|
||||
const descendant = spawn(process.execPath, ['-e', ${JSON.stringify(`
|
||||
const fs = require('node:fs');
|
||||
process.on('SIGTERM', () => {});
|
||||
fs.writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));
|
||||
setInterval(() => fs.appendFileSync(${JSON.stringify(heartbeat)}, '.'), 10);
|
||||
`)}], {stdio:'ignore'});
|
||||
const ready = setInterval(() => {
|
||||
if (!fs.existsSync(${JSON.stringify(pidFile)})) return;
|
||||
clearInterval(ready);
|
||||
${failure === "nonzero" ? "process.exit(7);" : "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"}
|
||||
}, 5);
|
||||
`);
|
||||
const started = Date.now();
|
||||
const code = await runRust({
|
||||
root,
|
||||
packages: ["ide-text"],
|
||||
command,
|
||||
timeoutMs: failure === "timeout" ? 500 : 3000,
|
||||
cleanupGraceMs: 100,
|
||||
});
|
||||
expect(Date.now() - started).toBeLessThan(5000);
|
||||
expect(code).toBe(1);
|
||||
const current = report(root);
|
||||
expect(current.sentinel).toBeUndefined();
|
||||
expect(current.errors.map((entry: { error: string }) => entry.error).join(" ")).toMatch(
|
||||
failure === "timeout" ? /timed out/ : /exit 7/,
|
||||
);
|
||||
expect(Object.keys(current.results[0].metrics)).toHaveLength(TEXT_BENCHMARKS);
|
||||
expect(current.meta.mode).toBe("execution");
|
||||
const runRoot = resolve(current.meta.artifactRoot, "..");
|
||||
expect(readFileSync(join(runRoot, "cargo.stderr.log"), "utf8")).toContain("partial cargo evidence");
|
||||
expect(JSON.parse(readFileSync(join(runRoot, "run.json"), "utf8")).error).toBeTruthy();
|
||||
expect(readFileSync(join(root, "target/perf/report-rust.md"), "utf8")).not.toContain("old passing report");
|
||||
expect(existsSync(pidFile)).toBe(true);
|
||||
const pid = Number(readFileSync(pidFile, "utf8"));
|
||||
ownedPids.push(pid);
|
||||
const before = existsSync(heartbeat) ? readFileSync(heartbeat, "utf8") : "";
|
||||
// Observe the real child's heartbeat after cleanup; fake timers cannot advance a subprocess.
|
||||
await new Promise((done) => setTimeout(done, 100));
|
||||
expect(existsSync(heartbeat) ? readFileSync(heartbeat, "utf8") : "").toBe(before);
|
||||
});
|
||||
|
||||
it("replaces old reports on spawn failure", async () => {
|
||||
const root = fixture();
|
||||
expect(
|
||||
await runRust({ root, packages: ["ide-text"], command: { file: join(root, "missing-cargo"), args: [] } }),
|
||||
).toBe(1);
|
||||
expect(report(root).errors[0].error).toMatch(/spawn failed/);
|
||||
expect(readFileSync(join(root, "target/perf/report-rust.md"), "utf8")).toContain("spawn failed");
|
||||
});
|
||||
|
||||
it.each(["missing", "malformed"])("reports %s artifacts after a successful child", async (kind) => {
|
||||
const root = fixture();
|
||||
const command = childCommand(
|
||||
kind === "malformed"
|
||||
? `fs.writeFileSync(path.join(process.env.CRITERION_HOME,'text/open_1mb/new/estimates.json'),'{');`
|
||||
: "",
|
||||
kind === "missing" ? [] : undefined,
|
||||
);
|
||||
expect(await runRust({ root, packages: ["ide-text"], command, timeoutMs: 3000 })).toBe(1);
|
||||
expect(report(root).errors.length).toBeGreaterThan(0);
|
||||
expect(readFileSync(join(root, "target/perf/report-rust.md"), "utf8")).not.toContain("old passing report");
|
||||
if (kind === "malformed") expect(Object.keys(report(root).results[0].metrics)).toHaveLength(TEXT_BENCHMARKS - 1);
|
||||
});
|
||||
|
||||
it("never reuses old or concurrent roots as current execution", async () => {
|
||||
const root = fixture();
|
||||
const earlier = recorded(root, "old", ["ide-text"], Date.now() - 60_000);
|
||||
const concurrent = recorded(root, "other", ["ide-text"], Date.now() - 30_000);
|
||||
const future = new Date(Date.now() + 60_000);
|
||||
utimesSync(join(concurrent, "criterion/text/open_1mb/new/estimates.json"), future, future);
|
||||
expect(await runRust({ root, packages: ["ide-text"], command: childCommand("", []), timeoutMs: 3000 })).toBe(1);
|
||||
const current = report(root);
|
||||
expect(current.results[0].metrics).toEqual({});
|
||||
expect(current.meta.artifactRoot).not.toBe(join(earlier, "criterion"));
|
||||
expect(current.meta.artifactRoot).not.toBe(join(concurrent, "criterion"));
|
||||
expect(current.errors[0].error).toMatch(/missing selected/);
|
||||
});
|
||||
|
||||
it("inspects selected scope and original dates without claiming a new execution", async () => {
|
||||
const root = fixture();
|
||||
const runRoot = recorded(root, "selected", ["ide-text", "ide-search"], Date.now() - 60_000);
|
||||
recorded(root, "unselected-newer", ["ide-heap"], Date.now() - 30_000);
|
||||
expect(await runRust({ root, packages: ["ide-text"], noRun: true })).toBe(0);
|
||||
const current = report(root);
|
||||
expect(current.meta.mode).toBe("inspection");
|
||||
expect(current.meta.artifactRoot).toBe(join(runRoot, "criterion"));
|
||||
expect(current.meta.artifactStartedAt).toBe(JSON.parse(readFileSync(join(runRoot, "run.json"), "utf8")).startedAt);
|
||||
expect(current.artifacts).toHaveLength(2 * TEXT_BENCHMARKS);
|
||||
expect(
|
||||
current.artifacts.every((file: { modifiedAt: string }) => Date.parse(file.modifiedAt) < Date.now() - 10_000),
|
||||
).toBe(true);
|
||||
expect(Object.keys(current.results[0].metrics)).toHaveLength(TEXT_BENCHMARKS);
|
||||
});
|
||||
|
||||
it("keeps legacy inspection useful and reports artifact dates", async () => {
|
||||
const root = fixture();
|
||||
const date = new Date(Date.now() - 120_000);
|
||||
artifacts(join(root, "target/criterion"), ["ide-text"], date);
|
||||
expect(await runRust({ root, packages: ["ide-text"], noRun: true })).toBe(0);
|
||||
const current = report(root);
|
||||
expect(current.meta.source).toBe("legacy inspection");
|
||||
expect(current.artifacts[0].modifiedAt).toBe(date.toISOString());
|
||||
});
|
||||
|
||||
it.each(["scope", "future", "running", "failed"])("rejects invalid recorded inspection: %s", async (kind) => {
|
||||
const root = fixture();
|
||||
const runRoot = recorded(
|
||||
root,
|
||||
"recorded",
|
||||
kind === "scope" ? ["ide-heap"] : ["ide-text"],
|
||||
Date.now() - 60_000,
|
||||
kind === "failed" ? "cargo failed earlier" : undefined,
|
||||
);
|
||||
const path = join(runRoot, "run.json");
|
||||
const manifest = JSON.parse(readFileSync(path, "utf8"));
|
||||
if (kind === "future") manifest.startedAt = new Date(Date.now() + 60_000).toISOString();
|
||||
if (kind === "running") delete manifest.finishedAt;
|
||||
writeFileSync(path, JSON.stringify(manifest));
|
||||
expect(await runRust({ root, packages: ["ide-text"], noRun: true })).toBe(1);
|
||||
const error = report(root).errors[0].error;
|
||||
expect(error).toMatch(
|
||||
{ scope: /covers selected/, future: /future/, running: /still running/, failed: /failed earlier/ }[kind]!,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects artifacts modified after their recorded completion", async () => {
|
||||
const root = fixture();
|
||||
const runRoot = recorded(root, "recorded", ["ide-text"], Date.now() - 60_000);
|
||||
const file = join(runRoot, "criterion/text/open_1mb/new/estimates.json");
|
||||
const now = new Date();
|
||||
utimesSync(file, now, now);
|
||||
expect(await runRust({ root, packages: ["ide-text"], noRun: true })).toBe(1);
|
||||
expect(report(root).errors[0].error).toMatch(/future Criterion artifact/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,253 @@
|
||||
import {
|
||||
closeSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
openSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { basename, join } from "node:path";
|
||||
import { evaluate, loadBudgets, printReport, type Budgets, type Report, type ScenarioResult } from "./report.ts";
|
||||
import { inspectRustArtifacts, rustBudgets, rustPaths, selectRustPackages, type RustArtifacts } from "./rust.ts";
|
||||
import { runCommand } from "./process.ts";
|
||||
|
||||
// Includes cold compilation and the complete Criterion workload. Tests inject shorter deadlines.
|
||||
const CARGO_TIMEOUT_MS = 60 * 60 * 1000;
|
||||
const CLEANUP_GRACE_MS = 1000;
|
||||
|
||||
type CargoOutcome = { error?: string };
|
||||
type RunManifest = {
|
||||
id: string;
|
||||
packages: string[];
|
||||
startedAt: string;
|
||||
finishedAt?: string;
|
||||
error?: string;
|
||||
cargo?: CargoOutcome;
|
||||
};
|
||||
|
||||
function message(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
/** Keep cargo output alongside partial Criterion artifacts, including spawn and timeout failures. */
|
||||
async function runCargo(
|
||||
command: { file: string; args: string[] },
|
||||
cwd: string,
|
||||
env: NodeJS.ProcessEnv,
|
||||
logs: string,
|
||||
timeoutMs: number,
|
||||
graceMs: number,
|
||||
): Promise<CargoOutcome> {
|
||||
const stdout = openSync(join(logs, "cargo.stdout.log"), "a");
|
||||
let stderr: number | undefined;
|
||||
try {
|
||||
stderr = openSync(join(logs, "cargo.stderr.log"), "a");
|
||||
await runCommand(command.file, command.args, {
|
||||
cwd,
|
||||
env,
|
||||
timeoutMs,
|
||||
graceMs,
|
||||
label: "cargo bench",
|
||||
stdio: ["ignore", stdout, stderr],
|
||||
});
|
||||
return {};
|
||||
} catch (error) {
|
||||
return { error: message(error) };
|
||||
} finally {
|
||||
closeSync(stdout);
|
||||
if (stderr !== undefined) closeSync(stderr);
|
||||
}
|
||||
}
|
||||
|
||||
function inspectionSource(criterion: string, packages: string[], now: number) {
|
||||
const runs = join(criterion, "runs");
|
||||
// Pre-manifest Criterion output is still useful, but is explicitly labelled inspection-only.
|
||||
if (!existsSync(runs)) return { criterion, kind: "legacy inspection" };
|
||||
const manifests = readdirSync(runs, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => {
|
||||
const root = join(runs, entry.name);
|
||||
const manifest = JSON.parse(readFileSync(join(root, "run.json"), "utf8")) as RunManifest;
|
||||
if (
|
||||
manifest.id !== entry.name ||
|
||||
!Array.isArray(manifest.packages) ||
|
||||
!manifest.packages.length ||
|
||||
manifest.packages.some((p) => typeof p !== "string") ||
|
||||
!Number.isFinite(Date.parse(manifest.startedAt)) ||
|
||||
Date.parse(manifest.startedAt) > now + 1000
|
||||
)
|
||||
throw new Error(`invalid or future Rust run manifest: ${root}`);
|
||||
selectRustPackages(manifest.packages);
|
||||
return { root, manifest };
|
||||
})
|
||||
.filter(({ manifest }) => packages.every((p) => manifest.packages.includes(p)))
|
||||
.sort(
|
||||
(a, b) =>
|
||||
Date.parse(b.manifest.startedAt) - Date.parse(a.manifest.startedAt) ||
|
||||
b.manifest.id.localeCompare(a.manifest.id),
|
||||
);
|
||||
const source = manifests[0];
|
||||
if (!source) throw new Error(`no Rust artifact run covers selected packages: ${packages.join(" ")}`);
|
||||
const { root, manifest } = source;
|
||||
if (!manifest.finishedAt) throw new Error(`Rust artifact run is still running or incomplete: ${root}`);
|
||||
if (
|
||||
!Number.isFinite(Date.parse(manifest.finishedAt)) ||
|
||||
Date.parse(manifest.finishedAt) < Date.parse(manifest.startedAt) ||
|
||||
Date.parse(manifest.finishedAt) > now + 1000
|
||||
)
|
||||
throw new Error(`invalid or future Rust run completion date: ${root}`);
|
||||
return { criterion: join(root, "criterion"), kind: "recorded run inspection", manifest };
|
||||
}
|
||||
|
||||
/** Execute or inspect one Rust trial; infrastructure failures are results, never stale passing reports. */
|
||||
export async function runRust(options: {
|
||||
root: string;
|
||||
packages?: string[];
|
||||
noRun?: boolean;
|
||||
saveBaseline?: boolean;
|
||||
failOnRegression?: boolean;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
command?: { file: string; args: string[] };
|
||||
timeoutMs?: number;
|
||||
cleanupGraceMs?: number;
|
||||
}): Promise<number> {
|
||||
const env = options.env ?? process.env;
|
||||
const out = join(options.root, "target", "perf");
|
||||
const baselineFile = join(out, "baseline-rust.json");
|
||||
const errors: string[] = [];
|
||||
let packages = options.packages ?? [];
|
||||
let selectedBudgets: Record<string, number> = {};
|
||||
let budgets: Budgets | undefined;
|
||||
let baseline: ScenarioResult[] | null = null;
|
||||
let evidence: RustArtifacts = { metrics: {}, artifacts: [], errors: [] };
|
||||
let manifest: RunManifest | undefined;
|
||||
let runRoot: string | undefined;
|
||||
let artifactRoot: string | undefined;
|
||||
let source = options.noRun ? "inspection (no cargo execution)" : "current execution";
|
||||
const startedAt = new Date().toISOString();
|
||||
mkdirSync(out, { recursive: true });
|
||||
writeFileSync(
|
||||
join(out, "report-rust.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
meta: { date: startedAt, mode: options.noRun ? "inspection" : "execution", packages },
|
||||
status: "running",
|
||||
results: [],
|
||||
artifacts: [],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
writeFileSync(join(out, "report-rust.md"), "# OpenIDEA performance report\n\nRun in progress; no passing verdict.\n");
|
||||
try {
|
||||
packages = selectRustPackages(options.packages);
|
||||
budgets = loadBudgets(join(options.root, "perf", "budgets.json"));
|
||||
selectedBudgets = rustBudgets(packages, budgets.rust);
|
||||
const paths = rustPaths(options.root, env);
|
||||
let artifactStarted: number | undefined;
|
||||
if (options.noRun) {
|
||||
const inspection = inspectionSource(paths.criterion, packages, Date.now());
|
||||
artifactRoot = inspection.criterion;
|
||||
source = inspection.kind;
|
||||
manifest = inspection.manifest;
|
||||
if (manifest?.error) errors.push(`recorded run failed: ${manifest.error}`);
|
||||
artifactStarted = manifest ? Date.parse(manifest.startedAt) : undefined;
|
||||
} else {
|
||||
const timeoutMs = options.timeoutMs ?? Number(env.PERF_RUST_TIMEOUT_MS ?? CARGO_TIMEOUT_MS);
|
||||
const graceMs = options.cleanupGraceMs ?? CLEANUP_GRACE_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647)
|
||||
throw new Error("invalid PERF_RUST_TIMEOUT_MS");
|
||||
if (!Number.isFinite(graceMs) || graceMs <= 0 || graceMs > 2_147_483_647)
|
||||
throw new Error("invalid Rust cleanup deadline");
|
||||
const runs = join(paths.criterion, "runs");
|
||||
mkdirSync(runs, { recursive: true });
|
||||
runRoot = mkdtempSync(join(runs, "run-"));
|
||||
artifactRoot = join(runRoot, "criterion");
|
||||
mkdirSync(artifactRoot);
|
||||
manifest = { id: basename(runRoot), packages, startedAt };
|
||||
writeFileSync(join(runRoot, "run.json"), JSON.stringify(manifest, null, 2));
|
||||
console.log(`Rust cargo logs and artifacts: ${runRoot}`);
|
||||
manifest.cargo = await runCargo(
|
||||
options.command ?? { file: "cargo", args: ["bench", ...packages.flatMap((p) => ["-p", p]), "--benches"] },
|
||||
options.root,
|
||||
{ ...env, CARGO_TARGET_DIR: paths.target, CRITERION_HOME: artifactRoot },
|
||||
runRoot,
|
||||
timeoutMs,
|
||||
graceMs,
|
||||
);
|
||||
if (manifest.cargo.error) errors.push(manifest.cargo.error);
|
||||
artifactStarted = Date.parse(startedAt);
|
||||
}
|
||||
evidence = inspectRustArtifacts(
|
||||
artifactRoot,
|
||||
packages,
|
||||
artifactStarted,
|
||||
options.noRun && manifest?.finishedAt ? Date.parse(manifest.finishedAt) : Date.now(),
|
||||
);
|
||||
errors.push(...evidence.errors);
|
||||
if (existsSync(baselineFile)) baseline = JSON.parse(readFileSync(baselineFile, "utf8")) as ScenarioResult[];
|
||||
} catch (error) {
|
||||
errors.push(message(error));
|
||||
baseline = null;
|
||||
}
|
||||
if (runRoot && manifest) {
|
||||
manifest.finishedAt = new Date().toISOString();
|
||||
if (errors.length) manifest.error = errors.join("\n");
|
||||
try {
|
||||
writeFileSync(join(runRoot, "run.json"), JSON.stringify(manifest, null, 2));
|
||||
} catch (error) {
|
||||
errors.push(`Rust run manifest: ${message(error)}`);
|
||||
}
|
||||
}
|
||||
const results: ScenarioResult[] = [
|
||||
{
|
||||
id: "rust",
|
||||
title: "criterion means (µs)",
|
||||
metrics: evidence.metrics,
|
||||
...(errors.length ? { error: errors.join("\n") } : {}),
|
||||
},
|
||||
];
|
||||
let report: Report;
|
||||
try {
|
||||
report = evaluate(results, selectedBudgets, baseline, {
|
||||
expected: ["rust"],
|
||||
enforceBudgets: true,
|
||||
tolerance: budgets?.regressionTolerance,
|
||||
floor: budgets?.regressionFloor,
|
||||
});
|
||||
} catch (error) {
|
||||
errors.push(`Rust report evaluation: ${message(error)}`);
|
||||
results[0]!.error = errors.join("\n");
|
||||
report = evaluate(results, selectedBudgets, null, { expected: ["rust"], enforceBudgets: true });
|
||||
}
|
||||
const meta = {
|
||||
date: new Date().toISOString(),
|
||||
unit: "µs (mean)",
|
||||
packages: packages.join(" "),
|
||||
mode: options.noRun ? "inspection" : "execution",
|
||||
source,
|
||||
artifactRoot: artifactRoot ?? "unavailable",
|
||||
artifactRunId: manifest?.id ?? "unrecorded",
|
||||
artifactStartedAt: manifest?.startedAt ?? "unknown (legacy artifacts; see file dates)",
|
||||
artifactFinishedAt: manifest?.finishedAt ?? "unknown",
|
||||
};
|
||||
mkdirSync(out, { recursive: true });
|
||||
writeFileSync(
|
||||
join(out, "report-rust.json"),
|
||||
JSON.stringify(
|
||||
{ meta, status: report.failures ? "failed" : "complete", ...report, results, artifacts: evidence.artifacts },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
writeFileSync(join(out, "report-rust.md"), printReport(report, meta));
|
||||
if (errors.length) console.error(errors.join("\n"));
|
||||
if (options.saveBaseline) {
|
||||
writeFileSync(baselineFile, JSON.stringify(results, null, 2));
|
||||
console.log(`Baseline saved to ${baselineFile}`);
|
||||
}
|
||||
return report.failures > 0 || (options.failOnRegression && report.regressions > 0) ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { evidenceJson, metricErrors, summarizeMetrics, summarizeTrials, type Trial } from "./trials.ts";
|
||||
|
||||
const measured = (metrics: Record<string, number>, index: number): Trial => {
|
||||
const errors = metricErrors(metrics, ["ms"]);
|
||||
return {
|
||||
phase: "measured",
|
||||
index,
|
||||
metrics,
|
||||
status: errors.length ? "invalid" : "ok",
|
||||
error: errors.join("; ") || undefined,
|
||||
};
|
||||
};
|
||||
|
||||
describe("trial integrity", () => {
|
||||
test.each([NaN, Infinity, -Infinity])("does not hide a nonfinite repetition (%s)", (bad) => {
|
||||
const trials = [measured({ ms: 1 }, 1), measured({ ms: bad }, 2), measured({ ms: 3 }, 3)];
|
||||
const summary = summarizeTrials(trials, 3);
|
||||
expect(summary.metrics.ms).toBeNaN();
|
||||
expect(summary.error).toContain("measured 2");
|
||||
expect(summary.counts).toMatchObject({ valid: 2, attempted: 3, planned: 3 });
|
||||
expect(JSON.parse(evidenceJson(trials))[1].metrics.ms).toBe(String(bad));
|
||||
});
|
||||
test("missing required metrics and missing repetitions cannot shrink the denominator", () => {
|
||||
const trials = [measured({ ms: 1 }, 1), measured({ other: 2 }, 2)];
|
||||
const summary = summarizeTrials(trials, 3);
|
||||
expect(summary.metrics.ms).toBeNaN();
|
||||
expect(summary.error).toContain("ms is missing");
|
||||
expect(summary.error).toContain("2/3 attempted");
|
||||
expect(summary.counts).toMatchObject({ valid: 1, attempted: 2, planned: 3 });
|
||||
});
|
||||
test("a failed warmup remains fatal alongside good measured runs", () => {
|
||||
const trials: Trial[] = [
|
||||
{ phase: "warmup", index: 1, status: "error", metrics: {}, error: "startup failed" },
|
||||
measured({ ms: 1 }, 1),
|
||||
];
|
||||
const summary = summarizeTrials(trials, 1);
|
||||
expect(summary.metrics.ms).toBe(1);
|
||||
expect(summary.error).toContain("warmup 1: startup failed");
|
||||
expect(summary.counts).toMatchObject({ warmupValid: 0, warmupAttempted: 1 });
|
||||
});
|
||||
test("retains worst correctness and median timing without filtering invalid entries", () => {
|
||||
expect(
|
||||
summarizeMetrics(
|
||||
[
|
||||
{ ms: 3, recall: 1 },
|
||||
{ ms: 1, recall: 0.5 },
|
||||
],
|
||||
["recall"],
|
||||
),
|
||||
).toEqual({ ms: 1, recall: 0.5 });
|
||||
expect(summarizeMetrics([{ ms: 1 }, { ms: NaN }]).ms).toBeNaN();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import { median } from "./stats.ts";
|
||||
|
||||
export interface Trial {
|
||||
phase: "warmup" | "measured";
|
||||
index: number;
|
||||
status: "ok" | "invalid" | "error" | "timeout";
|
||||
metrics: Record<string, number>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export function metricErrors(metrics: Record<string, number>, required: readonly string[]): string[] {
|
||||
const errors: string[] = [];
|
||||
if (!Object.keys(metrics).length) errors.push("no metrics produced");
|
||||
for (const key of new Set([...required, ...Object.keys(metrics)])) {
|
||||
if (!Number.isFinite(metrics[key])) errors.push(`${key} is missing or nonfinite (${String(metrics[key])})`);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
/** Missing or invalid repetitions poison the summary, never shrink its denominator. */
|
||||
export function summarizeMetrics(samples: readonly Record<string, number>[], minima: readonly string[] = []) {
|
||||
const keys = [...new Set(samples.flatMap((s) => Object.keys(s)))];
|
||||
return Object.fromEntries(
|
||||
keys.map((key) => {
|
||||
const values = samples.map((s) => s[key]!);
|
||||
return [key, values.every(Number.isFinite) ? (minima.includes(key) ? Math.min(...values) : median(values)) : NaN];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function summarizeTrials(trials: readonly Trial[], planned: number) {
|
||||
const measured = trials.filter((t) => t.phase === "measured");
|
||||
const errors = trials.filter((t) => t.status !== "ok").map((t) => `${t.phase} ${t.index}: ${t.error ?? t.status}`);
|
||||
if (measured.length !== planned) errors.push(`measured trials: ${measured.length}/${planned} attempted`);
|
||||
return {
|
||||
metrics: summarizeMetrics(measured.map((t) => t.metrics)),
|
||||
error: errors.length ? errors.join("; ") : undefined,
|
||||
counts: {
|
||||
planned,
|
||||
attempted: measured.length,
|
||||
valid: measured.filter((t) => t.status === "ok").length,
|
||||
warmupAttempted: trials.length - measured.length,
|
||||
warmupValid: trials.filter((t) => t.phase === "warmup" && t.status === "ok").length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const errorMessage = (error: unknown): string =>
|
||||
error instanceof AggregateError
|
||||
? `${error.message}: ${error.errors.map(errorMessage).join("; ")}`
|
||||
: error instanceof Error
|
||||
? error.message
|
||||
: String(error);
|
||||
|
||||
/** JSON has no nonfinite numbers; retain their identity rather than turning them into null. */
|
||||
export const evidenceJson = (value: unknown) =>
|
||||
JSON.stringify(value, (_key, v: unknown) => (typeof v === "number" && !Number.isFinite(v) ? String(v) : v), 2);
|
||||
+155
-65
@@ -12,16 +12,19 @@
|
||||
* pnpm perf -- --only typing --profile --unminified CPU profile + top functions
|
||||
*/
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import type { Server } from "node:http";
|
||||
import { join, resolve } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { chromium, type Browser } from "@playwright/test";
|
||||
import type { Browser, BrowserContext } from "@playwright/test";
|
||||
import { buildBundle, serveBundle } from "./lib/bundle.ts";
|
||||
import { Harness, type Metrics, type Scenario } from "./lib/harness.ts";
|
||||
import { topFunctions, type CpuProfile } from "./lib/profile.ts";
|
||||
import { installInPage } from "./lib/inpage.ts";
|
||||
import { countOption } from "./lib/options.ts";
|
||||
import { evaluate, loadBudgets, printReport, type ScenarioResult } from "./lib/report.ts";
|
||||
import { median } from "./lib/stats.ts";
|
||||
import { closeServer, DeadlineError, launchBrowser, withDeadline, type BrowserSession } from "./lib/lifecycle.ts";
|
||||
import { errorMessage, evidenceJson, metricErrors, summarizeTrials, type Trial } from "./lib/trials.ts";
|
||||
import { SCENARIOS } from "./scenarios.ts";
|
||||
|
||||
const ROOT = resolve(import.meta.dirname, "..");
|
||||
@@ -51,20 +54,31 @@ if (args.list) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const runs = Math.max(1, Number(args.runs));
|
||||
const warmup = Math.max(0, Number(args.warmup));
|
||||
const cpu = Math.max(1, Number(args.cpu));
|
||||
const runs = countOption("runs", args.runs!, 1);
|
||||
const warmup = countOption("warmup", args.warmup!, 0);
|
||||
const cpu = Number(args.cpu);
|
||||
if (!args.cpu?.trim() || !Number.isFinite(cpu) || cpu < 1) {
|
||||
throw new Error("--cpu must be a finite number >= 1");
|
||||
}
|
||||
// Unminified builds keep function names readable in profiles; their bundle size is meaningless.
|
||||
const DIST = join(OUT, args.unminified ? "dist-unminified" : "dist");
|
||||
const only = args.only?.split(",").map((s) => s.trim());
|
||||
const scenarios = only ? SCENARIOS.filter((s) => only.includes(s.id)) : SCENARIOS;
|
||||
if (only && scenarios.length !== only.length) {
|
||||
const known = new Set(SCENARIOS.map((s) => s.id));
|
||||
throw new Error(`unknown scenario(s): ${only.filter((o) => !known.has(o)).join(", ")} (see --list)`);
|
||||
const only = args.only === undefined ? undefined : [...new Set(args.only.split(",").map((s) => s.trim()))];
|
||||
const known = new Set(SCENARIOS.map((s) => s.id));
|
||||
if (known.size !== SCENARIOS.length) throw new Error("duplicate scenario ID");
|
||||
if (only?.some((id) => !known.has(id))) {
|
||||
throw new Error(`unknown scenario(s): ${only.filter((id) => !known.has(id)).join(", ")} (see --list)`);
|
||||
}
|
||||
const scenarios = only ? SCENARIOS.filter((s) => only.includes(s.id)) : SCENARIOS;
|
||||
if (!scenarios.length) throw new Error("empty workload");
|
||||
const budgets = loadBudgets(join(ROOT, "perf", "budgets.json"));
|
||||
for (const key of Object.keys(budgets.ui)) {
|
||||
const id = key.slice(0, key.indexOf("."));
|
||||
if (id !== "bundle" && !known.has(id)) throw new Error(`budget references unknown scenario: ${key}`);
|
||||
}
|
||||
const expected = [...(args.unminified ? [] : ["bundle"]), ...scenarios.map((s) => s.id)];
|
||||
|
||||
function build() {
|
||||
buildBundle(ROOT, DIST, args.unminified ? ["--minify", "false"] : []);
|
||||
async function build() {
|
||||
await buildBundle(ROOT, DIST, args.unminified ? ["--minify", "false"] : []);
|
||||
}
|
||||
|
||||
/** Initial-load JavaScript / CSS from the built index.html (gzip sizes). */
|
||||
@@ -86,12 +100,18 @@ interface Capture {
|
||||
}
|
||||
|
||||
async function runOnce(browser: Browser, url: string, scenario: Scenario, capture: Capture = {}): Promise<Metrics> {
|
||||
const context = await browser.newContext({
|
||||
baseURL: url,
|
||||
viewport: { width: 1400, height: 900 },
|
||||
deviceScaleFactor: 1,
|
||||
});
|
||||
try {
|
||||
let context: BrowserContext | undefined;
|
||||
let finished = false;
|
||||
const execute = async () => {
|
||||
context = await browser.newContext({
|
||||
baseURL: url,
|
||||
viewport: { width: 1400, height: 900 },
|
||||
deviceScaleFactor: 1,
|
||||
});
|
||||
if (finished) {
|
||||
await withDeadline(context.close(), 5000, "late context close");
|
||||
throw new Error("trial already ended");
|
||||
}
|
||||
await context.addInitScript(installInPage);
|
||||
const page = await context.newPage();
|
||||
const errors: string[] = [];
|
||||
@@ -122,83 +142,153 @@ async function runOnce(browser: Browser, url: string, scenario: Scenario, captur
|
||||
}
|
||||
if (errors.length) throw new Error(`page errors: ${errors.join("; ")}`);
|
||||
return metrics;
|
||||
};
|
||||
try {
|
||||
return await withDeadline(execute(), 5 * 60_000, `${scenario.id} trial`);
|
||||
} finally {
|
||||
await context.close();
|
||||
finished = true;
|
||||
if (context) await withDeadline(context.close(), 5000, "context close");
|
||||
}
|
||||
}
|
||||
|
||||
function aggregate(samples: Metrics[]): Metrics {
|
||||
const keys = [...new Set(samples.flatMap((s) => Object.keys(s)))];
|
||||
return Object.fromEntries(keys.map((k) => [k, median(samples.map((s) => s[k]!).filter(Number.isFinite))]));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
mkdirSync(join(OUT, "traces"), { recursive: true });
|
||||
mkdirSync(join(OUT, "profiles"), { recursive: true });
|
||||
if (!args["no-build"]) build();
|
||||
if (!existsSync(join(DIST, "index.html"))) throw new Error(`no build in ${DIST}; run without --no-build`);
|
||||
|
||||
const results: ScenarioResult[] = args.unminified
|
||||
? []
|
||||
: [{ id: "bundle", title: "Initial bundle (gzip)", metrics: bundleMetrics() }];
|
||||
const { server, url } = await serveBundle(DIST);
|
||||
const browser = await chromium
|
||||
.launch({
|
||||
const results: ScenarioResult[] = [];
|
||||
const trials: Record<string, Trial[]> = {};
|
||||
const counts: Record<string, unknown> = {};
|
||||
const infrastructure: string[] = [];
|
||||
const baselineFile = join(OUT, `baseline-${cpu}x.json`);
|
||||
let baseline: ScenarioResult[] | null = null;
|
||||
let runtime: BrowserSession | undefined;
|
||||
let server: Server | undefined;
|
||||
let browserVersion = "not started";
|
||||
let completed = false;
|
||||
const meta = { date: new Date().toISOString(), cpu, runs, warmup, channel: args.channel };
|
||||
const progress = () => {
|
||||
writeFileSync(
|
||||
join(OUT, "report.json"),
|
||||
evidenceJson({ meta, status: "running", expected, results, trials, counts }),
|
||||
);
|
||||
};
|
||||
progress();
|
||||
writeFileSync(join(OUT, "report.md"), "# OpenIDEA performance report\n\nRun in progress; no passing verdict.\n");
|
||||
try {
|
||||
baseline = existsSync(baselineFile) ? (JSON.parse(readFileSync(baselineFile, "utf8")) as ScenarioResult[]) : null;
|
||||
if (!args["no-build"]) await build();
|
||||
if (!existsSync(join(DIST, "index.html"))) throw new Error(`no build in ${DIST}; run without --no-build`);
|
||||
if (!args.unminified) results.push({ id: "bundle", title: "Initial bundle (gzip)", metrics: bundleMetrics() });
|
||||
const serving = await serveBundle(DIST);
|
||||
server = serving.server;
|
||||
runtime = await launchBrowser({
|
||||
channel: args.channel === "chromium" ? undefined : args.channel,
|
||||
headless: !args.headed,
|
||||
// Keep rAF / timers at full rate even when the window is not focused.
|
||||
args: [
|
||||
"--disable-renderer-backgrounding",
|
||||
"--disable-background-timer-throttling",
|
||||
"--disable-backgrounding-occluded-windows",
|
||||
],
|
||||
})
|
||||
.catch((e: Error) => {
|
||||
throw new Error(
|
||||
`${e.message}\nInstall the browser with \`pnpm exec playwright install chromium\` or pass --channel msedge|chrome.`,
|
||||
);
|
||||
});
|
||||
try {
|
||||
browserVersion = runtime.browser.version();
|
||||
for (const s of scenarios) {
|
||||
process.stdout.write(`${s.id.padEnd(14)} `);
|
||||
const samples: Metrics[] = [];
|
||||
try {
|
||||
for (let i = 0; i < warmup + runs; i++) {
|
||||
const attempts: Trial[] = (trials[s.id] = []);
|
||||
const required = Object.keys(budgets.ui)
|
||||
.filter((key) => key.startsWith(`${s.id}.`))
|
||||
.map((key) => key.slice(s.id.length + 1));
|
||||
for (let i = 0; i < warmup + runs; i++) {
|
||||
const phase = i < warmup ? "warmup" : "measured";
|
||||
const index = phase === "warmup" ? i + 1 : i - warmup + 1;
|
||||
try {
|
||||
const last = i === warmup + runs - 1;
|
||||
const m = await runOnce(browser, url, s, {
|
||||
const metrics = await runOnce(runtime.browser, serving.url, s, {
|
||||
trace: args.trace && last ? join(OUT, "traces", `${s.id}.json`) : undefined,
|
||||
profile: args.profile && last ? join(OUT, "profiles", `${s.id}.cpuprofile`) : undefined,
|
||||
});
|
||||
if (i >= warmup) samples.push(m);
|
||||
process.stdout.write(i < warmup ? "w" : ".");
|
||||
const errors = metricErrors(metrics, required);
|
||||
attempts.push({
|
||||
phase,
|
||||
index,
|
||||
status: errors.length ? "invalid" : "ok",
|
||||
metrics,
|
||||
error: errors.join("; ") || undefined,
|
||||
});
|
||||
process.stdout.write(errors.length ? "!" : phase === "warmup" ? "w" : ".");
|
||||
} catch (error) {
|
||||
attempts.push({
|
||||
phase,
|
||||
index,
|
||||
status: error instanceof DeadlineError ? "timeout" : "error",
|
||||
metrics: {},
|
||||
error: errorMessage(error),
|
||||
});
|
||||
process.stdout.write(" failed");
|
||||
progress();
|
||||
break;
|
||||
}
|
||||
results.push({ id: s.id, title: s.title, metrics: aggregate(samples) });
|
||||
console.log();
|
||||
} catch (e) {
|
||||
console.log(" failed");
|
||||
results.push({ id: s.id, title: s.title, metrics: {}, error: e instanceof Error ? e.message : String(e) });
|
||||
progress();
|
||||
}
|
||||
const summary = summarizeTrials(attempts, runs);
|
||||
counts[s.id] = summary.counts;
|
||||
results.push({ id: s.id, title: s.title, metrics: summary.metrics, error: summary.error });
|
||||
progress();
|
||||
console.log();
|
||||
}
|
||||
completed = true;
|
||||
} catch (error) {
|
||||
infrastructure.push(errorMessage(error));
|
||||
} finally {
|
||||
if (runtime) {
|
||||
try {
|
||||
await runtime.close();
|
||||
} catch (error) {
|
||||
infrastructure.push(`browser cleanup: ${errorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
if (server) {
|
||||
try {
|
||||
await closeServer(server);
|
||||
} catch (error) {
|
||||
infrastructure.push(`server cleanup: ${errorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
server.close();
|
||||
}
|
||||
|
||||
const baselineFile = join(OUT, `baseline-${cpu}x.json`);
|
||||
const baseline = existsSync(baselineFile)
|
||||
? (JSON.parse(readFileSync(baselineFile, "utf8")) as ScenarioResult[])
|
||||
: null;
|
||||
const budgets = loadBudgets(join(ROOT, "perf", "budgets.json"));
|
||||
const report = evaluate(results, budgets.ui, baseline, {
|
||||
expected,
|
||||
enforceBudgets: cpu === 1,
|
||||
tolerance: budgets.regressionTolerance,
|
||||
floor: budgets.regressionFloor,
|
||||
});
|
||||
const meta = { date: new Date().toISOString(), cpu, runs, channel: args.channel, browser: browser.version() };
|
||||
writeFileSync(join(OUT, "report.json"), JSON.stringify({ meta, ...report }, null, 2));
|
||||
writeFileSync(join(OUT, "report.md"), printReport(report, meta));
|
||||
if (args["save-baseline"]) {
|
||||
writeFileSync(baselineFile, JSON.stringify(results, null, 2));
|
||||
for (const error of infrastructure) report.errors.push({ id: "infrastructure", error });
|
||||
report.failures += infrastructure.length;
|
||||
const finalMeta = { ...meta, browser: browserVersion };
|
||||
writeFileSync(
|
||||
join(OUT, "report.json"),
|
||||
evidenceJson({
|
||||
meta: finalMeta,
|
||||
status: report.failures ? "failed" : "complete",
|
||||
...report,
|
||||
expected,
|
||||
results,
|
||||
trials,
|
||||
counts,
|
||||
infrastructure,
|
||||
}),
|
||||
);
|
||||
const trialTable = [
|
||||
"",
|
||||
"## Trials",
|
||||
"",
|
||||
"| Scenario | Valid / planned measured | Attempted measured | Valid / attempted warmup |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
...Object.entries(trials).map(([id, attempts]) => {
|
||||
const c = summarizeTrials(attempts, runs).counts;
|
||||
return `| ${id} | ${c.valid}/${c.planned} | ${c.attempted} | ${c.warmupValid}/${c.warmupAttempted} |`;
|
||||
}),
|
||||
].join("\n");
|
||||
writeFileSync(join(OUT, "report.md"), printReport(report, finalMeta) + trialTable + "\n");
|
||||
if (args["save-baseline"] && completed) {
|
||||
writeFileSync(baselineFile, evidenceJson(results));
|
||||
console.log(`Baseline saved to ${baselineFile}`);
|
||||
}
|
||||
const failed = report.failures > 0 || (args["fail-on-regression"] && report.regressions > 0);
|
||||
|
||||
+9
-63
@@ -4,19 +4,14 @@
|
||||
* and the local baseline.
|
||||
*
|
||||
* pnpm perf:rust run benches, then check
|
||||
* pnpm perf:rust -- --no-run check the latest criterion results only
|
||||
* pnpm perf:rust -- --no-run inspect recorded results (no execution)
|
||||
* pnpm perf:rust -- --package ide-text --save-baseline
|
||||
*/
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { resolve } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { evaluate, loadBudgets, printReport, type ScenarioResult } from "./lib/report.ts";
|
||||
import { runRust } from "./lib/rustRun.ts";
|
||||
|
||||
const ROOT = resolve(import.meta.dirname, "..");
|
||||
const TARGET = resolve(ROOT, process.env.CARGO_TARGET_DIR ?? "target");
|
||||
const OUT = join(ROOT, "target", "perf");
|
||||
const PACKAGES = ["ide-text", "ide-language", "ide-index", "ide-search", "ide-host", "ide-heap"];
|
||||
|
||||
const { values: args } = parseArgs({
|
||||
options: {
|
||||
@@ -27,59 +22,10 @@ const { values: args } = parseArgs({
|
||||
},
|
||||
});
|
||||
|
||||
const packages = args.package ?? PACKAGES;
|
||||
const started = Date.now();
|
||||
if (!args["no-run"]) {
|
||||
const r = spawnSync("cargo", ["bench", ...packages.flatMap((p) => ["-p", p]), "--benches"], {
|
||||
cwd: ROOT,
|
||||
stdio: "inherit",
|
||||
});
|
||||
if (r.status !== 0) throw new Error("cargo bench failed");
|
||||
}
|
||||
|
||||
/** Criterion results (`<id>/new/{benchmark,estimates}.json`) → mean in µs. */
|
||||
function collect(dir: string, out: Record<string, number>) {
|
||||
if (!existsSync(dir)) return;
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name === "report") continue;
|
||||
const sub = join(dir, entry.name);
|
||||
const bench = join(sub, "new", "benchmark.json");
|
||||
if (existsSync(bench) && (args["no-run"] || statSync(bench).mtimeMs >= started)) {
|
||||
const { full_id } = JSON.parse(readFileSync(bench, "utf8")) as { full_id: string };
|
||||
const est = JSON.parse(readFileSync(join(sub, "new", "estimates.json"), "utf8")) as {
|
||||
mean: { point_estimate: number };
|
||||
};
|
||||
out[full_id] = est.mean.point_estimate / 1000;
|
||||
}
|
||||
collect(sub, out);
|
||||
}
|
||||
}
|
||||
|
||||
const metrics: Record<string, number> = {};
|
||||
collect(join(TARGET, "criterion"), metrics);
|
||||
if (Object.keys(metrics).length === 0) throw new Error(`no criterion results under ${join(TARGET, "criterion")}`);
|
||||
|
||||
mkdirSync(OUT, { recursive: true });
|
||||
const budgets = loadBudgets(join(ROOT, "perf", "budgets.json"));
|
||||
const results: ScenarioResult[] = [{ id: "rust", title: "criterion means (µs)", metrics }];
|
||||
const baselineFile = join(OUT, "baseline-rust.json");
|
||||
const baseline = existsSync(baselineFile) ? (JSON.parse(readFileSync(baselineFile, "utf8")) as ScenarioResult[]) : null;
|
||||
// Only budgets of benches that ran: `--package` may select a subset.
|
||||
const rustBudgets = Object.fromEntries(
|
||||
Object.entries(budgets.rust)
|
||||
.filter(([id]) => args["no-run"] || id in metrics || !args.package)
|
||||
.map(([id, v]) => [`rust.${id}`, v]),
|
||||
);
|
||||
const report = evaluate(results, rustBudgets, baseline, {
|
||||
enforceBudgets: true,
|
||||
tolerance: budgets.regressionTolerance,
|
||||
floor: budgets.regressionFloor,
|
||||
process.exitCode = await runRust({
|
||||
root: ROOT,
|
||||
packages: args.package,
|
||||
noRun: args["no-run"],
|
||||
saveBaseline: args["save-baseline"],
|
||||
failOnRegression: args["fail-on-regression"],
|
||||
});
|
||||
const meta = { date: new Date().toISOString(), unit: "µs (mean)", packages: packages.join(" ") };
|
||||
writeFileSync(join(OUT, "report-rust.json"), JSON.stringify({ meta, ...report }, null, 2));
|
||||
writeFileSync(join(OUT, "report-rust.md"), printReport(report, meta));
|
||||
if (args["save-baseline"]) {
|
||||
writeFileSync(baselineFile, JSON.stringify(results, null, 2));
|
||||
console.log(`Baseline saved to ${baselineFile}`);
|
||||
}
|
||||
process.exit(report.failures > 0 || (args["fail-on-regression"] && report.regressions > 0) ? 1 : 0);
|
||||
@@ -136,7 +136,7 @@ const completion: Scenario = {
|
||||
const before = await h.counters();
|
||||
const narrow = await h.typeMeasured("etOrd", 150);
|
||||
const work = Harness.delta(before, await h.counters());
|
||||
const arrows = await h.pressMeasured(Array(20).fill("ArrowDown"), 60);
|
||||
const arrows = await h.completionArrowsMeasured(20, 60);
|
||||
await h.page.keyboard.press("Escape");
|
||||
return {
|
||||
openMs,
|
||||
@@ -310,13 +310,9 @@ const scroll: Scenario = {
|
||||
const smooth = frameSummary(await wheel(160, 120), refresh);
|
||||
const tbtMs = totalBlockingTime(await h.longTasksSince(t0));
|
||||
const fling = frameSummary(await wheel(60, 600), refresh);
|
||||
// Jump far: scrollbar drag / Ctrl+End style repositioning.
|
||||
const jumpMs = await h.measure(
|
||||
() => h.page.keyboard.press("Control+End"),
|
||||
".cm-content",
|
||||
"export class Repository",
|
||||
5_000,
|
||||
);
|
||||
// Jump far: document-end repositioning.
|
||||
await h.setCursor(0);
|
||||
const jumpMs = await h.jumpToEnd();
|
||||
return {
|
||||
...smooth,
|
||||
tbtMs,
|
||||
@@ -673,7 +669,7 @@ const popups: Scenario = {
|
||||
);
|
||||
// Editing the query: Backspace → list re-filtered and painted.
|
||||
const refresh = await h.refreshMs();
|
||||
const keyLat = await h.typeMeasured("\b\b\b\b\b\b", 80);
|
||||
const keyLat = await h.typeMeasured("\b\b\b\b\b\b", 80, "input");
|
||||
await h.page.keyboard.press("Escape");
|
||||
await h.page.locator('[role="dialog"]').waitFor({ state: "detached" });
|
||||
|
||||
|
||||
@@ -26,6 +26,11 @@ process.stdin.on("data", (chunk) => {
|
||||
if (buffer.length < end + 4 + length) return;
|
||||
const request = JSON.parse(buffer.subarray(end + 4, end + 4 + length));
|
||||
buffer = buffer.subarray(end + 4 + length);
|
||||
if (mode === "no-initialize" && request.method === "initialize") continue;
|
||||
if (mode === "bad-header") {
|
||||
process.stdout.write("Not-a-length: 0\r\n\r\n");
|
||||
continue;
|
||||
}
|
||||
const body = JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: request.id,
|
||||
|
||||
@@ -58,6 +58,27 @@ test("close waits for graceful EOF cleanup of a tool in its own group", options,
|
||||
await server.close();
|
||||
});
|
||||
|
||||
test("initialize timeout cleans up the server and its independent tool", options, async () => {
|
||||
const { home } = await fixture("no-initialize");
|
||||
// Exercise the actual stdio deadline, rather than replacing the child's clock.
|
||||
await assert.rejects(Server.start(home, { SMOKE_SERVER_MODE: "no-initialize" }, 500), /initialize timed out/);
|
||||
const pids = JSON.parse(readFileSync(join(home, "pids.json"), "utf8"));
|
||||
assertGone(pids.server);
|
||||
assertGone(pids.tool);
|
||||
});
|
||||
|
||||
test(
|
||||
"malformed protocol rejects startup and cleans up instead of throwing out of an event callback",
|
||||
options,
|
||||
async () => {
|
||||
const { home, start } = await fixture("bad-header");
|
||||
await assert.rejects(start(), /bad header/);
|
||||
const pids = JSON.parse(readFileSync(join(home, "pids.json"), "utf8"));
|
||||
assertGone(pids.server);
|
||||
assertGone(pids.tool);
|
||||
},
|
||||
);
|
||||
|
||||
test("failed initialize still runs graceful cleanup and preserves the error", options, async () => {
|
||||
const { home, start } = await fixture("reject");
|
||||
await assert.rejects(start(), { message: "initialize failed" });
|
||||
|
||||
+40
-7
@@ -1,3 +1,4 @@
|
||||
/// <reference lib="es2024.promise" />
|
||||
/**
|
||||
* `openidea-server --stdio`, spoken to as the desktop app does in remote development:
|
||||
* JSON-RPC with `Content-Length` framing over the server's stdin / stdout.
|
||||
@@ -121,12 +122,21 @@ export class Server {
|
||||
onEvent: (window: string | null, event: unknown) => void = () => {};
|
||||
|
||||
private closing: Promise<void> | undefined;
|
||||
private failure: unknown;
|
||||
// A plain field, not a parameter property: the evals load this file with Node's type stripping.
|
||||
private readonly child: ChildProcessWithoutNullStreams;
|
||||
|
||||
private constructor(child: ChildProcessWithoutNullStreams) {
|
||||
this.child = child;
|
||||
child.stdout.on("data", (chunk: Buffer) => this.receive(chunk));
|
||||
child.stdout.on("data", (chunk: Buffer) => {
|
||||
try {
|
||||
this.receive(chunk);
|
||||
} catch (error) {
|
||||
this.failure = error;
|
||||
for (const p of this.pending.values()) p.reject(error);
|
||||
this.pending.clear();
|
||||
}
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer) => {
|
||||
// The whole log, when asked for (OPENIDEA_SMOKE_LOG=<file>; OPENIDEA_LOG sets its level).
|
||||
if (process.env.OPENIDEA_SMOKE_LOG) appendFileSync(process.env.OPENIDEA_SMOKE_LOG, chunk);
|
||||
@@ -134,6 +144,7 @@ export class Server {
|
||||
if (this.stderr.length > 200) this.stderr.shift();
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
this.failure = error;
|
||||
for (const p of this.pending.values()) p.reject(error);
|
||||
this.pending.clear();
|
||||
});
|
||||
@@ -143,13 +154,14 @@ export class Server {
|
||||
this.pending.clear();
|
||||
});
|
||||
child.on("exit", (code) => {
|
||||
this.failure = new Error(`openidea-server exited (${code}): ${this.log()}`);
|
||||
for (const p of this.pending.values()) p.reject(new Error(`openidea-server exited (${code}): ${this.log()}`));
|
||||
this.pending.clear();
|
||||
});
|
||||
}
|
||||
|
||||
/** Starts a server whose settings, caches and tools live under `home`. */
|
||||
static async start(home: string, env: Record<string, string> = {}): Promise<Server> {
|
||||
static async start(home: string, env: Record<string, string> = {}, initializeTimeoutMs = 30_000): Promise<Server> {
|
||||
const child = spawn(serverBinary(), ["--stdio"], {
|
||||
env: { ...process.env, OPENIDEA_SERVER_HOME: home, ...env },
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
@@ -158,7 +170,7 @@ export class Server {
|
||||
});
|
||||
const server = new Server(child);
|
||||
try {
|
||||
await server.request("initialize", { version: PROTOCOL_VERSION });
|
||||
await server.request("initialize", { version: PROTOCOL_VERSION }, initializeTimeoutMs);
|
||||
return server;
|
||||
} catch (error) {
|
||||
await server.close();
|
||||
@@ -258,12 +270,33 @@ export class Server {
|
||||
if (errors.size > 1) throw new AggregateError(errors, "Smoke server cleanup failed");
|
||||
}
|
||||
|
||||
private request(method: string, params: unknown): Promise<unknown> {
|
||||
private request(method: string, params: unknown, timeoutMs = 5 * 60_000): Promise<unknown> {
|
||||
if (this.failure) return Promise.reject(this.failure);
|
||||
if (this.closing) return Promise.reject(new Error("openidea-server is closing"));
|
||||
const id = this.nextId++;
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pending.set(id, { resolve, reject });
|
||||
this.send({ jsonrpc: "2.0", id, method, params });
|
||||
const { promise, resolve, reject } = Promise.withResolvers<unknown>();
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(id);
|
||||
reject(new Error(`${method} timed out after ${timeoutMs} ms`));
|
||||
}, timeoutMs);
|
||||
this.pending.set(id, {
|
||||
resolve: (value) => {
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
},
|
||||
reject: (error) => {
|
||||
clearTimeout(timer);
|
||||
reject(error);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.send({ jsonrpc: "2.0", id, method, params });
|
||||
} catch (error) {
|
||||
clearTimeout(timer);
|
||||
this.pending.delete(id);
|
||||
reject(error);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
private send(message: unknown) {
|
||||
|
||||
@@ -18,6 +18,16 @@ pub async fn window_group_merge(window: WebviewWindow) -> IdeResult<()> {
|
||||
groups::persist(window.app_handle()).await
|
||||
}
|
||||
|
||||
/// Adds window `label` to this window's group as a new project tab and selects it. Unlike
|
||||
/// [`window_group_merge`], unrelated windows keep their own group.
|
||||
#[desktop_command(trust = always)]
|
||||
#[tauri::command]
|
||||
#[specta::specta]
|
||||
pub async fn window_group_join(window: WebviewWindow, label: String) -> IdeResult<()> {
|
||||
groups::join(window.app_handle(), window.label(), &label)?;
|
||||
groups::persist(window.app_handle()).await
|
||||
}
|
||||
|
||||
#[desktop_command(trust = always)]
|
||||
#[tauri::command]
|
||||
#[specta::specta]
|
||||
@@ -86,6 +96,7 @@ ide_command::desktop_domain!(
|
||||
window_group_close_tabs,
|
||||
window_group_current,
|
||||
window_group_detach,
|
||||
window_group_join,
|
||||
window_group_merge,
|
||||
window_group_reorder,
|
||||
window_group_restore,
|
||||
|
||||
@@ -139,11 +139,14 @@ fn announce_available(app: &AppHandle) {
|
||||
return;
|
||||
}
|
||||
let state = app.state::<AppState>();
|
||||
// Asking a window whether it is focused waits for the main thread, where
|
||||
// `app_update_window_ready` takes `ready`: never hold the lock across it.
|
||||
let focused = crate::windows::target_window(app);
|
||||
let live = crate::windows::project_window_labels(app);
|
||||
let owner = {
|
||||
let mut ready = state.updates.ready.lock();
|
||||
ready.retain(|label| app.get_webview_window(label).is_some());
|
||||
let focused = crate::windows::target_window(app);
|
||||
ready_owner(focused.as_ref().map(|window| window.label()), &crate::windows::project_window_labels(app), &ready)
|
||||
ready_owner(focused.as_ref().map(|window| window.label()), &live, &ready)
|
||||
};
|
||||
let Some(owner) = owner else { return };
|
||||
let update = {
|
||||
|
||||
@@ -105,7 +105,6 @@ fn native_select(app: &AppHandle, group: &Group, to: &WebviewWindow) -> IdeResul
|
||||
crate::macos::sync_tab_group(members, &anchor, to).map_err(|e| IdeError::internal(e.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn join(app: &AppHandle, anchor: &str, new_window: &str) -> IdeResult<()> {
|
||||
let state = app.state::<WindowsState>();
|
||||
state.groups.lock().join(anchor, new_window)?;
|
||||
|
||||
@@ -726,6 +726,7 @@ window_group_close always desktop bin
|
||||
window_group_close_tabs always desktop bindings sync
|
||||
window_group_current always desktop bindings sync
|
||||
window_group_detach always desktop bindings async
|
||||
window_group_join always desktop bindings async
|
||||
window_group_merge always desktop bindings async
|
||||
window_group_reorder always desktop bindings async
|
||||
window_group_restore always desktop bindings async
|
||||
|
||||
@@ -1918,6 +1918,11 @@ export const commands = {
|
||||
windowGroupCloseTabs: (keep: string, labels: string[]) => typedError<null, IdeError>(__TAURI_INVOKE("window_group_close_tabs", { keep, labels })),
|
||||
windowGroupCurrent: () => typedError<WindowTab[], IdeError>(__TAURI_INVOKE("window_group_current")),
|
||||
windowGroupDetach: (label: string) => typedError<null, IdeError>(__TAURI_INVOKE("window_group_detach", { label })),
|
||||
/**
|
||||
* Adds window `label` to this window's group as a new project tab and selects it. Unlike
|
||||
* [`window_group_merge`], unrelated windows keep their own group.
|
||||
*/
|
||||
windowGroupJoin: (label: string) => typedError<null, IdeError>(__TAURI_INVOKE("window_group_join", { label })),
|
||||
windowGroupMerge: () => typedError<null, IdeError>(__TAURI_INVOKE("window_group_merge")),
|
||||
windowGroupReorder: (labels: string[]) => typedError<null, IdeError>(__TAURI_INVOKE("window_group_reorder", { labels })),
|
||||
windowGroupRestore: () => typedError<null, IdeError>(__TAURI_INVOKE("window_group_restore")),
|
||||
@@ -3745,6 +3750,12 @@ export type CommitSettings = {
|
||||
doubleClickOpensDiff?: boolean | null,
|
||||
/** Team conventions for Generate Commit Message with Agent, added to its prompt. */
|
||||
agentInstructions?: string,
|
||||
/** The coding agent Generate Commit Message with Agent uses; empty follows the latest task's agent. */
|
||||
agent?: string,
|
||||
/** That agent's model for commit messages; empty keeps the model the agent would use. */
|
||||
agentModel?: string,
|
||||
/** That model's thinking level for commit messages; empty keeps the agent's. */
|
||||
agentEffort?: string,
|
||||
};
|
||||
|
||||
export type Commit_Deserialize = {
|
||||
@@ -8226,10 +8237,23 @@ export type ManifestDependency = {
|
||||
to: number,
|
||||
};
|
||||
|
||||
export type MarkdownCommandDirectory =
|
||||
/** The project's root folder. */
|
||||
"project" |
|
||||
/** The Markdown file's parent folder. */
|
||||
"file";
|
||||
|
||||
/** Languages & Frameworks › Markdown. Like IDEA's, it is application-level only. */
|
||||
export type MarkdownSettings = {
|
||||
/** Preview (and notebook Markdown cell) font size in px; unset follows the editor font size. */
|
||||
previewFontSize?: number | null,
|
||||
/**
|
||||
* "Detect commands that can be run right from Markdown files": gutter run icons on
|
||||
* shell code blocks and on command lines in code blocks.
|
||||
*/
|
||||
runCommands?: boolean,
|
||||
/** Where those commands run; unset asks when the project and the file's folder differ. */
|
||||
commandDirectory?: MarkdownCommandDirectory | null,
|
||||
};
|
||||
|
||||
/** A marketplace entry with what is installed. */
|
||||
|
||||
@@ -66,6 +66,7 @@ function show(view: EditorView): string {
|
||||
}
|
||||
|
||||
const javascript = await languages.find((l) => l.name === "JavaScript")!.load();
|
||||
const yaml = await languages.find((l) => l.name === "YAML")!.load();
|
||||
|
||||
describe("Move Statement", () => {
|
||||
it("moves a multi-line statement as a whole, never one of its lines", () => {
|
||||
@@ -203,6 +204,19 @@ describe("paste reformatting", () => {
|
||||
paste(v, "a();\n b();\n c();");
|
||||
expect(v.state.doc.toString()).toBe("if {\n a();\n b();\n c();\n}");
|
||||
});
|
||||
|
||||
it("pastes onto a blank line at the previous line's indent when the language has no indent engine", () => {
|
||||
const doc = "service:\n database:\n query-timeout: 30\n pool:\n validation-query: SELECT 1\n";
|
||||
const v = make(doc + "|", [yaml, pasteReindentFilter("indentEachLine")]);
|
||||
paste(v, "query-timeout: 30");
|
||||
expect(v.state.doc.toString()).toBe(doc + " query-timeout: 30");
|
||||
});
|
||||
|
||||
it("pastes a block onto a blank line at the indent of the nearest non-empty line above", () => {
|
||||
const v = make("outer:\n key: 1\n\n|\ntrailing: 2", [yaml, pasteReindentFilter("indentBlock")]);
|
||||
paste(v, "a: 1\n b: 2");
|
||||
expect(v.state.doc.toString()).toBe("outer:\n key: 1\n\n a: 1\n b: 2\ntrailing: 2");
|
||||
});
|
||||
});
|
||||
|
||||
describe("closing quotes", () => {
|
||||
|
||||
@@ -222,6 +222,15 @@ function leadingColumns(text: string, tabSize: number): number {
|
||||
return col;
|
||||
}
|
||||
|
||||
/** Indentation of the nearest non-empty line above `line` (1-based), for pastes onto a blank line in languages without an indent engine. */
|
||||
function previousLineIndent(state: EditorState, line: number): number | null {
|
||||
for (let n = line - 1; n >= 1; n--) {
|
||||
const prev = state.doc.line(n);
|
||||
if (/\S/.test(prev.text)) return leadingColumns(prev.text, state.tabSize);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Reformat on paste › Indent Block: re-indents a pasted block to the caret's indentation. */
|
||||
export function indentPastedText(
|
||||
text: string,
|
||||
@@ -277,7 +286,11 @@ export function pasteReindentFilter(mode: "indentBlock" | "indentEachLine") {
|
||||
let targetCol = leadingColumns(atIndent ? before : line.text, tabSize);
|
||||
let prefix = "";
|
||||
if (atIndent) {
|
||||
const proper = getIndentation(start, line.from);
|
||||
let proper = getIndentation(start, line.from);
|
||||
if (!targetCol && !proper) {
|
||||
const above = previousLineIndent(start, line.number);
|
||||
if (above) proper = above;
|
||||
}
|
||||
if (proper !== null && proper !== targetCol) {
|
||||
targetCol = proper;
|
||||
prefix = indentString(start, proper);
|
||||
|
||||
@@ -15,10 +15,10 @@ function parents(path: string): string[] {
|
||||
* For every path whose file name occurs more than once in `paths`, the
|
||||
* distinguishing directory hint. Paths with a unique name are absent.
|
||||
*/
|
||||
export function disambiguateTitles(paths: readonly string[]): Map<string, string> {
|
||||
export function disambiguateTitles(paths: readonly string[], titleOf = baseName): Map<string, string> {
|
||||
const groups = new Map<string, string[]>();
|
||||
for (const p of new Set(paths)) {
|
||||
const name = baseName(p);
|
||||
const name = titleOf(p);
|
||||
const list = groups.get(name);
|
||||
if (list) list.push(p);
|
||||
else groups.set(name, [p]);
|
||||
|
||||
@@ -161,7 +161,11 @@ function useTitleHints(): Map<string, string> {
|
||||
return out.sort();
|
||||
}),
|
||||
);
|
||||
return useMemo(() => disambiguateTitles(paths), [paths]);
|
||||
const labels = useFileDecorations(useShallow(() => paths.map((p) => getFileDecoration(p)?.tabTitle ?? basename(p))));
|
||||
return useMemo(() => {
|
||||
const byPath = new Map(paths.map((p, i) => [p, labels[i]!]));
|
||||
return disambiguateTitles(paths, (p) => byPath.get(p) ?? basename(p));
|
||||
}, [paths, labels]);
|
||||
}
|
||||
|
||||
const virtualData = (input: EditorInput) => input.data as VirtualInputData | undefined;
|
||||
@@ -180,6 +184,8 @@ function PaneView({ paneId }: { paneId: string }) {
|
||||
const dirty = tabs.map((tab, i) => inputDirty(tab.input, kinds) ?? textDirty[i] ?? false);
|
||||
const colors = useFileDecorations(useShallow(() => paths.map((p) => (p ? (getFileDecoration(p)?.color ?? "") : ""))));
|
||||
const tints = useFileDecorations(useShallow(() => paths.map((p) => (p ? (getFileDecoration(p)?.tint ?? "") : ""))));
|
||||
const tabLabels = useFileDecorations(useShallow(() => paths.map((p) => getFileDecoration(p)?.tabTitle)));
|
||||
const tabIcons = useFileDecorations(useShallow(() => paths.map((p) => getFileDecoration(p)?.tabIcon)));
|
||||
const errors = useFileDecorations(useShallow(() => paths.map((p) => !!p && !!getFileDecoration(p)?.error)));
|
||||
const fileColors = useFileColors("editorTabs");
|
||||
|
||||
@@ -200,7 +206,7 @@ function PaneView({ paneId }: { paneId: string }) {
|
||||
return tabs.map((t, i) => {
|
||||
const full = names[i]!;
|
||||
const dot = full.lastIndexOf(".");
|
||||
const name = t.input.kind === "file" && !showExtension && dot > 0 ? full.slice(0, dot) : full;
|
||||
const name = tabLabels[i] ?? (t.input.kind === "file" && !showExtension && dot > 0 ? full.slice(0, dot) : full);
|
||||
const dupe = !showDirectoryForNonUnique
|
||||
? null
|
||||
: t.input.kind === "file" && t.input.path
|
||||
@@ -223,7 +229,7 @@ function PaneView({ paneId }: { paneId: string }) {
|
||||
text: dupe ? `${name} — ${dupe}` : name,
|
||||
icon: showFileIcon
|
||||
? t.input.kind === "file" && t.input.path
|
||||
? file?.icon
|
||||
? (tabIcons[i] ?? file?.icon)
|
||||
: tabIcon(t.input, kinds)
|
||||
: undefined,
|
||||
italic: t.preview,
|
||||
@@ -255,6 +261,8 @@ function PaneView({ paneId }: { paneId: string }) {
|
||||
colors,
|
||||
errors,
|
||||
tints,
|
||||
tabLabels,
|
||||
tabIcons,
|
||||
fileColors,
|
||||
hints,
|
||||
showExtension,
|
||||
|
||||
@@ -274,6 +274,73 @@ describe("pickAgent", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("pickAgent with an agent chosen for commit messages", () => {
|
||||
const effort: AgentConfigOption = {
|
||||
id: "effort",
|
||||
name: "Effort",
|
||||
description: null,
|
||||
category: "thought_level",
|
||||
current: "high",
|
||||
values: ["low", "high"].map((value) => ({ value, name: value, description: null })),
|
||||
via: "option",
|
||||
};
|
||||
const claude = { ...provider("claude", true, [modelOption("smart")]), thoughtLevels: { fast: [effort] } };
|
||||
const tasks = [
|
||||
task("a", "claude", { updatedMs: 1, config: [modelOption("smart")] }),
|
||||
task("b", "codex", { updatedMs: 5 }),
|
||||
];
|
||||
const prefer = (agent: string, model = "", level = "") => ({ agent, agentModel: model, agentEffort: level });
|
||||
|
||||
it("uses that agent over the latest task's, on the model of its own latest task", () => {
|
||||
expect(pickAgent(tasks, [claude, provider("codex")], draft("codex"), prefer("claude"))).toEqual({
|
||||
provider: "claude",
|
||||
config: [{ id: "model", value: "smart" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("runs the chosen model at the chosen effort", () => {
|
||||
expect(pickAgent(tasks, [claude, provider("codex")], draft("codex"), prefer("claude", "fast", "low"))).toEqual({
|
||||
provider: "claude",
|
||||
config: [
|
||||
{ id: "model", value: "fast" },
|
||||
{ id: "effort", value: "low" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("reads the models from the agent's refreshed catalog", () => {
|
||||
const catalog = {
|
||||
config: [{ ...modelOption("smart"), values: [{ value: "mini", name: "mini", description: null }] }],
|
||||
thoughtLevels: { mini: [effort] },
|
||||
};
|
||||
expect(
|
||||
pickAgent([], [claude], draft("claude"), prefer("claude", "mini", "low"), new Map([["claude", catalog]])),
|
||||
).toEqual({
|
||||
provider: "claude",
|
||||
config: [
|
||||
{ id: "model", value: "mini" },
|
||||
{ id: "effort", value: "low" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the agent's model and effort for values it doesn't offer", () => {
|
||||
expect(pickAgent([], [claude], draft("claude"), prefer("claude", "gone", "low"))).toEqual({
|
||||
provider: "claude",
|
||||
config: [{ id: "model", value: "smart" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("follows the latest task when the chosen agent isn't installed", () => {
|
||||
expect(
|
||||
pickAgent(tasks, [provider("claude", false), provider("codex")], draft("codex"), prefer("claude", "fast")),
|
||||
).toEqual({
|
||||
provider: "codex",
|
||||
config: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateCommitMessage", () => {
|
||||
const changes = { revision: 3, roots: ["/p"], patch: PATCH };
|
||||
const notified = () => backend.notify.mock.calls.map(([n]) => n as { level: string; title: string; body?: string });
|
||||
|
||||
@@ -3,7 +3,14 @@
|
||||
* changes about to be committed and writes their message. It runs as a short task that
|
||||
* is deleted once the message is in the commit field.
|
||||
*/
|
||||
import type { AgentConfigChoice, AgentConfigOption, AgentEntry, AgentProvider, AgentTask } from "@/bindings";
|
||||
import type {
|
||||
AgentConfigChoice,
|
||||
AgentConfigOption,
|
||||
AgentEntry,
|
||||
AgentModelCatalog,
|
||||
AgentProvider,
|
||||
AgentTask,
|
||||
} from "@/bindings";
|
||||
import { executeAction } from "@/platform/actions";
|
||||
import { tx } from "@/platform/i18n";
|
||||
import { call, commands } from "@/platform/ipc";
|
||||
@@ -139,28 +146,74 @@ export function askingChoices(options: readonly AgentConfigOption[]): AgentConfi
|
||||
|
||||
const choicesFor = (options: readonly AgentConfigOption[]) => [...modelChoice(options), ...askingChoices(options)];
|
||||
|
||||
/** Settings › Version Control › Commit: the agent, model and thinking level to generate with; empty follows the agent. */
|
||||
export interface CommitAgentPreference {
|
||||
agent: string;
|
||||
agentModel: string;
|
||||
agentEffort: string;
|
||||
}
|
||||
|
||||
type ModelCatalog = Pick<AgentModelCatalog, "config" | "thoughtLevels">;
|
||||
|
||||
/**
|
||||
* The agent that wrote the code: the one of the project's latest task, on the model it
|
||||
* ran with. Without a task, the one the New Task page has selected.
|
||||
* ran with. Without a task, the one the New Task page has selected. An agent chosen in
|
||||
* the settings replaces it, on the model and thinking level chosen there when it offers them.
|
||||
*/
|
||||
export function pickAgent(
|
||||
tasks: Iterable<AgentTask>,
|
||||
providers: readonly AgentProvider[],
|
||||
draft: Draft,
|
||||
preference?: CommitAgentPreference,
|
||||
catalogs?: ReadonlyMap<string, ModelCatalog>,
|
||||
): CommitAgent | null {
|
||||
const installed = providers.filter((p) => p.installed);
|
||||
const chosen = preference?.agent ? installed.find((p) => p.id === preference.agent) : undefined;
|
||||
const candidates = chosen ? [chosen] : installed;
|
||||
const latest = [...tasks]
|
||||
.filter((t) => t.status !== "archived" && installed.some((p) => p.id === t.provider))
|
||||
.filter((t) => t.status !== "archived" && candidates.some((p) => p.id === t.provider))
|
||||
.sort((a, b) => (b.updatedMs ?? 0) - (a.updatedMs ?? 0))[0];
|
||||
if (latest) return { provider: latest.provider, config: choicesFor(latest.config) };
|
||||
const provider = installed.find((p) => p.id === draft.provider) ?? installed[0];
|
||||
const provider = latest
|
||||
? installed.find((p) => p.id === latest.provider)!
|
||||
: (chosen ?? installed.find((p) => p.id === draft.provider) ?? installed[0]);
|
||||
if (!provider) return null;
|
||||
const options = latest
|
||||
? latest.config
|
||||
: provider.id === draft.provider
|
||||
? draftConfig(provider, draft)
|
||||
: provider.config;
|
||||
if (!chosen || !preference || (!preference.agentModel && !preference.agentEffort)) {
|
||||
return { provider: provider.id, config: choicesFor(options) };
|
||||
}
|
||||
return {
|
||||
provider: provider.id,
|
||||
config: choicesFor(provider.id === draft.provider ? draftConfig(provider, draft) : provider.config),
|
||||
config: [...preferredModel(options, provider, preference, catalogs?.get(provider.id)), ...askingChoices(options)],
|
||||
};
|
||||
}
|
||||
|
||||
/** The model and thinking level of `preference` that the agent offers, else the ones it would run with. */
|
||||
function preferredModel(
|
||||
options: readonly AgentConfigOption[],
|
||||
provider: AgentProvider,
|
||||
preference: CommitAgentPreference,
|
||||
catalog: ModelCatalog | undefined,
|
||||
): AgentConfigChoice[] {
|
||||
const offered = (catalog?.config ?? provider.config).find((o) => o.category === "model");
|
||||
const fallback = modelChoice(options);
|
||||
const model =
|
||||
offered && preference.agentModel && offered.values.some((v) => v.value === preference.agentModel)
|
||||
? { id: offered.id, value: preference.agentModel }
|
||||
: fallback[0];
|
||||
if (!model) return [];
|
||||
const levels = (catalog?.thoughtLevels ?? provider.thoughtLevels)?.[model.value] ?? [];
|
||||
const level = levels.find((o) => o.category === "thought_level");
|
||||
const effort =
|
||||
level && preference.agentEffort && level.values.some((v) => v.value === preference.agentEffort)
|
||||
? [{ id: level.id, value: preference.agentEffort }]
|
||||
: [];
|
||||
return [model, ...effort];
|
||||
}
|
||||
|
||||
/** Resolves when the task's turn ends; rejects when it can't produce an answer. */
|
||||
function turnEnd(task: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -243,8 +296,9 @@ async function generate(): Promise<void> {
|
||||
notify({ level: "info", title: "No changes to describe", body: "Include changes in the commit first." });
|
||||
return;
|
||||
}
|
||||
const { tasks, providers, draft } = useAgents.getState();
|
||||
const agent = pickAgent(tasks.values(), providers, draft);
|
||||
const { commit } = workflowSettings().vcs;
|
||||
const { tasks, providers, draft, modelCatalogs } = useAgents.getState();
|
||||
const agent = pickAgent(tasks.values(), providers, draft, commit, modelCatalogs);
|
||||
if (!agent) {
|
||||
notify({
|
||||
level: "warning",
|
||||
@@ -255,7 +309,6 @@ async function generate(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
const { commit } = workflowSettings().vcs;
|
||||
const subjects = await recentSubjects(changes.roots[0]);
|
||||
const prompt = commitMessagePrompt(
|
||||
changes.patch,
|
||||
|
||||
@@ -173,6 +173,8 @@ export function SessionComposer({
|
||||
return task.config.map((o) => (o === model ? pickerModel : o));
|
||||
}, [task.config, model, candidateModels]);
|
||||
const [sending, setSending] = useState(false);
|
||||
const configChange = useRef<{ loadingModel: boolean } | null>(null);
|
||||
const [configuring, setConfiguring] = useState<typeof configChange.current>(null);
|
||||
const [mcpOpenRequest, setMcpOpenRequest] = useState(0);
|
||||
const [assetFocus, setAssetFocus] = useState(0);
|
||||
const agentCommands = useAgentCommands(task.provider, task.id, task.commands);
|
||||
@@ -191,7 +193,7 @@ export function SessionComposer({
|
||||
task.status === "running" &&
|
||||
(composer.images.length === 0 || !!task.capabilities.images);
|
||||
const send = (now: boolean) => {
|
||||
if (!canSend(composer) || sending || task.status === "archived") return;
|
||||
if (!canSend(composer) || sending || configChange.current || task.status === "archived") return;
|
||||
setSending(true);
|
||||
sendMessage(task.id, now && (whileRunning !== "steer" || steers))
|
||||
.catch((e: unknown) =>
|
||||
@@ -203,14 +205,30 @@ export function SessionComposer({
|
||||
)
|
||||
.finally(() => setSending(false));
|
||||
};
|
||||
const setOption = (id: string, value: string) =>
|
||||
void setTaskConfig(task.id, id, value).catch((e: unknown) =>
|
||||
notifications().notify({
|
||||
level: "error",
|
||||
title: "Couldn't change the setting",
|
||||
body: String(e),
|
||||
}),
|
||||
);
|
||||
const setOption = (id: string, value: string) => {
|
||||
const changingModel = id === model?.id;
|
||||
const loadingModel = changingModel && !model.values.some((v) => v.value === value);
|
||||
const pendingModel = model?.current != null && model.current !== task.model;
|
||||
const change = loadingModel || (changingModel && configChange.current) || pendingModel ? { loadingModel } : null;
|
||||
if (change) {
|
||||
configChange.current = change;
|
||||
setConfiguring(change);
|
||||
}
|
||||
void setTaskConfig(task.id, id, value)
|
||||
.catch((e: unknown) =>
|
||||
notifications().notify({
|
||||
level: "error",
|
||||
title: "Couldn't change the setting",
|
||||
body: String(e),
|
||||
}),
|
||||
)
|
||||
.finally(() => {
|
||||
if (change && configChange.current === change) {
|
||||
configChange.current = null;
|
||||
setConfiguring(null);
|
||||
}
|
||||
});
|
||||
};
|
||||
const openMcp = () => setMcpOpenRequest((n) => n + 1);
|
||||
const shown = useEditorShown();
|
||||
const effortAnnouncement = useEffortFocus(focusRoot, shown && task.status !== "archived", task.config, setOption);
|
||||
@@ -269,6 +287,7 @@ export function SessionComposer({
|
||||
canRefreshModels
|
||||
? {
|
||||
refreshing: requestingModels || (catalog?.refreshing ?? false),
|
||||
loadingSettings: configuring?.loadingModel,
|
||||
retryAtMs: catalog?.retryAtMs ?? null,
|
||||
error: requestingModels ? null : (catalog?.error ?? null),
|
||||
onOpen: () => void refreshTaskModels(task.id, false),
|
||||
@@ -303,7 +322,7 @@ export function SessionComposer({
|
||||
label={busy ? "Queue" : "Send"}
|
||||
pending={sending}
|
||||
variant={busy ? "secondary" : "primary"}
|
||||
disabled={!canSend(composer) || sending || task.status === "archived"}
|
||||
disabled={!canSend(composer) || sending || configuring !== null || task.status === "archived"}
|
||||
onClick={() => send(false)}
|
||||
/>
|
||||
</>
|
||||
|
||||
@@ -8,6 +8,7 @@ const workspaceUpdate = vi.fn();
|
||||
const projectCurrent = vi.fn();
|
||||
const windowOpenProject = vi.fn();
|
||||
const windowOpenFiles = vi.fn();
|
||||
const windowGroupJoin = vi.fn();
|
||||
const windowTakeLaunch = vi.fn();
|
||||
const windowCompleteLaunch = vi.fn();
|
||||
const wslResolve = vi.fn();
|
||||
@@ -44,6 +45,7 @@ vi.mock("@/platform/ipc", async (orig) => {
|
||||
projectCurrent: () => projectCurrent(),
|
||||
windowOpenProject: (...a: unknown[]) => windowOpenProject(...a),
|
||||
windowOpenFiles: (...a: unknown[]) => windowOpenFiles(...a),
|
||||
windowGroupJoin: (...a: unknown[]) => windowGroupJoin(...a),
|
||||
windowTakeLaunch: () => windowTakeLaunch(),
|
||||
windowCompleteLaunch: () => windowCompleteLaunch(),
|
||||
wslResolve: (p: string) => wslResolve(p),
|
||||
@@ -111,7 +113,8 @@ beforeEach(() => {
|
||||
workspaceAttach.mockReset();
|
||||
workspaceUpdate.mockReset();
|
||||
projectCurrent.mockReset();
|
||||
for (const f of [windowOpenProject, windowOpenFiles, windowTakeLaunch, wslResolve]) f.mockReset();
|
||||
for (const f of [windowOpenProject, windowOpenFiles, windowGroupJoin, windowTakeLaunch, wslResolve]) f.mockReset();
|
||||
windowGroupJoin.mockResolvedValue(ok(null));
|
||||
windowTakeLaunch.mockResolvedValue(null);
|
||||
windowCompleteLaunch.mockReset().mockResolvedValue(ok(null));
|
||||
wslResolve.mockResolvedValue(null);
|
||||
@@ -694,6 +697,37 @@ describe("Attach", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("Project tabs", () => {
|
||||
it("opens the project in its own window and joins that window into this window's group", async () => {
|
||||
useProject.setState(projectState("/p/a"));
|
||||
vi.mocked(ask).mockResolvedValueOnce({ button: "tab", checked: false });
|
||||
windowOpenProject.mockResolvedValue(ok({ kind: "opened", label: "main-2" }));
|
||||
await openProjectWithPolicy("/p/b", ["/p/b/file.txt"]);
|
||||
expect(confirmProjectTrust).toHaveBeenCalledWith("/p/b", "local");
|
||||
expect(windowOpenProject).toHaveBeenCalledWith("/p/b", true, ["/p/b/file.txt"]);
|
||||
expect(windowGroupJoin).toHaveBeenCalledWith("main-2");
|
||||
// The new window opens its own files, and this window keeps its project.
|
||||
expect(opened).toEqual([]);
|
||||
expect(openProject).not.toHaveBeenCalled();
|
||||
expect(confirmLeaveProject).not.toHaveBeenCalled();
|
||||
});
|
||||
it("groups the window that already has the project", async () => {
|
||||
useProject.setState(projectState("/p/a"));
|
||||
vi.mocked(ask).mockResolvedValueOnce({ button: "tab", checked: false });
|
||||
windowOpenProject.mockResolvedValue(ok({ kind: "focused", label: "main-3" }));
|
||||
await openProjectWithPolicy("/p/b");
|
||||
expect(windowGroupJoin).toHaveBeenCalledWith("main-3");
|
||||
});
|
||||
it("reports a window it could not group", async () => {
|
||||
useProject.setState(projectState("/p/a"));
|
||||
vi.mocked(ask).mockResolvedValueOnce({ button: "tab", checked: false });
|
||||
windowOpenProject.mockResolvedValue(ok({ kind: "opened", label: "main-2" }));
|
||||
windowGroupJoin.mockRejectedValueOnce(new Error("Window closed"));
|
||||
await openProjectWithPolicy("/p/b");
|
||||
expect(notify).toHaveBeenCalledWith(expect.objectContaining({ level: "error", body: "Window closed" }));
|
||||
});
|
||||
});
|
||||
|
||||
it("removes the first workspace member while retaining the workspace state root", async () => {
|
||||
const project = {
|
||||
root: "/workspace",
|
||||
|
||||
@@ -26,6 +26,7 @@ import { connectRemote, isRemoteWindow } from "@/platform/remote";
|
||||
import { holdSplash } from "@/platform/splash";
|
||||
import { editorService, maybeEditorService, notifications } from "@/workbench/contributions";
|
||||
import { chooseProjectWindow } from "@/workbench/projectWindow";
|
||||
import { openTabColorPicker } from "@/workbench/TabColorPicker";
|
||||
import { cycleWindowTab, useWindowGroup } from "@/platform/windowGroups";
|
||||
import { editorSwitching } from "@/editor/api";
|
||||
import { showWorkspaceDialog } from "./WorkspaceDialog";
|
||||
@@ -185,6 +186,10 @@ export async function openProjectWithPolicy(path: string, files: readonly string
|
||||
await attachProject(path, files);
|
||||
return;
|
||||
}
|
||||
if (mode === "tab") {
|
||||
await openProjectTab(path, files);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// A window that already has the project is focused instead.
|
||||
const outcome = await call(commands.windowOpenProject(path, false, [...files]));
|
||||
@@ -215,6 +220,19 @@ async function attachProject(path: string, files: readonly string[] = []): Promi
|
||||
}
|
||||
}
|
||||
|
||||
/** Opens `path` as a new project tab of this window: a window of its own, joined into this window's group. */
|
||||
async function openProjectTab(path: string, files: readonly string[]): Promise<void> {
|
||||
try {
|
||||
// `new_window` always answers with the window holding the project, focused or just created;
|
||||
// either way that window becomes this window's tab.
|
||||
const outcome = await openProjectWindow(path, true, files);
|
||||
if (outcome.kind === "current") return;
|
||||
await call(commands.windowGroupJoin(outcome.label));
|
||||
} catch (e) {
|
||||
openFailed(path, e);
|
||||
}
|
||||
}
|
||||
|
||||
/** LightEdit: opens the current file's folder as a project ("Open project in" applies) and the file there. */
|
||||
export async function openLightEditFileAsProject(): Promise<void> {
|
||||
if (!useProject.getState().lightEdit) return;
|
||||
@@ -418,6 +436,25 @@ export function windowActions(): Action[] {
|
||||
if (label) await call(commands.windowGroupDetach(label));
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "window.setTabColor",
|
||||
title: "Change Project Color…",
|
||||
category: t("app.category.window"),
|
||||
when: "windowGrouped",
|
||||
run: ({ arg }) => {
|
||||
const tabs = useWindowGroup.getState().tabs;
|
||||
const label = typeof arg === "string" ? arg : tabs.find((tab) => tab.current)?.label;
|
||||
const tab = tabs.find((t) => t.label === label);
|
||||
if (!tab?.root) return;
|
||||
const rect = document.querySelector(`[data-project-tab="${CSS.escape(tab.label)}"]`)?.getBoundingClientRect();
|
||||
openTabColorPicker({
|
||||
root: tab.root,
|
||||
name: tab.name,
|
||||
x: rect ? rect.left : window.innerWidth / 2 - 110,
|
||||
y: rect ? rect.bottom : 64,
|
||||
});
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "window.selectTab",
|
||||
title: "Select Project Tab",
|
||||
|
||||
@@ -97,4 +97,8 @@ export const buildToolsRunAnything: RunAnythingProvider = {
|
||||
const tool = lineTool(line);
|
||||
return !!tool && executeBuildLine(tool, line);
|
||||
},
|
||||
accepts(line) {
|
||||
const tool = lineTool(line);
|
||||
return !!tool && (tool === "maven" ? parseGoalLine(line) : parseTaskLine(line)).length > 0;
|
||||
},
|
||||
};
|
||||
@@ -76,8 +76,6 @@
|
||||
overflow: auto;
|
||||
position: relative;
|
||||
padding: 4px 0;
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--fs-code);
|
||||
color: var(--fg);
|
||||
background: var(--bg-app);
|
||||
-webkit-user-select: text;
|
||||
@@ -99,7 +97,6 @@
|
||||
left: 0;
|
||||
min-width: 100%;
|
||||
padding: 0 12px;
|
||||
line-height: 20px;
|
||||
white-space: pre;
|
||||
tab-size: 4;
|
||||
}
|
||||
@@ -141,9 +138,6 @@
|
||||
.detail {
|
||||
overflow: auto;
|
||||
padding: 4px 12px;
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--fs-code);
|
||||
line-height: 20px;
|
||||
color: var(--fg);
|
||||
background: var(--bg-app);
|
||||
-webkit-user-select: text;
|
||||
|
||||
@@ -5,7 +5,16 @@
|
||||
* its source (as IDEA: selection shows, navigation jumps).
|
||||
*/
|
||||
import { copySelectionOnKeyDown } from "@/ui/selectableText";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState, type MouseEvent, type ReactNode } from "react";
|
||||
import {
|
||||
useCallback,
|
||||
useEffect,
|
||||
useLayoutEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
type MouseEvent,
|
||||
type ReactNode,
|
||||
} from "react";
|
||||
import clsx from "clsx";
|
||||
import { useVirtualizer } from "@tanstack/react-virtual";
|
||||
import { basename, relativeDir, useProject } from "@/platform/project";
|
||||
@@ -41,6 +50,7 @@ import { openTarget, openUrl } from "@/features/run/navigate";
|
||||
import { navigateToMessage, rerunBuild, selectBuildNode, useBuildRun } from "./store";
|
||||
import styles from "./Build.module.css";
|
||||
import { WidestColumns } from "@/platform/consoleColumns";
|
||||
import { useConsoleFont } from "@/platform/consoleFont";
|
||||
import { tx } from "@/platform/i18n";
|
||||
|
||||
const SEVERITY_ICON: Record<Severity, string> = { error: "error", warning: "warning", info: "info", hint: "info" };
|
||||
@@ -203,8 +213,6 @@ function BuildTree() {
|
||||
);
|
||||
}
|
||||
|
||||
const LINE_HEIGHT = 20;
|
||||
|
||||
/**
|
||||
* One console line: file locations and URLs (compiler diagnostics, stack
|
||||
* frames) are links, resolved against the directory of the step that printed
|
||||
@@ -242,7 +250,7 @@ function ConsoleLineText({ line, root }: { line: ConsoleLine; root: string }) {
|
||||
return <>{out}</>;
|
||||
}
|
||||
|
||||
function BuildConsole() {
|
||||
function BuildConsole({ rowHeight }: { rowHeight: number }) {
|
||||
const root = useProject((s) => s.project?.root ?? "/");
|
||||
const lines = useBuildRun((s) => s.lines);
|
||||
const ref = useRef<HTMLDivElement>(null);
|
||||
@@ -252,12 +260,15 @@ function BuildConsole() {
|
||||
const virtualizer = useVirtualizer({
|
||||
count: lines.length,
|
||||
getScrollElement: () => ref.current,
|
||||
estimateSize: () => LINE_HEIGHT,
|
||||
estimateSize: () => rowHeight,
|
||||
overscan: 20,
|
||||
});
|
||||
useLayoutEffect(() => {
|
||||
virtualizer.measure();
|
||||
}, [rowHeight, virtualizer]);
|
||||
useEffect(() => {
|
||||
if (stick.current && lines.length) virtualizer.scrollToIndex(lines.length - 1, { align: "end" });
|
||||
}, [lines.length, virtualizer]);
|
||||
}, [lines.length, rowHeight, virtualizer]);
|
||||
return (
|
||||
<div
|
||||
ref={ref}
|
||||
@@ -268,7 +279,7 @@ function BuildConsole() {
|
||||
aria-label={tx("Build output")}
|
||||
onScroll={(e) => {
|
||||
const el = e.currentTarget;
|
||||
stick.current = el.scrollHeight - el.scrollTop - el.clientHeight < LINE_HEIGHT * 2;
|
||||
stick.current = el.scrollHeight - el.scrollTop - el.clientHeight < rowHeight * 2;
|
||||
}}
|
||||
>
|
||||
<div
|
||||
@@ -319,15 +330,16 @@ function MessageDetail({ message }: { message: BuildMessage }) {
|
||||
);
|
||||
}
|
||||
|
||||
function BuildOutput() {
|
||||
function BuildOutput({ rowHeight }: { rowHeight: number }) {
|
||||
const selected = useBuildRun((s) => s.selected);
|
||||
const messages = useBuildRun((s) => s.messages);
|
||||
const node = useMemo(() => (selected ? findNode(messages, selected) : undefined), [selected, messages]);
|
||||
return node?.kind === "message" ? <MessageDetail message={node.message} /> : <BuildConsole />;
|
||||
return node?.kind === "message" ? <MessageDetail message={node.message} /> : <BuildConsole rowHeight={rowHeight} />;
|
||||
}
|
||||
|
||||
export function BuildView() {
|
||||
const hasBuild = useBuildRun((s) => s.status !== null);
|
||||
const font = useConsoleFont();
|
||||
if (!hasBuild) {
|
||||
return (
|
||||
<div className={styles.view}>
|
||||
@@ -345,12 +357,22 @@ export function BuildView() {
|
||||
id="build-view-split"
|
||||
storageKey="openidea.build.split"
|
||||
>
|
||||
<SplitPanel id="build-tree" defaultSize="38%" minSize="120px" className={styles.pane}>
|
||||
<SplitPanel id="build-tree" defaultSize="33%" minSize="120px" className={styles.pane}>
|
||||
<BuildTree />
|
||||
</SplitPanel>
|
||||
<SplitHandle />
|
||||
<SplitPanel id="build-console" minSize="120px" className={styles.pane}>
|
||||
<BuildOutput />
|
||||
<SplitPanel
|
||||
id="build-console"
|
||||
minSize="120px"
|
||||
className={styles.pane}
|
||||
style={{
|
||||
fontFamily: font.family,
|
||||
fontSize: font.size,
|
||||
fontWeight: font.weight,
|
||||
lineHeight: `${font.rowHeight}px`,
|
||||
}}
|
||||
>
|
||||
<BuildOutput rowHeight={font.rowHeight} />
|
||||
</SplitPanel>
|
||||
</SplitGroup>
|
||||
</div>
|
||||
|
||||
@@ -14,6 +14,24 @@
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
}
|
||||
.results {
|
||||
display: flex;
|
||||
min-height: 0;
|
||||
height: 100%;
|
||||
}
|
||||
.toolbar {
|
||||
flex: none;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-1);
|
||||
padding: var(--space-1);
|
||||
border-inline-end: 1px solid var(--border);
|
||||
overflow-y: auto;
|
||||
}
|
||||
.tree {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
.tabs {
|
||||
min-width: 0;
|
||||
border: 0;
|
||||
@@ -38,10 +56,21 @@
|
||||
flex: none;
|
||||
color: var(--fg-muted);
|
||||
}
|
||||
.fileLocation {
|
||||
display: flex;
|
||||
gap: var(--space-1-5);
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
.fileName {
|
||||
flex: none;
|
||||
max-inline-size: 100%;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
color: var(--fg-strong);
|
||||
}
|
||||
.fileError {
|
||||
padding-block: var(--space-0-5);
|
||||
text-decoration-line: underline;
|
||||
text-decoration-style: wavy;
|
||||
text-decoration-color: var(--danger);
|
||||
@@ -51,18 +80,25 @@
|
||||
.selectedText {
|
||||
color: var(--fg-strong);
|
||||
}
|
||||
.muted {
|
||||
flex: none;
|
||||
.directory {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
color: var(--fg-subtle);
|
||||
}
|
||||
.count {
|
||||
flex: none;
|
||||
max-inline-size: 50%;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
color: var(--fg-subtle);
|
||||
margin-left: 4px;
|
||||
}
|
||||
.source {
|
||||
flex: none;
|
||||
margin-left: auto;
|
||||
padding-right: 8px;
|
||||
min-width: 0;
|
||||
max-inline-size: 25%;
|
||||
margin-inline-start: auto;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
color: var(--fg-faint);
|
||||
}
|
||||
.message {
|
||||
@@ -77,6 +113,7 @@
|
||||
.position {
|
||||
flex: none;
|
||||
color: var(--fg-subtle);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.sev-error {
|
||||
flex: none;
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
SplitHandle,
|
||||
SplitPanel,
|
||||
Tabs,
|
||||
Toolbar,
|
||||
Tree,
|
||||
fileIconName,
|
||||
showContextMenu,
|
||||
@@ -213,11 +214,11 @@ function useCurrentPath(): string | null {
|
||||
return currentProblemsPath(text, useSyncExternalStore(subscribeActiveInput, activeInput));
|
||||
}
|
||||
|
||||
export function ProblemsHeaderActions() {
|
||||
function ProblemsToolbar() {
|
||||
const show = useProblemsView((s) => s.show);
|
||||
const toggle = useProblemsView((s) => s.toggle);
|
||||
return (
|
||||
<>
|
||||
<Toolbar aria-label="Problems actions" orientation="vertical" className={styles.toolbar}>
|
||||
<IconButton icon="arrow-up" action="navigate.prevOccurrence" size="small" />
|
||||
<IconButton icon="arrow-down" action="navigate.nextOccurrence" size="small" />
|
||||
<IconButton icon="error" title="Show Errors" size="small" pressed={show.error} onClick={() => toggle("error")} />
|
||||
@@ -235,7 +236,7 @@ export function ProblemsHeaderActions() {
|
||||
pressed={show.info}
|
||||
onClick={() => toggle("info")}
|
||||
/>
|
||||
</>
|
||||
</Toolbar>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -337,33 +338,44 @@ export function ProblemsView() {
|
||||
|
||||
const renderNode = (n: Node, state: TreeRowState) => {
|
||||
if (n.kind === "file") {
|
||||
const name = basename(n.path);
|
||||
const directory = relativeDir(root, n.path);
|
||||
const c = countSeverities(n.items);
|
||||
const parts = [plural(c.error, "error"), plural(c.warning, "warning"), plural(c.info + c.hint, "info")].filter(
|
||||
Boolean,
|
||||
);
|
||||
const counts = [plural(c.error, "error"), plural(c.warning, "warning"), plural(c.info + c.hint, "info")]
|
||||
.filter(Boolean)
|
||||
.join(", ");
|
||||
return (
|
||||
<span className={styles.row}>
|
||||
<Icon name={fileIconName(basename(n.path), n.path)} size={16} className={styles.fileIcon} />
|
||||
<span
|
||||
className={clsx(styles.fileName, c.error > 0 && styles.fileError, state.selected && styles.selectedText)}
|
||||
>
|
||||
{basename(n.path)}
|
||||
<Icon name={fileIconName(name, n.path)} size={16} className={styles.fileIcon} />
|
||||
<span className={styles.fileLocation}>
|
||||
<span
|
||||
className={clsx(styles.fileName, c.error > 0 && styles.fileError, state.selected && styles.selectedText)}
|
||||
title={name}
|
||||
>
|
||||
{name}
|
||||
</span>
|
||||
{directory && (
|
||||
<span className={styles.directory} title={directory}>
|
||||
{directory}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
<span className={styles.count} title={counts}>
|
||||
{counts}
|
||||
</span>
|
||||
<span className={styles.muted}>{relativeDir(root, n.path)}</span>
|
||||
<span className={styles.count}>{parts.join(", ")}</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
const d = n.d;
|
||||
const source = d.source && (d.code ? `${d.source}(${d.code})` : d.source);
|
||||
return (
|
||||
<span className={styles.row} title={d.message}>
|
||||
<Icon name={SEVERITY_ICON[d.severity]} size={14} className={styles[`sev-${d.severity}`]} />
|
||||
<span className={clsx(styles.message, d.deprecated && styles.deprecated)}>{d.message.split("\n")[0]}</span>
|
||||
<span className={styles.position}>:{d.range.start.line + 1}</span>
|
||||
{d.source && (
|
||||
<span className={styles.source}>
|
||||
{d.source}
|
||||
{d.code ? `(${d.code})` : ""}
|
||||
{source && (
|
||||
<span className={styles.source} title={source}>
|
||||
{source}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
@@ -378,9 +390,10 @@ export function ProblemsView() {
|
||||
);
|
||||
|
||||
const selected = target(lead);
|
||||
const list = (
|
||||
const treeView = (
|
||||
<Tree<Node>
|
||||
aria-label="Problems"
|
||||
className={styles.tree}
|
||||
handleRef={tree}
|
||||
roots={roots}
|
||||
getKey={(n) => n.key}
|
||||
@@ -428,7 +441,11 @@ export function ProblemsView() {
|
||||
run: () => copy(n),
|
||||
},
|
||||
...inspectionMenu(n.kind === "diag" ? diagInspection(map, n.path, n.d) : null),
|
||||
...menuGroupItems("problems", { paths: [t.path], line: t.d.range.start.line, column: t.d.range.start.col }),
|
||||
...menuGroupItems("problems", {
|
||||
paths: [t.path],
|
||||
line: t.d.range.start.line,
|
||||
column: t.d.range.start.col,
|
||||
}),
|
||||
]);
|
||||
}}
|
||||
activateExpandable={false}
|
||||
@@ -436,6 +453,12 @@ export function ProblemsView() {
|
||||
empty={empty}
|
||||
/>
|
||||
);
|
||||
const list = (
|
||||
<div className={styles.results}>
|
||||
<ProblemsToolbar />
|
||||
{treeView}
|
||||
</div>
|
||||
);
|
||||
|
||||
return (
|
||||
<div className={styles.view}>
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
import { getContext } from "@/platform/context";
|
||||
import { clearProblems, installProblemsEvents, problemsReport, useProblems } from "./problemsStore";
|
||||
import { installProblemDecorations } from "./decorations";
|
||||
import { ProblemsHeaderActions, ProblemsTabs, ProblemsView, useProblemsHeaderMenu } from "./ProblemsView";
|
||||
import { ProblemsTabs, ProblemsView, useProblemsHeaderMenu } from "./ProblemsView";
|
||||
import { UsagesTabs, UsagesView, useUsagesHeaderMenu } from "./UsagesView";
|
||||
import { UsagesPopup } from "./UsagesPopup";
|
||||
import { findUsages, rerunActiveUsages, resetUsages, showUsageResults, showUsagesPopup } from "./usages";
|
||||
@@ -79,7 +79,6 @@ export function activate(): () => void {
|
||||
order: 30,
|
||||
component: ProblemsView,
|
||||
headerContent: ProblemsTabs,
|
||||
headerActions: ProblemsHeaderActions,
|
||||
headerMenu: useProblemsHeaderMenu,
|
||||
useBadge: useErrorBadge,
|
||||
}),
|
||||
|
||||
@@ -210,7 +210,14 @@ export function ImportDialog({ id, schema, table }: { id: string; schema: string
|
||||
<T>File:</T>
|
||||
</span>
|
||||
<div className={styles.formRow}>
|
||||
<Input readOnly aria-label="File" value={file ?? ""} placeholder="Choose a CSV or TSV file" />
|
||||
<Input
|
||||
className={styles.fill}
|
||||
readOnly
|
||||
aria-label="File"
|
||||
value={file ?? ""}
|
||||
placeholder="Choose a CSV or TSV file"
|
||||
title={file ?? undefined}
|
||||
/>
|
||||
<IconButton icon="folder" title="Choose File…" onClick={() => void choose()} />
|
||||
</div>
|
||||
<span className={styles.formLabel}>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* Record / Value / Aggregates side panel following the selection.
|
||||
*/
|
||||
import { writeClipboardText } from "@/platform/clipboard";
|
||||
import { useMemo, useRef, useState, type KeyboardEvent, type RefObject } from "react";
|
||||
import { useMemo, useRef, useState, type KeyboardEvent, type ReactNode, type RefObject } from "react";
|
||||
import clsx from "clsx";
|
||||
import { Button, Checkbox, Icon, IconButton, Input, Popup, SearchField, Select, Tabs, T, type MenuItem } from "@/ui";
|
||||
import { EXPORT_FORMATS, exportData, type ExportData, type ExportFormat } from "./export";
|
||||
@@ -99,6 +99,7 @@ export interface DataViewProps extends Pick<
|
||||
/** Rows from here on (the table editor's added rows) are never filtered or sorted. */
|
||||
fixedRows?: number;
|
||||
firstRowNumber?: number;
|
||||
pager?: ReactNode;
|
||||
/** The side panel shown (`null`: hidden). */
|
||||
panel: DataPanel | null;
|
||||
onPanelChange: (panel: DataPanel | null) => void;
|
||||
@@ -247,6 +248,7 @@ export function DataView(props: DataViewProps) {
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{props.pager && <div className={styles.pagerDock}>{props.pager}</div>}
|
||||
</div>
|
||||
{filterAt && (
|
||||
<FilterPopup
|
||||
|
||||
@@ -73,10 +73,11 @@
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
padding: 0 6px;
|
||||
padding: 0 var(--space-1);
|
||||
color: var(--fg-line-number);
|
||||
background: var(--bg-editor-gutter);
|
||||
border-right: 1px solid var(--border);
|
||||
border-bottom: 1px solid var(--border);
|
||||
user-select: none;
|
||||
}
|
||||
.cell {
|
||||
@@ -139,9 +140,10 @@
|
||||
.toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-0-5);
|
||||
height: var(--h-header);
|
||||
padding: 0 6px;
|
||||
min-height: 32px;
|
||||
padding: var(--space-0-5) var(--space-2);
|
||||
flex: none;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
@@ -233,14 +235,16 @@
|
||||
flex-direction: column;
|
||||
}
|
||||
.dbToolbar {
|
||||
height: 32px;
|
||||
min-height: 32px;
|
||||
flex: none;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: var(--space-0-5);
|
||||
padding: 0 6px;
|
||||
padding: var(--space-0-5) var(--space-2);
|
||||
}
|
||||
.dbToolbarSep {
|
||||
flex: none;
|
||||
width: 1px;
|
||||
height: 16px;
|
||||
margin: 0 4px;
|
||||
@@ -286,15 +290,22 @@
|
||||
.consoleBar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-1);
|
||||
height: var(--h-header);
|
||||
padding: 0 8px;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-0-5);
|
||||
min-height: 32px;
|
||||
padding: var(--space-0-5) var(--space-2);
|
||||
background: var(--bg-editor);
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-family: var(--font-ui);
|
||||
font-size: var(--fs-ui);
|
||||
color: var(--fg-secondary);
|
||||
}
|
||||
.consoleSchema {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: var(--space-1);
|
||||
padding-inline: var(--space-2);
|
||||
}
|
||||
|
||||
/* Inline statement errors in consoles. */
|
||||
:global(.cm-openidea-db-error) {
|
||||
@@ -307,9 +318,9 @@
|
||||
/* Table editor. */
|
||||
/* The table editor's WHERE | ORDER BY row (IDEA). */
|
||||
.filterBar {
|
||||
display: flex;
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 2fr) minmax(0, 3fr);
|
||||
flex: none;
|
||||
height: var(--h-header);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.filterField {
|
||||
@@ -318,29 +329,26 @@
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-1-5);
|
||||
padding: 0 8px;
|
||||
padding: 0 var(--space-2);
|
||||
color: var(--fg-muted);
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--fs-small);
|
||||
white-space: nowrap;
|
||||
}
|
||||
.filterField + .filterField {
|
||||
border-left: 1px solid var(--border);
|
||||
}
|
||||
.filterField > input {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
.filterField:focus-within {
|
||||
box-shadow: inset 0 -1px var(--focus-ring);
|
||||
}
|
||||
.pager {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-0-5);
|
||||
color: var(--fg-secondary);
|
||||
font-size: var(--fs-small);
|
||||
white-space: nowrap;
|
||||
.filterField:has(input[aria-invalid="true"]) {
|
||||
box-shadow: inset 0 -1px var(--danger);
|
||||
}
|
||||
.notice {
|
||||
padding: 4px 8px;
|
||||
padding: var(--space-1) var(--space-2);
|
||||
font-size: var(--fs-small);
|
||||
color: var(--fg-secondary);
|
||||
background: var(--bg-editor-banner);
|
||||
background: var(--bg-elev);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
@@ -427,6 +435,28 @@
|
||||
display: none;
|
||||
}
|
||||
/* Data source dialog. */
|
||||
.sourceLayout {
|
||||
display: grid;
|
||||
grid-template-rows: auto auto minmax(0, 1fr) auto;
|
||||
height: clamp(220px, calc(100dvh - 150px), 560px);
|
||||
}
|
||||
.sourceLayout .form {
|
||||
grid-template-columns: minmax(112px, 19%) minmax(0, 1fr);
|
||||
align-content: start;
|
||||
padding: var(--space-3) var(--space-4);
|
||||
}
|
||||
.sourceLayout .formLabel {
|
||||
text-align: left;
|
||||
}
|
||||
.sourceLayout .formRow {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.sourceLayout .formRow > .fill {
|
||||
flex: 1 1 160px;
|
||||
}
|
||||
.sourcePanel {
|
||||
overflow: auto;
|
||||
}
|
||||
.formSection {
|
||||
grid-column: 1 / -1;
|
||||
display: flex;
|
||||
@@ -549,10 +579,16 @@
|
||||
font-size: var(--fs-small);
|
||||
}
|
||||
.testResult {
|
||||
grid-column: 1 / -1;
|
||||
font-size: var(--fs-small);
|
||||
min-height: 16px;
|
||||
word-break: break-word;
|
||||
max-height: 25vh;
|
||||
padding: var(--space-2) var(--space-4);
|
||||
overflow: auto;
|
||||
overflow-wrap: anywhere;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
.testResult:empty {
|
||||
padding: 0;
|
||||
border: 0;
|
||||
}
|
||||
.ok {
|
||||
color: var(--success);
|
||||
@@ -737,17 +773,28 @@
|
||||
.modifyBody {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
padding: 14px;
|
||||
gap: var(--space-2);
|
||||
height: clamp(320px, 68vh, 560px);
|
||||
padding: var(--space-4);
|
||||
}
|
||||
.modifyMain {
|
||||
flex: 1;
|
||||
display: grid;
|
||||
grid-template-columns: 260px 1fr;
|
||||
height: 240px;
|
||||
grid-template-columns: minmax(108px, 34%) minmax(0, 1fr);
|
||||
min-height: 0;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-small);
|
||||
overflow: hidden;
|
||||
}
|
||||
.modifyBody > .previewBlock {
|
||||
flex: 0 0 38%;
|
||||
min-height: 0;
|
||||
}
|
||||
.modifyBody .sqlPreview {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
max-height: none;
|
||||
}
|
||||
.modifyColumns {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -786,13 +833,17 @@
|
||||
}
|
||||
.modifyProps {
|
||||
display: grid;
|
||||
grid-template-columns: max-content 1fr;
|
||||
grid-template-columns: minmax(0, 27%) minmax(0, 1fr);
|
||||
align-content: start;
|
||||
align-items: center;
|
||||
gap: var(--space-2) var(--space-3);
|
||||
padding: 12px;
|
||||
overflow: auto;
|
||||
}
|
||||
.modifyProps .formLabel {
|
||||
text-align: left;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.quickDoc {
|
||||
width: 460px;
|
||||
max-width: calc(100vw - 32px);
|
||||
@@ -831,9 +882,6 @@
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
}
|
||||
.dataView > .fill {
|
||||
min-width: 0;
|
||||
}
|
||||
.headerFilter {
|
||||
flex: none;
|
||||
display: inline-flex;
|
||||
@@ -938,7 +986,8 @@
|
||||
flex: none;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 320px;
|
||||
width: clamp(160px, 38%, 300px);
|
||||
max-width: 50%;
|
||||
min-height: 0;
|
||||
border-left: 1px solid var(--border);
|
||||
}
|
||||
@@ -1165,29 +1214,42 @@
|
||||
|
||||
/* Export / history / import dialogs. */
|
||||
.exportDialog {
|
||||
display: flex;
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 5fr) minmax(0, 6fr);
|
||||
gap: var(--space-4);
|
||||
height: 420px;
|
||||
padding: 14px;
|
||||
height: min(440px, 65vh);
|
||||
min-height: 340px;
|
||||
padding: var(--space-4);
|
||||
}
|
||||
.exportOptions {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-2);
|
||||
width: 300px;
|
||||
flex: none;
|
||||
}
|
||||
.exportOptions .formRow > input {
|
||||
flex: 1;
|
||||
}
|
||||
.exportOptions .formLabel,
|
||||
.exportPreview .formLabel {
|
||||
text-align: left;
|
||||
}
|
||||
.exportPreview {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
min-height: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
@media (max-width: 620px) {
|
||||
.exportDialog {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
height: auto;
|
||||
}
|
||||
.exportPreviewText {
|
||||
height: 200px;
|
||||
flex: none;
|
||||
}
|
||||
}
|
||||
.exportPreviewText,
|
||||
.historyPreview {
|
||||
flex: 1;
|
||||
@@ -1368,6 +1430,7 @@
|
||||
flex: none;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-1);
|
||||
padding: 4px 8px;
|
||||
background: var(--bg-elev);
|
||||
@@ -1375,8 +1438,8 @@
|
||||
font-size: var(--fs-ui);
|
||||
}
|
||||
.gridSearchField {
|
||||
flex: 0 1 360px;
|
||||
min-width: 160px;
|
||||
flex: 0 1 320px;
|
||||
min-width: min(100%, 160px);
|
||||
}
|
||||
.gridSearchCount {
|
||||
font-family: var(--font-mono);
|
||||
@@ -1396,14 +1459,19 @@
|
||||
background: var(--bg-text-search-result-current);
|
||||
}
|
||||
|
||||
/* IDEA's floating pager at the bottom of a result: |< < 1-500 of 501+ > >| and the page size. */
|
||||
.pagerDock {
|
||||
flex: none;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: var(--space-1) var(--space-2) var(--space-2);
|
||||
background: var(--bg-editor);
|
||||
}
|
||||
.resultPager {
|
||||
position: absolute;
|
||||
left: 50%;
|
||||
bottom: 12px;
|
||||
transform: translateX(-50%);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
max-width: 100%;
|
||||
gap: var(--space-0-5);
|
||||
padding: 2px 4px;
|
||||
border: 1px solid var(--border-popup, var(--border));
|
||||
|
||||
Loaded 100 of 187 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Заблокувати користувача