diff --git a/Cargo.lock b/Cargo.lock index 6d49a14d5..8565c4eb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4034,10 +4034,12 @@ name = "ide-agent" version = "0.1.25" dependencies = [ "dunce", + "encoding_rs", "ide-core", "ide-jsonrpc", "ide-tools", "ide-vcs", + "ide-vfs", "parking_lot", "serde", "serde_json", @@ -4202,6 +4204,7 @@ version = "0.1.25" dependencies = [ "dirs 6.0.0", "dunce", + "encoding_rs", "ide-agent", "ide-analysis", "ide-build", @@ -4603,6 +4606,7 @@ name = "ide-vcs" version = "0.1.25" dependencies = [ "dunce", + "encoding_rs", "gix", "ide-core", "ignore", diff --git a/crates/ide-agent/Cargo.toml b/crates/ide-agent/Cargo.toml index 6e4a53fba..046d1b244 100644 --- a/crates/ide-agent/Cargo.toml +++ b/crates/ide-agent/Cargo.toml @@ -6,6 +6,8 @@ edition.workspace = true license.workspace = true [dependencies] +encoding_rs.workspace = true +ide-vfs.workspace = true ide-core.workspace = true ide-jsonrpc.workspace = true ide-vcs.workspace = true diff --git a/crates/ide-agent/src/worktree.rs b/crates/ide-agent/src/worktree.rs index 7f385afca..b372604c3 100644 --- a/crates/ide-agent/src/worktree.rs +++ b/crates/ide-agent/src/worktree.rs @@ -96,7 +96,12 @@ pub(crate) fn without_untouched(changes: Vec, before: &BTreeMap IdeResult> { - repo_at(cwd)?.file_at(path, Some(rev)) + let repo = repo_at(cwd)?; + let encoding = match ide_vfs::load_text(path) { + Ok(loaded) => encoding_rs::Encoding::for_label(loaded.encoding.as_bytes()).unwrap_or(encoding_rs::UTF_8), + Err(_) => ide_vfs::detect_encoding(&repo.file_bytes_at(path, Some(rev))?.unwrap_or_default(), None), + }; + repo.file_at(path, Some(rev), encoding) } /// A short ASCII branch-safe name from the task title. diff --git a/crates/ide-host/Cargo.toml b/crates/ide-host/Cargo.toml index adc1344ca..4518fa0ff 100644 --- a/crates/ide-host/Cargo.toml +++ b/crates/ide-host/Cargo.toml @@ -10,6 +10,7 @@ name = "openidea-server" path = "src/bin/server.rs" [dependencies] +encoding_rs.workspace = true ide-core.workspace = true ide-text.workspace = true ide-vfs.workspace = true diff --git a/crates/ide-host/src/ops/vcs/commands.rs b/crates/ide-host/src/ops/vcs/commands.rs index 6140487d1..0c639cd1d 100644 --- a/crates/ide-host/src/ops/vcs/commands.rs +++ b/crates/ide-host/src/ops/vcs/commands.rs @@ -187,18 +187,24 @@ pub async fn vcs_branch_info(host: &Host, window: &str, repo: Option) - pub fn vcs_line_hunks(host: &Host, window: &str, path: PathBuf) -> IdeResult> { let project = host.project(window)?; let text = project.text_of(&path)?; - project.with_repo_of(&path, |r| r.line_hunks(&path, &text, false)) + project.with_repo_of(&path, |r| r.line_hunks(&path, &text, false, project.vcs_encoding(&path, None)?)) } pub async fn vcs_file_at(host: &Host, window: &str, path: PathBuf, rev: Option) -> IdeResult> { let project = host.project(window)?; - blocking(move || project.with_repo_of(&path, |r| r.file_at(&path, rev.as_deref()))).await + blocking(move || { + project.with_repo_of(&path, |r| r.file_at(&path, rev.as_deref(), project.vcs_encoding(&path, rev.as_deref())?)) + }) + .await } pub async fn vcs_blame(host: &Host, window: &str, path: PathBuf) -> IdeResult> { let project = host.project(window)?; let text = project.documents.text_of(&path); - blocking(move || project.with_repo_of(&path, |r| r.blame(&path, text.as_deref()))).await + blocking(move || { + project.with_repo_of(&path, |r| r.blame(&path, text.as_deref(), project.vcs_encoding(&path, None)?)) + }) + .await } pub async fn vcs_log(host: &Host, window: &str, repo: Option, query: LogQuery) -> IdeResult> { @@ -276,7 +282,7 @@ pub async fn vcs_unstage(host: &Host, window: &str, paths: Vec) -> IdeR pub async fn vcs_stage_patch(host: &Host, window: &str, patch: String) -> IdeResult<()> { let project = host.project(window)?; - blocking(move || project.with_repo(|r| r.stage_patch(&patch))).await + blocking(move || project.with_repo(|r| r.stage_patch(&patch, &|path| project.vcs_encoding(path, Some(":0"))))).await } pub async fn vcs_discard(host: &Host, window: &str, paths: Vec) -> IdeResult<()> { @@ -293,7 +299,12 @@ pub async fn vcs_commit( ) -> IdeResult { // Commit hooks may run for a long time. let project = host.project(window)?; - blocking(move || project.with_repo_at(repo.as_deref(), |r| r.commit_with(&message, &options))).await + blocking(move || { + project.with_repo_at(repo.as_deref(), |r| { + r.commit_with(&message, &options, &|path| project.vcs_encoding(path, None)) + }) + }) + .await } pub async fn vcs_checkout( @@ -464,12 +475,14 @@ pub async fn vcs_remote( /// The user's side is `ours` (sides swapped during a rebase, as in IDEA). pub async fn vcs_conflict_versions(host: &Host, window: &str, path: PathBuf) -> IdeResult { let project = host.project(window)?; - blocking(move || project.with_repo_of(&path, |r| r.merge_versions(&path))).await + blocking(move || project.with_repo_of(&path, |r| r.merge_versions(&path, project.vcs_encoding(&path, Some(":2"))?))) + .await } pub async fn vcs_diff_text(host: &Host, window: &str, path: PathBuf, staged: bool) -> IdeResult { let project = host.project(window)?; - blocking(move || project.with_repo_of(&path, |r| r.diff_text(&path, staged))).await + blocking(move || project.with_repo_of(&path, |r| r.diff_text(&path, staged, project.vcs_encoding(&path, None)?))) + .await } /// Line hunks between two arbitrary texts (diff viewer, commit dialog). @@ -481,13 +494,11 @@ pub fn vcs_diff_hunks(_host: &Host, _window: &str, old_text: String, new_text: S /// Indices refer to `vcs_diff_hunks(index_text, worktree_text)`. pub fn vcs_stage_hunks(host: &Host, window: &str, path: PathBuf, hunks: Vec) -> IdeResult<()> { let project = host.project(window)?; - let current = ide_vfs::load_text(&path)?.text; + let encoding = project.vcs_encoding(&path, Some(":0"))?; + let current = ide_vfs::load_text_as(&path, encoding.name())?.text; project.with_repo_of(&path, |r| { - let index_text = r.file_at(&path, Some(":0"))?.unwrap_or_default(); - let rel = path.strip_prefix(r.root()).unwrap_or(&path).to_string_lossy().replace('\\', "/"); let selected: Vec = hunks.iter().map(|&h| h as usize).collect(); - let patch = ide_vcs::build_partial_patch(&rel, &index_text, ¤t, &selected); - if patch.is_empty() { Ok(()) } else { r.stage_patch(&patch) } + r.stage_hunks(&path, ¤t, &selected, encoding) }) } diff --git a/crates/ide-host/src/project.rs b/crates/ide-host/src/project.rs index dcdeeb606..d959906aa 100644 --- a/crates/ide-host/src/project.rs +++ b/crates/ide-host/src/project.rs @@ -689,11 +689,29 @@ impl Project { } } + /// Use the open document's selected encoding, or the same detection/settings as opening it. + pub fn vcs_encoding(&self, path: &Path, rev: Option<&str>) -> IdeResult<&'static encoding_rs::Encoding> { + if let Some(id) = self.documents.by_path(path) { + let info = self.documents.snapshot(id)?; + return encoding_rs::Encoding::for_label(info.encoding.as_bytes()) + .ok_or_else(|| ide_core::IdeError::invalid("unknown document encoding")); + } + let defaults = self.documents.file_defaults(); + let fallback = defaults.fallback_for(path); + if path.exists() { + let loaded = ide_vfs::load_text_with_fallback(path, fallback)?; + return encoding_rs::Encoding::for_label(loaded.encoding.as_bytes()) + .ok_or_else(|| ide_core::IdeError::invalid("unknown file encoding")); + } + let bytes = self.with_repo_of(path, |r| r.file_bytes_at(path, rev))?.unwrap_or_default(); + Ok(ide_vfs::detect_encoding(&bytes, fallback)) + } + /// Current text of `path`: the open buffer when available, else the file on disk. pub fn text_of(&self, path: &Path) -> IdeResult { match self.documents.text_of(path) { Some(text) => Ok(text), - None => Ok(ide_vfs::load_text(path)?.text), + None => Ok(ide_vfs::load_text_with_fallback(path, self.documents.file_defaults().fallback_for(path))?.text), } } } diff --git a/crates/ide-host/tests/vcs_encoding.rs b/crates/ide-host/tests/vcs_encoding.rs new file mode 100644 index 000000000..0b34d18b5 --- /dev/null +++ b/crates/ide-host/tests/vcs_encoding.rs @@ -0,0 +1,111 @@ +use ide_core::{ChangeSet, NullSink, TextChange}; +use ide_host::{Host, HostConfig, ops::vcs}; +use std::{path::Path, process::Command, sync::Arc, time::Duration}; + +fn git(root: &Path, args: &[&str]) { + let out = Command::new("git").current_dir(root).args(args).output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn gbk_document_gutter_diff_stage_and_rollback_share_encoding() { + let temp = tempfile::tempdir().unwrap(); + let base = dunce::canonicalize(temp.path()).unwrap(); + let root = base.join("project"); + std::fs::create_dir(&root).unwrap(); + git(&root, &["init", "-b", "main"]); + git(&root, &["config", "user.name", "Test"]); + git(&root, &["config", "user.email", "test@example.com"]); + git(&root, &["config", "commit.gpgsign", "false"]); + git(&root, &["config", "core.autocrlf", "false"]); + let path = root.join("Example.java"); + let old = "// 中文注释\nclass Example {\n // 保持不变\n int value = 1;\n}\n"; + let original = ide_vfs::encode(old, "gbk", false).unwrap(); + std::fs::write(&path, &original).unwrap(); + std::fs::write(root.join("Utf8.java"), old).unwrap(); + git(&root, &["add", "."]); + git(&root, &["commit", "-m", "GBK Java"]); + let host = Host::new(HostConfig { + config_dir: base.join("config"), + data_dir: base.join("data"), + languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()), + syntax: Arc::default(), + sink_for: Box::new(|_| Arc::new(NullSink)), + owner: |w| w, + runtime: tokio::runtime::Handle::current(), + project_environment: false, + }); + let project = host.open_project("main", &root).unwrap(); + for _ in 0..500 { + if !project.repos().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let mut defaults = project.documents.file_defaults(); + defaults.project_fallback = "gbk".into(); + project.documents.set_file_defaults(defaults); + assert_eq!(project.text_of(&path).unwrap(), old); + assert_eq!(vcs::vcs_file_at(&host, "main", path.clone(), None).await.unwrap().as_deref(), Some(old)); + let doc = project.documents.open(&path).unwrap(); + assert_eq!(doc.encoding, "gbk"); + // A live document's explicit selection wins over later project defaults. + let mut defaults = project.documents.file_defaults(); + defaults.project_fallback = "windows-1252".into(); + project.documents.set_file_defaults(defaults); + assert!(vcs::vcs_line_hunks(&host, "main", path.clone()).unwrap().is_empty()); + let from = old.find("value = 1").unwrap() + "value = ".len(); + let offset = old[..from].encode_utf16().count() as u32; + project + .documents + .edit( + doc.id, + doc.version, + ChangeSet { changes: vec![TextChange { from: offset, to: offset + 1, insert: "2".into() }] }, + ) + .unwrap(); + let hunks = vcs::vcs_line_hunks(&host, "main", path.clone()).unwrap(); + assert_eq!(hunks.len(), 1); + assert_eq!((hunks[0].start, hunks[0].lines), (3, 1)); + assert_eq!(hunks[0].old_text, " int value = 1;\n"); + project.documents.save(doc.id).unwrap(); + vcs::vcs_stage_hunks(&host, "main", path.clone(), vec![0]).unwrap(); + let repo = ide_vcs::Repo::discover(&root).unwrap().unwrap(); + assert_eq!( + repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), + ide_vfs::encode(&old.replace("value = 1", "value = 2"), "gbk", false).unwrap() + ); + repo.unstage(std::slice::from_ref(&path)).unwrap(); + let snapshot = project.documents.snapshot(doc.id).unwrap(); + let line_start = + snapshot.text.split_inclusive('\n').take(3).map(|s| s.encode_utf16().count()).sum::() as u32; + project + .documents + .edit( + doc.id, + snapshot.version, + ChangeSet { + changes: vec![TextChange { + from: line_start, + to: line_start + " int value = 2;\n".encode_utf16().count() as u32, + insert: hunks[0].old_text.clone(), + }], + }, + ) + .unwrap(); + project.documents.save(doc.id).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), original); + assert!(vcs::vcs_line_hunks(&host, "main", path.clone()).unwrap().is_empty()); + assert!(vcs::vcs_blame(&host, "main", path.clone()).await.unwrap().iter().all(|l| l.author == "Test")); + // Missing working files still use configured encoding for historical content. + project.documents.close(doc.id); + let mut defaults = project.documents.file_defaults(); + defaults.project_fallback = "gbk".into(); + project.documents.set_file_defaults(defaults); + std::fs::remove_file(&path).unwrap(); + assert_eq!(vcs::vcs_file_at(&host, "main", path, None).await.unwrap().as_deref(), Some(old)); + let utf8 = root.join("Utf8.java"); + std::fs::remove_file(&utf8).unwrap(); + assert_eq!(vcs::vcs_file_at(&host, "main", utf8, None).await.unwrap().as_deref(), Some(old)); + host.close_project("main"); +} diff --git a/crates/ide-vcs/Cargo.toml b/crates/ide-vcs/Cargo.toml index 169063a40..9f74924e2 100644 --- a/crates/ide-vcs/Cargo.toml +++ b/crates/ide-vcs/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true [dependencies] +encoding_rs.workspace = true dunce.workspace = true ide-core.workspace = true ignore.workspace = true diff --git a/crates/ide-vcs/src/diff.rs b/crates/ide-vcs/src/diff.rs index f52f46406..9a2e935ff 100644 --- a/crates/ide-vcs/src/diff.rs +++ b/crates/ide-vcs/src/diff.rs @@ -174,6 +174,45 @@ fn quote_path(path: &str) -> String { out } +/// Apply selected line hunks while retaining the base's line separators. +pub(crate) fn selected_text(old: &str, new: &str, selected: &[usize]) -> String { + let old_norm = normalize_eol(old); + let new_norm = normalize_eol(new); + let old_lines = split_lines(&old_norm); + let new_lines = split_lines(&new_norm); + let old_raw = split_raw_lines(old); + let eol = if old.contains("\r\n") { + "\r\n" + } else if old.contains('\r') { + "\r" + } else { + "\n" + }; + let mut result = String::new(); + let mut cursor = 0; + for (index, (before, after)) in raw_hunks(&old_lines, &new_lines).iter().enumerate() { + if !selected.contains(&index) { + continue; + } + for line in &old_raw[cursor..before.start as usize] { + result.push_str(line); + } + for line in &new_lines[after.start as usize..after.end as usize] { + if let Some(body) = line.strip_suffix('\n') { + result.push_str(body); + result.push_str(eol); + } else { + result.push_str(line); + } + } + cursor = before.end as usize; + } + for line in &old_raw[cursor..] { + result.push_str(line); + } + result +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ide-vcs/src/lib.rs b/crates/ide-vcs/src/lib.rs index ceb0735c9..aa16e84f6 100644 --- a/crates/ide-vcs/src/lib.rs +++ b/crates/ide-vcs/src/lib.rs @@ -2,6 +2,7 @@ //! writes (commit, checkout, push/pull, stash, merge) shell out to `git` so //! hooks, credentials and signing behave exactly as on the command line. +use encoding_rs::Encoding; use ide_core::ignored::IgnoredNames; use ide_core::{CancellationToken, IdeResult}; use serde::{Deserialize, Serialize}; @@ -451,6 +452,7 @@ mod repo; mod roots; pub mod shelf; mod status; +mod text_encoding; mod write; pub use cli::{detected_git_executable, git_version, set_askpass_env, set_git_executable}; @@ -520,10 +522,24 @@ impl Repo { } /// Content of `path` at `rev` (default HEAD); `None` if absent there. /// `rev` may be any revision git understands, or `":"` / `":0"` for the index. - pub fn file_at(&self, path: &Path, rev: Option<&str>) -> IdeResult> { + pub fn file_at(&self, path: &Path, rev: Option<&str>, encoding: &'static Encoding) -> IdeResult> { let rel = self.rel_path(path)?; let repo = self.gix()?; - Ok(repo::blob_at(&repo, &rel, rev.unwrap_or("HEAD"))?.map(|b| String::from_utf8_lossy(&b).into_owned())) + Ok(repo::blob_at(&repo, &rel, rev.unwrap_or("HEAD"))?.map(|b| text_encoding::decode(&b, encoding).into_owned())) + } + /// Raw revision bytes, for encoding detection when the working file is absent. + pub fn file_bytes_at(&self, path: &Path, rev: Option<&str>) -> IdeResult>> { + repo::blob_at(&self.gix()?, &self.rel_path(path)?, rev.unwrap_or("HEAD")) + } + /// Stage selected decoded hunks, retaining the index encoding, BOM and line endings. + pub fn stage_hunks( + &self, + path: &Path, + current: &str, + selected: &[usize], + encoding: &'static Encoding, + ) -> IdeResult<()> { + write::stage_hunks(self, path, current, selected, encoding) } /// Hunks of `current_text` against HEAD (or the index when `against_index`). /// @@ -531,7 +547,13 @@ impl Repo { /// follows the removed block (equal to the line count when removed at the end). /// Files absent from the base yield one `Added` hunk; ignored files, binary files and /// paths outside the worktree yield no hunks. - pub fn line_hunks(&self, path: &Path, current_text: &str, against_index: bool) -> IdeResult> { + pub fn line_hunks( + &self, + path: &Path, + current_text: &str, + against_index: bool, + encoding: &'static Encoding, + ) -> IdeResult> { let Ok(rel) = self.rel_path(path) else { return Ok(Vec::new()) }; if rel.is_empty() || rel == ".git" || rel.starts_with(".git/") { return Ok(Vec::new()); @@ -539,8 +561,8 @@ impl Repo { let repo = self.gix()?; let base = repo::blob_at(&repo, &rel, if against_index { ":0" } else { "HEAD" })?; match base { - Some(base) if base.contains(&0) => Ok(Vec::new()), - Some(base) => Ok(diff::line_hunks(&String::from_utf8_lossy(&base), current_text)), + Some(base) if text_encoding::is_binary(&base, encoding) => Ok(Vec::new()), + Some(base) => Ok(diff::line_hunks(&text_encoding::decode(&base, encoding), current_text)), None => { if repo::is_ignored(&repo, &rel)? { return Ok(Vec::new()); @@ -551,8 +573,13 @@ impl Repo { } /// One entry per line of the current text (`current_text`, or the file on disk). /// Lines not yet committed carry an all-zero commit id and author "Not Committed Yet". - pub fn blame(&self, path: &Path, current_text: Option<&str>) -> IdeResult> { - write::blame(self, path, current_text) + pub fn blame( + &self, + path: &Path, + current_text: Option<&str>, + encoding: &'static Encoding, + ) -> IdeResult> { + write::blame(self, path, current_text, encoding) } /// Commits newest-first in topological order (children before parents). /// @@ -583,8 +610,8 @@ impl Repo { } /// Unified diff of a file: index↔worktree (`staged == false`, untracked files diff /// against /dev/null) or HEAD↔index (`staged == true`). Empty when unchanged. - pub fn diff_text(&self, path: &Path, staged: bool) -> IdeResult { - write::diff_text(self, path, staged) + pub fn diff_text(&self, path: &Path, staged: bool, encoding: &'static Encoding) -> IdeResult { + write::diff_text(self, path, staged, encoding) } /// `git config --get ` (all scopes); `None` when unset. @@ -602,8 +629,8 @@ impl Repo { } /// Applies a unified-diff patch to the index (partial staging of hunks/lines). /// Zero-context patches (see [`build_partial_patch`]) are accepted. - pub fn stage_patch(&self, patch: &str) -> IdeResult<()> { - write::stage_patch(self, patch) + pub fn stage_patch(&self, patch: &str, encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>) -> IdeResult<()> { + write::stage_patch(self, patch, encoding) } /// Discards unstaged changes: tracked paths are restored from the index (staged /// changes are kept), untracked files and directories are deleted. @@ -614,11 +641,16 @@ impl Repo { /// `amend` keeps the previous message. pub fn commit(&self, message: &str, amend: bool, sign_off: bool) -> IdeResult { let options = CommitOptions { amend, sign_off, author: None, run_hooks: true, partial: Vec::new() }; - write::commit(self, message, &options) + write::commit(self, message, &options, &|_| Ok(encoding_rs::UTF_8)) } /// Commits with the Commit Options popup's choices. - pub fn commit_with(&self, message: &str, options: &CommitOptions) -> IdeResult { - write::commit(self, message, options) + pub fn commit_with( + &self, + message: &str, + options: &CommitOptions, + encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>, + ) -> IdeResult { + write::commit(self, message, options, encoding) } /// Switches to `branch`. With `create`, creates it at HEAD. Without, a remote branch /// (`origin/x`) gets a local tracking branch and any other revision is checked out @@ -867,8 +899,8 @@ impl Repo { } /// The versions of a conflicted file with the user's work as `ours`, as IDEA's merge /// dialog shows them (sides swapped during a rebase). - pub fn merge_versions(&self, path: &Path) -> IdeResult { - let (base, ours, theirs) = self.conflict_versions(path)?; + pub fn merge_versions(&self, path: &Path, encoding: &'static Encoding) -> IdeResult { + let (base, ours, theirs) = self.conflict_versions(path, encoding)?; let sides = self.merge_sides()?; let (ours, theirs) = if sides.reversed { (theirs, ours) } else { (ours, theirs) }; let merged = match (&ours, &theirs) { @@ -878,11 +910,15 @@ impl Repo { Ok(MergeVersions { base, ours, theirs, sides, merged }) } /// For conflict resolution: (base, ours, theirs) versions of a conflicted file. - pub fn conflict_versions(&self, path: &Path) -> IdeResult<(Option, Option, Option)> { + pub fn conflict_versions( + &self, + path: &Path, + encoding: &'static Encoding, + ) -> IdeResult<(Option, Option, Option)> { let rel = self.rel_path(path)?; let repo = self.gix()?; let stage = |n: u8| -> IdeResult> { - Ok(repo::blob_at(&repo, &rel, &format!(":{n}"))?.map(|b| String::from_utf8_lossy(&b).into_owned())) + Ok(repo::blob_at(&repo, &rel, &format!(":{n}"))?.map(|b| text_encoding::decode(&b, encoding).into_owned())) }; Ok((stage(1)?, stage(2)?, stage(3)?)) } diff --git a/crates/ide-vcs/src/patch.rs b/crates/ide-vcs/src/patch.rs index 9bb54feaf..8192641ca 100644 --- a/crates/ide-vcs/src/patch.rs +++ b/crates/ide-vcs/src/patch.rs @@ -50,11 +50,12 @@ pub(crate) fn working_patch(r: &Repo, rel_paths: &[String], reverse: bool) -> Id let mut add: Vec<&str> = vec!["--literal-pathspecs", "add", "-A", "--"]; add.extend(rel_paths.iter().map(String::as_str)); run_env(r.root(), &add, None, &env)?; - let mut diff = vec!["diff", "--cached", "--binary", "--full-index", "-M", "--no-color", "--no-ext-diff"]; + let mut diff = + vec!["diff", "--cached", "--binary", "--full-index", "-M", "--no-color", "--no-ext-diff", "--no-textconv"]; if reverse { diff.push("-R"); } - let patch = run_env(r.root(), &diff, None, &env)?.stdout_str(); + let patch = lossless_diff(r, &diff, &env)?; let names = run_env(r.root(), &["diff", "--cached", "--name-status", "-z", "--no-renames"], None, &env)?; Ok((patch, crate::ops::parse_name_status(&names.stdout_str()))) } @@ -74,12 +75,12 @@ pub(crate) fn commits_patch(r: &Repo, commits: &[String], reverse: bool) -> IdeR let parent = commit.parent_ids().next().map(|p| p.detach().to_string()); (parent.unwrap_or_else(|| EMPTY_TREE.to_string()), id.to_string()) }; - let mut args = vec!["diff", "--binary", "--full-index", "-M", "--no-color", "--no-ext-diff"]; + let mut args = vec!["diff", "--binary", "--full-index", "-M", "--no-color", "--no-ext-diff", "--no-textconv"]; if reverse { args.push("-R"); } args.extend([parent.as_str(), id.as_str(), "--"]); - out.push_str(&run(r.root(), &args, None)?.stdout_str()); + out.push_str(&lossless_diff(r, &args, &[])?); } Ok(out) } @@ -266,6 +267,68 @@ pub(crate) fn parse_patch(patch: &str) -> Vec { out } +/// A patch crosses the string IPC boundary. Non-UTF-8 content must travel as a +/// Git binary patch, not replacement characters or an encoding-dependent string. +fn lossless_diff(r: &Repo, args: &[&str], env: &[(&str, &OsStr)]) -> IdeResult { + let output = run_env(r.root(), args, None, env)?.stdout; + if let Ok(text) = String::from_utf8(output) { + return Ok(text); + } + // info/attributes has highest precedence. Isolate it from the real repository + // so concurrent Git operations and user attributes are never modified. + let temp = tempfile::tempdir()?; + let git_dir = temp.path().to_string_lossy().into_owned(); + let format = run(r.root(), &["rev-parse", "--show-object-format"], None)?.stdout_str(); + run(r.root(), &["init", "--bare", "--quiet", &format!("--object-format={}", format.trim()), &git_dir], None)?; + std::fs::write(temp.path().join("info/attributes"), "* -diff\n")?; + if let Some(head) = r.head()?.1 { + std::fs::write(temp.path().join("HEAD"), format!("{head}\n"))?; + } + let objects = run(r.root(), &["rev-parse", "--path-format=absolute", "--git-path", "objects"], None)?.stdout_str(); + let mut isolated = env.to_vec(); + isolated.extend([ + ("GIT_DIR", temp.path().as_os_str()), + ("GIT_WORK_TREE", r.root().as_os_str()), + ("GIT_OBJECT_DIRECTORY", OsStr::new(objects.trim())), + ]); + String::from_utf8(run_env(r.root(), args, None, &isolated)?.stdout) + .map_err(|_| IdeError::invalid("Git produced a non-UTF-8 binary patch")) +} + +/// Encode only hunk payloads; patch headers and Git paths always remain UTF-8. +pub(crate) fn encode_text_patch( + r: &Repo, + patch: &str, + encoding: &dyn Fn(&std::path::Path) -> IdeResult<&'static encoding_rs::Encoding>, +) -> IdeResult> { + let mut out = Vec::new(); + let mut header = String::new(); + let mut enc = None; + for line in patch.split_inclusive('\n') { + if line.starts_with("diff --git ") { + header.clear(); + enc = None; + } + if line.starts_with("@@ ") { + let file = parse_patch(&header) + .into_iter() + .next() + .ok_or_else(|| IdeError::invalid("patch hunk has no file header"))?; + enc = Some(encoding(&r.abs_path(&file.path))?); + } else if let Some(enc) = enc { + if line.starts_with(['+', '-', ' ']) { + out.push(line.as_bytes()[0]); + out.extend(crate::text_encoding::encode(&line[1..], enc, false)?); + continue; + } + } else { + header.push_str(line); + } + out.extend_from_slice(line.as_bytes()); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ide-vcs/src/text_encoding.rs b/crates/ide-vcs/src/text_encoding.rs new file mode 100644 index 000000000..9d23171a4 --- /dev/null +++ b/crates/ide-vcs/src/text_encoding.rs @@ -0,0 +1,46 @@ +//! File content uses the document's encoding; Git metadata remains UTF-8. +use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE}; +use ide_core::{IdeError, IdeResult}; +use std::borrow::Cow; + +pub(crate) fn has_bom(bytes: &[u8], encoding: &'static Encoding) -> bool { + Encoding::for_bom(bytes).is_some_and(|(enc, _)| enc == encoding) +} + +pub(crate) fn decode<'a>(bytes: &'a [u8], encoding: &'static Encoding) -> Cow<'a, str> { + let body = match Encoding::for_bom(bytes) { + Some((enc, len)) if enc == encoding => &bytes[len..], + _ => bytes, + }; + encoding.decode_without_bom_handling(body).0 +} + +pub(crate) fn is_binary(bytes: &[u8], encoding: &'static Encoding) -> bool { + encoding != UTF_16LE && encoding != UTF_16BE && bytes.contains(&0) +} + +pub(crate) fn encode(text: &str, encoding: &'static Encoding, bom: bool) -> IdeResult> { + let mut out = Vec::new(); + if encoding == UTF_16LE || encoding == UTF_16BE { + let le = encoding == UTF_16LE; + if bom { + out.extend_from_slice(if le { &[0xff, 0xfe] } else { &[0xfe, 0xff] }); + } + for unit in text.encode_utf16() { + out.extend_from_slice(&if le { unit.to_le_bytes() } else { unit.to_be_bytes() }); + } + } else { + let (bytes, _, errors) = encoding.encode(text); + if errors { + return Err(IdeError::invalid(format!( + "text contains characters not representable in {}", + encoding.name() + ))); + } + if bom && encoding == UTF_8 { + out.extend_from_slice(&[0xef, 0xbb, 0xbf]); + } + out.extend_from_slice(&bytes); + } + Ok(out) +} diff --git a/crates/ide-vcs/src/write.rs b/crates/ide-vcs/src/write.rs index d34e3eb5e..b636b1b3b 100644 --- a/crates/ide-vcs/src/write.rs +++ b/crates/ide-vcs/src/write.rs @@ -2,9 +2,11 @@ use crate::cli::{self, run}; use crate::repo::{gx, resolve}; +use crate::text_encoding::{decode, encode, has_bom, is_binary}; use crate::{ BlameLine, CheckoutMode, CheckoutOptions, CheckoutOutcome, CommitOptions, DeletedBranch, DivergedLocal, Repo, }; +use encoding_rs::Encoding; use gix::bstr::{BStr, ByteSlice}; use ide_core::{CancellationToken, IdeError, IdeResult}; use std::path::{Path, PathBuf}; @@ -40,11 +42,33 @@ fn run_paths(r: &Repo, args: &[&str], paths: &[String]) -> IdeResult<()> { run(r.root(), &full, None).map(|_| ()) } -pub(crate) fn blame(r: &Repo, path: &Path, current_text: Option<&str>) -> IdeResult> { +pub(crate) fn blame( + r: &Repo, + path: &Path, + current_text: Option<&str>, + encoding: &'static Encoding, +) -> IdeResult> { let rel = r.rel_path(path)?; - let text: String = match current_text { - Some(t) => t.to_string(), - None => String::from_utf8_lossy(&std::fs::read(r.abs_path(&rel))?).into_owned(), + let disk = match std::fs::read(r.abs_path(&rel)) { + Ok(bytes) => bytes, + Err(_) if current_text.is_some() => Vec::new(), + Err(error) => return Err(error.into()), + }; + let text = current_text.map(str::to_owned).unwrap_or_else(|| decode(&disk, encoding).into_owned()); + let contents = match current_text { + Some(_) => { + let disk_text = decode(&disk, encoding); + let eol = if disk_text.contains("\r\n") { + "\r\n" + } else if disk_text.contains('\r') { + "\r" + } else { + "\n" + }; + let contents = crate::diff::normalize_eol(&text).replace('\n', eol); + encode(&contents, encoding, has_bom(&disk, encoding))? + } + None => disk, }; let line_count = text.split_inclusive('\n').count(); let tracked_in_head = { @@ -57,7 +81,7 @@ pub(crate) fn blame(r: &Repo, path: &Path, current_text: Option<&str>) -> IdeRes let out = run( r.root(), &["--literal-pathspecs", "blame", "--porcelain", "--contents", "-", "--", &rel], - Some(text.as_bytes()), + Some(&contents), )?; let mut lines = crate::blame::parse_porcelain(&out.stdout_str()); for line in &mut lines { @@ -87,7 +111,7 @@ fn uncommitted_lines(n: usize) -> Vec { .collect() } -pub(crate) fn diff_text(r: &Repo, path: &Path, staged: bool) -> IdeResult { +pub(crate) fn diff_text(r: &Repo, path: &Path, staged: bool, encoding: &'static Encoding) -> IdeResult { let rel = r.rel_path(path)?; let base = ["-c", "diff.noprefix=false", "-c", "diff.mnemonicPrefix=false", "-c", "diff.external=", "--literal-pathspecs"]; @@ -102,7 +126,7 @@ pub(crate) fn diff_text(r: &Repo, path: &Path, staged: bool) -> IdeResult Ok(out.stdout_str()), + Some(0) | Some(1) => Ok(decode_diff(&out.stdout, encoding)), code => Err(cli::failure(&args, &out.stderr, &out.stdout, code)), }; } @@ -111,7 +135,7 @@ pub(crate) fn diff_text(r: &Repo, path: &Path, staged: bool) -> IdeResult IdeResult<()> { @@ -128,7 +152,11 @@ pub(crate) fn unstage(r: &Repo, paths: &[PathBuf]) -> IdeResult<()> { } } -pub(crate) fn stage_patch(r: &Repo, patch: &str) -> IdeResult<()> { +pub(crate) fn stage_patch( + r: &Repo, + patch: &str, + encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>, +) -> IdeResult<()> { if patch.trim().is_empty() { return Ok(()); } @@ -136,12 +164,9 @@ pub(crate) fn stage_patch(r: &Repo, patch: &str) -> IdeResult<()> { if !patch.ends_with('\n') { patch.push('\n'); } - run( - r.root(), - &["apply", "--cached", "--unidiff-zero", "--whitespace=nowarn", "--recount", "-"], - Some(patch.as_bytes()), - ) - .map(|_| ()) + let bytes = crate::patch::encode_text_patch(r, &patch, encoding)?; + run(r.root(), &["apply", "--cached", "--unidiff-zero", "--whitespace=nowarn", "--recount", "-"], Some(&bytes)) + .map(|_| ()) } /// Paths with unresolved conflicts (index stages 1–3). @@ -224,7 +249,12 @@ pub(crate) fn discard(r: &Repo, paths: &[PathBuf]) -> IdeResult<()> { Ok(()) } -pub(crate) fn commit(r: &Repo, message: &str, options: &CommitOptions) -> IdeResult { +pub(crate) fn commit( + r: &Repo, + message: &str, + options: &CommitOptions, + encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>, +) -> IdeResult { let mut args = vec!["commit", "-q", "--cleanup=whitespace"]; if options.amend { args.push("--amend"); @@ -252,7 +282,7 @@ pub(crate) fn commit(r: &Repo, message: &str, options: &CommitOptions) -> IdeRes if options.partial.is_empty() { run(r.root(), &args, stdin)?; } else { - commit_partial(r, &args, stdin, &options.partial)?; + commit_partial(r, &args, stdin, &options.partial, encoding)?; } let repo = r.gix()?; Ok(repo.head_id().map_err(gx)?.detach().to_string()) @@ -277,6 +307,7 @@ fn commit_partial( commit_args: &[&str], stdin: Option<&[u8]>, partial: &[crate::PartialCommitFile], + encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>, ) -> IdeResult<()> { let tmp = TempIndex(r.git_dir.join("openidea-partial-index")); let index = r.git_dir.join("index"); @@ -295,10 +326,12 @@ fn commit_partial( IdeError::invalid(format!("{rel}: only changes of a file in HEAD can be committed partially")) })?; let disk = std::fs::read(r.abs_path(&rel))?; - if head.contains(&0) || disk.contains(&0) { + let encoding = encoding(&file.path.0)?; + let bom = has_bom(&head, encoding); + if is_binary(&head, encoding) || is_binary(&disk, encoding) { return Err(IdeError::invalid(format!("{rel}: binary files cannot be committed partially"))); } - let (head, disk) = (String::from_utf8_lossy(&head), String::from_utf8_lossy(&disk)); + let (head, disk) = (decode(&head, encoding), decode(&disk, encoding)); let hunks = crate::diff::line_hunks(&head, &disk); let mut selected = Vec::new(); for want in &file.hunks { @@ -311,15 +344,9 @@ fn commit_partial( } } cli::run_env(r.root(), &["--literal-pathspecs", "reset", "-q", "HEAD", "--", &rel], None, &env)?; - let patch = crate::diff::build_partial_patch(&rel, &head, &disk, &selected); - if !patch.is_empty() { - cli::run_env( - r.root(), - &["apply", "--cached", "--unidiff-zero", "--whitespace=nowarn", "--recount", "-"], - Some(patch.as_bytes()), - &env, - )?; - } + let text = crate::diff::selected_text(&head, &disk, &selected); + let bytes = encode(&text, encoding, bom)?; + write_index_blob(r, &rel, &bytes, &env)?; rels.push(rel); } cli::run_env(r.root(), commit_args, stdin, &env)?; @@ -798,3 +825,57 @@ fn push_remote(r: &Repo) -> IdeResult { }) .ok_or_else(|| IdeError::invalid("no remote configured")) } + +fn write_index_blob(r: &Repo, rel: &str, bytes: &[u8], env: &[(&str, &std::ffi::OsStr)]) -> IdeResult<()> { + let entry = + cli::run_env(r.root(), &["--literal-pathspecs", "ls-files", "--stage", "--", rel], None, env)?.stdout_str(); + let mode = entry.split_whitespace().next().unwrap_or("100644"); + let id = run(r.root(), &["hash-object", "-w", "--stdin"], Some(bytes))?.stdout_str(); + cli::run_env(r.root(), &["update-index", "--add", "--cacheinfo", mode, id.trim(), rel], None, env)?; + Ok(()) +} + +pub(crate) fn stage_hunks( + r: &Repo, + path: &Path, + current: &str, + selected: &[usize], + encoding: &'static Encoding, +) -> IdeResult<()> { + if selected.is_empty() { + return Ok(()); + } + let rel = r.rel_path(path)?; + let base = r.file_bytes_at(path, Some(":0"))?.unwrap_or_default(); + if is_binary(&base, encoding) { + return Err(IdeError::invalid("binary files cannot be staged partially")); + } + let old = decode(&base, encoding); + let hunks = crate::diff::line_hunks(&old, current); + if selected.iter().any(|&i| i >= hunks.len()) { + return Err(IdeError::invalid("hunk no longer exists")); + } + let text = crate::diff::selected_text(&old, current, selected); + let bytes = encode(&text, encoding, has_bom(&base, encoding))?; + write_index_blob(r, &rel, &bytes, &[]) +} + +// Git headers contain UTF-8 paths; only source lines use the file's encoding. +fn decode_diff(bytes: &[u8], encoding: &'static Encoding) -> String { + let mut hunks = false; + let mut text = String::new(); + for line in bytes.split_inclusive(|&b| b == b'\n') { + if line.starts_with(b"diff --git ") { + hunks = false; + } + if line.starts_with(b"@@ ") { + hunks = true; + } + if hunks { + text.push_str(&decode(line, encoding)); + } else { + text.push_str(&String::from_utf8_lossy(line)); + } + } + text +} diff --git a/crates/ide-vcs/tests/git_workflows.rs b/crates/ide-vcs/tests/git_workflows.rs index 073161f19..e33706286 100644 --- a/crates/ide-vcs/tests/git_workflows.rs +++ b/crates/ide-vcs/tests/git_workflows.rs @@ -651,7 +651,7 @@ fn rollback_hunk_via_partial_patch_leaves_other_hunks() { assert_eq!(hunks.len(), 2); let patch = build_partial_patch("f.txt", &original, &edited, &[0]); let repo = fx.repo(); - repo.stage_patch(&patch).unwrap(); + repo.stage_patch(&patch, &|_| Ok(encoding_rs::UTF_8)).unwrap(); repo.discard(&[fx.path("f.txt")]).unwrap(); assert_eq!(fx.read("f.txt"), original.replace("line 2\n", "line two\n")); } @@ -777,7 +777,7 @@ fn add_to_vcs_annotate_and_history() { repo.stage(&[fx.path("new/b.txt")]).unwrap(); assert_eq!(status_of(&repo.status().unwrap(), "new/b.txt").unwrap().staged, Some(ChangeKind::Added)); - let blame = repo.blame(&fx.path("a.txt"), Some("1\nchanged\nunsaved\n")).unwrap(); + let blame = repo.blame(&fx.path("a.txt"), Some("1\nchanged\nunsaved\n"), encoding_rs::UTF_8).unwrap(); assert_eq!(blame.len(), 3); assert_eq!(blame[0].commit, first); assert_eq!(blame[1].commit, second); @@ -785,8 +785,8 @@ fn add_to_vcs_annotate_and_history() { let history = repo.log(&LogQuery { path: Some(fx.path("a.txt").into()), ..Default::default() }).unwrap(); assert_eq!(history.iter().map(|c| c.id.as_str()).collect::>(), vec![second.as_str(), first.as_str()]); - assert_eq!(repo.file_at(&fx.path("a.txt"), Some(&first)).unwrap().as_deref(), Some("1\n2\n")); - let diff = repo.diff_text(&fx.path("new/b.txt"), true).unwrap(); + assert_eq!(repo.file_at(&fx.path("a.txt"), Some(&first), encoding_rs::UTF_8).unwrap().as_deref(), Some("1\n2\n")); + let diff = repo.diff_text(&fx.path("new/b.txt"), true, encoding_rs::UTF_8).unwrap(); assert!(diff.contains("+b"), "{diff}"); } diff --git a/crates/ide-vcs/tests/history_ops.rs b/crates/ide-vcs/tests/history_ops.rs index 7116a98c4..9e2b756bf 100644 --- a/crates/ide-vcs/tests/history_ops.rs +++ b/crates/ide-vcs/tests/history_ops.rs @@ -113,7 +113,7 @@ fn merge_versions_put_the_users_work_on_the_left() { let repo = fx.repo(); let topic = fx.git(&["rev-parse", "topic"]); assert!(repo.rebase("main").is_err()); - let v = repo.merge_versions(&fx.path("f.txt")).unwrap(); + let v = repo.merge_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap(); assert_eq!(v.ours.as_deref(), Some("topic\n")); assert_eq!(v.theirs.as_deref(), Some("main\n")); assert_eq!(v.base.as_deref(), Some("base\n")); @@ -129,7 +129,7 @@ fn merge_versions_put_the_users_work_on_the_left() { // Merge: not swapped; the merged branch is named. assert!(repo.merge("main").is_err()); - let v = repo.merge_versions(&fx.path("f.txt")).unwrap(); + let v = repo.merge_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap(); assert_eq!(v.ours.as_deref(), Some("topic\n")); assert_eq!(v.theirs.as_deref(), Some("main\n")); assert!(!v.sides.reversed); @@ -140,7 +140,7 @@ fn merge_versions_put_the_users_work_on_the_left() { // Cherry-pick: not swapped; the picked commit is named. let main = fx.git(&["rev-parse", "main"]); assert!(repo.cherry_pick(std::slice::from_ref(&main), &cancel, &out).is_err()); - let v = repo.merge_versions(&fx.path("f.txt")).unwrap(); + let v = repo.merge_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap(); assert_eq!(v.ours.as_deref(), Some("topic\n")); assert!(!v.sides.reversed); assert_eq!(v.sides.operation.as_deref(), Some("cherry-pick")); @@ -374,7 +374,7 @@ fn unshelve_with_conflict_leaves_merge_state() { assert_eq!(outcome.conflicts, vec!["f.txt".to_string()]); let text = fx.read("f.txt"); assert!(text.contains("<<<<<<<") && text.contains("SHELVED") && text.contains("COMMITTED"), "{text}"); - let (base, ours, theirs) = repo.conflict_versions(&fx.path("f.txt")).unwrap(); + let (base, ours, theirs) = repo.conflict_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap(); assert!(base.is_some() && ours.unwrap().contains("COMMITTED") && theirs.unwrap().contains("SHELVED")); // Kept on the shelf when not removing. assert_eq!(shelf::list(&shelf).unwrap().len(), 1); diff --git a/crates/ide-vcs/tests/repo.rs b/crates/ide-vcs/tests/repo.rs index b202cd78f..6b97eeadf 100644 --- a/crates/ide-vcs/tests/repo.rs +++ b/crates/ide-vcs/tests/repo.rs @@ -121,11 +121,11 @@ fn repository_without_commits() { assert!(repo.log(&LogQuery::default()).unwrap().is_empty()); assert!(repo.branches().unwrap().is_empty()); assert!(repo.stashes().unwrap().is_empty()); - assert_eq!(repo.file_at(Path::new("new.txt"), None).unwrap(), None); + assert_eq!(repo.file_at(Path::new("new.txt"), None, encoding_rs::UTF_8).unwrap(), None); let st = repo.status().unwrap(); assert_eq!(status_of(&st, "new.txt").unstaged, Some(ChangeKind::Untracked)); - let hunks = repo.line_hunks(&fx.path("new.txt"), "hello\n", false).unwrap(); + let hunks = repo.line_hunks(&fx.path("new.txt"), "hello\n", false, encoding_rs::UTF_8).unwrap(); assert_eq!(hunks.len(), 1); assert_eq!(hunks[0].kind, HunkKind::Added); @@ -244,7 +244,7 @@ fn merge_conflict_and_versions() { let st = repo.status().unwrap(); let s = status_of(&st, "c.txt"); assert_eq!((s.staged, s.unstaged), (None, Some(ChangeKind::Conflicted))); - let (base, ours, theirs) = repo.conflict_versions(&fx.path("c.txt")).unwrap(); + let (base, ours, theirs) = repo.conflict_versions(&fx.path("c.txt"), encoding_rs::UTF_8).unwrap(); assert_eq!(base.as_deref(), Some("base\n")); assert_eq!(ours.as_deref(), Some("ours\n")); assert_eq!(theirs.as_deref(), Some("theirs\n")); @@ -378,36 +378,36 @@ fn line_hunks_against_head_and_index() { let repo = fx.repo(); let p = fx.path("f.txt"); - assert!(repo.line_hunks(&p, "1\n2\n3\n4\n5\n", false).unwrap().is_empty()); - assert!(repo.line_hunks(&p, "1\r\n2\r\n3\r\n4\r\n5\r\n", false).unwrap().is_empty()); + assert!(repo.line_hunks(&p, "1\n2\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap().is_empty()); + assert!(repo.line_hunks(&p, "1\r\n2\r\n3\r\n4\r\n5\r\n", false, encoding_rs::UTF_8).unwrap().is_empty()); - let h = repo.line_hunks(&p, "0\n1\n2\n3\n4\n5\n", false).unwrap(); + let h = repo.line_hunks(&p, "0\n1\n2\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap(); assert_eq!((h.len(), h[0].kind, h[0].start, h[0].lines), (1, HunkKind::Added, 0, 1)); - let h = repo.line_hunks(&p, "1\n2\n3\n4\n5\n6\n", false).unwrap(); + let h = repo.line_hunks(&p, "1\n2\n3\n4\n5\n6\n", false, encoding_rs::UTF_8).unwrap(); assert_eq!((h[0].kind, h[0].start, h[0].lines), (HunkKind::Added, 5, 1)); - let h = repo.line_hunks(&p, "1\nX\n3\n4\n5\n", false).unwrap(); + let h = repo.line_hunks(&p, "1\nX\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap(); assert_eq!((h[0].kind, h[0].start, h[0].lines, h[0].old_text.as_str()), (HunkKind::Modified, 1, 1, "2\n")); - let h = repo.line_hunks(&p, "1\n4\n5\n", false).unwrap(); + let h = repo.line_hunks(&p, "1\n4\n5\n", false, encoding_rs::UTF_8).unwrap(); assert_eq!((h[0].kind, h[0].start, h[0].lines, h[0].old_start, h[0].old_lines), (HunkKind::Deleted, 1, 0, 1, 2)); - let h = repo.line_hunks(&p, "2\n3\n4\n5\n", false).unwrap(); + let h = repo.line_hunks(&p, "2\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap(); assert_eq!((h[0].kind, h[0].start), (HunkKind::Deleted, 0)); - let h = repo.line_hunks(&p, "1\n2\n3\n4\n", false).unwrap(); + let h = repo.line_hunks(&p, "1\n2\n3\n4\n", false, encoding_rs::UTF_8).unwrap(); assert_eq!((h[0].kind, h[0].start, h[0].old_start), (HunkKind::Deleted, 4, 4)); // Against the index: staged changes are not reported. fx.write("f.txt", "1\nX\n3\n4\n5\n"); fx.git(&["add", "f.txt"]); - assert!(repo.line_hunks(&p, "1\nX\n3\n4\n5\n", true).unwrap().is_empty()); - assert_eq!(repo.line_hunks(&p, "1\nX\n3\n4\n5\n", false).unwrap().len(), 1); + assert!(repo.line_hunks(&p, "1\nX\n3\n4\n5\n", true, encoding_rs::UTF_8).unwrap().is_empty()); + assert_eq!(repo.line_hunks(&p, "1\nX\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap().len(), 1); // New and ignored files. - let h = repo.line_hunks(&fx.path("new.txt"), "a\nb\n", false).unwrap(); + let h = repo.line_hunks(&fx.path("new.txt"), "a\nb\n", false, encoding_rs::UTF_8).unwrap(); assert_eq!((h.len(), h[0].kind, h[0].start, h[0].lines), (1, HunkKind::Added, 0, 2)); - assert!(repo.line_hunks(&fx.path("ignored.txt"), "a\n", false).unwrap().is_empty()); - assert!(repo.line_hunks(Path::new("/definitely/outside"), "a\n", false).unwrap().is_empty()); + assert!(repo.line_hunks(&fx.path("ignored.txt"), "a\n", false, encoding_rs::UTF_8).unwrap().is_empty()); + assert!(repo.line_hunks(Path::new("/definitely/outside"), "a\n", false, encoding_rs::UTF_8).unwrap().is_empty()); } #[test] @@ -419,7 +419,7 @@ fn blame_with_unsaved_edits() { let second = fx.commit_all("second"); let repo = fx.repo(); - let lines = repo.blame(&fx.path("b.txt"), None).unwrap(); + let lines = repo.blame(&fx.path("b.txt"), None, encoding_rs::UTF_8).unwrap(); assert_eq!(lines.len(), 3); assert_eq!(lines[0].commit, first); assert_eq!(lines[1].commit, second); @@ -427,7 +427,7 @@ fn blame_with_unsaved_edits() { assert_eq!(lines[0].author, "Test User"); assert!(lines[0].time_ms > 1.0e12); - let lines = repo.blame(&fx.path("b.txt"), Some("one\nedited\nthree\nfour")).unwrap(); + let lines = repo.blame(&fx.path("b.txt"), Some("one\nedited\nthree\nfour"), encoding_rs::UTF_8).unwrap(); assert_eq!(lines.len(), 4); assert_eq!(lines[0].commit, first); assert!(lines[1].commit.bytes().all(|b| b == b'0')); @@ -435,7 +435,7 @@ fn blame_with_unsaved_edits() { assert_eq!(lines[2].commit, first); assert_eq!(lines[3].author, "Not Committed Yet"); - let lines = repo.blame(&fx.path("untracked.txt"), Some("x\ny\n")).unwrap(); + let lines = repo.blame(&fx.path("untracked.txt"), Some("x\ny\n"), encoding_rs::UTF_8).unwrap(); assert_eq!(lines.len(), 2); assert!(lines.iter().all(|l| l.author == "Not Committed Yet")); } @@ -615,12 +615,12 @@ fn file_at_revisions_and_index() { fx.git(&["add", "f.txt"]); let repo = fx.repo(); let p = fx.path("f.txt"); - assert_eq!(repo.file_at(&p, None).unwrap().as_deref(), Some("v2\n")); - assert_eq!(repo.file_at(&p, Some(&c1)).unwrap().as_deref(), Some("v1\n")); - assert_eq!(repo.file_at(&p, Some("HEAD~1")).unwrap().as_deref(), Some("v1\n")); - assert_eq!(repo.file_at(&p, Some(":0")).unwrap().as_deref(), Some("v3\n")); - assert_eq!(repo.file_at(Path::new("missing.txt"), None).unwrap(), None); - assert_eq!(repo.file_at(&p, Some("no-such-rev")).unwrap_err().kind, ErrorKind::NotFound); + assert_eq!(repo.file_at(&p, None, encoding_rs::UTF_8).unwrap().as_deref(), Some("v2\n")); + assert_eq!(repo.file_at(&p, Some(&c1), encoding_rs::UTF_8).unwrap().as_deref(), Some("v1\n")); + assert_eq!(repo.file_at(&p, Some("HEAD~1"), encoding_rs::UTF_8).unwrap().as_deref(), Some("v1\n")); + assert_eq!(repo.file_at(&p, Some(":0"), encoding_rs::UTF_8).unwrap().as_deref(), Some("v3\n")); + assert_eq!(repo.file_at(Path::new("missing.txt"), None, encoding_rs::UTF_8).unwrap(), None); + assert_eq!(repo.file_at(&p, Some("no-such-rev"), encoding_rs::UTF_8).unwrap_err().kind, ErrorKind::NotFound); } #[test] @@ -636,7 +636,7 @@ fn file_at_in_a_blobless_clone() { git_in(fx.dir.parent().unwrap(), &["clone", "-q", "--filter=blob:none", &url, "clone"]); let repo = Repo::discover(&clone).unwrap().unwrap(); // v1's blob was never fetched; reading it fetches it on demand. - assert_eq!(repo.file_at(&clone.join("f.txt"), Some(&c1)).unwrap().as_deref(), Some("v1\n")); + assert_eq!(repo.file_at(&clone.join("f.txt"), Some(&c1), encoding_rs::UTF_8).unwrap().as_deref(), Some("v1\n")); } #[test] @@ -850,9 +850,12 @@ fn commit_with_author_and_without_hooks() { fx.write("a", "2\n"); repo.stage(&[fx.path("a")]).unwrap(); let hooked = CommitOptions { run_hooks: true, ..Default::default() }; - assert!(repo.commit_with("blocked", &hooked).is_err(), "the failing pre-commit hook runs"); + assert!( + repo.commit_with("blocked", &hooked, &|_| Ok(encoding_rs::UTF_8)).is_err(), + "the failing pre-commit hook runs" + ); let options = CommitOptions { author: Some(" Other ".into()), ..Default::default() }; - repo.commit_with("by other", &options).unwrap(); + repo.commit_with("by other", &options, &|_| Ok(encoding_rs::UTF_8)).unwrap(); assert_eq!(fx.git(&["log", "-1", "--format=%an <%ae>"]), "Other "); } @@ -932,7 +935,7 @@ fn smart_and_force_checkout_with_local_changes_in_the_way() { assert_eq!(repo.branch_info().unwrap().head.as_deref(), Some("other")); assert_eq!(repo.stashes().unwrap().len(), 1, "git keeps the stash after a conflicting pop"); // The stashed edit is the user's side of the merge, the checked-out branch the other. - let v = repo.merge_versions(&fx.path("f.txt")).unwrap(); + let v = repo.merge_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap(); assert_eq!(v.sides.operation.as_deref(), Some("unstash")); assert!(v.sides.reversed); assert_eq!(v.ours.as_deref(), Some("1\nmine\n3\n4\n5\n")); @@ -1049,7 +1052,7 @@ fn partial_commit_of_chosen_hunks_leaves_worktree_and_index_alone() { partial: vec![PartialCommitFile { path: fx.path("f.txt").into(), hunks: vec![range(&hunks[0])] }], ..Default::default() }; - repo.commit_with("partial", &options).unwrap(); + repo.commit_with("partial", &options, &|_| Ok(encoding_rs::UTF_8)).unwrap(); assert_eq!(fx.git(&["show", "HEAD:f.txt"]), "A\nb\nc\nd\ne\nf\ng\nh\ni\nj"); assert_eq!(fx.git(&["show", "HEAD:other.txt"]), "o2"); @@ -1070,21 +1073,21 @@ fn partial_commit_of_chosen_hunks_leaves_worktree_and_index_alone() { partial: vec![PartialCommitFile { path: fx.path("f.txt").into(), hunks: vec![range(&hunks[0])] }], ..Default::default() }; - let err = repo.commit_with("stale", &stale).unwrap_err(); + let err = repo.commit_with("stale", &stale, &|_| Ok(encoding_rs::UTF_8)).unwrap_err(); assert!(err.message.contains("changed"), "{}", err.message); assert_eq!(fx.git(&["rev-parse", "HEAD"]), before); assert_eq!(fx.git(&["diff", "--cached", "--name-only"]), ""); assert!(!fx.dir.join(".git/openidea-partial-index").exists()); // The remaining hunk commits on its own; then nothing is left. - let rest = diff_hunks(&repo.file_at(&fx.path("f.txt"), None).unwrap().unwrap(), edited); + let rest = diff_hunks(&repo.file_at(&fx.path("f.txt"), None, encoding_rs::UTF_8).unwrap().unwrap(), edited); assert_eq!(rest.len(), 1); let options = CommitOptions { partial: vec![PartialCommitFile { path: fx.path("f.txt").into(), hunks: vec![range(&rest[0])] }], ..Default::default() }; - repo.commit_with("rest", &options).unwrap(); - assert_eq!(repo.file_at(&fx.path("f.txt"), None).unwrap().as_deref(), Some(edited)); + repo.commit_with("rest", &options, &|_| Ok(encoding_rs::UTF_8)).unwrap(); + assert_eq!(repo.file_at(&fx.path("f.txt"), None, encoding_rs::UTF_8).unwrap().as_deref(), Some(edited)); assert!(repo.status().unwrap().iter().all(|f| f.rel_path != "f.txt")); assert_eq!(fx.read("staged.txt"), "s2\n"); @@ -1105,7 +1108,7 @@ fn partial_commit_of_chosen_hunks_leaves_worktree_and_index_alone() { partial: vec![PartialCommitFile { path: fx.path("f.txt").into(), hunks: vec![range(&hunks[0])] }], ..Default::default() }; - assert!(repo.commit_with("hooked", &options).is_err()); + assert!(repo.commit_with("hooked", &options, &|_| Ok(encoding_rs::UTF_8)).is_err()); assert_eq!(fx.git(&["rev-parse", "HEAD"]), head); assert_eq!(fx.git(&["diff", "--cached", "--name-only"]), ""); assert_eq!(fx.read("f.txt"), "Z\nb\nc\nd\ne\nf\ng\nh\ni\nJ\nk\nl\n"); @@ -1121,13 +1124,13 @@ fn partial_staging_with_built_patch() { let new = "l0\nl1\nl2\nCHANGED\nl4\nl5\nl6\nl8\nl9\nl10\n"; fx.write("p.txt", new); - let index_text = repo.file_at(&fx.path("p.txt"), Some(":0")).unwrap().unwrap(); + let index_text = repo.file_at(&fx.path("p.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap(); let hunks = ide_vcs::diff_hunks(&index_text, new); assert_eq!(hunks.len(), 4, "{hunks:#?}"); // Stage the modification and the trailing addition only. let patch = build_partial_patch("p.txt", &index_text, new, &[1, 3]); - repo.stage_patch(&patch).unwrap(); - let staged = repo.file_at(&fx.path("p.txt"), Some(":0")).unwrap().unwrap(); + repo.stage_patch(&patch, &|_| Ok(encoding_rs::UTF_8)).unwrap(); + let staged = repo.file_at(&fx.path("p.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap(); assert_eq!(staged, "l1\nl2\nCHANGED\nl4\nl5\nl6\nl7\nl8\nl9\nl10\n"); let st = repo.status().unwrap(); let s = status_of(&st, "p.txt"); @@ -1135,21 +1138,21 @@ fn partial_staging_with_built_patch() { // Then the rest, one hunk at a time against the updated index. loop { - let index_text = repo.file_at(&fx.path("p.txt"), Some(":0")).unwrap().unwrap(); + let index_text = repo.file_at(&fx.path("p.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap(); if ide_vcs::diff_hunks(&index_text, new).is_empty() { break; } - repo.stage_patch(&build_partial_patch("p.txt", &index_text, new, &[0])).unwrap(); + repo.stage_patch(&build_partial_patch("p.txt", &index_text, new, &[0]), &|_| Ok(encoding_rs::UTF_8)).unwrap(); } - assert_eq!(repo.file_at(&fx.path("p.txt"), Some(":0")).unwrap().unwrap(), new); + assert_eq!(repo.file_at(&fx.path("p.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap(), new); // Patches also apply to files without a trailing newline and CRLF files. fx.write("n.txt", "a\r\nb\r\nc"); fx.commit_all("crlf"); let new = "a\nB\nc\nd"; - let old = repo.file_at(&fx.path("n.txt"), Some(":0")).unwrap().unwrap(); - repo.stage_patch(&build_partial_patch("n.txt", &old, new, &[0, 1])).unwrap(); - assert_eq!(repo.file_at(&fx.path("n.txt"), Some(":0")).unwrap().unwrap(), "a\r\nB\r\nc\r\nd"); + let old = repo.file_at(&fx.path("n.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap(); + repo.stage_patch(&build_partial_patch("n.txt", &old, new, &[0, 1]), &|_| Ok(encoding_rs::UTF_8)).unwrap(); + assert_eq!(repo.file_at(&fx.path("n.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap(), "a\r\nB\r\nc\r\nd"); } #[test] @@ -1170,13 +1173,13 @@ fn discard_and_diff_text() { fx.write("keep/new.txt", "n\n"); fx.write("keep/k.txt", "changed\n"); - let unstaged = repo.diff_text(&fx.path("both.txt"), false).unwrap(); + let unstaged = repo.diff_text(&fx.path("both.txt"), false, encoding_rs::UTF_8).unwrap(); assert!(unstaged.contains("-staged\n+unstaged"), "{unstaged}"); - let staged = repo.diff_text(&fx.path("both.txt"), true).unwrap(); + let staged = repo.diff_text(&fx.path("both.txt"), true, encoding_rs::UTF_8).unwrap(); assert!(staged.contains("-one\n+staged"), "{staged}"); - let untracked = repo.diff_text(&fx.path("u.txt"), false).unwrap(); + let untracked = repo.diff_text(&fx.path("u.txt"), false, encoding_rs::UTF_8).unwrap(); assert!(untracked.contains("+untracked"), "{untracked}"); - assert!(repo.diff_text(&fx.path("t.txt"), true).unwrap().is_empty()); + assert!(repo.diff_text(&fx.path("t.txt"), true, encoding_rs::UTF_8).unwrap().is_empty()); repo.discard(&[fx.path("t.txt"), fx.path("both.txt"), fx.path("u.txt"), fx.path("udir"), fx.path("keep")]).unwrap(); assert_eq!(fx.read("t.txt"), "one\n"); @@ -1322,3 +1325,161 @@ fn path_log_simplifies_merges() { .collect(); assert_eq!(ids, vec![base]); } + +#[test] +fn gbk_java_unchanged_has_no_hunks() { + let fx = Fixture::new(); + let text = "// 中文注释\nclass Example {\n // 保持不变\n}\n"; + std::fs::write(fx.path("Example.java"), encoding_rs::GBK.encode(text).0).unwrap(); + fx.commit_all("GBK Java"); + for against_index in [false, true] { + assert!( + fx.repo().line_hunks(&fx.path("Example.java"), text, against_index, encoding_rs::GBK).unwrap().is_empty() + ); + } +} + +#[test] +fn encoded_java_hunks_staging_blame_and_partial_commit_preserve_bytes() { + for encoding in [encoding_rs::GBK, encoding_rs::SHIFT_JIS, encoding_rs::UTF_8] { + for eol in ["\n", "\r\n"] { + let fx = Fixture::new(); + let path = fx.path("中文.java"); + let old = "// 中文注釈\nclass Example {\n // 保存\n int value = 1;\n}\n".replace('\n', eol); + let changed = old.replace("中文注釈", "中文変更").replace("value = 1", "value = 2"); + let bytes = |text: &str| { + let (bytes, _, errors) = encoding.encode(text); + assert!(!errors); + bytes.into_owned() + }; + std::fs::write(&path, bytes(&old)).unwrap(); + fx.commit_all("encoded Java"); + let repo = fx.repo(); + assert_eq!(repo.file_at(&path, None, encoding).unwrap().as_deref(), Some(old.as_str())); + assert!( + repo.blame(&path, Some(&old.replace("\r\n", "\n")), encoding) + .unwrap() + .iter() + .all(|line| line.author == "Test User") + ); + let hunks = repo.line_hunks(&path, &old.replace("value = 1", "value = 2"), false, encoding).unwrap(); + assert_eq!(hunks.len(), 1); + assert_eq!((hunks[0].start, hunks[0].lines, hunks[0].kind), (3, 1, HunkKind::Modified)); + std::fs::write(&path, bytes(&changed)).unwrap(); + let hunks = repo.line_hunks(&path, &changed, false, encoding).unwrap(); + assert_eq!(hunks.len(), 2); + repo.stage_hunks(&path, &changed, &[0], encoding).unwrap(); + let selected = old.replace("中文注釈", "中文変更"); + assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), bytes(&selected)); + assert_eq!(std::fs::read(&path).unwrap(), bytes(&changed)); + assert!(repo.diff_text(&path, true, encoding).unwrap().contains("中文変更")); + repo.unstage(std::slice::from_ref(&path)).unwrap(); + let options = CommitOptions { + partial: vec![PartialCommitFile { + path: path.clone().into(), + hunks: vec![HunkRange { + start: hunks[0].start, + lines: hunks[0].lines, + old_start: hunks[0].old_start, + old_lines: hunks[0].old_lines, + }], + }], + ..Default::default() + }; + repo.commit_with("selected hunk", &options, &|_| Ok(encoding)).unwrap(); + assert_eq!(repo.file_bytes_at(&path, None).unwrap().unwrap(), bytes(&selected)); + assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), bytes(&selected)); + assert_eq!(std::fs::read(&path).unwrap(), bytes(&changed)); + } + } +} + +#[test] +fn gbk_patches_round_trip_even_with_text_attributes() { + let fx = Fixture::new(); + let path = fx.path("Example.java"); + let old = encoding_rs::GBK.encode("// 中文注释\nclass Example {}\n").0.into_owned(); + let new = encoding_rs::GBK.encode("// 修改注释\nclass Example {}\n").0.into_owned(); + fx.write(".gitattributes", "*.java diff\n"); + std::fs::write(&path, &old).unwrap(); + fx.commit_all("base"); + std::fs::write(&path, &new).unwrap(); + let repo = fx.repo(); + let patch = repo.create_patch(std::slice::from_ref(&path), false).unwrap(); + assert!(patch.contains("GIT binary patch")); + repo.discard(std::slice::from_ref(&path)).unwrap(); + repo.apply_patch(&patch).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), new); + let commit = fx.commit_all("change"); + let reverse = repo.create_commits_patch(&[commit], true).unwrap(); + repo.apply_patch(&reverse).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), old); +} + +#[test] +fn gbk_conflict_stages_are_readable() { + let fx = Fixture::new(); + let path = fx.path("Example.java"); + let write = |text: &str| std::fs::write(&path, encoding_rs::GBK.encode(text).0).unwrap(); + write("// 原始注释\n"); + fx.commit_all("base"); + fx.git(&["checkout", "-b", "other"]); + write("// 分支注释\n"); + fx.commit_all("other"); + fx.git(&["checkout", "main"]); + write("// 当前注释\n"); + fx.commit_all("ours"); + let out = Command::new("git").current_dir(&fx.dir).args(["merge", "other"]).output().unwrap(); + assert!(!out.status.success()); + let versions = fx.repo().merge_versions(&path, encoding_rs::GBK).unwrap(); + assert_eq!(versions.base.as_deref(), Some("// 原始注释\n")); + assert_eq!(versions.ours.as_deref(), Some("// 当前注释\n")); + assert_eq!(versions.theirs.as_deref(), Some("// 分支注释\n")); +} + +#[test] +fn utf16_and_utf8_boms_survive_partial_staging() { + for encoding in [encoding_rs::UTF_8, encoding_rs::UTF_16LE, encoding_rs::UTF_16BE] { + let fx = Fixture::new(); + let path = fx.path("Example.java"); + let old = "// 中文注释\r\nclass Example {}\r\n"; + let changed = old.replace("中文", "修改"); + let bytes = |text: &str| { + if encoding == encoding_rs::UTF_8 { + let mut bytes = vec![0xef, 0xbb, 0xbf]; + bytes.extend_from_slice(text.as_bytes()); + bytes + } else { + let le = encoding == encoding_rs::UTF_16LE; + let mut bytes = if le { vec![0xff, 0xfe] } else { vec![0xfe, 0xff] }; + for unit in text.encode_utf16() { + bytes.extend_from_slice(&if le { unit.to_le_bytes() } else { unit.to_be_bytes() }); + } + bytes + } + }; + std::fs::write(&path, bytes(old)).unwrap(); + fx.commit_all("BOM"); + let repo = fx.repo(); + assert!(repo.line_hunks(&path, old, false, encoding).unwrap().is_empty()); + repo.stage_hunks(&path, &changed, &[0], encoding).unwrap(); + assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), bytes(&changed)); + } +} + +#[test] +fn gbk_text_patch_keeps_utf8_paths_and_encoded_payload() { + let fx = Fixture::new(); + let path = fx.path("中文.java"); + let old = "// 中文注释\n"; + let new = "// 修改注释\n"; + std::fs::write(&path, encoding_rs::GBK.encode(old).0).unwrap(); + fx.commit_all("GBK"); + let repo = fx.repo(); + let patch = build_partial_patch("中文.java", old, new, &[0]); + repo.stage_patch(&patch, &|_| Ok(encoding_rs::GBK)).unwrap(); + assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), encoding_rs::GBK.encode(new).0.as_ref()); + let patch = build_partial_patch("中文.java", new, "// 😀\n", &[0]); + assert!(repo.stage_patch(&patch, &|_| Ok(encoding_rs::GBK)).is_err()); + assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), encoding_rs::GBK.encode(new).0.as_ref()); +} diff --git a/crates/ide-vfs/src/lib.rs b/crates/ide-vfs/src/lib.rs index 24c882e70..fa56b02c2 100644 --- a/crates/ide-vfs/src/lib.rs +++ b/crates/ide-vfs/src/lib.rs @@ -313,6 +313,21 @@ impl FsBatch { } } +/// Detect a revision's encoding with the same rules as files opened in the editor. +pub fn detect_encoding(bytes: &[u8], fallback: Option<&str>) -> &'static encoding_rs::Encoding { + let (_, detected, has_bom, binary) = encoding::decode(bytes); + if !binary + && !has_bom + && detected != encoding_rs::UTF_8 + && detected != encoding_rs::UTF_16LE + && detected != encoding_rs::UTF_16BE + { + fallback.and_then(|label| encoding_rs::Encoding::for_label(label.trim().as_bytes())).unwrap_or(detected) + } else { + detected + } +} + #[cfg(test)] mod tests { use super::*;