fix(properties): keep existing Unicode escapes when saving with native-to-ascii (#650)

With Transparent native-to-ascii conversion on, pasting or typing existing \uXXXX escapes into a .properties file and saving wrote them back as \u005CuXXXX, so the file on disk changed meaning.

Root cause: the shared properties encoder escaped the backslash of any valid ASCII Unicode escape. It now writes ASCII escapes as they are and only converts non-ASCII characters; a literal backslash right before a non-ASCII character is still protected. The same encoder serves every write path, so normal/auto save, find and replace in open and closed files, workspace edits, Local History restore and partial Git stage/commit are all covered.

Verified on macOS following the reporter's steps (paste escapes, save, reload externally); Windows and Linux not run.

Fixes #519

crates/ide-host/tests/properties.rs

@@ -104,7 +104,7 @@ async fn properties_search_git_and_document_coordinates_agree() {
         "main",
         opened.id,
         opened.version,
-        ChangeSet { changes: vec![TextChange { from: 10, to: 16, insert: "世界".into() }] },
+        ChangeSet { changes: vec![TextChange { from: 10, to: 16, insert: r"\u4E16\u754C".into() }] },
     )
     .unwrap();
     let hunks = vcs::vcs_line_hunks(&host, "main", path.clone()).await.unwrap();
@@ -129,7 +129,7 @@ async fn properties_search_git_and_document_coordinates_agree() {
     assert!(project.documents.snapshot(opened.id).unwrap().version > snap.version);
     assert!(vcs::vcs_line_hunks(&host, "main", path.clone()).await.unwrap().is_empty());
     // Partial commits identify hunks in displayed line coordinates as well.
-    let partial_text = displayed.replace("target", "世界").replace("logical", "changed");
+    let partial_text = displayed.replace("target", r"\u4E16\u754C").replace("logical", "changed");
     project.documents.replace_all(opened.id, &partial_text).unwrap();
     doc::save(&host, "main", opened.id).await.unwrap();
     let hunks = ide_vcs::diff_hunks(displayed, &partial_text);
@@ -169,7 +169,7 @@ async fn properties_search_git_and_document_coordinates_agree() {
         "main",
         RequestId(520),
         query,
-        "世界".into(),
+        r"\u4E16\u754C".into(),
         false,
         vec![ide_search::ReplaceTarget {
             path: closed_path.clone().into(),
@@ -181,5 +181,6 @@ async fn properties_search_git_and_document_coordinates_agree() {
     .unwrap();
     assert_eq!(std::fs::read_to_string(&closed_path).unwrap(), r"value=\u4F60\u597D\uD83D\uDE00\u4E16\u754C");
     assert!(project.documents.by_path(&closed_path).is_none());
+    assert_eq!(project.text_of(&closed_path).unwrap(), "value=你好😀世界");
     host.close_project("main");
 }

crates/ide-vcs/src/text_encoding.rs

@@ -3,7 +3,7 @@ use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE};
 use ide_core::{IdeError, IdeResult};
 use std::borrow::Cow;
 
-/// File charset plus an optional reversible ASCII representation (for properties).
+/// File charset plus an optional ASCII representation (for properties).
 /// The host supplies conversion functions so Git and documents share one codec.
 #[derive(Clone, Copy)]
 pub struct TextEncoding {

crates/ide-vfs/src/encoding.rs

@@ -62,20 +62,19 @@ pub fn decode_properties_escapes(text: &str) -> String {
 }
 
 /// Serializes displayed properties text as ASCII, using uppercase UTF-16 units.
+/// Existing ASCII Unicode escapes retain their spelling and meaning on disk.
 pub fn encode_properties_ascii(text: &str) -> String {
     use std::fmt::Write;
     let mut out = String::with_capacity(text.len());
     let mut rest = text;
     while let Some(c) = rest.chars().next() {
-        let current = rest;
         rest = &rest[c.len_utf8()..];
         if c == '\\' {
             if rest.starts_with('\\') {
                 out.push_str("\\\\");
                 rest = &rest[1..];
-            } else if unicode_character(current).is_some() || rest.chars().next().is_some_and(|next| !next.is_ascii()) {
-                // Keep a literal backslash from swallowing a generated escape,
-                // or becoming a Unicode escape on the next load.
+            } else if rest.chars().next().is_some_and(|next| !next.is_ascii()) {
+                // Keep a literal backslash from swallowing a generated escape.
                 out.push_str("\\u005C");
             } else {
                 out.push(c);
@@ -387,13 +386,32 @@ mod tests {
         for text in ["plain=ASCII\r\n", r"escaped=\\u0041", r"bad=\u12xz", r"bad=\uD800"] {
             assert_eq!(encode_properties_ascii(text), text);
         }
-        for text in ["你好😀", "\0", "\\中", "\\\\中", "\\\\\\😀", r"\u0041", r"\\u0041", r"\uD83D\uDE00"] {
+        for text in ["你好😀", "\0", "\\中", "\\\\中", "\\\\\\😀", r"\\u0041"] {
             let ascii = encode_properties_ascii(text);
             assert!(ascii.is_ascii());
             assert_eq!(decode_properties_escapes(&ascii), text, "{ascii}");
         }
     }
 
+    #[test]
+    fn properties_ascii_escapes_are_not_escaped_again() {
+        for text in [
+            r"test.key2=\u4F60\u597D",
+            r"test.key3=\u4f60\u597d",
+            r"ascii=\u0041\u005C\u000A",
+            r"emoji=\uD83D\uDE00",
+            r"slashes=\\u4F60 \\\u4F60 \\\\u4F60",
+            r"bad=\u123 \u12xz \uu0041 \uD800 \uDC00",
+        ] {
+            assert_eq!(encode_properties_ascii(text), text, "{text}");
+        }
+        let mixed = "raw=\\u4f60\\u597d; native=你好😀";
+        let expected = r"raw=\u4f60\u597d; native=\u4F60\u597D\uD83D\uDE00";
+        assert_eq!(encode_properties_ascii(mixed), expected);
+        assert_eq!(encode_properties_ascii(expected), expected);
+        assert_eq!(decode_properties_escapes(expected), "raw=你好; native=你好😀");
+    }
+
     #[test]
     fn properties_line_terminator_escapes_survive_an_edit() {
         let source = r"key=one\u000Atwo\u000Dthree";

crates/ide-workspace/tests/documents.rs

@@ -641,6 +641,57 @@ fn properties_conversion_uses_displayed_utf16_and_survives_reload() {
     assert!(!f.docs.is_dirty(doc.id).unwrap());
 }
 
+#[test]
+fn properties_pasted_escapes_survive_save_and_decode_on_reload() {
+    let pasted = "test.key1=Hello\ntest.key2=\\u4F60\\u597D\ntest.key3=\\u4f60\\u597d\n";
+    let displayed = "test.key1=Hello\ntest.key2=你好\ntest.key3=你好\n";
+    for encoding in ["utf-8", "iso-8859-1", "utf-16le"] {
+        for eol in ["\n", "\r\n"] {
+            let f = fixture();
+            let path = f.root.join("test_zh_CN.properties");
+            let source = pasted.replace('\n', eol);
+            fs::write(&path, ide_vfs::encode(&source, encoding, encoding == "utf-16le").unwrap()).unwrap();
+            let mut defaults = f.docs.file_defaults();
+            defaults.native_to_ascii = true;
+            defaults.properties_fallback = encoding.into();
+            f.docs.set_file_defaults(defaults.clone());
+            let doc = f.docs.open(&path).unwrap();
+            assert_eq!(doc.text, displayed);
+
+            f.docs.replace_all(doc.id, pasted).unwrap();
+            f.docs.save(doc.id).unwrap();
+            assert_eq!(fs::read(&path).unwrap(), source.as_bytes(), "{encoding} {eol:?}");
+            assert_eq!(f.docs.snapshot(doc.id).unwrap().text, pasted);
+            assert!(!f.docs.is_dirty(doc.id).unwrap());
+            f.docs.on_disk_changed(std::slice::from_ref(&path));
+            assert_eq!(f.docs.snapshot(doc.id).unwrap().text, pasted, "own saves do not reload");
+
+            f.docs.close(doc.id);
+            let reopened = f.docs.open(&path).unwrap();
+            assert_eq!(reopened.text, displayed);
+            // An edit followed by its inverse preserves the original escape casing.
+            f.docs.replace_all(reopened.id, pasted).unwrap();
+            f.docs.replace_all(reopened.id, displayed).unwrap();
+            f.docs.save(reopened.id).unwrap();
+            assert_eq!(fs::read(&path).unwrap(), source.as_bytes());
+
+            let external = format!("{source}test.key4=\\uD83D\\uDE00{eol}");
+            fs::write(&path, &external).unwrap();
+            f.docs.on_disk_changed(std::slice::from_ref(&path));
+            assert_eq!(f.docs.snapshot(reopened.id).unwrap().text, format!("{displayed}test.key4=😀\n"));
+            defaults.native_to_ascii = false;
+            f.docs.set_file_defaults(defaults.clone());
+            f.docs.reload_properties();
+            assert_eq!(f.docs.snapshot(reopened.id).unwrap().text, external.replace("\r\n", "\n"));
+            defaults.native_to_ascii = true;
+            f.docs.set_file_defaults(defaults);
+            f.docs.reload_properties();
+            assert_eq!(f.docs.snapshot(reopened.id).unwrap().text, format!("{displayed}test.key4=😀\n"));
+            assert_eq!(fs::read_to_string(&path).unwrap(), external);
+        }
+    }
+}
+
 #[test]
 fn properties_toggle_reloads_only_clean_properties_and_pins_dirty_codec() {
     let f = fixture();