Skip to content

Improve/modernize the handling of utf16 data in std::str #12317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

Improve/modernize the handling of utf16 data in std::str #12317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
493a4b6
std: iteratize str::is_utf16 & add tests.
huonw Feb 16, 2014
b7656d0
std: convert str::from_utf16 to an external iterator.
huonw Feb 16, 2014
a96cea4
str: provide lossy UTF-16 support.
huonw Feb 16, 2014
35b1b62
std: decode even numbered non-BMP planes in the UTF-16 decoder.
huonw Feb 16, 2014
4f841ee
std: make str::from_utf16 return an Option.
huonw Feb 16, 2014
c9b4538
str: add a function for truncating a vector of u16 at NUL.
huonw Feb 18, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/libnative/io/file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,9 @@ pub fn readdir(p: &CString) -> IoResult<~[Path]> {
else {
let fp_vec = vec::from_buf(
fp_buf, wcslen(fp_buf) as uint);
let fp_str = str::from_utf16(fp_vec);
let fp_trimmed = str::truncate_utf16_at_nul(fp_vec);
let fp_str = str::from_utf16(fp_trimmed)
.expect("rust_list_dir_wfd_fp_buf returned invalid UTF-16");
paths.push(Path::new(fp_str));
}
more_files = FindNextFileW(find_handle, wfd_ptr as HANDLE);
Expand Down
19 changes: 14 additions & 5 deletions src/libstd/os.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ pub fn getcwd() -> Path {
fail!();
}
}
Path::new(str::from_utf16(buf))
Path::new(str::from_utf16(str::truncate_utf16_at_nul(buf))
.expect("GetCurrentDirectoryW returned invalid UTF-16"))
}

#[cfg(windows)]
Expand Down Expand Up @@ -124,7 +125,12 @@ pub mod win32 {
}
if k != 0 && done {
let sub = buf.slice(0, k as uint);
res = option::Some(str::from_utf16(sub));
// We want to explicitly catch the case when the
// closure returned invalid UTF-16, rather than
// set `res` to None and continue.
let s = str::from_utf16(sub)
.expect("fill_utf16_buf_and_decode: closure created invalid UTF-16");
res = option::Some(s)
}
}
return res;
Expand Down Expand Up @@ -739,7 +745,8 @@ pub fn last_os_error() -> ~str {
fail!("[{}] FormatMessage failure", errno());
}

str::from_utf16(buf)
str::from_utf16(str::truncate_utf16_at_nul(buf))
.expect("FormatMessageW returned invalid UTF-16")
}
}

Expand Down Expand Up @@ -828,8 +835,10 @@ fn real_args() -> ~[~str] {
while *ptr.offset(len as int) != 0 { len += 1; }

// Push it onto the list.
args.push(vec::raw::buf_as_slice(ptr, len,
str::from_utf16));
let opt_s = vec::raw::buf_as_slice(ptr, len, |buf| {
str::from_utf16(str::truncate_utf16_at_nul(buf))
});
args.push(opt_s.expect("CommandLineToArgvW returned invalid UTF-16"));
}
}

Expand Down
Loading