syntex: Copy unstable librustc_unicode fns into libsyntax

Author: erickt

src/libsyntax/lib.rs

@@ -29,7 +29,6 @@
 #![feature(libc)]
 #![feature(rustc_private)]
 #![feature(staged_api)]
-#![feature(unicode)]
 
 extern crate arena;
 extern crate fmt_macros;
@@ -93,6 +92,7 @@ pub mod std_inject;
 pub mod str;
 pub mod test;
 pub mod visit;
+pub mod unicode;
 
 pub mod print {
     pub mod pp;

src/libsyntax/parse/lexer/mod.rs

@@ -16,6 +16,7 @@ use ext::tt::transcribe::tt_next_token;
 use parse::token::str_to_ident;
 use parse::token;
 use str::{char_at, escape_default};
+use unicode::char::{is_xid_start, is_xid_continue};
 
 use std::borrow::Cow;
 use std::char;
@@ -718,8 +719,10 @@ impl<'a> StringReader<'a> {
         // might be a float, but don't be greedy if this is actually an
         // integer literal followed by field/method access or a range pattern
         // (`0..2` and `12.foo()`)
-        if self.curr_is('.') && !self.nextch_is('.') && !self.nextch().unwrap_or('\0')
-                                                             .is_xid_start() {
+        if self.curr_is('.') &&
+            !self.nextch_is('.') &&
+            !is_xid_start(self.nextch().unwrap_or('\0'))
+        {
             // might have stuff after the ., and if it does, it needs to start
             // with a number
             self.bump();
@@ -1487,7 +1490,7 @@ fn ident_start(c: Option<char>) -> bool {
     (c >= 'a' && c <= 'z')
         || (c >= 'A' && c <= 'Z')
         || c == '_'
-        || (c > '\x7f' && c.is_xid_start())
+        || (c > '\x7f' && is_xid_start(c))
 }
 
 fn ident_continue(c: Option<char>) -> bool {
@@ -1497,7 +1500,7 @@ fn ident_continue(c: Option<char>) -> bool {
         || (c >= 'A' && c <= 'Z')
         || (c >= '0' && c <= '9')
         || c == '_'
-        || (c > '\x7f' && c.is_xid_continue())
+        || (c > '\x7f' && is_xid_continue(c))
 }
 
 #[cfg(test)]

src/libsyntax/unicode/char.rs

@@ -0,0 +1,50 @@
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use super::tables::{derived_property, charwidth};
+
+/// Returns whether the specified character satisfies the 'XID_Start'
+/// Unicode property.
+///
+/// 'XID_Start' is a Unicode Derived Property specified in
+/// [UAX #31](http://unicode.org/reports/tr31/#NFKC_Modifications),
+/// mostly similar to ID_Start but modified for closure under NFKx.
+#[unstable(feature = "unicode",
+           reason = "mainly needed for compiler internals")]
+#[inline]
+pub fn is_xid_start(ch: char) -> bool { derived_property::XID_Start(ch) }
+
+/// Returns whether the specified `char` satisfies the 'XID_Continue'
+/// Unicode property.
+///
+/// 'XID_Continue' is a Unicode Derived Property specified in
+/// [UAX #31](http://unicode.org/reports/tr31/#NFKC_Modifications),
+/// mostly similar to 'ID_Continue' but modified for closure under NFKx.
+#[unstable(feature = "unicode",
+           reason = "mainly needed for compiler internals")]
+#[inline]
+pub fn is_xid_continue(ch: char) -> bool { derived_property::XID_Continue(ch) }
+
+/// Returns this character's displayed width in columns, or `None` if it is a
+/// control character other than `'\x00'`.
+///
+/// `is_cjk` determines behavior for characters in the Ambiguous category:
+/// if `is_cjk` is `true`, these are 2 columns wide; otherwise, they are 1.
+/// In CJK contexts, `is_cjk` should be `true`, else it should be `false`.
+/// [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/)
+/// recommends that these characters be treated as 1 column (i.e.,
+/// `is_cjk` = `false`) if the context cannot be reliably determined.
+#[deprecated(reason = "use the crates.io `unicode-width` library instead",
+             since = "1.0.0")]
+#[unstable(feature = "unicode",
+           reason = "needs expert opinion. is_cjk flag stands out as ugly")]
+pub fn width(ch: char, is_cjk: bool) -> Option<usize> {
+    charwidth::width(ch, is_cjk)
+}

src/libsyntax/unicode/mod.rs

@@ -0,0 +1,12 @@
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+pub mod char;
+mod tables;