Parse and store suffixes on literals.

This adds an optional suffix at the end of a literal token:
`"foo"bar`. An actual use of a suffix in a expression (or other literal
that the compiler reads) is rejected in the parser.

This doesn't switch the handling of numbers to this system, and doesn't
outlaw illegal suffixes for them yet.

Author: huonw

src/librustdoc/html/highlight.rs

@@ -128,13 +128,17 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
                 }
             }
 
-            // text literals
-            token::Literal(token::Byte(..)) | token::Literal(token::Char(..)) |
-                token::Literal(token::Binary(..)) | token::Literal(token::BinaryRaw(..)) |
-                token::Literal(token::Str_(..)) | token::Literal(token::StrRaw(..)) => "string",
-
-            // number literals
-            token::Literal(token::Integer(..)) | token::Literal(token::Float(..)) => "number",
+            token::Literal(lit, _suf) => {
+                match lit {
+                    // text literals
+                    token::Byte(..) | token::Char(..) |
+                        token::Binary(..) | token::BinaryRaw(..) |
+                        token::Str_(..) | token::StrRaw(..) => "string",
+
+                    // number literals
+                    token::Integer(..) | token::Float(..) => "number",
+                }
+            }
 
             // keywords are also included in the identifier set
             token::Ident(ident, _is_mod_sep) => {

src/libsyntax/ast.rs

@@ -838,7 +838,7 @@ impl TokenTree {
                     tts: vec![TtToken(sp, token::Ident(token::str_to_ident("doc"),
                                                        token::Plain)),
                               TtToken(sp, token::Eq),
-                              TtToken(sp, token::Literal(token::Str_(name)))],
+                              TtToken(sp, token::Literal(token::Str_(name), None))],
                     close_span: sp,
                 }))
             }

src/libsyntax/diagnostics/plugin.rs

@@ -87,7 +87,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
         },
         [ast::TtToken(_, token::Ident(ref code, _)),
          ast::TtToken(_, token::Comma),
-         ast::TtToken(_, token::Literal(token::StrRaw(description, _)))] => {
+         ast::TtToken(_, token::Literal(token::StrRaw(description, _), None))] => {
             (code, Some(description))
         }
         _ => unreachable!()

src/libsyntax/ext/quote.rs

@@ -543,10 +543,13 @@ fn mk_delim(cx: &ExtCtxt, sp: Span, delim: token::DelimToken) -> P<ast::Expr> {
 #[allow(non_upper_case_globals)]
 fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
     macro_rules! mk_lit {
-        ($name: expr, $($args: expr),*) => {{
+        ($name: expr, $suffix: expr, $($args: expr),*) => {{
             let inner = cx.expr_call(sp, mk_token_path(cx, sp, $name), vec![$($args),*]);
-
-            cx.expr_call(sp, mk_token_path(cx, sp, "Literal"), vec![inner])
+            let suffix = match $suffix {
+                Some(name) => cx.expr_some(sp, mk_name(cx, sp, ast::Ident::new(name))),
+                None => cx.expr_none(sp)
+            };
+            cx.expr_call(sp, mk_token_path(cx, sp, "Literal"), vec![inner, suffix])
         }}
     }
     match *tok {
@@ -567,32 +570,32 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
                                 vec![mk_delim(cx, sp, delim)]);
         }
 
-        token::Literal(token::Byte(i)) => {
+        token::Literal(token::Byte(i), suf) => {
             let e_byte = mk_name(cx, sp, i.ident());
-            return mk_lit!("Byte", e_byte);
+            return mk_lit!("Byte", suf, e_byte);
         }
 
-        token::Literal(token::Char(i)) => {
+        token::Literal(token::Char(i), suf) => {
             let e_char = mk_name(cx, sp, i.ident());
-            return mk_lit!("Char", e_char);
+            return mk_lit!("Char", suf, e_char);
         }
 
-        token::Literal(token::Integer(i)) => {
+        token::Literal(token::Integer(i), suf) => {
             let e_int = mk_name(cx, sp, i.ident());
-            return mk_lit!("Integer", e_int);
+            return mk_lit!("Integer", suf, e_int);
         }
 
-        token::Literal(token::Float(fident)) => {
+        token::Literal(token::Float(fident), suf) => {
             let e_fident = mk_name(cx, sp, fident.ident());
-            return mk_lit!("Float", e_fident);
+            return mk_lit!("Float", suf, e_fident);
         }
 
-        token::Literal(token::Str_(ident)) => {
-            return mk_lit!("Str_", mk_name(cx, sp, ident.ident()))
+        token::Literal(token::Str_(ident), suf) => {
+            return mk_lit!("Str_", suf, mk_name(cx, sp, ident.ident()))
         }
 
-        token::Literal(token::StrRaw(ident, n)) => {
-            return mk_lit!("StrRaw", mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n))
+        token::Literal(token::StrRaw(ident, n), suf) => {
+            return mk_lit!("StrRaw", suf, mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n))
         }
 
         token::Ident(ident, style) => {

src/libsyntax/parse/lexer/mod.rs

@@ -369,6 +369,25 @@ impl<'a> StringReader<'a> {
         self.nextnextch() == Some(c)
     }
 
+    /// Eats <XID_start><XID_continue>*, if possible.
+    fn scan_optional_raw_name(&mut self) -> Option<ast::Name> {
+        if !ident_start(self.curr) {
+            return None
+        }
+        let start = self.last_pos;
+        while ident_continue(self.curr) {
+            self.bump();
+        }
+
+        self.with_str_from(start, |string| {
+            if string == "_" {
+                None
+            } else {
+                Some(token::intern(string))
+            }
+        })
+    }
+
     /// PRECONDITION: self.curr is not whitespace
     /// Eats any kind of comment.
     fn scan_comment(&mut self) -> Option<TokenAndSpan> {
@@ -638,7 +657,7 @@ impl<'a> StringReader<'a> {
     }
 
     /// Lex a LIT_INTEGER or a LIT_FLOAT
-    fn scan_number(&mut self, c: char) -> token::Token {
+    fn scan_number(&mut self, c: char) -> token::Lit {
         let mut num_digits;
         let mut base = 10;
         let start_bpos = self.last_pos;
@@ -655,17 +674,17 @@ impl<'a> StringReader<'a> {
                 }
                 'u' | 'i' => {
                     self.scan_int_suffix();
-                    return token::Literal(token::Integer(self.name_from(start_bpos)));
+                    return token::Integer(self.name_from(start_bpos));
                 },
                 'f' => {
                     let last_pos = self.last_pos;
                     self.scan_float_suffix();
                     self.check_float_base(start_bpos, last_pos, base);
-                    return token::Literal(token::Float(self.name_from(start_bpos)));
+                    return token::Float(self.name_from(start_bpos));
                 }
                 _ => {
                     // just a 0
-                    return token::Literal(token::Integer(self.name_from(start_bpos)));
+                    return token::Integer(self.name_from(start_bpos));
                 }
             }
         } else if c.is_digit_radix(10) {
@@ -678,7 +697,7 @@ impl<'a> StringReader<'a> {
             self.err_span_(start_bpos, self.last_pos, "no valid digits found for number");
             // eat any suffix
             self.scan_int_suffix();
-            return token::Literal(token::Integer(token::intern("0")));
+            return token::Integer(token::intern("0"));
         }
 
         // might be a float, but don't be greedy if this is actually an
@@ -696,25 +715,25 @@ impl<'a> StringReader<'a> {
             }
             let last_pos = self.last_pos;
             self.check_float_base(start_bpos, last_pos, base);
-            return token::Literal(token::Float(self.name_from(start_bpos)));
+            return token::Float(self.name_from(start_bpos));
         } else if self.curr_is('f') {
             // or it might be an integer literal suffixed as a float
             self.scan_float_suffix();
             let last_pos = self.last_pos;
             self.check_float_base(start_bpos, last_pos, base);
-            return token::Literal(token::Float(self.name_from(start_bpos)));
+            return token::Float(self.name_from(start_bpos));
         } else {
             // it might be a float if it has an exponent
             if self.curr_is('e') || self.curr_is('E') {
                 self.scan_float_exponent();
                 self.scan_float_suffix();
                 let last_pos = self.last_pos;
                 self.check_float_base(start_bpos, last_pos, base);
-                return token::Literal(token::Float(self.name_from(start_bpos)));
+                return token::Float(self.name_from(start_bpos));
             }
             // but we certainly have an integer!
             self.scan_int_suffix();
-            return token::Literal(token::Integer(self.name_from(start_bpos)));
+            return token::Integer(self.name_from(start_bpos));
         }
     }
 
@@ -967,7 +986,9 @@ impl<'a> StringReader<'a> {
         }
 
         if is_dec_digit(c) {
-            return self.scan_number(c.unwrap());
+            let num = self.scan_number(c.unwrap());
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(num, suffix)
         }
 
         if self.read_embedded_ident {
@@ -1126,17 +1147,19 @@ impl<'a> StringReader<'a> {
             }
             let id = if valid { self.name_from(start) } else { token::intern("0") };
             self.bump(); // advance curr past token
-            return token::Literal(token::Char(id));
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(token::Char(id), suffix);
           }
           'b' => {
             self.bump();
-            return match self.curr {
+            let lit = match self.curr {
                 Some('\'') => self.scan_byte(),
                 Some('"') => self.scan_byte_string(),
                 Some('r') => self.scan_raw_byte_string(),
                 _ => unreachable!()  // Should have been a token::Ident above.
             };
-
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(lit, suffix);
           }
           '"' => {
             let start_bpos = self.last_pos;
@@ -1157,7 +1180,8 @@ impl<'a> StringReader<'a> {
             let id = if valid { self.name_from(start_bpos + BytePos(1)) }
                      else { token::intern("??") };
             self.bump();
-            return token::Literal(token::Str_(id));
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(token::Str_(id), suffix);
           }
           'r' => {
             let start_bpos = self.last_pos;
@@ -1224,7 +1248,8 @@ impl<'a> StringReader<'a> {
             } else {
                 token::intern("??")
             };
-            return token::Literal(token::StrRaw(id, hash_count));
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(token::StrRaw(id, hash_count), suffix);
           }
           '-' => {
             if self.nextch_is('>') {
@@ -1293,7 +1318,7 @@ impl<'a> StringReader<'a> {
      || (self.curr_is('#') && self.nextch_is('!') && !self.nextnextch_is('['))
     }
 
-    fn scan_byte(&mut self) -> token::Token {
+    fn scan_byte(&mut self) -> token::Lit {
         self.bump();
         let start = self.last_pos;
 
@@ -1314,10 +1339,10 @@ impl<'a> StringReader<'a> {
 
         let id = if valid { self.name_from(start) } else { token::intern("??") };
         self.bump(); // advance curr past token
-        return token::Literal(token::Byte(id));
+        return token::Byte(id);
     }
 
-    fn scan_byte_string(&mut self) -> token::Token {
+    fn scan_byte_string(&mut self) -> token::Lit {
         self.bump();
         let start = self.last_pos;
         let mut valid = true;
@@ -1336,10 +1361,10 @@ impl<'a> StringReader<'a> {
         }
         let id = if valid { self.name_from(start) } else { token::intern("??") };
         self.bump();
-        return token::Literal(token::Binary(id));
+        return token::Binary(id);
     }
 
-    fn scan_raw_byte_string(&mut self) -> token::Token {
+    fn scan_raw_byte_string(&mut self) -> token::Lit {
         let start_bpos = self.last_pos;
         self.bump();
         let mut hash_count = 0u;
@@ -1387,9 +1412,9 @@ impl<'a> StringReader<'a> {
             self.bump();
         }
         self.bump();
-        return token::Literal(token::BinaryRaw(self.name_from_to(content_start_bpos,
-                                                                 content_end_bpos),
-                                               hash_count));
+        return token::BinaryRaw(self.name_from_to(content_start_bpos,
+                                                  content_end_bpos),
+                                hash_count);
     }
 }
 
@@ -1536,17 +1561,17 @@ mod test {
 
     #[test] fn character_a() {
         assert_eq!(setup(&mk_sh(), "'a'".to_string()).next_token().tok,
-                   token::Literal(token::Char(token::intern("a"))));
+                   token::Literal(token::Char(token::intern("a")), None));
     }
 
     #[test] fn character_space() {
         assert_eq!(setup(&mk_sh(), "' '".to_string()).next_token().tok,
-                   token::Literal(token::Char(token::intern(" "))));
+                   token::Literal(token::Char(token::intern(" ")), None));
     }
 
     #[test] fn character_escaped() {
         assert_eq!(setup(&mk_sh(), "'\\n'".to_string()).next_token().tok,
-                   token::Literal(token::Char(token::intern("\\n"))));
+                   token::Literal(token::Char(token::intern("\\n")), None));
     }
 
     #[test] fn lifetime_name() {
@@ -1558,7 +1583,38 @@ mod test {
         assert_eq!(setup(&mk_sh(),
                          "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token()
                                                                  .tok,
-                   token::Literal(token::StrRaw(token::intern("\"#a\\b\x00c\""), 3)));
+                   token::Literal(token::StrRaw(token::intern("\"#a\\b\x00c\""), 3), None));
+    }
+
+    #[test] fn literal_suffixes() {
+        macro_rules! test {
+            ($input: expr, $tok_type: ident, $tok_contents: expr) => {{
+                assert_eq!(setup(&mk_sh(), format!("{}suffix", $input)).next_token().tok,
+                           token::Literal(token::$tok_type(token::intern($tok_contents)),
+                                          Some(token::intern("suffix"))));
+                // with a whitespace separator:
+                assert_eq!(setup(&mk_sh(), format!("{} suffix", $input)).next_token().tok,
+                           token::Literal(token::$tok_type(token::intern($tok_contents)),
+                                          None));
+            }}
+        }
+
+        test!("'a'", Char, "a");
+        test!("b'a'", Byte, "a");
+        test!("\"a\"", Str_, "a");
+        test!("b\"a\"", Binary, "a");
+        test!("1234", Integer, "1234");
+        test!("0b101", Integer, "0b101");
+        test!("0xABC", Integer, "0xABC");
+        test!("1.0", Float, "1.0");
+        test!("1.0e10", Float, "1.0e10");
+
+        assert_eq!(setup(&mk_sh(), "r###\"raw\"###suffix".to_string()).next_token().tok,
+                   token::Literal(token::StrRaw(token::intern("raw"), 3),
+                                  Some(token::intern("suffix"))));
+        assert_eq!(setup(&mk_sh(), "br###\"raw\"###suffix".to_string()).next_token().tok,
+                   token::Literal(token::BinaryRaw(token::intern("raw"), 3),
+                                  Some(token::intern("suffix"))));
     }
 
     #[test] fn line_doc_comments() {
@@ -1574,7 +1630,7 @@ mod test {
             token::Comment => { },
             _ => panic!("expected a comment!")
         }
-        assert_eq!(lexer.next_token().tok, token::Literal(token::Char(token::intern("a"))));
+        assert_eq!(lexer.next_token().tok, token::Literal(token::Char(token::intern("a")), None));
     }
 
 }