Skip to content

Commit bbee088

Browse files
petrochenkovpetrochenkov
committed
Keep the original token in ast::Lit 
1 parent 4f76632 commit bbee088

File tree

10 files changed

+86
-59
lines changed

10 files changed

+86
-59
lines changed

‎src/librustc/hir/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1353,7 +1353,7 @@ pub struct Expr {
13531353

13541354
// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
13551355
#[cfg(target_arch = "x86_64")]
1356-
static_assert!(MEM_SIZE_OF_EXPR: std::mem::size_of::<Expr>() == 72);
1356+
static_assert!(MEM_SIZE_OF_EXPR: std::mem::size_of::<Expr>() == 80);
13571357

13581358
impl Expr {
13591359
pub fn precedence(&self) -> ExprPrecedence {

‎src/librustc/ich/impls_syntax.rs

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,8 @@ impl_stable_hash_for!(enum ::syntax::ast::LitIntType {
164164

165165
impl_stable_hash_for!(struct ::syntax::ast::Lit {
166166
node,
167+
token,
168+
suffix,
167169
span
168170
});
169171

@@ -284,6 +286,19 @@ for tokenstream::TokenStream {
284286
}
285287
}
286288

289+
impl_stable_hash_for!(enum token::Lit {
290+
Bool(val),
291+
Byte(val),
292+
Char(val),
293+
Err(val),
294+
Integer(val),
295+
Float(val),
296+
Str_(val),
297+
ByteStr(val),
298+
StrRaw(val, n),
299+
ByteStrRaw(val, n)
300+
});
301+
287302
fn hash_token<'a, 'gcx, W: StableHasherResult>(
288303
token: &token::Token,
289304
hcx: &mut StableHashingContext<'a>,
@@ -331,22 +346,8 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
331346
token::Token::CloseDelim(delim_token) => {
332347
std_hash::Hash::hash(&delim_token, hasher);
333348
}
334-
token::Token::Literal(ref lit, ref opt_name) => {
335-
mem::discriminant(lit).hash_stable(hcx, hasher);
336-
match *lit {
337-
token::Lit::Byte(val) |
338-
token::Lit::Char(val) |
339-
token::Lit::Err(val) |
340-
token::Lit::Integer(val) |
341-
token::Lit::Float(val) |
342-
token::Lit::Str_(val) |
343-
token::Lit::ByteStr(val) => val.hash_stable(hcx, hasher),
344-
token::Lit::StrRaw(val, n) |
345-
token::Lit::ByteStrRaw(val, n) => {
346-
val.hash_stable(hcx, hasher);
347-
n.hash_stable(hcx, hasher);
348-
}
349-
};
349+
token::Token::Literal(lit, opt_name) => {
350+
lit.hash_stable(hcx, hasher);
350351
opt_name.hash_stable(hcx, hasher);
351352
}
352353

‎src/librustdoc/html/highlight.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,8 @@ impl<'a> Classifier<'a> {
318318

319319
// Number literals.
320320
token::Integer(..) | token::Float(..) => Class::Number,
321+
322+
token::Bool(..) => panic!("literal token contains `Lit::Bool`"),
321323
}
322324
}
323325

‎src/libsyntax/ast.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ pub use crate::symbol::{Ident, Symbol as Name};
66
pub use crate::util::parser::ExprPrecedence;
77

88
use crate::ext::hygiene::{Mark, SyntaxContext};
9+
use crate::parse::token;
910
use crate::print::pprust;
1011
use crate::ptr::P;
1112
use crate::source_map::{dummy_spanned, respan, Spanned};
@@ -1354,6 +1355,8 @@ pub enum StrStyle {
13541355
#[derive(Clone, RustcEncodable, RustcDecodable, Debug, Hash, PartialEq)]
13551356
pub struct Lit {
13561357
pub node: LitKind,
1358+
pub token: token::Lit,
1359+
pub suffix: Option<Symbol>,
13571360
pub span: Span,
13581361
}
13591362

‎src/libsyntax/attr/mod.rs

Lines changed: 37 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,9 @@ impl Attribute {
350350
/* Constructors */
351351

352352
pub fn mk_name_value_item_str(ident: Ident, value: Spanned<Symbol>) -> MetaItem {
353-
let value = Lit { node: LitKind::Str(value.node, ast::StrStyle::Cooked), span: value.span };
353+
let node = LitKind::Str(value.node, ast::StrStyle::Cooked);
354+
let (token, suffix) = node.lit_token();
355+
let value = Lit { node, token, suffix, span: value.span };
354356
mk_name_value_item(ident.span.to(value.span), ident, value)
355357
}
356358

@@ -417,7 +419,9 @@ pub fn mk_spanned_attr_outer(sp: Span, id: AttrId, item: MetaItem) -> Attribute
417419

418420
pub fn mk_sugared_doc_attr(id: AttrId, text: Symbol, span: Span) -> Attribute {
419421
let style = doc_comment_style(&text.as_str());
420-
let lit = Lit { node: LitKind::Str(text, ast::StrStyle::Cooked), span };
422+
let node = LitKind::Str(text, ast::StrStyle::Cooked);
423+
let (token, suffix) = node.lit_token();
424+
let lit = Lit { node, token, suffix, span };
421425
Attribute {
422426
id,
423427
style,
@@ -562,7 +566,7 @@ impl MetaItemKind {
562566
tokens.next();
563567
return if let Some(TokenTree::Token(span, token)) = tokens.next() {
564568
LitKind::from_token(token)
565-
.map(|node| MetaItemKind::NameValue(Lit { node, span }))
569+
.map(|(node, token, suffix)| MetaItemKind::NameValue(Lit { node, token, suffix, span }))
566570
} else {
567571
None
568572
};
@@ -607,9 +611,9 @@ impl NestedMetaItem {
607611
where I: Iterator<Item = TokenTree>,
608612
{
609613
if let Some(TokenTree::Token(span, token)) = tokens.peek().cloned() {
610-
if let Some(node) = LitKind::from_token(token) {
614+
if let Some((node, token, suffix)) = LitKind::from_token(token) {
611615
tokens.next();
612-
return Some(NestedMetaItem::Literal(Lit { node, span }));
616+
return Some(NestedMetaItem::Literal(Lit { node, token, suffix, span }));
613617
}
614618
}
615619

@@ -625,67 +629,75 @@ impl Lit {
625629

626630
impl LitKind {
627631
fn token(&self) -> Token {
632+
match self.lit_token() {
633+
(token::Bool(symbol), _) => Token::Ident(Ident::with_empty_ctxt(symbol), false),
634+
(lit, suffix) => Token::Literal(lit, suffix),
635+
}
636+
}
637+
638+
pub(crate) fn lit_token(&self) -> (token::Lit, Option<Symbol>) {
628639
use std::ascii;
629640

630641
match *self {
631642
LitKind::Str(string, ast::StrStyle::Cooked) => {
632643
let escaped = string.as_str().escape_default().to_string();
633-
Token::Literal(token::Lit::Str_(Symbol::intern(&escaped)), None)
644+
(token::Lit::Str_(Symbol::intern(&escaped)), None)
634645
}
635646
LitKind::Str(string, ast::StrStyle::Raw(n)) => {
636-
Token::Literal(token::Lit::StrRaw(string, n), None)
647+
(token::Lit::StrRaw(string, n), None)
637648
}
638649
LitKind::ByteStr(ref bytes) => {
639650
let string = bytes.iter().cloned().flat_map(ascii::escape_default)
640651
.map(Into::<char>::into).collect::<String>();
641-
Token::Literal(token::Lit::ByteStr(Symbol::intern(&string)), None)
652+
(token::Lit::ByteStr(Symbol::intern(&string)), None)
642653
}
643654
LitKind::Byte(byte) => {
644655
let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
645-
Token::Literal(token::Lit::Byte(Symbol::intern(&string)), None)
656+
(token::Lit::Byte(Symbol::intern(&string)), None)
646657
}
647658
LitKind::Char(ch) => {
648659
let string: String = ch.escape_default().map(Into::<char>::into).collect();
649-
Token::Literal(token::Lit::Char(Symbol::intern(&string)), None)
660+
(token::Lit::Char(Symbol::intern(&string)), None)
650661
}
651662
LitKind::Int(n, ty) => {
652663
let suffix = match ty {
653664
ast::LitIntType::Unsigned(ty) => Some(Symbol::intern(ty.ty_to_string())),
654665
ast::LitIntType::Signed(ty) => Some(Symbol::intern(ty.ty_to_string())),
655666
ast::LitIntType::Unsuffixed => None,
656667
};
657-
Token::Literal(token::Lit::Integer(Symbol::intern(&n.to_string())), suffix)
668+
(token::Lit::Integer(Symbol::intern(&n.to_string())), suffix)
658669
}
659670
LitKind::Float(symbol, ty) => {
660-
Token::Literal(token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string())))
671+
(token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string())))
661672
}
662-
LitKind::FloatUnsuffixed(symbol) => Token::Literal(token::Lit::Float(symbol), None),
663-
LitKind::Bool(value) => Token::Ident(Ident::with_empty_ctxt(Symbol::intern(if value {
664-
"true"
665-
} else {
666-
"false"
667-
})), false),
668-
LitKind::Err(val) => Token::Literal(token::Lit::Err(val), None),
673+
LitKind::FloatUnsuffixed(symbol) => (token::Lit::Float(symbol), None),
674+
LitKind::Bool(value) => {
675+
let kw = if value { keywords::True } else { keywords::False };
676+
(token::Lit::Bool(kw.name()), None)
677+
}
678+
LitKind::Err(val) => (token::Lit::Err(val), None),
669679
}
670680
}
671681

672-
fn from_token(token: Token) -> Option<LitKind> {
682+
fn from_token(token: Token) -> Option<(LitKind, token::Lit, Option<Symbol>)> {
673683
match token {
674-
Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)),
675-
Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)),
684+
Token::Ident(ident, false) if ident.name == keywords::True.name() =>
685+
Some((LitKind::Bool(true), token::Bool(ident.name), None)),
686+
Token::Ident(ident, false) if ident.name == keywords::False.name() =>
687+
Some((LitKind::Bool(false), token::Bool(ident.name), None)),
676688
Token::Interpolated(nt) => match *nt {
677689
token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
678-
ExprKind::Lit(ref lit) => Some(lit.node.clone()),
690+
ExprKind::Lit(ref lit) => Some((lit.node.clone(), lit.token, lit.suffix)),
679691
_ => None,
680692
},
681693
_ => None,
682694
},
683695
Token::Literal(lit, suf) => {
684696
let (suffix_illegal, result) = parse::lit_token(lit, suf, None);
685-
if suffix_illegal && suf.is_some() {
697+
if result.is_none() || suffix_illegal && suf.is_some() {
686698
return None;
687699
}
688-
result
700+
Some((result.unwrap(), lit, suf))
689701
}
690702
_ => None,
691703
}

‎src/libsyntax/ext/build.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -698,7 +698,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
698698
}
699699

700700
fn expr_lit(&self, span: Span, node: ast::LitKind) -> P<ast::Expr> {
701-
self.expr(span, ast::ExprKind::Lit(ast::Lit { node, span }))
701+
let (token, suffix) = node.lit_token();
702+
self.expr(span, ast::ExprKind::Lit(ast::Lit { node, token, suffix, span }))
702703
}
703704
fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> {
704705
self.expr_lit(span, ast::LitKind::Int(i as u128,
@@ -1166,8 +1167,9 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
11661167

11671168
fn meta_name_value(&self, span: Span, name: ast::Name, node: ast::LitKind)
11681169
-> ast::MetaItem {
1170+
let (token, suffix) = node.lit_token();
11691171
attr::mk_name_value_item(span, Ident::with_empty_ctxt(name).with_span_pos(span),
1170-
ast::Lit { node, span })
1172+
ast::Lit { node, token, suffix, span })
11711173
}
11721174

11731175
fn item_use(&self, sp: Span,

‎src/libsyntax/parse/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,7 @@ crate fn lit_token(lit: token::Lit, suf: Option<Symbol>, diag: Option<(Span, &Ha
353353
use ast::LitKind;
354354

355355
match lit {
356+
token::Bool(_) => panic!("literal token contains `Lit::Bool`"),
356357
token::Byte(i) => {
357358
let lit_kind = match unescape_byte(&i.as_str()) {
358359
Ok(c) => LitKind::Byte(c),

‎src/libsyntax/parse/parser.rs

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2065,11 +2065,11 @@ impl<'a> Parser<'a> {
20652065
}
20662066

20672067
/// Matches `token_lit = LIT_INTEGER | ...`.
2068-
fn parse_lit_token(&mut self) -> PResult<'a, LitKind> {
2068+
fn parse_lit_token(&mut self) -> PResult<'a, (LitKind, token::Lit, Option<Symbol>)> {
20692069
let out = match self.token {
20702070
token::Interpolated(ref nt) => match **nt {
20712071
token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
2072-
ExprKind::Lit(ref lit) => { lit.node.clone() }
2072+
ExprKind::Lit(ref lit) => { (lit.node.clone(), lit.token, lit.suffix) }
20732073
_ => { return self.unexpected_last(&self.token); }
20742074
},
20752075
_ => { return self.unexpected_last(&self.token); }
@@ -2083,19 +2083,19 @@ impl<'a> Parser<'a> {
20832083
self.expect_no_suffix(sp, &format!("a {}", lit.literal_name()), suf)
20842084
}
20852085

2086-
result.unwrap()
2086+
(result.unwrap(), lit, suf)
20872087
}
20882088
token::Dot if self.look_ahead(1, |t| match t {
2089-
token::Literal(parse::token::Lit::Integer(_) , _) => true,
2089+
token::Literal(token::Lit::Integer(_) , _) => true,
20902090
_ => false,
20912091
}) => { // recover from `let x = .4;`
20922092
let lo = self.span;
20932093
self.bump();
20942094
if let token::Literal(
2095-
parse::token::Lit::Integer(val),
2095+
token::Lit::Integer(val),
20962096
suffix,
20972097
) = self.token {
2098-
let suffix = suffix.and_then(|s| {
2098+
let float_suffix = suffix.and_then(|s| {
20992099
let s = s.as_str();
21002100
if s == "f32" {
21012101
Some("f32")
@@ -2112,14 +2112,14 @@ impl<'a> Parser<'a> {
21122112
err.span_suggestion(
21132113
sp,
21142114
"must have an integer part",
2115-
format!("0.{}{}", val, suffix),
2115+
format!("0.{}{}", val, float_suffix),
21162116
Applicability::MachineApplicable,
21172117
);
21182118
err.emit();
2119-
return Ok(match suffix {
2120-
"f32" => ast::LitKind::Float(val, ast::FloatTy::F32),
2121-
"f64" => ast::LitKind::Float(val, ast::FloatTy::F64),
2122-
_ => ast::LitKind::FloatUnsuffixed(val),
2119+
return Ok(match float_suffix {
2120+
"f32" => (ast::LitKind::Float(val, ast::FloatTy::F32), token::Float(val), suffix),
2121+
"f64" => (ast::LitKind::Float(val, ast::FloatTy::F64), token::Float(val), suffix),
2122+
_ => (ast::LitKind::FloatUnsuffixed(val), token::Float(val), suffix),
21232123
});
21242124
} else {
21252125
unreachable!();
@@ -2135,14 +2135,14 @@ impl<'a> Parser<'a> {
21352135
/// Matches `lit = true | false | token_lit`.
21362136
crate fn parse_lit(&mut self) -> PResult<'a, Lit> {
21372137
let lo = self.span;
2138-
let node = if self.eat_keyword(keywords::True) {
2139-
LitKind::Bool(true)
2138+
let (node, token, suffix) = if self.eat_keyword(keywords::True) {
2139+
(LitKind::Bool(true), token::Bool(keywords::True.name()), None)
21402140
} else if self.eat_keyword(keywords::False) {
2141-
LitKind::Bool(false)
2141+
(LitKind::Bool(false), token::Bool(keywords::False.name()), None)
21422142
} else {
21432143
self.parse_lit_token()?
21442144
};
2145-
Ok(Lit { node, span: lo.to(self.prev_span) })
2145+
Ok(Lit { node, token, suffix, span: lo.to(self.prev_span) })
21462146
}
21472147

21482148
/// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`).

‎src/libsyntax/parse/token.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ impl DelimToken {
6161

6262
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
6363
pub enum Lit {
64+
Bool(ast::Name), // AST only, must never appear in a `Token`
6465
Byte(ast::Name),
6566
Char(ast::Name),
6667
Err(ast::Name),
@@ -72,9 +73,13 @@ pub enum Lit {
7273
ByteStrRaw(ast::Name, u16), /* raw byte str delimited by n hash symbols */
7374
}
7475

76+
#[cfg(target_arch = "x86_64")]
77+
static_assert!(MEM_SIZE_OF_LIT: mem::size_of::<Lit>() == 8);
78+
7579
impl Lit {
7680
crate fn literal_name(&self) -> &'static str {
7781
match *self {
82+
Bool(_) => panic!("literal token contains `Lit::Bool`"),
7883
Byte(_) => "byte literal",
7984
Char(_) => "char literal",
8085
Err(_) => "invalid literal",

‎src/libsyntax/print/pprust.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ pub fn token_to_string(tok: &Token) -> String {
225225
/* Literals */
226226
token::Literal(lit, suf) => {
227227
let mut out = match lit {
228+
token::Bool(_) => panic!("literal token contains `Lit::Bool`"),
228229
token::Byte(b) => format!("b'{}'", b),
229230
token::Char(c) => format!("'{}'", c),
230231
token::Err(c) => format!("'{}'", c),

0 commit comments

Comments
 (0)