Skip to content

Commit 2cd2070

Browse files
borsbors
committed
Auto merge of rust-lang#105160 - nnethercote:rm-Lit-token_lit, r=petrochenkov
Remove `token::Lit` from `ast::MetaItemLit`. Currently `ast::MetaItemLit` represents the literal kind twice. This PR removes that redundancy. Best reviewed one commit at a time. r? `@petrochenkov`
2 parents b397bc0 + 7e0c6db commit 2cd2070

File tree

28 files changed

+244
-171
lines changed

28 files changed

+244
-171
lines changed

‎compiler/rustc_ast/src/ast.rs

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1734,8 +1734,10 @@ pub enum StrStyle {
17341734
/// A literal in a meta item.
17351735
#[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)]
17361736
pub struct MetaItemLit {
1737-
/// The original literal token as written in source code.
1738-
pub token_lit: token::Lit,
1737+
/// The original literal as written in the source code.
1738+
pub symbol: Symbol,
1739+
/// The original suffix as written in the source code.
1740+
pub suffix: Option<Symbol>,
17391741
/// The "semantic" representation of the literal lowered from the original tokens.
17401742
/// Strings are unescaped, hexadecimal forms are eliminated, etc.
17411743
pub kind: LitKind,
@@ -1745,13 +1747,14 @@ pub struct MetaItemLit {
17451747
/// Similar to `MetaItemLit`, but restricted to string literals.
17461748
#[derive(Clone, Copy, Encodable, Decodable, Debug)]
17471749
pub struct StrLit {
1748-
/// The original literal token as written in source code.
1749-
pub style: StrStyle,
1750+
/// The original literal as written in source code.
17501751
pub symbol: Symbol,
1752+
/// The original suffix as written in source code.
17511753
pub suffix: Option<Symbol>,
1752-
pub span: Span,
1753-
/// The unescaped "semantic" representation of the literal lowered from the original token.
1754+
/// The semantic (unescaped) representation of the literal.
17541755
pub symbol_unescaped: Symbol,
1756+
pub style: StrStyle,
1757+
pub span: Span,
17551758
}
17561759

17571760
impl StrLit {
@@ -1797,8 +1800,9 @@ pub enum LitKind {
17971800
/// A string literal (`"foo"`). The symbol is unescaped, and so may differ
17981801
/// from the original token's symbol.
17991802
Str(Symbol, StrStyle),
1800-
/// A byte string (`b"foo"`).
1801-
ByteStr(Lrc<[u8]>),
1803+
/// A byte string (`b"foo"`). Not stored as a symbol because it might be
1804+
/// non-utf8, and symbols only allow utf8 strings.
1805+
ByteStr(Lrc<[u8]>, StrStyle),
18021806
/// A byte char (`b'f'`).
18031807
Byte(u8),
18041808
/// A character literal (`'a'`).
@@ -1823,7 +1827,7 @@ impl LitKind {
18231827

18241828
/// Returns `true` if this literal is byte literal string.
18251829
pub fn is_bytestr(&self) -> bool {
1826-
matches!(self, LitKind::ByteStr(_))
1830+
matches!(self, LitKind::ByteStr(..))
18271831
}
18281832

18291833
/// Returns `true` if this is a numeric literal.
@@ -3100,7 +3104,7 @@ mod size_asserts {
31003104
static_assert_size!(ItemKind, 112);
31013105
static_assert_size!(LitKind, 24);
31023106
static_assert_size!(Local, 72);
3103-
static_assert_size!(MetaItemLit, 48);
3107+
static_assert_size!(MetaItemLit, 40);
31043108
static_assert_size!(Param, 40);
31053109
static_assert_size!(Pat, 88);
31063110
static_assert_size!(Path, 24);

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
//! Functions dealing with attributes and meta items.
22
3-
use crate::ast;
43
use crate::ast::{AttrArgs, AttrArgsEq, AttrId, AttrItem, AttrKind, AttrStyle, AttrVec, Attribute};
54
use crate::ast::{DelimArgs, Expr, ExprKind, LitKind, MetaItemLit};
65
use crate::ast::{MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
7-
use crate::ast::{Path, PathSegment, StrStyle, DUMMY_NODE_ID};
6+
use crate::ast::{Path, PathSegment, DUMMY_NODE_ID};
87
use crate::ptr::P;
98
use crate::token::{self, CommentKind, Delimiter, Token};
109
use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
1110
use crate::tokenstream::{LazyAttrTokenStream, TokenStream};
1211
use crate::util::comments;
12+
use crate::util::literal::escape_string_symbol;
1313
use rustc_data_structures::sync::WorkerLocal;
1414
use rustc_index::bit_set::GrowableBitSet;
1515
use rustc_span::symbol::{sym, Ident, Symbol};
@@ -321,18 +321,6 @@ impl Attribute {
321321
}
322322
}
323323

324-
/* Constructors */
325-
326-
pub fn mk_name_value_item_str(ident: Ident, str: Symbol, str_span: Span) -> MetaItem {
327-
mk_name_value_item(ident, LitKind::Str(str, ast::StrStyle::Cooked), str_span)
328-
}
329-
330-
pub fn mk_name_value_item(ident: Ident, kind: LitKind, lit_span: Span) -> MetaItem {
331-
let lit = MetaItemLit { token_lit: kind.to_token_lit(), kind, span: lit_span };
332-
let span = ident.span.to(lit_span);
333-
MetaItem { path: Path::from_ident(ident), kind: MetaItemKind::NameValue(lit), span }
334-
}
335-
336324
pub struct AttrIdGenerator(WorkerLocal<Cell<u32>>);
337325

338326
#[cfg(debug_assertions)]
@@ -408,7 +396,7 @@ pub fn mk_attr_name_value_str(
408396
val: Symbol,
409397
span: Span,
410398
) -> Attribute {
411-
let lit = LitKind::Str(val, StrStyle::Cooked).to_token_lit();
399+
let lit = token::Lit::new(token::Str, escape_string_symbol(val), None);
412400
let expr = P(Expr {
413401
id: DUMMY_NODE_ID,
414402
kind: ExprKind::Lit(lit),

‎compiler/rustc_ast/src/util/literal.rs

Lines changed: 94 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,31 @@
11
//! Code related to parsing literals.
22
3-
use crate::ast::{self, LitKind, MetaItemLit};
3+
use crate::ast::{self, LitKind, MetaItemLit, StrStyle};
44
use crate::token::{self, Token};
55
use rustc_lexer::unescape::{byte_from_char, unescape_byte, unescape_char, unescape_literal, Mode};
66
use rustc_span::symbol::{kw, sym, Symbol};
77
use rustc_span::Span;
8-
use std::ascii;
8+
use std::{ascii, fmt, str};
9+
10+
// Escapes a string, represented as a symbol. Reuses the original symbol,
11+
// avoiding interning, if no changes are required.
12+
pub fn escape_string_symbol(symbol: Symbol) -> Symbol {
13+
let s = symbol.as_str();
14+
let escaped = s.escape_default().to_string();
15+
if s == escaped { symbol } else { Symbol::intern(&escaped) }
16+
}
17+
18+
// Escapes a char.
19+
pub fn escape_char_symbol(ch: char) -> Symbol {
20+
let s: String = ch.escape_default().map(Into::<char>::into).collect();
21+
Symbol::intern(&s)
22+
}
23+
24+
// Escapes a byte string.
25+
pub fn escape_byte_str_symbol(bytes: &[u8]) -> Symbol {
26+
let s = bytes.escape_ascii().to_string();
27+
Symbol::intern(&s)
28+
}
929

1030
#[derive(Debug)]
1131
pub enum LitError {
@@ -115,9 +135,9 @@ impl LitKind {
115135
}
116136
});
117137
error?;
118-
LitKind::ByteStr(buf.into())
138+
LitKind::ByteStr(buf.into(), StrStyle::Cooked)
119139
}
120-
token::ByteStrRaw(_) => {
140+
token::ByteStrRaw(n) => {
121141
let s = symbol.as_str();
122142
let bytes = if s.contains('\r') {
123143
let mut buf = Vec::with_capacity(s.len());
@@ -136,69 +156,95 @@ impl LitKind {
136156
symbol.to_string().into_bytes()
137157
};
138158

139-
LitKind::ByteStr(bytes.into())
159+
LitKind::ByteStr(bytes.into(), StrStyle::Raw(n))
140160
}
141161
token::Err => LitKind::Err,
142162
})
143163
}
164+
}
144165

145-
/// Attempts to recover a token from semantic literal.
146-
/// This function is used when the original token doesn't exist (e.g. the literal is created
147-
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
148-
pub fn to_token_lit(&self) -> token::Lit {
149-
let (kind, symbol, suffix) = match *self {
150-
LitKind::Str(symbol, ast::StrStyle::Cooked) => {
151-
// Don't re-intern unless the escaped string is different.
152-
let s = symbol.as_str();
153-
let escaped = s.escape_default().to_string();
154-
let symbol = if s == escaped { symbol } else { Symbol::intern(&escaped) };
155-
(token::Str, symbol, None)
156-
}
157-
LitKind::Str(symbol, ast::StrStyle::Raw(n)) => (token::StrRaw(n), symbol, None),
158-
LitKind::ByteStr(ref bytes) => {
159-
let string = bytes.escape_ascii().to_string();
160-
(token::ByteStr, Symbol::intern(&string), None)
166+
impl fmt::Display for LitKind {
167+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
168+
match *self {
169+
LitKind::Byte(b) => {
170+
let b: String = ascii::escape_default(b).map(Into::<char>::into).collect();
171+
write!(f, "b'{}'", b)?;
161172
}
162-
LitKind::Byte(byte) => {
163-
let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
164-
(token::Byte, Symbol::intern(&string), None)
173+
LitKind::Char(ch) => write!(f, "'{}'", escape_char_symbol(ch))?,
174+
LitKind::Str(sym, StrStyle::Cooked) => write!(f, "\"{}\"", escape_string_symbol(sym))?,
175+
LitKind::Str(sym, StrStyle::Raw(n)) => write!(
176+
f,
177+
"r{delim}\"{string}\"{delim}",
178+
delim = "#".repeat(n as usize),
179+
string = sym
180+
)?,
181+
LitKind::ByteStr(ref bytes, StrStyle::Cooked) => {
182+
write!(f, "b\"{}\"", escape_byte_str_symbol(bytes))?
165183
}
166-
LitKind::Char(ch) => {
167-
let string: String = ch.escape_default().map(Into::<char>::into).collect();
168-
(token::Char, Symbol::intern(&string), None)
184+
LitKind::ByteStr(ref bytes, StrStyle::Raw(n)) => {
185+
// Unwrap because raw byte string literals can only contain ASCII.
186+
let symbol = str::from_utf8(bytes).unwrap();
187+
write!(
188+
f,
189+
"br{delim}\"{string}\"{delim}",
190+
delim = "#".repeat(n as usize),
191+
string = symbol
192+
)?;
169193
}
170194
LitKind::Int(n, ty) => {
171-
let suffix = match ty {
172-
ast::LitIntType::Unsigned(ty) => Some(ty.name()),
173-
ast::LitIntType::Signed(ty) => Some(ty.name()),
174-
ast::LitIntType::Unsuffixed => None,
175-
};
176-
(token::Integer, sym::integer(n), suffix)
195+
write!(f, "{}", n)?;
196+
match ty {
197+
ast::LitIntType::Unsigned(ty) => write!(f, "{}", ty.name())?,
198+
ast::LitIntType::Signed(ty) => write!(f, "{}", ty.name())?,
199+
ast::LitIntType::Unsuffixed => {}
200+
}
177201
}
178202
LitKind::Float(symbol, ty) => {
179-
let suffix = match ty {
180-
ast::LitFloatType::Suffixed(ty) => Some(ty.name()),
181-
ast::LitFloatType::Unsuffixed => None,
182-
};
183-
(token::Float, symbol, suffix)
203+
write!(f, "{}", symbol)?;
204+
match ty {
205+
ast::LitFloatType::Suffixed(ty) => write!(f, "{}", ty.name())?,
206+
ast::LitFloatType::Unsuffixed => {}
207+
}
184208
}
185-
LitKind::Bool(value) => {
186-
let symbol = if value { kw::True } else { kw::False };
187-
(token::Bool, symbol, None)
209+
LitKind::Bool(b) => write!(f, "{}", if b { "true" } else { "false" })?,
210+
LitKind::Err => {
211+
// This only shows up in places like `-Zunpretty=hir` output, so we
212+
// don't bother to produce something useful.
213+
write!(f, "<bad-literal>")?;
188214
}
189-
// This only shows up in places like `-Zunpretty=hir` output, so we
190-
// don't bother to produce something useful.
191-
LitKind::Err => (token::Err, Symbol::intern("<bad-literal>"), None),
192-
};
215+
}
193216

194-
token::Lit::new(kind, symbol, suffix)
217+
Ok(())
195218
}
196219
}
197220

198221
impl MetaItemLit {
199-
/// Converts token literal into a meta item literal.
222+
/// Converts a token literal into a meta item literal.
200223
pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<MetaItemLit, LitError> {
201-
Ok(MetaItemLit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
224+
Ok(MetaItemLit {
225+
symbol: token_lit.symbol,
226+
suffix: token_lit.suffix,
227+
kind: LitKind::from_token_lit(token_lit)?,
228+
span,
229+
})
230+
}
231+
232+
/// Cheaply converts a meta item literal into a token literal.
233+
pub fn as_token_lit(&self) -> token::Lit {
234+
let kind = match self.kind {
235+
LitKind::Bool(_) => token::Bool,
236+
LitKind::Str(_, ast::StrStyle::Cooked) => token::Str,
237+
LitKind::Str(_, ast::StrStyle::Raw(n)) => token::StrRaw(n),
238+
LitKind::ByteStr(_, ast::StrStyle::Cooked) => token::ByteStr,
239+
LitKind::ByteStr(_, ast::StrStyle::Raw(n)) => token::ByteStrRaw(n),
240+
LitKind::Byte(_) => token::Byte,
241+
LitKind::Char(_) => token::Char,
242+
LitKind::Int(..) => token::Integer,
243+
LitKind::Float(..) => token::Float,
244+
LitKind::Err => token::Err,
245+
};
246+
247+
token::Lit::new(kind, self.symbol, self.suffix)
202248
}
203249

204250
/// Converts an arbitrary token into meta item literal.

‎compiler/rustc_ast_lowering/src/expr.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
9797
}
9898
ExprKind::IncludedBytes(bytes) => hir::ExprKind::Lit(respan(
9999
self.lower_span(e.span),
100-
LitKind::ByteStr(bytes.clone()),
100+
LitKind::ByteStr(bytes.clone(), StrStyle::Cooked),
101101
)),
102102
ExprKind::Cast(expr, ty) => {
103103
let expr = self.lower_expr(expr);

‎compiler/rustc_ast_lowering/src/lib.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -958,7 +958,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
958958
lit
959959
} else {
960960
MetaItemLit {
961-
token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None),
961+
symbol: kw::Empty,
962+
suffix: None,
962963
kind: LitKind::Err,
963964
span: DUMMY_SP,
964965
}

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
376376
}
377377

378378
fn print_meta_item_lit(&mut self, lit: &ast::MetaItemLit) {
379-
self.print_token_literal(lit.token_lit, lit.span)
379+
self.print_token_literal(lit.as_token_lit(), lit.span)
380380
}
381381

382382
fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) {

‎compiler/rustc_ast_pretty/src/pprust/state/expr.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ use crate::pp::Breaks::Inconsistent;
22
use crate::pprust::state::{AnnNode, IterDelimited, PrintState, State, INDENT_UNIT};
33

44
use rustc_ast::ptr::P;
5+
use rustc_ast::token;
6+
use rustc_ast::util::literal::escape_byte_str_symbol;
57
use rustc_ast::util::parser::{self, AssocOp, Fixity};
68
use rustc_ast::{self as ast, BlockCheckMode};
79

@@ -323,7 +325,7 @@ impl<'a> State<'a> {
323325
self.print_token_literal(*token_lit, expr.span);
324326
}
325327
ast::ExprKind::IncludedBytes(bytes) => {
326-
let lit = ast::LitKind::ByteStr(bytes.clone()).to_token_lit();
328+
let lit = token::Lit::new(token::ByteStr, escape_byte_str_symbol(bytes), None);
327329
self.print_token_literal(lit, expr.span)
328330
}
329331
ast::ExprKind::Cast(expr, ty) => {

‎compiler/rustc_builtin_macros/src/concat_bytes.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ fn invalid_type_err(
6969
Ok(ast::LitKind::Int(_, _)) => {
7070
cx.span_err(span, "numeric literal is not a `u8`");
7171
}
72-
Ok(ast::LitKind::ByteStr(_) | ast::LitKind::Byte(_)) => unreachable!(),
72+
Ok(ast::LitKind::ByteStr(..) | ast::LitKind::Byte(_)) => unreachable!(),
7373
Err(err) => {
7474
report_lit_error(&cx.sess.parse_sess, err, token_lit, span);
7575
}
@@ -97,7 +97,7 @@ fn handle_array_element(
9797
)) if val <= u8::MAX.into() => Some(val as u8),
9898

9999
Ok(ast::LitKind::Byte(val)) => Some(val),
100-
Ok(ast::LitKind::ByteStr(_)) => {
100+
Ok(ast::LitKind::ByteStr(..)) => {
101101
if !*has_errors {
102102
cx.struct_span_err(expr.span, "cannot concatenate doubly nested array")
103103
.note("byte strings are treated as arrays of bytes")
@@ -174,7 +174,7 @@ pub fn expand_concat_bytes(
174174
Ok(ast::LitKind::Byte(val)) => {
175175
accumulator.push(val);
176176
}
177-
Ok(ast::LitKind::ByteStr(ref bytes)) => {
177+
Ok(ast::LitKind::ByteStr(ref bytes, _)) => {
178178
accumulator.extend_from_slice(&bytes);
179179
}
180180
_ => {

‎compiler/rustc_builtin_macros/src/derive.rs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::cfg_eval::cfg_eval;
22

33
use rustc_ast as ast;
4-
use rustc_ast::{token, GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
4+
use rustc_ast::{GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
55
use rustc_errors::{struct_span_err, Applicability};
66
use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
77
use rustc_feature::AttributeTemplate;
@@ -130,9 +130,11 @@ fn report_bad_target(sess: &Session, item: &Annotatable, span: Span) -> bool {
130130
}
131131

132132
fn report_unexpected_meta_item_lit(sess: &Session, lit: &ast::MetaItemLit) {
133-
let help_msg = match lit.token_lit.kind {
134-
token::Str if rustc_lexer::is_ident(lit.token_lit.symbol.as_str()) => {
135-
format!("try using `#[derive({})]`", lit.token_lit.symbol)
133+
let help_msg = match lit.kind {
134+
ast::LitKind::Str(_, ast::StrStyle::Cooked)
135+
if rustc_lexer::is_ident(lit.symbol.as_str()) =>
136+
{
137+
format!("try using `#[derive({})]`", lit.symbol)
136138
}
137139
_ => "for example, write `#[derive(Debug)]` for `Debug`".to_string(),
138140
};

‎compiler/rustc_expand/src/base.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1231,7 +1231,7 @@ pub fn expr_to_spanned_string<'a>(
12311231
Err(match expr.kind {
12321232
ast::ExprKind::Lit(token_lit) => match ast::LitKind::from_token_lit(token_lit) {
12331233
Ok(ast::LitKind::Str(s, style)) => return Ok((s, style, expr.span)),
1234-
Ok(ast::LitKind::ByteStr(_)) => {
1234+
Ok(ast::LitKind::ByteStr(..)) => {
12351235
let mut err = cx.struct_span_err(expr.span, err_msg);
12361236
let span = expr.span.shrink_to_lo();
12371237
err.span_suggestion(

0 commit comments

Comments
 (0)