Skip to content

Commit a7e8629

Browse files
petrochenkovpetrochenkov
committed
macro_rules: Less hacky heuristic for tt macro parameter spans
1 parent 767453e commit a7e8629

12 files changed

+85
-81
lines changed

‎compiler/rustc_ast/src/tokenstream.rs

+1-22
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ use rustc_span::{sym, Span, Symbol, DUMMY_SP};
2626
use smallvec::{smallvec, SmallVec};
2727

2828
use std::borrow::Cow;
29-
use std::{cmp, fmt, iter, mem};
29+
use std::{cmp, fmt, iter};
3030

3131
/// When the main Rust parser encounters a syntax-extension invocation, it
3232
/// parses the arguments to the invocation as a token tree. This is a very
@@ -81,14 +81,6 @@ impl TokenTree {
8181
}
8282
}
8383

84-
/// Modify the `TokenTree`'s span in-place.
85-
pub fn set_span(&mut self, span: Span) {
86-
match self {
87-
TokenTree::Token(token, _) => token.span = span,
88-
TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span),
89-
}
90-
}
91-
9284
/// Create a `TokenTree::Token` with alone spacing.
9385
pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
9486
TokenTree::Token(Token::new(kind, span), Spacing::Alone)
@@ -461,19 +453,6 @@ impl TokenStream {
461453
t1.next().is_none() && t2.next().is_none()
462454
}
463455

464-
/// Applies the supplied function to each `TokenTree` and its index in `self`, returning a new `TokenStream`
465-
///
466-
/// It is equivalent to `TokenStream::new(self.trees().cloned().enumerate().map(|(i, tt)| f(i, tt)).collect())`.
467-
pub fn map_enumerated_owned(
468-
mut self,
469-
mut f: impl FnMut(usize, TokenTree) -> TokenTree,
470-
) -> TokenStream {
471-
let owned = Lrc::make_mut(&mut self.0); // clone if necessary
472-
// rely on vec's in-place optimizations to avoid another allocation
473-
*owned = mem::take(owned).into_iter().enumerate().map(|(i, tree)| f(i, tree)).collect();
474-
self
475-
}
476-
477456
/// Create a token stream containing a single token with alone spacing. The
478457
/// spacing used for the final token in a constructed stream doesn't matter
479458
/// because it's never used. In practice we arbitrarily use

‎compiler/rustc_expand/src/mbe.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ enum TokenTree {
7272
/// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
7373
Sequence(DelimSpan, SequenceRepetition),
7474
/// e.g., `$var`.
75-
MetaVar(Span, Ident),
75+
MetaVar(Span, Ident, bool),
7676
/// e.g., `$var:expr`. Only appears on the LHS.
7777
MetaVarDecl(Span, Ident /* name to bind */, Option<NonterminalKind>),
7878
/// A meta-variable expression inside `${...}`.
@@ -97,7 +97,7 @@ impl TokenTree {
9797
fn span(&self) -> Span {
9898
match *self {
9999
TokenTree::Token(Token { span, .. })
100-
| TokenTree::MetaVar(span, _)
100+
| TokenTree::MetaVar(span, ..)
101101
| TokenTree::MetaVarDecl(span, _, _) => span,
102102
TokenTree::Delimited(span, ..)
103103
| TokenTree::MetaVarExpr(span, _)

‎compiler/rustc_expand/src/mbe/macro_check.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ fn check_binders(
242242
// the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the
243243
// LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon
244244
// MetaVar(fragment) and not as MetaVarDecl(y, fragment).
245-
TokenTree::MetaVar(span, name) => {
245+
TokenTree::MetaVar(span, name, _) => {
246246
if macros.is_empty() {
247247
sess.dcx.span_bug(span, "unexpected MetaVar in lhs");
248248
}
@@ -342,7 +342,7 @@ fn check_occurrences(
342342
TokenTree::MetaVarDecl(span, _name, _kind) => {
343343
sess.dcx.span_bug(span, "unexpected MetaVarDecl in rhs")
344344
}
345-
TokenTree::MetaVar(span, name) => {
345+
TokenTree::MetaVar(span, name, _) => {
346346
let name = MacroRulesNormalizedIdent::new(name);
347347
check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name);
348348
}

‎compiler/rustc_expand/src/mbe/macro_rules.rs

+3-34
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use crate::mbe::transcribe::transcribe;
1010

1111
use rustc_ast as ast;
1212
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
13-
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
13+
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
1414
use rustc_ast::{NodeId, DUMMY_NODE_ID};
1515
use rustc_ast_pretty::pprust;
1616
use rustc_attr::{self as attr, TransparencyError};
@@ -213,45 +213,14 @@ fn expand_macro<'cx>(
213213
let arm_span = rhses[i].span();
214214

215215
// rhs has holes ( `$id` and `$(...)` that need filled)
216-
let mut tts = match transcribe(cx, &named_matches, rhs, rhs_span, transparency) {
216+
let tts = match transcribe(cx, &named_matches, rhs, rhs_span, transparency) {
217217
Ok(tts) => tts,
218218
Err(mut err) => {
219219
err.emit();
220220
return DummyResult::any(arm_span);
221221
}
222222
};
223223

224-
// Replace all the tokens for the corresponding positions in the macro, to maintain
225-
// proper positions in error reporting, while maintaining the macro_backtrace.
226-
if tts.len() == rhs.tts.len() {
227-
tts = tts.map_enumerated_owned(|i, mut tt| {
228-
let rhs_tt = &rhs.tts[i];
229-
let ctxt = tt.span().ctxt();
230-
match (&mut tt, rhs_tt) {
231-
// preserve the delim spans if able
232-
(
233-
TokenTree::Delimited(target_sp, ..),
234-
mbe::TokenTree::Delimited(source_sp, ..),
235-
) => {
236-
target_sp.open = source_sp.open.with_ctxt(ctxt);
237-
target_sp.close = source_sp.close.with_ctxt(ctxt);
238-
}
239-
(
240-
TokenTree::Delimited(target_sp, ..),
241-
mbe::TokenTree::MetaVar(source_sp, ..),
242-
) => {
243-
target_sp.open = source_sp.with_ctxt(ctxt);
244-
target_sp.close = source_sp.with_ctxt(ctxt).shrink_to_hi();
245-
}
246-
_ => {
247-
let sp = rhs_tt.span().with_ctxt(ctxt);
248-
tt.set_span(sp);
249-
}
250-
}
251-
tt
252-
});
253-
}
254-
255224
if cx.trace_macros() {
256225
let msg = format!("to `{}`", pprust::tts_to_string(&tts));
257226
trace_macros_note(&mut cx.expansions, sp, msg);
@@ -1409,7 +1378,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
14091378
fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
14101379
match tt {
14111380
mbe::TokenTree::Token(token) => pprust::token_to_string(token).into(),
1412-
mbe::TokenTree::MetaVar(_, name) => format!("${name}"),
1381+
mbe::TokenTree::MetaVar(_, name, _) => format!("${name}"),
14131382
mbe::TokenTree::MetaVarDecl(_, name, Some(kind)) => format!("${name}:{kind}"),
14141383
mbe::TokenTree::MetaVarDecl(_, name, None) => format!("${name}:"),
14151384
_ => panic!(

‎compiler/rustc_expand/src/mbe/quoted.rs

+12-4
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ pub(super) fn parse(
5454
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
5555
let tree = parse_tree(tree, &mut trees, parsing_patterns, sess, node_id, features, edition);
5656
match tree {
57-
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
57+
TokenTree::MetaVar(start_sp, ident, _) if parsing_patterns => {
5858
let span = match trees.next() {
5959
Some(&tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
6060
match trees.next() {
@@ -202,10 +202,18 @@ fn parse_tree<'a>(
202202
// If we didn't find a metavar expression above, then we must have a
203203
// repetition sequence in the macro (e.g. `$(pat)*`). Parse the
204204
// contents of the sequence itself
205-
let sequence = parse(tts, parsing_patterns, sess, node_id, features, edition);
205+
let mut sequence =
206+
parse(tts, parsing_patterns, sess, node_id, features, edition);
206207
// Get the Kleene operator and optional separator
207208
let (separator, kleene) =
208209
parse_sep_and_kleene_op(&mut trees, delim_span.entire(), sess);
210+
if let [TokenTree::MetaVar(.., undelimited_seq)] = &mut *sequence
211+
&& separator.is_none()
212+
&& matches!(kleene.op, KleeneOp::ZeroOrMore | KleeneOp::OneOrMore)
213+
{
214+
// See the comment on `fn maybe_use_metavar_location`.
215+
*undelimited_seq = true;
216+
}
209217
// Count the number of captured "names" (i.e., named metavars)
210218
let num_captures =
211219
if parsing_patterns { count_metavar_decls(&sequence) } else { 0 };
@@ -223,7 +231,7 @@ fn parse_tree<'a>(
223231
if ident.name == kw::Crate && !is_raw {
224232
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
225233
} else {
226-
TokenTree::MetaVar(span, ident)
234+
TokenTree::MetaVar(span, ident, false)
227235
}
228236
}
229237

@@ -245,7 +253,7 @@ fn parse_tree<'a>(
245253
let msg =
246254
format!("expected identifier, found `{}`", pprust::token_to_string(token),);
247255
sess.dcx.span_err(token.span, msg);
248-
TokenTree::MetaVar(token.span, Ident::empty())
256+
TokenTree::MetaVar(token.span, Ident::empty(), false)
249257
}
250258

251259
// There are no more tokens. Just return the `$` we already have.

‎compiler/rustc_expand/src/mbe/transcribe.rs

+51-3
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ pub(super) fn transcribe<'a>(
228228
}
229229

230230
// Replace the meta-var with the matched token tree from the invocation.
231-
mbe::TokenTree::MetaVar(mut sp, mut original_ident) => {
231+
mbe::TokenTree::MetaVar(mut sp, mut original_ident, undelimited_seq) => {
232232
// Find the matched nonterminal from the macro invocation, and use it to replace
233233
// the meta-var.
234234
let ident = MacroRulesNormalizedIdent::new(original_ident);
@@ -237,7 +237,7 @@ pub(super) fn transcribe<'a>(
237237
MatchedTokenTree(tt) => {
238238
// `tt`s are emitted into the output stream directly as "raw tokens",
239239
// without wrapping them into groups.
240-
result.push(tt.clone());
240+
result.push(maybe_use_metavar_location(cx, tt, sp, *undelimited_seq));
241241
}
242242
MatchedNonterminal(nt) => {
243243
// Other variables are emitted into the output stream as groups with
@@ -302,6 +302,54 @@ pub(super) fn transcribe<'a>(
302302
}
303303
}
304304

305+
/// Usually metavariables `$var` produce interpolated tokens, which have an additional place for
306+
/// keeping both the original span and the metavariable span. For `tt` metavariables that's not the
307+
/// case however, and there's no place for keeping a second span. So we try to give the single
308+
/// produced span a location that would be most useful in practice (the hygiene part of the span
309+
/// must not be changed).
310+
///
311+
/// Different locations are useful for different purposes:
312+
/// - The original location is useful when we need to report a diagnostic for the original token in
313+
/// isolation, without combining it with any surrounding tokens. This case occurs, but it is not
314+
/// very common in practice.
315+
/// - The metavariable location is useful when we need to somehow combine the token span with spans
316+
/// of its surrounding tokens. This is the most common way to use token spans.
317+
///
318+
/// So this function replaces the original location with the metavariable location in all cases
319+
/// except these two:
320+
/// - The metavariable is an element of undelimited sequence `$($tt)*`.
321+
/// These are typically used for passing larger amounts of code, and tokens in that code usually
322+
/// combine with each other and not with tokens outside of the sequence.
323+
/// - The metavariable span comes from a different crate, then we prefer the more local span.
324+
///
325+
/// FIXME: Find a way to keep both original and metavariable spans for all tokens without
326+
/// regressing compilation time too much. Several experiments for adding such spans were made in
327+
/// the past (PR #95580, #118517, #118671) and all showed some regressions.
328+
fn maybe_use_metavar_location(
329+
cx: &ExtCtxt<'_>,
330+
orig_tt: &TokenTree,
331+
metavar_span: Span,
332+
undelimited_seq: bool,
333+
) -> TokenTree {
334+
if undelimited_seq || cx.source_map().is_imported(metavar_span) {
335+
return orig_tt.clone();
336+
}
337+
338+
match orig_tt {
339+
TokenTree::Token(Token { kind, span }, spacing) => {
340+
let span = metavar_span.with_ctxt(span.ctxt());
341+
TokenTree::Token(Token { kind: kind.clone(), span }, *spacing)
342+
}
343+
TokenTree::Delimited(dspan, dspacing, delimiter, tts) => {
344+
let dspan = DelimSpan::from_pair(
345+
metavar_span.shrink_to_lo().with_ctxt(dspan.open.ctxt()),
346+
metavar_span.shrink_to_hi().with_ctxt(dspan.close.ctxt()),
347+
);
348+
TokenTree::Delimited(dspan, *dspacing, *delimiter, tts.clone())
349+
}
350+
}
351+
}
352+
305353
/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using
306354
/// the set of matches `interpolations`.
307355
///
@@ -402,7 +450,7 @@ fn lockstep_iter_size(
402450
size.with(lockstep_iter_size(tt, interpolations, repeats))
403451
})
404452
}
405-
TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => {
453+
TokenTree::MetaVar(_, name, _) | TokenTree::MetaVarDecl(_, name, _) => {
406454
let name = MacroRulesNormalizedIdent::new(*name);
407455
match lookup_cur_matched(name, interpolations, repeats) {
408456
Some(matched) => match matched {

‎tests/ui/macros/issue-118786.stderr

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ LL | macro_rules! $macro_name {
66
|
77
help: change the delimiters to curly braces
88
|
9-
LL | macro_rules! {} {
10-
| ~ +
9+
LL | macro_rules! {$macro_name} {
10+
| + +
1111
help: add a semicolon
1212
|
1313
LL | macro_rules! $macro_name; {

‎tests/ui/parser/issues/issue-68091-unicode-ident-after-if.stderr

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
error: missing condition for `if` expression
2-
--> $DIR/issue-68091-unicode-ident-after-if.rs:3:14
2+
--> $DIR/issue-68091-unicode-ident-after-if.rs:3:13
33
|
44
LL | $($c)ö* {}
5-
| ^ - if this block is the condition of the `if` expression, then it must be followed by another block
6-
| |
7-
| expected condition here
5+
| ^ - if this block is the condition of the `if` expression, then it must be followed by another block
6+
| |
7+
| expected condition here
88

99
error: aborting due to 1 previous error
1010

‎tests/ui/parser/issues/issue-68092-unicode-ident-after-incomplete-expr.stderr

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
error: macro expansion ends with an incomplete expression: expected expression
2-
--> $DIR/issue-68092-unicode-ident-after-incomplete-expr.rs:3:14
2+
--> $DIR/issue-68092-unicode-ident-after-incomplete-expr.rs:3:13
33
|
44
LL | $($c)ö*
5-
| ^ expected expression
5+
| ^ expected expression
66

77
error: aborting due to 1 previous error
88

‎tests/ui/proc-macro/capture-macro-rules-invoke.stdout

+1-1
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
271271
span: $DIR/capture-macro-rules-invoke.rs:47:19: 47:20 (#0),
272272
},
273273
],
274-
span: $DIR/capture-macro-rules-invoke.rs:47:13: 47:22 (#0),
274+
span: $DIR/capture-macro-rules-invoke.rs:15:60: 15:63 (#0),
275275
},
276276
Punct {
277277
ch: ',',

‎tests/ui/proc-macro/expand-expr.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ expand_expr_is!("hello", stringify!(hello));
3737
expand_expr_is!("10 + 20", stringify!(10 + 20));
3838

3939
macro_rules! echo_tts {
40-
($($t:tt)*) => { $($t)* }; //~ ERROR: expected expression, found `$`
40+
($($t:tt)*) => { $($t)* };
4141
}
4242

4343
macro_rules! echo_lit {
@@ -109,7 +109,7 @@ expand_expr_fail!("string"; hello); //~ ERROR: expected one of `.`, `?`, or an o
109109

110110
// Invalid expressions produce errors in addition to returning `Err(())`.
111111
expand_expr_fail!($); //~ ERROR: expected expression, found `$`
112-
expand_expr_fail!(echo_tts!($));
112+
expand_expr_fail!(echo_tts!($)); //~ ERROR: expected expression, found `$`
113113
expand_expr_fail!(echo_pm!($)); //~ ERROR: expected expression, found `$`
114114

115115
// We get errors reported and recover during macro expansion if the macro

‎tests/ui/proc-macro/expand-expr.stderr

+3-3
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ LL | expand_expr_fail!($);
1111
| ^ expected expression
1212

1313
error: expected expression, found `$`
14-
--> $DIR/expand-expr.rs:40:23
14+
--> $DIR/expand-expr.rs:112:29
1515
|
16-
LL | ($($t:tt)*) => { $($t)* };
17-
| ^^^^ expected expression
16+
LL | expand_expr_fail!(echo_tts!($));
17+
| ^ expected expression
1818

1919
error: expected expression, found `$`
2020
--> $DIR/expand-expr.rs:113:28

0 commit comments

Comments
 (0)