Skip to content

Commit 4c098db

Browse files
nnethercotennethercote
committed
Remove NtVis.
We now use invisible delimiters for expanded `vis` fragments, instead of `Token::Interpolated`.
1 parent 481b5fa commit 4c098db

File tree

8 files changed

+110
-19
lines changed

8 files changed

+110
-19
lines changed

‎compiler/rustc_ast/src/ast_traits.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,6 @@ impl HasTokens for Nonterminal {
206206
Nonterminal::NtTy(ty) => ty.tokens(),
207207
Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
208208
Nonterminal::NtPath(path) => path.tokens(),
209-
Nonterminal::NtVis(vis) => vis.tokens(),
210209
Nonterminal::NtBlock(block) => block.tokens(),
211210
}
212211
}
@@ -219,7 +218,6 @@ impl HasTokens for Nonterminal {
219218
Nonterminal::NtTy(ty) => ty.tokens_mut(),
220219
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
221220
Nonterminal::NtPath(path) => path.tokens_mut(),
222-
Nonterminal::NtVis(vis) => vis.tokens_mut(),
223221
Nonterminal::NtBlock(block) => block.tokens_mut(),
224222
}
225223
}

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -889,7 +889,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
889889
visit_lazy_tts(vis, tokens);
890890
}
891891
token::NtPath(path) => vis.visit_path(path),
892-
token::NtVis(visib) => vis.visit_vis(visib),
893892
}
894893
}
895894

‎compiler/rustc_ast/src/token.rs

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -963,6 +963,15 @@ impl Token {
963963
}
964964
}
965965

966+
/// Is this an invisible open delimiter at the start of a token sequence
967+
/// from an expanded metavar?
968+
pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
969+
match self.kind {
970+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
971+
_ => None,
972+
}
973+
}
974+
966975
pub fn glue(&self, joint: &Token) -> Option<Token> {
967976
let kind = match self.kind {
968977
Eq => match joint.kind {
@@ -1066,7 +1075,6 @@ pub enum Nonterminal {
10661075
/// Stuff inside brackets for attributes
10671076
NtMeta(P<ast::AttrItem>),
10681077
NtPath(P<ast::Path>),
1069-
NtVis(P<ast::Visibility>),
10701078
}
10711079

10721080
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -1163,7 +1171,6 @@ impl Nonterminal {
11631171
NtTy(ty) => ty.span,
11641172
NtMeta(attr_item) => attr_item.span(),
11651173
NtPath(path) => path.span,
1166-
NtVis(vis) => vis.span,
11671174
}
11681175
}
11691176

@@ -1178,7 +1185,6 @@ impl Nonterminal {
11781185
NtTy(..) => "type",
11791186
NtMeta(..) => "attribute",
11801187
NtPath(..) => "path",
1181-
NtVis(..) => "visibility",
11821188
}
11831189
}
11841190
}
@@ -1205,7 +1211,6 @@ impl fmt::Debug for Nonterminal {
12051211
NtLiteral(..) => f.pad("NtLiteral(..)"),
12061212
NtMeta(..) => f.pad("NtMeta(..)"),
12071213
NtPath(..) => f.pad("NtPath(..)"),
1208-
NtVis(..) => f.pad("NtVis(..)"),
12091214
}
12101215
}
12111216
}

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,6 @@ impl TokenStream {
471471
Nonterminal::NtTy(ty) => TokenStream::from_ast(ty),
472472
Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
473473
Nonterminal::NtPath(path) => TokenStream::from_ast(path),
474-
Nonterminal::NtVis(vis) => TokenStream::from_ast(vis),
475474
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
476475
}
477476
}

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ use std::mem;
22

33
use rustc_ast::ExprKind;
44
use rustc_ast::mut_visit::{self, MutVisitor};
5-
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, LitKind, Nonterminal, Token, TokenKind};
5+
use rustc_ast::token::{
6+
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
7+
TokenKind,
8+
};
69
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
710
use rustc_data_structures::fx::FxHashMap;
811
use rustc_data_structures::sync::Lrc;
@@ -253,7 +256,6 @@ pub(super) fn transcribe<'a>(
253256
}
254257
}
255258

256-
// Replace the meta-var with the matched token tree from the invocation.
257259
mbe::TokenTree::MetaVar(mut sp, mut original_ident) => {
258260
// Find the matched nonterminal from the macro invocation, and use it to replace
259261
// the meta-var.
@@ -273,6 +275,33 @@ pub(super) fn transcribe<'a>(
273275
// some of the unnecessary whitespace.
274276
let ident = MacroRulesNormalizedIdent::new(original_ident);
275277
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
278+
// We wrap the tokens in invisible delimiters, unless they are already wrapped
279+
// in invisible delimiters with the same `MetaVarKind`. Because there is no
280+
// point in having multiple layers of invisible delimiters of the same
281+
// `MetaVarKind`. Indeed, some proc macros can't handle them.
282+
let mut mk_delimited = |mv_kind, mut stream: TokenStream| {
283+
if stream.len() == 1 {
284+
let tree = stream.trees().next().unwrap();
285+
if let TokenTree::Delimited(_, _, delim, inner) = tree
286+
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mvk)) = delim
287+
&& mv_kind == *mvk
288+
{
289+
stream = inner.clone();
290+
}
291+
}
292+
293+
// Emit as a token stream within `Delimiter::Invisible` to maintain
294+
// parsing priorities.
295+
marker.visit_span(&mut sp);
296+
// Both the open delim and close delim get the same span, which covers the
297+
// `$foo` in the decl macro RHS.
298+
TokenTree::Delimited(
299+
DelimSpan::from_single(sp),
300+
DelimSpacing::new(Spacing::Alone, Spacing::Alone),
301+
Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)),
302+
stream,
303+
)
304+
};
276305
let tt = match cur_matched {
277306
MatchedSingle(ParseNtResult::Tt(tt)) => {
278307
// `tt`s are emitted into the output stream directly as "raw tokens",
@@ -289,6 +318,9 @@ pub(super) fn transcribe<'a>(
289318
let kind = token::NtLifetime(*ident, *is_raw);
290319
TokenTree::token_alone(kind, sp)
291320
}
321+
MatchedSingle(ParseNtResult::Vis(vis)) => {
322+
mk_delimited(MetaVarKind::Vis, TokenStream::from_ast(vis))
323+
}
292324
MatchedSingle(ParseNtResult::Nt(nt)) => {
293325
// Other variables are emitted into the output stream as groups with
294326
// `Delimiter::Invisible` to maintain parsing priorities.

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -720,6 +720,43 @@ impl<'a> Parser<'a> {
720720
if !self.eat_keyword(kw) { self.unexpected() } else { Ok(()) }
721721
}
722722

723+
/// Consume a sequence produced by a metavar expansion, if present.
724+
fn eat_metavar_seq<T>(
725+
&mut self,
726+
mv_kind: MetaVarKind,
727+
f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
728+
) -> Option<T> {
729+
self.eat_metavar_seq_with_matcher(|mvk| mvk == mv_kind, f)
730+
}
731+
732+
/// A slightly more general form of `eat_metavar_seq`, for use with the
733+
/// `MetaVarKind` variants that have parameters, where an exact match isn't
734+
/// desired.
735+
fn eat_metavar_seq_with_matcher<T>(
736+
&mut self,
737+
match_mv_kind: impl Fn(MetaVarKind) -> bool,
738+
mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
739+
) -> Option<T> {
740+
if let token::OpenDelim(delim) = self.token.kind
741+
&& let Delimiter::Invisible(token::InvisibleOrigin::MetaVar(mv_kind)) = delim
742+
&& match_mv_kind(mv_kind)
743+
{
744+
self.bump();
745+
let res = f(self).expect("failed to reparse {mv_kind:?}");
746+
if let token::CloseDelim(delim) = self.token.kind
747+
&& let Delimiter::Invisible(token::InvisibleOrigin::MetaVar(mv_kind)) = delim
748+
&& match_mv_kind(mv_kind)
749+
{
750+
self.bump();
751+
Some(res)
752+
} else {
753+
panic!("no close delim when reparsing {mv_kind:?}");
754+
}
755+
} else {
756+
None
757+
}
758+
}
759+
723760
/// Is the given keyword `kw` followed by a non-reserved identifier?
724761
fn is_kw_followed_by_ident(&self, kw: Symbol) -> bool {
725762
self.token.is_keyword(kw) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident())
@@ -1461,7 +1498,11 @@ impl<'a> Parser<'a> {
14611498
/// so emit a proper diagnostic.
14621499
// Public for rustfmt usage.
14631500
pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
1464-
maybe_whole!(self, NtVis, |vis| vis.into_inner());
1501+
if let Some(vis) = self
1502+
.eat_metavar_seq(MetaVarKind::Vis, |this| this.parse_visibility(FollowedByType::Yes))
1503+
{
1504+
return Ok(vis);
1505+
}
14651506

14661507
if !self.eat_keyword(kw::Pub) {
14671508
// We need a span for our `Spanned<VisibilityKind>`, but there's inherently no
@@ -1699,7 +1740,8 @@ pub enum ParseNtResult {
16991740
Tt(TokenTree),
17001741
Ident(Ident, IdentIsRaw),
17011742
Lifetime(Ident, IdentIsRaw),
1743+
Vis(P<ast::Visibility>),
17021744

1703-
/// This case will eventually be removed, along with `Token::Interpolate`.
1745+
/// This variant will eventually be removed, along with `Token::Interpolate`.
17041746
Nt(Lrc<Nonterminal>),
17051747
}

‎compiler/rustc_parse/src/parser/nonterminal.rs

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,7 @@ impl<'a> Parser<'a> {
5555
| NtMeta(_)
5656
| NtPath(_) => true,
5757

58-
NtItem(_)
59-
| NtBlock(_)
60-
| NtVis(_) => false,
58+
NtItem(_) | NtBlock(_) => false,
6159
}
6260
}
6361

@@ -87,7 +85,7 @@ impl<'a> Parser<'a> {
8785
NonterminalKind::Ident => get_macro_ident(token).is_some(),
8886
NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
8987
NonterminalKind::Vis => match token.kind {
90-
// The follow-set of :vis + "priv" keyword + interpolated
88+
// The follow-set of :vis + "priv" keyword + interpolated/metavar-expansion.
9189
token::Comma
9290
| token::Ident(..)
9391
| token::NtIdent(..)
@@ -101,7 +99,7 @@ impl<'a> Parser<'a> {
10199
token::NtLifetime(..) => true,
102100
token::Interpolated(nt) => match &**nt {
103101
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
104-
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
102+
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) => false,
105103
},
106104
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
107105
MetaVarKind::Block
@@ -207,8 +205,9 @@ impl<'a> Parser<'a> {
207205
}
208206
NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(ForceCollect::Yes)?)),
209207
NonterminalKind::Vis => {
210-
NtVis(P(self
211-
.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?))
208+
return Ok(ParseNtResult::Vis(P(self.collect_tokens_no_attrs(|this| {
209+
this.parse_visibility(FollowedByType::Yes)
210+
})?)));
212211
}
213212
NonterminalKind::Lifetime => {
214213
// We want to keep `'keyword` parsing, just like `keyword` is still

‎tests/ui/macros/block-to-expr-metavar.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
//@ check-pass
2+
//
3+
// A test case where a `block` fragment specifier is interpreted as an `expr`
4+
// fragment specifier. It's an interesting case for the handling of invisible
5+
// delimiters.
6+
7+
macro_rules! m_expr {
8+
($e:expr) => { const _CURRENT: u32 = $e; };
9+
}
10+
11+
macro_rules! m_block {
12+
($b:block) => ( m_expr!($b); );
13+
}
14+
15+
fn main() {
16+
m_block!({ 1 });
17+
}

0 commit comments

Comments
 (0)