Skip to content

Commit 4286768

Browse files
borsbors
committed
Auto merge of #133436 - nnethercote:rm-NtVis-NtTy, r=<try>
Remove `NtVis` and `NtTy` The next part of #124141. The first actual remove of `Nonterminal` variants. `NtVis` is a simple case that doesn't get much use, but `NtTy` is more complex. r? `@petrochenkov`
2 parents f2abf82 + 5552feb commit 4286768

20 files changed

+197
-60
lines changed

‎compiler/rustc_ast/src/ast_traits.rs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -203,10 +203,8 @@ impl HasTokens for Nonterminal {
203203
Nonterminal::NtStmt(stmt) => stmt.tokens(),
204204
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
205205
Nonterminal::NtPat(pat) => pat.tokens(),
206-
Nonterminal::NtTy(ty) => ty.tokens(),
207206
Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
208207
Nonterminal::NtPath(path) => path.tokens(),
209-
Nonterminal::NtVis(vis) => vis.tokens(),
210208
Nonterminal::NtBlock(block) => block.tokens(),
211209
}
212210
}
@@ -216,10 +214,8 @@ impl HasTokens for Nonterminal {
216214
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
217215
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
218216
Nonterminal::NtPat(pat) => pat.tokens_mut(),
219-
Nonterminal::NtTy(ty) => ty.tokens_mut(),
220217
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
221218
Nonterminal::NtPath(path) => path.tokens_mut(),
222-
Nonterminal::NtVis(vis) => vis.tokens_mut(),
223219
Nonterminal::NtBlock(block) => block.tokens_mut(),
224220
}
225221
}

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -884,7 +884,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
884884
}),
885885
token::NtPat(pat) => vis.visit_pat(pat),
886886
token::NtExpr(expr) => vis.visit_expr(expr),
887-
token::NtTy(ty) => vis.visit_ty(ty),
888887
token::NtLiteral(expr) => vis.visit_expr(expr),
889888
token::NtMeta(item) => {
890889
let AttrItem { unsafety: _, path, args, tokens } = item.deref_mut();
@@ -893,7 +892,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
893892
visit_lazy_tts(vis, tokens);
894893
}
895894
token::NtPath(path) => vis.visit_path(path),
896-
token::NtVis(visib) => vis.visit_vis(visib),
897895
}
898896
}
899897

‎compiler/rustc_ast/src/token.rs

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -660,7 +660,6 @@ impl Token {
660660
| NtMeta(..)
661661
| NtPat(..)
662662
| NtPath(..)
663-
| NtTy(..)
664663
),
665664
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
666665
MetaVarKind::Expr { .. } |
@@ -689,7 +688,7 @@ impl Token {
689688
Lifetime(..) | // lifetime bound in trait object
690689
Lt | BinOp(Shl) | // associated path
691690
PathSep => true, // global path
692-
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
691+
Interpolated(ref nt) => matches!(&**nt, NtPath(..)),
693692
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
694693
MetaVarKind::Ty |
695694
MetaVarKind::Path
@@ -963,6 +962,15 @@ impl Token {
963962
}
964963
}
965964

965+
/// Is this an invisible open delimiter at the start of a token sequence
966+
/// from an expanded metavar?
967+
pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
968+
match self.kind {
969+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
970+
_ => None,
971+
}
972+
}
973+
966974
pub fn glue(&self, joint: &Token) -> Option<Token> {
967975
let kind = match self.kind {
968976
Eq => match joint.kind {
@@ -1061,12 +1069,10 @@ pub enum Nonterminal {
10611069
NtStmt(P<ast::Stmt>),
10621070
NtPat(P<ast::Pat>),
10631071
NtExpr(P<ast::Expr>),
1064-
NtTy(P<ast::Ty>),
10651072
NtLiteral(P<ast::Expr>),
10661073
/// Stuff inside brackets for attributes
10671074
NtMeta(P<ast::AttrItem>),
10681075
NtPath(P<ast::Path>),
1069-
NtVis(P<ast::Visibility>),
10701076
}
10711077

10721078
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -1160,10 +1166,8 @@ impl Nonterminal {
11601166
NtStmt(stmt) => stmt.span,
11611167
NtPat(pat) => pat.span,
11621168
NtExpr(expr) | NtLiteral(expr) => expr.span,
1163-
NtTy(ty) => ty.span,
11641169
NtMeta(attr_item) => attr_item.span(),
11651170
NtPath(path) => path.span,
1166-
NtVis(vis) => vis.span,
11671171
}
11681172
}
11691173

@@ -1175,10 +1179,8 @@ impl Nonterminal {
11751179
NtPat(..) => "pattern",
11761180
NtExpr(..) => "expression",
11771181
NtLiteral(..) => "literal",
1178-
NtTy(..) => "type",
11791182
NtMeta(..) => "attribute",
11801183
NtPath(..) => "path",
1181-
NtVis(..) => "visibility",
11821184
}
11831185
}
11841186
}
@@ -1201,11 +1203,9 @@ impl fmt::Debug for Nonterminal {
12011203
NtStmt(..) => f.pad("NtStmt(..)"),
12021204
NtPat(..) => f.pad("NtPat(..)"),
12031205
NtExpr(..) => f.pad("NtExpr(..)"),
1204-
NtTy(..) => f.pad("NtTy(..)"),
12051206
NtLiteral(..) => f.pad("NtLiteral(..)"),
12061207
NtMeta(..) => f.pad("NtMeta(..)"),
12071208
NtPath(..) => f.pad("NtPath(..)"),
1208-
NtVis(..) => f.pad("NtVis(..)"),
12091209
}
12101210
}
12111211
}

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -468,10 +468,8 @@ impl TokenStream {
468468
}
469469
Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
470470
Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
471-
Nonterminal::NtTy(ty) => TokenStream::from_ast(ty),
472471
Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
473472
Nonterminal::NtPath(path) => TokenStream::from_ast(path),
474-
Nonterminal::NtVis(vis) => TokenStream::from_ast(vis),
475473
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
476474
}
477475
}

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ use std::mem;
22

33
use rustc_ast::ExprKind;
44
use rustc_ast::mut_visit::{self, MutVisitor};
5-
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, LitKind, Nonterminal, Token, TokenKind};
5+
use rustc_ast::token::{
6+
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
7+
TokenKind,
8+
};
69
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
710
use rustc_data_structures::fx::FxHashMap;
811
use rustc_data_structures::sync::Lrc;
@@ -253,7 +256,6 @@ pub(super) fn transcribe<'a>(
253256
}
254257
}
255258

256-
// Replace the meta-var with the matched token tree from the invocation.
257259
mbe::TokenTree::MetaVar(mut sp, mut original_ident) => {
258260
// Find the matched nonterminal from the macro invocation, and use it to replace
259261
// the meta-var.
@@ -273,6 +275,33 @@ pub(super) fn transcribe<'a>(
273275
// some of the unnecessary whitespace.
274276
let ident = MacroRulesNormalizedIdent::new(original_ident);
275277
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
278+
// We wrap the tokens in invisible delimiters, unless they are already wrapped
279+
// in invisible delimiters with the same `MetaVarKind`. Because there is no
280+
// point in having multiple layers of invisible delimiters of the same
281+
// `MetaVarKind`. Indeed, some proc macros can't handle them.
282+
let mut mk_delimited = |mv_kind, mut stream: TokenStream| {
283+
if stream.len() == 1 {
284+
let tree = stream.trees().next().unwrap();
285+
if let TokenTree::Delimited(_, _, delim, inner) = tree
286+
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mvk)) = delim
287+
&& mv_kind == *mvk
288+
{
289+
stream = inner.clone();
290+
}
291+
}
292+
293+
// Emit as a token stream within `Delimiter::Invisible` to maintain
294+
// parsing priorities.
295+
marker.visit_span(&mut sp);
296+
// Both the open delim and close delim get the same span, which covers the
297+
// `$foo` in the decl macro RHS.
298+
TokenTree::Delimited(
299+
DelimSpan::from_single(sp),
300+
DelimSpacing::new(Spacing::Alone, Spacing::Alone),
301+
Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)),
302+
stream,
303+
)
304+
};
276305
let tt = match cur_matched {
277306
MatchedSingle(ParseNtResult::Tt(tt)) => {
278307
// `tt`s are emitted into the output stream directly as "raw tokens",
@@ -289,6 +318,12 @@ pub(super) fn transcribe<'a>(
289318
let kind = token::NtLifetime(*ident, *is_raw);
290319
TokenTree::token_alone(kind, sp)
291320
}
321+
MatchedSingle(ParseNtResult::Ty(ty)) => {
322+
mk_delimited(MetaVarKind::Ty, TokenStream::from_ast(ty))
323+
}
324+
MatchedSingle(ParseNtResult::Vis(vis)) => {
325+
mk_delimited(MetaVarKind::Vis, TokenStream::from_ast(vis))
326+
}
292327
MatchedSingle(ParseNtResult::Nt(nt)) => {
293328
// Other variables are emitted into the output stream as groups with
294329
// `Delimiter::Invisible` to maintain parsing priorities.

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 72 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -116,12 +116,16 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath {
116116
($self: expr, $allow_qpath_recovery: expr) => {
117117
if $allow_qpath_recovery
118118
&& $self.may_recover()
119-
&& $self.look_ahead(1, |t| t == &token::PathSep)
120-
&& let token::Interpolated(nt) = &$self.token.kind
121-
&& let token::NtTy(ty) = &**nt
119+
&& let Some(token::MetaVarKind::Ty) = $self.token.is_metavar_seq()
120+
&& $self.check_noexpect_past_close_delim(&token::PathSep)
122121
{
123-
let ty = ty.clone();
124-
$self.bump();
122+
// Reparse the type, then move to recovery.
123+
let ty = $self
124+
.eat_metavar_seq(token::MetaVarKind::Ty, |this| {
125+
this.parse_ty_no_question_mark_recover()
126+
})
127+
.expect("metavar seq ty");
128+
125129
return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_token.span, ty);
126130
}
127131
};
@@ -611,6 +615,24 @@ impl<'a> Parser<'a> {
611615
self.token == *tok
612616
}
613617

618+
// Check the first token after the delimiter that closes the current
619+
// delimited sequence. (Panics if used in the outermost token stream, which
620+
// has no delimiters.) It uses a clone of the relevant tree cursor to skip
621+
// past the entire `TokenTree::Delimited` in a single step, avoiding the
622+
// need for unbounded token lookahead.
623+
//
624+
// Primarily used when `self.token` matches
625+
// `OpenDelim(Delimiter::Invisible(_))`, to look ahead through the current
626+
// metavar expansion.
627+
fn check_noexpect_past_close_delim(&self, tok: &TokenKind) -> bool {
628+
let mut tree_cursor = self.token_cursor.stack.last().unwrap().0.clone();
629+
let tt = tree_cursor.next_ref();
630+
matches!(
631+
tt,
632+
Some(ast::tokenstream::TokenTree::Token(token::Token { kind, .. }, _)) if kind == tok
633+
)
634+
}
635+
614636
/// Consumes a token 'tok' if it exists. Returns whether the given token was present.
615637
///
616638
/// the main purpose of this function is to reduce the cluttering of the suggestions list
@@ -720,6 +742,43 @@ impl<'a> Parser<'a> {
720742
if !self.eat_keyword(kw) { self.unexpected() } else { Ok(()) }
721743
}
722744

745+
/// Consume a sequence produced by a metavar expansion, if present.
746+
fn eat_metavar_seq<T>(
747+
&mut self,
748+
mv_kind: MetaVarKind,
749+
f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
750+
) -> Option<T> {
751+
self.eat_metavar_seq_with_matcher(|mvk| mvk == mv_kind, f)
752+
}
753+
754+
/// A slightly more general form of `eat_metavar_seq`, for use with the
755+
/// `MetaVarKind` variants that have parameters, where an exact match isn't
756+
/// desired.
757+
fn eat_metavar_seq_with_matcher<T>(
758+
&mut self,
759+
match_mv_kind: impl Fn(MetaVarKind) -> bool,
760+
mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
761+
) -> Option<T> {
762+
if let token::OpenDelim(delim) = self.token.kind
763+
&& let Delimiter::Invisible(token::InvisibleOrigin::MetaVar(mv_kind)) = delim
764+
&& match_mv_kind(mv_kind)
765+
{
766+
self.bump();
767+
let res = f(self).expect("failed to reparse {mv_kind:?}");
768+
if let token::CloseDelim(delim) = self.token.kind
769+
&& let Delimiter::Invisible(token::InvisibleOrigin::MetaVar(mv_kind)) = delim
770+
&& match_mv_kind(mv_kind)
771+
{
772+
self.bump();
773+
Some(res)
774+
} else {
775+
panic!("no close delim when reparsing {mv_kind:?}");
776+
}
777+
} else {
778+
None
779+
}
780+
}
781+
723782
/// Is the given keyword `kw` followed by a non-reserved identifier?
724783
fn is_kw_followed_by_ident(&self, kw: Symbol) -> bool {
725784
self.token.is_keyword(kw) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident())
@@ -1461,7 +1520,11 @@ impl<'a> Parser<'a> {
14611520
/// so emit a proper diagnostic.
14621521
// Public for rustfmt usage.
14631522
pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
1464-
maybe_whole!(self, NtVis, |vis| vis.into_inner());
1523+
if let Some(vis) = self
1524+
.eat_metavar_seq(MetaVarKind::Vis, |this| this.parse_visibility(FollowedByType::Yes))
1525+
{
1526+
return Ok(vis);
1527+
}
14651528

14661529
if !self.eat_keyword(kw::Pub) {
14671530
// We need a span for our `Spanned<VisibilityKind>`, but there's inherently no
@@ -1699,7 +1762,9 @@ pub enum ParseNtResult {
16991762
Tt(TokenTree),
17001763
Ident(Ident, IdentIsRaw),
17011764
Lifetime(Ident, IdentIsRaw),
1765+
Ty(P<ast::Ty>),
1766+
Vis(P<ast::Visibility>),
17021767

1703-
/// This case will eventually be removed, along with `Token::Interpolate`.
1768+
/// This variant will eventually be removed, along with `Token::Interpolate`.
17041769
Nt(Lrc<Nonterminal>),
17051770
}

‎compiler/rustc_parse/src/parser/nonterminal.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,11 @@ impl<'a> Parser<'a> {
5050
NtStmt(_)
5151
| NtPat(_)
5252
| NtExpr(_)
53-
| NtTy(_)
5453
| NtLiteral(_) // `true`, `false`
5554
| NtMeta(_)
5655
| NtPath(_) => true,
5756

58-
NtItem(_)
59-
| NtBlock(_)
60-
| NtVis(_) => false,
57+
NtItem(_) | NtBlock(_) => false,
6158
}
6259
}
6360

@@ -87,7 +84,7 @@ impl<'a> Parser<'a> {
8784
NonterminalKind::Ident => get_macro_ident(token).is_some(),
8885
NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
8986
NonterminalKind::Vis => match token.kind {
90-
// The follow-set of :vis + "priv" keyword + interpolated
87+
// The follow-set of :vis + "priv" keyword + interpolated/metavar-expansion.
9188
token::Comma
9289
| token::Ident(..)
9390
| token::NtIdent(..)
@@ -101,7 +98,7 @@ impl<'a> Parser<'a> {
10198
token::NtLifetime(..) => true,
10299
token::Interpolated(nt) => match &**nt {
103100
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
104-
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
101+
NtItem(_) | NtPat(_) | NtMeta(_) | NtPath(_) => false,
105102
},
106103
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
107104
MetaVarKind::Block
@@ -188,7 +185,9 @@ impl<'a> Parser<'a> {
188185
NtLiteral(self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?)
189186
}
190187
NonterminalKind::Ty => {
191-
NtTy(self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?)
188+
return Ok(ParseNtResult::Ty(
189+
self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?,
190+
));
192191
}
193192
// this could be handled like a token, since it is one
194193
NonterminalKind::Ident => {
@@ -207,8 +206,9 @@ impl<'a> Parser<'a> {
207206
}
208207
NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(ForceCollect::Yes)?)),
209208
NonterminalKind::Vis => {
210-
NtVis(P(self
211-
.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?))
209+
return Ok(ParseNtResult::Vis(P(self.collect_tokens_no_attrs(|this| {
210+
this.parse_visibility(FollowedByType::Yes)
211+
})?)));
212212
}
213213
NonterminalKind::Lifetime => {
214214
// We want to keep `'keyword` parsing, just like `keyword` is still

‎compiler/rustc_parse/src/parser/path.rs

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use std::mem;
22

33
use ast::token::IdentIsRaw;
44
use rustc_ast::ptr::P;
5-
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
5+
use rustc_ast::token::{self, Delimiter, MetaVarKind, Token, TokenKind};
66
use rustc_ast::{
77
self as ast, AngleBracketedArg, AngleBracketedArgs, AnonConst, AssocItemConstraint,
88
AssocItemConstraintKind, BlockCheckMode, GenericArg, GenericArgs, Generics, ParenthesizedArgs,
@@ -198,13 +198,14 @@ impl<'a> Parser<'a> {
198198

199199
maybe_whole!(self, NtPath, |path| reject_generics_if_mod_style(self, path.into_inner()));
200200

201-
if let token::Interpolated(nt) = &self.token.kind {
202-
if let token::NtTy(ty) = &**nt {
203-
if let ast::TyKind::Path(None, path) = &ty.kind {
204-
let path = path.clone();
205-
self.bump();
206-
return Ok(reject_generics_if_mod_style(self, path));
207-
}
201+
if let Some(MetaVarKind::Ty) = self.token.is_metavar_seq() {
202+
let mut snapshot = self.create_snapshot_for_diagnostic();
203+
let ty = snapshot
204+
.eat_metavar_seq(MetaVarKind::Ty, |this| this.parse_ty_no_question_mark_recover())
205+
.expect("metavar seq ty");
206+
if let ast::TyKind::Path(None, path) = ty.into_inner().kind {
207+
self.restore_snapshot(snapshot);
208+
return Ok(reject_generics_if_mod_style(self, path));
208209
}
209210
}
210211

0 commit comments

Comments
 (0)