Skip to content

Commit 848d2d2

Browse files
nnethercotennethercote
committed
[xtk-ui 1 failure] Don't skip invisible delimiters.
1 parent 8fc2809 commit 848d2d2

File tree

13 files changed

+261
-137
lines changed

13 files changed

+261
-137
lines changed

‎compiler/rustc_ast/src/token.rs

Lines changed: 23 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -62,26 +62,6 @@ pub enum InvisibleSource {
6262
// Converted from `proc_macro::Delimiter` in
6363
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
6464
ProcMacro,
65-
66-
// Converted from `TokenKind::Interpolated` in
67-
// `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
68-
FlattenToken,
69-
}
70-
71-
impl Delimiter {
72-
// Should the parser skip these delimiters? Only happens for certain kinds
73-
// of invisible delimiters. Once all interpolated nonterminals are removed,
74-
// the answer should become `false` for all kinds, whereupon this function
75-
// can be removed.
76-
pub fn skip(&self) -> bool {
77-
match self {
78-
Delimiter::Invisible(src) => match src {
79-
InvisibleSource::FlattenToken | InvisibleSource::ProcMacro => true,
80-
InvisibleSource::MetaVar(_) => false,
81-
},
82-
Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
83-
}
84-
}
8565
}
8666

8767
// Note that the suffix is *not* considered when deciding the `LitKind` in this
@@ -136,21 +116,8 @@ impl Lit {
136116
match token.uninterpolate().kind {
137117
Ident(name, false) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
138118
Literal(token_lit) => Some(token_lit),
139-
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Literal))) => {
140-
panic!("njn: FROM_TOKEN (1)");
141-
// if let NtExpr(expr) | NtLiteral(expr) = &**nt
142-
// && let ast::ExprKind::Lit(token_lit) = expr.kind =>
143-
// {
144-
// Some(token_lit)
145-
// }
146-
}
147-
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Expr))) => {
148-
panic!("njn: FROM_TOKEN (2)");
149-
// if let NtExpr(expr) | NtLiteral(expr) = &**nt
150-
// && let ast::ExprKind::Lit(token_lit) = expr.kind =>
151-
// {
152-
// Some(token_lit)
153-
// }
119+
OpenDelim(Delimiter::Invisible(source)) => {
120+
panic!("njn: from_token {source:?}");
154121
}
155122
_ => None,
156123
}
@@ -415,8 +382,8 @@ impl Token {
415382
match self.kind {
416383
InterpolatedIdent(_, _, uninterpolated_span)
417384
| InterpolatedLifetime(_, uninterpolated_span) => uninterpolated_span,
418-
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(kind))) => {
419-
panic!("njn: uninterpolated_span {kind:?}");
385+
OpenDelim(Delimiter::Invisible(source)) => {
386+
panic!("njn: uninterpolated_span {source:?}");
420387
}
421388
_ => self.span,
422389
}
@@ -473,8 +440,8 @@ impl Token {
473440
NonterminalKind::Expr |
474441
NonterminalKind::Literal |
475442
NonterminalKind::Path
476-
)))
477-
=> true,
443+
))) |
444+
OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
478445
_ => false,
479446
}
480447
}
@@ -501,7 +468,8 @@ impl Token {
501468
NonterminalKind::PatWithOr |
502469
NonterminalKind::Path |
503470
NonterminalKind::Literal
504-
))) => true,
471+
))) |
472+
OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
505473
_ => false,
506474
}
507475
}
@@ -524,7 +492,8 @@ impl Token {
524492
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
525493
NonterminalKind::Ty |
526494
NonterminalKind::Path
527-
))) => true,
495+
))) |
496+
OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
528497
_ => false,
529498
}
530499
}
@@ -536,6 +505,7 @@ impl Token {
536505
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
537506
NonterminalKind::Block | NonterminalKind::Expr | NonterminalKind::Literal,
538507
))) => true,
508+
OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
539509
_ => self.can_begin_literal_maybe_minus(),
540510
}
541511
}
@@ -592,7 +562,8 @@ impl Token {
592562
Ident(name, false) if name.is_bool_lit() => true,
593563
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
594564
NonterminalKind::Literal | NonterminalKind::Expr,
595-
))) => true,
565+
)))
566+
| OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
596567
_ => false,
597568
}
598569
}
@@ -658,6 +629,7 @@ impl Token {
658629
/// Would `maybe_reparse_metavar_expr` in `parser.rs` return `Ok(..)`?
659630
/// That is, is this a pre-parsed expression dropped into the token stream
660631
/// (which happens while parsing the result of macro expansion)?
632+
// njn: proc macro?
661633
pub fn is_metavar_expr(&self) -> bool {
662634
matches!(
663635
self.is_metavar_seq(),
@@ -672,6 +644,7 @@ impl Token {
672644

673645
/// Are we at a block from a metavar (`$b:block`)?
674646
pub fn is_metavar_block(&self) -> bool {
647+
// njn: handle proc-macro here too?
675648
matches!(self.is_metavar_seq(), Some(NonterminalKind::Block))
676649
}
677650

@@ -687,6 +660,7 @@ impl Token {
687660
pub fn is_path_start(&self) -> bool {
688661
self == &ModSep
689662
|| self.is_qpath_start()
663+
// njn: proc macro?
690664
|| matches!(self.is_metavar_seq(), Some(NonterminalKind::Path))
691665
|| self.is_path_segment_keyword()
692666
|| self.is_ident() && !self.is_reserved_ident()
@@ -760,6 +734,13 @@ impl Token {
760734
}
761735
}
762736

737+
/// Is this an invisible open delimiter at the start of a token sequence
738+
/// from a proc macro?
739+
// njn: need to use this more
740+
pub fn is_proc_macro_seq(&self) -> bool {
741+
matches!(self.kind, OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)))
742+
}
743+
763744
pub fn glue(&self, joint: &Token) -> Option<Token> {
764745
let kind = match self.kind {
765746
Eq => match joint.kind {

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
1616
use crate::ast::AttrStyle;
1717
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
18-
use crate::token::{self, Delimiter, InvisibleSource, Token, TokenKind};
18+
use crate::token::{self, Delimiter, InvisibleSource, NonterminalKind, Token, TokenKind};
1919
use crate::AttrVec;
2020

2121
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -456,9 +456,10 @@ impl TokenStream {
456456
Token::new(token::Ident(name, is_raw), uninterpolated_span),
457457
spacing,
458458
),
459+
// njn: not actually a metavar, but we know the kind
459460
token::InterpolatedLifetime(name, uninterpolated_span) => TokenTree::Delimited(
460461
DelimSpan::from_single(token.span),
461-
Delimiter::Invisible(InvisibleSource::FlattenToken),
462+
Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Lifetime)),
462463
TokenStream::token_alone(token::Lifetime(name), uninterpolated_span),
463464
),
464465
_ => TokenTree::Token(*token, spacing),
@@ -474,6 +475,7 @@ impl TokenStream {
474475
}
475476
}
476477

478+
// njn: do we still need this?
477479
#[must_use]
478480
pub fn flattened(&self) -> TokenStream {
479481
fn can_skip(stream: &TokenStream) -> bool {

‎compiler/rustc_expand/src/mbe/macro_rules.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -721,8 +721,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
721721
ident == sym::compile_error &&
722722
let mbe::TokenTree::Token(bang) = bang &&
723723
let TokenKind::Not = bang.kind &&
724-
let mbe::TokenTree::Delimited(_, del) = args &&
725-
!del.delim.skip()
724+
let mbe::TokenTree::Delimited(..) = args
726725
{
727726
true
728727
} else {

‎compiler/rustc_expand/src/mbe/quoted.rs

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -147,17 +147,9 @@ fn parse_tree<'a>(
147147
match tree {
148148
// `tree` is a `$` token. Look at the next token in `trees`
149149
&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
150-
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
151-
// during parsing.
152-
let mut next = outer_trees.next();
153-
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
154-
match next {
155-
Some(tokenstream::TokenTree::Delimited(_, delim, tts)) if delim.skip() => {
156-
trees = Box::new(tts.trees());
157-
next = trees.next();
158-
}
159-
_ => trees = Box::new(outer_trees),
160-
}
150+
let next = outer_trees.next();
151+
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>> =
152+
Box::new(outer_trees);
161153

162154
match next {
163155
// `tree` is followed by a delimited set of token trees.

‎compiler/rustc_parse/src/parser/expr.rs

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -709,6 +709,7 @@ impl<'a> Parser<'a> {
709709
}
710710

711711
/// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
712+
// njn: needs adjusting for ProcMacro?
712713
fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
713714
match self.prev_token.kind {
714715
TokenKind::InterpolatedIdent(..) | TokenKind::InterpolatedLifetime(..) => {
@@ -1363,16 +1364,39 @@ impl<'a> Parser<'a> {
13631364
/// correctly if called from `parse_dot_or_call_expr()`.
13641365
fn parse_expr_bottom(&mut self) -> PResult<'a, P<Expr>> {
13651366
maybe_recover_from_interpolated_ty_qpath!(self, true);
1366-
//eprintln!("AAA {:?}", self.token);
1367+
//eprintln!("AAA {:?}", self.token.kind);
13671368
maybe_reparse_metavar_expr!(self);
1368-
//eprintln!("BBB {:?}", self.token);
1369+
//eprintln!("BBB {:?}", self.token.kind);
1370+
1371+
if let token::OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) = self.token.kind
1372+
{
1373+
//eprintln!("BUMP {:?}", self.token.kind);
1374+
self.bump();
1375+
// njn: parse_expr_force_collect?
1376+
match self.parse_expr() {
1377+
Ok(expr) => {
1378+
match self.expect(&token::CloseDelim(Delimiter::Invisible(
1379+
InvisibleSource::ProcMacro,
1380+
))) {
1381+
Ok(_) => {
1382+
return Ok(expr);
1383+
}
1384+
Err(_) => panic!("njn: no invisible close delim: {:?}", self.token),
1385+
}
1386+
}
1387+
Err(_) => {
1388+
panic!("njn: bad expr parse");
1389+
}
1390+
}
1391+
}
1392+
//eprintln!("CCC {:?}", self.token.kind);
13691393

13701394
// Outer attributes are already parsed and will be
13711395
// added to the return value after the fact.
13721396

13731397
// Note: when adding new syntax here, don't forget to adjust `TokenKind::can_begin_expr()`.
13741398
let lo = self.token.span;
1375-
if let token::Literal(_) = self.token.kind {
1399+
let res = if let token::Literal(_) = self.token.kind {
13761400
// This match arm is a special-case of the `_` match arm below and
13771401
// could be removed without changing functionality, but it's faster
13781402
// to have it here, especially for programs with large constants.
@@ -1473,7 +1497,9 @@ impl<'a> Parser<'a> {
14731497
}
14741498
} else {
14751499
self.parse_expr_lit()
1476-
}
1500+
};
1501+
//eprintln!("DDD {:?} -> {:#?}", self.token.kind, res);
1502+
res
14771503
}
14781504

14791505
fn parse_expr_lit(&mut self) -> PResult<'a, P<Expr>> {
@@ -1964,6 +1990,7 @@ impl<'a> Parser<'a> {
19641990
&mut self,
19651991
mk_lit_char: impl FnOnce(Symbol, Span) -> L,
19661992
) -> PResult<'a, L> {
1993+
// njn: proc macro too?
19671994
if let Some(NonterminalKind::Expr | NonterminalKind::Literal) = self.token.is_metavar_seq()
19681995
{
19691996
// njn: not checking for ExprKind::Err
@@ -2090,6 +2117,9 @@ impl<'a> Parser<'a> {
20902117
None
20912118
}
20922119
}
2120+
token::OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => {
2121+
panic!("njn: maybe_parse_token_lit");
2122+
}
20932123
_ => None,
20942124
}
20952125
}
@@ -2157,6 +2187,25 @@ impl<'a> Parser<'a> {
21572187
pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
21582188
maybe_reparse_metavar_expr!(self);
21592189

2190+
// if let token::OpenDelim(Delimiter::Invisible) = self.token.kind {
2191+
// // njn: need parse_expr for negative case?
2192+
// self.bump();
2193+
// // njn: parse_expr_force_collect?
2194+
// match self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus()) {
2195+
// Ok(expr) => {
2196+
// match self.expect(&token::CloseDelim(Delimiter::Invisible)) {
2197+
// Ok(_) => {
2198+
// return Ok(expr);
2199+
// }
2200+
// Err(_) => panic!("njn: no invisible close delim: {:?}", self.token),
2201+
// }
2202+
// }
2203+
// Err(_) => {
2204+
// panic!("njn: bad expr parse");
2205+
// }
2206+
// }
2207+
// }
2208+
21602209
let lo = self.token.span;
21612210
let minus_present = self.eat(&token::BinOp(token::Minus));
21622211
let (token_lit, span) = self.parse_token_lit()?;

‎compiler/rustc_parse/src/parser/item.rs

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,36 @@ impl<'a> Parser<'a> {
130130
return Ok(Some(item.into_inner()));
131131
}
132132

133+
if let token::OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) = self.token.kind
134+
{
135+
//eprintln!("ITEM BUMP {:?}", self.token.kind);
136+
self.bump();
137+
match self.parse_item(ForceCollect::Yes) {
138+
Ok(Some(mut item)) => {
139+
match self.expect(&token::CloseDelim(Delimiter::Invisible(
140+
InvisibleSource::ProcMacro,
141+
))) {
142+
Ok(_) => {
143+
attrs.prepend_to_nt_inner(&mut item.attrs);
144+
return Ok(Some(item.into_inner()));
145+
}
146+
Err(_) => panic!("njn: no invisible close delim 1: {:?}", self.token),
147+
}
148+
}
149+
Ok(None) => {
150+
panic!("njn: missing item {:?}", self.token);
151+
// match self.expect(&token::CloseDelim(Delimiter::Invisible)) {
152+
// Ok(_) => return Ok(None),
153+
// // njn: hitting on tests/ui/proc-macro/issue-75734-pp-paren.rs, hmm
154+
// Err(_) => panic!("njn: no invisible close delim 2: {:?}", self.token),
155+
// }
156+
}
157+
Err(_) => {
158+
panic!("njn: bad item parse");
159+
}
160+
}
161+
}
162+
133163
let item =
134164
self.collect_tokens_trailing_token(attrs, force_collect, |this: &mut Self, attrs| {
135165
let item =

0 commit comments

Comments
 (0)