Skip to content

Commit 7a915dd

Browse files
matthiaskrgrmatthiaskrgr
authored
Rollup merge of #96682 - nnethercote:show-invisible-delims, r=petrochenkov
Show invisible delimeters (within comments) when pretty printing. Because invisible syntax is really hard to work with! r? `@petrochenkov`
2 parents 5f8a2f6 + 5022d76 commit 7a915dd

17 files changed

+110
-68
lines changed

‎compiler/rustc_ast/src/token.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,12 @@ pub enum Delimiter {
5050
Brace,
5151
/// `[ ... ]`
5252
Bracket,
53-
/// `Ø ... Ø`
53+
/// `/*«*/ ... /*»*/`
5454
/// An invisible delimiter, that may, for example, appear around tokens coming from a
5555
/// "macro variable" `$var`. It is important to preserve operator priorities in cases like
5656
/// `$var * 3` where `$var` is `1 + 2`.
57-
/// Invisible delimiters might not survive roundtrip of a token stream through a string.
57+
/// Invisible delimiters are not directly writable in normal Rust code except as comments.
58+
/// Therefore, they might not survive a roundtrip of a token stream through a string.
5859
Invisible,
5960
}
6061

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -590,15 +590,29 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
590590
self.nbsp();
591591
}
592592
self.word("{");
593-
if !tts.is_empty() {
593+
let empty = tts.is_empty();
594+
if !empty {
594595
self.space();
595596
}
596597
self.ibox(0);
597598
self.print_tts(tts, convert_dollar_crate);
598599
self.end();
599-
let empty = tts.is_empty();
600600
self.bclose(span, empty);
601601
}
602+
Some(Delimiter::Invisible) => {
603+
self.word("/*«*/");
604+
let empty = tts.is_empty();
605+
if !empty {
606+
self.space();
607+
}
608+
self.ibox(0);
609+
self.print_tts(tts, convert_dollar_crate);
610+
self.end();
611+
if !empty {
612+
self.space();
613+
}
614+
self.word("/*»*/");
615+
}
602616
Some(delim) => {
603617
let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
604618
self.word(token_str);
@@ -772,9 +786,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
772786
token::CloseDelim(Delimiter::Bracket) => "]".into(),
773787
token::OpenDelim(Delimiter::Brace) => "{".into(),
774788
token::CloseDelim(Delimiter::Brace) => "}".into(),
775-
token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
776-
"".into()
777-
}
789+
token::OpenDelim(Delimiter::Invisible) => "/*«*/".into(),
790+
token::CloseDelim(Delimiter::Invisible) => "/*»*/".into(),
778791
token::Pound => "#".into(),
779792
token::Dollar => "$".into(),
780793
token::Question => "?".into(),

‎library/proc_macro/src/lib.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -703,11 +703,12 @@ pub enum Delimiter {
703703
/// `[ ... ]`
704704
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
705705
Bracket,
706-
/// `Ø ... Ø`
706+
/// `/*«*/ ... /*»*/`
707707
/// An invisible delimiter, that may, for example, appear around tokens coming from a
708708
/// "macro variable" `$var`. It is important to preserve operator priorities in cases like
709709
/// `$var * 3` where `$var` is `1 + 2`.
710-
/// Invisible delimiters might not survive roundtrip of a token stream through a string.
710+
/// Invisible delimiters are not directly writable in normal Rust code except as comments.
711+
/// Therefore, they might not survive a roundtrip of a token stream through a string.
711712
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
712713
None,
713714
}

‎src/test/ui/proc-macro/auxiliary/expand-expr.rs

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,15 @@ use std::str::FromStr;
1212

1313
#[proc_macro]
1414
pub fn expand_expr_is(input: TokenStream) -> TokenStream {
15+
expand_expr_is_inner(input, false)
16+
}
17+
18+
#[proc_macro]
19+
pub fn expand_expr_is_trim(input: TokenStream) -> TokenStream {
20+
expand_expr_is_inner(input, true)
21+
}
22+
23+
fn expand_expr_is_inner(input: TokenStream, trim_invisible: bool) -> TokenStream {
1524
let mut iter = input.into_iter();
1625
let mut expected_tts = Vec::new();
1726
loop {
@@ -22,14 +31,18 @@ pub fn expand_expr_is(input: TokenStream) -> TokenStream {
2231
}
2332
}
2433

25-
let expected = expected_tts.into_iter().collect::<TokenStream>();
26-
let expanded = iter.collect::<TokenStream>().expand_expr().expect("expand_expr failed");
27-
assert!(
28-
expected.to_string() == expanded.to_string(),
29-
"assert failed\nexpected: `{}`\nexpanded: `{}`",
30-
expected.to_string(),
31-
expanded.to_string()
32-
);
34+
// If requested, trim the "invisible" delimiters at the start and end.
35+
let expected = expected_tts.into_iter().collect::<TokenStream>().to_string();
36+
let expected = if trim_invisible {
37+
let len1 = "/*«*/ ".len();
38+
let len2 = " /*»*/".len();
39+
&expected[len1..expected.len() - len2]
40+
} else {
41+
&expected[..]
42+
};
43+
let expanded = iter.collect::<TokenStream>().expand_expr().unwrap().to_string();
44+
45+
assert_eq!(expected, expanded);
3346

3447
TokenStream::new()
3548
}

‎src/test/ui/proc-macro/capture-macro-rules-invoke.stdout

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
PRINT-BANG INPUT (DISPLAY): self
2+
PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ self /*»*/
23
PRINT-BANG INPUT (DEBUG): TokenStream [
34
Group {
45
delimiter: None,
@@ -13,8 +14,10 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
1314
]
1415
PRINT-BANG INPUT (DISPLAY): 1 + 1, { "a" }, let a = 1;, String, my_name, 'a, my_val = 30,
1516
std::option::Option, pub(in some::path) , [a b c], -30
16-
PRINT-BANG RE-COLLECTED (DISPLAY): 1 + 1, { "a" }, let a = 1, String, my_name, 'a, my_val = 30,
17-
std :: option :: Option, pub(in some :: path), [a b c], - 30
17+
PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 1 + 1 /*»*/, /*«*/ { "a" } /*»*/, /*«*/ let a = 1 /*»*/, /*«*/
18+
String /*»*/, my_name, /*«*/ 'a /*»*/, /*«*/ my_val = 30 /*»*/, /*«*/
19+
std :: option :: Option /*»*/, /*«*/ pub(in some :: path) /*»*/, [a b c],
20+
/*«*/ - 30 /*»*/
1821
PRINT-BANG INPUT (DEBUG): TokenStream [
1922
Group {
2023
delimiter: None,
@@ -295,6 +298,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
295298
},
296299
]
297300
PRINT-BANG INPUT (DISPLAY): (a, b)
301+
PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ (a, b) /*»*/
298302
PRINT-BANG INPUT (DEBUG): TokenStream [
299303
Group {
300304
delimiter: None,

‎src/test/ui/proc-macro/capture-unglued-token.stdout

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
PRINT-BANG INPUT (DISPLAY): Vec<u8>
2-
PRINT-BANG RE-COLLECTED (DISPLAY): Vec < u8 >
2+
PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ Vec < u8 > /*»*/
33
PRINT-BANG INPUT (DEBUG): TokenStream [
44
Group {
55
delimiter: None,

‎src/test/ui/proc-macro/expand-expr.rs

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
extern crate expand_expr;
44

5-
use expand_expr::{
6-
check_expand_expr_file, echo_pm, expand_expr_fail, expand_expr_is, recursive_expand,
7-
};
5+
use expand_expr::{check_expand_expr_file, echo_pm, expand_expr_fail, expand_expr_is};
6+
use expand_expr::{expand_expr_is_trim, recursive_expand};
7+
88

99
// Check builtin macros can be expanded.
1010

@@ -47,21 +47,21 @@ macro_rules! echo_expr {
4747

4848
macro_rules! simple_lit {
4949
($l:literal) => {
50-
expand_expr_is!($l, $l);
51-
expand_expr_is!($l, echo_lit!($l));
52-
expand_expr_is!($l, echo_expr!($l));
53-
expand_expr_is!($l, echo_tts!($l));
54-
expand_expr_is!($l, echo_pm!($l));
50+
expand_expr_is_trim!($l, $l);
51+
expand_expr_is_trim!($l, echo_lit!($l));
52+
expand_expr_is_trim!($l, echo_expr!($l));
53+
expand_expr_is_trim!($l, echo_tts!($l));
54+
expand_expr_is_trim!($l, echo_pm!($l));
5555
const _: () = {
5656
macro_rules! mac {
5757
() => {
5858
$l
5959
};
6060
}
61-
expand_expr_is!($l, mac!());
62-
expand_expr_is!($l, echo_expr!(mac!()));
63-
expand_expr_is!($l, echo_tts!(mac!()));
64-
expand_expr_is!($l, echo_pm!(mac!()));
61+
expand_expr_is_trim!($l, mac!());
62+
expand_expr_is_trim!($l, echo_expr!(mac!()));
63+
expand_expr_is_trim!($l, echo_tts!(mac!()));
64+
expand_expr_is_trim!($l, echo_pm!(mac!()));
6565
};
6666
};
6767
}

‎src/test/ui/proc-macro/expr-stmt-nonterminal-tokens.stdout

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = #[allow(warnings)] 0 ; 0 }, }
2-
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = #[allow(warnings)] #[allow(warnings)] 0 ; 0 }, }
2+
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E
3+
{ V = { let _ = /*«*/ #[allow(warnings)] #[allow(warnings)] 0 /*»*/ ; 0 }, }
34
PRINT-DERIVE INPUT (DEBUG): TokenStream [
45
Ident {
56
ident: "enum",
@@ -123,7 +124,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
123124
},
124125
]
125126
PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { 0; } ; 0 }, }
126-
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { 0 } ; 0 }, }
127+
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ 0 /*»*/ } ; 0 }, }
127128
PRINT-DERIVE INPUT (DEBUG): TokenStream [
128129
Ident {
129130
ident: "enum",
@@ -203,6 +204,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
203204
},
204205
]
205206
PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { {} } ; 0 }, }
207+
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ {} /*»*/ } ; 0 }, }
206208
PRINT-DERIVE INPUT (DEBUG): TokenStream [
207209
Ident {
208210
ident: "enum",
@@ -281,7 +283,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
281283
},
282284
]
283285
PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { PATH; } ; 0 }, }
284-
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { PATH } ; 0 }, }
286+
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ PATH /*»*/ } ; 0 }, }
285287
PRINT-DERIVE INPUT (DEBUG): TokenStream [
286288
Ident {
287289
ident: "enum",
@@ -359,7 +361,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
359361
},
360362
]
361363
PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { 0 + 1; } ; 0 }, }
362-
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { 0 + 1 } ; 0 }, }
364+
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ 0 + 1 /*»*/ } ; 0 }, }
363365
PRINT-DERIVE INPUT (DEBUG): TokenStream [
364366
Ident {
365367
ident: "enum",
@@ -450,7 +452,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
450452
},
451453
]
452454
PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { PATH + 1; } ; 0 }, }
453-
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { PATH + 1 } ; 0 }, }
455+
PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ PATH + 1 /*»*/ } ; 0 }, }
454456
PRINT-DERIVE INPUT (DEBUG): TokenStream [
455457
Ident {
456458
ident: "enum",

‎src/test/ui/proc-macro/issue-75734-pp-paren.stdout

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
9696
},
9797
]
9898
PRINT-BANG INPUT (DISPLAY): 1 + 1 * 2
99+
PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 1 + 1 /*»*/ * 2
99100
PRINT-BANG INPUT (DEBUG): TokenStream [
100101
Group {
101102
delimiter: None,

‎src/test/ui/proc-macro/issue-78675-captured-inner-attrs.stdout

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
PRINT-BANG INPUT (DISPLAY): foo! { #[fake_attr] mod bar {
22
#![doc = r" Foo"]
33
} }
4-
PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): foo! { #[fake_attr] mod bar { #! [doc = r" Foo"] } }
4+
PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): foo! { #[fake_attr] /*«*/ mod bar { #! [doc = r" Foo"] } /*»*/ }
55
PRINT-BANG INPUT (DEBUG): TokenStream [
66
Ident {
77
ident: "foo",

‎src/test/ui/proc-macro/issue-80760-empty-stmt.stdout

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
PRINT-BANG INPUT (DISPLAY): ;
2+
PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ ; /*»*/
23
PRINT-BANG INPUT (DEBUG): TokenStream [
34
Group {
45
delimiter: None,

‎src/test/ui/proc-macro/nested-nonterminal-tokens.stdout

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
PRINT-BANG INPUT (DISPLAY): 0 + 1 + 2 + 3
2+
PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 0 + 1 + 2 /*»*/ + 3
3+
PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): /*«*/ /*«*/ /*«*/ 0 /*»*/ + 1 /*»*/ + 2 /*»*/ + 3
24
PRINT-BANG INPUT (DEBUG): TokenStream [
35
Group {
46
delimiter: None,

‎src/test/ui/proc-macro/nodelim-groups.stdout

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
PRINT-BANG INPUT (DISPLAY): "hi" 1 + (25) + 1 (1 + 1)
2+
PRINT-BANG RE-COLLECTED (DISPLAY): "hi" /*«*/ 1 + (25) + 1 /*»*/ (1 + 1)
23
PRINT-BANG INPUT (DEBUG): TokenStream [
34
Literal {
45
kind: Str,
@@ -71,6 +72,9 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
7172
},
7273
]
7374
PRINT-BANG INPUT (DISPLAY): "hi" "hello".len() + "world".len() (1 + 1)
75+
PRINT-BANG RE-COLLECTED (DISPLAY): "hi" /*«*/ "hello".len() + "world".len() /*»*/ (1 + 1)
76+
PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): "hi" /*«*/ /*«*/ "hello".len() /*»*/ + /*«*/ "world".len() /*»*/ /*»*/
77+
(1 + 1)
7478
PRINT-BANG INPUT (DEBUG): TokenStream [
7579
Literal {
7680
kind: Str,

‎src/test/ui/proc-macro/nonterminal-expansion.stdout

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
PRINT-ATTR_ARGS INPUT (DISPLAY): a, line!(), b
2-
PRINT-ATTR_ARGS RE-COLLECTED (DISPLAY): a, line! (), b
2+
PRINT-ATTR_ARGS RE-COLLECTED (DISPLAY): a, /*«*/ line! () /*»*/, b
33
PRINT-ATTR_ARGS INPUT (DEBUG): TokenStream [
44
Ident {
55
ident: "a",

‎src/test/ui/proc-macro/nonterminal-token-hygiene.stdout

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
PRINT-BANG INPUT (DISPLAY): struct S;
2-
PRINT-BANG RE-COLLECTED (DISPLAY): struct S ;
2+
PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ struct S ; /*»*/
33
PRINT-BANG INPUT (DEBUG): TokenStream [
44
Group {
55
delimiter: None,

‎src/test/ui/proc-macro/parent-source-spans.rs

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,16 @@ use parent_source_spans::parent_source_spans;
88

99
macro one($a:expr, $b:expr) {
1010
two!($a, $b);
11-
//~^ ERROR first parent: "hello"
12-
//~| ERROR second parent: "world"
11+
//~^ ERROR first parent: /*«*/ "hello" /*»*/
12+
//~| ERROR second parent: /*«*/ "world" /*»*/
1313
}
1414

1515
macro two($a:expr, $b:expr) {
1616
three!($a, $b);
17-
//~^ ERROR first final: "hello"
18-
//~| ERROR second final: "world"
19-
//~| ERROR first final: "yay"
20-
//~| ERROR second final: "rust"
17+
//~^ ERROR first final: /*«*/ "hello" /*»*/
18+
//~| ERROR second final: /*«*/ "world" /*»*/
19+
//~| ERROR first final: /*«*/ "yay" /*»*/
20+
//~| ERROR second final: /*«*/ "rust" /*»*/
2121
}
2222

2323
// forwarding tokens directly doesn't create a new source chain
@@ -34,16 +34,16 @@ macro four($($tokens:tt)*) {
3434

3535
fn main() {
3636
one!("hello", "world");
37-
//~^ ERROR first grandparent: "hello"
38-
//~| ERROR second grandparent: "world"
39-
//~| ERROR first source: "hello"
40-
//~| ERROR second source: "world"
37+
//~^ ERROR first grandparent: /*«*/ "hello" /*»*/
38+
//~| ERROR second grandparent: /*«*/ "world" /*»*/
39+
//~| ERROR first source: /*«*/ "hello" /*»*/
40+
//~| ERROR second source: /*«*/ "world" /*»*/
4141

4242
two!("yay", "rust");
43-
//~^ ERROR first parent: "yay"
44-
//~| ERROR second parent: "rust"
45-
//~| ERROR first source: "yay"
46-
//~| ERROR second source: "rust"
43+
//~^ ERROR first parent: /*«*/ "yay" /*»*/
44+
//~| ERROR second parent: /*«*/ "rust" /*»*/
45+
//~| ERROR first source: /*«*/ "yay" /*»*/
46+
//~| ERROR second source: /*«*/ "rust" /*»*/
4747

4848
three!("hip", "hop");
4949
//~^ ERROR first final: "hip"

0 commit comments

Comments
 (0)