Skip to content

Commit bb398ca

Browse files
committed
Remove hacks in make_token_stream.
`make_tokenstream` has three commented hacks, and a comment at the top referring to #67062. These hacks have no observable effect, at least as judged by running the test suite. The hacks were added in #82608, with an explanation [here](#82608 (comment)). It appears that one of the following is true: (a) they never did anything useful, (b) they do something useful but we have no test coverage for them, or (c) something has changed in the meantime that means they are no longer necessary. This commit removes the hacks and the comments, in the hope that (b) is not true.
1 parent baaa3b6 commit bb398ca

File tree

1 file changed

+0
-38
lines changed

1 file changed

+0
-38
lines changed

compiler/rustc_parse/src/parser/attr_wrapper.rs

-38
Original file line numberDiff line numberDiff line change
@@ -388,12 +388,6 @@ impl<'a> Parser<'a> {
388388
/// Converts a flattened iterator of tokens (including open and close delimiter tokens)
389389
/// into a `TokenStream`, creating a `TokenTree::Delimited` for each matching pair
390390
/// of open and close delims.
391-
// FIXME(#67062): Currently, we don't parse `Invisible`-delimited groups correctly,
392-
// which can cause us to end up with mismatched `Invisible` delimiters in our
393-
// captured tokens. This function contains several hacks to work around this -
394-
// essentially, we throw away mismatched `Invisible` delimiters when we encounter them.
395-
// Once we properly parse `Invisible` delimiters, they can be captured just like any
396-
// other tokens, and these hacks can be removed.
397391
fn make_token_stream(
398392
mut iter: impl Iterator<Item = (FlatToken, Spacing)>,
399393
break_last_token: bool,
@@ -412,35 +406,10 @@ fn make_token_stream(
412406
stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] });
413407
}
414408
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
415-
// HACK: If we encounter a mismatched `Invisible` delimiter at the top
416-
// level, just ignore it.
417-
if matches!(delim, Delimiter::Invisible)
418-
&& (stack.len() == 1
419-
|| !matches!(
420-
stack.last_mut().unwrap().open_delim_sp.unwrap().0,
421-
Delimiter::Invisible
422-
))
423-
{
424-
token_and_spacing = iter.next();
425-
continue;
426-
}
427409
let frame_data = stack
428410
.pop()
429411
.unwrap_or_else(|| panic!("Token stack was empty for token: {:?}", token));
430412

431-
// HACK: If our current frame has a mismatched opening `Invisible` delimiter,
432-
// merge our current frame with the one above it. That is, transform
433-
// `[ { < first second } third ]` into `[ { first second } third ]`
434-
if !matches!(delim, Delimiter::Invisible)
435-
&& matches!(frame_data.open_delim_sp.unwrap().0, Delimiter::Invisible)
436-
{
437-
stack.last_mut().unwrap().inner.extend(frame_data.inner);
438-
// Process our closing delimiter again, this time at the previous
439-
// frame in the stack
440-
token_and_spacing = Some((token, spacing));
441-
continue;
442-
}
443-
444413
let (open_delim, open_sp) = frame_data.open_delim_sp.unwrap();
445414
assert_eq!(
446415
open_delim, delim,
@@ -472,13 +441,6 @@ fn make_token_stream(
472441
}
473442
token_and_spacing = iter.next();
474443
}
475-
// HACK: If we don't have a closing `Invisible` delimiter for our last
476-
// frame, merge the frame with the top-level frame. That is,
477-
// turn `< first second` into `first second`
478-
if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == Delimiter::Invisible {
479-
let temp_buf = stack.pop().unwrap();
480-
stack.last_mut().unwrap().inner.extend(temp_buf.inner);
481-
}
482444
let mut final_buf = stack.pop().expect("Missing final buf!");
483445
if break_last_token {
484446
let (last_token, spacing) = final_buf.inner.pop().unwrap();

0 commit comments

Comments
 (0)