Skip to content

Some parser and AST cleanups #114353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Aug 3, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 3 additions & 29 deletions compiler/rustc_ast/src/ast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
//! - [`Generics`], [`GenericParam`], [`WhereClause`]: Metadata associated with generic parameters.
//! - [`EnumDef`] and [`Variant`]: Enum declaration.
//! - [`MetaItemLit`] and [`LitKind`]: Literal expressions.
//! - [`MacroDef`], [`MacStmtStyle`], [`MacCall`], [`MacDelimiter`]: Macro definition and invocation.
//! - [`MacroDef`], [`MacStmtStyle`], [`MacCall`]: Macro definition and invocation.
//! - [`Attribute`]: Metadata associated with item.
//! - [`UnOp`], [`BinOp`], and [`BinOpKind`]: Unary and binary operators.

Expand Down Expand Up @@ -1693,15 +1693,15 @@ where
#[derive(Clone, Encodable, Decodable, Debug)]
pub struct DelimArgs {
pub dspan: DelimSpan,
pub delim: MacDelimiter,
pub delim: Delimiter, // Note: `Delimiter::Invisible` never occurs
pub tokens: TokenStream,
}

impl DelimArgs {
/// Whether a macro with these arguments needs a semicolon
/// when used as a standalone item or statement.
pub fn need_semicolon(&self) -> bool {
!matches!(self, DelimArgs { delim: MacDelimiter::Brace, .. })
!matches!(self, DelimArgs { delim: Delimiter::Brace, .. })
}
}

Expand All @@ -1717,32 +1717,6 @@ where
}
}

#[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Debug, HashStable_Generic)]
pub enum MacDelimiter {
Parenthesis,
Bracket,
Brace,
}

impl MacDelimiter {
pub fn to_token(self) -> Delimiter {
match self {
MacDelimiter::Parenthesis => Delimiter::Parenthesis,
MacDelimiter::Bracket => Delimiter::Bracket,
MacDelimiter::Brace => Delimiter::Brace,
}
}

pub fn from_token(delim: Delimiter) -> Option<MacDelimiter> {
match delim {
Delimiter::Parenthesis => Some(MacDelimiter::Parenthesis),
Delimiter::Bracket => Some(MacDelimiter::Bracket),
Delimiter::Brace => Some(MacDelimiter::Brace),
Delimiter::Invisible => None,
}
}
}

/// Represents a macro definition.
#[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)]
pub struct MacroDef {
Expand Down
14 changes: 6 additions & 8 deletions compiler/rustc_ast/src/attr/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use crate::ast::{AttrArgs, AttrArgsEq, AttrId, AttrItem, AttrKind, AttrStyle, AttrVec, Attribute};
use crate::ast::{DelimArgs, Expr, ExprKind, LitKind, MetaItemLit};
use crate::ast::{MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
use crate::ast::{Path, PathSegment, DUMMY_NODE_ID};
use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter, Token};
Expand Down Expand Up @@ -196,7 +196,7 @@ impl AttrItem {

fn meta_item_list(&self) -> Option<ThinVec<NestedMetaItem>> {
match &self.args {
AttrArgs::Delimited(args) if args.delim == MacDelimiter::Parenthesis => {
AttrArgs::Delimited(args) if args.delim == Delimiter::Parenthesis => {
MetaItemKind::list_from_tokens(args.tokens.clone())
}
AttrArgs::Delimited(_) | AttrArgs::Eq(..) | AttrArgs::Empty => None,
Expand Down Expand Up @@ -402,11 +402,9 @@ impl MetaItemKind {
fn from_attr_args(args: &AttrArgs) -> Option<MetaItemKind> {
match args {
AttrArgs::Empty => Some(MetaItemKind::Word),
AttrArgs::Delimited(DelimArgs {
dspan: _,
delim: MacDelimiter::Parenthesis,
tokens,
}) => MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List),
AttrArgs::Delimited(DelimArgs { dspan: _, delim: Delimiter::Parenthesis, tokens }) => {
MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List)
}
AttrArgs::Delimited(..) => None,
AttrArgs::Eq(_, AttrArgsEq::Ast(expr)) => match expr.kind {
ExprKind::Lit(token_lit) => {
Expand Down Expand Up @@ -578,7 +576,7 @@ pub fn mk_attr_nested_word(
let path = Path::from_ident(outer_ident);
let attr_args = AttrArgs::Delimited(DelimArgs {
dspan: DelimSpan::from_single(span),
delim: MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens: inner_tokens,
});
mk_attr(g, style, path, attr_args, span)
Expand Down
2 changes: 0 additions & 2 deletions compiler/rustc_ast/src/token.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,6 @@ pub enum BinOpToken {
/// Describes how a sequence of token trees is delimited.
/// Cannot use `proc_macro::Delimiter` directly because this
/// structure should implement some additional traits.
/// The `None` variant is also renamed to `Invisible` to be
/// less confusing and better convey the semantics.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[derive(Encodable, Decodable, Hash, HashStable_Generic)]
pub enum Delimiter {
Expand Down
6 changes: 3 additions & 3 deletions compiler/rustc_ast_pretty/src/pprust/state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
Some(MacHeader::Path(&item.path)),
false,
None,
delim.to_token(),
*delim,
tokens,
true,
span,
Expand Down Expand Up @@ -640,7 +640,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
Some(MacHeader::Keyword(kw)),
has_bang,
Some(*ident),
macro_def.body.delim.to_token(),
macro_def.body.delim,
&macro_def.body.tokens.clone(),
true,
sp,
Expand Down Expand Up @@ -1240,7 +1240,7 @@ impl<'a> State<'a> {
Some(MacHeader::Path(&m.path)),
true,
None,
m.args.delim.to_token(),
m.args.delim,
&m.args.tokens.clone(),
true,
m.span(),
Expand Down
5 changes: 3 additions & 2 deletions compiler/rustc_builtin_macros/src/assert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ use crate::edition_panic::use_panic_2021;
use crate::errors;
use rustc_ast::ptr::P;
use rustc_ast::token;
use rustc_ast::token::Delimiter;
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, MacDelimiter, Path, PathSegment, UnOp};
use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, Path, PathSegment, UnOp};
use rustc_ast_pretty::pprust;
use rustc_errors::PResult;
use rustc_expand::base::{DummyResult, ExtCtxt, MacEager, MacResult};
Expand Down Expand Up @@ -58,7 +59,7 @@ pub fn expand_assert<'cx>(
path: panic_path(),
args: P(DelimArgs {
dspan: DelimSpan::from_single(call_site_span),
delim: MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens,
}),
})),
Expand Down
7 changes: 4 additions & 3 deletions compiler/rustc_builtin_macros/src/assert/context.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
use rustc_ast::{
ptr::P,
token,
token::Delimiter,
tokenstream::{DelimSpan, TokenStream, TokenTree},
BinOpKind, BorrowKind, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MacDelimiter, MethodCall,
Mutability, Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID,
BinOpKind, BorrowKind, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MethodCall, Mutability,
Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID,
};
use rustc_ast_pretty::pprust;
use rustc_data_structures::fx::FxHashSet;
Expand Down Expand Up @@ -179,7 +180,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
path: panic_path,
args: P(DelimArgs {
dspan: DelimSpan::from_single(self.span),
delim: MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens: initial.into_iter().chain(captures).collect::<TokenStream>(),
}),
})),
Expand Down
3 changes: 2 additions & 1 deletion compiler/rustc_builtin_macros/src/edition_panic.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use rustc_ast::ptr::P;
use rustc_ast::token::Delimiter;
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
use rustc_ast::*;
use rustc_expand::base::*;
Expand Down Expand Up @@ -60,7 +61,7 @@ fn expand<'cx>(
},
args: P(DelimArgs {
dspan: DelimSpan::from_single(sp),
delim: MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens: tts,
}),
})),
Expand Down
3 changes: 2 additions & 1 deletion compiler/rustc_expand/src/placeholders.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use crate::expand::{AstFragment, AstFragmentKind};
use rustc_ast as ast;
use rustc_ast::mut_visit::*;
use rustc_ast::ptr::P;
use rustc_ast::token::Delimiter;
use rustc_data_structures::fx::FxHashMap;
use rustc_span::source_map::DUMMY_SP;
use rustc_span::symbol::Ident;
Expand All @@ -18,7 +19,7 @@ pub fn placeholder(
path: ast::Path { span: DUMMY_SP, segments: ThinVec::new(), tokens: None },
args: P(ast::DelimArgs {
dspan: ast::tokenstream::DelimSpan::dummy(),
delim: ast::MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens: ast::tokenstream::TokenStream::new(Vec::new()),
}),
})
Expand Down
6 changes: 3 additions & 3 deletions compiler/rustc_parse/src/parser/attr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ impl<'a> Parser<'a> {
pub(super) fn parse_outer_attributes(&mut self) -> PResult<'a, AttrWrapper> {
let mut outer_attrs = ast::AttrVec::new();
let mut just_parsed_doc_comment = false;
let start_pos = self.token_cursor.num_next_calls;
let start_pos = self.num_bump_calls;
loop {
let attr = if self.check(&token::Pound) {
let prev_outer_attr_sp = outer_attrs.last().map(|attr| attr.span);
Expand Down Expand Up @@ -277,7 +277,7 @@ impl<'a> Parser<'a> {
pub(crate) fn parse_inner_attributes(&mut self) -> PResult<'a, ast::AttrVec> {
let mut attrs = ast::AttrVec::new();
loop {
let start_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap();
let start_pos: u32 = self.num_bump_calls.try_into().unwrap();
// Only try to parse if it is an inner attribute (has `!`).
let attr = if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) {
Some(self.parse_attribute(InnerAttrPolicy::Permitted)?)
Expand All @@ -298,7 +298,7 @@ impl<'a> Parser<'a> {
None
};
if let Some(attr) = attr {
let end_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap();
let end_pos: u32 = self.num_bump_calls.try_into().unwrap();
// If we are currently capturing tokens, mark the location of this inner attribute.
// If capturing ends up creating a `LazyAttrTokenStream`, we will include
// this replace range with it, removing the inner attribute from the final
Expand Down
22 changes: 9 additions & 13 deletions compiler/rustc_parse/src/parser/attr_wrapper.rs
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ impl<'a> Parser<'a> {

let start_token = (self.token.clone(), self.token_spacing);
let cursor_snapshot = self.token_cursor.clone();
let start_pos = self.num_bump_calls;

let has_outer_attrs = !attrs.attrs.is_empty();
let prev_capturing = std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes);
Expand Down Expand Up @@ -273,8 +274,7 @@ impl<'a> Parser<'a> {

let replace_ranges_end = self.capture_state.replace_ranges.len();

let cursor_snapshot_next_calls = cursor_snapshot.num_next_calls;
let mut end_pos = self.token_cursor.num_next_calls;
let mut end_pos = self.num_bump_calls;

let mut captured_trailing = false;

Expand All @@ -301,12 +301,12 @@ impl<'a> Parser<'a> {
// then extend the range of captured tokens to include it, since the parser
// was not actually bumped past it. When the `LazyAttrTokenStream` gets converted
// into an `AttrTokenStream`, we will create the proper token.
if self.token_cursor.break_last_token {
if self.break_last_token {
assert!(!captured_trailing, "Cannot set break_last_token and have trailing token");
end_pos += 1;
}

let num_calls = end_pos - cursor_snapshot_next_calls;
let num_calls = end_pos - start_pos;

// If we have no attributes, then we will never need to
// use any replace ranges.
Expand All @@ -316,7 +316,7 @@ impl<'a> Parser<'a> {
// Grab any replace ranges that occur *inside* the current AST node.
// We will perform the actual replacement when we convert the `LazyAttrTokenStream`
// to an `AttrTokenStream`.
let start_calls: u32 = cursor_snapshot_next_calls.try_into().unwrap();
let start_calls: u32 = start_pos.try_into().unwrap();
self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end]
.iter()
.cloned()
Expand All @@ -331,7 +331,7 @@ impl<'a> Parser<'a> {
start_token,
num_calls,
cursor_snapshot,
break_last_token: self.token_cursor.break_last_token,
break_last_token: self.break_last_token,
replace_ranges,
});

Expand Down Expand Up @@ -359,14 +359,10 @@ impl<'a> Parser<'a> {
// with a `FlatToken::AttrTarget`. If this AST node is inside an item
// that has `#[derive]`, then this will allow us to cfg-expand this
// AST node.
let start_pos =
if has_outer_attrs { attrs.start_pos } else { cursor_snapshot_next_calls };
let start_pos = if has_outer_attrs { attrs.start_pos } else { start_pos };
let new_tokens = vec![(FlatToken::AttrTarget(attr_data), Spacing::Alone)];

assert!(
!self.token_cursor.break_last_token,
"Should not have unglued last token with cfg attr"
);
assert!(!self.break_last_token, "Should not have unglued last token with cfg attr");
let range: Range<u32> = (start_pos.try_into().unwrap())..(end_pos.try_into().unwrap());
self.capture_state.replace_ranges.push((range, new_tokens));
self.capture_state.replace_ranges.extend(inner_attr_replace_ranges);
Expand Down Expand Up @@ -464,6 +460,6 @@ mod size_asserts {
use rustc_data_structures::static_assert_size;
// tidy-alphabetical-start
static_assert_size!(AttrWrapper, 16);
static_assert_size!(LazyAttrTokenStreamImpl, 120);
static_assert_size!(LazyAttrTokenStreamImpl, 104);
// tidy-alphabetical-end
}
4 changes: 2 additions & 2 deletions compiler/rustc_parse/src/parser/expr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1167,7 +1167,7 @@ impl<'a> Parser<'a> {
DestructuredFloat::TrailingDot(sym, sym_span, dot_span) => {
assert!(suffix.is_none());
// Analogous to `Self::break_and_eat`
self.token_cursor.break_last_token = true;
self.break_last_token = true;
// This might work, in cases like `1. 2`, and might not,
// in cases like `offset_of!(Ty, 1.)`. It depends on what comes
// after the float-like token, and therefore we have to make
Expand Down Expand Up @@ -2599,7 +2599,7 @@ impl<'a> Parser<'a> {

// Recover from missing expression in `for` loop
if matches!(expr.kind, ExprKind::Block(..))
&& !matches!(self.token.kind, token::OpenDelim(token::Delimiter::Brace))
&& !matches!(self.token.kind, token::OpenDelim(Delimiter::Brace))
&& self.may_recover()
{
self.sess
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_parse/src/parser/item.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
use rustc_ast::util::case::Case;
use rustc_ast::MacCall;
use rustc_ast::{self as ast, AttrVec, Attribute, DUMMY_NODE_ID};
use rustc_ast::{Async, Const, Defaultness, IsAuto, Mutability, Unsafe, UseTree, UseTreeKind};
use rustc_ast::{BindingAnnotation, Block, FnDecl, FnSig, Param, SelfKind};
use rustc_ast::{EnumDef, FieldDef, Generics, TraitRef, Ty, TyKind, Variant, VariantData};
use rustc_ast::{FnHeader, ForeignItem, Path, PathSegment, Visibility, VisibilityKind};
use rustc_ast::{MacCall, MacDelimiter};
use rustc_ast_pretty::pprust;
use rustc_errors::{
struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult,
Expand Down Expand Up @@ -1968,7 +1968,7 @@ impl<'a> Parser<'a> {
let arrow = TokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>`
let tokens = TokenStream::new(vec![params, arrow, body]);
let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi());
P(DelimArgs { dspan, delim: MacDelimiter::Brace, tokens })
P(DelimArgs { dspan, delim: Delimiter::Brace, tokens })
} else {
return self.unexpected();
};
Expand Down
Loading