Skip to content

Commit 8cd0c08

Browse files
jseyfriedjseyfried
committed
Introduce syntax::parse::parser::TokenCursor.
1 parent 8dca72b commit 8cd0c08

File tree

2 files changed

+173
-115
lines changed

2 files changed

+173
-115
lines changed

‎src/libsyntax/parse/parser.rs

+127-47
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
// except according to those terms.
1010

1111
use abi::{self, Abi};
12-
use ast::BareFnTy;
12+
use ast::{AttrStyle, BareFnTy};
1313
use ast::{RegionTyParamBound, TraitTyParamBound, TraitBoundModifier};
1414
use ast::Unsafety;
1515
use ast::{Mod, Arg, Arm, Attribute, BindingMode, TraitItemKind};
@@ -46,21 +46,21 @@ use errors::{self, DiagnosticBuilder};
4646
use parse::{self, classify, token};
4747
use parse::common::SeqSep;
4848
use parse::lexer::TokenAndSpan;
49+
use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
4950
use parse::obsolete::ObsoleteSyntax;
5051
use parse::{new_sub_parser_from_file, ParseSess, Directory, DirectoryOwnership};
5152
use util::parser::{AssocOp, Fixity};
5253
use print::pprust;
5354
use ptr::P;
5455
use parse::PResult;
55-
use tokenstream::{Delimited, TokenTree};
56+
use tokenstream::{self, Delimited, TokenTree, TokenStream};
5657
use symbol::{Symbol, keywords};
5758
use util::ThinVec;
5859

5960
use std::collections::HashSet;
60-
use std::mem;
61+
use std::{cmp, mem, slice};
6162
use std::path::{Path, PathBuf};
6263
use std::rc::Rc;
63-
use std::slice;
6464

6565
bitflags! {
6666
flags Restrictions: u8 {
@@ -175,12 +175,108 @@ pub struct Parser<'a> {
175175
/// into modules, and sub-parsers have new values for this name.
176176
pub root_module_name: Option<String>,
177177
pub expected_tokens: Vec<TokenType>,
178-
pub tts: Vec<(TokenTree, usize)>,
178+
token_cursor: TokenCursor,
179179
pub desugar_doc_comments: bool,
180180
/// Whether we should configure out of line modules as we parse.
181181
pub cfg_mods: bool,
182182
}
183183

184+
struct TokenCursor {
185+
frame: TokenCursorFrame,
186+
stack: Vec<TokenCursorFrame>,
187+
}
188+
189+
struct TokenCursorFrame {
190+
delim: token::DelimToken,
191+
span: Span,
192+
open_delim: bool,
193+
tree_cursor: tokenstream::Cursor,
194+
close_delim: bool,
195+
}
196+
197+
impl TokenCursorFrame {
198+
fn new(sp: Span, delimited: &Delimited) -> Self {
199+
TokenCursorFrame {
200+
delim: delimited.delim,
201+
span: sp,
202+
open_delim: delimited.delim == token::NoDelim,
203+
tree_cursor: delimited.tts.iter().cloned().collect::<TokenStream>().into_trees(),
204+
close_delim: delimited.delim == token::NoDelim,
205+
}
206+
}
207+
}
208+
209+
impl TokenCursor {
210+
fn next(&mut self) -> TokenAndSpan {
211+
loop {
212+
let tree = if !self.frame.open_delim {
213+
self.frame.open_delim = true;
214+
Delimited { delim: self.frame.delim, tts: Vec::new() }.open_tt(self.frame.span)
215+
} else if let Some(tree) = self.frame.tree_cursor.next() {
216+
tree
217+
} else if !self.frame.close_delim {
218+
self.frame.close_delim = true;
219+
Delimited { delim: self.frame.delim, tts: Vec::new() }.close_tt(self.frame.span)
220+
} else if let Some(frame) = self.stack.pop() {
221+
self.frame = frame;
222+
continue
223+
} else {
224+
return TokenAndSpan { tok: token::Eof, sp: self.frame.span }
225+
};
226+
227+
match tree {
228+
TokenTree::Token(sp, tok) => return TokenAndSpan { tok: tok, sp: sp },
229+
TokenTree::Delimited(sp, ref delimited) => {
230+
let frame = TokenCursorFrame::new(sp, delimited);
231+
self.stack.push(mem::replace(&mut self.frame, frame));
232+
}
233+
}
234+
}
235+
}
236+
237+
fn next_desugared(&mut self) -> TokenAndSpan {
238+
let (sp, name) = match self.next() {
239+
TokenAndSpan { sp, tok: token::DocComment(name) } => (sp, name),
240+
tok @ _ => return tok,
241+
};
242+
243+
let stripped = strip_doc_comment_decoration(&name.as_str());
244+
245+
// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
246+
// required to wrap the text.
247+
let mut num_of_hashes = 0;
248+
let mut count = 0;
249+
for ch in stripped.chars() {
250+
count = match ch {
251+
'"' => 1,
252+
'#' if count > 0 => count + 1,
253+
_ => 0,
254+
};
255+
num_of_hashes = cmp::max(num_of_hashes, count);
256+
}
257+
258+
let body = TokenTree::Delimited(sp, Rc::new(Delimited {
259+
delim: token::Bracket,
260+
tts: vec![TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"))),
261+
TokenTree::Token(sp, token::Eq),
262+
TokenTree::Token(sp, token::Literal(
263+
token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))],
264+
}));
265+
266+
self.stack.push(mem::replace(&mut self.frame, TokenCursorFrame::new(sp, &Delimited {
267+
delim: token::NoDelim,
268+
tts: if doc_comment_style(&name.as_str()) == AttrStyle::Inner {
269+
[TokenTree::Token(sp, token::Pound), TokenTree::Token(sp, token::Not), body]
270+
.iter().cloned().collect()
271+
} else {
272+
[TokenTree::Token(sp, token::Pound), body].iter().cloned().collect()
273+
},
274+
})));
275+
276+
self.next()
277+
}
278+
}
279+
184280
#[derive(PartialEq, Eq, Clone)]
185281
pub enum TokenType {
186282
Token(token::Token),
@@ -313,10 +409,6 @@ impl<'a> Parser<'a> {
313409
directory: Option<Directory>,
314410
desugar_doc_comments: bool)
315411
-> Self {
316-
let tt = TokenTree::Delimited(syntax_pos::DUMMY_SP, Rc::new(Delimited {
317-
delim: token::NoDelim,
318-
tts: tokens,
319-
}));
320412
let mut parser = Parser {
321413
sess: sess,
322414
token: token::Underscore,
@@ -328,7 +420,13 @@ impl<'a> Parser<'a> {
328420
directory: Directory { path: PathBuf::new(), ownership: DirectoryOwnership::Owned },
329421
root_module_name: None,
330422
expected_tokens: Vec::new(),
331-
tts: if tt.len() > 0 { vec![(tt, 0)] } else { Vec::new() },
423+
token_cursor: TokenCursor {
424+
frame: TokenCursorFrame::new(syntax_pos::DUMMY_SP, &Delimited {
425+
delim: token::NoDelim,
426+
tts: tokens,
427+
}),
428+
stack: Vec::new(),
429+
},
332430
desugar_doc_comments: desugar_doc_comments,
333431
cfg_mods: true,
334432
};
@@ -346,28 +444,9 @@ impl<'a> Parser<'a> {
346444
}
347445

348446
fn next_tok(&mut self) -> TokenAndSpan {
349-
loop {
350-
let tok = if let Some((tts, i)) = self.tts.pop() {
351-
let tt = tts.get_tt(i);
352-
if i + 1 < tts.len() {
353-
self.tts.push((tts, i + 1));
354-
}
355-
if let TokenTree::Token(sp, tok) = tt {
356-
TokenAndSpan { tok: tok, sp: sp }
357-
} else {
358-
self.tts.push((tt, 0));
359-
continue
360-
}
361-
} else {
362-
TokenAndSpan { tok: token::Eof, sp: self.span }
363-
};
364-
365-
match tok.tok {
366-
token::DocComment(name) if self.desugar_doc_comments => {
367-
self.tts.push((TokenTree::Token(tok.sp, token::DocComment(name)), 0));
368-
}
369-
_ => return tok,
370-
}
447+
match self.desugar_doc_comments {
448+
true => self.token_cursor.next_desugared(),
449+
false => self.token_cursor.next(),
371450
}
372451
}
373452

@@ -972,19 +1051,16 @@ impl<'a> Parser<'a> {
9721051
F: FnOnce(&token::Token) -> R,
9731052
{
9741053
if dist == 0 {
975-
return f(&self.token);
976-
}
977-
let mut tok = token::Eof;
978-
if let Some(&(ref tts, mut i)) = self.tts.last() {
979-
i += dist - 1;
980-
if i < tts.len() {
981-
tok = match tts.get_tt(i) {
982-
TokenTree::Token(_, tok) => tok,
983-
TokenTree::Delimited(_, delimited) => token::OpenDelim(delimited.delim),
984-
};
985-
}
1054+
return f(&self.token)
9861055
}
987-
f(&tok)
1056+
1057+
f(&match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) {
1058+
Some(tree) => match tree {
1059+
TokenTree::Token(_, tok) => tok,
1060+
TokenTree::Delimited(_, delimited) => token::OpenDelim(delimited.delim),
1061+
},
1062+
None => token::CloseDelim(self.token_cursor.frame.delim),
1063+
})
9881064
}
9891065
pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> {
9901066
self.sess.span_diagnostic.struct_span_fatal(self.span, m)
@@ -2569,10 +2645,14 @@ impl<'a> Parser<'a> {
25692645
pub fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
25702646
match self.token {
25712647
token::OpenDelim(..) => {
2572-
let tt = self.tts.pop().unwrap().0;
2573-
self.span = tt.span();
2648+
let frame = mem::replace(&mut self.token_cursor.frame,
2649+
self.token_cursor.stack.pop().unwrap());
2650+
self.span = frame.span;
25742651
self.bump();
2575-
return Ok(tt);
2652+
return Ok(TokenTree::Delimited(frame.span, Rc::new(Delimited {
2653+
delim: frame.delim,
2654+
tts: frame.tree_cursor.original_stream().trees().collect(),
2655+
})));
25762656
},
25772657
token::CloseDelim(_) | token::Eof => unreachable!(),
25782658
_ => Ok(TokenTree::Token(self.span, self.bump_and_get())),

‎src/libsyntax/tokenstream.rs

+46-68
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,11 @@
2222
//! and a borrowed TokenStream is sufficient to build an owned TokenStream without taking
2323
//! ownership of the original.
2424
25-
use ast::{self, AttrStyle, LitKind};
25+
use ast::{self, LitKind};
2626
use syntax_pos::{BytePos, Span, DUMMY_SP};
2727
use codemap::Spanned;
2828
use ext::base;
2929
use ext::tt::{macro_parser, quoted};
30-
use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
3130
use parse::{self, Directory};
3231
use parse::token::{self, Token, Lit};
3332
use print::pprust;
@@ -103,72 +102,6 @@ pub enum TokenTree {
103102
}
104103

105104
impl TokenTree {
106-
pub fn len(&self) -> usize {
107-
match *self {
108-
TokenTree::Token(_, token::DocComment(name)) => {
109-
match doc_comment_style(&name.as_str()) {
110-
AttrStyle::Outer => 2,
111-
AttrStyle::Inner => 3,
112-
}
113-
}
114-
TokenTree::Delimited(_, ref delimed) => match delimed.delim {
115-
token::NoDelim => delimed.tts.len(),
116-
_ => delimed.tts.len() + 2,
117-
},
118-
TokenTree::Token(..) => 0,
119-
}
120-
}
121-
122-
pub fn get_tt(&self, index: usize) -> TokenTree {
123-
match (self, index) {
124-
(&TokenTree::Token(sp, token::DocComment(_)), 0) => TokenTree::Token(sp, token::Pound),
125-
(&TokenTree::Token(sp, token::DocComment(name)), 1)
126-
if doc_comment_style(&name.as_str()) == AttrStyle::Inner => {
127-
TokenTree::Token(sp, token::Not)
128-
}
129-
(&TokenTree::Token(sp, token::DocComment(name)), _) => {
130-
let stripped = strip_doc_comment_decoration(&name.as_str());
131-
132-
// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
133-
// required to wrap the text.
134-
let num_of_hashes = stripped.chars()
135-
.scan(0, |cnt, x| {
136-
*cnt = if x == '"' {
137-
1
138-
} else if *cnt != 0 && x == '#' {
139-
*cnt + 1
140-
} else {
141-
0
142-
};
143-
Some(*cnt)
144-
})
145-
.max()
146-
.unwrap_or(0);
147-
148-
TokenTree::Delimited(sp, Rc::new(Delimited {
149-
delim: token::Bracket,
150-
tts: vec![TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"))),
151-
TokenTree::Token(sp, token::Eq),
152-
TokenTree::Token(sp, token::Literal(
153-
token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))],
154-
}))
155-
}
156-
(&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
157-
delimed.tts[index].clone()
158-
}
159-
(&TokenTree::Delimited(span, ref delimed), _) => {
160-
if index == 0 {
161-
return delimed.open_tt(span);
162-
}
163-
if index == delimed.tts.len() + 1 {
164-
return delimed.close_tt(span);
165-
}
166-
delimed.tts[index - 1].clone()
167-
}
168-
_ => panic!("Cannot expand a token tree"),
169-
}
170-
}
171-
172105
/// Use this token tree as a matcher to parse given tts.
173106
pub fn parse(cx: &base::ExtCtxt, mtch: &[quoted::TokenTree], tts: &[TokenTree])
174107
-> macro_parser::NamedParseResult {
@@ -416,6 +349,51 @@ impl Cursor {
416349
}
417350
})
418351
}
352+
353+
pub fn original_stream(self) -> TokenStream {
354+
match self.0 {
355+
CursorKind::Empty => TokenStream::empty(),
356+
CursorKind::Tree(tree, _) => tree.into(),
357+
CursorKind::Stream(cursor) => TokenStream::concat_rc_slice({
358+
cursor.stack.get(0).cloned().map(|(stream, _)| stream).unwrap_or(cursor.stream)
359+
}),
360+
}
361+
}
362+
363+
pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
364+
fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result<TokenTree, usize> {
365+
for stream in streams {
366+
n = match stream.kind {
367+
TokenStreamKind::Tree(ref tree) if n == 0 => return Ok(tree.clone()),
368+
TokenStreamKind::Tree(..) => n - 1,
369+
TokenStreamKind::Stream(ref stream) => match look_ahead(stream, n) {
370+
Ok(tree) => return Ok(tree),
371+
Err(n) => n,
372+
},
373+
_ => n,
374+
};
375+
}
376+
377+
Err(n)
378+
}
379+
380+
match self.0 {
381+
CursorKind::Empty | CursorKind::Tree(_, true) => Err(n),
382+
CursorKind::Tree(ref tree, false) => look_ahead(&[tree.clone().into()], n),
383+
CursorKind::Stream(ref cursor) => {
384+
look_ahead(&cursor.stream[cursor.index ..], n).or_else(|mut n| {
385+
for &(ref stream, index) in cursor.stack.iter().rev() {
386+
n = match look_ahead(&stream[index..], n) {
387+
Ok(tree) => return Ok(tree),
388+
Err(n) => n,
389+
}
390+
}
391+
392+
Err(n)
393+
})
394+
}
395+
}.ok()
396+
}
419397
}
420398

421399
impl fmt::Display for TokenStream {

0 commit comments

Comments
 (0)