Auto merge of #40202 - jseyfried:integrate_tokenstream, r=nrc

syntax: integrate `TokenStream`

Use `TokenStream` instead of `Vec<TokenTree>` in `TokenTree::Delimited` and elsewhere.
r? @nrc

Author: bors

src/libproc_macro/lib.rs

@@ -101,7 +101,7 @@ pub mod __internal {
 
     pub fn token_stream_parse_items(stream: TokenStream) -> Result<Vec<P<ast::Item>>, LexError> {
         with_parse_sess(move |sess| {
-            let mut parser = parse::new_parser_from_ts(sess, stream.inner);
+            let mut parser = parse::stream_to_parser(sess, stream.inner);
             let mut items = Vec::new();
 
             while let Some(item) = try!(parser.parse_item().map_err(super::parse_to_lex_err)) {
@@ -177,9 +177,8 @@ impl FromStr for TokenStream {
         __internal::with_parse_sess(|sess| {
             let src = src.to_string();
             let name = "<proc-macro source code>".to_string();
-            let tts = parse::parse_tts_from_source_str(name, src, sess);
-
-            Ok(__internal::token_stream_wrap(tts.into_iter().collect()))
+            let stream = parse::parse_stream_from_source_str(name, src, sess);
+            Ok(__internal::token_stream_wrap(stream))
         })
     }
 }

src/libproc_macro_plugin/qquote.rs

@@ -17,7 +17,7 @@ use syntax::symbol::Symbol;
 use syntax::tokenstream::{self, Delimited, TokenTree, TokenStream};
 use syntax_pos::DUMMY_SP;
 
-use std::rc::Rc;
+use std::iter;
 
 pub fn qquote<'cx>(stream: TokenStream) -> TokenStream {
     stream.quote()
@@ -49,10 +49,7 @@ macro_rules! quote_tree {
 }
 
 fn delimit(delim: token::DelimToken, stream: TokenStream) -> TokenStream {
-    TokenTree::Delimited(DUMMY_SP, Rc::new(Delimited {
-        delim: delim,
-        tts: stream.trees().cloned().collect(),
-    })).into()
+    TokenTree::Delimited(DUMMY_SP, Delimited { delim: delim, tts: stream.into() }).into()
 }
 
 macro_rules! quote {
@@ -75,9 +72,9 @@ impl Quote for TokenStream {
             return quote!(::syntax::tokenstream::TokenStream::empty());
         }
 
-        struct Quote<'a>(tokenstream::Cursor<'a>);
+        struct Quote(iter::Peekable<tokenstream::Cursor>);
 
-        impl<'a> Iterator for Quote<'a> {
+        impl Iterator for Quote {
             type Item = TokenStream;
 
             fn next(&mut self) -> Option<TokenStream> {
@@ -89,25 +86,18 @@ impl Quote for TokenStream {
                     _ => false,
                 };
 
-                self.0.next().cloned().map(|tree| {
+                self.0.next().map(|tree| {
                     let quoted_tree = if is_unquote { tree.into() } else { tree.quote() };
                     quote!(::syntax::tokenstream::TokenStream::from((unquote quoted_tree)),)
                 })
             }
         }
 
-        let quoted = Quote(self.trees()).collect::<TokenStream>();
+        let quoted = Quote(self.trees().peekable()).collect::<TokenStream>();
         quote!([(unquote quoted)].iter().cloned().collect::<::syntax::tokenstream::TokenStream>())
     }
 }
 
-impl Quote for Vec<TokenTree> {
-    fn quote(&self) -> TokenStream {
-        let stream = self.iter().cloned().collect::<TokenStream>();
-        quote!((quote stream).trees().cloned().collect::<::std::vec::Vec<_> >())
-    }
-}
-
 impl Quote for TokenTree {
     fn quote(&self) -> TokenStream {
         match *self {
@@ -123,12 +113,12 @@ impl Quote for TokenTree {
     }
 }
 
-impl Quote for Rc<Delimited> {
+impl Quote for Delimited {
     fn quote(&self) -> TokenStream {
-        quote!(::std::rc::Rc::new(::syntax::tokenstream::Delimited {
+        quote!(::syntax::tokenstream::Delimited {
             delim: (quote self.delim),
-            tts: (quote self.tts),
-        }))
+            tts: (quote self.stream()).into(),
+        })
     }
 }
 

src/librustc/hir/mod.rs

@@ -40,7 +40,7 @@ use syntax::ast::{Ident, Name, NodeId, DUMMY_NODE_ID, AsmDialect};
 use syntax::ast::{Attribute, Lit, StrStyle, FloatTy, IntTy, UintTy, MetaItem};
 use syntax::ptr::P;
 use syntax::symbol::{Symbol, keywords};
-use syntax::tokenstream::TokenTree;
+use syntax::tokenstream::TokenStream;
 use syntax::util::ThinVec;
 
 use std::collections::BTreeMap;
@@ -471,7 +471,7 @@ pub struct MacroDef {
     pub attrs: HirVec<Attribute>,
     pub id: NodeId,
     pub span: Span,
-    pub body: HirVec<TokenTree>,
+    pub body: TokenStream,
 }
 
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]

src/librustc_incremental/calculate_svh/svh_visitor.rs

@@ -866,8 +866,8 @@ impl<'a, 'hash, 'tcx> visit::Visitor<'tcx> for StrictVersionHashVisitor<'a, 'has
         debug!("visit_macro_def: st={:?}", self.st);
         SawMacroDef.hash(self.st);
         hash_attrs!(self, &macro_def.attrs);
-        for tt in &macro_def.body {
-            self.hash_token_tree(tt);
+        for tt in macro_def.body.trees() {
+            self.hash_token_tree(&tt);
         }
         visit::walk_macro_def(self, macro_def)
     }
@@ -1033,15 +1033,9 @@ impl<'a, 'hash, 'tcx> StrictVersionHashVisitor<'a, 'hash, 'tcx> {
             }
             tokenstream::TokenTree::Delimited(span, ref delimited) => {
                 hash_span!(self, span);
-                let tokenstream::Delimited {
-                    ref delim,
-                    ref tts,
-                } = **delimited;
-
-                delim.hash(self.st);
-                tts.len().hash(self.st);
-                for sub_tt in tts {
-                    self.hash_token_tree(sub_tt);
+                delimited.delim.hash(self.st);
+                for sub_tt in delimited.stream().trees() {
+                    self.hash_token_tree(&sub_tt);
                 }
             }
         }

src/librustc_metadata/cstore_impl.rs

@@ -34,7 +34,7 @@ use std::rc::Rc;
 
 use syntax::ast;
 use syntax::attr;
-use syntax::parse::filemap_to_tts;
+use syntax::parse::filemap_to_stream;
 use syntax::symbol::Symbol;
 use syntax_pos::{mk_sp, Span};
 use rustc::hir::svh::Svh;
@@ -401,7 +401,7 @@ impl CrateStore for cstore::CStore {
 
         let filemap = sess.parse_sess.codemap().new_filemap(source_name, None, def.body);
         let local_span = mk_sp(filemap.start_pos, filemap.end_pos);
-        let body = filemap_to_tts(&sess.parse_sess, filemap);
+        let body = filemap_to_stream(&sess.parse_sess, filemap);
 
         // Mark the attrs as used
         let attrs = data.get_item_attrs(id.index);
@@ -419,7 +419,7 @@ impl CrateStore for cstore::CStore {
             id: ast::DUMMY_NODE_ID,
             span: local_span,
             attrs: attrs,
-            body: body,
+            body: body.into(),
         })
     }
 

src/librustc_metadata/encoder.rs

@@ -853,9 +853,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
     /// Serialize the text of exported macros
     fn encode_info_for_macro_def(&mut self, macro_def: &hir::MacroDef) -> Entry<'tcx> {
+        use syntax::print::pprust;
         Entry {
             kind: EntryKind::MacroDef(self.lazy(&MacroDef {
-                body: ::syntax::print::pprust::tts_to_string(&macro_def.body)
+                body: pprust::tts_to_string(&macro_def.body.trees().collect::<Vec<_>>()),
             })),
             visibility: self.lazy(&ty::Visibility::Public),
             span: self.lazy(&macro_def.span),

src/librustc_resolve/build_reduced_graph.rs

@@ -516,7 +516,7 @@ impl<'a> Resolver<'a> {
             expansion: Cell::new(LegacyScope::Empty),
         });
         self.invocations.insert(mark, invocation);
-        macro_rules.body = mark_tts(&macro_rules.body, mark);
+        macro_rules.body = mark_tts(macro_rules.stream(), mark).into();
         let ext = Rc::new(macro_rules::compile(&self.session.parse_sess, &macro_rules));
         self.macro_map.insert(def_id, ext.clone());
         ext

src/librustc_resolve/macros.rs

@@ -545,7 +545,7 @@ impl<'a> Resolver<'a> {
 
     pub fn define_macro(&mut self, item: &ast::Item, legacy_scope: &mut LegacyScope<'a>) {
         let tts = match item.node {
-            ast::ItemKind::Mac(ref mac) => &mac.node.tts,
+            ast::ItemKind::Mac(ref mac) => mac.node.stream(),
             _ => unreachable!(),
         };
 
@@ -562,7 +562,7 @@ impl<'a> Resolver<'a> {
             attrs: item.attrs.clone(),
             id: ast::DUMMY_NODE_ID,
             span: item.span,
-            body: mark_tts(tts, mark),
+            body: mark_tts(tts, mark).into(),
         };
 
         *legacy_scope = LegacyScope::Binding(self.arenas.alloc_legacy_binding(LegacyBinding {

src/librustc_save_analysis/span_utils.rs

@@ -284,7 +284,7 @@ impl<'a> SpanUtils<'a> {
     pub fn signature_string_for_span(&self, span: Span) -> String {
         let mut toks = self.retokenise_span(span);
         toks.real_token();
-        let mut toks = toks.parse_all_token_trees().unwrap().into_iter();
+        let mut toks = toks.parse_all_token_trees().unwrap().trees();
         let mut prev = toks.next().unwrap();
 
         let first_span = prev.span();

src/librustdoc/visit_ast.rs

@@ -211,7 +211,8 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
                     };
 
                     // FIXME(jseyfried) merge with `self.visit_macro()`
-                    let matchers = def.body.chunks(4).map(|arm| arm[0].span()).collect();
+                    let tts = def.stream().trees().collect::<Vec<_>>();
+                    let matchers = tts.chunks(4).map(|arm| arm[0].span()).collect();
                     om.macros.push(Macro {
                         def_id: def_id,
                         attrs: def.attrs.clone().into(),
@@ -520,8 +521,9 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
 
     // convert each exported_macro into a doc item
     fn visit_local_macro(&self, def: &hir::MacroDef) -> Macro {
+        let tts = def.body.trees().collect::<Vec<_>>();
         // Extract the spans of all matchers. They represent the "interface" of the macro.
-        let matchers = def.body.chunks(4).map(|arm| arm[0].span()).collect();
+        let matchers = tts.chunks(4).map(|arm| arm[0].span()).collect();
 
         Macro {
             def_id: self.cx.tcx.hir.local_def_id(def.id),

src/libsyntax/ast.rs

@@ -24,7 +24,7 @@ use ext::hygiene::SyntaxContext;
 use print::pprust;
 use ptr::P;
 use symbol::{Symbol, keywords};
-use tokenstream::{TokenTree};
+use tokenstream::{ThinTokenStream, TokenStream};
 
 use std::collections::HashSet;
 use std::fmt;
@@ -1033,7 +1033,13 @@ pub type Mac = Spanned<Mac_>;
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
 pub struct Mac_ {
     pub path: Path,
-    pub tts: Vec<TokenTree>,
+    pub tts: ThinTokenStream,
+}
+
+impl Mac_ {
+    pub fn stream(&self) -> TokenStream {
+        self.tts.clone().into()
+    }
 }
 
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
@@ -1915,7 +1921,13 @@ pub struct MacroDef {
     pub attrs: Vec<Attribute>,
     pub id: NodeId,
     pub span: Span,
-    pub body: Vec<TokenTree>,
+    pub body: ThinTokenStream,
+}
+
+impl MacroDef {
+    pub fn stream(&self) -> TokenStream {
+        self.body.clone().into()
+    }
 }
 
 #[cfg(test)]

src/libsyntax/ext/base.rs

@@ -188,10 +188,7 @@ impl<F> AttrProcMacro for F
 
 /// Represents a thing that maps token trees to Macro Results
 pub trait TTMacroExpander {
-    fn expand<'cx>(&self,
-                   ecx: &'cx mut ExtCtxt,
-                   span: Span,
-                   token_tree: &[tokenstream::TokenTree])
+    fn expand<'cx>(&self, ecx: &'cx mut ExtCtxt, span: Span, input: TokenStream)
                    -> Box<MacResult+'cx>;
 }
 
@@ -200,15 +197,11 @@ pub type MacroExpanderFn =
                 -> Box<MacResult+'cx>;
 
 impl<F> TTMacroExpander for F
-    where F : for<'cx> Fn(&'cx mut ExtCtxt, Span, &[tokenstream::TokenTree])
-                          -> Box<MacResult+'cx>
+    where F: for<'cx> Fn(&'cx mut ExtCtxt, Span, &[tokenstream::TokenTree]) -> Box<MacResult+'cx>
 {
-    fn expand<'cx>(&self,
-                   ecx: &'cx mut ExtCtxt,
-                   span: Span,
-                   token_tree: &[tokenstream::TokenTree])
+    fn expand<'cx>(&self, ecx: &'cx mut ExtCtxt, span: Span, input: TokenStream)
                    -> Box<MacResult+'cx> {
-        (*self)(ecx, span, token_tree)
+        (*self)(ecx, span, &input.trees().collect::<Vec<_>>())
     }
 }
 
@@ -654,9 +647,8 @@ impl<'a> ExtCtxt<'a> {
         expand::MacroExpander::new(self, true)
     }
 
-    pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree])
-        -> parser::Parser<'a> {
-        parse::tts_to_parser(self.parse_sess, tts.to_vec())
+    pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree]) -> parser::Parser<'a> {
+        parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect())
     }
     pub fn codemap(&self) -> &'a CodeMap { self.parse_sess.codemap() }
     pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }