syntax: point quote tokens at the site of quote-using-extension invocation.

Author: graydon

src/libsyntax/ext/base.rs

@@ -163,6 +163,7 @@ trait ext_ctxt {
     fn codemap() -> @CodeMap;
     fn parse_sess() -> parse::parse_sess;
     fn cfg() -> ast::crate_cfg;
+    fn call_site() -> span;
     fn print_backtrace();
     fn backtrace() -> Option<@ExpnInfo>;
     fn mod_push(mod_name: ast::ident);
@@ -195,6 +196,12 @@ fn mk_ctxt(parse_sess: parse::parse_sess,
         fn codemap() -> @CodeMap { self.parse_sess.cm }
         fn parse_sess() -> parse::parse_sess { self.parse_sess }
         fn cfg() -> ast::crate_cfg { self.cfg }
+        fn call_site() -> span {
+            match self.backtrace {
+                Some(@ExpandedFrom({call_site: cs, _})) => cs,
+                None => self.bug(~"missing top span")
+            }
+        }
         fn print_backtrace() { }
         fn backtrace() -> Option<@ExpnInfo> { self.backtrace }
         fn mod_push(i: ast::ident) { self.mod_path.push(i); }

src/libsyntax/ext/build.rs

@@ -145,6 +145,24 @@ fn mk_glob_use(cx: ext_ctxt, sp: span,
       vis: ast::private,
       span: sp}
 }
+fn mk_local(cx: ext_ctxt, sp: span, mutbl: bool,
+            ident: ast::ident, ex: @ast::expr) -> @ast::stmt {
+
+    let pat : @ast::pat = @{id: cx.next_id(),
+                            node: ast::pat_ident(ast::bind_by_value,
+                                                 mk_raw_path(sp, ~[ident]),
+                                                 None),
+                           span: sp};
+    let ty : @ast::Ty = @{ id: cx.next_id(), node: ast::ty_infer, span: sp };
+    let local : @ast::local = @{node: {is_mutbl: mutbl,
+                                       ty: ty,
+                                       pat: pat,
+                                       init: Some(ex),
+                                       id: cx.next_id()},
+                                span: sp};
+    let decl = {node: ast::decl_local(~[local]), span: sp};
+    @{ node: ast::stmt_decl(@decl, cx.next_id()), span: sp }
+}
 fn mk_block(cx: ext_ctxt, sp: span,
             view_items: ~[@ast::view_item],
             stmts: ~[@ast::stmt],

src/libsyntax/ext/expand.rs

@@ -50,10 +50,11 @@ fn expand_expr(exts: HashMap<~str, syntax_extension>, cx: ext_ctxt,
                         fmt!("%s can only be used as a decorator", *extname));
                   }
                   Some(normal({expander: exp, span: exp_sp})) => {
-                    let expanded = exp(cx, (*mac).span, args, body);
 
                     cx.bt_push(ExpandedFrom({call_site: s,
                                 callie: {name: *extname, span: exp_sp}}));
+                    let expanded = exp(cx, (*mac).span, args, body);
+
                     //keep going, outside-in
                     let fully_expanded = fld.fold_expr(expanded).node;
                     cx.bt_pop();
@@ -90,6 +91,9 @@ fn expand_expr(exts: HashMap<~str, syntax_extension>, cx: ext_ctxt,
                                   fmt!("macro undefined: '%s'", *extname))
                   }
                   Some(normal_tt({expander: exp, span: exp_sp})) => {
+                    cx.bt_push(ExpandedFrom({call_site: s,
+                                callie: {name: *extname, span: exp_sp}}));
+
                     let expanded = match exp(cx, (*mac).span, (*tts)) {
                       mr_expr(e) => e,
                       mr_any(expr_maker,_,_) => expr_maker(),
@@ -98,22 +102,21 @@ fn expand_expr(exts: HashMap<~str, syntax_extension>, cx: ext_ctxt,
                                          *extname))
                     };
 
-                    cx.bt_push(ExpandedFrom({call_site: s,
-                                callie: {name: *extname, span: exp_sp}}));
                     //keep going, outside-in
                     let fully_expanded = fld.fold_expr(expanded).node;
                     cx.bt_pop();
 
                     (fully_expanded, s)
                   }
                   Some(normal({expander: exp, span: exp_sp})) => {
+                    cx.bt_push(ExpandedFrom({call_site: s,
+                                callie: {name: *extname, span: exp_sp}}));
+
                     //convert the new-style invoc for the old-style macro
                     let arg = base::tt_args_to_original_flavor(cx, pth.span,
                                                                (*tts));
                     let expanded = exp(cx, (*mac).span, arg, None);
 
-                    cx.bt_push(ExpandedFrom({call_site: s,
-                                callie: {name: *extname, span: exp_sp}}));
                     //keep going, outside-in
                     let fully_expanded = fld.fold_expr(expanded).node;
                     cx.bt_pop();
@@ -296,6 +299,8 @@ fn expand_stmt(exts: HashMap<~str, syntax_extension>, cx: ext_ctxt,
             cx.span_fatal(pth.span, fmt!("macro undefined: '%s'", *extname)),
 
         Some(normal_tt({expander: exp, span: exp_sp})) => {
+            cx.bt_push(ExpandedFrom(
+                {call_site: sp, callie: {name: *extname, span: exp_sp}}));
             let expanded = match exp(cx, mac.span, tts) {
                 mr_expr(e) =>
                     @{node: stmt_expr(e, cx.next_id()), span: e.span},
@@ -305,8 +310,6 @@ fn expand_stmt(exts: HashMap<~str, syntax_extension>, cx: ext_ctxt,
                     fmt!("non-stmt macro in stmt pos: %s", *extname))
             };
 
-            cx.bt_push(ExpandedFrom(
-                {call_site: sp, callie: {name: *extname, span: exp_sp}}));
             //keep going, outside-in
             let fully_expanded = fld.fold_stmt(expanded).node;
             cx.bt_pop();
@@ -315,15 +318,15 @@ fn expand_stmt(exts: HashMap<~str, syntax_extension>, cx: ext_ctxt,
         }
 
         Some(normal({expander: exp, span: exp_sp})) => {
+            cx.bt_push(ExpandedFrom({call_site: sp,
+                                      callie: {name: *extname,
+                                               span: exp_sp}}));
             //convert the new-style invoc for the old-style macro
             let arg = base::tt_args_to_original_flavor(cx, pth.span, tts);
             let exp_expr = exp(cx, mac.span, arg, None);
             let expanded = @{node: stmt_expr(exp_expr, cx.next_id()),
                              span: exp_expr.span};
 
-            cx.bt_push(ExpandedFrom({call_site: sp,
-                                      callie: {name: *extname,
-                                               span: exp_sp}}));
             //keep going, outside-in
             let fully_expanded = fld.fold_stmt(expanded).node;
             cx.bt_pop();

src/libsyntax/ext/quote.rs

@@ -86,65 +86,6 @@ fn id_ext(cx: ext_ctxt, str: ~str) -> ast::ident {
     cx.parse_sess().interner.intern(@str)
 }
 
-fn mk_option_span(cx: ext_ctxt,
-                  qsp: span,
-                  sp: Option<span>) -> @ast::expr {
-    match sp {
-        None => build::mk_path(cx, qsp, ids_ext(cx, ~[~"None"])),
-        Some(sp) => {
-            build::mk_call(cx, qsp,
-                           ids_ext(cx, ~[~"Some"]),
-                           ~[build::mk_managed(cx, qsp,
-                                               mk_span(cx, qsp, sp))])
-        }
-    }
-}
-
-fn mk_span(cx: ext_ctxt, qsp: span, sp: span) -> @ast::expr {
-
-    let e_expn_info = match sp.expn_info {
-        None => build::mk_path(cx, qsp, ids_ext(cx, ~[~"None"])),
-        Some(@codemap::ExpandedFrom(ref cr)) => {
-            let e_callee =
-                build::mk_rec_e(
-                    cx, qsp,
-                    ~[{ident: id_ext(cx, ~"name"),
-                       ex: build::mk_uniq_str(cx, qsp,
-                                              (*cr).callie.name)},
-                      {ident: id_ext(cx, ~"span"),
-                       ex: mk_option_span(cx, qsp, (*cr).callie.span)}]);
-
-            let e_expn_info_ =
-                build::mk_call(
-                    cx, qsp,
-                    ids_ext(cx, ~[~"expanded_from"]),
-                    ~[build::mk_rec_e(
-                        cx, qsp,
-                        ~[{ident: id_ext(cx, ~"call_site"),
-                           ex: mk_span(cx, qsp, (*cr).call_site)},
-                          {ident: id_ext(cx, ~"callie"),
-                           ex: e_callee}])]);
-
-            build::mk_call(cx, qsp,
-                           ids_ext(cx, ~[~"Some"]),
-                           ~[build::mk_managed(cx, qsp, e_expn_info_)])
-        }
-    };
-
-    let span_path = ids_ext(cx, ~[~"span"]);
-
-    build::mk_struct_e(cx, qsp,
-                       span_path,
-                    ~[{ident: id_ext(cx, ~"lo"),
-                       ex: mk_bytepos(cx, qsp, sp.lo) },
-
-                      {ident: id_ext(cx, ~"hi"),
-                       ex: mk_bytepos(cx, qsp, sp.hi) },
-
-                      {ident: id_ext(cx, ~"expn_info"),
-                       ex: e_expn_info}])
-}
-
 // Lift an ident to the expr that evaluates to that ident.
 fn mk_ident(cx: ext_ctxt, sp: span, ident: ast::ident) -> @ast::expr {
     let e_meth = build::mk_access(cx, sp,
@@ -321,59 +262,121 @@ fn mk_token(cx: ext_ctxt, sp: span, tok: token::Token) -> @ast::expr {
 }
 
 
-fn mk_tt(cx: ext_ctxt, sp: span, tt: &ast::token_tree) -> @ast::expr {
+fn mk_tt(cx: ext_ctxt, sp: span, tt: &ast::token_tree)
+    -> ~[@ast::stmt] {
+
     match *tt {
+
         ast::tt_tok(sp, ref tok) => {
+            let e_sp = build::mk_path(cx, sp,
+                                      ids_ext(cx, ~[~"sp"]));
             let e_tok =
                 build::mk_call(cx, sp,
                                ids_ext(cx, ~[~"tt_tok"]),
-                               ~[mk_span(cx, sp, sp),
-                                 mk_token(cx, sp, (*tok))]);
-            build::mk_uniq_vec_e(cx, sp, ~[e_tok])
-        }
+                               ~[e_sp, mk_token(cx, sp, *tok)]);
+            let e_push =
+                build::mk_call_(cx, sp,
+                                build::mk_access(cx, sp,
+                                                 ids_ext(cx, ~[~"tt"]),
+                                                 id_ext(cx, ~"push")),
+                                ~[e_tok]);
+            ~[build::mk_stmt(cx, sp, e_push)]
 
-        ast::tt_delim(ref tts) => {
-            let e_delim =
-                build::mk_call(cx, sp,
-                               ids_ext(cx, ~[~"tt_delim"]),
-                               ~[mk_tts(cx, sp, (*tts))]);
-            build::mk_uniq_vec_e(cx, sp, ~[e_delim])
         }
 
+        ast::tt_delim(ref tts) => mk_tts(cx, sp, *tts),
         ast::tt_seq(*) => fail ~"tt_seq in quote!",
 
-        ast::tt_nonterminal(sp, ident) =>
-        build::mk_copy(cx, sp, build::mk_path(cx, sp, ~[ident]))
+        ast::tt_nonterminal(sp, ident) => {
+            let e_push =
+                build::mk_call_(cx, sp,
+                                build::mk_access
+                                (cx, sp,
+                                 ids_ext(cx, ~[~"tt"]),
+                                 id_ext(cx, ~"push_all_move")),
+                                ~[build::mk_path(cx, sp, ~[ident])]);
+            ~[build::mk_stmt(cx, sp, e_push)]            
+        }
     }
 }
 
-fn mk_tts(cx: ext_ctxt, sp: span, tts: &[ast::token_tree]) -> @ast::expr {
-    let e_tts = tts.map(|tt| mk_tt(cx, sp, tt));
-    build::mk_call(cx, sp,
-                   ids_ext(cx, ~[~"vec", ~"concat"]),
-                   ~[build::mk_slice_vec_e(cx, sp, e_tts)])
+fn mk_tts(cx: ext_ctxt, sp: span, tts: &[ast::token_tree])
+    -> ~[@ast::stmt] {
+    let mut ss = ~[];
+    for tts.each |tt| {
+        ss.push_all_move(mk_tt(cx, sp, tt));
+    }
+    ss
 }
 
 fn expand_tts(cx: ext_ctxt,
               sp: span,
               tts: ~[ast::token_tree]) -> @ast::expr {
+
     // NB: It appears that the main parser loses its mind if we consider
     // $foo as a tt_nonterminal during the main parse, so we have to re-parse
     // under quote_depth > 0. This is silly and should go away; the _guess_ is
     // it has to do with transition away from supporting old-style macros, so
     // try removing it when enough of them are gone.
+
     let p = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(), tts);
     p.quote_depth += 1u;
     let tts = p.parse_all_token_trees();
     p.abort_if_errors();
 
     // We want to emit a block expression that does a sequence of 'use's to
-    // import the runtime module, followed by a tt expression.
+    // import the runtime module, followed by a tt-building expression.
+
     let uses = ~[ build::mk_glob_use(cx, sp, ids_ext(cx, ~[~"syntax",
                                                            ~"ext",
                                                            ~"quote",
                                                            ~"rt"])) ];
-    build::mk_block(cx, sp, uses, ~[], Some(mk_tts(cx, sp, tts)))
+
+    // We also bind a single value, sp, to ext_cx.call_site()
+    //
+    // This causes every span in a token-tree quote to be attributed to the
+    // call site of the extension using the quote. We can't really do much
+    // better since the source of the quote may well be in a library that
+    // was not even parsed by this compilation run, that the user has no
+    // source code for (eg. in libsyntax, which they're just _using_).
+    //
+    // The old quasiquoter had an elaborate mechanism for denoting input
+    // file locations from which quotes originated; unfortunately this
+    // relied on feeding the source string of the quote back into the
+    // compiler (which we don't really want to do) and, in any case, only
+    // pushed the problem a very small step further back: an error
+    // resulting from a parse of the resulting quote is still attributed to
+    // the site the string literal occured, which was in a source file
+    // _other_ than the one the user has control over. For example, an
+    // error in a quote from the protocol compiler, invoked in user code
+    // using proto! for example, will be attributed to the pipec.rs file in
+    // libsyntax, which the user might not even have source to (unless they
+    // happen to have a compiler on hand). Over all, the phase distinction
+    // just makes quotes "hard to attribute". Possibly this could be fixed
+    // by recreating some of the original qq machinery in the tt regime
+    // (pushing fake FileMaps onto the parser to account for original sites
+    // of quotes, for example) but at this point it seems not likely to be
+    // worth the hassle.
+
+    let e_sp = build::mk_call_(cx, sp,
+                               build::mk_access(cx, sp,
+                                                ids_ext(cx, ~[~"ext_cx"]),
+                                                id_ext(cx, ~"call_site")),
+                               ~[]);
+
+    let stmt_let_sp = build::mk_local(cx, sp, false,
+                                      id_ext(cx, ~"sp"),
+                                      e_sp);
+    
+    let stmt_let_tt = build::mk_local(cx, sp, true,
+                                      id_ext(cx, ~"tt"),
+                                      build::mk_uniq_vec_e(cx, sp, ~[]));
+
+    build::mk_block(cx, sp, uses,
+                    ~[stmt_let_sp,
+                      stmt_let_tt] + mk_tts(cx, sp, tts),
+                    Some(build::mk_path(cx, sp,
+                                        ids_ext(cx, ~[~"tt"]))))
 }
 
 fn expand_parse_call(cx: ext_ctxt,