Try to support pre and post-change metavars

Author: Veykril

Cargo.lock

@@ -110,6 +110,7 @@ nohash-hasher = "0.2.0"
 rayon = "1.8.0"
 rust-analyzer-salsa = "0.17.0-pre.4"
 rustc-hash = "1.1.0"
+semver = "1.0.14"
 serde = { version = "1.0.192", features = ["derive"] }
 serde_json = "1.0.108"
 smallvec = { version = "1.10.0", features = [

Cargo.toml

@@ -16,6 +16,7 @@ la-arena.workspace = true
 rust-analyzer-salsa.workspace = true
 rustc-hash.workspace = true
 triomphe.workspace = true
+semver.workspace = true
 
 # local deps
 cfg.workspace = true

crates/base-db/Cargo.toml

@@ -11,6 +11,7 @@ use std::{fmt, mem, ops, str::FromStr};
 use cfg::CfgOptions;
 use la_arena::{Arena, Idx};
 use rustc_hash::{FxHashMap, FxHashSet};
+use semver::Version;
 use syntax::SmolStr;
 use triomphe::Arc;
 use vfs::{file_set::FileSet, AbsPathBuf, AnchoredPath, FileId, VfsPath};
@@ -258,7 +259,7 @@ impl ReleaseChannel {
 
     pub fn from_str(str: &str) -> Option<Self> {
         Some(match str {
-            "" => ReleaseChannel::Stable,
+            "" | "stable" => ReleaseChannel::Stable,
             "nightly" => ReleaseChannel::Nightly,
             _ if str.starts_with("beta") => ReleaseChannel::Beta,
             _ => return None,
@@ -289,7 +290,7 @@ pub struct CrateData {
     // things. This info does need to be somewhat present though as to prevent deduplication from
     // happening across different workspaces with different layouts.
     pub target_layout: TargetLayoutLoadResult,
-    pub channel: Option<ReleaseChannel>,
+    pub toolchain: Option<Version>,
 }
 
 impl CrateData {
@@ -346,6 +347,10 @@ impl CrateData {
 
         slf_deps.eq(other_deps)
     }
+
+    pub fn channel(&self) -> Option<ReleaseChannel> {
+        self.toolchain.as_ref().and_then(|v| ReleaseChannel::from_str(&v.pre))
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -427,7 +432,7 @@ impl CrateGraph {
         is_proc_macro: bool,
         origin: CrateOrigin,
         target_layout: Result<Arc<str>, Arc<str>>,
-        channel: Option<ReleaseChannel>,
+        toolchain: Option<Version>,
     ) -> CrateId {
         let data = CrateData {
             root_file_id,
@@ -441,7 +446,7 @@ impl CrateGraph {
             origin,
             target_layout,
             is_proc_macro,
-            channel,
+            toolchain,
         };
         self.arena.alloc(data)
     }

crates/base-db/src/input.rs

@@ -23,6 +23,8 @@ pub use salsa::{self, Cancelled};
 pub use span::{FilePosition, FileRange};
 pub use vfs::{file_set::FileSet, AnchoredPath, AnchoredPathBuf, FileId, VfsPath};
 
+pub use semver::{BuildMetadata, Prerelease, Version, VersionReq};
+
 #[macro_export]
 macro_rules! impl_intern_key {
     ($name:ident) => {

crates/base-db/src/lib.rs

@@ -1,8 +1,10 @@
 //! Defines database & queries for macro expansion.
 
+use std::sync::OnceLock;
+
 use base_db::{
     salsa::{self, debug::DebugQueryTable},
-    CrateId, Edition, FileId, SourceDatabase,
+    CrateId, Edition, FileId, SourceDatabase, VersionReq,
 };
 use either::Either;
 use limit::Limit;
@@ -45,32 +47,68 @@ pub struct DeclarativeMacroExpander {
     pub transparency: Transparency,
 }
 
+// FIXME: Remove this once we drop support for 1.76
+static REQUIREMENT: OnceLock<VersionReq> = OnceLock::new();
+
 impl DeclarativeMacroExpander {
     pub fn expand(
         &self,
         db: &dyn ExpandDatabase,
         tt: tt::Subtree,
         call_id: MacroCallId,
     ) -> ExpandResult<tt::Subtree> {
+        let toolchain = &db.crate_graph()[db.lookup_intern_macro_call(call_id).def.krate].toolchain;
+        let new_meta_vars = toolchain.as_ref().map_or(false, |version| {
+            REQUIREMENT.get_or_init(|| VersionReq::parse(">=1.76").unwrap()).matches(
+                &base_db::Version {
+                    pre: base_db::Prerelease::EMPTY,
+                    build: base_db::BuildMetadata::EMPTY,
+                    major: version.major,
+                    minor: version.minor,
+                    patch: version.patch,
+                },
+            )
+        });
         match self.mac.err() {
             Some(e) => ExpandResult::new(
                 tt::Subtree::empty(tt::DelimSpan::DUMMY),
                 ExpandError::other(format!("invalid macro definition: {e}")),
             ),
             None => self
                 .mac
-                .expand(&tt, |s| s.ctx = apply_mark(db, s.ctx, call_id, self.transparency))
+                .expand(
+                    &tt,
+                    |s| s.ctx = apply_mark(db, s.ctx, call_id, self.transparency),
+                    new_meta_vars,
+                )
                 .map_err(Into::into),
         }
     }
 
-    pub fn expand_unhygienic(&self, tt: tt::Subtree) -> ExpandResult<tt::Subtree> {
+    pub fn expand_unhygienic(
+        &self,
+        db: &dyn ExpandDatabase,
+        tt: tt::Subtree,
+        krate: CrateId,
+    ) -> ExpandResult<tt::Subtree> {
+        let toolchain = &db.crate_graph()[krate].toolchain;
+        let new_meta_vars = toolchain.as_ref().map_or(false, |version| {
+            REQUIREMENT.get_or_init(|| VersionReq::parse(">=1.76").unwrap()).matches(
+                &base_db::Version {
+                    pre: base_db::Prerelease::EMPTY,
+                    build: base_db::BuildMetadata::EMPTY,
+                    major: version.major,
+                    minor: version.minor,
+                    patch: version.patch,
+                },
+            )
+        });
         match self.mac.err() {
             Some(e) => ExpandResult::new(
                 tt::Subtree::empty(tt::DelimSpan::DUMMY),
                 ExpandError::other(format!("invalid macro definition: {e}")),
             ),
-            None => self.mac.expand(&tt, |_| ()).map_err(Into::into),
+            None => self.mac.expand(&tt, |_| (), new_meta_vars).map_err(Into::into),
         }
     }
 }
@@ -278,7 +316,7 @@ pub fn expand_speculative(
             expander.expand(db, actual_macro_call, &adt, span_map)
         }
         MacroDefKind::Declarative(it) => {
-            db.decl_macro_expander(loc.krate, it).expand_unhygienic(tt)
+            db.decl_macro_expander(loc.krate, it).expand_unhygienic(db, tt, loc.def.krate)
         }
         MacroDefKind::BuiltIn(it, _) => it.expand(db, actual_macro_call, &tt).map_err(Into::into),
         MacroDefKind::BuiltInEager(it, _) => {
@@ -525,7 +563,8 @@ fn decl_macro_expander(
     def_crate: CrateId,
     id: AstId<ast::Macro>,
 ) -> Arc<DeclarativeMacroExpander> {
-    let is_2021 = db.crate_graph()[def_crate].edition >= Edition::Edition2021;
+    let crate_data = &db.crate_graph()[def_crate];
+    let is_2021 = crate_data.edition >= Edition::Edition2021;
     let (root, map) = parse_with_map(db, id.file_id);
     let root = root.syntax_node();
 
@@ -549,13 +588,25 @@ fn decl_macro_expander(
             _ => None,
         }
     };
+    let toolchain = crate_data.toolchain.as_ref();
+    let new_meta_vars = toolchain.as_ref().map_or(false, |version| {
+        REQUIREMENT.get_or_init(|| VersionReq::parse(">=1.76").unwrap()).matches(
+            &base_db::Version {
+                pre: base_db::Prerelease::EMPTY,
+                build: base_db::BuildMetadata::EMPTY,
+                major: version.major,
+                minor: version.minor,
+                patch: version.patch,
+            },
+        )
+    });
 
     let (mac, transparency) = match id.to_ptr(db).to_node(&root) {
         ast::Macro::MacroRules(macro_rules) => (
             match macro_rules.token_tree() {
                 Some(arg) => {
                     let tt = mbe::syntax_node_to_token_tree(arg.syntax(), map.as_ref());
-                    let mac = mbe::DeclarativeMacro::parse_macro_rules(&tt, is_2021);
+                    let mac = mbe::DeclarativeMacro::parse_macro_rules(&tt, is_2021, new_meta_vars);
                     mac
                 }
                 None => mbe::DeclarativeMacro::from_err(
@@ -569,7 +620,7 @@ fn decl_macro_expander(
             match macro_def.body() {
                 Some(arg) => {
                     let tt = mbe::syntax_node_to_token_tree(arg.syntax(), map.as_ref());
-                    let mac = mbe::DeclarativeMacro::parse_macro2(&tt, is_2021);
+                    let mac = mbe::DeclarativeMacro::parse_macro2(&tt, is_2021, new_meta_vars);
                     mac
                 }
                 None => mbe::DeclarativeMacro::from_err(

crates/hir-expand/src/db.rs

@@ -693,7 +693,7 @@ impl<'a> CompletionContext<'a> {
         let krate = scope.krate();
         let module = scope.module();
 
-        let toolchain = db.crate_graph()[krate.into()].channel;
+        let toolchain = db.crate_graph()[krate.into()].channel();
         // `toolchain == None` means we're in some detached files. Since we have no information on
         // the toolchain being used, let's just allow unstable items to be listed.
         let is_nightly = matches!(toolchain, Some(base_db::ReleaseChannel::Nightly) | None);

crates/ide-completion/src/context.rs

@@ -492,7 +492,7 @@ fn get_doc_base_urls(
     let Some(krate) = def.krate(db) else { return Default::default() };
     let Some(display_name) = krate.display_name(db) else { return Default::default() };
     let crate_data = &db.crate_graph()[krate.into()];
-    let channel = crate_data.channel.map_or("nightly", ReleaseChannel::as_str);
+    let channel = crate_data.channel().unwrap_or(ReleaseChannel::Nightly).as_str();
 
     let (web_base, local_base) = match &crate_data.origin {
         // std and co do not specify `html_root_url` any longer so we gotta handwrite this ourself.

crates/ide/src/doc_links.rs

@@ -40,7 +40,7 @@ pub(crate) fn shuffle_crate_graph(db: &mut RootDatabase) {
             data.is_proc_macro,
             data.origin.clone(),
             data.target_layout.clone(),
-            data.channel,
+            data.toolchain.clone(),
         );
         new_proc_macros.insert(new_id, proc_macros[&old_id].clone());
         map.insert(old_id, new_id);

crates/ide/src/shuffle_crate_graph.rs

@@ -20,7 +20,10 @@ fn benchmark_parse_macro_rules() {
     let rules = macro_rules_fixtures_tt();
     let hash: usize = {
         let _pt = bench("mbe parse macro rules");
-        rules.values().map(|it| DeclarativeMacro::parse_macro_rules(it, true).rules.len()).sum()
+        rules
+            .values()
+            .map(|it| DeclarativeMacro::parse_macro_rules(it, true, true).rules.len())
+            .sum()
     };
     assert_eq!(hash, 1144);
 }
@@ -38,7 +41,7 @@ fn benchmark_expand_macro_rules() {
         invocations
             .into_iter()
             .map(|(id, tt)| {
-                let res = rules[&id].expand(&tt, |_| ());
+                let res = rules[&id].expand(&tt, |_| (), true);
                 assert!(res.err.is_none());
                 res.value.token_trees.len()
             })
@@ -50,7 +53,7 @@ fn benchmark_expand_macro_rules() {
 fn macro_rules_fixtures() -> FxHashMap<String, DeclarativeMacro<DummyTestSpanData>> {
     macro_rules_fixtures_tt()
         .into_iter()
-        .map(|(id, tt)| (id, DeclarativeMacro::parse_macro_rules(&tt, true)))
+        .map(|(id, tt)| (id, DeclarativeMacro::parse_macro_rules(&tt, true, true)))
         .collect()
 }
 
@@ -105,7 +108,7 @@ fn invocation_fixtures(
                     for op in rule.lhs.iter() {
                         collect_from_op(op, &mut subtree, &mut seed);
                     }
-                    if it.expand(&subtree, |_| ()).err.is_none() {
+                    if it.expand(&subtree, |_| (), true).err.is_none() {
                         res.push((name.clone(), subtree));
                         break;
                     }

crates/mbe/src/benchmark.rs

@@ -16,6 +16,7 @@ pub(crate) fn expand_rules<S: Span>(
     input: &tt::Subtree<S>,
     marker: impl Fn(&mut S) + Copy,
     is_2021: bool,
+    new_meta_vars: bool,
 ) -> ExpandResult<tt::Subtree<S>> {
     let mut match_: Option<(matcher::Match<S>, &crate::Rule<S>)> = None;
     for rule in rules {
@@ -26,7 +27,7 @@ pub(crate) fn expand_rules<S: Span>(
             // Unconditionally returning the transcription here makes the
             // `test_repeat_bad_var` test fail.
             let ExpandResult { value, err: transcribe_err } =
-                transcriber::transcribe(&rule.rhs, &new_match.bindings, marker);
+                transcriber::transcribe(&rule.rhs, &new_match.bindings, marker, new_meta_vars);
             if transcribe_err.is_none() {
                 return ExpandResult::ok(value);
             }
@@ -45,7 +46,7 @@ pub(crate) fn expand_rules<S: Span>(
     if let Some((match_, rule)) = match_ {
         // if we got here, there was no match without errors
         let ExpandResult { value, err: transcribe_err } =
-            transcriber::transcribe(&rule.rhs, &match_.bindings, marker);
+            transcriber::transcribe(&rule.rhs, &match_.bindings, marker, new_meta_vars);
         ExpandResult { value, err: match_.err.or(transcribe_err) }
     } else {
         ExpandResult::new(

crates/mbe/src/expander.rs

@@ -131,8 +131,9 @@ pub(super) fn transcribe<S: Span>(
     template: &MetaTemplate<S>,
     bindings: &Bindings<S>,
     marker: impl Fn(&mut S) + Copy,
+    new_meta_vars: bool,
 ) -> ExpandResult<tt::Subtree<S>> {
-    let mut ctx = ExpandCtx { bindings, nesting: Vec::new() };
+    let mut ctx = ExpandCtx { bindings, nesting: Vec::new(), new_meta_vars };
     let mut arena: Vec<tt::TokenTree<S>> = Vec::new();
     expand_subtree(&mut ctx, template, None, &mut arena, marker)
 }
@@ -152,6 +153,7 @@ struct NestingState {
 struct ExpandCtx<'a, S> {
     bindings: &'a Bindings<S>,
     nesting: Vec<NestingState>,
+    new_meta_vars: bool,
 }
 
 fn expand_subtree<S: Span>(
@@ -284,7 +286,13 @@ fn expand_subtree<S: Span>(
                     }
                 }
 
-                let c = match count(ctx, binding, 0, *depth) {
+                let res = if ctx.new_meta_vars {
+                    count(ctx, binding, 0, depth.unwrap_or(0))
+                } else {
+                    count_old(ctx, binding, 0, *depth)
+                };
+
+                let c = match res {
                     Ok(c) => c,
                     Err(e) => {
                         // XXX: It *might* make sense to emit a dummy integer value like `0` here.
@@ -548,3 +556,32 @@ fn count<S>(
         Binding::Fragment(_) | Binding::Missing(_) => Ok(1),
     }
 }
+
+fn count_old<S>(
+    ctx: &ExpandCtx<'_, S>,
+    binding: &Binding<S>,
+    our_depth: usize,
+    count_depth: Option<usize>,
+) -> Result<usize, CountError> {
+    match binding {
+        Binding::Nested(bs) => match count_depth {
+            None => bs.iter().map(|b| count_old(ctx, b, our_depth + 1, None)).sum(),
+            Some(0) => Ok(bs.len()),
+            Some(d) => bs.iter().map(|b| count_old(ctx, b, our_depth + 1, Some(d - 1))).sum(),
+        },
+        Binding::Empty => Ok(0),
+        Binding::Fragment(_) | Binding::Missing(_) => {
+            if our_depth == 0 {
+                // `${count(t)}` is placed inside the innermost repetition. This includes cases
+                // where `t` is not a repeated fragment.
+                Err(CountError::Misplaced)
+            } else if count_depth.is_none() {
+                Ok(1)
+            } else {
+                // We've reached at the innermost repeated fragment, but the user wants us to go
+                // further!
+                Err(CountError::OutOfBounds)
+            }
+        }
+    }
+}