Update proc macro2 (rust-lang#455)

* Update to proc_macro2 0.4 and related

* Update to proc_macro2 0.4 and related

* Update to proc_macro2 0.4 and related

* Add proc_macro_gen feature

* Update to the new rustfmt cli

* A few proc-macro2 stylistic updates

* Disable RUST_BACKTRACE by default

* Allow rustfmt failure for now

* Disable proc-macro2 nightly feature in verify-x86

Currently this causes bugs on nightly due to upstream rustc bugs, this should be
temporary

* Attempt to thwart mergefunc

* Use static relocation model on i686

Author: lu-zero

.travis.yml

@@ -48,13 +48,14 @@ matrix:
       before_script:
       - rustup component add rustfmt-preview
       script:
-      - cargo fmt --all -- --write-mode=diff
+      - cargo fmt --all -- --check
     - env: CLIPPY=On TARGET=x86_64-unknown-linux-gnu NO_ADD=1
       script: |
         cargo install clippy
         cargo clippy --all -- -D clippy-pedantic
   allow_failures:
     - env: CLIPPY=On TARGET=x86_64-unknown-linux-gnu NO_ADD=1
+    - env: RUSTFMT=On TARGET=x86_64-unknown-linux-gnu NO_ADD=1
 
 before_install:
   # FIXME (travis-ci/travis-ci#8920) shouldn't be necessary...

ci/run.sh

@@ -7,7 +7,7 @@ set -ex
 # Tests are all super fast anyway, and they fault often enough on travis that
 # having only one thread increases debuggability to be worth it.
 export RUST_TEST_THREADS=1
-export RUST_BACKTRACE=full
+#export RUST_BACKTRACE=full
 #export RUST_TEST_NOCAPTURE=1
 
 RUSTFLAGS="$RUSTFLAGS --cfg stdsimd_strict"
@@ -28,6 +28,15 @@ case ${TARGET} in
     powerpc64-*)
         export STDSIMD_DISABLE_ASSERT_INSTR=1
         ;;
+
+    # On 32-bit use a static relocation model which avoids some extra
+    # instructions when dealing with static data, notably allowing some
+    # instruction assertion checks to pass below the 20 instruction limit. If
+    # this is the default, dynamic, then too many instructions are generated
+    # when we assert the instruction for a function and it causes tests to fail.
+    i686-* | i586-*)
+        export RUSTFLAGS="${RUSTFLAGS} -C relocation-model=static"
+        ;;
     *)
         ;;
 esac

crates/assert-instr-macro/Cargo.toml

@@ -8,6 +8,6 @@ proc-macro = true
 test = false
 
 [dependencies]
-proc-macro2 = { version = "0.3", features = ["nightly"] }
-quote = "0.5"
-syn = { version = "0.13", features = ["full"] }
+proc-macro2 = { version = "0.4", features = ["nightly"] }
+quote = "0.6"
+syn = { version = "0.14", features = ["full"] }

crates/assert-instr-macro/src/lib.rs

@@ -48,14 +48,15 @@ pub fn assert_instr(
     use quote::ToTokens;
     let instr_str = instr
         .clone()
-        .into_tokens()
+        .into_token_stream()
         .to_string()
         .replace('.', "_")
         .replace(|c: char| c.is_whitespace(), "");
-    let assert_name = syn::Ident::from(
-        &format!("assert_{}_{}", name.as_ref(), instr_str)[..],
+    let assert_name = syn::Ident::new(
+        &format!("assert_{}_{}", name, instr_str),
+        name.span(),
     );
-    let shim_name = syn::Ident::from(format!("{}_shim", name.as_ref()));
+    let shim_name = syn::Ident::new(&format!("{}_shim", name), name.span());
     let mut inputs = Vec::new();
     let mut input_vals = Vec::new();
     let ret = &func.decl.output;
@@ -64,7 +65,7 @@ pub fn assert_instr(
             syn::FnArg::Captured(ref c) => c,
             ref v => panic!(
                 "arguments must not have patterns: `{:?}`",
-                v.clone().into_tokens()
+                v.clone().into_token_stream()
             ),
         };
         let ident = match capture.pat {
@@ -74,7 +75,7 @@ pub fn assert_instr(
         match invoc
             .args
             .iter()
-            .find(|a| a.0 == ident.as_ref())
+            .find(|a| *ident == a.0)
         {
             Some(&(_, ref tts)) => {
                 input_vals.push(quote! { #tts });
@@ -95,7 +96,7 @@ pub fn assert_instr(
                 .expect("attr.path.segments.first() failed")
                 .value()
                 .ident
-                .as_ref()
+                .to_string()
                 .starts_with("target")
         })
         .collect::<Vec<_>>();
@@ -108,15 +109,27 @@ pub fn assert_instr(
     } else {
         syn::LitStr::new("C", proc_macro2::Span::call_site())
     };
+    let shim_name_str = format!("{}{}", shim_name, assert_name);
     let to_test = quote! {
         #attrs
         unsafe extern #abi fn #shim_name(#(#inputs),*) #ret {
+            // The compiler in optimized mode by default runs a pass called
+            // "mergefunc" where it'll merge functions that look identical.
+            // Turns out some intrinsics produce identical code and they're
+            // folded together, meaning that one just jumps to another. This
+            // messes up our inspection of the disassembly of this function and
+            // we're not a huge fan of that.
+            //
+            // To thwart this pass and prevent functions from being merged we
+            // generate some code that's hopefully very tight in terms of
+            // codegen but is otherwise unique to prevent code from being
+            // folded.
+            ::stdsimd_test::_DONT_DEDUP = #shim_name_str;
             #name(#(#input_vals),*)
         }
     };
 
-    let tts: TokenStream = quote_spanned! {
-        proc_macro2::Span::call_site() =>
+    let tts: TokenStream = quote! {
         #[test]
         #[allow(non_snake_case)]
         #maybe_ignore
@@ -169,7 +182,7 @@ where
     T: Clone + IntoIterator,
     T::Item: quote::ToTokens,
 {
-    fn to_tokens(&self, tokens: &mut quote::Tokens) {
+    fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) {
         for item in self.0.clone() {
             item.to_tokens(tokens);
         }

crates/coresimd/src/lib.rs

@@ -10,7 +10,7 @@
 #![allow(dead_code)]
 #![allow(unused_features)]
 #![feature(const_fn, link_llvm_intrinsics, platform_intrinsics, repr_simd,
-           simd_ffi, asm,
+           simd_ffi, asm, proc_macro_gen,
            integer_atomics, stmt_expr_attributes, core_intrinsics,
            crate_in_paths, no_core, attr_literals, rustc_attrs, stdsimd,
            staged_api, core_float, core_slice_ext, align_offset,

crates/simd-test-macro/Cargo.toml

@@ -8,5 +8,5 @@ proc-macro = true
 test = false
 
 [dependencies]
-proc-macro2 = { version = "0.3", features = ["nightly"] }
-quote = "0.5"
+proc-macro2 = { version = "0.4", features = ["nightly"] }
+quote = "0.6"

crates/simd-test-macro/src/lib.rs

@@ -12,7 +12,7 @@ extern crate quote;
 
 use std::env;
 
-use proc_macro2::{Literal, Span, Term, TokenStream, TokenTree};
+use proc_macro2::{Literal, Span, Ident, TokenStream, TokenTree};
 
 fn string(s: &str) -> TokenTree {
     Literal::string(s).into()
@@ -29,11 +29,11 @@ pub fn simd_test(
         panic!("expected #[simd_test(enable = \"feature\")]");
     }
     match &tokens[0] {
-        TokenTree::Term(tt) if tt.to_string() == "enable" => {}
+        TokenTree::Ident(tt) if tt.to_string() == "enable" => {}
         _ => panic!("expected #[simd_test(enable = \"feature\")]"),
     }
     match &tokens[1] {
-        TokenTree::Op(tt) if tt.op() == '=' => {}
+        TokenTree::Punct(tt) if tt.as_char() == '=' => {}
         _ => panic!("expected #[simd_test(enable = \"feature\")]"),
     }
     let enable_feature = match &tokens[2] {
@@ -53,9 +53,9 @@ pub fn simd_test(
     let item = TokenStream::from(item);
     let name = find_name(item.clone());
 
-    let name: TokenStream = name.as_str().parse().expect(&format!(
+    let name: TokenStream = name.to_string().parse().expect(&format!(
         "failed to parse name: {}",
-        name.clone().as_str()
+        name.to_string()
     ));
 
     let target = env::var("TARGET")
@@ -87,9 +87,9 @@ pub fn simd_test(
         }
         t => panic!("unknown target: {}", t),
     };
-    let macro_test = proc_macro2::Term::new(macro_test, Span::call_site());
+    let macro_test = Ident::new(macro_test, Span::call_site());
 
-    let mut cfg_target_features = quote::Tokens::new();
+    let mut cfg_target_features = TokenStream::empty();
     use quote::ToTokens;
     for feature in target_features {
         let q = quote_spanned! {
@@ -119,18 +119,18 @@ pub fn simd_test(
     ret.into()
 }
 
-fn find_name(item: TokenStream) -> Term {
+fn find_name(item: TokenStream) -> Ident {
     let mut tokens = item.into_iter();
     while let Some(tok) = tokens.next() {
-        if let TokenTree::Term(word) = tok {
-            if word.as_str() == "fn" {
+        if let TokenTree::Ident(word) = tok {
+            if word == "fn" {
                 break;
             }
         }
     }
 
     match tokens.next() {
-        Some(TokenTree::Term(word)) => word,
+        Some(TokenTree::Ident(word)) => word,
         _ => panic!("failed to find function name"),
     }
 }

crates/stdsimd-test/src/lib.rs

@@ -405,3 +405,6 @@ pub fn assert_skip_test_ok(name: &str) {
         name
     );
 }
+
+// See comment in `assert-instr-macro` crate for why this exists
+pub static mut _DONT_DEDUP: &'static str = "";

crates/stdsimd-verify/Cargo.toml

@@ -4,9 +4,9 @@ version = "0.1.0"
 authors = ["Alex Crichton <[email protected]>"]
 
 [dependencies]
-proc-macro2 = { version = "0.3", features = ["nightly"] }
-quote = "0.5"
-syn = { version = "0.13", features = ["full"] }
+proc-macro2 = "0.4"
+quote = "0.6"
+syn = { version = "0.14", features = ["full"] }
 
 [lib]
 proc-macro = true

crates/stdsimd-verify/src/lib.rs

@@ -12,11 +12,6 @@ use std::io::Read;
 use std::path::Path;
 
 use proc_macro::TokenStream;
-use quote::Tokens;
-
-macro_rules! my_quote {
-    ($($t:tt)*) => (quote_spanned!(proc_macro2::Span::call_site() => $($t)*))
-}
 
 #[proc_macro]
 pub fn x86_functions(input: TokenStream) -> TokenStream {
@@ -56,7 +51,7 @@ pub fn x86_functions(input: TokenStream) -> TokenStream {
     let functions = functions
         .iter()
         .map(|&(ref f, path)| {
-            let name = f.ident;
+            let name = &f.ident;
             // println!("{}", name);
             let mut arguments = Vec::new();
             for input in f.decl.inputs.iter() {
@@ -67,19 +62,19 @@ pub fn x86_functions(input: TokenStream) -> TokenStream {
                 arguments.push(to_type(ty));
             }
             let ret = match f.decl.output {
-                syn::ReturnType::Default => my_quote! { None },
+                syn::ReturnType::Default => quote! { None },
                 syn::ReturnType::Type(_, ref t) => {
                     let ty = to_type(t);
-                    my_quote! { Some(#ty) }
+                    quote! { Some(#ty) }
                 }
             };
             let instrs = find_instrs(&f.attrs);
             let target_feature = match find_target_feature(&f.attrs) {
-                Some(i) => my_quote! { Some(#i) },
-                None => my_quote! { None },
+                Some(i) => quote! { Some(#i) },
+                None => quote! { None },
             };
             let required_const = find_required_const(&f.attrs);
-            my_quote! {
+            quote! {
                 Function {
                     name: stringify!(#name),
                     arguments: &[#(#arguments),*],
@@ -93,43 +88,43 @@ pub fn x86_functions(input: TokenStream) -> TokenStream {
         })
         .collect::<Vec<_>>();
 
-    let ret = my_quote! { #input: &[Function] = &[#(#functions),*]; };
+    let ret = quote! { #input: &[Function] = &[#(#functions),*]; };
     // println!("{}", ret);
     ret.into()
 }
 
-fn to_type(t: &syn::Type) -> Tokens {
+fn to_type(t: &syn::Type) -> proc_macro2::TokenStream {
     match *t {
-        syn::Type::Path(ref p) => match extract_path_ident(&p.path).as_ref() {
-            "__m128" => my_quote! { &M128 },
-            "__m128d" => my_quote! { &M128D },
-            "__m128i" => my_quote! { &M128I },
-            "__m256" => my_quote! { &M256 },
-            "__m256d" => my_quote! { &M256D },
-            "__m256i" => my_quote! { &M256I },
-            "__m64" => my_quote! { &M64 },
-            "bool" => my_quote! { &BOOL },
-            "f32" => my_quote! { &F32 },
-            "f64" => my_quote! { &F64 },
-            "i16" => my_quote! { &I16 },
-            "i32" => my_quote! { &I32 },
-            "i64" => my_quote! { &I64 },
-            "i8" => my_quote! { &I8 },
-            "u16" => my_quote! { &U16 },
-            "u32" => my_quote! { &U32 },
-            "u64" => my_quote! { &U64 },
-            "u8" => my_quote! { &U8 },
-            "CpuidResult" => my_quote! { &CPUID },
+        syn::Type::Path(ref p) => match extract_path_ident(&p.path).to_string().as_ref() {
+            "__m128" => quote! { &M128 },
+            "__m128d" => quote! { &M128D },
+            "__m128i" => quote! { &M128I },
+            "__m256" => quote! { &M256 },
+            "__m256d" => quote! { &M256D },
+            "__m256i" => quote! { &M256I },
+            "__m64" => quote! { &M64 },
+            "bool" => quote! { &BOOL },
+            "f32" => quote! { &F32 },
+            "f64" => quote! { &F64 },
+            "i16" => quote! { &I16 },
+            "i32" => quote! { &I32 },
+            "i64" => quote! { &I64 },
+            "i8" => quote! { &I8 },
+            "u16" => quote! { &U16 },
+            "u32" => quote! { &U32 },
+            "u64" => quote! { &U64 },
+            "u8" => quote! { &U8 },
+            "CpuidResult" => quote! { &CPUID },
             s => panic!("unspported type: {}", s),
         },
         syn::Type::Ptr(syn::TypePtr { ref elem, .. })
         | syn::Type::Reference(syn::TypeReference { ref elem, .. }) => {
             let tokens = to_type(&elem);
-            my_quote! { &Type::Ptr(#tokens) }
+            quote! { &Type::Ptr(#tokens) }
         }
         syn::Type::Slice(_) => panic!("unsupported slice"),
         syn::Type::Array(_) => panic!("unsupported array"),
-        syn::Type::Tuple(_) => my_quote! { &TUPLE },
+        syn::Type::Tuple(_) => quote! { &TUPLE },
         _ => panic!("unsupported type"),
     }
 }
@@ -145,7 +140,7 @@ fn extract_path_ident(path: &syn::Path) -> syn::Ident {
         syn::PathArguments::None => {}
         _ => panic!("unsupported path that has path arguments"),
     }
-    path.segments.first().unwrap().value().ident
+    path.segments.first().unwrap().value().ident.clone()
 }
 
 fn walk(root: &Path, files: &mut Vec<(syn::File, String)>) {