Merge pull request #1283 from epage/email

fix(token): Improve email detection by prefering that over base64

Author: epage

crates/typos/src/tokens.rs

@@ -188,15 +188,15 @@ mod parser {
                 // - Update `is_ignore_char` to make sure `sep1` doesn't eat it all up
                 // - Make sure you always consume it
                 terminated(uuid_literal, peek(sep1)),
-                terminated(hash_literal, peek(sep1)),
+                terminated(email_literal, peek(sep1)),
+                terminated(url_literal, peek(sep1)),
+                terminated(jwt, peek(sep1)),
                 terminated(base64_literal, peek(sep1)), // base64 should be quoted or something
+                terminated(hash_literal, peek(sep1)),
                 terminated(ordinal_literal, peek(sep1)),
                 terminated(hex_literal, peek(sep1)),
                 terminated(dec_literal, peek(sep1)), // Allow digit-prefixed words
-                terminated(email_literal, peek(sep1)),
-                terminated(url_literal, peek(sep1)),
                 terminated(css_color, peek(sep1)),
-                terminated(jwt, peek(sep1)),
                 c_escape,
                 printf,
                 other,
@@ -1653,7 +1653,7 @@ mod test {
     fn tokenize_ignore_email() {
         let parser = TokenizerBuilder::new().build();
 
-        let input = "Good [email protected] Bye";
+        let input = "Good [email protected] if=@nam iff=@nam Bye";
         let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect();
         assert_data_eq!(
             actual.to_debug(),
@@ -1667,7 +1667,7 @@ mod test {
     Identifier {
         token: "Bye",
         case: None,
-        offset: 25,
+        offset: 42,
     },
 ]
 
@@ -1686,7 +1686,7 @@ mod test {
     Identifier {
         token: "Bye",
         case: None,
-        offset: 25,
+        offset: 42,
     },
 ]