perf: use memchr for split_into_lines

closes #73

Author: Boshen

src/helpers.rs

@@ -469,26 +469,38 @@ pub fn split_into_potential_tokens(source: &str) -> PotentialTokens {
   }
 }
 
-// /[^\n]+\n?|\n/g
-pub fn split_into_lines(source: &str) -> Vec<&str> {
-  let mut results = Vec::new();
-  let mut i = 0;
-  let bytes = source.as_bytes();
-  while i < bytes.len() {
-    let cc = bytes[i];
-    if cc == 10 {
-      results.push("\n");
-      i += 1;
-    } else {
-      let mut j = i + 1;
-      while j < bytes.len() && bytes[j] != 10 {
-        j += 1;
+/// Split the string with a needle, each string will contain the needle.
+///
+/// Copied and modified from https://github.com/rust-lang/cargo/blob/30efe860c0e4adc1a6d7057ad223dc6e47d34edf/src/cargo/sources/registry/index.rs#L1048-L1072
+fn split(haystack: &str, needle: u8) -> impl Iterator<Item = &str> {
+  struct Split<'a> {
+    haystack: &'a str,
+    needle: u8,
+  }
+
+  impl<'a> Iterator for Split<'a> {
+    type Item = &'a str;
+
+    fn next(&mut self) -> Option<&'a str> {
+      if self.haystack.is_empty() {
+        return None;
       }
-      results.push(&source[i..(j + 1).min(bytes.len())]);
-      i = j + 1;
+      let (ret, remaining) =
+        match memchr::memchr(self.needle, self.haystack.as_bytes()) {
+          Some(pos) => (&self.haystack[..=pos], &self.haystack[pos + 1..]),
+          None => (self.haystack, ""),
+        };
+      self.haystack = remaining;
+      Some(ret)
     }
   }
-  results
+
+  Split { haystack, needle }
+}
+
+// /[^\n]+\n?|\n/g
+pub fn split_into_lines(source: &str) -> Vec<&str> {
+  split(source, b'\n').collect()
 }
 
 pub fn get_generated_source_info(source: &str) -> GeneratedInfo {