Merge branch 'optimize_hashtables'

Author: Byron

Cargo.lock

@@ -174,6 +174,7 @@ members = [
     "git-path",
     "git-repository",
     "gitoxide-core",
+    "git-hashtable",
     "git-tui",
     "git-tix",
 

Cargo.toml

@@ -156,6 +156,7 @@ is usable to some extend.
   * [git-worktree](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-worktree)
   * [git-bitmap](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-bitmap)
   * [git-date](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-date)
+  * [git-hashtable](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-hashtable)
 * **idea** _(just a name placeholder)_
   * [git-note](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-note)
   * [git-fetchhead](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-fetchhead)

README.md

@@ -11,6 +11,12 @@
 * [x] decode the chunk file table of contents and provide convenient API
 * [x] write the table of contents
 
+### git-hashtable
+
+* [x] hashmap
+* [x] hashset
+
+
 ### git-object
 * *decode (zero-copy)* borrowed objects
     * [x] commit

crate-status.md

@@ -32,7 +32,7 @@ echo "in root: gitoxide CLI"
 (enter git-config && indent cargo diet -n --package-size-limit 120KB)
 (enter git-config-value && indent cargo diet -n --package-size-limit 20KB)
 (enter git-command && indent cargo diet -n --package-size-limit 5KB)
-(enter git-hash && indent cargo diet -n --package-size-limit 20KB)
+(enter git-hash && indent cargo diet -n --package-size-limit 30KB)
 (enter git-chunk && indent cargo diet -n --package-size-limit 10KB)
 (enter git-rebase && indent cargo diet -n --package-size-limit 5KB)
 (enter git-sequencer && indent cargo diet -n --package-size-limit 5KB)
@@ -43,6 +43,7 @@ echo "in root: gitoxide CLI"
 (enter git-url && indent cargo diet -n --package-size-limit 20KB)
 (enter git-validate && indent cargo diet -n --package-size-limit 5KB)
 (enter git-date && indent cargo diet -n --package-size-limit 15KB)
+(enter git-hashtable && indent cargo diet -n --package-size-limit 5KB)
 (enter git-filter && indent cargo diet -n --package-size-limit 5KB)
 (enter git-lfs && indent cargo diet -n --package-size-limit 5KB)
 (enter git-note && indent cargo diet -n --package-size-limit 5KB)

etc/check-package-size.sh

@@ -1,15 +1,29 @@
+use std::hash::{Hash, Hasher};
 use std::{borrow::Borrow, convert::TryInto, fmt, ops::Deref};
 
 use crate::{borrowed::oid, Kind, SIZE_OF_SHA1_DIGEST};
 
 /// An owned hash identifying objects, most commonly Sha1
-#[derive(PartialEq, Eq, Hash, Ord, PartialOrd, Clone, Copy)]
+#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy)]
 #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
 pub enum ObjectId {
     /// A SHA 1 hash digest
     Sha1([u8; SIZE_OF_SHA1_DIGEST]),
 }
 
+// False positive: https://github.com/rust-lang/rust-clippy/issues/2627
+// ingoring some fields while hashing is perfectly valid and just leads to
+// increased HashCollisions. One Sha1 being a prefix of another Sha256 is
+// extremly unlikely to begin with so it doesn't matter.
+// This implementation matches the `Hash` implementation for `oid`
+// and allows the usage of custom Hashers that only copy a truncated ShaHash
+#[allow(clippy::derive_hash_xor_eq)]
+impl Hash for ObjectId {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        state.write(self.as_slice())
+    }
+}
+
 #[allow(missing_docs)]
 pub mod decode {
     use std::str::FromStr;

git-hash/src/object_id.rs

@@ -0,0 +1,20 @@
+[package]
+name = "git-hashtable"
+version = "0.1.0"
+repository = "https://github.com/Byron/gitoxide"
+license = "MIT/Apache-2.0"
+description = "A crate that provides hashtable based data structures optimized to utilize ObjectId keys"
+authors = ["Pascal Kuthe <[email protected]>"]
+edition = "2021"
+include = ["src/**/*"]
+
+[lib]
+doctest = false
+
+[dependencies]
+hashbrown = { version = "0.13.1", default-features = false, features = [
+  "inline-more",
+  "raw"
+] }
+git-hash = { version = "^0.10.0", path = "../git-hash" }
+

git-hashtable/Cargo.toml

@@ -0,0 +1,47 @@
+//! Customized HashMap and Hasher implementation optimized for using `ObjectId`s as keys.
+//!
+//! The crate mirrors `std::collections` in layout for familiarity.
+#![deny(missing_docs, rust_2018_idioms)]
+#![forbid(unsafe_code)]
+
+use git_hash::ObjectId;
+
+pub use hashbrown::{hash_map, hash_set, raw, Equivalent};
+
+///
+pub mod hash {
+    /// A Hasher for usage with HashMap keys that are already robust hashes (like an `ObjectId`).
+    /// The first `8` bytes of the hash are used as the `HashMap` hash
+    #[derive(Default, Clone, Copy)]
+    pub struct Hasher(u64);
+
+    impl std::hash::Hasher for Hasher {
+        fn finish(&self) -> u64 {
+            self.0
+        }
+
+        #[inline(always)]
+        fn write(&mut self, bytes: &[u8]) {
+            self.0 = u64::from_ne_bytes(bytes[..8].try_into().unwrap());
+        }
+    }
+
+    /// A Hasher for usage with HashMap keys that are already robust hashes (like an `ObjectId`).
+    /// The first `8` bytes of the hash are used as the `HashMap` hash
+    #[derive(Default, Clone, Copy)]
+    pub struct Builder;
+    impl std::hash::BuildHasher for Builder {
+        type Hasher = Hasher;
+
+        fn build_hasher(&self) -> Self::Hasher {
+            Hasher::default()
+        }
+    }
+}
+
+/// A HashMap for usage with keys that are already robust hashes (like an `ObjectId`).
+/// The first `8` bytes of the hash are used as the `HashMap` hash
+pub type HashMap<K, V> = hashbrown::HashMap<K, V, hash::Builder>;
+/// A HashSet for usage with keys that are already robust hashes (like an `ObjectId`).
+/// The first `8` bytes of the hash are used as the `HashMap` hash
+pub type HashSet<T = ObjectId> = hashbrown::HashSet<T, hash::Builder>;

git-hashtable/src/lib.rs

@@ -45,6 +45,7 @@ git-object = { version = "^0.23.0", path = "../git-object" }
 git-traverse = { version = "^0.19.0", path = "../git-traverse" }
 git-diff = { version = "^0.23.0", path = "../git-diff" }
 git-tempfile = { version = "^3.0.0", path = "../git-tempfile" }
+git-hashtable = { version = "^0.1.0", path = "../git-hashtable" }
 
 smallvec = "1.3.0"
 memmap2 = "0.5.0"
@@ -55,7 +56,6 @@ thiserror = "1.0.26"
 uluru = { version = "3.0.0", optional = true }
 clru = { version = "0.5.0", optional = true }
 dashmap = "5.1.0"
-hash_hasher = "2.0.3"
 
 document-features = { version = "0.2.0", optional = true }
 

git-pack/Cargo.toml

@@ -29,7 +29,7 @@ mod memory {
 
     /// An LRU cache with hash map backing and an eviction rule based on the memory usage for object data in bytes.
     pub struct MemoryCappedHashmap {
-        inner: clru::CLruCache<Key, Entry, hash_hasher::HashBuildHasher, CustomScale>,
+        inner: clru::CLruCache<Key, Entry, git_hashtable::hash::Builder, CustomScale>,
         free_list: Vec<Vec<u8>>,
         debug: git_features::cache::Debug,
     }
@@ -45,7 +45,7 @@ mod memory {
             MemoryCappedHashmap {
                 inner: clru::CLruCache::with_config(
                     clru::CLruCacheConfig::new(NonZeroUsize::new(memory_cap_in_bytes).expect("non zero"))
-                        .with_hasher(hash_hasher::HashBuildHasher::default())
+                        .with_hasher(git_hashtable::hash::Builder::default())
                         .with_scale(CustomScale),
                 ),
                 free_list: Vec::new(),

git-pack/src/cache/object.rs

@@ -5,7 +5,6 @@ use std::{
 
 use git_features::{parallel, progress::Progress};
 use git_hash::ObjectId;
-use hash_hasher::{HashBuildHasher, HashedSet};
 
 use crate::{data::output, find};
 
@@ -65,7 +64,7 @@ where
         inner: objects_ids,
         size: chunk_size,
     };
-    let seen_objs = dashmap::DashSet::<ObjectId, HashBuildHasher>::default();
+    let seen_objs = dashmap::DashSet::<ObjectId, git_hashtable::hash::Builder>::default();
     let progress = Arc::new(parking_lot::Mutex::new(progress));
 
     parallel::in_parallel(
@@ -120,7 +119,7 @@ where
     Oid: Into<ObjectId>,
     IterErr: std::error::Error,
 {
-    let seen_objs = RefCell::new(HashedSet::<ObjectId>::default());
+    let seen_objs = RefCell::new(git_hashtable::HashSet::default());
 
     let (mut buf1, mut buf2) = (Vec::new(), Vec::new());
     expand::this(

git-pack/src/data/output/count/objects/mod.rs

@@ -6,17 +6,17 @@ mod trait_impls {
     use std::{cell::RefCell, hash::Hash};
 
     use dashmap::DashSet;
-    use hash_hasher::{HashBuildHasher, HashedSet};
+    use git_hashtable::HashSet;
 
     use super::InsertImmutable;
 
-    impl<T: Eq + Hash> InsertImmutable<T> for DashSet<T, HashBuildHasher> {
+    impl<T: Eq + Hash> InsertImmutable<T> for DashSet<T, git_hashtable::hash::Builder> {
         fn insert(&self, item: T) -> bool {
             self.insert(item)
         }
     }
 
-    impl<T: Eq + Hash> InsertImmutable<T> for RefCell<HashedSet<T>> {
+    impl<T: Eq + Hash> InsertImmutable<T> for RefCell<HashSet<T>> {
         fn insert(&self, item: T) -> bool {
             self.borrow_mut().insert(item)
         }

git-pack/src/data/output/count/objects/util.rs

@@ -115,6 +115,7 @@ git-credentials = { version = "^0.7.0", path = "../git-credentials" }
 git-prompt = { version = "^0.2.0", path = "../git-prompt" }
 git-index = { version = "^0.9.1", path = "../git-index" }
 git-worktree = { version = "^0.9.0", path = "../git-worktree" }
+git-hashtable = { version = "^0.1.0", path = "../git-hashtable" }
 
 once_cell = "1.14.0"
 signal-hook = { version = "0.3.9", default-features = false }

git-repository/Cargo.toml

@@ -20,6 +20,7 @@ pub mod describe {
     use std::borrow::Cow;
 
     use git_hash::ObjectId;
+    use git_hashtable::HashMap;
     use git_odb::Find;
 
     use crate::{bstr::BStr, ext::ObjectIdExt, Repository};
@@ -66,10 +67,7 @@ pub mod describe {
     }
 
     impl SelectRef {
-        fn names(
-            &self,
-            repo: &Repository,
-        ) -> Result<git_revision::hash_hasher::HashedMap<ObjectId, Cow<'static, BStr>>, Error> {
+        fn names(&self, repo: &Repository) -> Result<HashMap<ObjectId, Cow<'static, BStr>>, Error> {
             let platform = repo.references()?;
 
             Ok(match self {

git-repository/src/commit.rs

@@ -19,9 +19,9 @@ serde1 = [ "serde", "git-hash/serde1", "git-object/serde1" ]
 git-hash = { version = "^0.10.0", path = "../git-hash" }
 git-object = { version = "^0.23.0", path = "../git-object" }
 git-date = { version = "^0.3.0", path = "../git-date" }
+git-hashtable = { version = "^0.1.0", path = "../git-hashtable" }
 
 bstr = { version = "1.0.1", default-features = false, features = ["std"]}
-hash_hasher = "2.0.3"
 thiserror = "1.0.26"
 serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }
 document-features = { version = "0.2.1", optional = true }