Minor adjustments to the worktree structure.

Author: Byron

gitoxide-core/src/index/checkout.rs

@@ -4,7 +4,7 @@ use std::{
 };
 
 use anyhow::bail;
-use gix::{odb::FindExt, worktree::index::checkout, Progress};
+use gix::{odb::FindExt, worktree::checkout, Progress};
 
 use crate::{
     index,
@@ -55,7 +55,7 @@ pub fn checkout_exclusive(
         progress.info(format!("Skipping {} DIR/SYMLINK/COMMIT entries", num_skipped));
     }
 
-    let opts = gix::worktree::index::checkout::Options {
+    let opts = gix::worktree::checkout::Options {
         fs: gix::fs::Capabilities::probe(dest_directory),
 
         destination_is_initially_empty: true,
@@ -80,7 +80,7 @@ pub fn checkout_exclusive(
         files_updated,
         bytes_written,
     } = match repo {
-        Some(repo) => gix::worktree::index::checkout(
+        Some(repo) => gix::worktree::checkout(
             &mut index,
             dest_directory,
             {
@@ -103,7 +103,7 @@ pub fn checkout_exclusive(
             should_interrupt,
             opts,
         ),
-        None => gix::worktree::index::checkout(
+        None => gix::worktree::checkout(
             &mut index,
             dest_directory,
             |_, buf| {

gitoxide-core/src/repository/clone.rs

@@ -98,7 +98,7 @@ pub(crate) mod function {
             }
         };
 
-        if let Some(gix::worktree::index::checkout::Outcome { collisions, errors, .. }) = outcome {
+        if let Some(gix::worktree::checkout::Outcome { collisions, errors, .. }) = outcome {
             if !(collisions.is_empty() && errors.is_empty()) {
                 let mut messages = Vec::new();
                 if !errors.is_empty() {

gix-worktree/src/fs/cache/mod.rs renamed to gix-worktree/src/cache/mod.rs

@@ -5,7 +5,7 @@ use bstr::{BStr, ByteSlice};
 use gix_hash::oid;
 
 use super::Cache;
-use crate::fs::PathOidMapping;
+use crate::PathOidMapping;
 
 #[derive(Clone)]
 pub enum State {

gix-worktree/src/fs/cache/platform.rs renamed to gix-worktree/src/cache/platform.rs

@@ -3,7 +3,7 @@ use std::path::Path;
 use bstr::ByteSlice;
 use gix_hash::oid;
 
-use crate::fs::{
+use crate::{
     cache::{Platform, State},
     PathOidMapping,
 };

gix-worktree/src/fs/cache/state/attributes.rs renamed to gix-worktree/src/cache/state/attributes.rs

@@ -1,12 +1,12 @@
-use crate::fs::cache::state::{AttributeMatchGroup, Attributes};
+use crate::cache::state::{AttributeMatchGroup, Attributes};
 use gix_glob::pattern::Case;
 use std::path::PathBuf;
 
 /// Decide where to read `.gitattributes` files from.
 #[derive(Default, Debug, Clone, Copy)]
 pub enum Source {
     /// Retrieve attribute files from an attribute list, see
-    /// [State::attribute_list_from_index()][crate::fs::cache::State::attribute_list_from_index()].
+    /// [State::attribute_list_from_index()][crate::cache::State::attribute_list_from_index()].
     ///
     /// The attribute list is typically produced from an index. If a tree should be the source, build an attribute list
     /// from a tree instead.

gix-worktree/src/fs/cache/state/ignore.rs renamed to gix-worktree/src/cache/state/ignore.rs

@@ -1,5 +1,5 @@
-use crate::fs::cache::state::IgnoreMatchGroup;
-use crate::fs::PathOidMapping;
+use crate::cache::state::IgnoreMatchGroup;
+use crate::PathOidMapping;
 use bstr::{BStr, BString, ByteSlice};
 use gix_glob::pattern::Case;
 use gix_hash::oid;

gix-worktree/src/fs/cache/state/mod.rs renamed to gix-worktree/src/cache/state/mod.rs

@@ -2,7 +2,7 @@ use bstr::ByteSlice;
 use gix_glob::pattern::Case;
 use std::path::PathBuf;
 
-use crate::fs::{cache::State, PathOidMapping};
+use crate::{cache::State, PathOidMapping};
 
 type AttributeMatchGroup = gix_attributes::Search;
 type IgnoreMatchGroup = gix_ignore::Search;

gix-worktree/src/checkout/chunk.rs

@@ -0,0 +1,183 @@
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use bstr::BStr;
+use gix_features::progress::Progress;
+use gix_hash::oid;
+
+use crate::checkout::entry;
+use crate::{checkout, Cache};
+
+mod reduce {
+    use std::{
+        marker::PhantomData,
+        sync::atomic::{AtomicUsize, Ordering},
+    };
+
+    use gix_features::progress::Progress;
+
+    use crate::checkout;
+
+    pub struct Reduce<'a, 'entry, P1, P2, E> {
+        pub files: &'a mut P1,
+        pub bytes: &'a mut P2,
+        pub num_files: &'a AtomicUsize,
+        pub aggregate: super::Outcome<'entry>,
+        pub marker: PhantomData<E>,
+    }
+
+    impl<'a, 'entry, P1, P2, E> gix_features::parallel::Reduce for Reduce<'a, 'entry, P1, P2, E>
+    where
+        P1: Progress,
+        P2: Progress,
+        E: std::error::Error + Send + Sync + 'static,
+    {
+        type Input = Result<super::Outcome<'entry>, checkout::Error<E>>;
+        type FeedProduce = ();
+        type Output = super::Outcome<'entry>;
+        type Error = checkout::Error<E>;
+
+        fn feed(&mut self, item: Self::Input) -> Result<Self::FeedProduce, Self::Error> {
+            let item = item?;
+            let super::Outcome {
+                bytes_written,
+                delayed,
+                errors,
+                collisions,
+            } = item;
+            self.aggregate.bytes_written += bytes_written;
+            self.aggregate.delayed.extend(delayed);
+            self.aggregate.errors.extend(errors);
+            self.aggregate.collisions.extend(collisions);
+
+            self.bytes.set(self.aggregate.bytes_written as usize);
+            self.files.set(self.num_files.load(Ordering::Relaxed));
+
+            Ok(())
+        }
+
+        fn finalize(self) -> Result<Self::Output, Self::Error> {
+            Ok(self.aggregate)
+        }
+    }
+}
+pub use reduce::Reduce;
+
+#[derive(Default)]
+pub struct Outcome<'a> {
+    pub collisions: Vec<checkout::Collision>,
+    pub errors: Vec<checkout::ErrorRecord>,
+    pub delayed: Vec<(&'a mut gix_index::Entry, &'a BStr)>,
+    pub bytes_written: u64,
+}
+
+#[derive(Clone)]
+pub struct Context<'a, Find: Clone> {
+    pub find: Find,
+    pub path_cache: Cache,
+    pub buf: Vec<u8>,
+    pub options: checkout::Options,
+    /// We keep these shared so that there is the chance for printing numbers that aren't looking like
+    /// multiple of chunk sizes. Purely cosmetic. Otherwise it's the same as `files`.
+    pub num_files: &'a AtomicUsize,
+}
+
+pub fn process<'entry, Find, E>(
+    entries_with_paths: impl Iterator<Item = (&'entry mut gix_index::Entry, &'entry BStr)>,
+    files: &mut impl Progress,
+    bytes: &mut impl Progress,
+    ctx: &mut Context<'_, Find>,
+) -> Result<Outcome<'entry>, checkout::Error<E>>
+where
+    Find: for<'a> FnMut(&oid, &'a mut Vec<u8>) -> Result<gix_object::BlobRef<'a>, E> + Clone,
+    E: std::error::Error + Send + Sync + 'static,
+{
+    let mut delayed = Vec::new();
+    let mut collisions = Vec::new();
+    let mut errors = Vec::new();
+    let mut bytes_written = 0;
+
+    for (entry, entry_path) in entries_with_paths {
+        // TODO: write test for that
+        if entry.flags.contains(gix_index::entry::Flags::SKIP_WORKTREE) {
+            files.inc();
+            continue;
+        }
+
+        // Symlinks always have to be delayed on windows as they have to point to something that exists on creation.
+        // And even if not, there is a distinction between file and directory symlinks, hence we have to check what the target is
+        // before creating it.
+        // And to keep things sane, we just do the same on non-windows as well which is similar to what git does and adds some safety
+        // around writing through symlinks (even though we handle this).
+        // This also means that we prefer content in files over symlinks in case of collisions, which probably is for the better, too.
+        if entry.mode == gix_index::entry::Mode::SYMLINK {
+            delayed.push((entry, entry_path));
+            continue;
+        }
+
+        bytes_written +=
+            checkout_entry_handle_result(entry, entry_path, &mut errors, &mut collisions, files, bytes, ctx)? as u64;
+    }
+
+    Ok(Outcome {
+        bytes_written,
+        errors,
+        collisions,
+        delayed,
+    })
+}
+
+pub fn checkout_entry_handle_result<Find, E>(
+    entry: &mut gix_index::Entry,
+    entry_path: &BStr,
+    errors: &mut Vec<checkout::ErrorRecord>,
+    collisions: &mut Vec<checkout::Collision>,
+    files: &mut impl Progress,
+    bytes: &mut impl Progress,
+    Context {
+        find,
+        path_cache,
+        buf,
+        options,
+        num_files,
+    }: &mut Context<'_, Find>,
+) -> Result<usize, checkout::Error<E>>
+where
+    Find: for<'a> FnMut(&oid, &'a mut Vec<u8>) -> Result<gix_object::BlobRef<'a>, E> + Clone,
+    E: std::error::Error + Send + Sync + 'static,
+{
+    let res = entry::checkout(
+        entry,
+        entry_path,
+        entry::Context { find, path_cache, buf },
+        options.clone(),
+    );
+    files.inc();
+    num_files.fetch_add(1, Ordering::SeqCst);
+    match res {
+        Ok(object_size) => {
+            bytes.inc_by(object_size);
+            Ok(object_size)
+        }
+        Err(checkout::Error::Io(err)) if gix_fs::symlink::is_collision_error(&err) => {
+            // We are here because a file existed or was blocked by a directory which shouldn't be possible unless
+            // we are on a file insensitive file system.
+            files.fail(format!("{}: collided ({:?})", entry_path, err.kind()));
+            collisions.push(checkout::Collision {
+                path: entry_path.into(),
+                error_kind: err.kind(),
+            });
+            Ok(0)
+        }
+        Err(err) => {
+            if options.keep_going {
+                errors.push(checkout::ErrorRecord {
+                    path: entry_path.into(),
+                    error: Box::new(err),
+                });
+                Ok(0)
+            } else {
+                Err(err)
+            }
+        }
+    }
+}

gix-worktree/src/index/entry.rs renamed to gix-worktree/src/checkout/entry.rs

@@ -6,11 +6,11 @@ use gix_index::entry::Stat;
 use gix_index::Entry;
 use io_close::Close;
 
-use crate::{fs, index};
+use crate::Cache;
 
 pub struct Context<'a, Find> {
     pub find: &'a mut Find,
-    pub path_cache: &'a mut fs::Cache,
+    pub path_cache: &'a mut Cache,
     pub buf: &'a mut Vec<u8>,
 }
 
@@ -19,7 +19,7 @@ pub fn checkout<Find, E>(
     entry: &mut Entry,
     entry_path: &BStr,
     Context { find, path_cache, buf }: Context<'_, Find>,
-    index::checkout::Options {
+    crate::checkout::Options {
         fs: gix_fs::Capabilities {
             symlink,
             executable_bit,
@@ -28,21 +28,21 @@ pub fn checkout<Find, E>(
         destination_is_initially_empty,
         overwrite_existing,
         ..
-    }: index::checkout::Options,
-) -> Result<usize, index::checkout::Error<E>>
+    }: crate::checkout::Options,
+) -> Result<usize, crate::checkout::Error<E>>
 where
     Find: for<'a> FnMut(&oid, &'a mut Vec<u8>) -> Result<gix_object::BlobRef<'a>, E>,
     E: std::error::Error + Send + Sync + 'static,
 {
-    let dest_relative = gix_path::try_from_bstr(entry_path).map_err(|_| index::checkout::Error::IllformedUtf8 {
+    let dest_relative = gix_path::try_from_bstr(entry_path).map_err(|_| crate::checkout::Error::IllformedUtf8 {
         path: entry_path.to_owned(),
     })?;
     let is_dir = Some(entry.mode == gix_index::entry::Mode::COMMIT || entry.mode == gix_index::entry::Mode::DIR);
     let dest = path_cache.at_path(dest_relative, is_dir, &mut *find)?.path();
 
     let object_size = match entry.mode {
         gix_index::entry::Mode::FILE | gix_index::entry::Mode::FILE_EXECUTABLE => {
-            let obj = find(&entry.id, buf).map_err(|err| index::checkout::Error::Find {
+            let obj = find(&entry.id, buf).map_err(|err| crate::checkout::Error::Find {
                 err,
                 oid: entry.id,
                 path: dest.to_path_buf(),
@@ -77,13 +77,13 @@ where
             obj.data.len()
         }
         gix_index::entry::Mode::SYMLINK => {
-            let obj = find(&entry.id, buf).map_err(|err| index::checkout::Error::Find {
+            let obj = find(&entry.id, buf).map_err(|err| crate::checkout::Error::Find {
                 err,
                 oid: entry.id,
                 path: dest.to_path_buf(),
             })?;
             let symlink_destination = gix_path::try_from_byte_slice(obj.data)
-                .map_err(|_| index::checkout::Error::IllformedUtf8 { path: obj.data.into() })?;
+                .map_err(|_| crate::checkout::Error::IllformedUtf8 { path: obj.data.into() })?;
 
             if symlink {
                 try_write_or_unlink(dest, overwrite_existing, |p| {