feat: shallow support for clone operations.

TODO: more elaborate docs

Author: Byron

crate-status.md

@@ -637,10 +637,14 @@ See its [README.md](https://github.com/Byron/gitoxide/blob/main/gix-lock/README.
         * [x] find remote itself
             - [ ] respect `branch.<name>.merge` in the returned remote.
     * **remotes**  
-        * [ ] clone 
+        * [x] clone 
           * [ ] shallow
+            * [ ] include-tags when shallow is used (needs separate fetch)
+            * [ ] prune non-existing shallow commits
           * [ ] [bundles](https://git-scm.com/docs/git-bundle)
         * [x] fetch
+           * [ ] 'ref-in-want'
+           * [ ] standard negotiation algorithms (right now we only have a 'naive' one)
         * [ ] push
         * [x] ls-refs
         * [x] ls-refs with ref-spec filter

gix/src/clone/fetch/mod.rs

@@ -121,6 +121,7 @@ impl PrepareFetch {
             .with_reflog_message(RefLogMessage::Override {
                 message: reflog_message.clone(),
             })
+            .with_shallow(self.shallow.clone())
             .receive(should_interrupt)?;
 
         util::append_config_to_repo_config(repo, config);
@@ -184,6 +185,12 @@ impl PrepareFetch {
         self.remote_name = Some(crate::remote::name::validated(name)?);
         Ok(self)
     }
+
+    /// Make this clone a shallow one with the respective choice of shallow-ness.
+    pub fn with_shallow(mut self, shallow: crate::remote::fetch::Shallow) -> Self {
+        self.shallow = shallow;
+        self
+    }
 }
 
 /// Consumption

gix/src/clone/mod.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::result_large_err)]
 use std::convert::TryInto;
 
-use crate::{bstr::BString, config::tree::gitoxide};
+use crate::{bstr::BString, config::tree::gitoxide, remote};
 
 type ConfigureRemoteFn =
     Box<dyn FnMut(crate::Remote<'_>) -> Result<crate::Remote<'_>, Box<dyn std::error::Error + Send + Sync>>>;
@@ -22,6 +22,9 @@ pub struct PrepareFetch {
     /// The url to clone from
     #[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
     url: gix_url::Url,
+    /// How to handle shallow clones
+    #[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
+    shallow: remote::fetch::Shallow,
 }
 
 /// The error returned by [`PrepareFetch::new()`].
@@ -99,6 +102,7 @@ impl PrepareFetch {
             repo: Some(repo),
             remote_name: None,
             configure_remote: None,
+            shallow: remote::fetch::Shallow::NoChange,
         })
     }
 }

gix/src/remote/connection/fetch/error.rs

@@ -28,6 +28,17 @@ pub enum Error {
         path: std::path::PathBuf,
         source: std::io::Error,
     },
+    #[error(transparent)]
+    ShallowOpen(#[from] crate::shallow::open::Error),
+    #[error("Server lack feature {feature:?}: {description}")]
+    MissingServerFeature {
+        feature: &'static str,
+        description: &'static str,
+    },
+    #[error("Could not write 'shallow' file to incorporate remote updates after fetching")]
+    WriteShallowFile(#[from] crate::shallow::write::Error),
+    #[error("'shallow' file could not be locked in preparation for writing changes")]
+    LockShallowFile(#[from] gix_lock::acquire::Error),
 }
 
 impl gix_protocol::transport::IsSpuriousError for Error {

gix/src/remote/connection/fetch/mod.rs

@@ -149,6 +149,7 @@ where
             dry_run: DryRun::No,
             reflog_message: None,
             write_packed_refs: WritePackedRefs::Never,
+            shallow: Default::default(),
         })
     }
 }
@@ -179,6 +180,7 @@ where
     dry_run: DryRun,
     reflog_message: Option<RefLogMessage>,
     write_packed_refs: WritePackedRefs,
+    shallow: remote::fetch::Shallow,
 }
 
 /// Builder
@@ -212,6 +214,14 @@ where
         self.reflog_message = reflog_message.into();
         self
     }
+
+    /// Define what to do when the current repository is a shallow clone.
+    ///
+    /// *Has no effect if the current repository is not as shallow clone.*
+    pub fn with_shallow(mut self, shallow: remote::fetch::Shallow) -> Self {
+        self.shallow = shallow;
+        self
+    }
 }
 
 impl<'remote, 'repo, T, P> Drop for Prepare<'remote, 'repo, T, P>

gix/src/remote/connection/fetch/negotiate.rs

@@ -1,3 +1,5 @@
+use gix_refspec::RefSpec;
+
 /// The way the negotiation is performed
 #[derive(Copy, Clone)]
 pub(crate) enum Algorithm {
@@ -16,6 +18,7 @@ pub enum Error {
 /// Negotiate one round with `algo` by looking at `ref_map` and adjust `arguments` to contain the haves and wants.
 /// If this is not the first round, the `previous_response` is set with the last recorded server response.
 /// Returns `true` if the negotiation is done from our side so the server won't keep asking.
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn one_round(
     algo: Algorithm,
     round: usize,
@@ -24,10 +27,12 @@ pub(crate) fn one_round(
     fetch_tags: crate::remote::fetch::Tags,
     arguments: &mut gix_protocol::fetch::Arguments,
     _previous_response: Option<&gix_protocol::fetch::Response>,
+    wants_shallow_change: Option<&[RefSpec]>,
 ) -> Result<bool, Error> {
     let tag_refspec_to_ignore = fetch_tags
         .to_refspec()
         .filter(|_| matches!(fetch_tags, crate::remote::fetch::Tags::Included));
+    let non_wildcard_specs_only = wants_shallow_change;
     match algo {
         Algorithm::Naive => {
             assert_eq!(round, 1, "Naive always finishes after the first round, and claims.");
@@ -42,6 +47,14 @@ pub(crate) fn one_round(
                 }) {
                     continue;
                 }
+                if non_wildcard_specs_only
+                    .and_then(|refspecs| mapping.spec_index.get(refspecs, &ref_map.extra_refspecs))
+                    .map_or(false, |spec| {
+                        spec.to_ref().local().map_or(false, |ref_| ref_.contains(&b'*'))
+                    })
+                {
+                    continue;
+                }
                 let have_id = mapping.local.as_ref().and_then(|name| {
                     repo.find_reference(name)
                         .ok()
@@ -50,7 +63,7 @@ pub(crate) fn one_round(
                 match have_id {
                     Some(have_id) => {
                         if let Some(want_id) = mapping.remote.as_id() {
-                            if want_id != have_id {
+                            if want_id != have_id || wants_shallow_change.is_some() {
                                 arguments.want(want_id);
                                 arguments.have(have_id);
                             }

gix/src/remote/connection/fetch/receive_pack.rs

@@ -1,16 +1,17 @@
 use std::sync::atomic::AtomicBool;
 
 use gix_odb::FindExt;
+use gix_protocol::fetch::Arguments;
 use gix_protocol::transport::client::Transport;
 
 use crate::{
     remote,
     remote::{
         connection::fetch::config,
         fetch,
-        fetch::{negotiate, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Status},
+        fetch::{negotiate, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Shallow, Status},
     },
-    Progress,
+    Progress, Repository,
 };
 
 impl<'remote, 'repo, T, P> Prepare<'remote, 'repo, T, P>
@@ -82,10 +83,17 @@ where
         let mut arguments = gix_protocol::fetch::Arguments::new(protocol_version, fetch_features);
         if matches!(con.remote.fetch_tags, crate::remote::fetch::Tags::Included) {
             if !arguments.can_use_include_tag() {
-                unimplemented!("we expect servers to support 'include-tag', otherwise we have to implement another pass to fetch attached tags separately");
+                return Err(Error::MissingServerFeature {
+                    feature: "include-tag",
+                    description:
+                        // NOTE: if this is an issue, we could probably do what's proposed here.
+                        "To make this work we would have to implement another pass to fetch attached tags separately",
+                });
             }
             arguments.use_include_tag();
         }
+        let (shallow_commits, shallow_lock) = add_shallow_args(&mut arguments, &self.shallow, repo)?;
+
         let mut previous_response = None::<gix_protocol::fetch::Response>;
         let mut round = 1;
 
@@ -108,6 +116,7 @@ where
                 con.remote.fetch_tags,
                 &mut arguments,
                 previous_response.as_ref(),
+                (self.shallow != Shallow::NoChange).then(|| con.remote.refspecs(remote::Direction::Fetch)),
             ) {
                 Ok(_) if arguments.is_empty() => {
                     gix_protocol::indicate_end_of_interaction(&mut con.transport).await.ok();
@@ -146,6 +155,7 @@ where
                 if !sideband_all {
                     setup_remote_progress(progress, &mut reader);
                 }
+                previous_response = Some(response);
                 break 'negotiation reader;
             } else {
                 previous_response = Some(response);
@@ -187,6 +197,12 @@ where
             gix_protocol::indicate_end_of_interaction(&mut con.transport).await.ok();
         }
 
+        if let Some((response, shallow_lock)) = previous_response.zip(shallow_lock) {
+            if !response.shallow_updates().is_empty() {
+                crate::shallow::write(shallow_lock, shallow_commits, response.shallow_updates())?;
+            }
+        }
+
         let update_refs = refs::update(
             repo,
             self.reflog_message
@@ -221,6 +237,56 @@ where
     }
 }
 
+fn add_shallow_args(
+    args: &mut Arguments,
+    shallow: &Shallow,
+    repo: &Repository,
+) -> Result<(Option<crate::shallow::Commits>, Option<gix_lock::File>), Error> {
+    let expect_change = *shallow != Shallow::NoChange;
+    let shallow_lock = expect_change
+        .then(|| {
+            gix_lock::File::acquire_to_update_resource(repo.shallow_file(), gix_lock::acquire::Fail::Immediately, None)
+        })
+        .transpose()?;
+
+    let shallow_commits = repo.shallow_commits()?;
+    if (shallow_commits.is_some() || expect_change) && !args.can_use_shallow() {
+        // NOTE: if this is an issue, we can always unshallow the repo ourselves.
+        return Err(Error::MissingServerFeature {
+            feature: "shallow",
+            description: "shallow clones need server support to remain shallow, otherwise bigger than expected packs are sent effectively unshallowing the repository",
+        });
+    }
+    if let Some(shallow_commits) = &shallow_commits {
+        for commit in shallow_commits.iter() {
+            args.shallow(commit);
+        }
+    }
+    match shallow {
+        Shallow::NoChange => {}
+        Shallow::DepthAtRemote(commits) => args.deepen(commits.get() as usize),
+        Shallow::Deepen(commits) => {
+            args.deepen(*commits as usize);
+            args.deepen_relative();
+        }
+        Shallow::Since { cutoff } => {
+            args.deepen_since(cutoff.seconds_since_unix_epoch as usize);
+        }
+        Shallow::Exclude {
+            remote_refs,
+            since_cutoff,
+        } => {
+            if let Some(cutoff) = since_cutoff {
+                args.deepen_since(cutoff.seconds_since_unix_epoch as usize);
+            }
+            for ref_ in remote_refs {
+                args.deepen_not(ref_.as_ref().as_bstr());
+            }
+        }
+    }
+    Ok((shallow_commits, shallow_lock))
+}
+
 fn setup_remote_progress<P>(
     progress: &mut P,
     reader: &mut Box<dyn gix_protocol::transport::client::ExtendedBufRead + Unpin + '_>,

gix/src/remote/fetch.rs

@@ -52,6 +52,52 @@ impl Tags {
     }
 }
 
+/// Describe how shallow clones are handled when fetching, with variants defining how the *shallow boundary* is handled.
+///
+/// The *shallow boundary* is a set of commits whose parents are not present in the repository.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Shallow {
+    /// Fetch all changes from the remote without affecting the shallow boundary at all.
+    NoChange,
+    /// Receive update to `depth` commits in the history of the refs to fetch (from the viewpoint of the remote),
+    /// with the value of `1` meaning to receive only the commit a ref is pointing to.
+    ///
+    /// This may update the shallow boundary to increase or decrease the amount of available history.
+    DepthAtRemote(std::num::NonZeroU32),
+    /// Increase the number of commits and thus expand the shallow boundary by `depth` commits as seen from our local
+    /// shallow boundary, with a value of `0` having no effect.
+    Deepen(u32),
+    /// Set the shallow boundary at the `cutoff` time, meaning that there will be no commits beyond that time.
+    Since {
+        /// The date beyond which there will be no history.
+        cutoff: gix_date::Time,
+    },
+    /// Receive all history excluding all commits reachable from `remote_refs`. These can be long or short
+    /// ref names or tag names.
+    Exclude {
+        /// The ref names to exclude, short or long. Note that ambiguous short names will cause the remote to abort
+        /// without an error message being transferred (because the protocol does not support it)
+        remote_refs: Vec<gix_ref::PartialName>,
+        /// If some, this field has the same meaning as [`Shallow::Since`] which can be used in combination
+        /// with excluded references.
+        since_cutoff: Option<gix_date::Time>,
+    },
+}
+
+impl Default for Shallow {
+    fn default() -> Self {
+        Shallow::NoChange
+    }
+}
+
+impl Shallow {
+    /// Produce a variant that causes the repository to loose its shallow boundary, effectively by extending it
+    /// beyond all limits.
+    pub fn unshallow() -> Self {
+        Shallow::DepthAtRemote((i32::MAX as u32).try_into().expect("valid at compile time"))
+    }
+}
+
 /// Information about the relationship between our refspecs, and remote references with their local counterparts.
 #[derive(Default, Debug, Clone)]
 #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))]

gix/src/repository/shallow.rs

@@ -12,8 +12,12 @@ impl Repository {
             .map_or(false, |m| m.is_file() && m.len() > 0)
     }
 
-    /// Return a shared list of shallow commits which is updated automatically if the in-memory snapshot has become stale as the underlying file
-    /// on disk has changed.
+    /// Return a shared list of shallow commits which is updated automatically if the in-memory snapshot has become stale
+    /// as the underlying file on disk has changed.
+    ///
+    /// The list of shallow commits represents the shallow boundary, beyond which we are lacking all (parent) commits.
+    /// Note that the list is never empty, as `Ok(None)` is returned in that case indicating the repository
+    /// isn't a shallow clone.
     ///
     /// The shared list is shared across all clones of this repository.
     pub fn shallow_commits(&self) -> Result<Option<crate::shallow::Commits>, crate::shallow::open::Error> {