feat: shallow support for clone operations.

TODO: more elaborate docs

Author: Byron

crate-status.md

@@ -637,10 +637,14 @@ See its [README.md](https://github.com/Byron/gitoxide/blob/main/gix-lock/README.
         * [x] find remote itself
             - [ ] respect `branch.<name>.merge` in the returned remote.
     * **remotes**  
-        * [ ] clone 
+        * [x] clone 
           * [ ] shallow
+            * [ ] include-tags when shallow is used (needs separate fetch)
+            * [ ] prune non-existing shallow commits
           * [ ] [bundles](https://git-scm.com/docs/git-bundle)
         * [x] fetch
+           * [ ] 'ref-in-want'
+           * [ ] standard negotiation algorithms (right now we only have a 'naive' one)
         * [ ] push
         * [x] ls-refs
         * [x] ls-refs with ref-spec filter

gix/src/clone/fetch/mod.rs

@@ -121,6 +121,7 @@ impl PrepareFetch {
             .with_reflog_message(RefLogMessage::Override {
                 message: reflog_message.clone(),
             })
+            .with_shallow(self.shallow.clone())
             .receive(should_interrupt)?;
 
         util::append_config_to_repo_config(repo, config);
@@ -184,6 +185,12 @@ impl PrepareFetch {
         self.remote_name = Some(crate::remote::name::validated(name)?);
         Ok(self)
     }
+
+    /// Make this clone a shallow one with the respective choice of shallow-ness.
+    pub fn with_shallow(mut self, shallow: crate::remote::fetch::Shallow) -> Self {
+        self.shallow = shallow;
+        self
+    }
 }
 
 /// Consumption

gix/src/clone/mod.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::result_large_err)]
 use std::convert::TryInto;
 
-use crate::{bstr::BString, config::tree::gitoxide};
+use crate::{bstr::BString, config::tree::gitoxide, remote};
 
 type ConfigureRemoteFn =
     Box<dyn FnMut(crate::Remote<'_>) -> Result<crate::Remote<'_>, Box<dyn std::error::Error + Send + Sync>>>;
@@ -22,6 +22,9 @@ pub struct PrepareFetch {
     /// The url to clone from
     #[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
     url: gix_url::Url,
+    /// How to handle shallow clones
+    #[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
+    shallow: remote::fetch::Shallow,
 }
 
 /// The error returned by [`PrepareFetch::new()`].
@@ -99,6 +102,7 @@ impl PrepareFetch {
             repo: Some(repo),
             remote_name: None,
             configure_remote: None,
+            shallow: remote::fetch::Shallow::NoChange,
         })
     }
 }

gix/src/config/tree/sections/clone.rs

@@ -7,6 +7,8 @@ impl Clone {
     /// The `clone.defaultRemoteName` key.
     pub const DEFAULT_REMOTE_NAME: keys::RemoteName =
         keys::RemoteName::new_remote_name("defaultRemoteName", &config::Tree::CLONE);
+    /// The `clone.rejectShallow` key.
+    pub const REJECT_SHALLOW: keys::Boolean = keys::Boolean::new_boolean("rejectShallow", &config::Tree::CLONE);
 }
 
 impl Section for Clone {
@@ -15,6 +17,6 @@ impl Section for Clone {
     }
 
     fn keys(&self) -> &[&dyn Key] {
-        &[&Self::DEFAULT_REMOTE_NAME]
+        &[&Self::DEFAULT_REMOTE_NAME, &Self::REJECT_SHALLOW]
     }
 }

gix/src/remote/connection/fetch/error.rs

@@ -28,6 +28,21 @@ pub enum Error {
         path: std::path::PathBuf,
         source: std::io::Error,
     },
+    #[error(transparent)]
+    ShallowOpen(#[from] crate::shallow::open::Error),
+    #[error("Server lack feature {feature:?}: {description}")]
+    MissingServerFeature {
+        feature: &'static str,
+        description: &'static str,
+    },
+    #[error("Could not write 'shallow' file to incorporate remote updates after fetching")]
+    WriteShallowFile(#[from] crate::shallow::write::Error),
+    #[error("'shallow' file could not be locked in preparation for writing changes")]
+    LockShallowFile(#[from] gix_lock::acquire::Error),
+    #[error("Could not obtain configuration to learn if shallow remotes should be rejected")]
+    RejectShallowRemoteConfig(#[from] config::boolean::Error),
+    #[error("Receiving objects from shallow remotes is prohibited due to the value of `clone.rejectShallow`")]
+    RejectShallowRemote,
 }
 
 impl gix_protocol::transport::IsSpuriousError for Error {

gix/src/remote/connection/fetch/mod.rs

@@ -149,6 +149,7 @@ where
             dry_run: DryRun::No,
             reflog_message: None,
             write_packed_refs: WritePackedRefs::Never,
+            shallow: Default::default(),
         })
     }
 }
@@ -179,6 +180,7 @@ where
     dry_run: DryRun,
     reflog_message: Option<RefLogMessage>,
     write_packed_refs: WritePackedRefs,
+    shallow: remote::fetch::Shallow,
 }
 
 /// Builder
@@ -212,6 +214,14 @@ where
         self.reflog_message = reflog_message.into();
         self
     }
+
+    /// Define what to do when the current repository is a shallow clone.
+    ///
+    /// *Has no effect if the current repository is not as shallow clone.*
+    pub fn with_shallow(mut self, shallow: remote::fetch::Shallow) -> Self {
+        self.shallow = shallow;
+        self
+    }
 }
 
 impl<'remote, 'repo, T, P> Drop for Prepare<'remote, 'repo, T, P>

gix/src/remote/connection/fetch/negotiate.rs

@@ -1,3 +1,5 @@
+use gix_refspec::RefSpec;
+
 /// The way the negotiation is performed
 #[derive(Copy, Clone)]
 pub(crate) enum Algorithm {
@@ -16,6 +18,7 @@ pub enum Error {
 /// Negotiate one round with `algo` by looking at `ref_map` and adjust `arguments` to contain the haves and wants.
 /// If this is not the first round, the `previous_response` is set with the last recorded server response.
 /// Returns `true` if the negotiation is done from our side so the server won't keep asking.
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn one_round(
     algo: Algorithm,
     round: usize,
@@ -24,13 +27,15 @@ pub(crate) fn one_round(
     fetch_tags: crate::remote::fetch::Tags,
     arguments: &mut gix_protocol::fetch::Arguments,
     _previous_response: Option<&gix_protocol::fetch::Response>,
+    wants_shallow_change: Option<&[RefSpec]>,
 ) -> Result<bool, Error> {
     let tag_refspec_to_ignore = fetch_tags
         .to_refspec()
         .filter(|_| matches!(fetch_tags, crate::remote::fetch::Tags::Included));
+    let non_wildcard_specs_only = wants_shallow_change;
     match algo {
         Algorithm::Naive => {
-            assert_eq!(round, 1, "Naive always finishes after the first round, and claims.");
+            assert_eq!(round, 1, "Naive always finishes after the first round, it claims.");
             let mut has_missing_tracking_branch = false;
             for mapping in &ref_map.mappings {
                 if tag_refspec_to_ignore.map_or(false, |tag_spec| {
@@ -42,6 +47,14 @@ pub(crate) fn one_round(
                 }) {
                     continue;
                 }
+                if non_wildcard_specs_only
+                    .and_then(|refspecs| mapping.spec_index.get(refspecs, &ref_map.extra_refspecs))
+                    .map_or(false, |spec| {
+                        spec.to_ref().local().map_or(false, |ref_| ref_.contains(&b'*'))
+                    })
+                {
+                    continue;
+                }
                 let have_id = mapping.local.as_ref().and_then(|name| {
                     repo.find_reference(name)
                         .ok()
@@ -50,7 +63,7 @@ pub(crate) fn one_round(
                 match have_id {
                     Some(have_id) => {
                         if let Some(want_id) = mapping.remote.as_id() {
-                            if want_id != have_id {
+                            if want_id != have_id || wants_shallow_change.is_some() {
                                 arguments.want(want_id);
                                 arguments.have(have_id);
                             }
@@ -65,10 +78,11 @@ pub(crate) fn one_round(
                 }
             }
 
-            if has_missing_tracking_branch {
+            if has_missing_tracking_branch || (wants_shallow_change.is_some() && arguments.is_empty()) {
                 if let Ok(Some(r)) = repo.head_ref() {
                     if let Some(id) = r.target().try_id() {
                         arguments.have(id);
+                        arguments.want(id);
                     }
                 }
             }

gix/src/remote/connection/fetch/receive_pack.rs

@@ -1,16 +1,18 @@
 use std::sync::atomic::AtomicBool;
 
 use gix_odb::FindExt;
+use gix_protocol::fetch::Arguments;
 use gix_protocol::transport::client::Transport;
 
+use crate::config::tree::Clone;
 use crate::{
     remote,
     remote::{
         connection::fetch::config,
         fetch,
-        fetch::{negotiate, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Status},
+        fetch::{negotiate, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Shallow, Status},
     },
-    Progress,
+    Progress, Repository,
 };
 
 impl<'remote, 'repo, T, P> Prepare<'remote, 'repo, T, P>
@@ -82,10 +84,17 @@ where
         let mut arguments = gix_protocol::fetch::Arguments::new(protocol_version, fetch_features);
         if matches!(con.remote.fetch_tags, crate::remote::fetch::Tags::Included) {
             if !arguments.can_use_include_tag() {
-                unimplemented!("we expect servers to support 'include-tag', otherwise we have to implement another pass to fetch attached tags separately");
+                return Err(Error::MissingServerFeature {
+                    feature: "include-tag",
+                    description:
+                        // NOTE: if this is an issue, we could probably do what's proposed here.
+                        "To make this work we would have to implement another pass to fetch attached tags separately",
+                });
             }
             arguments.use_include_tag();
         }
+        let (shallow_commits, mut shallow_lock) = add_shallow_args(&mut arguments, &self.shallow, repo)?;
+
         let mut previous_response = None::<gix_protocol::fetch::Response>;
         let mut round = 1;
 
@@ -108,6 +117,7 @@ where
                 con.remote.fetch_tags,
                 &mut arguments,
                 previous_response.as_ref(),
+                (self.shallow != Shallow::NoChange).then(|| con.remote.refspecs(remote::Direction::Fetch)),
             ) {
                 Ok(_) if arguments.is_empty() => {
                     gix_protocol::indicate_end_of_interaction(&mut con.transport).await.ok();
@@ -146,11 +156,26 @@ where
                 if !sideband_all {
                     setup_remote_progress(progress, &mut reader);
                 }
+                previous_response = Some(response);
                 break 'negotiation reader;
             } else {
                 previous_response = Some(response);
             }
         };
+        let previous_response = previous_response.expect("knowledge of a pack means a response was received");
+        if !previous_response.shallow_updates().is_empty() && shallow_lock.is_none() {
+            let reject_shallow_remote = repo
+                .config
+                .resolved
+                .boolean_filter_by_key("clone.rejectShallow", &mut repo.filter_config_section())
+                .map(|val| Clone::REJECT_SHALLOW.enrich_error(val))
+                .transpose()?
+                .unwrap_or(false);
+            if reject_shallow_remote {
+                return Err(Error::RejectShallowRemote);
+            }
+            shallow_lock = acquire_shallow_lock(repo).map(Some)?;
+        }
 
         let options = gix_pack::bundle::write::Options {
             thread_limit: config::index_threads(repo)?,
@@ -187,6 +212,12 @@ where
             gix_protocol::indicate_end_of_interaction(&mut con.transport).await.ok();
         }
 
+        if let Some(shallow_lock) = shallow_lock {
+            if !previous_response.shallow_updates().is_empty() {
+                crate::shallow::write(shallow_lock, shallow_commits, previous_response.shallow_updates())?;
+            }
+        }
+
         let update_refs = refs::update(
             repo,
             self.reflog_message
@@ -221,6 +252,57 @@ where
     }
 }
 
+fn acquire_shallow_lock(repo: &Repository) -> Result<gix_lock::File, Error> {
+    gix_lock::File::acquire_to_update_resource(repo.shallow_file(), gix_lock::acquire::Fail::Immediately, None)
+        .map_err(Into::into)
+}
+
+fn add_shallow_args(
+    args: &mut Arguments,
+    shallow: &Shallow,
+    repo: &Repository,
+) -> Result<(Option<crate::shallow::Commits>, Option<gix_lock::File>), Error> {
+    let expect_change = *shallow != Shallow::NoChange;
+    let shallow_lock = expect_change.then(|| acquire_shallow_lock(repo)).transpose()?;
+
+    let shallow_commits = repo.shallow_commits()?;
+    if (shallow_commits.is_some() || expect_change) && !args.can_use_shallow() {
+        // NOTE: if this is an issue, we can always unshallow the repo ourselves.
+        return Err(Error::MissingServerFeature {
+            feature: "shallow",
+            description: "shallow clones need server support to remain shallow, otherwise bigger than expected packs are sent effectively unshallowing the repository",
+        });
+    }
+    if let Some(shallow_commits) = &shallow_commits {
+        for commit in shallow_commits.iter() {
+            args.shallow(commit);
+        }
+    }
+    match shallow {
+        Shallow::NoChange => {}
+        Shallow::DepthAtRemote(commits) => args.deepen(commits.get() as usize),
+        Shallow::Deepen(commits) => {
+            args.deepen(*commits as usize);
+            args.deepen_relative();
+        }
+        Shallow::Since { cutoff } => {
+            args.deepen_since(cutoff.seconds_since_unix_epoch as usize);
+        }
+        Shallow::Exclude {
+            remote_refs,
+            since_cutoff,
+        } => {
+            if let Some(cutoff) = since_cutoff {
+                args.deepen_since(cutoff.seconds_since_unix_epoch as usize);
+            }
+            for ref_ in remote_refs {
+                args.deepen_not(ref_.as_ref().as_bstr());
+            }
+        }
+    }
+    Ok((shallow_commits, shallow_lock))
+}
+
 fn setup_remote_progress<P>(
     progress: &mut P,
     reader: &mut Box<dyn gix_protocol::transport::client::ExtendedBufRead + Unpin + '_>,