Skip to content

Commit 4e89c19

Browse files
committed
feat: shallow support for clone operations.
TODO: more elaborate docs
1 parent 397ed90 commit 4e89c19

File tree

17 files changed

+479
-21
lines changed

17 files changed

+479
-21
lines changed

crate-status.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -637,10 +637,14 @@ See its [README.md](https://github.com/Byron/gitoxide/blob/main/gix-lock/README.
637637
* [x] find remote itself
638638
- [ ] respect `branch.<name>.merge` in the returned remote.
639639
* **remotes**
640-
* [ ] clone
640+
* [x] clone
641641
* [ ] shallow
642+
* [ ] include-tags when shallow is used (needs separate fetch)
643+
* [ ] prune non-existing shallow commits
642644
* [ ] [bundles](https://git-scm.com/docs/git-bundle)
643645
* [x] fetch
646+
* [ ] 'ref-in-want'
647+
* [ ] standard negotiation algorithms (right now we only have a 'naive' one)
644648
* [ ] push
645649
* [x] ls-refs
646650
* [x] ls-refs with ref-spec filter

gix/src/clone/fetch/mod.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ impl PrepareFetch {
121121
.with_reflog_message(RefLogMessage::Override {
122122
message: reflog_message.clone(),
123123
})
124+
.with_shallow(self.shallow.clone())
124125
.receive(should_interrupt)?;
125126

126127
util::append_config_to_repo_config(repo, config);
@@ -184,6 +185,12 @@ impl PrepareFetch {
184185
self.remote_name = Some(crate::remote::name::validated(name)?);
185186
Ok(self)
186187
}
188+
189+
/// Make this clone a shallow one with the respective choice of shallow-ness.
190+
pub fn with_shallow(mut self, shallow: crate::remote::fetch::Shallow) -> Self {
191+
self.shallow = shallow;
192+
self
193+
}
187194
}
188195

189196
/// Consumption

gix/src/clone/mod.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#![allow(clippy::result_large_err)]
22
use std::convert::TryInto;
33

4-
use crate::{bstr::BString, config::tree::gitoxide};
4+
use crate::{bstr::BString, config::tree::gitoxide, remote};
55

66
type ConfigureRemoteFn =
77
Box<dyn FnMut(crate::Remote<'_>) -> Result<crate::Remote<'_>, Box<dyn std::error::Error + Send + Sync>>>;
@@ -22,6 +22,9 @@ pub struct PrepareFetch {
2222
/// The url to clone from
2323
#[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
2424
url: gix_url::Url,
25+
/// How to handle shallow clones
26+
#[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
27+
shallow: remote::fetch::Shallow,
2528
}
2629

2730
/// The error returned by [`PrepareFetch::new()`].
@@ -99,6 +102,7 @@ impl PrepareFetch {
99102
repo: Some(repo),
100103
remote_name: None,
101104
configure_remote: None,
105+
shallow: remote::fetch::Shallow::NoChange,
102106
})
103107
}
104108
}

gix/src/config/tree/sections/clone.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ impl Clone {
77
/// The `clone.defaultRemoteName` key.
88
pub const DEFAULT_REMOTE_NAME: keys::RemoteName =
99
keys::RemoteName::new_remote_name("defaultRemoteName", &config::Tree::CLONE);
10+
/// The `clone.rejectShallow` key.
11+
pub const REJECT_SHALLOW: keys::Boolean = keys::Boolean::new_boolean("rejectShallow", &config::Tree::CLONE);
1012
}
1113

1214
impl Section for Clone {
@@ -15,6 +17,6 @@ impl Section for Clone {
1517
}
1618

1719
fn keys(&self) -> &[&dyn Key] {
18-
&[&Self::DEFAULT_REMOTE_NAME]
20+
&[&Self::DEFAULT_REMOTE_NAME, &Self::REJECT_SHALLOW]
1921
}
2022
}

gix/src/remote/connection/fetch/error.rs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,21 @@ pub enum Error {
2828
path: std::path::PathBuf,
2929
source: std::io::Error,
3030
},
31+
#[error(transparent)]
32+
ShallowOpen(#[from] crate::shallow::open::Error),
33+
#[error("Server lack feature {feature:?}: {description}")]
34+
MissingServerFeature {
35+
feature: &'static str,
36+
description: &'static str,
37+
},
38+
#[error("Could not write 'shallow' file to incorporate remote updates after fetching")]
39+
WriteShallowFile(#[from] crate::shallow::write::Error),
40+
#[error("'shallow' file could not be locked in preparation for writing changes")]
41+
LockShallowFile(#[from] gix_lock::acquire::Error),
42+
#[error("Could not obtain configuration to learn if shallow remotes should be rejected")]
43+
RejectShallowRemoteConfig(#[from] config::boolean::Error),
44+
#[error("Receiving objects from shallow remotes is prohibited due to the value of `clone.rejectShallow`")]
45+
RejectShallowRemote,
3146
}
3247

3348
impl gix_protocol::transport::IsSpuriousError for Error {

gix/src/remote/connection/fetch/mod.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ where
149149
dry_run: DryRun::No,
150150
reflog_message: None,
151151
write_packed_refs: WritePackedRefs::Never,
152+
shallow: Default::default(),
152153
})
153154
}
154155
}
@@ -179,6 +180,7 @@ where
179180
dry_run: DryRun,
180181
reflog_message: Option<RefLogMessage>,
181182
write_packed_refs: WritePackedRefs,
183+
shallow: remote::fetch::Shallow,
182184
}
183185

184186
/// Builder
@@ -212,6 +214,14 @@ where
212214
self.reflog_message = reflog_message.into();
213215
self
214216
}
217+
218+
/// Define what to do when the current repository is a shallow clone.
219+
///
220+
/// *Has no effect if the current repository is not as shallow clone.*
221+
pub fn with_shallow(mut self, shallow: remote::fetch::Shallow) -> Self {
222+
self.shallow = shallow;
223+
self
224+
}
215225
}
216226

217227
impl<'remote, 'repo, T, P> Drop for Prepare<'remote, 'repo, T, P>

gix/src/remote/connection/fetch/negotiate.rs

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use gix_refspec::RefSpec;
2+
13
/// The way the negotiation is performed
24
#[derive(Copy, Clone)]
35
pub(crate) enum Algorithm {
@@ -16,6 +18,7 @@ pub enum Error {
1618
/// Negotiate one round with `algo` by looking at `ref_map` and adjust `arguments` to contain the haves and wants.
1719
/// If this is not the first round, the `previous_response` is set with the last recorded server response.
1820
/// Returns `true` if the negotiation is done from our side so the server won't keep asking.
21+
#[allow(clippy::too_many_arguments)]
1922
pub(crate) fn one_round(
2023
algo: Algorithm,
2124
round: usize,
@@ -24,13 +27,15 @@ pub(crate) fn one_round(
2427
fetch_tags: crate::remote::fetch::Tags,
2528
arguments: &mut gix_protocol::fetch::Arguments,
2629
_previous_response: Option<&gix_protocol::fetch::Response>,
30+
wants_shallow_change: Option<&[RefSpec]>,
2731
) -> Result<bool, Error> {
2832
let tag_refspec_to_ignore = fetch_tags
2933
.to_refspec()
3034
.filter(|_| matches!(fetch_tags, crate::remote::fetch::Tags::Included));
35+
let non_wildcard_specs_only = wants_shallow_change;
3136
match algo {
3237
Algorithm::Naive => {
33-
assert_eq!(round, 1, "Naive always finishes after the first round, and claims.");
38+
assert_eq!(round, 1, "Naive always finishes after the first round, it claims.");
3439
let mut has_missing_tracking_branch = false;
3540
for mapping in &ref_map.mappings {
3641
if tag_refspec_to_ignore.map_or(false, |tag_spec| {
@@ -42,6 +47,14 @@ pub(crate) fn one_round(
4247
}) {
4348
continue;
4449
}
50+
if non_wildcard_specs_only
51+
.and_then(|refspecs| mapping.spec_index.get(refspecs, &ref_map.extra_refspecs))
52+
.map_or(false, |spec| {
53+
spec.to_ref().local().map_or(false, |ref_| ref_.contains(&b'*'))
54+
})
55+
{
56+
continue;
57+
}
4558
let have_id = mapping.local.as_ref().and_then(|name| {
4659
repo.find_reference(name)
4760
.ok()
@@ -50,7 +63,7 @@ pub(crate) fn one_round(
5063
match have_id {
5164
Some(have_id) => {
5265
if let Some(want_id) = mapping.remote.as_id() {
53-
if want_id != have_id {
66+
if want_id != have_id || wants_shallow_change.is_some() {
5467
arguments.want(want_id);
5568
arguments.have(have_id);
5669
}
@@ -65,10 +78,11 @@ pub(crate) fn one_round(
6578
}
6679
}
6780

68-
if has_missing_tracking_branch {
81+
if has_missing_tracking_branch || (wants_shallow_change.is_some() && arguments.is_empty()) {
6982
if let Ok(Some(r)) = repo.head_ref() {
7083
if let Some(id) = r.target().try_id() {
7184
arguments.have(id);
85+
arguments.want(id);
7286
}
7387
}
7488
}

gix/src/remote/connection/fetch/receive_pack.rs

Lines changed: 85 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
use std::sync::atomic::AtomicBool;
22

33
use gix_odb::FindExt;
4+
use gix_protocol::fetch::Arguments;
45
use gix_protocol::transport::client::Transport;
56

7+
use crate::config::tree::Clone;
68
use crate::{
79
remote,
810
remote::{
911
connection::fetch::config,
1012
fetch,
11-
fetch::{negotiate, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Status},
13+
fetch::{negotiate, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Shallow, Status},
1214
},
13-
Progress,
15+
Progress, Repository,
1416
};
1517

1618
impl<'remote, 'repo, T, P> Prepare<'remote, 'repo, T, P>
@@ -82,10 +84,17 @@ where
8284
let mut arguments = gix_protocol::fetch::Arguments::new(protocol_version, fetch_features);
8385
if matches!(con.remote.fetch_tags, crate::remote::fetch::Tags::Included) {
8486
if !arguments.can_use_include_tag() {
85-
unimplemented!("we expect servers to support 'include-tag', otherwise we have to implement another pass to fetch attached tags separately");
87+
return Err(Error::MissingServerFeature {
88+
feature: "include-tag",
89+
description:
90+
// NOTE: if this is an issue, we could probably do what's proposed here.
91+
"To make this work we would have to implement another pass to fetch attached tags separately",
92+
});
8693
}
8794
arguments.use_include_tag();
8895
}
96+
let (shallow_commits, mut shallow_lock) = add_shallow_args(&mut arguments, &self.shallow, repo)?;
97+
8998
let mut previous_response = None::<gix_protocol::fetch::Response>;
9099
let mut round = 1;
91100

@@ -108,6 +117,7 @@ where
108117
con.remote.fetch_tags,
109118
&mut arguments,
110119
previous_response.as_ref(),
120+
(self.shallow != Shallow::NoChange).then(|| con.remote.refspecs(remote::Direction::Fetch)),
111121
) {
112122
Ok(_) if arguments.is_empty() => {
113123
gix_protocol::indicate_end_of_interaction(&mut con.transport).await.ok();
@@ -146,11 +156,26 @@ where
146156
if !sideband_all {
147157
setup_remote_progress(progress, &mut reader);
148158
}
159+
previous_response = Some(response);
149160
break 'negotiation reader;
150161
} else {
151162
previous_response = Some(response);
152163
}
153164
};
165+
let previous_response = previous_response.expect("knowledge of a pack means a response was received");
166+
if !previous_response.shallow_updates().is_empty() && shallow_lock.is_none() {
167+
let reject_shallow_remote = repo
168+
.config
169+
.resolved
170+
.boolean_filter_by_key("clone.rejectShallow", &mut repo.filter_config_section())
171+
.map(|val| Clone::REJECT_SHALLOW.enrich_error(val))
172+
.transpose()?
173+
.unwrap_or(false);
174+
if reject_shallow_remote {
175+
return Err(Error::RejectShallowRemote);
176+
}
177+
shallow_lock = acquire_shallow_lock(repo).map(Some)?;
178+
}
154179

155180
let options = gix_pack::bundle::write::Options {
156181
thread_limit: config::index_threads(repo)?,
@@ -187,6 +212,12 @@ where
187212
gix_protocol::indicate_end_of_interaction(&mut con.transport).await.ok();
188213
}
189214

215+
if let Some(shallow_lock) = shallow_lock {
216+
if !previous_response.shallow_updates().is_empty() {
217+
crate::shallow::write(shallow_lock, shallow_commits, previous_response.shallow_updates())?;
218+
}
219+
}
220+
190221
let update_refs = refs::update(
191222
repo,
192223
self.reflog_message
@@ -221,6 +252,57 @@ where
221252
}
222253
}
223254

255+
fn acquire_shallow_lock(repo: &Repository) -> Result<gix_lock::File, Error> {
256+
gix_lock::File::acquire_to_update_resource(repo.shallow_file(), gix_lock::acquire::Fail::Immediately, None)
257+
.map_err(Into::into)
258+
}
259+
260+
fn add_shallow_args(
261+
args: &mut Arguments,
262+
shallow: &Shallow,
263+
repo: &Repository,
264+
) -> Result<(Option<crate::shallow::Commits>, Option<gix_lock::File>), Error> {
265+
let expect_change = *shallow != Shallow::NoChange;
266+
let shallow_lock = expect_change.then(|| acquire_shallow_lock(repo)).transpose()?;
267+
268+
let shallow_commits = repo.shallow_commits()?;
269+
if (shallow_commits.is_some() || expect_change) && !args.can_use_shallow() {
270+
// NOTE: if this is an issue, we can always unshallow the repo ourselves.
271+
return Err(Error::MissingServerFeature {
272+
feature: "shallow",
273+
description: "shallow clones need server support to remain shallow, otherwise bigger than expected packs are sent effectively unshallowing the repository",
274+
});
275+
}
276+
if let Some(shallow_commits) = &shallow_commits {
277+
for commit in shallow_commits.iter() {
278+
args.shallow(commit);
279+
}
280+
}
281+
match shallow {
282+
Shallow::NoChange => {}
283+
Shallow::DepthAtRemote(commits) => args.deepen(commits.get() as usize),
284+
Shallow::Deepen(commits) => {
285+
args.deepen(*commits as usize);
286+
args.deepen_relative();
287+
}
288+
Shallow::Since { cutoff } => {
289+
args.deepen_since(cutoff.seconds_since_unix_epoch as usize);
290+
}
291+
Shallow::Exclude {
292+
remote_refs,
293+
since_cutoff,
294+
} => {
295+
if let Some(cutoff) = since_cutoff {
296+
args.deepen_since(cutoff.seconds_since_unix_epoch as usize);
297+
}
298+
for ref_ in remote_refs {
299+
args.deepen_not(ref_.as_ref().as_bstr());
300+
}
301+
}
302+
}
303+
Ok((shallow_commits, shallow_lock))
304+
}
305+
224306
fn setup_remote_progress<P>(
225307
progress: &mut P,
226308
reader: &mut Box<dyn gix_protocol::transport::client::ExtendedBufRead + Unpin + '_>,

0 commit comments

Comments
 (0)