feat: patchable init mirroring

Cargo.toml

@@ -4,7 +4,7 @@ resolver = "2"
 
 [workspace.dependencies]
 clap = { version = "4.5.27", features = ["derive"] }
-git2 = "0.20.0"
+git2 = "0.20.1"
 serde = { version = "1.0.217", features = ["derive"] }
 snafu = "0.8.5"
 tempfile = "3.16.0"

druid/stackable/patches/30.0.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/druid.git"
+upstream = "https://github.com/stackabletech/druid.git"
 base = "09d36ee324747f1407705c27618b6d415c3fa8a9"

druid/stackable/patches/30.0.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/druid.git"
+upstream = "https://github.com/stackabletech/druid.git"
 base = "a30af7a91d528e5c3a90356a5592abc7119191c6"

druid/stackable/patches/31.0.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/druid.git"
+upstream = "https://github.com/stackabletech/druid.git"
 base = "520482cb9638e452b0553595b4f29bb397a63758"

hadoop/stackable/patches/3.3.4/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hadoop.git"
+upstream = "https://github.com/stackabletech/hadoop.git"
 base = "a585a73c3e02ac62350c136643a5e7f6095a3dbb"

hadoop/stackable/patches/3.3.6/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hadoop.git"
+upstream = "https://github.com/stackabletech/hadoop.git"
 base = "1be78238728da9266a4f88195058f08fd012bf9c"

hadoop/stackable/patches/3.4.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hadoop.git"
+upstream = "https://github.com/stackabletech/hadoop.git"
 base = "bd8b77f398f626bb7791783192ee7a5dfaeec760"

hadoop/stackable/patches/3.4.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hadoop.git"
+upstream = "https://github.com/stackabletech/hadoop.git"
 base = "4d7825309348956336b8f06a08322b78422849b1"

hbase/stackable/patches/2.4.18/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hbase.git"
+upstream = "https://github.com/stackabletech/hbase.git"
 base = "a1767f4d76859c0068720a6c1e5cb78282ebfe1e"

hbase/stackable/patches/2.6.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hbase.git"
+upstream = "https://github.com/stackabletech/hbase.git"
 base = "de99f8754135ea69adc39da48d2bc2b2710a5366"

hbase/stackable/patches/2.6.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hbase.git"
+upstream = "https://github.com/stackabletech/hbase.git"
 base = "7ed50b4dd742269a78875fb32112215f831284ff"

hive/stackable/patches/3.1.3/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hive.git"
+upstream = "https://github.com/stackabletech/hive.git"
 base = "4df4d75bf1e16fe0af75aad0b4179c34c07fc975"

hive/stackable/patches/4.0.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hive.git"
+upstream = "https://github.com/stackabletech/hive.git"
 base = "183f8cb41d3dbed961ffd27999876468ff06690c"

hive/stackable/patches/4.0.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hive.git"
+upstream = "https://github.com/stackabletech/hive.git"
 base = "3af4517eb8cfd9407ad34ed78a0b48b57dfaa264"

kafka/stackable/patches/3.7.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/kafka.git"
+upstream = "https://github.com/stackabletech/kafka.git"
 base = "e2494e6ffb89f8288ed2aeb9b5596c755210bffd"

kafka/stackable/patches/3.7.2/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/kafka.git"
+upstream = "https://github.com/stackabletech/kafka.git"
 base = "79a8f2b5f44f9d5a6867190d1dfc463d08d60b82"

kafka/stackable/patches/3.8.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/kafka.git"
+upstream = "https://github.com/stackabletech/kafka.git"
 base = "771b9576b00ecf5b64ab6e8bedf04156fbdb5cd6"

kafka/stackable/patches/3.9.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/kafka.git"
+upstream = "https://github.com/stackabletech/kafka.git"
 base = "84caaa6e9da06435411510a81fa321d4f99c351f"

nifi/stackable/patches/1.27.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/nifi.git"
+upstream = "https://github.com/stackabletech/nifi.git"
 base = "e0c4461d90bd4f6e5f2b81765bcff5cd97ed3e18"

nifi/stackable/patches/1.28.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/nifi.git"
+upstream = "https://github.com/stackabletech/nifi.git"
 base = "883338fe28883733417d10f6ffa9319e75f5ea06"

nifi/stackable/patches/2.2.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/nifi.git"
+upstream = "https://github.com/stackabletech/nifi.git"
 base = "b33ffac8aa10992482f7fa54e6cfccc46a5e8e27"

omid/stackable/patches/1.1.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/phoenix-omid.git"
+upstream = "https://github.com/stackabletech/phoenix-omid.git"
 base = "3b9e16b7537adbc90a7403507fb8aabd8d1fab0c"

omid/stackable/patches/1.1.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/phoenix-omid.git"
+upstream = "https://github.com/stackabletech/phoenix-omid.git"
 base = "cd546d58d93f380fec9bf65dbfa618f53493f662"

omid/stackable/patches/1.1.2/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/phoenix-omid.git"
+upstream = "https://github.com/stackabletech/phoenix-omid.git"
 base = "88812c9e127063f3b3016262f81ea3e8b48ec157"

rust/patchable/src/main.rs

@@ -9,8 +9,8 @@ use std::{fs::File, io::Write, path::PathBuf};
 
 use git2::{Oid, Repository};
 use serde::{Deserialize, Serialize};
-use snafu::{OptionExt, ResultExt as _, Snafu};
-use tracing_indicatif::IndicatifLayer;
+use snafu::{ensure, OptionExt, ResultExt as _, Snafu};
+use tracing_indicatif::{span_ext::IndicatifSpanExt, IndicatifLayer};
 use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _};
 
 #[derive(clap::Parser)]
@@ -149,6 +149,11 @@ enum Cmd {
         /// Refs (such as tags and branches) will be resolved to commit IDs.
         #[clap(long)]
         base: String,
+
+        /// Assume a fork exists at stackabletech/<repo_name> and push the base ref to it.
+        /// The fork URL will be stored in patchable.toml instead of the original upstream.
+        #[clap(long)]
+        forked: bool,
     },
 
     /// Shows the patch directory for a given product version
@@ -197,6 +202,23 @@ pub enum Error {
         path: PathBuf,
     },
 
+    #[snafu(display("failed to parse upstream URL {url:?} to extract repository name"))]
+    ParseUpstreamUrl { url: String },
+    #[snafu(display("failed to add temporary fork remote for {url:?}"))]
+    AddForkRemote { source: git2::Error, url: String },
+    #[snafu(display("failed to push commit {commit} (as {refspec}) to fork {url:?}"))]
+    PushToFork {
+        source: git2::Error,
+        url: String,
+        refspec: String,
+        commit: Oid,
+    },
+    #[snafu(display("failed to delete remote {name}"))]
+    DeleteRemote {
+        source: git2::Error,
+        name: String,
+    },
+
     #[snafu(display("failed to find images repository"))]
     FindImagesRepo { source: repo::Error },
     #[snafu(display("images repository has no work directory"))]
@@ -397,7 +419,12 @@ fn main() -> Result<()> {
             );
         }
 
-        Cmd::Init { pv, upstream, base } => {
+        Cmd::Init {
+            pv,
+            upstream,
+            base,
+            forked,
+        } => {
             let ctx = ProductVersionContext {
                 pv,
                 images_repo_root,
@@ -414,8 +441,93 @@ fn main() -> Result<()> {
             // --base can be a reference, but patchable.toml should always have a resolved commit id,
             // so that it cannot be changed under our feet (without us knowing so, anyway...).
             tracing::info!(?base, "resolving base commit-ish");
-            let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream)
+
+            let (base_commit, upstream) = if forked {
+                // Parse e.g. "https://github.com/apache/druid.git" into "druid"
+                let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).context(ParseUpstreamUrlSnafu { url: &upstream })?;
+
+                ensure!(!repo_name.is_empty(), ParseUpstreamUrlSnafu { url: &upstream });
+
+                let fork_url = format!("https://github.com/stackabletech/{}.git", repo_name);
+                tracing::info!(%fork_url, "using fork repository");
+
+                // Fetch from original upstream using a temporary remote name
+                tracing::info!(upstream = upstream, %base, "fetching base ref from original upstream");
+                let base_commit_oid = repo::resolve_and_fetch_commitish(
+                    &product_repo,
+                    &base,
+                    &upstream
+                )
                 .context(FetchBaseCommitSnafu)?;
+
+                tracing::info!(commit = %base_commit_oid, "fetched base commit OID");
+
+                // Add fork remote
+                let temp_fork_remote = "patchable_fork_push";
+                let mut fork_remote = product_repo
+                    .remote(temp_fork_remote, &fork_url)
+                    .context(AddForkRemoteSnafu { url: fork_url.clone() })?;
+
+                // Push the base commit to the fork
+                tracing::info!(commit = %base_commit_oid, base = base, url = fork_url, "pushing commit to fork");
+                let mut callbacks = git2::RemoteCallbacks::new();
+                callbacks.credentials(|_url, username_from_url, _allowed_types| {
+                    git2::Cred::credential_helper(
+                        &git2::Config::open_default().unwrap(), // Use default git config
+                        _url,
+                        username_from_url,
+                    )
+                });
+
+                // Add progress tracking for push operation
+                let span_push = tracing::info_span!("pushing");
+                span_push.pb_set_style(&utils::progress_bar_style());
+                let _ = span_push.enter();
+                let mut quant_push = utils::Quantizer::percent();
+                callbacks.push_transfer_progress(move |current, total, _| {
+                    if total > 0 {
+                        quant_push.update_span_progress(current, total, &span_push);
+                    }
+                });
+
+                let mut push_options = git2::PushOptions::new();
+                push_options.remote_callbacks(callbacks);
+
+                // Check if the reference is a tag or branch by inspecting the git repository
+                let refspec = {
+                    let tag_ref = format!("refs/tags/{}", base);
+                    let is_tag = product_repo
+                        .find_reference(&tag_ref)
+                        .is_ok();
+
+                    if is_tag {
+                        format!("{}:refs/tags/{}", base_commit_oid, base)
+                    } else {
+                        // Assume it's a branch as default behavior
+                        format!("{}:refs/heads/{}", base_commit_oid, base)
+                    }
+                };
+
+                tracing::info!(refspec = refspec, "constructed push refspec");
+
+                fork_remote
+                    .push(&[&refspec], Some(&mut push_options))
+                    .context(PushToForkSnafu {
+                        url: fork_url.clone(),
+                        refspec: &refspec,
+                        commit: base_commit_oid,
+                    })?;
+
+                product_repo.remote_delete(temp_fork_remote)
+                    .context(DeleteRemoteSnafu { name: temp_fork_remote.to_string() })?;
+
+                tracing::info!("successfully pushed base ref to fork");
+
+                (base_commit_oid, fork_url)
+            } else {
+                (repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?, upstream)
+            };
+
             tracing::info!(?base, base.commit = ?base_commit, "resolved base commit");
 
             tracing::info!("saving configuration");

rust/patchable/src/repo.rs

@@ -180,7 +180,9 @@ pub fn resolve_and_fetch_commitish(
                     &[commitish],
                     Some(
                         FetchOptions::new()
-                            .update_fetchhead(true)
+                            // Tags need to be present to later determine whether `commitish` is a tag or not
+                            // Patchable needs to know this when initializing a repository with the `--forked` option (to construct the refspec)
+                            .download_tags(git2::AutotagOption::Auto)
                             .remote_callbacks(callbacks)
                             // TODO: could be 1, CLI option maybe?
                             .depth(0),
@@ -193,8 +195,7 @@ pub fn resolve_and_fetch_commitish(
                     refs: vec![commitish.to_string()],
                 })?;
             tracing::info!("fetched base commit");
-            // FETCH_HEAD is written by Remote::fetch to be the last reference fetched
-            repo.revparse_single("FETCH_HEAD")
+            repo.revparse_single(commitish)
                 .and_then(|obj| obj.peel_to_commit())
         }
         Err(err) => Err(err),

spark-k8s/stackable/patches/3.5.2/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/spark.git"
+upstream = "https://github.com/stackabletech/spark.git"
 base = "bb7846dd487f259994fdc69e18e03382e3f64f42"

spark-k8s/stackable/patches/3.5.5/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/spark.git"
+upstream = "https://github.com/stackabletech/spark.git"
 base = "7c29c664cdc9321205a98a14858aaf8daaa19db2"

trino-storage-connector/stackable/patches/451/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/snowlift/trino-storage.git"
+upstream = "https://github.com/stackabletech/trino-storage.git"
 base = "b6e5825bb84a4f1a3f89ff45ea39ce349313f60a"

trino-storage-connector/stackable/patches/455/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/snowlift/trino-storage.git"
+upstream = "https://github.com/stackabletech/trino-storage.git"
 base = "869a735d8be527117a19150e161ad8ca69317578"

trino-storage-connector/stackable/patches/470/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/snowlift/trino-storage.git"
+upstream = "https://github.com/stackabletech/trino-storage.git"
 base = "1b25d617940f14a844a43ee34aa705f7d11fbaf9"

trino/stackable/patches/451/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/trinodb/trino.git"
+upstream = "https://github.com/stackabletech/trino.git"
 base = "2c974f7cb1d71e1f9f466941a317190a474fc432"

trino/stackable/patches/455/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/trinodb/trino.git"
+upstream = "https://github.com/stackabletech/trino.git"
 base = "e212460ea0aa663f0de9b16fecd480c4ad6490cc"

trino/stackable/patches/470/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/trinodb/trino.git"
+upstream = "https://github.com/stackabletech/trino.git"
 base = "05bc059cf0c9263e4ee8be2c1ad69753d0dd4faf"

zookeeper/stackable/patches/3.9.2/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/zookeeper.git"
+upstream = "https://github.com/stackabletech/zookeeper.git"
 base = "e454e8c7283100c7caec6dcae2bc82aaecb63023"

zookeeper/stackable/patches/3.9.3/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/zookeeper.git"
+upstream = "https://github.com/stackabletech/zookeeper.git"
 base = "c26634f34490bb0ea7a09cc51e05ede3b4e320ee"