Run git gc periodically on the crates.io index (#975)

Author: ohaddahan

src/config.rs

@@ -30,6 +30,8 @@ pub struct Config {
     pub(crate) max_file_size_html: usize,
     // The most memory that can be used to parse an HTML file
     pub(crate) max_parse_memory: usize,
+    // Time between 'git gc --auto' calls in seconds
+    pub(crate) registry_gc_interval: u64,
 }
 
 impl Config {
@@ -61,6 +63,7 @@ impl Config {
             // LOL HTML only uses as much memory as the size of the start tag!
             // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380
             max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?,
+            registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?,
         })
     }
 

src/docbuilder/queue.rs

@@ -84,4 +84,8 @@ impl DocBuilder {
 
         Ok(processed)
     }
+
+    pub fn run_git_gc(&self) {
+        self.index.run_git_gc();
+    }
 }

src/index/mod.rs

@@ -1,4 +1,7 @@
-use std::path::{Path, PathBuf};
+use std::{
+    path::{Path, PathBuf},
+    process::Command,
+};
 
 use url::Url;
 
@@ -71,6 +74,22 @@ impl Index {
     pub fn api(&self) -> &Api {
         &self.api
     }
+
+    pub fn run_git_gc(&self) {
+        let gc = Command::new("git")
+            .arg("-C")
+            .arg(&self.path)
+            .args(&["gc", "--auto"])
+            .output();
+
+        if let Err(err) = gc {
+            log::error!(
+                "failed to run `git gc --auto`\npath: {:#?}\nerror: {:#?}",
+                &self.path,
+                err
+            );
+        }
+    }
 }
 
 impl Clone for Index {

src/utils/daemon.rs

@@ -10,16 +10,20 @@ use chrono::{Timelike, Utc};
 use failure::Error;
 use log::{debug, error, info};
 use std::thread;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> {
     let pool = context.pool()?;
     let build_queue = context.build_queue()?;
+    let config = context.config()?;
+
     thread::Builder::new()
         .name("registry index reader".to_string())
         .spawn(move || {
             // space this out to prevent it from clashing against the queue-builder thread on launch
             thread::sleep(Duration::from_secs(30));
+
+            let mut last_gc = Instant::now();
             loop {
                 let mut doc_builder =
                     DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone());
@@ -34,6 +38,10 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res
                     }
                 }
 
+                if last_gc.elapsed().as_secs() >= config.registry_gc_interval {
+                    doc_builder.run_git_gc();
+                    last_gc = Instant::now();
+                }
                 thread::sleep(Duration::from_secs(60));
             }
         })?;