Run git gc periodically on the crates.io index (#778)

.gitignore

@@ -8,3 +8,4 @@ target
 .vagrant
 .rustwide
 .rustwide-docker
+.idea/

src/config.rs

@@ -30,6 +30,9 @@ pub struct Config {
     pub(crate) max_file_size_html: usize,
     // The most memory that can be used to parse an HTML file
     pub(crate) max_parse_memory: usize,
+
+    // Time between 'git gc' calls in seconds
+    pub(crate) registry_gc_interval: u64,
 }
 
 impl Config {
@@ -61,6 +64,7 @@ impl Config {
             // LOL HTML only uses as much memory as the size of the start tag!
             // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380
             max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?,
+            registry_gc_interval: env("REGISTRY_GC_INTERVAL", 60 * 60)?,
         })
     }
 

src/utils/daemon.rs

@@ -11,20 +11,32 @@ use crate::{
 use chrono::{Timelike, Utc};
 use failure::Error;
 use log::{debug, error, info};
+use std::process::Command;
 use std::sync::Arc;
 use std::thread;
-use std::time::Duration;
+use std::time::{Duration, Instant};
+
+fn run_git_gc() {
+    Command::new("git")
+        .args(&["gc"])
+        .output()
+        .expect("Failed to execute git gc");
+}
 
 fn start_registry_watcher(
     opts: DocBuilderOptions,
     pool: Pool,
     build_queue: Arc<BuildQueue>,
+    config: Arc<Config>,
 ) -> Result<(), Error> {
     thread::Builder::new()
         .name("registry index reader".to_string())
         .spawn(move || {
             // space this out to prevent it from clashing against the queue-builder thread on launch
             thread::sleep(Duration::from_secs(30));
+            let mut last_gc = Instant::now();
+            run_git_gc();
+
             loop {
                 let mut doc_builder =
                     DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone());
@@ -39,6 +51,10 @@ fn start_registry_watcher(
                     }
                 }
 
+                if last_gc.elapsed().as_secs() >= config.registry_gc_interval {
+                    run_git_gc();
+                    last_gc = Instant::now();
+                }
                 thread::sleep(Duration::from_secs(60));
             }
         })?;
@@ -60,7 +76,12 @@ pub fn start_daemon(
 
     if enable_registry_watcher {
         // check new crates every minute
-        start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone())?;
+        start_registry_watcher(
+            dbopts.clone(),
+            db.clone(),
+            build_queue.clone(),
+            config.clone(),
+        )?;
     }
 
     // build new crates every minute