Run git gc periodically on the crates.io index (#778)

.git_hooks/pre-commit

@@ -0,0 +1,19 @@
+#!/usr/bin/env sh
+if ! cargo fmt -- --check ; then
+  printf "\n"
+  printf "\033[0;31mpre-commit hook failed during:\033[0m\n"
+  printf "\033[0;31m\tcargo fmt -- --check\033[0m\n"
+  exit 1
+fi
+
+if ! cargo clippy --locked -- -D warnings ; then
+  printf "\n"
+  printf "\033[0;31mpre-commit hook failed during:\033[0m\n"
+  printf "\033[0;31m\tclippy --locked -- -D warning\033[0m\n"
+  exit 1
+fi
+
+printf "\n"
+printf "\033[0;32mpre-commit hook succeeded\033[0m\n"
+exit 0
+

README.md

@@ -22,6 +22,12 @@ The recommended way to develop docs.rs is a combination of `cargo run` for
 the main binary and [docker-compose](https://docs.docker.com/compose/) for the external services.
 This gives you reasonable incremental build times without having to add new users and packages to your host machine.
 
+### Git Hooks
+For ease of use, `git_hooks` directory contains useful `git hooks` to make your development easier.
+```bash
+cd .git/hooks && ln -s ../../.git_hooks/* . && cd ../..
+```
+
 ### Dependencies
 
 Docs.rs requires at least the following native C dependencies.

src/config.rs

@@ -30,6 +30,8 @@ pub struct Config {
     pub(crate) max_file_size_html: usize,
     // The most memory that can be used to parse an HTML file
     pub(crate) max_parse_memory: usize,
+    // Time between 'git gc' calls in seconds
+    pub(crate) registry_gc_interval: u64,
 }
 
 impl Config {
@@ -61,6 +63,7 @@ impl Config {
             // LOL HTML only uses as much memory as the size of the start tag!
             // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380
             max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?,
+            registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?,
         })
     }
 

src/docbuilder/queue.rs

@@ -7,6 +7,10 @@ use crates_index_diff::ChangeKind;
 use log::{debug, error};
 
 impl DocBuilder {
+    pub fn run_git_gc(&self) {
+        self.index.run_git_gc();
+    }
+
     /// Updates registry index repository and adds new crates into build queue.
     /// Returns the number of crates added
     pub fn get_new_crates(&mut self) -> Result<usize> {

src/index/mod.rs

@@ -1,4 +1,5 @@
 use std::path::{Path, PathBuf};
+use std::process::Command;
 
 use url::Url;
 
@@ -50,6 +51,14 @@ impl Index {
         Ok(Self { path, api })
     }
 
+    pub fn run_git_gc(&self) {
+        let cmd = format!("cd {} && gc --auto", self.path.to_str().unwrap());
+        let gc = Command::new("sh").args(&["-c", cmd.as_str()]).output();
+        if let Err(err) = gc {
+            log::error!("Failed to run `{}`: {:?}", cmd, err);
+        }
+    }
+
     pub(crate) fn diff(&self) -> Result<crates_index_diff::Index> {
         let diff = crates_index_diff::Index::from_path_or_cloned(&self.path)
             .context("re-opening registry index for diff")?;

src/utils/daemon.rs

@@ -13,18 +13,21 @@ use failure::Error;
 use log::{debug, error, info};
 use std::sync::Arc;
 use std::thread;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 fn start_registry_watcher(
     opts: DocBuilderOptions,
     pool: Pool,
     build_queue: Arc<BuildQueue>,
+    config: Arc<Config>,
 ) -> Result<(), Error> {
     thread::Builder::new()
         .name("registry index reader".to_string())
         .spawn(move || {
             // space this out to prevent it from clashing against the queue-builder thread on launch
             thread::sleep(Duration::from_secs(30));
+            let mut last_gc = Instant::now();
+
             loop {
                 let mut doc_builder =
                     DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone());
@@ -39,6 +42,10 @@ fn start_registry_watcher(
                     }
                 }
 
+                if last_gc.elapsed().as_secs() >= config.registry_gc_interval {
+                    doc_builder.run_git_gc();
+                    last_gc = Instant::now();
+                }
                 thread::sleep(Duration::from_secs(60));
             }
         })?;
@@ -60,7 +67,12 @@ pub fn start_daemon(
 
     if enable_registry_watcher {
         // check new crates every minute
-        start_registry_watcher(dbopts.clone(), db.clone(), build_queue.clone())?;
+        start_registry_watcher(
+            dbopts.clone(),
+            db.clone(),
+            build_queue.clone(),
+            config.clone(),
+        )?;
     }
 
     // build new crates every minute

src/web/metrics.rs

@@ -129,7 +129,7 @@ pub static MAX_DB_CONNECTIONS: Lazy<IntGauge> = Lazy::new(|| {
     .unwrap()
 });
 
-#[cfg(not(windows))]
+#[cfg(target_os = "linux")]
 pub static OPEN_FILE_DESCRIPTORS: Lazy<IntGauge> = Lazy::new(|| {
     register_int_gauge!(
         "docsrs_open_file_descriptors",
@@ -138,7 +138,7 @@ pub static OPEN_FILE_DESCRIPTORS: Lazy<IntGauge> = Lazy::new(|| {
     .unwrap()
 });
 
-#[cfg(not(windows))]
+#[cfg(target_os = "linux")]
 pub static CURRENTLY_RUNNING_THREADS: Lazy<IntGauge> = Lazy::new(|| {
     register_int_gauge!(
         "docsrs_running_threads",