Add gitlab support

Author: GuillaumeGomez

README.md

@@ -191,7 +191,10 @@ docker-compose run -- database add-directory <DIRECTORY> [PREFIX]
 # You need to set CRATESFYI_GITHUB_USERNAME, CRATESFYI_GITHUB_ACCESSTOKEN
 # environment variables in order to run this command.
 # You can set this environment variables in ~/.cratesfyi.env file.
-docker-compose run -- database update-github-fields
+cargo run -- database update-github-fields
+# Updates github stats for crates.
+# You need to set CRATESFYI_GITLAB_ACCESSTOKEN environment variable.
+cargo run -- database update-gitlab-fields
 ```
 
 If you want to explore or edit database manually, you can connect to the database

src/bin/cratesfyi.rs

@@ -377,6 +377,12 @@ enum DatabaseSubcommand {
     /// Backfill GitHub stats for crates.
     BackfillGithubStats,
 
+    /// Updates gitlab stats for crates.
+    UpdateGitlabFields,
+
+    /// Backfill Gitlab stats for crates.
+    BackfillGitlabStats,
+
     /// Updates info for a crate from the registry's API
     UpdateCrateRegistryFields {
         #[structopt(name = "CRATE")]
@@ -436,6 +442,18 @@ impl DatabaseSubcommand {
                     .backfill_repositories()?;
             }
 
+            Self::UpdateGitlabFields => {
+                docs_rs::utils::GitlabUpdater::new(ctx.config()?, ctx.pool()?)?
+                    .ok_or_else(|| failure::format_err!("missing Gitlab token"))?
+                    .update_all_crates()?;
+            }
+
+            Self::BackfillGitlabStats => {
+                docs_rs::utils::GitlabUpdater::new(ctx.config()?, ctx.pool()?)?
+                    .ok_or_else(|| failure::format_err!("missing Gitlab token"))?
+                    .backfill_repositories()?;
+            }
+
             Self::UpdateCrateRegistryFields { name } => {
                 let index = ctx.index()?;
 

src/config.rs

@@ -30,6 +30,10 @@ pub struct Config {
     pub(crate) github_accesstoken: Option<String>,
     pub(crate) github_updater_min_rate_limit: u32,
 
+    // Gitlab authentication
+    pub(crate) gitlab_accesstoken: Option<String>,
+    pub(crate) gitlab_updater_min_rate_limit: u32,
+
     // Max size of the files served by the docs.rs frontend
     pub(crate) max_file_size: usize,
     pub(crate) max_file_size_html: usize,
@@ -77,6 +81,9 @@ impl Config {
             github_accesstoken: maybe_env("CRATESFYI_GITHUB_ACCESSTOKEN")?,
             github_updater_min_rate_limit: env("DOCSRS_GITHUB_UPDATER_MIN_RATE_LIMIT", 2500)?,
 
+            gitlab_accesstoken: maybe_env("CRATESFYI_GITLAB_ACCESSTOKEN")?,
+            gitlab_updater_min_rate_limit: env("DOCSRS_GITLAB_UPDATER_MIN_RATE_LIMIT", 2500)?,
+
             max_file_size: env("DOCSRS_MAX_FILE_SIZE", 50 * 1024 * 1024)?,
             max_file_size_html: env("DOCSRS_MAX_FILE_SIZE_HTML", 50 * 1024 * 1024)?,
             // LOL HTML only uses as much memory as the size of the start tag!

src/db/add_package.rs

@@ -38,6 +38,7 @@ pub(crate) fn add_package_into_database(
     has_examples: bool,
     compression_algorithms: std::collections::HashSet<CompressionAlgorithm>,
     github_repo: Option<String>,
+    gitlab_repo: Option<String>,
 ) -> Result<i32> {
     debug!("Adding package into database");
     let crate_id = initialize_package_in_database(conn, metadata_pkg)?;
@@ -55,12 +56,14 @@ pub(crate) fn add_package_into_database(
             homepage_url, description, description_long, readme,
             authors, keywords, have_examples, downloads, files,
             doc_targets, is_library, doc_rustc_version,
-            documentation_url, default_target, features, github_repo
+            documentation_url, default_target, features,
+            github_repo, gitlab_repo
          )
          VALUES (
             $1,  $2,  $3,  $4,  $5,  $6,  $7,  $8,  $9,
             $10, $11, $12, $13, $14, $15, $16, $17, $18,
-            $19, $20, $21, $22, $23, $24, $25, $26, $27
+            $19, $20, $21, $22, $23, $24, $25, $26, $27,
+            $28
          )
          ON CONFLICT (crate_id, version) DO UPDATE
             SET release_time = $3,
@@ -87,7 +90,8 @@ pub(crate) fn add_package_into_database(
                 documentation_url = $24,
                 default_target = $25,
                 features = $26,
-                github_repo = $27
+                github_repo = $27,
+                gitlab_repo = $28
          RETURNING id",
         &[
             &crate_id,
@@ -117,6 +121,7 @@ pub(crate) fn add_package_into_database(
             &default_target,
             &features,
             &github_repo,
+            &gitlab_repo,
         ],
     )?;
 

src/db/migrate.rs

@@ -618,6 +618,43 @@ pub fn migrate(version: Option<Version>, conn: &mut Client) -> CratesfyiResult<(
             DROP INDEX github_repos_stars_idx;
             ",
         ),
+        migration!(
+            context,
+            27,
+            // description
+            "Add the gitlab_repositories table to contain Gitlab information",
+            // upgrade query
+            "
+                CREATE TABLE gitlab_repos (
+                    id VARCHAR PRIMARY KEY NOT NULL,
+                    name VARCHAR NOT NULL,
+                    description VARCHAR,
+                    last_commit TIMESTAMP,
+                    stars INT NOT NULL,
+                    forks INT NOT NULL,
+                    issues INT NOT NULL,
+                    updated_at TIMESTAMP NOT NULL
+                );
+
+                ALTER TABLE releases ADD COLUMN gitlab_repo VARCHAR
+                    REFERENCES gitlab_repos(id) ON DELETE SET NULL;
+
+                CREATE INDEX releases_gitlab_repo_idx ON releases (gitlab_repo);
+                CREATE INDEX gitlab_repos_stars_idx ON gitlab_repos(stars DESC);
+
+                ALTER TABLE gitlab_repos
+                    ALTER updated_at TYPE timestamptz USING updated_at AT TIME ZONE 'UTC',
+                    ALTER last_commit TYPE timestamptz USING last_commit AT TIME ZONE 'UTC';
+            ",
+            // downgrade query
+            "
+                ALTER TABLE releases DROP COLUMN gitlab_repo;
+                DROP TABLE gitlab_repos;
+
+                DROP INDEX releases_github_repo_idx;
+                DROP INDEX github_repos_stars_idx;
+            "
+        ),
     ];
 
     for migration in migrations {

src/docbuilder/rustwide_builder.rs

@@ -7,7 +7,9 @@ use crate::docbuilder::{crates::crates_from_path, Limits};
 use crate::error::Result;
 use crate::index::api::ReleaseData;
 use crate::storage::CompressionAlgorithms;
-use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata, GithubUpdater};
+use crate::utils::{
+    copy_doc_dir, parse_rustc_version, CargoMetadata, GithubUpdater, GitlabUpdater,
+};
 use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
 use crate::{Config, Context, Index, Metrics, Storage};
 use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET};
@@ -394,6 +396,7 @@ impl RustwideBuilder {
 
                 let cargo_metadata = res.cargo_metadata.root();
                 let github_repo = self.get_github_repo(&mut conn, cargo_metadata)?;
+                let gitlab_repo = self.get_gitlab_repo(&mut conn, cargo_metadata)?;
 
                 let release_id = add_package_into_database(
                     &mut conn,
@@ -408,6 +411,7 @@ impl RustwideBuilder {
                     has_examples,
                     algs,
                     github_repo,
+                    gitlab_repo,
                 )?;
 
                 if let Some(doc_coverage) = res.doc_coverage {
@@ -701,6 +705,34 @@ impl RustwideBuilder {
             }
         }
     }
+
+    fn get_gitlab_repo(
+        &self,
+        conn: &mut Client,
+        metadata: &MetadataPackage,
+    ) -> Result<Option<String>> {
+        let updater = match GitlabUpdater::new(self.config.clone(), self.db.clone())? {
+            Some(updater) => updater,
+            None => {
+                warn!("did not collect Gitlab stats as no token was provided");
+                return Ok(None);
+            }
+        };
+        let repo = match &metadata.repository {
+            Some(url) => url,
+            None => {
+                debug!("did not collect Gitlab stats as no repository URL was present");
+                return Ok(None);
+            }
+        };
+        match updater.load_repository(conn, repo) {
+            Ok(repo) => Ok(repo),
+            Err(err) => {
+                warn!("failed to collect Gitlab stats: {}", err);
+                Ok(None)
+            }
+        }
+    }
 }
 
 struct FullBuildResult {

src/utils/daemon.rs

@@ -3,7 +3,7 @@
 //! This daemon will start web server, track new packages and build them
 
 use crate::{
-    utils::{queue_builder, update_release_activity, GithubUpdater},
+    utils::{queue_builder, update_release_activity, GithubUpdater, GitlabUpdater},
     Context, DocBuilder, RustwideBuilder,
 };
 use chrono::{Timelike, Utc};
@@ -93,7 +93,7 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res
         },
     )?;
 
-    if let Some(github_updater) = GithubUpdater::new(config, context.pool()?)? {
+    if let Some(github_updater) = GithubUpdater::new(config.clone(), context.pool()?)? {
         cron(
             "github stats updater",
             Duration::from_secs(60 * 60),
@@ -106,6 +106,19 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res
         log::warn!("GitHub stats updater not started as no token was provided");
     }
 
+    if let Some(gitlab_updater) = GitlabUpdater::new(config, context.pool()?)? {
+        cron(
+            "gitlab stats updater",
+            Duration::from_secs(60 * 60),
+            move || {
+                gitlab_updater.update_all_crates()?;
+                Ok(())
+            },
+        )?;
+    } else {
+        log::warn!("Gitlab stats updater not started as no token was provided");
+    }
+
     // Never returns; `server` blocks indefinitely when dropped
     // NOTE: if a failure occurred earlier in `start_daemon`, the server will _not_ be joined -
     // instead it will get killed when the process exits.

src/utils/github_updater.rs

@@ -28,7 +28,7 @@ const GRAPHQL_UPDATE: &str = "query($ids: [ID!]!) {
             description
             stargazerCount
             forkCount
-            issues(states: [OPEN]) { totalCount }
+            issues { totalCount }
         }
     }
     rateLimit {
@@ -44,7 +44,7 @@ const GRAPHQL_SINGLE: &str = "query($owner: String!, $repo: String!) {
         description
         stargazerCount
         forkCount
-        issues(states: [OPEN]) { totalCount }
+        issues { totalCount }
     }
 }";