Skip to content

Commit d8a76f8

Browse files
GuillaumeGomezGuillaumeGomez
committed
Unify Updaters
1 parent 686fe63 commit d8a76f8

File tree

11 files changed

+178
-238
lines changed

11 files changed

+178
-238
lines changed

‎README.md

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -190,11 +190,10 @@ docker-compose run -- database add-directory <DIRECTORY> [PREFIX]
190190
# Updates github stats for crates.
191191
# You need to set CRATESFYI_GITHUB_USERNAME, CRATESFYI_GITHUB_ACCESSTOKEN
192192
# environment variables in order to run this command.
193+
# Even though it's not mandatory, setting CRATESFYI_GITLAB_ACCESSTOKEN
194+
# environment variable might be a good idea too.
193195
# You can set this environment variables in ~/.cratesfyi.env file.
194-
cargo run -- database update-github-fields
195-
# Updates github stats for crates.
196-
# You need to set CRATESFYI_GITLAB_ACCESSTOKEN environment variable.
197-
cargo run -- database update-gitlab-fields
196+
cargo run -- database update-repositories-fields
198197
```
199198

200199
If you want to explore or edit database manually, you can connect to the database

‎src/bin/cratesfyi.rs

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use std::path::PathBuf;
44
use std::sync::Arc;
55

66
use docs_rs::db::{self, add_path_into_database, Pool, PoolClient};
7-
use docs_rs::utils::{remove_crate_priority, set_crate_priority};
7+
use docs_rs::utils::{remove_crate_priority, set_crate_priority, Updater};
88
use docs_rs::{
99
BuildQueue, Config, Context, DocBuilder, Index, Metrics, PackageKind, RustwideBuilder, Server,
1010
Storage,
@@ -371,17 +371,11 @@ enum DatabaseSubcommand {
371371
version: Option<i64>,
372372
},
373373

374-
/// Updates github stats for crates.
375-
UpdateGithubFields,
374+
/// Updates Github/Gitlab stats for crates.
375+
UpdateRepositoriesFields,
376376

377-
/// Backfill GitHub stats for crates.
378-
BackfillGithubStats,
379-
380-
/// Updates gitlab stats for crates.
381-
UpdateGitlabFields,
382-
383-
/// Backfill Gitlab stats for crates.
384-
BackfillGitlabStats,
377+
/// Backfill GitHub/Gitlab stats for crates.
378+
BackfillRepositoriesStats,
385379

386380
/// Updates info for a crate from the registry's API
387381
UpdateCrateRegistryFields {
@@ -430,28 +424,34 @@ impl DatabaseSubcommand {
430424
.context("Failed to run database migrations")?;
431425
}
432426

433-
Self::UpdateGithubFields => {
434-
docs_rs::utils::GithubUpdater::new(ctx.config()?, ctx.pool()?)?
435-
.ok_or_else(|| failure::format_err!("missing GitHub token"))?
436-
.update_all_crates()?;
437-
}
438-
439-
Self::BackfillGithubStats => {
440-
docs_rs::utils::GithubUpdater::new(ctx.config()?, ctx.pool()?)?
441-
.ok_or_else(|| failure::format_err!("missing GitHub token"))?
442-
.backfill_repositories()?;
443-
}
444-
445-
Self::UpdateGitlabFields => {
446-
docs_rs::utils::GitlabUpdater::new(ctx.config()?, ctx.pool()?)?
447-
.ok_or_else(|| failure::format_err!("missing Gitlab token"))?
448-
.update_all_crates()?;
427+
Self::UpdateRepositoriesFields => {
428+
let mut errors = Vec::new();
429+
match docs_rs::utils::GithubUpdater::new(ctx.config()?, ctx.pool()?)? {
430+
Some(up) => up.update_all_crates()?,
431+
None => errors.push("missing GitHub token"),
432+
}
433+
match docs_rs::utils::GitlabUpdater::new(ctx.config()?, ctx.pool()?)? {
434+
Some(up) => up.update_all_crates()?,
435+
None => errors.push("missing Gitlab token"),
436+
}
437+
if !errors.is_empty() {
438+
return Err(failure::format_err!("{}", errors.join("\n")));
439+
}
449440
}
450441

451-
Self::BackfillGitlabStats => {
452-
docs_rs::utils::GitlabUpdater::new(ctx.config()?, ctx.pool()?)?
453-
.ok_or_else(|| failure::format_err!("missing Gitlab token"))?
454-
.backfill_repositories()?;
442+
Self::BackfillRepositoriesStats => {
443+
let mut errors = Vec::new();
444+
match docs_rs::utils::GithubUpdater::new(ctx.config()?, ctx.pool()?)? {
445+
Some(up) => up.backfill_repositories()?,
446+
None => errors.push("missing GitHub token"),
447+
}
448+
match docs_rs::utils::GitlabUpdater::new(ctx.config()?, ctx.pool()?)? {
449+
Some(up) => up.backfill_repositories()?,
450+
None => errors.push("missing Gitlab token"),
451+
}
452+
if !errors.is_empty() {
453+
return Err(failure::format_err!("{}", errors.join("\n")));
454+
}
455455
}
456456

457457
Self::UpdateCrateRegistryFields { name } => {

‎src/config.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ pub struct Config {
3232

3333
// Gitlab authentication
3434
pub(crate) gitlab_accesstoken: Option<String>,
35-
pub(crate) gitlab_updater_min_rate_limit: u32,
3635

3736
// Max size of the files served by the docs.rs frontend
3837
pub(crate) max_file_size: usize,
@@ -82,7 +81,6 @@ impl Config {
8281
github_updater_min_rate_limit: env("DOCSRS_GITHUB_UPDATER_MIN_RATE_LIMIT", 2500)?,
8382

8483
gitlab_accesstoken: maybe_env("CRATESFYI_GITLAB_ACCESSTOKEN")?,
85-
gitlab_updater_min_rate_limit: env("DOCSRS_GITLAB_UPDATER_MIN_RATE_LIMIT", 2500)?,
8684

8785
max_file_size: env("DOCSRS_MAX_FILE_SIZE", 50 * 1024 * 1024)?,
8886
max_file_size_html: env("DOCSRS_MAX_FILE_SIZE_HTML", 50 * 1024 * 1024)?,

‎src/db/migrate.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -697,7 +697,7 @@ pub fn migrate(version: Option<Version>, conn: &mut Client) -> CratesfyiResult<(
697697
ALTER TABLE releases
698698
DROP COLUMN repository;
699699
700-
DROP TABLE repository;
700+
DROP TABLE repositories;
701701
"
702702
),
703703
];

‎src/docbuilder/rustwide_builder.rs

Lines changed: 19 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use crate::error::Result;
88
use crate::index::api::ReleaseData;
99
use crate::storage::CompressionAlgorithms;
1010
use crate::utils::{
11-
copy_doc_dir, parse_rustc_version, CargoMetadata, GithubUpdater, GitlabUpdater,
11+
copy_doc_dir, parse_rustc_version, CargoMetadata, GithubUpdater, GitlabUpdater, Updater,
1212
};
1313
use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
1414
use crate::{Config, Context, Index, Metrics, Storage};
@@ -395,10 +395,7 @@ impl RustwideBuilder {
395395
};
396396

397397
let cargo_metadata = res.cargo_metadata.root();
398-
let repository = match self.get_github_repo(&mut conn, cargo_metadata)? {
399-
Some(r) => Some(r),
400-
None => self.get_gitlab_repo(&mut conn, cargo_metadata)?,
401-
};
398+
let repository = self.get_repo(&mut conn, cargo_metadata)?;
402399

403400
let release_id = add_package_into_database(
404401
&mut conn,
@@ -679,57 +676,44 @@ impl RustwideBuilder {
679676
}
680677
}
681678

682-
fn get_github_repo(
683-
&self,
684-
conn: &mut Client,
685-
metadata: &MetadataPackage,
686-
) -> Result<Option<i32>> {
687-
let updater = match GithubUpdater::new(self.config.clone(), self.db.clone())? {
688-
Some(updater) => updater,
689-
None => {
690-
warn!("did not collect GitHub stats as no token was provided");
691-
return Ok(None);
692-
}
693-
};
694-
let repo = match &metadata.repository {
695-
Some(url) => url,
696-
None => {
697-
debug!("did not collect GitHub stats as no repository URL was present");
698-
return Ok(None);
699-
}
700-
};
701-
match updater.load_repository(conn, repo) {
702-
Ok(repo) => Ok(repo),
703-
Err(err) => {
704-
warn!("failed to collect GitHub stats: {}", err);
705-
Ok(None)
706-
}
679+
fn get_repo(&self, conn: &mut Client, metadata: &MetadataPackage) -> Result<Option<i32>> {
680+
macro_rules! return_if_ok_some {
681+
($typ:ty) => {
682+
let data = self.get_repo_inner::<$typ>(conn, metadata);
683+
if matches!(data, Ok(Some(_))) {
684+
return data;
685+
}
686+
};
707687
}
688+
689+
// The `GitlabUpdated is a bit more permissive so better to put it at the end.
690+
return_if_ok_some!(GithubUpdater);
691+
return_if_ok_some!(GitlabUpdater);
692+
Ok(None)
708693
}
709694

710-
fn get_gitlab_repo(
695+
fn get_repo_inner<T: Updater>(
711696
&self,
712697
conn: &mut Client,
713698
metadata: &MetadataPackage,
714699
) -> Result<Option<i32>> {
715-
let updater = match GitlabUpdater::new(self.config.clone(), self.db.clone())? {
700+
let updater = match T::new(self.config.clone(), self.db.clone())? {
716701
Some(updater) => updater,
717702
None => {
718-
warn!("did not collect Gitlab stats as no token was provided");
719703
return Ok(None);
720704
}
721705
};
722706
let repo = match &metadata.repository {
723707
Some(url) => url,
724708
None => {
725-
debug!("did not collect Gitlab stats as no repository URL was present");
709+
debug!("did not collect GitHub stats as no repository URL was present");
726710
return Ok(None);
727711
}
728712
};
729713
match updater.load_repository(conn, repo) {
730714
Ok(repo) => Ok(repo),
731715
Err(err) => {
732-
warn!("failed to collect Gitlab stats: {}", err);
716+
warn!("failed to collect GitHub stats: {}", err);
733717
Ok(None)
734718
}
735719
}

‎src/test/fakes.rs

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -364,19 +364,20 @@ struct FakeGithubStats {
364364
}
365365

366366
impl FakeGithubStats {
367-
fn create(&self, conn: &mut Client) -> Result<String, Error> {
367+
fn create(&self, conn: &mut Client) -> Result<i32, Error> {
368368
let existing_count: i64 = conn
369369
.query_one("SELECT COUNT(*) FROM repositories;", &[])?
370370
.get(0);
371-
let id = base64::encode(format!("FAKE ID {}", existing_count));
371+
let host_id = base64::encode(format!("FAKE ID {}", existing_count));
372372

373-
conn.execute(
373+
let data = conn.query(
374374
"INSERT INTO repositories (host, host_id, name, description, last_commit, stars, forks, issues, updated_at)
375-
VALUES ('github', $1, $2, 'Fake description!', NOW(), $3, $4, $5, NOW());",
376-
&[&id, &self.repo, &self.stars, &self.forks, &self.issues],
375+
VALUES ('github', $1, $2, 'Fake description!', NOW(), $3, $4, $5, NOW())
376+
RETURNING id;",
377+
&[&host_id, &self.repo, &self.stars, &self.forks, &self.issues],
377378
)?;
378379

379-
Ok(id)
380+
Ok(data[0].get(0))
380381
}
381382
}
382383

‎src/utils/daemon.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
//! This daemon will start web server, track new packages and build them
44
55
use crate::{
6-
utils::{queue_builder, update_release_activity, GithubUpdater, GitlabUpdater},
6+
utils::{queue_builder, update_release_activity, GithubUpdater, GitlabUpdater, Updater},
77
Context, DocBuilder, RustwideBuilder,
88
};
99
use chrono::{Timelike, Utc};
@@ -115,8 +115,6 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res
115115
Ok(())
116116
},
117117
)?;
118-
} else {
119-
log::warn!("Gitlab stats updater not started as no token was provided");
120118
}
121119

122120
// Never returns; `server` blocks indefinitely when dropped

0 commit comments

Comments
 (0)