Skip to content

Commit 6b6c774

Browse files
GuillaumeGomezGuillaumeGomez
committed
Create RepositoryStatsUpdater type to use as interface over updaters
1 parent 506b9bf commit 6b6c774

File tree

7 files changed

+141
-97
lines changed

7 files changed

+141
-97
lines changed

‎src/bin/cratesfyi.rs

Lines changed: 3 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use std::path::PathBuf;
44
use std::sync::Arc;
55

66
use docs_rs::db::{self, add_path_into_database, Pool, PoolClient};
7-
use docs_rs::utils::{remove_crate_priority, set_crate_priority, Updater};
7+
use docs_rs::utils::{remove_crate_priority, set_crate_priority, RepositoryStatsUpdater};
88
use docs_rs::{
99
BuildQueue, Config, Context, DocBuilder, Index, Metrics, PackageKind, RustwideBuilder, Server,
1010
Storage,
@@ -425,33 +425,11 @@ impl DatabaseSubcommand {
425425
}
426426

427427
Self::UpdateRepositoriesFields => {
428-
let mut errors = Vec::new();
429-
match docs_rs::utils::GithubUpdater::new(ctx.config()?, ctx.pool()?)? {
430-
Some(up) => up.update_all_crates()?,
431-
None => errors.push("missing GitHub token"),
432-
}
433-
match docs_rs::utils::GitlabUpdater::new(ctx.config()?, ctx.pool()?)? {
434-
Some(up) => up.update_all_crates()?,
435-
None => errors.push("missing Gitlab token"),
436-
}
437-
if !errors.is_empty() {
438-
return Err(failure::format_err!("{}", errors.join("\n")));
439-
}
428+
RepositoryStatsUpdater::update_all_crates(&ctx)?;
440429
}
441430

442431
Self::BackfillRepositoriesStats => {
443-
let mut errors = Vec::new();
444-
match docs_rs::utils::GithubUpdater::new(ctx.config()?, ctx.pool()?)? {
445-
Some(up) => up.backfill_repositories()?,
446-
None => errors.push("missing GitHub token"),
447-
}
448-
match docs_rs::utils::GitlabUpdater::new(ctx.config()?, ctx.pool()?)? {
449-
Some(up) => up.backfill_repositories()?,
450-
None => errors.push("missing Gitlab token"),
451-
}
452-
if !errors.is_empty() {
453-
return Err(failure::format_err!("{}", errors.join("\n")));
454-
}
432+
RepositoryStatsUpdater::backfill_repositories(&ctx)?;
455433
}
456434

457435
Self::UpdateCrateRegistryFields { name } => {

‎src/docbuilder/rustwide_builder.rs

Lines changed: 7 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,7 @@ use crate::docbuilder::{crates::crates_from_path, Limits};
77
use crate::error::Result;
88
use crate::index::api::ReleaseData;
99
use crate::storage::CompressionAlgorithms;
10-
use crate::utils::{
11-
copy_doc_dir, parse_rustc_version, CargoMetadata, GithubUpdater, GitlabUpdater, Updater,
12-
};
10+
use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata, RepositoryStatsUpdater};
1311
use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
1412
use crate::{Config, Context, Index, Metrics, Storage};
1513
use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET};
@@ -677,46 +675,12 @@ impl RustwideBuilder {
677675
}
678676

679677
fn get_repo(&self, conn: &mut Client, metadata: &MetadataPackage) -> Result<Option<i32>> {
680-
macro_rules! return_if_ok_some {
681-
($typ:ty) => {
682-
let data = self.get_repo_inner::<$typ>(conn, metadata);
683-
if matches!(data, Ok(Some(_))) {
684-
return data;
685-
}
686-
};
687-
}
688-
689-
// The `GitlabUpdated is a bit more permissive so better to put it at the end.
690-
return_if_ok_some!(GithubUpdater);
691-
return_if_ok_some!(GitlabUpdater);
692-
Ok(None)
693-
}
694-
695-
fn get_repo_inner<T: Updater>(
696-
&self,
697-
conn: &mut Client,
698-
metadata: &MetadataPackage,
699-
) -> Result<Option<i32>> {
700-
let updater = match T::new(self.config.clone(), self.db.clone())? {
701-
Some(updater) => updater,
702-
None => {
703-
return Ok(None);
704-
}
705-
};
706-
let repo = match &metadata.repository {
707-
Some(url) => url,
708-
None => {
709-
debug!("did not collect GitHub stats as no repository URL was present");
710-
return Ok(None);
711-
}
712-
};
713-
match updater.load_repository(conn, repo) {
714-
Ok(repo) => Ok(repo),
715-
Err(err) => {
716-
warn!("failed to collect GitHub stats: {}", err);
717-
Ok(None)
718-
}
719-
}
678+
RepositoryStatsUpdater::load_repository(
679+
conn,
680+
metadata,
681+
self.config.clone(),
682+
self.db.clone(),
683+
)
720684
}
721685
}
722686

‎src/utils/daemon.rs

Lines changed: 3 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
//! This daemon will start web server, track new packages and build them
44
55
use crate::{
6-
utils::{queue_builder, update_release_activity, GithubUpdater, GitlabUpdater, Updater},
6+
utils::{queue_builder, update_release_activity, RepositoryStatsUpdater},
77
Context, DocBuilder, RustwideBuilder,
88
};
99
use chrono::{Timelike, Utc};
@@ -93,29 +93,7 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res
9393
},
9494
)?;
9595

96-
if let Some(github_updater) = GithubUpdater::new(config.clone(), context.pool()?)? {
97-
cron(
98-
"github stats updater",
99-
Duration::from_secs(60 * 60),
100-
move || {
101-
github_updater.update_all_crates()?;
102-
Ok(())
103-
},
104-
)?;
105-
} else {
106-
log::warn!("GitHub stats updater not started as no token was provided");
107-
}
108-
109-
if let Some(gitlab_updater) = GitlabUpdater::new(config, context.pool()?)? {
110-
cron(
111-
"gitlab stats updater",
112-
Duration::from_secs(60 * 60),
113-
move || {
114-
gitlab_updater.update_all_crates()?;
115-
Ok(())
116-
},
117-
)?;
118-
}
96+
RepositoryStatsUpdater::start_crons(config, context)?;
11997

12098
// Never returns; `server` blocks indefinitely when dropped
12199
// NOTE: if a failure occurred earlier in `start_daemon`, the server will _not_ be joined -
@@ -125,7 +103,7 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res
125103
.map_err(|_| failure::err_msg("web server panicked"))
126104
}
127105

128-
fn cron<F>(name: &'static str, interval: Duration, exec: F) -> Result<(), Error>
106+
pub(crate) fn cron<F>(name: &'static str, interval: Duration, exec: F) -> Result<(), Error>
129107
where
130108
F: Fn() -> Result<(), Error> + Send + 'static,
131109
{

‎src/utils/github_updater.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,10 @@ impl Updater for GithubUpdater {
205205
None => None,
206206
}
207207
}
208+
209+
fn name() -> &'static str {
210+
"Github"
211+
}
208212
}
209213

210214
impl GithubUpdater {

‎src/utils/gitlab_updater.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,10 @@ impl Updater for GitlabUpdater {
219219
None => None,
220220
}
221221
}
222+
223+
fn name() -> &'static str {
224+
"Gitlab"
225+
}
222226
}
223227

224228
impl GitlabUpdater {

‎src/utils/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ pub use self::queue::{get_crate_priority, remove_crate_priority, set_crate_prior
1010
pub use self::queue_builder::queue_builder;
1111
pub use self::release_activity_updater::update_release_activity;
1212
pub(crate) use self::rustc_version::parse_rustc_version;
13-
pub use self::updater::Updater;
1413
pub(crate) use self::updater::{RepositoryName, APP_USER_AGENT};
14+
pub use self::updater::{RepositoryStatsUpdater, Updater};
1515

1616
#[cfg(test)]
1717
pub(crate) use self::cargo_metadata::{Dependency, Target};

‎src/utils/updater.rs

Lines changed: 119 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
use crate::error::Result;
2-
use crate::{db::Pool, Config};
3-
use log::trace;
2+
use crate::utils::daemon::cron;
3+
use crate::utils::{GithubUpdater, GitlabUpdater, MetadataPackage};
4+
use crate::{db::Pool, Config, Context};
5+
use log::{debug, trace, warn};
46
use postgres::Client;
57
use std::sync::Arc;
8+
use std::time::Duration;
69

710
pub const APP_USER_AGENT: &str = concat!(
811
env!("CARGO_PKG_NAME"),
@@ -18,9 +21,11 @@ pub trait Updater {
1821
fn load_repository(&self, conn: &mut Client, url: &str) -> Result<Option<i32>>;
1922
fn update_all_crates(&self) -> Result<()>;
2023
fn repository_name(url: &str) -> Option<RepositoryName>;
24+
fn name() -> &'static str;
2125
fn delete_repository(&self, conn: &mut Client, host_id: &str, host: &str) -> Result<()> {
2226
trace!(
23-
"removing Gitlab repository stats for host ID `{}` and host `{}`",
27+
"removing {} repository stats for host ID `{}` and host `{}`",
28+
Self::name(),
2429
host_id,
2530
host
2631
);
@@ -32,6 +37,117 @@ pub trait Updater {
3237
}
3338
}
3439

40+
pub struct RepositoryStatsUpdater;
41+
42+
impl RepositoryStatsUpdater {
43+
pub(crate) fn load_repository(
44+
conn: &mut Client,
45+
metadata: &MetadataPackage,
46+
config: Arc<Config>,
47+
db: Pool,
48+
) -> Result<Option<i32>> {
49+
macro_rules! return_if_ok_some {
50+
($typ:ty) => {
51+
let data =
52+
Self::load_repository_inner::<$typ>(conn, metadata, config.clone(), db.clone());
53+
if matches!(data, Ok(Some(_))) {
54+
return data;
55+
}
56+
};
57+
}
58+
59+
// The `GitlabUpdater is a bit more permissive so better to put it at the end.
60+
return_if_ok_some!(GithubUpdater);
61+
return_if_ok_some!(GitlabUpdater);
62+
Ok(None)
63+
}
64+
65+
fn load_repository_inner<T: Updater>(
66+
conn: &mut Client,
67+
metadata: &MetadataPackage,
68+
config: Arc<Config>,
69+
db: Pool,
70+
) -> Result<Option<i32>> {
71+
let updater = match T::new(config, db)? {
72+
Some(updater) => updater,
73+
None => {
74+
return Ok(None);
75+
}
76+
};
77+
let repo = match &metadata.repository {
78+
Some(url) => url,
79+
None => {
80+
debug!(
81+
"did not collect {} stats as no repository URL was present",
82+
T::name()
83+
);
84+
return Ok(None);
85+
}
86+
};
87+
match updater.load_repository(conn, repo) {
88+
Ok(repo) => Ok(repo),
89+
Err(err) => {
90+
warn!("failed to collect {} stats: {}", T::name(), err);
91+
Ok(None)
92+
}
93+
}
94+
}
95+
96+
pub fn start_crons(config: Arc<Config>, context: &dyn Context) -> Result<()> {
97+
macro_rules! start_cron {
98+
($typ:ty) => {
99+
if let Some(updater) = <$typ>::new(config.clone(), context.pool()?)? {
100+
cron(
101+
concat!(stringify!($typ), " stats updater"),
102+
Duration::from_secs(60 * 60),
103+
move || {
104+
updater.update_all_crates()?;
105+
Ok(())
106+
},
107+
)?;
108+
} else {
109+
log::warn!(
110+
"{} stats updater not started as no token was provided",
111+
<$typ>::name()
112+
);
113+
}
114+
};
115+
}
116+
117+
start_cron!(GithubUpdater);
118+
start_cron!(GitlabUpdater);
119+
Ok(())
120+
}
121+
122+
pub fn update_all_crates(ctx: &dyn Context) -> Result<()> {
123+
fn inner<T: Updater>(ctx: &dyn Context) -> Result<()> {
124+
match T::new(ctx.config()?, ctx.pool()?)? {
125+
Some(up) => up.update_all_crates()?,
126+
None => warn!("missing {} token", T::name()),
127+
}
128+
Ok(())
129+
}
130+
131+
inner::<GithubUpdater>(ctx)?;
132+
inner::<GitlabUpdater>(ctx)?;
133+
Ok(())
134+
}
135+
136+
pub fn backfill_repositories(ctx: &dyn Context) -> Result<()> {
137+
fn inner<T: Updater>(ctx: &dyn Context) -> Result<()> {
138+
match T::new(ctx.config()?, ctx.pool()?)? {
139+
Some(up) => up.backfill_repositories()?,
140+
None => warn!("missing {} token", T::name()),
141+
}
142+
Ok(())
143+
}
144+
145+
inner::<GithubUpdater>(ctx)?;
146+
inner::<GitlabUpdater>(ctx)?;
147+
Ok(())
148+
}
149+
}
150+
35151
#[derive(Debug, Eq, PartialEq)]
36152
pub struct RepositoryName<'a> {
37153
pub owner: &'a str,

0 commit comments

Comments
 (0)