extend and improve data-collection for release-activity graph

Author: syphar

src/db/migrate.rs

@@ -618,6 +618,28 @@ pub fn migrate(version: Option<Version>, conn: &mut Client) -> CratesfyiResult<(
             DROP INDEX github_repos_stars_idx;
             ",
         ),
+        migration!(
+            context, 
+            27, 
+            "create materialized view for release-statistics", 
+            "
+            CREATE MATERIALIZED VIEW releases_statistics AS 
+            SELECT
+                release_time::date as date,
+                count(*) AS counts,
+                count(CASE WHEN is_library = TRUE AND build_status = FALSE THEN 1 ELSE 0 END) AS failures
+            FROM
+                releases
+            GROUP BY
+                release_time::date
+            ;
+            CREATE INDEX releases_statistics_date_idx ON releases_statistics (date);
+            ",
+            "
+            DROP INDEX releases_statistics_date_idx;
+            DROP MATERIALIZED VIEW releases_statistics;
+            ",
+        ),
     ];
 
     for migration in migrations {

src/utils/release_activity_updater.rs

@@ -1,70 +1,8 @@
 use crate::error::Result;
-use chrono::{Duration, Utc};
 use postgres::Client;
-use serde_json::{Map, Value};
 
 pub fn update_release_activity(conn: &mut Client) -> Result<()> {
-    let mut dates = Vec::with_capacity(30);
-    let mut crate_counts = Vec::with_capacity(30);
-    let mut failure_counts = Vec::with_capacity(30);
-
-    for day in 0..30 {
-        let rows = conn.query(
-            format!(
-                "SELECT COUNT(*)
-                 FROM releases
-                 WHERE release_time < NOW() - INTERVAL '{} day' AND
-                       release_time > NOW() - INTERVAL '{} day'",
-                day,
-                day + 1
-            )
-            .as_str(),
-            &[],
-        )?;
-        let failures_count_rows = conn.query(
-            format!(
-                "SELECT COUNT(*)
-                 FROM releases
-                 WHERE is_library = TRUE AND
-                       build_status = FALSE AND
-                       release_time < NOW() - INTERVAL '{} day' AND
-                       release_time > NOW() - INTERVAL '{} day'",
-                day,
-                day + 1
-            )
-            .as_str(),
-            &[],
-        )?;
-
-        let release_count: i64 = rows[0].get(0);
-        let failure_count: i64 = failures_count_rows[0].get(0);
-        let now = Utc::now().naive_utc();
-        let date = now - Duration::days(day);
-
-        dates.push(format!("{}", date.format("%d %b")));
-        crate_counts.push(release_count);
-        failure_counts.push(failure_count);
-    }
-
-    dates.reverse();
-    crate_counts.reverse();
-    failure_counts.reverse();
-
-    let map = {
-        let mut map = Map::new();
-        map.insert("dates".to_owned(), serde_json::to_value(dates)?);
-        map.insert("counts".to_owned(), serde_json::to_value(crate_counts)?);
-        map.insert("failures".to_owned(), serde_json::to_value(failure_counts)?);
-
-        Value::Object(map)
-    };
-
-    conn.query(
-        "INSERT INTO config (name, value) VALUES ('release_activity', $1)
-         ON CONFLICT (name) DO UPDATE
-            SET value = $1 WHERE config.name = 'release_activity'",
-        &[&map],
-    )?;
+    conn.execute("REFRESH MATERIALIZED VIEW releases_statistics", &[])?;
 
     Ok(())
 }

src/web/releases.rs

@@ -7,7 +7,7 @@ use crate::{
     web::{error::Nope, match_version, page::WebPage, redirect_base},
     BuildQueue, Config,
 };
-use chrono::{DateTime, Utc};
+use chrono::{DateTime, NaiveDate, Utc};
 use iron::{
     headers::{ContentType, Expires, HttpDate},
     mime::{Mime, SubLevel, TopLevel},
@@ -17,7 +17,7 @@ use iron::{
 use postgres::Client;
 use router::Router;
 use serde::Serialize;
-use serde_json::Value;
+use serde_json::{json, Value};
 
 /// Number of release in home page
 const RELEASES_IN_HOME: i64 = 15;
@@ -668,20 +668,34 @@ impl_webpage! {
 
 pub fn activity_handler(req: &mut Request) -> IronResult<Response> {
     let mut conn = extension!(req, Pool).get()?;
-    let activity_data: Value = ctry!(
+
+    let data: Vec<(NaiveDate, i64, i64)> = ctry!(
         req,
         conn.query(
-            "SELECT value FROM config WHERE name = 'release_activity'",
-            &[]
-        ),
+            "SELECT 
+                date,
+                counts, 
+                failures 
+            FROM releases_statistics
+            WHERE date >= CURRENT_DATE - INTERVAL '30 days' 
+            ORDER BY date DESC
+            ",
+            &[],
+        )
     )
-    .iter()
-    .next()
-    .map_or(Value::Null, |row| row.get("value"));
+    .into_iter()
+    .map(|row| (row.get(0), row.get(1), row.get(2)))
+    .collect();
 
     ReleaseActivity {
         description: "Monthly release activity",
-        activity_data,
+        activity_data: json!(
+            {
+                "dates": data.iter().map(|&d| d.0.format("%d %b").to_string()).collect::<Vec<_>>(),
+                "counts": data.iter().map(|&d| d.1).collect::<Vec<_>>(),
+                "failures": data.iter().map(|&d| d.2).collect::<Vec<_>>(),
+            }
+        ),
     }
     .into_response(req)
 }