build: create a util directory (#3558)

* Introduces a new directory called `util` in the gulp setup. This should make the gulp setup more clear and maintainable.
* The utility `task_helpers` no longer includes random utilities like for Firebase. Those have been moved to their own util file (similar as in `e2e/`)
* Refactors the Github Status utility because it can be useful for other things as well (e.g upcoming coverage, payload)

Author: devversion

tools/gulp/tasks/aot.ts

@@ -1,7 +1,7 @@
 import {task} from 'gulp';
 import {join} from 'path';
 import {DIST_ROOT} from '../constants';
-import {execNodeTask, sequenceTask} from '../task_helpers';
+import {execNodeTask, sequenceTask} from '../util/task_helpers';
 
 /** Copies the source files of the demo-app to the dist folder. */
 task('aot:copy', [':build:devapp:scss', ':build:devapp:assets']);

tools/gulp/tasks/clean.ts

@@ -1,6 +1,6 @@
 import {task} from 'gulp';
 import {DIST_ROOT} from '../constants';
-import {cleanTask} from '../task_helpers';
+import {cleanTask} from '../util/task_helpers';
 
 
 task('clean', cleanTask(DIST_ROOT));

tools/gulp/tasks/components.ts

@@ -7,7 +7,7 @@ import {
 import {
   sassBuildTask, tsBuildTask, execNodeTask, copyTask, sequenceTask,
   triggerLivereload
-} from '../task_helpers';
+} from '../util/task_helpers';
 
 // No typings for these.
 const inlineResources = require('../../../scripts/release/inline-resources');

tools/gulp/tasks/coverage.ts

@@ -2,7 +2,8 @@ import {task} from 'gulp';
 import {existsSync} from 'fs-extra';
 import {COVERAGE_RESULT_FILE} from '../constants';
 import {spawnSync} from 'child_process';
-import {isTravisPushBuild, openFirebaseDashboardDatabase} from '../task_helpers';
+import {isTravisPushBuild} from '../util/travis-ci';
+import {openFirebaseDashboardDatabase} from '../util/firebase';
 
 task('coverage:upload', () => {
   if (!existsSync(COVERAGE_RESULT_FILE)) {

tools/gulp/tasks/development.ts

@@ -5,7 +5,7 @@ import {DIST_ROOT, SOURCE_ROOT} from '../constants';
 import {
   sassBuildTask, tsBuildTask, copyTask, buildAppTask, vendorTask,
   serverTask, sequenceTask, triggerLivereload
-} from '../task_helpers';
+} from '../util/task_helpers';
 
 
 const appDir = path.join(SOURCE_ROOT, 'demo-app');

tools/gulp/tasks/e2e.ts

@@ -5,7 +5,7 @@ import {SOURCE_ROOT, DIST_ROOT, PROJECT_ROOT} from '../constants';
 import {
   tsBuildTask, copyTask, buildAppTask, execNodeTask,
   vendorTask, sequenceTask, serverTask
-} from '../task_helpers';
+} from '../util/task_helpers';
 
 const gulpRunSequence = require('run-sequence');
 const gulpConnect = require('gulp-connect');

tools/gulp/tasks/lint.ts

@@ -1,5 +1,5 @@
 import gulp = require('gulp');
-import {execNodeTask} from '../task_helpers';
+import {execNodeTask} from '../util/task_helpers';
 
 gulp.task('lint', ['tslint', 'stylelint', 'madge']);
 

tools/gulp/tasks/payload.ts

@@ -2,8 +2,9 @@ import {task} from 'gulp';
 import {join} from 'path';
 import {statSync, readFileSync} from 'fs';
 import {DIST_COMPONENTS_ROOT} from '../constants';
-import {openFirebaseDashboardDatabase, isTravisPushBuild} from '../task_helpers';
 import {spawnSync} from 'child_process';
+import {isTravisPushBuild} from '../util/travis-ci';
+import {openFirebaseDashboardDatabase} from '../util/firebase';
 
 // Those imports lack types.
 const uglifyJs = require('uglify-js');

tools/gulp/tasks/release.ts

@@ -5,7 +5,7 @@ import gulpRunSequence = require('run-sequence');
 import path = require('path');
 import minimist = require('minimist');
 
-import {execTask, cleanTask} from '../task_helpers';
+import {execTask, cleanTask} from '../util/task_helpers';
 import {DIST_COMPONENTS_ROOT} from '../constants';
 
 const argv = minimist(process.argv.slice(3));

tools/gulp/tasks/screenshots.ts

@@ -2,8 +2,8 @@ import {task} from 'gulp';
 import {readdirSync, statSync, existsSync, mkdirp} from 'fs-extra';
 import * as path from 'path';
 import * as admin from 'firebase-admin';
-import {openScreenshotsBucket, openFirebaseScreenshotsDatabase} from '../task_helpers';
-import {updateGithubStatus} from '../util-functions';
+import {openScreenshotsBucket, openFirebaseScreenshotsDatabase} from '../util/firebase';
+import {setGithubStatus} from '../util/github';
 
 const imageDiff = require('image-diff');
 
@@ -19,7 +19,7 @@ task('screenshots', () => {
     return getScreenshotFiles(database)
       .then((files: any[]) => downloadAllGoldsAndCompare(files, database, prNumber))
       .then((results: boolean) => updateResult(database, prNumber, results))
-      .then((result: boolean) => updateGithubStatus(result, prNumber))
+      .then((result: boolean) => updateGithubStatus(prNumber, result))
       .then(() => uploadScreenshots('diff', prNumber))
       .then(() => uploadScreenshots('test', prNumber))
       .then(() => updateTravis(database, prNumber))
@@ -159,3 +159,13 @@ function setScreenFilenames(database: admin.database.Database,
     database.ref(FIREBASE_FILELIST);
   return filelistDatabase.set(filenames);
 }
+
+/** Updates the Github Status of the given Pullrequest. */
+function updateGithubStatus(prNumber: number, result: boolean) {
+  setGithubStatus(process.env['TRAVIS_PULL_REQUEST_SHA'], {
+    result: result,
+    name: 'Screenshot Tests',
+    description: `Screenshot Tests ${result ? 'passed' : 'failed'})`,
+    url: `http://material2-screenshots.firebaseapp.com/${prNumber}`
+  });
+}

tools/gulp/tasks/serve.ts

@@ -1,5 +1,5 @@
 import {task} from 'gulp';
-import {serverTask} from '../task_helpers';
+import {serverTask} from '../util/task_helpers';
 
 
 task('serve', serverTask());

tools/gulp/tasks/unit-test.ts

@@ -3,7 +3,7 @@ import path = require('path');
 import gulpMerge = require('merge2');
 
 import {PROJECT_ROOT, COMPONENTS_DIR} from '../constants';
-import {sequenceTask} from '../task_helpers';
+import {sequenceTask} from '../util/task_helpers';
 
 const karma = require('karma');
 const runSequence = require('run-sequence');

tools/gulp/util/firebase.ts

@@ -0,0 +1,61 @@
+const firebaseAdmin = require('firebase-admin');
+const gcloud = require('google-cloud');
+
+/** Opens a connection to the firebase realtime database. */
+export function openFirebaseDashboardDatabase() {
+  // Initialize the Firebase application with admin credentials.
+  // Credentials need to be for a Service Account, which can be created in the Firebase console.
+  firebaseAdmin.initializeApp({
+    credential: firebaseAdmin.credential.cert({
+      project_id: 'material2-dashboard',
+      client_email: 'firebase-adminsdk-ch1ob@material2-dashboard.iam.gserviceaccount.com',
+      // In Travis CI the private key will be incorrect because the line-breaks are escaped.
+      // The line-breaks need to persist in the service account private key.
+      private_key: (process.env['MATERIAL2_FIREBASE_PRIVATE_KEY'] || '').replace(/\\n/g, '\n')
+    }),
+    databaseURL: 'https://material2-dashboard.firebaseio.com'
+  });
+
+  return firebaseAdmin.database();
+}
+
+/**
+ * Open Google Cloud Storage for screenshots.
+ * The files uploaded to google cloud are also available to firebase storage.
+ */
+export function openScreenshotsBucket() {
+  let gcs = gcloud.storage({
+    projectId: 'material2-screenshots',
+    credentials: {
+      client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
+      private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'])
+    },
+  });
+
+  // Reference the existing appspot bucket.
+  return gcs.bucket('material2-screenshots.appspot.com');
+}
+
+/** Opens a connection to the firebase database for screenshots. */
+export function openFirebaseScreenshotsDatabase() {
+  // Initialize the Firebase application with admin credentials.
+  // Credentials need to be for a Service Account, which can be created in the Firebase console.
+  let screenshotApp = firebaseAdmin.initializeApp({
+    credential: firebaseAdmin.credential.cert({
+      project_id: 'material2-screenshots',
+      client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
+      private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'])
+    }),
+    databaseURL: 'https://material2-screenshots.firebaseio.com'
+  }, 'material2-screenshots');
+
+  return screenshotApp.database();
+}
+
+/** Decodes a Travis CI variable that is public in favor for PRs. */
+export function decode(str: string): string {
+  // In Travis CI the private key will be incorrect because the line-breaks are escaped.
+  // The line-breaks need to persist in the service account private key.
+  return (str || '').split('\\n').reverse().join('\\n').replace(/\\n/g, '\n');
+}
+

tools/gulp/util-functions.ts renamed to tools/gulp/util/github.ts

@@ -1,28 +1,37 @@
+import {MATERIAL_VERSION} from '../constants';
+
 const request = require('request');
 
-/** Update github pr status to success/failure */
-export function updateGithubStatus(result: boolean, prNumber: string) {
-  let state = result ? 'success' : 'failure';
-  let sha = process.env['TRAVIS_PULL_REQUEST_SHA'];
+/** Data that must be specified to set a Github PR status. */
+export type GithubStatusData = {
+  result: boolean;
+  name: string;
+  description: string;
+  url: string;
+};
+
+/** Function that sets a Github commit status */
+export function setGithubStatus(commitSHA: number, statusData: GithubStatusData) {
+  let state = statusData.result ? 'success' : 'failure';
   let token = decode(process.env['MATERIAL2_GITHUB_STATUS_TOKEN']);
 
   let data = JSON.stringify({
     state: state,
-    target_url: `http://material2-screenshots.firebaseapp.com/${prNumber}`,
-    context: 'screenshot-diff',
-    description: `Screenshot test ${state}`
+    target_url: statusData.url,
+    context: statusData.name,
+    description: statusData.description
   });
 
   let headers =  {
     'Authorization': `token ${token}`,
-    'User-Agent': 'ScreenshotDiff/1.0.0',
+    'User-Agent': `${statusData.name}/${MATERIAL_VERSION}`,
     'Content-Type': 'application/json',
     'Content-Length': Buffer.byteLength(data)
   };
 
   return new Promise((resolve) => {
     request({
-      url: `https://api.github.com/repos/angular/material2/statuses/${sha}`,
+      url: `https://api.github.com/repos/angular/material2/statuses/${commitSHA}`,
       method: 'POST',
       form: data,
       headers: headers

tools/gulp/task_helpers.ts renamed to tools/gulp/util/task_helpers.ts

@@ -2,7 +2,7 @@ import * as child_process from 'child_process';
 import * as fs from 'fs';
 import * as gulp from 'gulp';
 import * as path from 'path';
-import {NPM_VENDOR_FILES, PROJECT_ROOT, DIST_ROOT, SASS_AUTOPREFIXER_OPTIONS} from './constants';
+import {NPM_VENDOR_FILES, PROJECT_ROOT, DIST_ROOT, SASS_AUTOPREFIXER_OPTIONS} from '../constants';
 
 
 /** Those imports lack typings. */
@@ -14,8 +14,7 @@ const gulpSourcemaps = require('gulp-sourcemaps');
 const gulpAutoprefixer = require('gulp-autoprefixer');
 const gulpConnect = require('gulp-connect');
 const resolveBin = require('resolve-bin');
-const firebaseAdmin = require('firebase-admin');
-const gcloud = require('google-cloud');
+
 
 
 /** If the string passed in is a glob, returns it, otherwise append '**\/*' to it. */
@@ -185,66 +184,3 @@ export function sequenceTask(...args: any[]) {
     );
   };
 }
-
-/** Opens a connection to the firebase realtime database. */
-export function openFirebaseDashboardDatabase() {
-  // Initialize the Firebase application with admin credentials.
-  // Credentials need to be for a Service Account, which can be created in the Firebase console.
-  firebaseAdmin.initializeApp({
-    credential: firebaseAdmin.credential.cert({
-      project_id: 'material2-dashboard',
-      client_email: 'firebase-adminsdk-ch1ob@material2-dashboard.iam.gserviceaccount.com',
-      // In Travis CI the private key will be incorrect because the line-breaks are escaped.
-      // The line-breaks need to persist in the service account private key.
-      private_key: (process.env['MATERIAL2_FIREBASE_PRIVATE_KEY'] || '').replace(/\\n/g, '\n')
-    }),
-    databaseURL: 'https://material2-dashboard.firebaseio.com'
-  });
-
-  return firebaseAdmin.database();
-}
-
-/** Whether gulp currently runs inside of Travis as a push. */
-export function isTravisPushBuild() {
-  return process.env['TRAVIS_PULL_REQUEST'] === 'false';
-}
-
-/**
- * Open Google Cloud Storage for screenshots.
- * The files uploaded to google cloud are also available to firebase storage.
- */
-export function openScreenshotsBucket() {
-  let gcs = gcloud.storage({
-    projectId: 'material2-screenshots',
-    credentials: {
-      client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
-      private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'])
-    },
-  });
-
-  // Reference an existing bucket.
-  return gcs.bucket('material2-screenshots.appspot.com');
-}
-
-/** Opens a connection to the firebase realtime database for screenshots. */
-export function openFirebaseScreenshotsDatabase() {
-  // Initialize the Firebase application with admin credentials.
-  // Credentials need to be for a Service Account, which can be created in the Firebase console.
-  let screenshotApp = firebaseAdmin.initializeApp({
-    credential: firebaseAdmin.credential.cert({
-      project_id: 'material2-screenshots',
-      client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
-      private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'])
-    }),
-    databaseURL: 'https://material2-screenshots.firebaseio.com'
-  }, 'material2-screenshots');
-
-  return screenshotApp.database();
-}
-
-/** Decode the token for Travis to use. */
-function decode(str: string): string {
-  // In Travis CI the private key will be incorrect because the line-breaks are escaped.
-  // The line-breaks need to persist in the service account private key.
-  return (str || '').split('\\n').reverse().join('\\n').replace(/\\n/g, '\n');
-}

tools/gulp/util/travis-ci.ts

@@ -0,0 +1,4 @@
+/** Whether gulp currently runs inside of Travis as a push. */
+export function isTravisPushBuild() {
+  return process.env['TRAVIS_PULL_REQUEST'] === 'false';
+}