build: share release output between circle jobs (#13838)

Also support parallel Karma instances on CircleCI

Author: devversion

.circleci/config.yml

@@ -60,6 +60,12 @@ var_9: &docker-firefox-image
   # See the PR that fixes this: https://github.com/angular/angular/pull/26435
   - image: circleci/node:10.12-browsers
 
+# Attaches the release output which has been stored in the workspace to the current job.
+# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
+var_10: &attach_release_output
+  attach_workspace:
+      at: dist/releases
+
 # -----------------------------
 # Container version of CircleCI
 # -----------------------------
@@ -86,8 +92,6 @@ jobs:
       - run: bazel build src/...
       - run: bazel test src/...
 
-      - *save_cache
-
   # ------------------------------------------------------------------------------------------
   # Job that runs the unit tests on locally installed browsers (Chrome and Firefox headless).
   # The available browsers are installed through the angular/ngcontainer Docker image.
@@ -106,15 +110,11 @@ jobs:
       # variable which has been configured above
       - run: yarn gulp ci:test
 
-      - *save_cache
-
   # ----------------------------------------------------------------
-  # Job that runs the e2e tests with Protractor and Chrome w/ Xvfb. We cannot use
-  # Chrome headless because our tests rely on APIs which are not testable in headless mode.
-  # For example: the Fullscreen browser API.
+  # Job that runs the e2e tests with Protractor and Chrome Headless
   # ----------------------------------------------------------------
   e2e_tests:
-    docker: *docker-firefox-image
+    <<: *job_defaults
     resource_class: xlarge
     steps:
       - *checkout_code
@@ -123,8 +123,6 @@ jobs:
 
       - run: yarn gulp ci:e2e
 
-      - *save_cache
-
   # ----------------------------------------------------------------------------
   # Job that runs the unit tests on Browserstack. The browsers that will be used
   # to run the unit tests on Browserstack are set in: test/browser-providers.js
@@ -135,46 +133,41 @@ jobs:
     environment:
       BROWSER_STACK_USERNAME: "angularteam1"
       BROWSER_STACK_ACCESS_KEY: "CaXMeMHD9pr5PHg8N7Jq"
+    parallelism: 2
     steps:
       - *checkout_code
       - *restore_cache
       - *yarn_install
 
       - run: ./scripts/circleci/run-browserstack-tests.sh
 
-      - *save_cache
-
-  # --------------------------------------
-  # Job that builds the demo-app with AOT
-  # --------------------------------------
-  demo_app_aot:
+  # -----------------------------------------------------------------------------------------
+  # Job that builds the demo-app with AOT. In order to speed up this job, the release output
+  # from the workspace storage will be attached to this job.
+  # -----------------------------------------------------------------------------------------
+  build_demoapp_aot:
     <<: *job_defaults
-    resource_class: xlarge
     steps:
       - *checkout_code
       - *restore_cache
       - *yarn_install
+      - *attach_release_output
 
       - run: yarn gulp ci:aot
 
-      - *save_cache
-
   # -------------------------------------------------------------------------
   # Job that pre-render's the universal app with `@angular/platform-server`.
   # This verifies that Angular Material can be rendered within Node.
   # -------------------------------------------------------------------------
   prerender_build:
     <<: *job_defaults
-    resource_class: xlarge
     steps:
     - *checkout_code
     - *restore_cache
     - *yarn_install
 
     - run: yarn gulp ci:prerender
 
-    - *save_cache
-
   # ----------------------------------
   # Lint job. Runs the gulp lint task.
   # ----------------------------------
@@ -187,6 +180,27 @@ jobs:
 
       - run: yarn gulp ci:lint
 
+  # -------------------------------------------------------------------------------------------
+  # Job that builds all release packages with Gulp. The built packages can be then used in the
+  # same workflow to publish snapshot builds or test the demo-app with the release packages.
+  # -------------------------------------------------------------------------------------------
+  build_release_packages:
+    <<: *job_defaults
+    resource_class: xlarge
+    steps:
+      - *checkout_code
+      - *restore_cache
+      - *yarn_install
+
+      - run: yarn gulp ci:build-release-packages
+
+      # Store the release output in the workspace storage. This means that other jobs
+      # in the same workflow can attach the release output to their job.
+      - persist_to_workspace:
+          root: dist/releases
+          paths:
+            - "**/*"
+
       - *save_cache
 
 # ----------------------------------------------------------------------------------------
@@ -214,7 +228,13 @@ workflows:
     jobs:
       - e2e_tests
       - prerender_build
-      - demo_app_aot
+
+  release_output:
+    jobs:
+      - build_release_packages
+      - build_demoapp_aot:
+          requires:
+            - build_release_packages
 
   # Lint workflow. As we want to lint in one job, this is a workflow with just one job.
   lint:

scripts/browserstack/start-tunnel.sh

@@ -31,7 +31,7 @@ rm ${tunnelFileName}
 ARGS=""
 
 if [ ! -z "${CIRCLE_BUILD_NUM}" ]; then
-  ARGS="${ARGS} --local-identifier ${CIRCLE_BUILD_NUM}"
+  ARGS="${ARGS} --local-identifier ${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX}"
 fi
 
 echo "Starting Browserstack Local in the background, logging into: ${tunnelLogFile}"

src/demo-app/tsconfig-aot.json

@@ -14,6 +14,8 @@
     "outDir": "../../dist/packages/demo-app",
     "rootDirs": [
       ".",
+      // Include the package output here because otherwise NGC won't be able to load
+      // the SCSS files.
       "../../dist/packages/demo-app"
     ],
     "paths": {
@@ -26,7 +28,7 @@
       "@angular/cdk-experimental/*": ["../../dist/releases/cdk-experimental/*"],
       "@angular/cdk-experimental": ["../../dist/releases/cdk-experimental"],
       "@angular/material-moment-adapter": ["../../dist/releases/material-moment-adapter"],
-      "@angular/material-examples": ["../../dist/packages/material-examples"]
+      "@angular/material-examples": ["../../dist/releases/material-examples"]
     }
   },
   "files": [

test/karma.conf.js

@@ -101,7 +101,9 @@ module.exports = (config) => {
   });
 
   if (process.env['CIRCLECI']) {
-    const tunnelIdentifier = process.env['CIRCLE_BUILD_NUM'];
+    const instanceIndex = Number(process.env['CIRCLE_NODE_INDEX']);
+    const maxParallelInstances = Number(process.env['CIRCLE_NODE_TOTAL']);
+    const tunnelIdentifier = `${process.env['CIRCLE_BUILD_NUM']}-${instanceIndex}`;
     const buildIdentifier = `angular-material-${tunnelIdentifier}`;
     const testPlatform = process.env['TEST_PLATFORM'];
 
@@ -110,8 +112,13 @@ module.exports = (config) => {
       config.browserStack.tunnelIdentifier = tunnelIdentifier;
     }
 
-    // Configure Karma launch the browsers that belong to the given test platform.
-    config.browsers = platformMap[testPlatform];
+    const platformBrowsers = platformMap[testPlatform];
+    const browserInstanceChunks = splitBrowsersIntoInstances(
+        platformBrowsers, maxParallelInstances);
+
+    // Configure Karma to launch the browsers that belong to the given test platform
+    // and instance.
+    config.browsers = browserInstanceChunks[instanceIndex];
   }
 
   if (process.env['TRAVIS']) {
@@ -150,3 +157,21 @@ module.exports = (config) => {
     config.browsers = platformMap[platform];
   }
 };
+
+/**
+ * Splits the specified browsers into a maximum amount of chunks. The chunk of browsers
+ * are being created deterministically and therefore we get reproducible tests when executing
+ * the same CircleCI instance multiple times.
+ */
+function splitBrowsersIntoInstances(browsers, maxInstances) {
+  let chunks = [];
+  let assignedBrowsers = 0;
+
+  for (let i = 0; i < maxInstances; i++) {
+    const chunkSize = Math.floor((browsers.length - assignedBrowsers) / (maxInstances - i));
+    chunks[i] = browsers.slice(assignedBrowsers, assignedBrowsers + chunkSize);
+    assignedBrowsers += chunkSize;
+  }
+
+  return chunks;
+}

tools/gulp/packages.ts

@@ -24,3 +24,13 @@ cdkPackage.copySecondaryEntryPointStylesToRoot = true;
 // Build and copy the schematics of the CDK and Material package.
 cdkPackage.hasSchematics = true;
 materialPackage.hasSchematics = true;
+
+/** List of all build packages defined for this project. */
+export const allBuildPackages = [
+  cdkPackage,
+  materialPackage,
+  cdkExperimentalPackage,
+  materialExperimentalPackage,
+  momentAdapterPackage,
+  examplesPackage
+];

tools/gulp/tasks/aot.ts

@@ -11,8 +11,24 @@ const demoAppSource = join(packagesDir, 'demo-app');
 /** Path to the tsconfig file that builds the AOT files. */
 const tsconfigFile = join(demoAppSource, 'tsconfig-aot.json');
 
+/**
+ * Build the demo-app wit the release output in order confirm that the library is
+ * working with AOT compilation enabled.
+ */
+task('build-aot', sequenceTask(
+  'clean',
+  ['build-aot:release-packages', 'build-aot:assets'],
+  'build-aot:compiler-cli'
+));
+
+/**
+ * Task that can be used to build the demo-app with AOT without building the
+ * release output. This can be run if the release output is already built.
+ */
+task('build-aot:no-release-build', sequenceTask('build-aot:assets', 'build-aot:compiler-cli'));
+
 /** Builds the demo-app assets and builds the required release packages. */
-task('aot:deps', sequenceTask(
+task('build-aot:release-packages', sequenceTask(
   [
     'cdk:build-release',
     'material:build-release',
@@ -21,15 +37,15 @@ task('aot:deps', sequenceTask(
     'material-moment-adapter:build-release',
     'material-examples:build-release',
   ],
-  // Build the assets after the releases have been built, because the demo-app assets import
-  // SCSS files from the release packages.
-  [':build:devapp:assets', ':build:devapp:scss'],
 ));
 
-/** Build the demo-app and a release to confirm that the library is AOT-compatible. */
-task('aot:build', sequenceTask('clean', 'aot:deps', 'aot:compiler-cli'));
+/**
+ * Task that builds the assets which are required for building with AOT. Since the demo-app uses
+ * Sass files, we need to provide the transpiled CSS sources in the package output.
+ */
+task('build-aot:assets', [':build:devapp:assets', ':build:devapp:scss']);
 
 /** Build the demo-app and a release to confirm that the library is AOT-compatible. */
-task('aot:compiler-cli', execNodeTask(
+task('build-aot:compiler-cli', execNodeTask(
   '@angular/compiler-cli', 'ngc', ['-p', tsconfigFile]
 ));

tools/gulp/tasks/ci.ts

@@ -1,5 +1,6 @@
 import {task} from 'gulp';
-
+import {sequenceTask} from 'material2-build-tools';
+import {allBuildPackages} from '../packages';
 
 task('ci:lint', ['lint']);
 
@@ -8,8 +9,11 @@ task('ci:test', ['test:single-run'], () => process.exit(0));
 
 task('ci:e2e', ['e2e']);
 
-/** Task to verify that all components work with AOT compilation. */
-task('ci:aot', ['aot:build']);
+/**
+ * Task to verify that all components work with AOT compilation. This task requires the
+ * release output to be built already.
+ */
+task('ci:aot', ['build-aot:no-release-build']);
 
 /** Task which reports the size of the library and stores it in a database. */
 task('ci:payload', ['payload']);
@@ -19,3 +23,9 @@ task('ci:coverage', ['coverage:upload']);
 
 /** Task that verifies if all Material components are working with platform-server. */
 task('ci:prerender', ['prerender']);
+
+/** Task that builds all release packages. */
+task('ci:build-release-packages', sequenceTask(
+  'clean',
+  allBuildPackages.map(buildPackage => `${buildPackage.name}:build-release`)
+));