[patch port] ci: setup automatic deployment for the docs app (#24552)

* ci: setup automatic deployment for the docs app

See the following document for the overall plan that this
commit implements:

https://docs.google.com/document/d/1xkrSOFa6WeFqyg1cTwMhl_wB8ygbVwdSxr3K2-cps14/edit

* build: do not reserve `GOOGLE_APPLICATION_CREDENTIALS` for RBE auth

We currently configure RBE by setting `GOOGLE_APPLICATION_CREDENTIALS`
into the `$BASH_ENV` variable, ensuring RBE is configured everywhere on CI.

This worked nicely but now with automatic docs deployment turned out to
be problematic since it prevents scripts from defining `GOOGLE_APPLICATION_CREDENTIALS`
themselves/overriding it. the reason is that `$BASH_ENV` always runs in new
child processes (like when firebase is initialized) and then overrides the credentials
back to the RBE service key.

We can simplify this code by using a dedicated Bazel flag.

Author: devversion

.circleci/config.yml

@@ -86,8 +86,6 @@ var_14: &publish_branches_filter
       - master
       # 6.0.x, 7.1.x, etc.
       - /\d+\.\d+\.x/
-      # 6.x, 7.x, 8.x etc
-      - /\d+\.x/
 
 # Branch filter that is usually applied to all jobs. Since there is no way within CircleCI to
 # exclude a branch for all defined jobs, we need to manually specify the filters for each job.
@@ -395,6 +393,42 @@ jobs:
       - store_artifacts:
           path: dist/release-archives
 
+  # ----------------------------------------
+  # Job that publishes the docs site
+  # ----------------------------------------
+  deploy_docs_site:
+    docker:
+      - image: *docker-browser-image
+    resource_class: xlarge
+    environment:
+      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
+    steps:
+      - checkout_and_rebase
+      - *restore_cache
+      - *setup_bazel_ci_config
+      - *setup_bazel_remote_execution
+      - *yarn_install
+      - *setup_bazel_binary
+
+      - run: yarn ci-push-deploy-docs-app
+      - *slack_notify_on_failure
+
+  # ----------------------------------------
+  # Job that monitors the docs site, ensuring
+  # the docs site is online and works as expected. 
+  # ----------------------------------------
+  monitor_docs_site:
+    docker:
+      - image: *docker-browser-image
+    resource_class: xlarge
+    steps:
+      - checkout_and_rebase
+      - *restore_cache
+      - *yarn_install
+
+      - run: yarn ci-docs-monitor-test
+      - *slack_notify_on_failure
+
   # ----------------------------------------
   # Job that publishes the build snapshots
   # ----------------------------------------
@@ -609,6 +643,12 @@ workflows:
           filters: *publish_branches_filter
           requires:
             - build_release_packages
+      - deploy_docs_site:
+          filters: *publish_branches_filter
+          requires:
+            - lint
+            - build_release_packages
+            - tests_local_browsers
 
   # Snapshot tests workflow that is scheduled to run all specified jobs every hour.
   # This workflow runs various jobs against the Angular snapshot builds from Github.
@@ -623,6 +663,8 @@ workflows:
           filters: *only_main_branch_filter
       - snapshot_linker_tests:
           filters: *only_main_branch_filter
+      - monitor_docs_site:
+          filters: *only_main_branch_filter
 
     triggers:
       - schedule:

package.json

@@ -17,7 +17,7 @@
     "postinstall": "node tools/postinstall/apply-patches.js",
     "build": "ts-node --project scripts/tsconfig.json ./scripts/build-packages-dist.ts",
     "build-and-check-release-output": "ts-node --project scripts/tsconfig.json scripts/build-and-check-release-output.ts",
-    "build-docs-content": "node ./scripts/build-docs-content.js",
+    "build-docs-content": "ts-node --project scripts/tsconfig.json ./scripts/build-docs-content.ts",
     "dev-app": "ibazel run //src/dev-app:devserver",
     "test": "node ./scripts/run-component-tests.js",
     "test-local": "yarn -s test --local",
@@ -48,6 +48,8 @@
     "check-mdc-exports": "ts-node --project scripts/tsconfig.json scripts/check-mdc-exports.ts",
     "check-tooling-setup": "yarn tsc --project tools/tsconfig.json && yarn tsc --project .ng-dev/tsconfig.json",
     "tsc": "node ./node_modules/typescript/bin/tsc",
+    "ci-push-deploy-docs-app": "ts-node --project scripts/tsconfig.json scripts/docs-deploy/deploy-ci-push.ts",
+    "ci-docs-monitor-test": "ts-node --project scripts/tsconfig.json scripts/docs-deploy/monitoring/ci-test.ts",
     "prepare": "husky install"
   },
   "version": "13.2.6",
@@ -73,7 +75,7 @@
     "@angular/bazel": "13.2.0",
     "@angular/cli": "13.2.0",
     "@angular/compiler-cli": "13.2.0",
-    "@angular/dev-infra-private": "https://github.com/angular/dev-infra-private-builds.git#b6656cffbd46bb3637b09b08561e5f1a147603c9",
+    "@angular/dev-infra-private": "https://github.com/angular/dev-infra-private-builds.git#7544378ad9aa94cdfe256aaaa923c107f45175a2",
     "@angular/localize": "13.2.0",
     "@angular/platform-browser-dynamic": "13.2.0",
     "@angular/platform-server": "13.2.0",
@@ -217,7 +219,8 @@
     "typescript-4.4": "npm:[email protected]",
     "vrsource-tslint-rules": "6.0.0",
     "yaml": "^1.10.2",
-    "yargs": "^17.3.1"
+    "yargs": "^17.3.1",
+    "zx": "^4.3.0"
   },
   "resolutions": {
     "@angular/dev-infra-private/typescript": "~4.5.2",

scripts/bazel/setup-remote-execution.sh

@@ -11,21 +11,10 @@ fi
 
 # Decode the GCP token that is needed to authenticate the Bazel remote execution.
 openssl aes-256-cbc -d -in scripts/bazel/gcp_token -md md5 -k ${GCP_DECRYPT_TOKEN} \
-  -out $HOME/.gcp_credentials
-
-# Set the "GOOGLE_APPLICATION_CREDENTIALS" environment variable. It should point to the GCP credentials
-# file. Bazel will then automatically picks up the credentials from that variable.
-# https://docs.bazel.build/versions/main/command-line-reference.html#flag--google_default_credentials
-# https://cloud.google.com/docs/authentication/production.
-if [[ ! -z "${BASH_ENV}" ]]; then
-  # CircleCI uses the `BASH_ENV` variable for environment variables.
-  echo "export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/.gcp_credentials" >> ${BASH_ENV}
-elif [[ ! -z "${GITHUB_ENV}" ]]; then
-  # Github actions use the `GITHUB_ENV` variable for environment variables.
-  echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/.gcp_credentials" >> ${GITHUB_ENV}
-fi
+  -out $HOME/.gcp_rbe_credentials
 
 # Update the project Bazel configuration to always use remote execution.
 # Note: We add the remote config flag to the user bazelrc file that is not tracked
 # by Git. This is necessary to avoid stamping builds with `.with-local-changes`.
 echo "build --config=remote" >> .bazelrc.user
+echo "build:remote --google_credentials=\"$HOME/.gcp_rbe_credentials\"" >> .bazelrc.user

scripts/build-docs-content.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+/**
+ * Script that builds the docs content NPM package and moves it into a conveniently
+ * accessible distribution directory (the project `dist/` directory).
+ */
+
+import {cd, chmod, cp, exec, mkdir, rm, set} from 'shelljs';
+
+import {BuiltPackage} from '@angular/dev-infra-private/ng-dev';
+import {join} from 'path';
+
+/** Path to the project directory. */
+const projectDir = join(__dirname, '../');
+
+/** Path to the distribution directory. */
+const distDir = join(projectDir, 'dist/');
+
+/**
+ * Path to the directory where the docs-content package is copied to. Note: When
+ * changing the path, also change the path in the docs-content deploy script.
+ */
+const outputDir = join(distDir, 'docs-content-pkg');
+
+/** Command that runs Bazel. */
+const bazelCmd = process.env.BAZEL || `yarn -s bazel`;
+
+/**
+ * Builds the docs content NPM package in snapshot mode.
+ *
+ * @returns an object describing the built package and its output path.
+ */
+export function buildDocsContentPackage(): BuiltPackage {
+  // ShellJS should exit if a command fails.
+  set('-e');
+
+  // Go to project directory.
+  cd(projectDir);
+
+  /** Path to the bazel bin output directory. */
+  const bazelBinPath = exec(`${bazelCmd} info bazel-bin`).stdout.trim();
+
+  /** Path where the NPM package is built into by Bazel. */
+  const bazelBinOutDir = join(bazelBinPath, 'src/components-examples/npm_package');
+
+  // Clean the output directory to ensure that the docs-content package
+  // will not contain outdated files from previous builds.
+  rm('-rf', outputDir);
+  mkdir('-p', distDir);
+
+  // Build the docs-content package with the snapshot-build mode. That will help
+  // determining which commit is associated with the built docs-content.
+  exec(`${bazelCmd} build src/components-examples:npm_package --config=snapshot-build`);
+
+  // Copy the package output into the dist path. Also update the permissions
+  // as Bazel by default marks files in the bazel-out as readonly.
+  cp('-R', bazelBinOutDir, outputDir);
+  chmod('-R', 'u+w', outputDir);
+
+  return {
+    name: '@angular/components-examples',
+    outputPath: outputDir,
+  };
+}
+
+if (require.main === module) {
+  const builtPackage = buildDocsContentPackage();
+  console.info(`Built docs-content into: ${builtPackage.outputPath}`);
+}

scripts/build-docs-content.ts

@@ -21,7 +21,7 @@ const releaseTargetTag = 'release-package';
 const projectDir = join(__dirname, '../');
 
 /** Command that runs Bazel. */
-const bazelCmd = process.env.BAZEL_COMMAND || `bazel`;
+const bazelCmd = process.env.BAZEL || `yarn -s bazel`;
 
 /** Command that queries Bazel for all release package targets. */
 const queryPackagesCmd =
@@ -69,7 +69,8 @@ function buildReleasePackages(distPath: string, isSnapshotBuild: boolean): Built
   const targets = exec(queryPackagesCmd, true).split(/\r?\n/);
   const packageNames = getPackageNamesOfTargets(targets);
   const bazelBinPath = exec(`${bazelCmd} info bazel-bin`, true);
-  const getOutputPath = (pkgName: string) => join(bazelBinPath, 'src', pkgName, 'npm_package');
+  const getBazelOutputPath = (pkgName: string) => join(bazelBinPath, 'src', pkgName, 'npm_package');
+  const getDistPath = (pkgName: string) => join(distPath, pkgName);
 
   // Build with "--config=release" or `--config=snapshot-build` so that Bazel
   // runs the workspace stamping script. The stamping script ensures that the
@@ -80,7 +81,7 @@ function buildReleasePackages(distPath: string, isSnapshotBuild: boolean): Built
   // a workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1219. We need to
   // do this to ensure that the version placeholders are properly populated.
   packageNames.forEach(pkgName => {
-    const outputPath = getOutputPath(pkgName);
+    const outputPath = getBazelOutputPath(pkgName);
     if (test('-d', outputPath)) {
       chmod('-R', 'u+w', outputPath);
       rm('-rf', outputPath);
@@ -96,18 +97,17 @@ function buildReleasePackages(distPath: string, isSnapshotBuild: boolean): Built
 
   // Copy the package output into the specified distribution folder.
   packageNames.forEach(pkgName => {
-    const outputPath = getOutputPath(pkgName);
-    const targetFolder = join(distPath, pkgName);
+    const outputPath = getBazelOutputPath(pkgName);
+    const targetFolder = getDistPath(pkgName);
     console.log(`> Copying package output to "${targetFolder}"`);
     cp('-R', outputPath, targetFolder);
     chmod('-R', 'u+w', targetFolder);
   });
 
   return packageNames.map(pkg => {
-    const outputPath = getOutputPath(pkg);
     return {
       name: `@angular/${pkg}`,
-      outputPath,
+      outputPath: getDistPath(pkg),
     };
   });
 }

scripts/build-packages-dist.ts

@@ -24,7 +24,7 @@ docsDistPath="${projectPath}/dist/docs"
 docsContentPath="${projectPath}/tmp/material2-docs-content"
 
 # Path to the build output of the Bazel "@angular/components-examples" NPM package.
-# Note: When changing this, also change the path in `scripts/build-docs-content.js`.
+# Note: When changing this, also change the path in `scripts/build-docs-content.ts`.
 examplesPackagePath="${projectPath}/dist/docs-content-pkg/"
 
 # Git clone URL for the material2-docs-content repository.

scripts/deploy/publish-docs-content.sh

@@ -0,0 +1,3 @@
+### Docs app deployment
+
+https://docs.google.com/document/d/1xkrSOFa6WeFqyg1cTwMhl_wB8ygbVwdSxr3K2-cps14/edit?usp=sharing

scripts/docs-deploy/README.md

@@ -0,0 +1,63 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+import {$} from 'zx';
+import {projectDir} from './utils';
+
+/** Git repository HTTP url pointing to the docs repository. */
+export const docsRepoUrl = 'https://github.com/angular/material.angular.io.git';
+
+/**
+ * Clones the docs repository for the given major into a
+ * temporary directory.
+ *
+ * @returns An absolute path to the temporary directory.
+ */
+export async function cloneDocsRepositoryForMajor(major: number): Promise<string> {
+  const repoTmpDir = path.join(projectDir, 'tmp/docs-repo');
+  const baseCloneArgs = [docsRepoUrl, repoTmpDir, '--single-branch', '--depth=1'];
+  const majorDocsBranchName = getDocsBranchNameForMajor(major);
+
+  // Cleanup the temporary repository directory if it exists.
+  try {
+    await fs.promises.rm(repoTmpDir, {recursive: true});
+  } catch {}
+
+  // Clone the docs app (either the main branch, or a dedicated major branch if available).
+  if (await hasUpstreamDocsBranch(majorDocsBranchName)) {
+    console.log(`Cloning docs app with dedicated branch: ${majorDocsBranchName}`);
+    await $`git clone ${baseCloneArgs} --branch=${majorDocsBranchName}`;
+  } else {
+    console.log(`Cloning docs app with default branch (no dedicated branch for major).`);
+    await $`git clone ${baseCloneArgs}`;
+  }
+
+  return repoTmpDir;
+}
+
+/**
+ * Gets whether the specified branch exists in the specified remote URL.
+ */
+async function hasUpstreamDocsBranch(branchName: string): Promise<boolean> {
+  try {
+    const proc = await $`git ls-remote ${docsRepoUrl} refs/heads/${branchName}`;
+    return proc.stdout.trim() !== '';
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Gets the name of a potential dedicated branch for this major in the
+ * docs repository.
+ *
+ * e.g. if a branch like `13.x` exists and we intend to deploy v13, then
+ * this branch can be used as revision for the docs-app.
+ *
+ * More details on why this is preferred:
+ * https://docs.google.com/document/d/1xkrSOFa6WeFqyg1cTwMhl_wB8ygbVwdSxr3K2-cps14/edit#heading=h.nsf3ag63jpwu.
+ */
+function getDocsBranchNameForMajor(major: number): string {
+  return 'firebase-target';
+  // TODO return `${major}.x`;
+}