build: run unit tests with partial compilation and Angular linker (#22979)

Instead of running e2e tests against a linker-processed application, we
run all of our unit tests using partial compilation and the Angular
linker. This ensures we test our components more throroughly against
the Angular linker & partial compilation. This also solves our CI
flakiness issue due to the large amount of webdriver e2e tests running
concurrently on CI (especially since all e2e tests ran on snapshot
builds).

Author: devversion

.circleci/config.yml

@@ -197,8 +197,7 @@ jobs:
 
       # Exclude release and docs packages here as those will be built within
       # the "build_release_packages" and "publish_snapshots" jobs.
-      # TODO(devversion): remove target exclusion once https://github.com/bazelbuild/rules_nodejs/pull/2646 is available.
-      - run: bazel build --build_tag_filters=-docs-package,-release-package -- src/... -//src/e2e-app:devserver_with_linked_declarations.MF
+      - run: bazel build --build_tag_filters=-docs-package,-release-package -- src/...
       - *slack_notify_on_failure
 
   # -----------------------------------
@@ -220,8 +219,7 @@ jobs:
 
       # Exclude release and docs packages here as those will be built within
       # the "build_release_packages" and "publish_snapshots" jobs.
-      # TODO(devversion): remove target exclusion once https://github.com/bazelbuild/rules_nodejs/pull/2646 is available.
-      - run: bazel build --build_tag_filters=-docs-package,-release-package --config=view-engine -- src/... -//src/e2e-app:devserver_with_linked_declarations.MF
+      - run: bazel build --build_tag_filters=-docs-package,-release-package --config=view-engine -- src/...
       - *slack_notify_on_failure
 
   # --------------------------------------------------------------------------------------------
@@ -283,8 +281,7 @@ jobs:
       - *yarn_install
       - *setup_bazel_binary
 
-      # TODO(devversion): remove target exclusion once https://github.com/bazelbuild/rules_nodejs/pull/2646 is available.
-      - run: bazel test --build_tag_filters=-e2e --test_tag_filters=-e2e --build_tests_only -- src/... -//src/e2e-app:devserver_with_linked_declarations.MF
+      - run: bazel test --build_tag_filters=-e2e --test_tag_filters=-e2e --build_tests_only -- src/...
       - *slack_notify_on_failure
 
   # ----------------------------------------------------------------------------
@@ -536,8 +533,7 @@ jobs:
     - *yarn_install_loose_lockfile
     - *setup_bazel_binary
 
-    # TODO(devversion): remove target exclusion once https://github.com/bazelbuild/rules_nodejs/pull/2646 is available.
-    - run: bazel test --build_tag_filters=-e2e --test_tag_filters=-e2e --build_tests_only -- src/... -//src/e2e-app:devserver_with_linked_declarations.MF
+    - run: bazel test --build_tag_filters=-e2e --test_tag_filters=-e2e --build_tests_only -- src/...
     - *slack_notify_on_failure
 
   # ----------------------------------------------------------------------------
@@ -558,8 +554,7 @@ jobs:
       - *setup_bazel_binary
 
       # Run project tests with NGC and View Engine.
-      # TODO(devversion): remove target exclusion once https://github.com/bazelbuild/rules_nodejs/pull/2646 is available.
-      - run: bazel test --build_tag_filters=-docs-package,-e2e --test_tag_filters=-e2e --config=view-engine --build_tests_only -- src/... -//src/e2e-app:devserver_with_linked_declarations.MF
+      - run: bazel test --build_tag_filters=-docs-package,-e2e --test_tag_filters=-e2e --config=view-engine --build_tests_only -- src/...
       - *slack_notify_on_failure
 
   # ----------------------------------------------------------------------------
@@ -580,8 +575,7 @@ jobs:
       - *setup_bazel_binary
 
       # Run project tests with NGC and View Engine.
-      # TODO(devversion): remove target exclusion once https://github.com/bazelbuild/rules_nodejs/pull/2646 is available.
-      - run: bazel test --build_tag_filters=-docs-package,-e2e --test_tag_filters=-e2e --config=view-engine --build_tests_only -- src/... -//src/e2e-app:devserver_with_linked_declarations.MF
+      - run: bazel test --build_tag_filters=-docs-package,-e2e --test_tag_filters=-e2e --config=view-engine --build_tests_only -- src/...
       - *slack_notify_on_failure
 
   # ----------------------------------------------------------------------------
@@ -654,8 +648,7 @@ jobs:
 
       # Setup the components repository to use the MDC snapshot builds.
       # Run project tests with the MDC canary builds.
-      # TODO(devversion): remove target exclusion once https://github.com/bazelbuild/rules_nodejs/pull/2646 is available.
-      - run: bazel test --build_tag_filters=-docs-package,-e2e --test_tag_filters=-e2e --build_tests_only -- src/... -//src/e2e-app:devserver_with_linked_declarations.MF
+      - run: bazel test --build_tag_filters=-docs-package,-e2e --test_tag_filters=-e2e --build_tests_only -- src/...
       - *slack_notify_on_failure
 
 # ----------------------------------------------------------------------------------------

package.json

@@ -45,7 +45,7 @@
     "approve-size-tests": "node ./scripts/approve-size-golden.js",
     "integration-tests": "bazel test --test_tag_filters=-view-engine-only,-linker-integration-test --build_tests_only -- //integration/... -//integration/size-test/...",
     "integration-tests:view-engine": "bazel test --test_tag_filters=view-engine-only --build_tests_only -- //integration/... -//integration/size-test/...",
-    "integration-tests:partial-ivy": "bazel test --//tools:partial_compilation=True --test_tag_filters=partial-compilation-integration --build_tests_only -- //integration/... //src/...",
+    "integration-tests:partial-ivy": "bazel test --//tools:partial_compilation=True --test_tag_filters=partial-compilation-integration,-firefox --build_tests_only -- //integration/... //src/...",
     "integration-tests:size-test": "bazel test //integration/size-test/...",
     "check-mdc-tests": "ts-node --project scripts/tsconfig.json scripts/check-mdc-tests.ts",
     "check-mdc-exports": "ts-node --project scripts/tsconfig.json scripts/check-mdc-exports.ts",

src/e2e-app/BUILD.bazel

@@ -1,4 +1,3 @@
-load("@npm//@bazel/rollup:index.bzl", "rollup_bundle")
 load("@npm//@bazel/concatjs:index.bzl", "concatjs_devserver")
 load("//:packages.bzl", "getAngularUmdTargets")
 load("//tools:defaults.bzl", "ng_module", "sass_binary")
@@ -117,45 +116,3 @@ concatjs_devserver(
     tags = ["manual"],
     deps = [":e2e-app"],
 )
-
-# Bundles the `e2e-app` using rollup while running the Angular linker to transform
-# partial declarations to their corresponding Angular definitions. This target should
-# only be run if the e2e-app is built in partial compilation mode. Technically this
-# target could be run with full compilation mode too, but we want to raise an error if
-# the partial compilation is not enabled (as the mode has been forgotten most likely).
-rollup_bundle(
-    name = "linked_app_bundle",
-    testonly = True,
-    config_file = ":rollup-with-linker.config.js",
-    entry_points = {
-        "main.ts": "bundle",
-    },
-    sourcemap = "false",
-    tags = ["manual"],
-    deps = select(
-        {
-            # We rely on the partial compilation build setting here to ensure that the target
-            # cannot be built if partial compilation is not enabled.
-            "//tools:partial_compilation_enabled": [
-                ":e2e-app",
-                "@npm//rollup-plugin-node-resolve",
-                "@npm//@angular/compiler-cli",
-                "@npm//@rollup/plugin-babel",
-                "@npm//@rollup/plugin-commonjs",
-                "@npm//glob",
-            ],
-        },
-        no_match_error = "Partial compilation needs to be enabled. The following flag enables partial " +
-                         "compilation: --//tools:partial_compilation=True",
-    ),
-)
-
-# Basic web server that serves the e2e-app processed by the Angular linker.
-concatjs_devserver(
-    name = "devserver_with_linked_declarations",
-    testonly = True,
-    additional_root_paths = ["npm/node_modules"],
-    port = 4200,
-    static_files = [":linked_app_bundle"] + devserverIndexHtmlDependencies,
-    tags = ["manual"],
-)

src/e2e-app/rollup-with-linker.config.js

@@ -1,6 +1,6 @@
 load("//tools:defaults.bzl", "protractor_web_test_suite")
 
-def e2e_test_suite(name, data = [], tags = ["e2e", "partial-compilation-integration"], deps = []):
+def e2e_test_suite(name, data = [], tags = ["e2e"], deps = []):
     protractor_web_test_suite(
         name = name,
         configuration = "//src/e2e-app:protractor.conf.js",
@@ -9,12 +9,7 @@ def e2e_test_suite(name, data = [], tags = ["e2e", "partial-compilation-integrat
             "@npm//@axe-core/webdriverjs",
         ] + data,
         on_prepare = "//src/e2e-app:start-devserver.js",
-        # Based on whether the partial compilation mode is enabled, test either with the default e2e-app
-        # server, or test with a server that processed all sources with the Angular linker.
-        server = select({
-            "//conditions:default": "//src/e2e-app:devserver",
-            "//tools:partial_compilation_enabled": "//src/e2e-app:devserver_with_linked_declarations",
-        }),
+        server = "//src/e2e-app:devserver",
         tags = tags,
         deps = deps,
     )

src/e2e-app/test_suite.bzl

@@ -9,6 +9,7 @@ load("@npm//@bazel/typescript:index.bzl", _ts_library = "ts_library")
 load("//:packages.bzl", "VERSION_PLACEHOLDER_REPLACEMENTS", "getAngularUmdTargets")
 load("//:rollup-globals.bzl", "ROLLUP_GLOBALS")
 load("//tools/markdown-to-html:index.bzl", _markdown_to_html = "markdown_to_html")
+load("//tools/linker-process:index.bzl", "linker_process")
 
 _DEFAULT_TSCONFIG_BUILD = "//src:bazel-tsconfig-build.json"
 _DEFAULT_TSCONFIG_TEST = "//src:tsconfig-test"
@@ -161,8 +162,22 @@ def ng_e2e_test_library(deps = [], tsconfig = None, **kwargs):
 
 def karma_web_test_suite(name, **kwargs):
     web_test_args = {}
+    test_deps = ["//tools/rxjs:rxjs_umd_modules"] + kwargs.get("deps", [])
+
     kwargs["srcs"] = ["@npm//:node_modules/tslib/tslib.js"] + getAngularUmdTargets() + kwargs.get("srcs", [])
-    kwargs["deps"] = ["//tools/rxjs:rxjs_umd_modules"] + kwargs.get("deps", [])
+    kwargs["tags"] = ["partial-compilation-integration"] + kwargs.get("tags", [])
+    kwargs["deps"] = select({
+        # Based on whether partial compilation is enabled, use the linker processed dependencies.
+        "//tools:partial_compilation_enabled": ["%s_linker_processed_deps" % name],
+        "//conditions:default": test_deps,
+    })
+
+    linker_process(
+        name = "%s_linker_processed_deps" % name,
+        srcs = test_deps,
+        testonly = True,
+        tags = ["manual"],
+    )
 
     # Set up default browsers if no explicit `browsers` have been specified.
     if not hasattr(kwargs, "browsers"):

tools/defaults.bzl

@@ -0,0 +1,30 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")
+load("//tools:defaults.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "sources",
+    srcs = glob(["**/*.ts"]),
+    deps = [
+        "@npm//@angular/compiler-cli",
+        "@npm//@babel/core",
+        "@npm//@babel/traverse",
+        "@npm//@types/node",
+    ],
+)
+
+# Exposes the `linker-process` tool as executable so that it can be used as
+# build tool (for `ctx.actions.run`) within the `linker_process` custom Bazel rule.
+nodejs_binary(
+    name = "linker-process",
+    data = [
+        ":sources",
+    ],
+    entry_point = ":linker-process.ts",
+    templated_args = [
+        # TODO(josephperrott): update dependency usages to no longer need bazel patch module resolver
+        # See: https://github.com/bazelbuild/rules_nodejs/wiki#--bazel_patch_module_resolver-now-defaults-to-false-2324
+        "--bazel_patch_module_resolver",
+    ],
+)

tools/linker-process/BUILD.bazel

@@ -0,0 +1,100 @@
+"""
+ Exposes a custom Bazel rule for processing sources, which are extracted from Bazel
+ targets, with the Angular linker plugin.
+"""
+
+load("@build_bazel_rules_nodejs//:providers.bzl", "JSModuleInfo", "JSNamedModuleInfo")
+
+"""
+Gets the Bazel manifest path for a given file. Manifest paths are used within Bazel runfile
+manifests and are formatted as followed: `<workspace_name>/<workspace_relative_file_path>`
+"""
+
+def _to_manifest_path(ctx, file):
+    # If a file resides outside of the current workspace, we omit the leading `../`
+    # segment as the rest will contain the workspace name. e.g. `../npm/node_modules/<..>`.
+    if file.short_path.startswith("../"):
+        return file.short_path[3:]
+    else:
+        return ctx.workspace_name + "/" + file.short_path
+
+"""Extracts all source files from the specified list of dependencies."""
+
+def _extract_source_files(deps):
+    depsets = []
+    for dep in deps:
+        if JSNamedModuleInfo in dep:
+            depsets.append(dep[JSNamedModuleInfo].sources)
+        elif JSModuleInfo in dep:
+            depsets.append(dep[JSModuleInfo].sources)
+        elif hasattr(dep, "files"):
+            depsets.append(dep.files)
+    return depset(transitive = depsets)
+
+def _linker_process(ctx):
+    args = ctx.actions.args()
+    sources = _extract_source_files(ctx.attr.srcs)
+    tmp_dir_name = ctx.label.name
+
+    # The output directory manifest path. e.g `angular_material/src/cdk/a11y/linker_processed`.
+    output_dir_manifest_path = "%s/%s/%s" % (ctx.workspace_name, ctx.label.package, tmp_dir_name)
+
+    # The output directory exec path. e.g `bazel_out/<..>/src/cdk/a11y/linker_processed`.
+    output_dir_exec_path = "%s/%s/%s" % (ctx.bin_dir.path, ctx.label.package, tmp_dir_name)
+
+    # Given the sources being transformed and written to a new location, the AMD module names
+    # need to be rewritten. This file maps AMD modules as per the new location to the AMD modules
+    # as they appear in the sources. i.e. we generate AMD module aliases.
+    amd_module_mapping_file = ctx.actions.declare_file("%s/_module_mappings.js" % tmp_dir_name)
+
+    args.add(output_dir_exec_path)
+    args.add(output_dir_manifest_path)
+    args.add(amd_module_mapping_file.path)
+
+    outputs = [amd_module_mapping_file]
+
+    # Iterate through the determined sources and pass them to the tool as argument.
+    for input_file in sources.to_list():
+        output_pkg_path = _to_manifest_path(ctx, input_file)
+        args.add("%s:%s" % (input_file.path, output_pkg_path))
+        outputs.append(ctx.actions.declare_file("%s/%s" % (tmp_dir_name, output_pkg_path)))
+
+    # Support passing arguments through a parameter file. This is necessary because on Windows
+    # there is an argument limit and we need to handle a large amount of input files. Bazel
+    # switches between parameter file and normal argument passing based on the operating system.
+    # Read more here: https://docs.bazel.build/versions/master/skylark/lib/Args.html#use_param_file
+    args.use_param_file(param_file_arg = "--param-file=%s", use_always = True)
+
+    ctx.actions.run(
+        inputs = sources,
+        outputs = outputs,
+        executable = ctx.executable._linker_process_tool,
+        arguments = [args],
+        progress_message = "NgLinkerProcess",
+    )
+
+    outputs_depset = depset(outputs)
+
+    return [
+        DefaultInfo(files = outputs_depset),
+    ]
+
+"""
+  Rule definition for the "linker_process" rule that can process a list of targets
+  with the Angular linker. The processed files can be retrieved through the default
+  files provider, or through the `JSNamedModuleInfo` provider.
+"""
+linker_process = rule(
+    implementation = _linker_process,
+    attrs = {
+        "srcs": attr.label_list(
+            allow_files = True,
+            doc = """List of sources that should be processed with the Angular linker.""",
+        ),
+        "_linker_process_tool": attr.label(
+            default = Label("//tools/linker-process"),
+            executable = True,
+            cfg = "host",
+        ),
+    },
+)