refactor: decouple migrations from CLI devkit (#19004)

We want to refactor our update-tool to no longer rely
on specifics from the Angular CLI devkit. This has various benefits:

* Ability to run migrations inside Google.
* Ability to run migrations in a standalone tool (for people not using the CLI!)
* No tight dependency on the CLI devkit API. We could easily switch tools if needed.
* Ability to wrap migrations in tslint rules (or other tools).
* And potentially more. These are just the ones that came to mind yet.

Author: devversion

src/cdk/schematics/BUILD.bazel

@@ -32,6 +32,7 @@ ts_library(
         "@npm//@angular-devkit/schematics",
         # TODO(devversion): Only include jasmine for test sources (See: tsconfig types).
         "@npm//@types/jasmine",
+        "@npm//@types/glob",
         "@npm//@types/node",
         "@npm//glob",
         "@npm//parse5",
@@ -73,6 +74,7 @@ ts_library(
         ":schematics",
         "//src/cdk/schematics/testing",
         "//src/cdk/schematics/update-tool",
+        "//src/cdk/testing/private",
         "@npm//@angular-devkit/schematics",
         "@npm//@schematics/angular",
         "@npm//@types/jasmine",

src/cdk/schematics/index.spec.ts

@@ -0,0 +1,13 @@
+/**
+ * Path to the schematic collection for non-migration schematics. Needs to use
+ * the workspace path as otherwise the resolution won't work on Windows.
+ */
+export const COLLECTION_PATH = require
+    .resolve('angular_material/src/cdk/schematics/collection.json');
+
+/**
+ * Path to the schematic collection that includes the migrations. Needs to use
+ * the workspace path as otherwise the resolution won't work on Windows.
+ */
+export const MIGRATION_PATH = require
+  .resolve('angular_material/src/cdk/schematics/migration.json');

src/cdk/schematics/ng-add/index.spec.ts

@@ -1,5 +1,6 @@
 import {Tree} from '@angular-devkit/schematics';
 import {SchematicTestRunner} from '@angular-devkit/schematics/testing';
+import {COLLECTION_PATH} from '../index.spec';
 import {createTestApp, getFileContent} from '../testing';
 import {addPackageToPackageJson} from './package-config';
 
@@ -8,7 +9,7 @@ describe('CDK ng-add', () => {
   let appTree: Tree;
 
   beforeEach(async () => {
-    runner = new SchematicTestRunner('schematics', require.resolve('../collection.json'));
+    runner = new SchematicTestRunner('schematics', COLLECTION_PATH);
     appTree = await createTestApp(runner);
   });
 

src/cdk/schematics/ng-generate/drag-drop/index.spec.ts

@@ -1,6 +1,7 @@
 import {SchematicTestRunner} from '@angular-devkit/schematics/testing';
 import {getProjectFromWorkspace} from '@angular/cdk/schematics';
 import {getWorkspace} from '@schematics/angular/utility/config';
+import {COLLECTION_PATH} from '../../index.spec';
 import {createTestApp, getFileContent} from '../../testing';
 import {Schema} from './schema';
 
@@ -16,7 +17,7 @@ describe('CDK drag-drop schematic', () => {
   };
 
   beforeEach(() => {
-    runner = new SchematicTestRunner('schematics', require.resolve('../../collection.json'));
+    runner = new SchematicTestRunner('schematics', COLLECTION_PATH);
   });
 
   it('should create drag-drop files and add them to module', async () => {

src/cdk/schematics/ng-update/devkit-file-system.ts

@@ -0,0 +1,59 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {normalize} from '@angular-devkit/core';
+import {Tree, UpdateRecorder} from '@angular-devkit/schematics';
+import {relative} from 'path';
+import {FileSystem} from '../update-tool/file-system';
+
+/** File system that leverages the virtual tree from the CLI devkit. */
+export class DevkitFileSystem implements FileSystem {
+  private _updateRecorderCache = new Map<string, UpdateRecorder>();
+
+  constructor(private _tree: Tree, private _workspaceFsPath: string) {}
+
+  resolve(fsFilePath: string) {
+    return normalize(relative(this._workspaceFsPath, fsFilePath)) as string;
+  }
+
+  edit(fsFilePath: string) {
+    const treeFilePath = this.resolve(fsFilePath);
+    if (this._updateRecorderCache.has(treeFilePath)) {
+      return this._updateRecorderCache.get(treeFilePath)!;
+    }
+    const recorder = this._tree.beginUpdate(treeFilePath);
+    this._updateRecorderCache.set(treeFilePath, recorder);
+    return recorder;
+  }
+
+  commitEdits() {
+    this._updateRecorderCache.forEach(r => this._tree.commitUpdate(r));
+    this._updateRecorderCache.clear();
+  }
+
+  exists(fsFilePath: string) {
+    return this._tree.exists(this.resolve(fsFilePath));
+  }
+
+  overwrite(fsFilePath: string, content: string) {
+    this._tree.overwrite(this.resolve(fsFilePath), content);
+  }
+
+  create(fsFilePath: string, content: string) {
+    this._tree.create(this.resolve(fsFilePath), content);
+  }
+
+  delete(fsFilePath: string) {
+    this._tree.delete(this.resolve(fsFilePath));
+  }
+
+  read(fsFilePath: string) {
+    const buffer = this._tree.read(this.resolve(fsFilePath));
+    return buffer !== null ? buffer.toString() : null;
+  }
+}

src/cdk/schematics/ng-update/devkit-migration-rule.ts

@@ -0,0 +1,161 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Rule, SchematicContext, Tree} from '@angular-devkit/schematics';
+import {NodePackageInstallTask} from '@angular-devkit/schematics/tasks';
+import {WorkspaceProject} from '@schematics/angular/utility/workspace-models';
+import {sync as globSync} from 'glob';
+import {join} from 'path';
+
+import {UpdateProject} from '../update-tool';
+import {TargetVersion} from '../update-tool/target-version';
+import {getTargetTsconfigPath, getWorkspaceConfigGracefully} from '../utils/project-tsconfig-paths';
+
+import {DevkitFileSystem} from './devkit-file-system';
+import {DevkitContext, DevkitMigrationCtor} from './devkit-migration';
+import {AttributeSelectorsMigration} from './migrations/attribute-selectors';
+import {ClassInheritanceMigration} from './migrations/class-inheritance';
+import {ClassNamesMigration} from './migrations/class-names';
+import {ConstructorSignatureMigration} from './migrations/constructor-signature';
+import {CssSelectorsMigration} from './migrations/css-selectors';
+import {ElementSelectorsMigration} from './migrations/element-selectors';
+import {InputNamesMigration} from './migrations/input-names';
+import {MethodCallArgumentsMigration} from './migrations/method-call-arguments';
+import {MiscTemplateMigration} from './migrations/misc-template';
+import {OutputNamesMigration} from './migrations/output-names';
+import {PropertyNamesMigration} from './migrations/property-names';
+import {UpgradeData} from './upgrade-data';
+
+
+/** List of migrations which run for the CDK update. */
+export const cdkMigrations: DevkitMigrationCtor<UpgradeData>[] = [
+  AttributeSelectorsMigration,
+  ClassInheritanceMigration,
+  ClassNamesMigration,
+  ConstructorSignatureMigration,
+  CssSelectorsMigration,
+  ElementSelectorsMigration,
+  InputNamesMigration,
+  MethodCallArgumentsMigration,
+  MiscTemplateMigration,
+  OutputNamesMigration,
+  PropertyNamesMigration,
+];
+
+type NullableMigration = DevkitMigrationCtor<UpgradeData|null>;
+
+type PostMigrationFn =
+    (context: SchematicContext, targetVersion: TargetVersion, hasFailure: boolean) => void;
+
+/**
+ * Creates a Angular schematic rule that runs the upgrade for the
+ * specified target version.
+ */
+export function createMigrationSchematicRule(
+    targetVersion: TargetVersion, extraMigrations: NullableMigration[], upgradeData: UpgradeData,
+    onMigrationCompleteFn?: PostMigrationFn): Rule {
+  return async (tree: Tree, context: SchematicContext) => {
+    const logger = context.logger;
+    const workspace = getWorkspaceConfigGracefully(tree);
+
+    if (workspace === null) {
+      logger.error('Could not find workspace configuration file.');
+      return;
+    }
+
+    // Keep track of all project source files which have been checked/migrated. This is
+    // necessary because multiple TypeScript projects can contain the same source file and
+    // we don't want to check these again, as this would result in duplicated failure messages.
+    const analyzedFiles = new Set<string>();
+    // The CLI uses the working directory as the base directory for the virtual file system tree.
+    const workspaceFsPath = process.cwd();
+    const fileSystem = new DevkitFileSystem(tree, workspaceFsPath);
+    const projectNames = Object.keys(workspace.projects);
+    const migrations: NullableMigration[] = [...cdkMigrations, ...extraMigrations];
+    let hasFailures = false;
+
+    for (const projectName of projectNames) {
+      const project = workspace.projects[projectName];
+      const buildTsconfigPath = getTargetTsconfigPath(project, 'build');
+      const testTsconfigPath = getTargetTsconfigPath(project, 'test');
+
+      if (!buildTsconfigPath && !testTsconfigPath) {
+        logger.warn(`Could not find TypeScript project for project: ${projectName}`);
+        continue;
+      }
+      if (buildTsconfigPath !== null) {
+        runMigrations(project, buildTsconfigPath, false);
+      }
+      if (testTsconfigPath !== null) {
+        runMigrations(project, testTsconfigPath, true);
+      }
+    }
+
+    let runPackageManager = false;
+    // Run the global post migration static members for all migrations.
+    migrations.forEach(m => {
+      const actionResult =
+          m.globalPostMigration !== undefined ? m.globalPostMigration(tree, context) : null;
+      if (actionResult) {
+        runPackageManager = runPackageManager || actionResult.runPackageManager;
+      }
+    });
+
+    // If a migration requested the package manager to run, we run it as an
+    // asynchronous post migration task. We cannot run it synchronously,
+    // as file changes from the current migration task are not applied to
+    // the file system yet.
+    if (runPackageManager) {
+      context.addTask(new NodePackageInstallTask({quiet: false}));
+    }
+
+    if (onMigrationCompleteFn) {
+      onMigrationCompleteFn(context, targetVersion, hasFailures);
+    }
+
+    /** Runs the migrations for the specified workspace project. */
+    function runMigrations(project: WorkspaceProject, tsconfigPath: string, isTestTarget: boolean) {
+      const projectRootFsPath = join(workspaceFsPath, project.root);
+      const tsconfigFsPath = join(workspaceFsPath, tsconfigPath);
+      const program = UpdateProject.createProgramFromTsconfig(tsconfigFsPath, fileSystem);
+      const updateContext: DevkitContext = {
+        workspaceFsPath,
+        isTestTarget,
+        project,
+        tree,
+      };
+
+      const updateProject = new UpdateProject(
+        updateContext,
+        program,
+        fileSystem,
+        analyzedFiles,
+        context.logger,
+      );
+
+      // In some applications, developers will have global stylesheets which are not
+      // specified in any Angular component. Therefore we glob up all CSS and SCSS files
+      // outside of node_modules and dist. The files will be read by the individual
+      // stylesheet rules and checked.
+      // TODO: rework this to collect global stylesheets from the workspace config. COMP-280.
+      const additionalStylesheets = globSync(
+        '!(node_modules|dist)/**/*.+(css|scss)',
+        {absolute: true, cwd: projectRootFsPath, nodir: true});
+
+      const result =
+        updateProject.migrate(migrations, targetVersion, upgradeData, additionalStylesheets);
+
+      // Commit all recorded edits in the update recorder. We apply the edits after all
+      // migrations ran because otherwise offsets in the TypeScript program would be
+      // shifted and individual migrations could no longer update the same source file.
+      fileSystem.commitEdits();
+
+      hasFailures = hasFailures || result.hasFailures;
+    }
+  };
+}

src/cdk/schematics/ng-update/devkit-migration.ts

@@ -0,0 +1,35 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {SchematicContext, Tree} from '@angular-devkit/schematics';
+import {WorkspaceProject} from '@schematics/angular/utility/workspace-models';
+import {Constructor, Migration, PostMigrationAction} from '../update-tool/migration';
+
+export type DevkitContext = {
+  /** Devkit tree for the current migrations. Can be used to insert/remove files. */
+  tree: Tree,
+  /** Workspace project the migrations run against. */
+  project: WorkspaceProject,
+  /** Absolute file system path to the workspace */
+  workspaceFsPath: string,
+  /** Whether the migrations run for a test target. */
+  isTestTarget: boolean,
+};
+
+export abstract class DevkitMigration<Data> extends Migration<Data, DevkitContext> {
+  /**
+   * Optional static method that will be called once the migration of all project
+   * targets has been performed. This method can be used to make changes respecting the
+   * migration result of all individual targets. e.g. removing HammerJS if it
+   * is not needed in any project target.
+   */
+  static globalPostMigration?(tree: Tree, context: SchematicContext): PostMigrationAction;
+}
+
+export type DevkitMigrationCtor<Data> = Constructor<DevkitMigration<Data>> &
+    {[m in keyof typeof DevkitMigration]: (typeof DevkitMigration)[m]};

src/cdk/schematics/ng-update/index.ts

@@ -9,26 +9,26 @@
 import {Rule, SchematicContext} from '@angular-devkit/schematics';
 import {TargetVersion} from '../update-tool/target-version';
 import {cdkUpgradeData} from './upgrade-data';
-import {createUpgradeRule} from './upgrade-rules';
+import {createMigrationSchematicRule} from './devkit-migration-rule';
 
 /** Entry point for the migration schematics with target of Angular CDK 6.0.0 */
 export function updateToV6(): Rule {
-  return createUpgradeRule(TargetVersion.V6, [], cdkUpgradeData, onMigrationComplete);
+  return createMigrationSchematicRule(TargetVersion.V6, [], cdkUpgradeData, onMigrationComplete);
 }
 
 /** Entry point for the migration schematics with target of Angular CDK 7.0.0 */
 export function updateToV7(): Rule {
-  return createUpgradeRule(TargetVersion.V7, [], cdkUpgradeData, onMigrationComplete);
+  return createMigrationSchematicRule(TargetVersion.V7, [], cdkUpgradeData, onMigrationComplete);
 }
 
 /** Entry point for the migration schematics with target of Angular CDK 8.0.0 */
 export function updateToV8(): Rule {
-  return createUpgradeRule(TargetVersion.V8, [], cdkUpgradeData, onMigrationComplete);
+  return createMigrationSchematicRule(TargetVersion.V8, [], cdkUpgradeData, onMigrationComplete);
 }
 
 /** Entry point for the migration schematics with target of Angular CDK 9.0.0 */
 export function updateToV9(): Rule {
-  return createUpgradeRule(TargetVersion.V9, [], cdkUpgradeData, onMigrationComplete);
+  return createMigrationSchematicRule(TargetVersion.V9, [], cdkUpgradeData, onMigrationComplete);
 }
 
 /** Function that will be called when the migration completed. */

src/cdk/schematics/ng-update/upgrade-rules/attribute-selectors-rule.ts renamed to src/cdk/schematics/ng-update/migrations/attribute-selectors.ts

@@ -8,21 +8,21 @@
 
 import * as ts from 'typescript';
 import {ResolvedResource} from '../../update-tool/component-resource-collector';
-import {MigrationRule} from '../../update-tool/migration-rule';
+import {Migration} from '../../update-tool/migration';
 import {AttributeSelectorUpgradeData} from '../data/attribute-selectors';
 import {findAllSubstringIndices} from '../typescript/literal';
-import {getVersionUpgradeData, RuleUpgradeData} from '../upgrade-data';
+import {getVersionUpgradeData, UpgradeData} from '../upgrade-data';
 
 /**
- * Migration rule that walks through every string literal, template and stylesheet
+ * Migration that walks through every string literal, template and stylesheet
  * in order to switch deprecated attribute selectors to the updated selector.
  */
-export class AttributeSelectorsRule extends MigrationRule<RuleUpgradeData> {
+export class AttributeSelectorsMigration extends Migration<UpgradeData> {
   /** Required upgrade changes for specified target version. */
   data = getVersionUpgradeData(this, 'attributeSelectors');
 
   // Only enable the migration rule if there is upgrade data.
-  ruleEnabled = this.data.length !== 0;
+  enabled = this.data.length !== 0;
 
   visitNode(node: ts.Node) {
     if (ts.isStringLiteralLike(node)) {
@@ -68,8 +68,8 @@ export class AttributeSelectorsRule extends MigrationRule<RuleUpgradeData> {
   }
 
   private _replaceSelector(filePath: string, start: number, data: AttributeSelectorUpgradeData) {
-    const updateRecorder = this.getUpdateRecorder(filePath);
-    updateRecorder.remove(start, data.replace.length);
-    updateRecorder.insertRight(start, data.replaceWith);
+    this.fileSystem.edit(filePath)
+      .remove(start, data.replace.length)
+      .insertRight(start, data.replaceWith);
   }
 }