MQE-893: Add flag to robo generate: tests which accepts a specific set of tests to execute

bin/static-checks

@@ -8,7 +8,7 @@ set -e
 echo "==============================="
 echo "         CODE SNIFFER"
 echo "==============================="
-vendor/bin/phpcs ./src --standard=./dev/tests/static/Magento
+vendor/bin/phpcs ./src --standard=./dev/tests/static/Magento --ignore=src/Magento/FunctionalTestingFramework/Group,src/Magento/FunctionalTestingFramework/AcceptanceTester.php
 vendor/bin/phpcs ./dev/tests/unit --standard=./dev/tests/static/Magento
 vendor/bin/phpcs ./dev/tests/verification --standard=./dev/tests/static/Magento --ignore=dev/tests/verification/_generated
 echo ""
@@ -22,7 +22,7 @@ echo ""
 echo "==============================="
 echo "       MESS DETECTOR"
 echo "==============================="
-vendor/bin/phpmd ./src text /dev/tests/static/Magento/CodeMessDetector/ruleset.xml --exclude _generated
+vendor/bin/phpmd ./src text /dev/tests/static/Magento/CodeMessDetector/ruleset.xml --exclude _generated,src/Magento/FunctionalTestingFramework/Group,src/Magento/FunctionalTestingFramework/AcceptanceTester.php
 echo ""
 
 echo "==============================="

bin/static-checks.bat

@@ -5,15 +5,15 @@
 
 @echo off
 @echo ===============================PHP CODE SNIFFER REPORT===============================
-call vendor\bin\phpcs .\src --standard=.\dev\tests\static\Magento
+call vendor\bin\phpcs .\src --standard=.\dev\tests\static\Magento --ignore=src\Magento\FunctionalTestingFramework\Group,src\Magento\FunctionalTestingFramework\AcceptanceTester.php
 call vendor\bin\phpcs .\dev\tests\unit --standard=.\dev\tests\static\Magento
 call vendor\bin\phpcs .\dev\tests\verification --standard=.\dev\tests\static\Magento --ignore=dev\tests\verification\_generated
 
 @echo ===============================COPY PASTE DETECTOR REPORT===============================
 call vendor\bin\phpcpd .\src
 
 @echo "===============================PHP MESS DETECTOR REPORT===============================
-vendor\bin\phpmd .\src text \dev\tests\static\Magento\CodeMessDetector\ruleset.xml --exclude _generated
+vendor\bin\phpmd .\src text \dev\tests\static\Magento\CodeMessDetector\ruleset.xml --exclude _generated,src\Magento\FunctionalTestingFramework\Group,src\Magento\FunctionalTestingFramework\AcceptanceTester.php
 
 @echo ===============================MAGENTO COPYRIGHT REPORT===============================
 echo msgbox "INFO:Copyright check currently not run as part of .bat implementation" > "%temp%\popup.vbs"

dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php

@@ -76,15 +76,6 @@ public function testSortWithSuites()
 
         AspectMock::double(TestObjectHandler::class, ['getInstance' => $mockHandler])->make();
 
-        // mock a suite object
-        $mockSuite = AspectMock::double(SuiteObject::class, [
-            'getBeforeHook' => null,
-            'getAfterHook' => null,
-            'getName' => 'mockSuite1'
-        ])->make();
-        $mockSuiteHandler = AspectMock::double(SuiteObjectHandler::class, ['getObject' => $mockSuite])->make();
-        AspectMock::double(SuiteObjectHandler::class, ['getInstance' => $mockSuiteHandler]);
-
         // create test to size array
         $sampleTestArray = [
             'test1' => 100,
@@ -96,8 +87,7 @@ public function testSortWithSuites()
 
         // create mock suite references
         $sampleSuiteArray = [
-            'mockTest1' => ['mockSuite1'],
-            'mockTest2' => ['mockSuite1']
+            'mockSuite1' => ['mockTest1', 'mockTest2']
         ];
 
         // perform sort

src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php

@@ -78,6 +78,26 @@ public function getAllObjects()
         return $this->suiteObjects;
     }
 
+    /**
+     * Function which return all tests referenced by suites.
+     *
+     * @return array
+     */
+    public function getAllTestReferences()
+    {
+        $testsReferencedInSuites = [];
+        $suites = $this->getAllObjects();
+
+        foreach ($suites as $suite) {
+            /** @var SuiteObject $suite */
+            $test_keys = array_keys($suite->getTests());
+            $testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]);
+            $testsReferencedInSuites = array_merge_recursive($testsReferencedInSuites, $testToSuiteName);
+        }
+
+        return $testsReferencedInSuites;
+    }
+
     /**
      * Method to parse all suite data xml into objects.
      *

src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

@@ -6,14 +6,13 @@
 
 namespace Magento\FunctionalTestingFramework\Suite;
 
-use Magento\Framework\Phrase;
-use Magento\Framework\Validator\Exception;
+use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
 use Magento\FunctionalTestingFramework\Suite\Generators\GroupClassGenerator;
 use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
 use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject;
+use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
 use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
 use Magento\FunctionalTestingFramework\Util\Manifest\BaseTestManifest;
-use Magento\FunctionalTestingFramework\Util\Manifest\ParallelTestManifest;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
 use Symfony\Component\Yaml\Yaml;
 
@@ -75,15 +74,23 @@ public static function getInstance()
      */
     public function generateAllSuites($testManifest)
     {
-        $suites = SuiteObjectHandler::getInstance()->getAllObjects();
-        if (get_class($testManifest) == ParallelTestManifest::class) {
-            /** @var  ParallelTestManifest $testManifest */
-            $suites = $testManifest->getSorter()->getResultingSuites();
+        $suites = array_keys(SuiteObjectHandler::getInstance()->getAllObjects());
+        if ($testManifest != null) {
+            $suites = $testManifest->getSuiteConfig();
         }
 
-        foreach ($suites as $suite) {
-            // during a parallel config run we must generate only after we have data around how a suite will be split
-            $this->generateSuiteFromObject($suite);
+        foreach ($suites as $suiteName => $suiteContent) {
+            $firstElement = array_values($suiteContent)[0];
+
+            // if the first element is a string we know that we simply have an array of tests
+            if (is_string($firstElement)) {
+                $this->generateSuiteFromTest($suiteName, $suiteContent);
+            }
+
+            // if our first element is an array we know that we have split the suites
+            if (is_array($firstElement)) {
+                $this->generateSplitSuiteFromTest($suiteName, $suiteContent);
+            }
         }
     }
 
@@ -96,11 +103,22 @@ public function getTestsReferencedInSuites()
     {
         $testsReferencedInSuites = [];
         $suites = SuiteObjectHandler::getInstance()->getAllObjects();
+
+        // see if we have a specific suite configuration.
+        if (!empty($this->suiteReferences)) {
+            $suites = array_intersect_key($suites, $this->suiteReferences);
+        }
+
         foreach ($suites as $suite) {
             /** @var SuiteObject $suite */
             $test_keys = array_keys($suite->getTests());
-            $testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]);
 
+            // see if we need to filter which tests we'll be generating.
+            if (array_key_exists($suite->getName(), $this->suiteReferences)) {
+                $test_keys = $this->suiteReferences[$suite->getName()] ?? $test_keys;
+            }
+
+            $testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]);
             $testsReferencedInSuites = array_merge_recursive($testsReferencedInSuites, $testToSuiteName);
         }
 
@@ -117,35 +135,125 @@ public function getTestsReferencedInSuites()
     public function generateSuite($suiteName)
     {
         /**@var SuiteObject $suite **/
-        $suite = SuiteObjectHandler::getInstance()->getObject($suiteName);
-        $this->generateSuiteFromObject($suite);
+        $this->generateSuiteFromTest($suiteName, []);
     }
 
     /**
-     * Function which takes a suite object and generates all relevant supporting files and classes.
+     * Function which takes a suite name and a set of test names. The function then generates all relevant supporting
+     * files and classes for the suite. The function takes an optional argument for suites which are split by a parallel
+     * run so that any pre/post conditions can be duplicated.
      *
-     * @param SuiteObject $suiteObject
+     * @param string $suiteName
+     * @param array $tests
+     * @param string $originalSuiteName
      * @return void
      */
-    public function generateSuiteFromObject($suiteObject)
+    private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteName = null)
     {
-        $suiteName = $suiteObject->getName();
         $relativePath = TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . $suiteName;
-        $fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath;
-        $groupNamespace = null;
+        $fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath . DIRECTORY_SEPARATOR;
 
         DirSetupUtil::createGroupDir($fullPath);
-        $this->generateRelevantGroupTests($suiteName, $suiteObject->getTests());
 
-        if ($suiteObject->requiresGroupFile()) {
-            // if the suite requires a group file, generate it and set the namespace
-            $groupNamespace = $this->groupClassGenerator->generateGroupClass($suiteObject);
+        $relevantTests = [];
+        if (!empty($tests)) {
+            $this->validateTestsReferencedInSuite($suiteName, $tests, $originalSuiteName);
+            foreach ($tests as $testName) {
+                $relevantTests[$testName] = TestObjectHandler::getInstance()->getObject($testName);
+            }
+        } else {
+            $relevantTests = SuiteObjectHandler::getInstance()->getObject($suiteName)->getTests();
         }
 
+        $this->generateRelevantGroupTests($suiteName, $relevantTests);
+        $groupNamespace = $this->generateGroupFile($suiteName, $relevantTests, $originalSuiteName);
+
         $this->appendEntriesToConfig($suiteName, $fullPath, $groupNamespace);
         print "Suite ${suiteName} generated to ${relativePath}.\n";
     }
 
+    /**
+     * Function which validates tests passed in as custom configuration against the configuration defined by the user to
+     * prevent possible invalid test configurations from executing.
+     *
+     * @param string $suiteName
+     * @param array $testsReferenced
+     * @param string $originalSuiteName
+     * @return void
+     * @throws TestReferenceException
+     */
+    private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $originalSuiteName)
+    {
+        $suiteRef = $originalSuiteName ?? $suiteName;
+        $possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests();
+        $invalidTestRef = null;
+        $errorMsg = "Cannot reference tests not declared as part of {$suiteRef}:\n ";
+
+        array_walk($testsReferenced, function ($value) use (&$invalidTestRef, $possibleTestRef, &$errorMsg) {
+            if (!array_key_exists($value, $possibleTestRef)) {
+                $invalidTestRef.= "\t{$value}\n";
+            }
+        });
+
+        if ($invalidTestRef != null) {
+            throw new TestReferenceException($errorMsg . $invalidTestRef);
+        }
+    }
+
+    /**
+     * Function for generating split groups of tests (following a parallel execution). Takes a paralle suite config
+     * and generates applicable suites.
+     *
+     * @param string $suiteName
+     * @param array $suiteContent
+     * @return void
+     */
+    private function generateSplitSuiteFromTest($suiteName, $suiteContent)
+    {
+        foreach ($suiteContent as $suiteSplitName => $tests) {
+            $this->generateSuiteFromTest($suiteSplitName, $tests, $suiteName);
+        }
+    }
+
+    /**
+     * Function which takes a suite name, array of tests, and an original suite name. The function takes these args
+     * and generates a group file which captures suite level preconditions.
+     *
+     * @param string $suiteName
+     * @param array $tests
+     * @param string $originalSuiteName
+     * @return null|string
+     */
+    private function generateGroupFile($suiteName, $tests, $originalSuiteName)
+    {
+        // if there's an original suite name we know that this test came from a split group.
+        if ($originalSuiteName) {
+            // create the new suite object
+            /** @var SuiteObject $originalSuite */
+            $originalSuite = SuiteObjectHandler::getInstance()->getObject($originalSuiteName);
+            $suiteObject = new SuiteObject(
+                $suiteName,
+                $tests,
+                [],
+                $originalSuite->getHooks()
+            );
+        } else {
+            $suiteObject = SuiteObjectHandler::getInstance()->getObject($suiteName);
+            // we have to handle the case when there is a custom configuration for an existing suite.
+            if (count($suiteObject->getTests()) != count($tests)) {
+                return $this->generateGroupFile($suiteName, $tests, $suiteName);
+            }
+        }
+
+        if (!$suiteObject->requiresGroupFile()) {
+            // if we do not require a group file we don't need a namespace
+            return null;
+        }
+
+        // if the suite requires a group file, generate it and set the namespace
+        return $this->groupClassGenerator->generateGroupClass($suiteObject);
+    }
+
     /**
      * Function which accepts a suite name and suite path and appends a new group entry to the codeception.yml.dist
      * file in order to register the set of tests as a new group. Also appends group object location if required
@@ -219,7 +327,7 @@ private static function clearPreviousSessionConfigEntries()
     private function generateRelevantGroupTests($path, $tests)
     {
         $testGenerator = TestGenerator::getInstance($path, $tests);
-        $testGenerator->createAllTestFiles('suite');
+        $testGenerator->createAllTestFiles(null, []);
     }
 
     /**

src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php

@@ -6,6 +6,8 @@
 
 namespace Magento\FunctionalTestingFramework\Util\Manifest;
 
+use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
+use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
 
 abstract class BaseTestManifest
@@ -24,16 +26,25 @@ abstract class BaseTestManifest
      */
     protected $relativeDirPath;
 
+    /**
+     * Suite configuration in the format suite name to test name. Overwritten during a custom configuration.
+     *
+     * @var array
+     */
+    protected $suiteConfiguration;
+
     /**
      * TestManifest constructor.
      *
      * @param string $path
      * @param string $runConfig
+     * @param array $suiteConfiguration
      */
-    public function __construct($path, $runConfig)
+    public function __construct($path, $runConfig, $suiteConfiguration)
     {
         $this->runTypeConfig = $runConfig;
         $this->relativeDirPath = substr($path, strlen(dirname(dirname(TESTS_BP))) + 1);
+        $this->suiteConfiguration = $suiteConfiguration;
     }
 
     /**
@@ -57,9 +68,41 @@ abstract public function addTest($testObject);
     /**
      * Function which generates the actual manifest(s) once the relevant tests have been added to the array.
      *
-     * @param array $testsReferencedInSuites
-     * @param int|null $nodes
      * @return void
      */
-    abstract public function generate($testsReferencedInSuites, $nodes = null);
+    abstract public function generate();
+
+    /**
+     * Getter for the suite configuration.
+     *
+     * @return array
+     */
+    public function getSuiteConfig()
+    {
+        if ($this->suiteConfiguration === null) {
+            return [];
+        }
+
+        $suiteToTestNames = [];
+        if (empty($this->suiteConfiguration)) {
+            // if there is no configuration passed we can assume the user wants all suites generated as specified.
+            foreach (SuiteObjectHandler::getInstance()->getAllObjects() as $suite => $suiteObj) {
+                /** @var SuiteObject $suitObj */
+                $suiteToTestNames[$suite] = array_keys($suiteObj->getTests());
+            }
+        } else {
+            // we need to loop through the configuration to make sure we capture suites with no specific config
+            foreach ($this->suiteConfiguration as $suiteName => $test) {
+                if (empty($test)) {
+                    $suiteToTestNames[$suiteName] =
+                        array_keys(SuiteObjectHandler::getInstance()->getObject($suiteName)->getTests());
+                    continue;
+                }
+
+                $suiteToTestNames[$suiteName] = $test;
+            }
+        }
+
+        return $suiteToTestNames;
+    }
 }