Merge pull request #2 from magento-commerce/imported-magento-magento2-functional-testing-framework-797

[Imported] MQE-1800: Allow generate:tests to continue running even if one test f…

Author: jilu1

bin/functional

@@ -7,5 +7,6 @@ echo "==============================="
 echo "   EXECUTE Functional Tests    "
 echo "==============================="
 bin/mftf build:project
+bin/mftf run:test DeprecatedDevDocsTest -f
 bin/mftf run:test DevDocsTest -f
 bin/mftf run:test FormatCurrencyTest -f

composer.json

@@ -63,7 +63,7 @@
     },
     "autoload-dev": {
         "psr-4": {
-            "tests\\unit\\": "dev/tests/unit"
+            "tests\\": "dev/tests"
         }
     },
     "scripts": {

composer.lock

@@ -16,6 +16,7 @@
 use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor;
 use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser;
+use Magento\FunctionalTestingFramework\Util\GenerationErrorHandler;
 use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest;
 use tests\unit\Util\MagentoTestCase;
 use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory;
@@ -50,7 +51,6 @@ public function setUp(): void
 
     /**
      * Tests generating a single suite given a set of parsed test data
-     * @throws \Exception
      */
     public function testGenerateSuite()
     {
@@ -86,7 +86,6 @@ public function testGenerateSuite()
 
     /**
      * Tests generating all suites given a set of parsed test data
-     * @throws \Exception
      */
     public function testGenerateAllSuites()
     {
@@ -123,28 +122,36 @@ public function testGenerateAllSuites()
 
     /**
      * Tests attempting to generate a suite with no included/excluded tests and no hooks
-     * @throws \Exception
      */
     public function testGenerateEmptySuite()
     {
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockTestData = $testDataArrayBuilder
+            ->withName('test')
+            ->withAnnotations()
+            ->withTestActions()
+            ->build();
+
         $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
         $mockData = $suiteDataArrayBuilder
             ->withName('basicTestSuite')
             ->build();
         unset($mockData['suites']['basicTestSuite'][TestObjectExtractor::TEST_BEFORE_HOOK]);
         unset($mockData['suites']['basicTestSuite'][TestObjectExtractor::TEST_AFTER_HOOK]);
 
-        $mockTestData = null;
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // set expected error message
-        $this->expectExceptionMessage("Suites must not be empty. Suite: \"basicTestSuite\"");
+        $this->expectExceptionMessage("Suite basicTestSuite is not defined in xml or is invalid");
 
         // parse and generate suite object with mocked data
         $mockSuiteGenerator = SuiteGenerator::getInstance();
         $mockSuiteGenerator->generateSuite("basicTestSuite");
     }
 
+    /**
+     * Tests generating all suites with a suite containing invalid test reference
+     */
     public function testInvalidSuiteTestPair()
     {
         // Mock Suite1 => Test1 and Suite2 => Test2
@@ -179,15 +186,20 @@ public function testInvalidSuiteTestPair()
         $suiteConfig = ['Suite2' => ['Test1']];
         $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
 
-        // Set up Expected Exception
-        $this->expectException(TestReferenceException::class);
-        $this->expectExceptionMessageMatches('(Suite: "Suite2" Tests: "Test1")');
-
         // parse and generate suite object with mocked data and manifest
         $mockSuiteGenerator = SuiteGenerator::getInstance();
         $mockSuiteGenerator->generateAllSuites($manifest);
+
+        // assert that no exception for generateAllSuites and suite generation error is stored in GenerationErrorHandler
+        $errMessage = 'Cannot reference tests which are not declared as part of suite (Suite: "Suite2" Tests: "Test1")';
+        TestLoggingUtil::getInstance()->validateMockLogStatement('error', $errMessage, []);
+        $suiteErrors = GenerationErrorHandler::getInstance()->getErrorsByType('suite');
+        $this->assertArrayHasKey('Suite2', $suiteErrors);
     }
 
+    /**
+     * Tests generating all suites with a non-existing suite
+     */
     public function testNonExistentSuiteTestPair()
     {
         $testDataArrayBuilder = new TestDataArrayBuilder();
@@ -203,19 +215,22 @@ public function testNonExistentSuiteTestPair()
         $suiteConfig = ['Suite3' => ['Test1']];
         $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
 
-        // Set up Expected Exception
-        $this->expectException(TestReferenceException::class);
-        $this->expectExceptionMessageMatches('#Suite3 is not defined#');
-
         // parse and generate suite object with mocked data and manifest
         $mockSuiteGenerator = SuiteGenerator::getInstance();
         $mockSuiteGenerator->generateAllSuites($manifest);
+
+        // assert that no exception for generateAllSuites and suite generation error is stored in GenerationErrorHandler
+        $errMessage = 'Suite Suite3 is not defined in xml or is invalid.';
+        TestLoggingUtil::getInstance()->validateMockLogStatement('error', $errMessage, []);
+        $suiteErrors = GenerationErrorHandler::getInstance()->getErrorsByType('suite');
+        $this->assertArrayHasKey('Suite3', $suiteErrors);
     }
 
     /**
      * Function used to set mock for parser return and force init method to run between tests.
      *
      * @param array $testData
+     * @param array $suiteData
      * @throws \Exception
      */
     private function setMockTestAndSuiteParserOutput($testData, $suiteData)
@@ -271,6 +286,14 @@ private function setMockTestAndSuiteParserOutput($testData, $suiteData)
         $property->setValue($instance, $instance);
     }
 
+    /**
+     * clean up function runs after each test
+     */
+    public function tearDown(): void
+    {
+        GenerationErrorHandler::getInstance()->reset();
+    }
+
     /**
      * clean up function runs after all tests
      */

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -20,9 +20,16 @@
 use tests\unit\Util\ObjectHandlerUtil;
 use tests\unit\Util\TestDataArrayBuilder;
 use tests\unit\Util\MockModuleResolverBuilder;
+use tests\unit\Util\TestLoggingUtil;
+use Magento\FunctionalTestingFramework\Util\GenerationErrorHandler;
 
 class TestObjectHandlerTest extends MagentoTestCase
 {
+    public function setUp(): void
+    {
+        TestLoggingUtil::getInstance()->setMockLoggingUtil();
+    }
+
     /**
      * Basic test to validate array => test object conversion.
      *
@@ -111,8 +118,7 @@ public function testGetTestObject()
      */
     public function testGetTestWithFileName()
     {
-        $this->markTestIncomplete();
-        //TODO
+        $this->markTestIncomplete('TODO');
     }
 
     /**
@@ -254,10 +260,13 @@ public function testGetAllTestObjectsWithInvalidExtends()
         ObjectHandlerUtil::mockTestObjectHandlerWitData(array_merge($testOne, $testTwo));
 
         $toh = TestObjectHandler::getInstance();
-
-        $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class);
-        $this->expectExceptionMessage("Mftf Test can not extend from itself: " . "testOne");
         $toh->getAllObjects();
+
+        // assert that no exception for getAllObjects and test generation error is stored in GenerationErrorHandler
+        $errorMessage = '/' . preg_quote("Mftf Test can not extend from itself: " . "testOne") . '/';
+        TestLoggingUtil::getInstance()->validateMockLogStatmentRegex('error', $errorMessage, []);
+        $testErrors = GenerationErrorHandler::getInstance()->getErrorsByType('test');
+        $this->assertArrayHasKey('testOne', $testErrors);
     }
 
     /**
@@ -360,6 +369,8 @@ public function testGetTestObjectWhenEnablePause()
      */
     public function tearDown(): void
     {
+        TestLoggingUtil::getInstance()->clearMockLoggingUtil();
         AspectMock::clean();
+        parent::tearDownAfterClass();
     }
 }

dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php

@@ -12,6 +12,7 @@
 use Monolog\Logger;
 use PHPUnit\Framework\TestCase;
 use tests\unit\Util\TestLoggingUtil;
+use Magento\FunctionalTestingFramework\Util\GenerationErrorHandler;
 
 class AnnotationExtractorTest extends TestCase
 {
@@ -195,14 +196,22 @@ public function testTestCaseIdUniqueness()
         $extractor = new AnnotationExtractor();
         $extractor->extractAnnotations($firstTestAnnotation, "firstTest");
         $extractor->extractAnnotations($secondTestannotation, "secondTest");
+        $extractor->validateTestCaseIdTitleUniqueness();
 
-        //Expect Exception
-        $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\XmlException::class);
-        $this->expectExceptionMessage("TestCaseId and Title pairs must be unique:\n\n" .
-            "TestCaseId: 'MQE-0001' Title: 'TEST TITLE' in Tests 'firstTest', 'secondTest'");
+        // assert that no exception for validateTestCaseIdTitleUniqueness
+        // and validation error is stored in GenerationErrorHandler
+        $errorMessage = '/'
+            . preg_quote("TestCaseId and Title pairs is not unique in Tests 'firstTest', 'secondTest'")
+            . '/';
+        TestLoggingUtil::getInstance()->validateMockLogStatmentRegex('error', $errorMessage, []);
+        $testErrors = GenerationErrorHandler::getInstance()->getErrorsByType('test');
+        $this->assertArrayHasKey('firstTest', $testErrors);
+        $this->assertArrayHasKey('secondTest', $testErrors);
+    }
 
-        //Trigger Exception
-        $extractor->validateTestCaseIdTitleUniqueness();
+    public function tearDown(): void
+    {
+        GenerationErrorHandler::getInstance()->reset();
     }
 
     /**