MQE-1800: Allow generate:tests to continue running even if one test fails generation

Author: jilu1

dev/tests/unit/Magento/FunctionalTestFramework/Util/GenerationErrorHandlerTest.php

@@ -242,6 +242,93 @@ public function testGetAllErrorsDuplicate()
         $this->assertEquals($expectedSuiteErrors, GenerationErrorHandler::getInstance()->getErrorsByType('suite'));
     }
 
+    /**
+     * Test get all error messages
+     *
+     * @param string $expectedErrMessages
+     * @param array  $errors
+     * @dataProvider getAllErrorMessagesDataProvider
+     */
+    public function testGetAllErrorMessages($expectedErrMessages, $errors)
+    {
+        $handler = GenerationErrorHandler::getInstance();
+        $handler->reset();
+
+        $property = new \ReflectionProperty(GenerationErrorHandler::class, 'errors');
+        $property->setAccessible(true);
+        $property->setValue($handler, $errors);
+
+        // Assert getAllErrorMessages
+        $this->assertEquals($expectedErrMessages, GenerationErrorHandler::getInstance()->getAllErrorMessages());
+    }
+
+    /**
+     * Data provider for testGetAllErrorMessages()
+     *
+     * @return array
+     */
+    public function getAllErrorMessagesDataProvider()
+    {
+        return [
+            ['', []],
+            ['', [
+                    'test' => [],
+                    'suite' => [],
+                ]
+            ],
+            ['TestError1'
+                . PHP_EOL
+                . 'TestError2'
+                . PHP_EOL
+                . 'TestError3'
+                . PHP_EOL
+                . 'SuiteError1'
+                . PHP_EOL
+                . 'SuiteError2'
+                . PHP_EOL
+                . 'SuiteError3'
+                . PHP_EOL
+                . 'SuiteError4',
+                [
+                    'test' => [
+                        'Sameple1Test' => [
+                            'message' => [
+                                0 => 'TestError1',
+                                1 => 'TestError2'
+                            ],
+                            'generated' => [
+                                0 => false,
+                                1 => false
+                            ],
+                        ],
+                        'Sameple2Test' => [
+                            'message' => 'TestError3',
+                            'generated' => true,
+                        ],
+                    ],
+                    'suite' => [
+                        'Sameple1Suite' => [
+                            'message' => 'SuiteError1',
+                            'generated' => true,
+                        ],
+                        'Sameple2Suite' => [
+                            'message' => [
+                                0 => 'SuiteError2',
+                                1 => 'SuiteError3',
+                                2 => 'SuiteError4',
+                            ],
+                            'generated' => [
+                                0 => false,
+                                1 => true,
+                                2 => false,
+                            ],
+                        ],
+                    ],
+                ]
+            ],
+        ];
+    }
+
     /**
      * Test reset
      */

src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

@@ -140,7 +140,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
                     // $testConfiguration['tests'] cannot be empty if $tests is not empty
                     TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
                 } elseif (empty($testConfiguration['suites'])) {
-                    throw new FastFailException('Invalid input');
+                    throw new FastFailException(
+                        !empty(GenerationErrorHandler::getInstance()->getAllErrors())
+                            ?
+                            GenerationErrorHandler::getInstance()->getAllErrorMessages()
+                            :
+                            'Invalid input'
+                    );
                 }
             } catch (FastFailException $e) {
                 throw $e;

src/Magento/FunctionalTestingFramework/Util/GenerationErrorHandler.php

@@ -78,6 +78,28 @@ public function getAllErrors()
         return $this->errors;
     }
 
+    /**
+     * Return all error message in a string
+     *
+     * @return string
+     */
+    public function getAllErrorMessages()
+    {
+        $errMessages = '';
+
+        foreach ($this->errors as $type => $errors) {
+            foreach ($errors as $error) {
+                if (is_array($error['message'])) {
+                    $errMessages .= (!empty($errMessages) ? PHP_EOL : '') . implode(PHP_EOL, $error['message']);
+                } else {
+                    $errMessages .= (!empty($errMessages) ? PHP_EOL : '') . $error['message'];
+                }
+            }
+        }
+
+        return $errMessages;
+    }
+
     /**
      * Return errors for given type
      *