Merge pull request #787 from magento/MQE-2269

MQE-2269: Test manifest run stops at first failure encountered when E…

Author: jilu1

src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

@@ -33,6 +33,7 @@ class BaseGenerateCommand extends Command
     const MFTF_NOTICES = "Placeholder text for MFTF notices\n";
     const CODECEPT_RUN = 'codecept:run';
     const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . ' functional ';
+    const CODECEPT_RUN_OPTION_NO_EXIT = ' --no-exit ';
 
     /**
      * Enable pause()

src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php

@@ -81,12 +81,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         // Delete the Codeception failed file just in case it exists from any previous test runs
         $this->deleteFailedFile();
 
-        foreach ($manifestFile as $manifestLine) {
-            if (empty($manifestLine)) {
+        for ($line = 0; $line < count($manifestFile); $line++) {
+            if (empty($manifestFile[$line])) {
                 continue;
             }
 
-            $this->runManifestLine($manifestLine, $output);
+            if ($line == count($manifestFile) - 1) {
+                $this->runManifestLine($manifestFile[$line], $output, true);
+            } else {
+                $this->runManifestLine($manifestFile[$line], $output);
+            }
+
             $this->aggregateFailed();
         }
 
@@ -103,18 +108,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int
      *
      * @param string          $manifestLine
      * @param OutputInterface $output
+     * @param boolean         $exit
      * @return void
      * @throws \Exception
      *
      * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure
      */
-    private function runManifestLine(string $manifestLine, OutputInterface $output)
+    private function runManifestLine($manifestLine, $output, $exit = false)
     {
         if (getenv('ENABLE_PAUSE') === 'true') {
             $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_FUNCTIONAL
-                . '--verbose --steps --debug ' . $manifestLine;
+                . '--verbose --steps --debug ';
+            if (!$exit) {
+                $codeceptionCommand .= BaseGenerateCommand::CODECEPT_RUN_OPTION_NO_EXIT;
+            }
+            $codeceptionCommand .= $manifestLine;
             $input = new StringInput($codeceptionCommand);
-            $command = $this->getApplication()->find('codecept:run');
+            $command = $this->getApplication()->find(BaseGenerateCommand::CODECEPT_RUN);
             $subReturnCode = $command->run($input, $output);
         } else {
             $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept")

src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

@@ -135,8 +135,8 @@ private function runTests(array $tests, OutputInterface $output)
             TestGenerator::DEFAULT_DIR .
             DIRECTORY_SEPARATOR ;
 
-        foreach ($tests as $test) {
-            $testName = $test . 'Cest.php';
+        for ($i = 0; $i < count($tests); $i++) {
+            $testName = $tests[$i] . 'Cest.php';
             if (!realpath($testsDirectory . $testName)) {
                 throw new TestFrameworkException(
                     $testName . " is not available under " . $testsDirectory
@@ -145,6 +145,9 @@ private function runTests(array $tests, OutputInterface $output)
 
             if ($this->pauseEnabled()) {
                 $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps --debug';
+                if ($i != count($tests) - 1) {
+                    $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+                }
                 $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output));
             } else {
                 $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps';
@@ -169,10 +172,18 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
             $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept')
                 . ' run functional --verbose --steps ';
         }
+
+        $count = count($suitesConfig);
+        $index = 0;
         //for tests in suites, run them as a group to run before and after block
         foreach (array_keys($suitesConfig) as $suite) {
             $fullCommand = $codeceptionCommand . " -g {$suite}";
+
+            $index += 1;
             if ($this->pauseEnabled()) {
+                if ($index != $count) {
+                    $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+                }
                 $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output));
             } else {
                 $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -116,13 +116,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $testManifestList = $this->readTestManifestFile();
         $returnCode = 0;
-        foreach ($testManifestList as $testCommand) {
+        for ($i = 0; $i < count($testManifestList); $i++) {
             if ($this->pauseEnabled()) {
-                $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testCommand . ' --debug';
+                $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testManifestList[$i] . ' --debug ';
+                if ($i != count($testManifestList) - 1) {
+                    $codeceptionCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+                }
                 $returnCode = $this->codeceptRunTest($codeceptionCommand, $output);
             } else {
                 $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
-                $codeceptionCommand .= $testCommand;
+                $codeceptionCommand .= $testManifestList[$i];
 
                 $process = new Process($codeceptionCommand);
                 $process->setWorkingDirectory(TESTS_BP);
@@ -142,6 +145,7 @@ function ($type, $buffer) use ($output) {
                 );
             }
         }
+
         foreach ($this->failedList as $test) {
             $this->writeFailedTestToFile($test, $this->testsFailedFile);
         }

src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

@@ -52,6 +52,7 @@ protected function configure()
      * @throws \Exception
      *
      * @SuppressWarnings(PHPMD.UnusedLocalVariable)
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
@@ -102,10 +103,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $exitCode = -1;
         $returnCodes = [];
-        foreach ($groups as $group) {
-            $codeceptionCommandString = $commandString . " -g {$group}";
+        for ($i = 0; $i < count($groups); $i++) {
+            $codeceptionCommandString = $commandString . ' -g ' . $groups[$i];
 
             if ($this->pauseEnabled()) {
+                if ($i != count($groups) - 1) {
+                    $codeceptionCommandString .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+                }
                 $returnCodes[] = $this->codeceptRunTest($codeceptionCommandString, $output);
             } else {
                 $process = new Process($codeceptionCommandString);