Merge pull request #4 from magento-commerce/MQE-2272

MQE-2272: run:failed only runs last failed test when multiple tests a…

Author: jilu1

src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

@@ -36,6 +36,7 @@ class BaseGenerateCommand extends Command
     const CODECEPT_RUN = 'codecept:run';
     const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . ' functional ';
     const CODECEPT_RUN_OPTION_NO_EXIT = ' --no-exit ';
+    const FAILED_FILE = 'failed';
 
     /**
      * Enable pause()
@@ -44,13 +45,34 @@ class BaseGenerateCommand extends Command
      */
     private $enablePause = null;
 
+    /**
+     * Full path to '_output' dir
+     *
+     * @var string
+     */
+    private $testsOutputDir = null;
+
+    /**
+     *  String contains all 'failed' tests
+     *
+     * @var string
+     */
+    private $allFailed;
+
     /**
      * Console output style
      *
      * @var SymfonyStyle
      */
     protected $ioStyle = null;
 
+    /**
+     * Full path to 'failed' file
+     *
+     * @var string
+     */
+    protected $testsFailedFile = null;
+
     /**
      * Configures the base command.
      *
@@ -270,4 +292,76 @@ protected function codeceptRunTest(string $commandStr, OutputInterface $output)
         $command = $this->getApplication()->find(self::CODECEPT_RUN);
         return $command->run($input, $output);
     }
+
+    /**
+     * Return tests _output directory
+     *
+     * @return string
+     * @throws TestFrameworkException
+     */
+    protected function getTestsOutputDir()
+    {
+        if (!$this->testsOutputDir) {
+            $this->testsOutputDir = FilePathFormatter::format(TESTS_BP) .
+                "tests" .
+                DIRECTORY_SEPARATOR .
+                "_output" .
+                DIRECTORY_SEPARATOR;
+        }
+
+        return $this->testsOutputDir;
+    }
+
+    /**
+     * Save 'failed' tests
+     *
+     * @return void
+     */
+    protected function appendRunFailed()
+    {
+        try {
+            if (!$this->testsFailedFile) {
+                $this->testsFailedFile = $this->getTestsOutputDir() . self::FAILED_FILE;
+            }
+
+            if (file_exists($this->testsFailedFile)) {
+                // Save 'failed' tests
+                $contents = file_get_contents($this->testsFailedFile);
+                if ($contents !== false && !empty($contents)) {
+                    $this->allFailed .= trim($contents) . PHP_EOL;
+                }
+            }
+        } catch (TestFrameworkException $e) {
+        }
+    }
+
+    /**
+     * Apply 'allFailed' in 'failed' file
+     *
+     * @return void
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
+     */
+    protected function applyAllFailed()
+    {
+        try {
+            if (!$this->testsFailedFile) {
+                $this->testsFailedFile = $this->getTestsOutputDir() . self::FAILED_FILE;
+            }
+
+            if (!empty($this->allFailed)) {
+                // Update 'failed' with content from 'allFailed'
+                if (file_exists($this->testsFailedFile)) {
+                    rename($this->testsFailedFile, $this->testsFailedFile . '.copy');
+                }
+                if (file_put_contents($this->testsFailedFile, $this->allFailed) === false
+                    && file_exists($this->testsFailedFile . '.copy')) {
+                    rename($this->testsFailedFile . '.copy', $this->testsFailedFile);
+                }
+                if (file_exists($this->testsFailedFile . '.copy')) {
+                    unlink($this->testsFailedFile . '.copy');
+                }
+            }
+        } catch (TestFrameworkException $e) {
+        }
+    }
 }

src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

@@ -117,6 +117,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $this->runTestsInSuite($testConfigArray['suites'], $output);
         }
 
+        // Add all failed tests in 'failed' file
+        $this->applyAllFailed();
+
         return max($this->returnCode, $generationErrorCode);
     }
 
@@ -161,6 +164,9 @@ private function runTests(array $tests, OutputInterface $output)
                 $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps';
                 $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));
             }
+
+            // Save failed tests
+            $this->appendRunFailed();
         }
     }
 
@@ -196,6 +202,9 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
             } else {
                 $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));
             }
+
+            // Save failed tests
+            $this->appendRunFailed();
         }
     }
 

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -23,11 +23,6 @@ class RunTestFailedCommand extends BaseGenerateCommand
      */
     const DEFAULT_TEST_GROUP = 'default';
 
-    /**
-     * @var string
-     */
-    private $testsFailedFile;
-
     /**
      * @var string
      */
@@ -69,14 +64,8 @@ protected function configure()
      */
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
-        $testsOutputDir = FilePathFormatter::format(TESTS_BP) .
-            "tests" .
-            DIRECTORY_SEPARATOR .
-            "_output" .
-            DIRECTORY_SEPARATOR;
-
-        $this->testsFailedFile = $testsOutputDir . "failed";
-        $this->testsReRunFile = $testsOutputDir . "rerun_tests";
+        $this->testsFailedFile = $this->getTestsOutputDir() . self::FAILED_FILE;
+        $this->testsReRunFile = $this->getTestsOutputDir() . "rerun_tests";
         $this->testsManifestFile= FilePathFormatter::format(TESTS_MODULE_PATH) .
             "_generated" .
             DIRECTORY_SEPARATOR .

src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

@@ -130,8 +130,14 @@ function ($type, $buffer) use ($output) {
                     }
                 );
             }
+
+            // Save failed tests
+            $this->appendRunFailed();
         }
 
+        // Add all failed tests in 'failed' file
+        $this->applyAllFailed();
+
         foreach ($returnCodes as $returnCode) {
             if ($returnCode != 0) {
                 return $returnCode;