MFTF 2.3.7 - Merge to Master

CHANGELOG.md

@@ -1,6 +1,29 @@
 Magento Functional Testing Framework Changelog
 ================================================
 
+2.3.7
+-----
+### Enhancements
+* Traceability
+    * Test generation errors output xml filename where they were encountered, as well as xml parent nodes where applicable.
+    * Duplicate element detection now outputs parent element where duplicate was found.
+* Maintainability
+    * Standalone MFTF can now be pointed at a Magento installation folder to generate and execute tests.
+        * See DevDocs for more information.
+    * MFTF now checks for `test` and `actionGroup` elements that have the same `name` in the same file.
+* Customizability
+    * Updated prefered syntax for `actionGroup` `argument`s that use `xml.data` (old syntax is still supported)
+        * Old: `xml.data`
+        * New: `{{xml.data}}`
+* Modularity
+    * `ModuleResolver` now utilizes each Magento module's `registration.php` to map MFTF test material directories.
+### Fixes
+* The `waitForPageLoad` action now correctly uses the given `timeout` attribute for all of its checks.
+* Firefox compatibility issues in javascript error logging were fixed.
+* Fixed an issue where arguments containing `-` would not properly resolve parameterized selectors.
+* Fixed an issue where actions using `parameterArray` would not resolve `$persisted.data$` references.
+* Fixed an issue where composer installations of Magento would fail to parse MFTF materials under a path `vendor/magento/module-<module>/`
+
 2.3.6
 -----
 ### Enhancements

bin/mftf

@@ -12,7 +12,7 @@ if (PHP_SAPI !== 'cli') {
 }
 
 $autoloadPath = realpath(__DIR__ . '/../../../autoload.php');
-$testBootstrapPath = realpath(__DIR__ . '/../dev/tests/functional/_bootstrap.php');
+$testBootstrapPath = realpath(__DIR__ . '/../dev/tests/functional/standalone_bootstrap.php');
 
 try {
     if (file_exists($autoloadPath)) {
@@ -29,7 +29,7 @@ try {
 try {
     $application = new Symfony\Component\Console\Application();
     $application->setName('Magento Functional Testing Framework CLI');
-    $application->setVersion('2.3.6');
+    $application->setVersion('2.3.7');
     /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */
     $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList;
     foreach ($commandList->getCommands() as $command) {

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.3.6",
+    "version": "2.3.7",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

dev/tests/_bootstrap.php

@@ -40,7 +40,8 @@
     'MAGENTO_BASE_URL' => 'http://baseurl:8080',
     'MAGENTO_BACKEND_NAME' => 'admin',
     'MAGENTO_ADMIN_USERNAME' => 'admin',
-    'MAGENTO_ADMIN_PASSWORD' => 'admin123'
+    'MAGENTO_ADMIN_PASSWORD' => 'admin123',
+    'DEFAULT_TIMEZONE' => 'America/Los_Angeles'
 ];
 
 foreach ($TEST_ENVS as $key => $value) {

dev/tests/functional/_bootstrap.php → dev/tests/functional/standalone_bootstrap.php

@@ -12,31 +12,30 @@
 }
 
 defined('PROJECT_ROOT') || define('PROJECT_ROOT', dirname(dirname(dirname(__DIR__))));
+
 require_once realpath(PROJECT_ROOT . '/vendor/autoload.php');
 
 //Load constants from .env file
-defined('FW_BP') || define('FW_BP', PROJECT_ROOT);
-
-// add the debug flag here
-$debug_mode = $_ENV['MFTF_DEBUG'] ?? false;
-if (!(bool)$debug_mode && extension_loaded('xdebug')) {
-    xdebug_disable();
-}
-
-$RELATIVE_TESTS_MODULE_PATH = '/tests/functional/tests/MFTF';
-
-defined('MAGENTO_BP') || define('MAGENTO_BP', PROJECT_ROOT);
-defined('TESTS_BP') || define('TESTS_BP', dirname(dirname(__DIR__)));
-defined('TESTS_MODULE_PATH') || define('TESTS_MODULE_PATH', realpath(TESTS_BP . $RELATIVE_TESTS_MODULE_PATH));
-
-if (file_exists(TESTS_BP . DIRECTORY_SEPARATOR . '.env')) {
-    $env = new \Dotenv\Loader(TESTS_BP . DIRECTORY_SEPARATOR . '.env');
+$envFilePath = dirname(dirname(__DIR__));
+if (file_exists($envFilePath . DIRECTORY_SEPARATOR . '.env')) {
+    $env = new \Dotenv\Loader($envFilePath . DIRECTORY_SEPARATOR . '.env');
     $env->load();
 
     foreach ($_ENV as $key => $var) {
         defined($key) || define($key, $var);
     }
 
+    if (array_key_exists('MAGENTO_BP', $_ENV)) {
+        // TODO REMOVE THIS CODE ONCE WE HAVE STOPPED SUPPORTING dev/tests/acceptance PATH
+        // define TEST_PATH and TEST_MODULE_PATH
+        defined('TESTS_BP') || define('TESTS_BP', dirname(dirname(__DIR__)));
+        $RELATIVE_TESTS_MODULE_PATH = '/tests/functional/tests/MFTF';
+        defined('TESTS_MODULE_PATH') || define(
+            'TESTS_MODULE_PATH',
+            realpath(TESTS_BP . $RELATIVE_TESTS_MODULE_PATH)
+        );
+    }
+
     defined('MAGENTO_CLI_COMMAND_PATH') || define(
         'MAGENTO_CLI_COMMAND_PATH',
         'dev/tests/acceptance/utils/command.php'
@@ -45,4 +44,28 @@
 
     defined('MAGENTO_CLI_COMMAND_PARAMETER') || define('MAGENTO_CLI_COMMAND_PARAMETER', 'command');
     $env->setEnvironmentVariable('MAGENTO_CLI_COMMAND_PARAMETER', MAGENTO_CLI_COMMAND_PARAMETER);
+
+    defined('DEFAULT_TIMEZONE') || define('DEFAULT_TIMEZONE', 'America/Los_Angeles');
+    $env->setEnvironmentVariable('DEFAULT_TIMEZONE', DEFAULT_TIMEZONE);
+
+    try {
+        new DateTimeZone(DEFAULT_TIMEZONE);
+    } catch (\Exception $e) {
+        throw new \Exception("Invalid DEFAULT_TIMEZONE in .env: " . DEFAULT_TIMEZONE . PHP_EOL);        
+    }
+
+}
+
+defined('FW_BP') || define('FW_BP', PROJECT_ROOT);
+defined('MAGENTO_BP') || define('MAGENTO_BP', PROJECT_ROOT);
+defined('TESTS_BP') || define('TESTS_BP', dirname(dirname(__DIR__)));
+
+$RELATIVE_TESTS_MODULE_PATH = '/tests/functional/tests/MFTF';
+defined('TESTS_MODULE_PATH') || define('TESTS_MODULE_PATH', realpath(TESTS_BP . $RELATIVE_TESTS_MODULE_PATH));
+
+
+// add the debug flag here
+$debug_mode = $_ENV['MFTF_DEBUG'] ?? false;
+if (!(bool)$debug_mode && extension_loaded('xdebug')) {
+    xdebug_disable();
 }

dev/tests/unit/Magento/FunctionalTestFramework/Test/Config/ActionGroupDomTest.php

@@ -25,7 +25,7 @@ public function testActionGroupDomStepKeyValidation()
          </actionGroups>";
 
         $exceptionCollector = new ExceptionCollector();
-        $actionDom = new ActionGroupDom($sampleXml, 'dupeStepKeyActionGroup.xml', $exceptionCollector);
+        new ActionGroupDom($sampleXml, 'dupeStepKeyActionGroup.xml', $exceptionCollector);
 
         $this->expectException(\Exception::class);
         $exceptionCollector->throwException();
@@ -47,4 +47,25 @@ public function testActionGroupDomInvalidXmlValidation()
         $this->expectExceptionMessage("XML Parse Error: invalid.xml\n");
         new ActionGroupDom($sampleXml, 'invalid.xml', $exceptionCollector);
     }
+
+    /**
+     * Test detection of two ActionGroups with the same Name in the same file.
+     */
+    public function testActionGroupDomDuplicateActionGroupsValidation()
+    {
+        $sampleXml = '<actionGroups>
+            <actionGroup name="actionGroupName">
+                <wait time="1" stepKey="key1" />
+            </actionGroup>
+            <actionGroup name="actionGroupName">
+                <wait time="1" stepKey="key1" />
+            </actionGroup>
+        </actionGroups>';
+
+        $exceptionCollector = new ExceptionCollector();
+        new ActionGroupDom($sampleXml, 'dupeNameActionGroup.xml', $exceptionCollector);
+        $this->expectException(\Exception::class);
+        $this->expectExceptionMessageRegExp("/name: actionGroupName is used more than once./");
+        $exceptionCollector->throwException();
+    }
 }

dev/tests/unit/Magento/FunctionalTestFramework/Test/Config/DomTest.php

@@ -0,0 +1,55 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Tests\unit\Magento\FunctionalTestFramework\Test\Config;
+
+use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector;
+use Magento\FunctionalTestingFramework\Config\Dom\ValidationException;
+use Magento\FunctionalTestingFramework\Test\Config\ActionGroupDom;
+use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+
+class DomTest extends MagentoTestCase
+{
+    /**
+     * Test Test duplicate step key validation
+     */
+    public function testTestStepKeyDuplicateValidation()
+    {
+        $sampleXml = '<tests>
+            <test name="testName">
+                <comment userInput="step1" stepKey="key1"/>
+                <comment userInput="step2" stepKey="key1"/>
+            </test>
+        </tests>';
+
+        $exceptionCollector = new ExceptionCollector();
+        new ActionGroupDom($sampleXml, 'dupeStepKeyTest.xml', $exceptionCollector);
+
+        $this->expectException(\Exception::class);
+        $this->expectExceptionMessageRegExp("/stepKey: key1 is used more than once. \(Parent: testName\)/");
+        $exceptionCollector->throwException();
+    }
+
+    /**
+     * Test detection of two Tests with the same Name in the same file.
+     */
+    public function testTestNameDuplicateValidation()
+    {
+        $sampleXml = '<tests>
+            <test name="testName">
+                <comment userInput="step1" stepKey="key1"/>
+            </test>
+            <test name="testName">
+                <comment userInput="step1" stepKey="key1"/>
+            </test>
+        </tests>';
+
+        $exceptionCollector = new ExceptionCollector();
+        new ActionGroupDom($sampleXml, 'dupeTestsTest.xml', $exceptionCollector);
+        $this->expectException(\Exception::class);
+        $this->expectExceptionMessageRegExp("/name: testName is used more than once./");
+        $exceptionCollector->throwException();
+    }
+}

dev/tests/unit/Magento/FunctionalTestFramework/Test/Objects/ActionObjectTest.php

@@ -370,6 +370,21 @@ public function testTooManyArgumentException()
         $actionObject->resolveReferences();
     }
 
+    /**
+     * Action object should throw an exception if the timezone provided is not valid.
+     */
+    public function testInvalidTimezoneException()
+    {
+        $this->expectException(TestReferenceException::class);
+
+        $actionObject = new ActionObject('key123', 'generateDate', [
+            'timezone' => "INVALID_TIMEZONE"
+        ]);
+
+        // Call the method under test
+        $actionObject->resolveReferences();
+    }
+
     private function mockSectionHandlerWithElement($elementObject)
     {
         $sectionObject = new SectionObject('SectionObject', ['elementObject' => $elementObject]);

dev/tests/unit/Magento/FunctionalTestFramework/Util/ActionMergeUtilTest.php

@@ -0,0 +1,34 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace Tests\unit\Magento\FunctionalTestFramework\Test\Handlers;
+
+use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
+use Magento\FunctionalTestingFramework\Test\Util\ActionMergeUtil;
+use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+
+class ActionMergeUtilTest extends MagentoTestCase
+{
+    /**
+     * Test Exception Handler for merging actions
+     *
+     * @throws \Exception
+     */
+    public function testMergeActionsException()
+    {
+        $testActionMergeUtil = new ActionMergeUtil(null, null);
+
+        $actionObject = new ActionObject('fakeAction', 'comment', [
+            'userInput' => '{{someEntity.entity}}'
+        ]);
+
+        $this->expectExceptionMessage("Could not resolve entity reference \"{{someEntity.entity}}\" " .
+            "in Action with stepKey \"fakeAction\".\n" .
+            "Exception occurred parsing action at StepKey \"fakeAction\"");
+
+        $testActionMergeUtil->resolveActionSteps(["merge123" => $actionObject]);
+    }
+}