Skip to content

Commit e2b88bf

Browse files
imeron2433imeron2433
authored
MQE-893: Add flag to robo generate: tests which accepts a specific set of tests to execute
MQE-893: Add flag to robo generate: tests which accepts a specific set of tests to execute
2 parents 734f356 + 81e078d commit e2b88bf

File tree

12 files changed

+372
-164
lines changed

12 files changed

+372
-164
lines changed

‎bin/static-checks

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ set -e
88
echo "==============================="
99
echo " CODE SNIFFER"
1010
echo "==============================="
11-
vendor/bin/phpcs ./src --standard=./dev/tests/static/Magento
11+
vendor/bin/phpcs ./src --standard=./dev/tests/static/Magento --ignore=src/Magento/FunctionalTestingFramework/Group,src/Magento/FunctionalTestingFramework/AcceptanceTester.php
1212
vendor/bin/phpcs ./dev/tests/unit --standard=./dev/tests/static/Magento
1313
vendor/bin/phpcs ./dev/tests/verification --standard=./dev/tests/static/Magento --ignore=dev/tests/verification/_generated
1414
echo ""
@@ -22,7 +22,7 @@ echo ""
2222
echo "==============================="
2323
echo " MESS DETECTOR"
2424
echo "==============================="
25-
vendor/bin/phpmd ./src text /dev/tests/static/Magento/CodeMessDetector/ruleset.xml --exclude _generated
25+
vendor/bin/phpmd ./src text /dev/tests/static/Magento/CodeMessDetector/ruleset.xml --exclude _generated,src/Magento/FunctionalTestingFramework/Group,src/Magento/FunctionalTestingFramework/AcceptanceTester.php
2626
echo ""
2727

2828
echo "==============================="

‎bin/static-checks.bat

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@
55

66
@echo off
77
@echo ===============================PHP CODE SNIFFER REPORT===============================
8-
call vendor\bin\phpcs .\src --standard=.\dev\tests\static\Magento
8+
call vendor\bin\phpcs .\src --standard=.\dev\tests\static\Magento --ignore=src\Magento\FunctionalTestingFramework\Group,src\Magento\FunctionalTestingFramework\AcceptanceTester.php
99
call vendor\bin\phpcs .\dev\tests\unit --standard=.\dev\tests\static\Magento
1010
call vendor\bin\phpcs .\dev\tests\verification --standard=.\dev\tests\static\Magento --ignore=dev\tests\verification\_generated
1111

1212
@echo ===============================COPY PASTE DETECTOR REPORT===============================
1313
call vendor\bin\phpcpd .\src
1414

1515
@echo "===============================PHP MESS DETECTOR REPORT===============================
16-
vendor\bin\phpmd .\src text \dev\tests\static\Magento\CodeMessDetector\ruleset.xml --exclude _generated
16+
vendor\bin\phpmd .\src text \dev\tests\static\Magento\CodeMessDetector\ruleset.xml --exclude _generated,src\Magento\FunctionalTestingFramework\Group,src\Magento\FunctionalTestingFramework\AcceptanceTester.php
1717

1818
@echo ===============================MAGENTO COPYRIGHT REPORT===============================
1919
echo msgbox "INFO:Copyright check currently not run as part of .bat implementation" > "%temp%\popup.vbs"

‎dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php

+1-11
Original file line numberDiff line numberDiff line change
@@ -76,15 +76,6 @@ public function testSortWithSuites()
7676

7777
AspectMock::double(TestObjectHandler::class, ['getInstance' => $mockHandler])->make();
7878

79-
// mock a suite object
80-
$mockSuite = AspectMock::double(SuiteObject::class, [
81-
'getBeforeHook' => null,
82-
'getAfterHook' => null,
83-
'getName' => 'mockSuite1'
84-
])->make();
85-
$mockSuiteHandler = AspectMock::double(SuiteObjectHandler::class, ['getObject' => $mockSuite])->make();
86-
AspectMock::double(SuiteObjectHandler::class, ['getInstance' => $mockSuiteHandler]);
87-
8879
// create test to size array
8980
$sampleTestArray = [
9081
'test1' => 100,
@@ -96,8 +87,7 @@ public function testSortWithSuites()
9687

9788
// create mock suite references
9889
$sampleSuiteArray = [
99-
'mockTest1' => ['mockSuite1'],
100-
'mockTest2' => ['mockSuite1']
90+
'mockSuite1' => ['mockTest1', 'mockTest2']
10191
];
10292

10393
// perform sort

‎src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php

+20
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,26 @@ public function getAllObjects()
7878
return $this->suiteObjects;
7979
}
8080

81+
/**
82+
* Function which return all tests referenced by suites.
83+
*
84+
* @return array
85+
*/
86+
public function getAllTestReferences()
87+
{
88+
$testsReferencedInSuites = [];
89+
$suites = $this->getAllObjects();
90+
91+
foreach ($suites as $suite) {
92+
/** @var SuiteObject $suite */
93+
$test_keys = array_keys($suite->getTests());
94+
$testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]);
95+
$testsReferencedInSuites = array_merge_recursive($testsReferencedInSuites, $testToSuiteName);
96+
}
97+
98+
return $testsReferencedInSuites;
99+
}
100+
81101
/**
82102
* Method to parse all suite data xml into objects.
83103
*

‎src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

+132-24
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,13 @@
66

77
namespace Magento\FunctionalTestingFramework\Suite;
88

9-
use Magento\Framework\Phrase;
10-
use Magento\Framework\Validator\Exception;
9+
use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
1110
use Magento\FunctionalTestingFramework\Suite\Generators\GroupClassGenerator;
1211
use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
1312
use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject;
13+
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
1414
use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
1515
use Magento\FunctionalTestingFramework\Util\Manifest\BaseTestManifest;
16-
use Magento\FunctionalTestingFramework\Util\Manifest\ParallelTestManifest;
1716
use Magento\FunctionalTestingFramework\Util\TestGenerator;
1817
use Symfony\Component\Yaml\Yaml;
1918

@@ -75,15 +74,23 @@ public static function getInstance()
7574
*/
7675
public function generateAllSuites($testManifest)
7776
{
78-
$suites = SuiteObjectHandler::getInstance()->getAllObjects();
79-
if (get_class($testManifest) == ParallelTestManifest::class) {
80-
/** @var ParallelTestManifest $testManifest */
81-
$suites = $testManifest->getSorter()->getResultingSuites();
77+
$suites = array_keys(SuiteObjectHandler::getInstance()->getAllObjects());
78+
if ($testManifest != null) {
79+
$suites = $testManifest->getSuiteConfig();
8280
}
8381

84-
foreach ($suites as $suite) {
85-
// during a parallel config run we must generate only after we have data around how a suite will be split
86-
$this->generateSuiteFromObject($suite);
82+
foreach ($suites as $suiteName => $suiteContent) {
83+
$firstElement = array_values($suiteContent)[0];
84+
85+
// if the first element is a string we know that we simply have an array of tests
86+
if (is_string($firstElement)) {
87+
$this->generateSuiteFromTest($suiteName, $suiteContent);
88+
}
89+
90+
// if our first element is an array we know that we have split the suites
91+
if (is_array($firstElement)) {
92+
$this->generateSplitSuiteFromTest($suiteName, $suiteContent);
93+
}
8794
}
8895
}
8996

@@ -96,11 +103,22 @@ public function getTestsReferencedInSuites()
96103
{
97104
$testsReferencedInSuites = [];
98105
$suites = SuiteObjectHandler::getInstance()->getAllObjects();
106+
107+
// see if we have a specific suite configuration.
108+
if (!empty($this->suiteReferences)) {
109+
$suites = array_intersect_key($suites, $this->suiteReferences);
110+
}
111+
99112
foreach ($suites as $suite) {
100113
/** @var SuiteObject $suite */
101114
$test_keys = array_keys($suite->getTests());
102-
$testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]);
103115

116+
// see if we need to filter which tests we'll be generating.
117+
if (array_key_exists($suite->getName(), $this->suiteReferences)) {
118+
$test_keys = $this->suiteReferences[$suite->getName()] ?? $test_keys;
119+
}
120+
121+
$testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]);
104122
$testsReferencedInSuites = array_merge_recursive($testsReferencedInSuites, $testToSuiteName);
105123
}
106124

@@ -117,35 +135,125 @@ public function getTestsReferencedInSuites()
117135
public function generateSuite($suiteName)
118136
{
119137
/**@var SuiteObject $suite **/
120-
$suite = SuiteObjectHandler::getInstance()->getObject($suiteName);
121-
$this->generateSuiteFromObject($suite);
138+
$this->generateSuiteFromTest($suiteName, []);
122139
}
123140

124141
/**
125-
* Function which takes a suite object and generates all relevant supporting files and classes.
142+
* Function which takes a suite name and a set of test names. The function then generates all relevant supporting
143+
* files and classes for the suite. The function takes an optional argument for suites which are split by a parallel
144+
* run so that any pre/post conditions can be duplicated.
126145
*
127-
* @param SuiteObject $suiteObject
146+
* @param string $suiteName
147+
* @param array $tests
148+
* @param string $originalSuiteName
128149
* @return void
129150
*/
130-
public function generateSuiteFromObject($suiteObject)
151+
private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteName = null)
131152
{
132-
$suiteName = $suiteObject->getName();
133153
$relativePath = TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . $suiteName;
134-
$fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath;
135-
$groupNamespace = null;
154+
$fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath . DIRECTORY_SEPARATOR;
136155

137156
DirSetupUtil::createGroupDir($fullPath);
138-
$this->generateRelevantGroupTests($suiteName, $suiteObject->getTests());
139157

140-
if ($suiteObject->requiresGroupFile()) {
141-
// if the suite requires a group file, generate it and set the namespace
142-
$groupNamespace = $this->groupClassGenerator->generateGroupClass($suiteObject);
158+
$relevantTests = [];
159+
if (!empty($tests)) {
160+
$this->validateTestsReferencedInSuite($suiteName, $tests, $originalSuiteName);
161+
foreach ($tests as $testName) {
162+
$relevantTests[$testName] = TestObjectHandler::getInstance()->getObject($testName);
163+
}
164+
} else {
165+
$relevantTests = SuiteObjectHandler::getInstance()->getObject($suiteName)->getTests();
143166
}
144167

168+
$this->generateRelevantGroupTests($suiteName, $relevantTests);
169+
$groupNamespace = $this->generateGroupFile($suiteName, $relevantTests, $originalSuiteName);
170+
145171
$this->appendEntriesToConfig($suiteName, $fullPath, $groupNamespace);
146172
print "Suite ${suiteName} generated to ${relativePath}.\n";
147173
}
148174

175+
/**
176+
* Function which validates tests passed in as custom configuration against the configuration defined by the user to
177+
* prevent possible invalid test configurations from executing.
178+
*
179+
* @param string $suiteName
180+
* @param array $testsReferenced
181+
* @param string $originalSuiteName
182+
* @return void
183+
* @throws TestReferenceException
184+
*/
185+
private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $originalSuiteName)
186+
{
187+
$suiteRef = $originalSuiteName ?? $suiteName;
188+
$possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests();
189+
$invalidTestRef = null;
190+
$errorMsg = "Cannot reference tests not declared as part of {$suiteRef}:\n ";
191+
192+
array_walk($testsReferenced, function ($value) use (&$invalidTestRef, $possibleTestRef, &$errorMsg) {
193+
if (!array_key_exists($value, $possibleTestRef)) {
194+
$invalidTestRef.= "\t{$value}\n";
195+
}
196+
});
197+
198+
if ($invalidTestRef != null) {
199+
throw new TestReferenceException($errorMsg . $invalidTestRef);
200+
}
201+
}
202+
203+
/**
204+
* Function for generating split groups of tests (following a parallel execution). Takes a paralle suite config
205+
* and generates applicable suites.
206+
*
207+
* @param string $suiteName
208+
* @param array $suiteContent
209+
* @return void
210+
*/
211+
private function generateSplitSuiteFromTest($suiteName, $suiteContent)
212+
{
213+
foreach ($suiteContent as $suiteSplitName => $tests) {
214+
$this->generateSuiteFromTest($suiteSplitName, $tests, $suiteName);
215+
}
216+
}
217+
218+
/**
219+
* Function which takes a suite name, array of tests, and an original suite name. The function takes these args
220+
* and generates a group file which captures suite level preconditions.
221+
*
222+
* @param string $suiteName
223+
* @param array $tests
224+
* @param string $originalSuiteName
225+
* @return null|string
226+
*/
227+
private function generateGroupFile($suiteName, $tests, $originalSuiteName)
228+
{
229+
// if there's an original suite name we know that this test came from a split group.
230+
if ($originalSuiteName) {
231+
// create the new suite object
232+
/** @var SuiteObject $originalSuite */
233+
$originalSuite = SuiteObjectHandler::getInstance()->getObject($originalSuiteName);
234+
$suiteObject = new SuiteObject(
235+
$suiteName,
236+
$tests,
237+
[],
238+
$originalSuite->getHooks()
239+
);
240+
} else {
241+
$suiteObject = SuiteObjectHandler::getInstance()->getObject($suiteName);
242+
// we have to handle the case when there is a custom configuration for an existing suite.
243+
if (count($suiteObject->getTests()) != count($tests)) {
244+
return $this->generateGroupFile($suiteName, $tests, $suiteName);
245+
}
246+
}
247+
248+
if (!$suiteObject->requiresGroupFile()) {
249+
// if we do not require a group file we don't need a namespace
250+
return null;
251+
}
252+
253+
// if the suite requires a group file, generate it and set the namespace
254+
return $this->groupClassGenerator->generateGroupClass($suiteObject);
255+
}
256+
149257
/**
150258
* Function which accepts a suite name and suite path and appends a new group entry to the codeception.yml.dist
151259
* file in order to register the set of tests as a new group. Also appends group object location if required
@@ -219,7 +327,7 @@ private static function clearPreviousSessionConfigEntries()
219327
private function generateRelevantGroupTests($path, $tests)
220328
{
221329
$testGenerator = TestGenerator::getInstance($path, $tests);
222-
$testGenerator->createAllTestFiles('suite');
330+
$testGenerator->createAllTestFiles(null, []);
223331
}
224332

225333
/**

‎src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php

+47-4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
namespace Magento\FunctionalTestingFramework\Util\Manifest;
88

9+
use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
10+
use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject;
911
use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
1012

1113
abstract class BaseTestManifest
@@ -24,16 +26,25 @@ abstract class BaseTestManifest
2426
*/
2527
protected $relativeDirPath;
2628

29+
/**
30+
* Suite configuration in the format suite name to test name. Overwritten during a custom configuration.
31+
*
32+
* @var array
33+
*/
34+
protected $suiteConfiguration;
35+
2736
/**
2837
* TestManifest constructor.
2938
*
3039
* @param string $path
3140
* @param string $runConfig
41+
* @param array $suiteConfiguration
3242
*/
33-
public function __construct($path, $runConfig)
43+
public function __construct($path, $runConfig, $suiteConfiguration)
3444
{
3545
$this->runTypeConfig = $runConfig;
3646
$this->relativeDirPath = substr($path, strlen(dirname(dirname(TESTS_BP))) + 1);
47+
$this->suiteConfiguration = $suiteConfiguration;
3748
}
3849

3950
/**
@@ -57,9 +68,41 @@ abstract public function addTest($testObject);
5768
/**
5869
* Function which generates the actual manifest(s) once the relevant tests have been added to the array.
5970
*
60-
* @param array $testsReferencedInSuites
61-
* @param int|null $nodes
6271
* @return void
6372
*/
64-
abstract public function generate($testsReferencedInSuites, $nodes = null);
73+
abstract public function generate();
74+
75+
/**
76+
* Getter for the suite configuration.
77+
*
78+
* @return array
79+
*/
80+
public function getSuiteConfig()
81+
{
82+
if ($this->suiteConfiguration === null) {
83+
return [];
84+
}
85+
86+
$suiteToTestNames = [];
87+
if (empty($this->suiteConfiguration)) {
88+
// if there is no configuration passed we can assume the user wants all suites generated as specified.
89+
foreach (SuiteObjectHandler::getInstance()->getAllObjects() as $suite => $suiteObj) {
90+
/** @var SuiteObject $suitObj */
91+
$suiteToTestNames[$suite] = array_keys($suiteObj->getTests());
92+
}
93+
} else {
94+
// we need to loop through the configuration to make sure we capture suites with no specific config
95+
foreach ($this->suiteConfiguration as $suiteName => $test) {
96+
if (empty($test)) {
97+
$suiteToTestNames[$suiteName] =
98+
array_keys(SuiteObjectHandler::getInstance()->getObject($suiteName)->getTests());
99+
continue;
100+
}
101+
102+
$suiteToTestNames[$suiteName] = $test;
103+
}
104+
}
105+
106+
return $suiteToTestNames;
107+
}
65108
}

0 commit comments

Comments
 (0)