Skip to content

Commit e1f94a0

Browse files
hubertgrzeskowiakmichael-o
hubertgrzeskowiak
authored and
michael-o
committed
[SUREFIRE-2276] JUnit5's TestTemplate failures treated as flakes with retries
This closes #788
1 parent d24adb4 commit e1f94a0

File tree

5 files changed

+222
-1
lines changed

5 files changed

+222
-1
lines changed

‎surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,45 @@ public void testJupiterEngineWithFailureInTestTemplateProvider() {
296296
.assertContainsText("Encountered failure in TestTemplate provideTestTemplateInvocationContexts()");
297297
}
298298

299+
@Test
300+
public void testJupiterEngineWithTestTemplateNotClassifiedAsFlake() {
301+
unpack("junit5-testtemplate-bug", "-" + jupiter)
302+
.setTestToRun("FieldSettingTest")
303+
.sysProp("junit5.version", jupiter)
304+
.maven()
305+
.withFailure()
306+
.executeTest()
307+
.verifyTextInLog("AssertionFailedError")
308+
.assertTestSuiteResults(2, 0, 1, 0, 0);
309+
310+
unpack("junit5-testtemplate-bug", "-" + jupiter)
311+
.debugLogging()
312+
.setTestToRun("FieldSettingTest")
313+
.sysProp("junit5.version", jupiter)
314+
// The tests are failing deterministically, so rerunning them should not change the result
315+
.sysProp("surefire.rerunFailingTestsCount", "1")
316+
.maven()
317+
.withFailure()
318+
.executeTest()
319+
.verifyTextInLog("AssertionFailedError")
320+
.assertTestSuiteResults(2, 0, 1, 0, 0);
321+
}
322+
323+
@Test
324+
public void testJupiterEngineWithParameterizedTestsNotClassifiedAsFlake() {
325+
unpack("junit5-testtemplate-bug", "-" + jupiter)
326+
.debugLogging()
327+
.setTestToRun("ParamsContextTest")
328+
.sysProp("junit5.version", jupiter)
329+
// The tests are failing deterministically, so rerunning them should not change the result
330+
.sysProp("surefire.rerunFailingTestsCount", "1")
331+
.maven()
332+
.withFailure()
333+
.executeTest()
334+
.verifyTextInLog("AssertionFailedError")
335+
.assertTestSuiteResults(2, 0, 1, 0, 0);
336+
}
337+
299338
@Test
300339
public void testJupiterEngineWithAssertionsFailNoParameters() {
301340
// `Assertions.fail()` not supported until 5.2.0

‎surefire-its/src/test/resources/junit5-testtemplate-bug/pom.xml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<groupId>org.apache.maven.plugins.surefire</groupId>
8+
<artifactId>surefire-junit-testtemplate-retry-bug</artifactId>
9+
<version>1.0-SNAPSHOT</version>
10+
<name>Test for JUnit 5 TestTemplate with retries</name>
11+
12+
<properties>
13+
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
14+
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
15+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
16+
</properties>
17+
18+
<dependencies>
19+
<dependency>
20+
<groupId>org.junit.jupiter</groupId>
21+
<artifactId>junit-jupiter-engine</artifactId>
22+
<version>${junit5.version}</version>
23+
<scope>test</scope>
24+
</dependency>
25+
<dependency>
26+
<groupId>org.junit.jupiter</groupId>
27+
<artifactId>junit-jupiter-api</artifactId>
28+
<version>${junit5.version}</version>
29+
<scope>test</scope>
30+
</dependency>
31+
<dependency>
32+
<groupId>org.junit.jupiter</groupId>
33+
<artifactId>junit-jupiter-params</artifactId>
34+
<version>${junit5.version}</version>
35+
<scope>test</scope>
36+
</dependency>
37+
</dependencies>
38+
<build>
39+
<plugins>
40+
<plugin>
41+
<artifactId>maven-compiler-plugin</artifactId>
42+
<version>3.8.1</version>
43+
<configuration>
44+
<encoding>UTF-8</encoding>
45+
</configuration>
46+
</plugin>
47+
<plugin>
48+
<groupId>org.apache.maven.plugins</groupId>
49+
<artifactId>maven-surefire-plugin</artifactId>
50+
<version>${surefire.version}</version>
51+
</plugin>
52+
</plugins>
53+
</build>
54+
55+
</project>

‎surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/FieldSettingTest.java

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package pkg;
2+
3+
import org.junit.jupiter.api.TestTemplate;
4+
import org.junit.jupiter.api.extension.BeforeEachCallback;
5+
import org.junit.jupiter.api.extension.ExtendWith;
6+
import org.junit.jupiter.api.extension.Extension;
7+
import org.junit.jupiter.api.extension.ExtensionContext;
8+
import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
9+
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
10+
11+
import java.util.Arrays;
12+
import java.util.List;
13+
import java.util.stream.Stream;
14+
15+
import static org.junit.jupiter.api.Assertions.assertEquals;
16+
17+
public class FieldSettingTest {
18+
private int testValue = 42;
19+
20+
// We're calling this in the provider underneath
21+
public void setTestValue(int testValue) {
22+
this.testValue = testValue;
23+
}
24+
25+
@TestTemplate
26+
@ExtendWith(FieldSettingContextProvider.class)
27+
public void testTemplatePartiallyFails() {
28+
assertEquals(42, testValue);
29+
}
30+
}
31+
32+
33+
class FieldSettingContextProvider implements TestTemplateInvocationContextProvider {
34+
@Override
35+
public boolean supportsTestTemplate(ExtensionContext extensionContext) {
36+
return true;
37+
}
38+
39+
@Override
40+
public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(ExtensionContext extensionContext) {
41+
return Stream.of(context(0), context(42));
42+
}
43+
44+
private TestTemplateInvocationContext context(int parameter) {
45+
return new TestTemplateInvocationContext() {
46+
@Override
47+
public String getDisplayName(int invocationIndex) {
48+
return String.format("[%d] %s", invocationIndex, parameter);
49+
}
50+
51+
@Override
52+
public List<Extension> getAdditionalExtensions() {
53+
return getBeforeEachCallbacks(parameter);
54+
}
55+
};
56+
}
57+
58+
private List<Extension> getBeforeEachCallbacks(int value) {
59+
return Arrays.asList((BeforeEachCallback) ctx ->
60+
((FieldSettingTest) ctx.getRequiredTestInstance()).setTestValue(value)
61+
);
62+
}
63+
}

‎surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/ParamsContextTest.java

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package pkg;
2+
3+
import org.junit.jupiter.api.TestTemplate;
4+
import org.junit.jupiter.api.extension.ExtendWith;
5+
import org.junit.jupiter.api.extension.Extension;
6+
import org.junit.jupiter.api.extension.ExtensionContext;
7+
import org.junit.jupiter.api.extension.ParameterContext;
8+
import org.junit.jupiter.api.extension.ParameterResolver;
9+
import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
10+
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
11+
12+
import java.util.Collections;
13+
import java.util.List;
14+
import java.util.stream.Stream;
15+
16+
import static org.junit.jupiter.api.Assertions.assertEquals;
17+
18+
public class ParamsContextTest {
19+
20+
@TestTemplate
21+
@ExtendWith(ParamsContextProvider.class)
22+
void testTemplatePartiallyFails(int value) {
23+
assertEquals(42, value);
24+
}
25+
}
26+
27+
class ParamsContextProvider implements TestTemplateInvocationContextProvider {
28+
29+
@Override
30+
public boolean supportsTestTemplate(ExtensionContext context) {
31+
return true;
32+
}
33+
34+
@Override
35+
public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(ExtensionContext context) {
36+
return Stream.of(invocationContext(0), invocationContext(42));
37+
}
38+
39+
private TestTemplateInvocationContext invocationContext(int parameter) {
40+
return new TestTemplateInvocationContext() {
41+
@Override
42+
public String getDisplayName(int invocationIndex) {
43+
return String.format("[%d] %s", invocationIndex, parameter);
44+
}
45+
46+
@Override
47+
public List<Extension> getAdditionalExtensions() {
48+
return Collections.singletonList(new ParameterResolver() {
49+
@Override
50+
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) {
51+
return true;
52+
}
53+
54+
@Override
55+
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) {
56+
return parameter;
57+
}
58+
});
59+
}
60+
};
61+
}
62+
}

‎surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,12 +317,14 @@ private String[] toClassMethodName(TestIdentifier testIdentifier) {
317317
boolean needsSpaceSeparator = isNotBlank(parentDisplay) && !display.startsWith("[");
318318
String methodDisplay = parentDisplay + (needsSpaceSeparator ? " " : "") + display;
319319

320+
boolean isParameterized = isNotBlank(methodSource.getMethodParameterTypes());
320321
boolean hasParameterizedParent = collectAllTestIdentifiersInHierarchy(testIdentifier)
321322
.filter(identifier -> !identifier.getSource().isPresent())
322323
.map(TestIdentifier::getLegacyReportingName)
323324
.anyMatch(legacyReportingName -> legacyReportingName.matches("^\\[.+]$"));
325+
boolean isTestTemplate = testIdentifier.getLegacyReportingName().matches("^.*\\[\\d+]$");
324326

325-
boolean parameterized = isNotBlank(methodSource.getMethodParameterTypes()) || hasParameterizedParent;
327+
boolean parameterized = isParameterized || hasParameterizedParent || isTestTemplate;
326328
String methodName = methodSource.getMethodName();
327329
String description = testIdentifier.getLegacyReportingName();
328330
boolean equalDescriptions = methodDisplay.equals(description);

0 commit comments

Comments
 (0)