Skip to content

Commit 5a54a6a

Browse files
westsewestse
authored
Improve backward-compatibility testing (#546)
- Attempts to be exhaustive on backward-compatibility coverage to raise confidence that future refactors will not introduce unintended regressions. - Each incompatible condition is tested separately by comparing two spec files where the only difference is that fine-grained incompatible condition. - In some cases the current behavior appears incorrect. Tests are still added for these to avoid unintended regression, but are given TODO comments for later follow-up. - Pre-existing tests have been removed if they are redundant to avoid confusion and to follow the convention. Closes #545
1 parent c0fa0fe commit 5a54a6a

File tree

142 files changed

+5407
-829
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

142 files changed

+5407
-829
lines changed

‎core/src/main/java/org/openapitools/openapidiff/core/model/ChangedSecurityScheme.java

+7
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ public DiffResult isCoreChanged() {
4444
&& !changedBearerFormat
4545
&& !changedOpenIdConnectUrl
4646
&& (changedScopes == null || changedScopes.getIncreased().isEmpty())) {
47+
48+
// TODO: Dead code removal opportunity for changedType and changedIn. It appears that
49+
// SecuritySchemaDiff will never be given the chance to detect differences TYPE and
50+
// IN differences because that case has already been detected and filtered out by
51+
// SecurityRequirementsDiff and recorded as a dropped requirement in
52+
// ChangedSecurityRequirements.
53+
4754
return DiffResult.COMPATIBLE;
4855
}
4956
return DiffResult.INCOMPATIBLE;

‎core/src/test/java/org/openapitools/openapidiff/core/BackwardCompatibilityTest.java

-49
This file was deleted.

‎core/src/test/java/org/openapitools/openapidiff/core/TestUtils.java

+15
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import static org.slf4j.LoggerFactory.getLogger;
55

66
import org.openapitools.openapidiff.core.model.ChangedOpenApi;
7+
import org.openapitools.openapidiff.core.model.DiffResult;
78
import org.slf4j.Logger;
89

910
public class TestUtils {
@@ -25,6 +26,20 @@ public static void assertOpenApiChangedEndpoints(String oldSpec, String newSpec)
2526
assertThat(changedOpenApi.getChangedOperations()).isNotEmpty();
2627
}
2728

29+
public static void assertSpecUnchanged(String oldSpec, String newSpec) {
30+
ChangedOpenApi changedOpenApi = OpenApiCompare.fromLocations(oldSpec, newSpec);
31+
LOG.info("Result: {}", changedOpenApi.isChanged().getValue());
32+
assertThat(changedOpenApi.isUnchanged()).isTrue();
33+
}
34+
35+
public static void assertSpecChangedButCompatible(String oldSpec, String newSpec) {
36+
ChangedOpenApi changedOpenApi = OpenApiCompare.fromLocations(oldSpec, newSpec);
37+
DiffResult diffResult = changedOpenApi.isChanged();
38+
LOG.info("Result: {}", diffResult.getValue());
39+
assertThat(diffResult.isDifferent()).isTrue();
40+
assertThat(diffResult.isCompatible()).isTrue();
41+
}
42+
2843
public static void assertOpenApiBackwardCompatible(
2944
String oldSpec, String newSpec, boolean isDiff) {
3045
ChangedOpenApi changedOpenApi = OpenApiCompare.fromLocations(oldSpec, newSpec);

‎core/src/test/java/org/openapitools/openapidiff/core/backcompat/ApiResponseBCTest.java

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package org.openapitools.openapidiff.core.backcompat;
2+
3+
import static org.openapitools.openapidiff.core.TestUtils.assertOpenApiBackwardIncompatible;
4+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecChangedButCompatible;
5+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecUnchanged;
6+
7+
import org.junit.jupiter.api.Test;
8+
9+
public class ApiResponseBCTest {
10+
private final String BASE = "bc_response_apiresponse_base.yaml";
11+
12+
@Test
13+
public void unchanged() {
14+
assertSpecUnchanged(BASE, BASE);
15+
}
16+
17+
@Test
18+
public void changedButCompatible() {
19+
assertSpecChangedButCompatible(BASE, "bc_response_apiresponse_changed_but_compatible.yaml");
20+
}
21+
22+
@Test
23+
public void decreased() {
24+
assertOpenApiBackwardIncompatible(BASE, "bc_response_apiresponse_decreased.yaml");
25+
}
26+
}

‎core/src/test/java/org/openapitools/openapidiff/core/backcompat/ContentBCTest.java

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package org.openapitools.openapidiff.core.backcompat;
2+
3+
import static org.openapitools.openapidiff.core.TestUtils.assertOpenApiBackwardIncompatible;
4+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecChangedButCompatible;
5+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecUnchanged;
6+
7+
import org.junit.jupiter.api.Test;
8+
9+
public class ContentBCTest {
10+
private final String BASE = "bc_content_base.yaml";
11+
12+
@Test
13+
public void unchanged() {
14+
assertSpecUnchanged(BASE, BASE);
15+
}
16+
17+
@Test
18+
public void changedButCompatible() {
19+
assertSpecChangedButCompatible(BASE, "bc_content_changed_but_compatible.yaml");
20+
}
21+
22+
@Test
23+
public void requestDecreased() {
24+
assertOpenApiBackwardIncompatible(BASE, "bc_request_content_decreased.yaml");
25+
}
26+
27+
@Test
28+
public void responseDecreased() {
29+
assertOpenApiBackwardIncompatible(BASE, "bc_response_content_decreased.yaml");
30+
}
31+
}

‎core/src/test/java/org/openapitools/openapidiff/core/backcompat/EnumBCTest.java

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package org.openapitools.openapidiff.core.backcompat;
2+
3+
import static org.openapitools.openapidiff.core.TestUtils.assertOpenApiBackwardIncompatible;
4+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecChangedButCompatible;
5+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecUnchanged;
6+
7+
import org.junit.jupiter.api.Test;
8+
9+
public class EnumBCTest {
10+
private final String BASE = "bc_enum_base.yaml";
11+
12+
@Test
13+
public void unchanged() {
14+
assertSpecUnchanged(BASE, BASE);
15+
}
16+
17+
@Test
18+
public void changedButCompatible() {
19+
assertSpecChangedButCompatible(BASE, "bc_enum_changed_but_compatible.yaml");
20+
}
21+
22+
@Test
23+
public void requestDecreased() {
24+
assertOpenApiBackwardIncompatible(BASE, "bc_request_enum_decreased.yaml");
25+
}
26+
27+
@Test
28+
public void responseIncreased() {
29+
assertOpenApiBackwardIncompatible(BASE, "bc_response_enum_increased.yaml");
30+
}
31+
}

‎core/src/test/java/org/openapitools/openapidiff/core/backcompat/HeaderBCTest.java

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package org.openapitools.openapidiff.core.backcompat;
2+
3+
import static org.openapitools.openapidiff.core.TestUtils.assertOpenApiBackwardIncompatible;
4+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecChangedButCompatible;
5+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecUnchanged;
6+
7+
import org.junit.jupiter.api.Test;
8+
9+
public class HeaderBCTest {
10+
private final String BASE = "bc_response_header_base.yaml";
11+
12+
@Test
13+
public void responseHeaderUnchanged() {
14+
assertSpecUnchanged(BASE, BASE);
15+
}
16+
17+
@Test
18+
public void responseHeaderDeprecated() {
19+
assertSpecChangedButCompatible(BASE, "bc_response_header_deprecated.yaml");
20+
}
21+
22+
@Test
23+
public void responseHeaderRequiredAdded() {
24+
assertOpenApiBackwardIncompatible(BASE, "bc_response_header_required_added.yaml");
25+
}
26+
27+
@Test
28+
public void responseHeaderRequiredDeleted() {
29+
assertOpenApiBackwardIncompatible(BASE, "bc_response_header_required_deleted.yaml");
30+
}
31+
32+
@Test
33+
public void responseHeaderExplode() {
34+
String RESPONSE_HEADER_EXPLODE = "bc_response_header_explode.yaml";
35+
assertOpenApiBackwardIncompatible(BASE, RESPONSE_HEADER_EXPLODE);
36+
}
37+
}

‎core/src/test/java/org/openapitools/openapidiff/core/backcompat/HeadersBCTest.java

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package org.openapitools.openapidiff.core.backcompat;
2+
3+
import static org.openapitools.openapidiff.core.TestUtils.assertOpenApiBackwardIncompatible;
4+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecChangedButCompatible;
5+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecUnchanged;
6+
7+
import org.junit.jupiter.api.Test;
8+
9+
public class HeadersBCTest {
10+
private final String BASE = "bc_response_headers_base.yaml";
11+
12+
@Test
13+
public void responseHeadersUnchanged() {
14+
assertSpecUnchanged(BASE, BASE);
15+
}
16+
17+
@Test
18+
public void responseHeadersIncreased() {
19+
assertSpecChangedButCompatible(BASE, "bc_response_headers_increased.yaml");
20+
}
21+
22+
@Test
23+
public void responseHeadersMissing() {
24+
assertOpenApiBackwardIncompatible(BASE, "bc_response_headers_missing.yaml");
25+
}
26+
}

‎core/src/test/java/org/openapitools/openapidiff/core/backcompat/MaxLengthBCTest.java

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package org.openapitools.openapidiff.core.backcompat;
2+
3+
import static org.openapitools.openapidiff.core.TestUtils.assertOpenApiBackwardIncompatible;
4+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecUnchanged;
5+
6+
import org.junit.jupiter.api.Test;
7+
8+
public class MaxLengthBCTest {
9+
private final String BASE = "bc_maxlength_base.yaml";
10+
11+
@Test
12+
public void maxLengthUnchanged() {
13+
assertSpecUnchanged(BASE, BASE);
14+
}
15+
16+
@Test
17+
public void requestMaxLengthDecreased() {
18+
assertOpenApiBackwardIncompatible(BASE, "bc_request_maxlength_decreased.yaml");
19+
}
20+
21+
@Test
22+
public void responseMaxLengthIncreased() {
23+
assertOpenApiBackwardIncompatible(BASE, "bc_response_maxlength_increased.yaml");
24+
}
25+
}

‎core/src/test/java/org/openapitools/openapidiff/core/backcompat/NumericRangeBCTest.java

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package org.openapitools.openapidiff.core.backcompat;
2+
3+
import static org.openapitools.openapidiff.core.TestUtils.assertOpenApiBackwardIncompatible;
4+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecChangedButCompatible;
5+
import static org.openapitools.openapidiff.core.TestUtils.assertSpecUnchanged;
6+
7+
import org.junit.jupiter.api.Test;
8+
9+
public class NumericRangeBCTest {
10+
private final String BASE = "bc_numericrange_base.yaml";
11+
12+
@Test
13+
public void unchanged() {
14+
assertSpecUnchanged(BASE, BASE);
15+
}
16+
17+
@Test
18+
public void changedButCompatible() {
19+
// TODO: Fix bug where response range-narrowing is deemed incompatible, then test here
20+
assertSpecChangedButCompatible(BASE, "bc_numericrange_changed_but_compatible.yaml");
21+
}
22+
23+
@Test
24+
public void requestExclusiveMaxCreated() {
25+
assertOpenApiBackwardIncompatible(BASE, "bc_request_numericrange_exclusive_max_created.yaml");
26+
}
27+
28+
@Test
29+
public void requestExclusiveMaxSet() {
30+
assertOpenApiBackwardIncompatible(BASE, "bc_request_numericrange_exclusive_max_set.yaml");
31+
}
32+
33+
@Test
34+
public void requestExclusiveMinCreated() {
35+
assertOpenApiBackwardIncompatible(BASE, "bc_request_numericrange_exclusive_min_created.yaml");
36+
}
37+
38+
@Test
39+
public void requestExclusiveMinSet() {
40+
assertOpenApiBackwardIncompatible(BASE, "bc_request_numericrange_exclusive_min_set.yaml");
41+
}
42+
43+
@Test
44+
public void requestMaxAdded() {
45+
assertOpenApiBackwardIncompatible(BASE, "bc_request_numericrange_max_added.yaml");
46+
}
47+
48+
@Test
49+
public void requestMaxDecreased() {
50+
assertOpenApiBackwardIncompatible(BASE, "bc_request_numericrange_max_decreased.yaml");
51+
}
52+
53+
@Test
54+
public void requestMinAdded() {
55+
assertOpenApiBackwardIncompatible(BASE, "bc_request_numericrange_min_added.yaml");
56+
}
57+
58+
@Test
59+
public void requestMinIncreased() {
60+
assertOpenApiBackwardIncompatible(BASE, "bc_request_numericrange_min_increased.yaml");
61+
}
62+
63+
@Test
64+
public void responseExclusiveMaxDeleted() {
65+
// TODO: Should be incompatible because clients may be unprepared for wider range
66+
// (test added to avoid unintentional regression)
67+
assertSpecChangedButCompatible(BASE, "bc_response_numericrange_exclusive_max_deleted.yaml");
68+
}
69+
70+
@Test
71+
public void responseExclusiveMaxUnset() {
72+
assertOpenApiBackwardIncompatible(BASE, "bc_response_numericrange_exclusive_max_unset.yaml");
73+
}
74+
75+
@Test
76+
public void responseExclusiveMinDeleted() {
77+
// TODO: Should be incompatible because clients may be unprepared for wider range
78+
// (test added to avoid unintentional regression)
79+
assertSpecChangedButCompatible(BASE, "bc_response_numericrange_exclusive_min_deleted.yaml");
80+
}
81+
82+
@Test
83+
public void responseExclusiveMinUnset() {
84+
assertOpenApiBackwardIncompatible(BASE, "bc_response_numericrange_exclusive_min_unset.yaml");
85+
}
86+
87+
@Test
88+
public void responseMaxDeleted() {
89+
// TODO: Should be incompatible because clients may be unprepared for wider range
90+
// (test added to avoid unintentional regression)
91+
assertSpecChangedButCompatible(BASE, "bc_response_numericrange_max_deleted.yaml");
92+
}
93+
94+
@Test
95+
public void responseMaxIncreased() {
96+
// TODO: Should be incompatible because clients may be unprepared
97+
// (test added to avoid unintentional regression)
98+
assertSpecChangedButCompatible(BASE, "bc_response_numericrange_max_increased.yaml");
99+
}
100+
101+
@Test
102+
public void responseMinDecreased() {
103+
// TODO: Should be incompatible because clients may be unprepared for wider range
104+
// (test added to avoid unintentional regression)
105+
assertSpecChangedButCompatible(BASE, "bc_response_numericrange_min_decreased.yaml");
106+
}
107+
108+
@Test
109+
public void responseMinDeleted() {
110+
// TODO: Should be incompatible because clients may be unprepared for wider range
111+
// (test added to avoid unintentional regression)
112+
assertSpecChangedButCompatible(BASE, "bc_response_numericrange_min_deleted.yaml");
113+
}
114+
}

0 commit comments

Comments
 (0)