Skip to content

Commit c99d928

Browse files
eramongodberamongodb
authored
CXX-2665 audit and refactor EVG matrices to use supported distros and extended coverage (#1328)
* Silence venv activation error messages during post-task commands * Print uncaught exception message during component execution * Avoid failure to spawn mongocryptd from failing API examples * Address -Wsign-conversion warnings on ubuntu2204-arm64 * Fix expansion of uninstall directories on RHEL * Add TEST_WITH_CSFLE to opt out of CSFLE setup routines * Move handling of lib vs. lib64 from config into scripts * Use DYLD_FALLBACK_LIBRARY_PATH instead of DYLD_LIBRARY_PATH * Use Astral UV to acquire clang-tidy on RHEL * Refactor matrices to use highest-availability hosts * Use rhel80 as the default Linux distro of choice * Remove unused distros from distros.py * Reduce large+small distro boilerplate * Group tasks on non-limited distros into display tasks * Ignore std::bad_alloc exceptions when allocating examples::big_string * Reduce tasks on limited distros to csfle, latest, replica only * Update path to llvm-symbolizer for print_stacktrace=1 output * Use consistent LLVM toolchain for compiler and symbolizer
1 parent 5d1930f commit c99d928

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+14226
-2204
lines changed

‎.evergreen/config_generator/components/atlas_search_indexes.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ def functions():
4646

4747

4848
def tasks():
49-
distro_name = 'ubuntu2004'
49+
distro_name = 'rhel80'
5050
distro = find_large_distro(distro_name)
5151

5252
return [

‎.evergreen/config_generator/components/clang_tidy.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from config_generator.components.funcs.install_c_driver import InstallCDriver
2+
from config_generator.components.funcs.install_uv import InstallUV
23
from config_generator.components.funcs.setup import Setup
34

45
from config_generator.etc.distros import compiler_to_vars, find_small_distro
@@ -24,6 +25,7 @@ class ClangTidy(Function):
2425
'cc_compiler',
2526
'cxx_compiler',
2627
'distro_id',
28+
'UV_INSTALL_DIR',
2729
],
2830
script='etc/run-clang-tidy.sh',
2931
)
@@ -42,7 +44,7 @@ def functions():
4244

4345

4446
def tasks():
45-
distro_name = 'ubuntu2004'
47+
distro_name = 'rhel80'
4648
distro = find_small_distro(distro_name)
4749

4850
return [
@@ -52,6 +54,7 @@ def tasks():
5254
run_on=distro.name,
5355
commands=[
5456
Setup.call(),
57+
InstallUV.call(),
5558
InstallCDriver.call(compiler='clang'),
5659
ClangTidy.call(compiler='clang'),
5760
],

‎.evergreen/config_generator/components/compile_only.py

Lines changed: 76 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
from config_generator.components.funcs.compile import Compile
22
from config_generator.components.funcs.install_uv import InstallUV
3-
from config_generator.components.funcs.fetch_c_driver_source import FetchCDriverSource
3+
from config_generator.components.funcs.install_c_driver import InstallCDriver
44
from config_generator.components.funcs.setup import Setup
55

6-
from config_generator.etc.distros import find_large_distro, make_distro_str
6+
from config_generator.etc.distros import compiler_to_vars, find_large_distro, make_distro_str
77

8-
from shrub.v3.evg_build_variant import BuildVariant
8+
from shrub.v3.evg_build_variant import BuildVariant, DisplayTask
9+
from shrub.v3.evg_command import KeyValueParam, expansions_update
910
from shrub.v3.evg_task import EvgTask, EvgTaskRef
1011

1112
from itertools import product
@@ -17,112 +18,110 @@
1718
# pylint: disable=line-too-long
1819
# fmt: off
1920
MATRIX = [
20-
('rhel79', None, ['Release'], ['shared'], ['impls']),
21+
('rhel80', 'gcc', ['Debug', 'Release'], ['shared', 'static'], [11, 17, 20, 23]),
22+
('rhel80', 'clang', ['Debug', 'Release'], ['shared', 'static'], [11, 17, 20, 23]),
2123

22-
('rhel81-power8', None, ['Release'], ['shared'], [None]),
23-
('rhel83-zseries', None, ['Release'], ['shared'], [None]),
24+
('ubuntu2004-arm64', 'gcc', ['Debug', 'Release'], ['shared', 'static'], [11, 17, 20, 23]),
25+
('ubuntu2004-arm64', 'clang', ['Debug', 'Release'], ['shared', 'static'], [11, 17, 20, 23]),
2426

25-
('ubuntu2004', None, ['Debug'], ['shared'], [None]),
26-
('ubuntu2004', 'gcc', ['Debug'], ['shared'], [None]),
27-
('ubuntu2004', 'clang', ['Debug'], ['shared'], [None]),
27+
('rhel8-power', None, ['Debug', 'Release'], ['shared', 'static'], [11, 17]),
28+
('rhel8-zseries', None, ['Debug', 'Release'], ['shared', 'static'], [11, 17]),
2829

29-
('windows-64-vs2015', 'vs2015x64', ['Debug', 'Release'], ['shared'], [None]),
30-
('windows-vsCurrent', 'vs2017x64', ['Debug', 'Release'], ['shared'], [None]),
31-
('windows-vsCurrent', 'vs2019x64', ['Debug', 'Release'], ['shared'], [None]),
32-
('windows-vsCurrent', 'vs2022x64', ['Debug', 'Release'], ['shared'], [None]),
30+
('macos-14-arm64', None, ['Debug', 'Release'], ['shared', 'static'], [11, 17]),
31+
('macos-14', None, ['Debug', 'Release'], ['shared', 'static'], [11, 17]),
32+
33+
('windows-64-vs2015', 'vs2015x64', ['Debug', 'Release'], ['shared', 'static'], [11, ]), # CXX-3215
34+
('windows-vsCurrent', 'vs2017x64', ['Debug', 'Release'], ['shared', 'static'], [11, 17, ]),
35+
('windows-vsCurrent', 'vs2019x64', ['Debug', 'Release'], ['shared', 'static'], [11, 17, 20, ]),
36+
('windows-vsCurrent', 'vs2022x64', ['Debug', 'Release'], ['shared', 'static'], [11, 17, 20, 23]),
3337
]
3438
# fmt: on
3539
# pylint: enable=line-too-long
3640

3741

38-
def generate_tasks():
39-
res = []
40-
41-
for distro_name, compiler, build_types, link_types, polyfills in MATRIX:
42-
for build_type, link_type, polyfill in product(build_types, link_types, polyfills):
42+
def tasks():
43+
for distro_name, compiler, build_types, link_types, cxx_standards in MATRIX:
44+
for build_type, link_type, cxx_standard in product(build_types, link_types, cxx_standards):
4345
distro = find_large_distro(distro_name)
4446

4547
name = f'{TAG}-{make_distro_str(distro_name, compiler, None)}'
4648
tags = [TAG, distro_name]
4749

50+
if cxx_standard is not None:
51+
name += f'-cxx{cxx_standard}'
52+
tags += [f'cxx{cxx_standard}']
53+
4854
if compiler is not None:
4955
tags.append(compiler)
5056

5157
name += f'-{build_type.lower()}-{link_type}'
5258
tags += [build_type.lower(), link_type]
5359

54-
if polyfill is not None:
55-
name += f'-{polyfill}'
56-
tags.append(polyfill)
60+
updates = []
61+
compile_vars = {}
62+
63+
updates += [KeyValueParam(key='build_type', value=build_type)]
64+
updates += [KeyValueParam(key=key, value=value) for key, value in compiler_to_vars(compiler).items()]
5765

58-
patchable = None
66+
if cxx_standard is not None:
67+
compile_vars |= {'REQUIRED_CXX_STANDARD': cxx_standard}
68+
69+
if link_type == 'static':
70+
compile_vars |= {'USE_STATIC_LIBS': 1}
5971

6072
# PowerPC and zSeries are limited resources.
61-
if any(pattern in distro_name for pattern in ['power8', 'zseries']):
62-
patchable = False
63-
64-
# In etc/calc_release_version.py:
65-
# error: unknown option `format=...'
66-
# usage: git tag ...
67-
# or: ...
68-
if distro_name == 'rhel79':
69-
patchable = False
70-
71-
res.append(
72-
EvgTask(
73-
name=name,
74-
tags=tags,
75-
run_on=distro.name,
76-
patchable=patchable,
77-
commands=[
78-
Setup.call(),
79-
FetchCDriverSource.call(),
80-
] + (
81-
# DEVPROD-13875 + astral-sh/uv/issues/10231.
82-
[] if "vs2015" in distro_name else [InstallUV.call()]
83-
) + [
84-
Compile.call(
85-
build_type=build_type,
86-
compiler=compiler,
87-
polyfill=polyfill,
88-
)
89-
],
73+
patchable = False if any(pattern in distro_name for pattern in ['power', 'zseries']) else None
74+
75+
commands = [expansions_update(updates=updates)] if updates else []
76+
commands += [
77+
Setup.call(),
78+
InstallCDriver.call(),
79+
] + (
80+
# DEVPROD-13875 + astral-sh/uv/issues/10231.
81+
[] if "vs2015" in distro_name else [InstallUV.call()]
82+
) + [
83+
Compile.call(
84+
build_type=build_type,
85+
compiler=compiler,
9086
)
87+
]
88+
89+
yield EvgTask(
90+
name=name,
91+
tags=tags,
92+
run_on=distro.name,
93+
patchable=patchable,
94+
commands=commands,
9195
)
9296

93-
return res
94-
95-
96-
TASKS = generate_tasks()
97-
98-
99-
def tasks():
100-
res = TASKS.copy()
101-
102-
# PowerPC and zSeries are limited resources.
103-
for task in res:
104-
if any(pattern in task.run_on for pattern in ["power8", "zseries"]):
105-
task.patchable = False
106-
107-
return res
108-
10997

11098
def variants():
11199
tasks = []
112100

113101
one_day = 1440 # Seconds.
114102

115103
# PowerPC and zSeries are limited resources.
116-
tasks = [
117-
EvgTaskRef(name=f'.{TAG} .rhel81-power8', batchtime=one_day),
118-
EvgTaskRef(name=f'.{TAG} .rhel83-zseries', batchtime=one_day),
119-
EvgTaskRef(name=f'.{TAG} !.rhel81-power8 !.rhel83-zseries'),
104+
limited_distros = [
105+
'rhel8-power',
106+
'rhel8-zseries',
120107
]
121108

122-
return [
123-
BuildVariant(
124-
name=f'{TAG}-matrix',
125-
display_name=f'{TAG}-matrix',
126-
tasks=tasks,
127-
),
109+
distros = sorted(list({entry[0] for entry in MATRIX}))
110+
batched = [distro for distro in distros if distro in limited_distros]
111+
tasks = [
112+
EvgTaskRef(name=f'.{TAG} .{distro}', batchtime=one_day) for distro in batched
113+
] + [
114+
EvgTaskRef(name=f'.{TAG}' + ''.join(f' !.{distro}' for distro in batched))
128115
]
116+
117+
yield BuildVariant(
118+
name=f'{TAG}-matrix',
119+
display_name=f'{TAG}-matrix',
120+
tasks=tasks,
121+
display_tasks=[
122+
DisplayTask(
123+
name=f'{TAG}-matrix',
124+
execution_tasks=[f'.{TAG}' + ''.join(f' !.{distro}' for distro in batched)],
125+
)
126+
],
127+
)

‎.evergreen/config_generator/components/docker_build.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
# pylint: disable=line-too-long
1616
# fmt: off
1717
MATRIX = [
18-
('ubuntu2204-arm64'),
19-
('ubuntu2204'),
18+
('ubuntu2204-arm64'), # `docker` is not available on RHEL distros by default.
19+
('ubuntu2204'), # `docker` is not available on RHEL distros by default.
2020
]
2121
# fmt: on
2222
# pylint: enable=line-too-long

‎.evergreen/config_generator/components/funcs/stop_mongod.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ class StopMongod(Function):
1313
set -o pipefail
1414
if cd drivers-evergreen-tools/.evergreen/orchestration 2>/dev/null; then
1515
. ../venv-utils.sh
16-
if venvactivate venv; then
17-
mongo-orchestration stop
16+
if venvactivate venv 2>/dev/null; then
17+
mongo-orchestration stop
1818
fi
1919
fi
2020
'''

‎.evergreen/config_generator/components/funcs/test.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ class Test(Function):
2929
'example_projects_cxxflags',
3030
'example_projects_ldflags',
3131
'generator',
32-
'lib_dir',
3332
'MONGOCXX_TEST_TOPOLOGY',
3433
'MONGODB_API_VERSION',
3534
'platform',

‎.evergreen/config_generator/components/funcs/test_atlas_connectivity.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,11 @@ class TestAtlasConnectivity(Function):
1313
script='''\
1414
export MONGOC_INSTALL_PREFIX=$(pwd)/../mongoc
1515
export MONGOCXX_INSTALL_PREFIX=$(pwd)/build/install
16-
export LIB_DIR=${lib_dir}
16+
if [[ "${distro_id}" == rhel* ]]; then
17+
export LIB_DIR=lib64
18+
else
19+
export LIB_DIR=lib
20+
fi
1721
export BUILD_TYPE=${build_type}
1822
export BUILD_DIR=$(pwd)/build
1923

‎.evergreen/config_generator/components/funcs/test_auth.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ class TestAuth(Function):
1212
script='''\
1313
export MONGOC_INSTALL_PREFIX=$(pwd)/../mongoc
1414
export MONGOCXX_INSTALL_PREFIX=$(pwd)/build/install
15-
export LIB_DIR=${lib_dir}
15+
if [[ "${distro_id}" == rhel* ]]; then
16+
export LIB_DIR=lib64
17+
else
18+
export LIB_DIR=lib
19+
fi
1620
export BUILD_TYPE=${build_type}
1721
export BUILD_DIR=$(pwd)/build
1822
export URI="mongodb://bob:pwd123@localhost"

0 commit comments

Comments
 (0)