Skip to content

Commit 1e66cec

Browse files
lucasssvazlucasssvaz
committed
ci(performance): Add performance tests to CI
1 parent cf44890 commit 1e66cec

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+8246
-52
lines changed

‎.github/scripts/sketch_utils.sh

+14-8
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,12 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
7070

7171
len=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn | length' $sketchdir/cfg.json`
7272
fqbn=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn' $sketchdir/cfg.json`
73+
74+
# Skip if the target is not in the config file.
75+
if [ -z $fqbn ]; then
76+
echo "No FQBN found for chip: $target. Skipping."
77+
exit 0
78+
fi
7379
else
7480
# Since we are passing options, we will end up with only one FQBN to
7581
# build.
@@ -121,7 +127,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
121127
fi
122128

123129
if [ -z "$fqbn" ]; then
124-
echo "No FQBN passed or unvalid chip: $target"
130+
echo "No FQBN passed or invalid chip: $target"
125131
exit 1
126132
fi
127133

@@ -139,7 +145,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
139145
echo "Skipping $sketchname for target $target"
140146
exit 0
141147
fi
142-
148+
143149
ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
144150
if [ -n "$ARDUINO_BUILD_DIR" ]; then
145151
build_dir="$ARDUINO_BUILD_DIR"
@@ -177,7 +183,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
177183
--build-path "$build_dir" \
178184
$xtra_opts "${sketchdir}" \
179185
> $output_file
180-
186+
181187
exit_status=$?
182188
if [ $exit_status -ne 0 ]; then
183189
echo ""ERROR: Compilation failed with error code $exit_status""
@@ -198,11 +204,11 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
198204
# Extract the desired substring using sed
199205
lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||")
200206
#append json file where key is fqbn, sketch name, sizes -> extracted values
201-
echo "{\"name\": \"$lib_sketch_name\",
207+
echo "{\"name\": \"$lib_sketch_name\",
202208
\"sizes\": [{
203-
\"flash_bytes\": $flash_bytes,
204-
\"flash_percentage\": $flash_percentage,
205-
\"ram_bytes\": $ram_bytes,
209+
\"flash_bytes\": $flash_bytes,
210+
\"flash_percentage\": $flash_percentage,
211+
\"ram_bytes\": $ram_bytes,
206212
\"ram_percentage\": $ram_percentage
207213
}]
208214
}," >> "$sizes_file"
@@ -386,7 +392,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
386392
if [ $log_compilation ]; then
387393
#echo board,target and start of sketches to sizes_file json
388394
echo "{ \"board\": \"$fqbn\",
389-
\"target\": \"$target\",
395+
\"target\": \"$target\",
390396
\"sketches\": [" >> "$sizes_file"
391397
fi
392398

‎.github/scripts/tests_build.sh

+25-7
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,20 @@
22

33
USAGE="
44
USAGE:
5-
${0} -c <chunk_build_opts>
6-
Example: ${0} -c -t esp32 -i 0 -m 15
5+
${0} -c -type <test_type> <chunk_build_opts>
6+
Example: ${0} -c -type validation -t esp32 -i 0 -m 15
77
${0} -s sketch_name <build_opts>
88
Example: ${0} -s hello_world -t esp32
99
${0} -clean
1010
Remove build and test generated files
1111
"
1212

1313
function clean(){
14-
rm -rf tests/*/build*/
1514
rm -rf tests/.pytest_cache
16-
rm -rf tests/*/__pycache__/
17-
rm -rf tests/*/*.xml
15+
find tests/ -type d -name 'build*' -exec rm -rf "{}" \+
16+
find tests/ -type d -name '__pycache__' -exec rm -rf "{}" \+
17+
find tests/ -name '*.xml' -exec rm -rf "{}" \+
18+
find tests/ -name 'result_*.json' -exec rm -rf "{}" \+
1819
}
1920

2021
SCRIPTS_DIR="./.github/scripts"
@@ -35,6 +36,10 @@ while [ ! -z "$1" ]; do
3536
echo "$USAGE"
3637
exit 0
3738
;;
39+
-type )
40+
shift
41+
test_type=$1
42+
;;
3843
-clean )
3944
clean
4045
exit 0
@@ -52,12 +57,25 @@ source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
5257

5358
args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH"
5459

60+
if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
61+
if [ -n "$sketch" ]; then
62+
tmp_sketch_path=$(find tests -name $sketch.ino)
63+
test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
64+
echo "Sketch $sketch test type: $test_type"
65+
test_folder="$PWD/tests/$test_type"
66+
else
67+
test_folder="$PWD/tests"
68+
fi
69+
else
70+
test_folder="$PWD/tests/$test_type"
71+
fi
72+
5573
if [ $chunk_build -eq 1 ]; then
5674
BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
57-
args+=" -p $PWD/tests"
75+
args+=" -p $test_folder"
5876
else
5977
BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
60-
args+=" -s $PWD/tests/$sketch"
78+
args+=" -s $test_folder/$sketch"
6179
fi
6280

6381
${BUILD_CMD} ${args} $*

‎.github/scripts/tests_run.sh

+37-9
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,20 @@ function run_test() {
1010

1111
if [ $options -eq 0 ] && [ -f $sketchdir/cfg.json ]; then
1212
len=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn | length' $sketchdir/cfg.json`
13+
14+
# Skip if no configuration for the target
15+
if [ $len -eq 0 ]; then
16+
echo "No configuration for target $target. Skipping test."
17+
return 0
18+
fi
1319
else
1420
len=1
1521
fi
1622

1723
if [ $len -eq 1 ]; then
18-
# build_dir="tests/$sketchname/build"
24+
# build_dir="$sketchdir/build"
1925
build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
20-
report_file="tests/$sketchname/$sketchname.xml"
26+
report_file="$sketchdir/$sketchname.xml"
2127
fi
2228

2329
for i in `seq 0 $(($len - 1))`
@@ -28,9 +34,9 @@ function run_test() {
2834
fi
2935

3036
if [ $len -ne 1 ]; then
31-
# build_dir="tests/$sketchname/build$i"
37+
# build_dir="$sketchdir/build$i"
3238
build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
33-
report_file="tests/$sketchname/$sketchname$i.xml"
39+
report_file="$sketchdir/$sketchname$i.xml"
3440
fi
3541

3642
pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file
@@ -79,6 +85,10 @@ while [ ! -z "$1" ]; do
7985
echo "$USAGE"
8086
exit 0
8187
;;
88+
-type )
89+
shift
90+
test_type=$1
91+
;;
8292
* )
8393
break
8494
;;
@@ -88,21 +98,39 @@ done
8898

8999
source ${SCRIPTS_DIR}/install-arduino-ide.sh
90100

101+
# If sketch is provided and test type is not, test type is inferred from the sketch path
102+
if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
103+
if [ -n "$sketch" ]; then
104+
tmp_sketch_path=$(find tests -name $sketch.ino)
105+
test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
106+
echo "Sketch $sketch test type: $test_type"
107+
test_folder="$PWD/tests/$test_type"
108+
else
109+
test_folder="$PWD/tests"
110+
fi
111+
else
112+
test_folder="$PWD/tests/$test_type"
113+
fi
114+
91115
if [ $chunk_run -eq 0 ]; then
92-
run_test $target $PWD/tests/$sketch/$sketch.ino $options $erase
116+
if [ -z $sketch ]; then
117+
echo "ERROR: Sketch name is required for single test run"
118+
exit 1
119+
fi
120+
run_test $target $test_folder/$sketch/$sketch.ino $options $erase
93121
else
94122
if [ "$chunk_max" -le 0 ]; then
95123
echo "ERROR: Chunks count must be positive number"
96-
return 1
124+
exit 1
97125
fi
98126

99127
if [ "$chunk_index" -ge "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
100128
echo "ERROR: Chunk index must be less than chunks count"
101-
return 1
129+
exit 1
102130
fi
103131

104132
set +e
105-
${COUNT_SKETCHES} $PWD/tests $target
133+
${COUNT_SKETCHES} $test_folder $target
106134
sketchcount=$?
107135
set -e
108136
sketches=$(cat sketches.txt)
@@ -123,7 +151,7 @@ else
123151
start_index=$(( $chunk_index * $chunk_size ))
124152
if [ "$sketchcount" -le "$start_index" ]; then
125153
echo "Skipping job"
126-
return 0
154+
exit 0
127155
fi
128156

129157
end_index=$(( $(( $chunk_index + 1 )) * $chunk_size ))

‎.github/workflows/hil.yml

+58-27
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@ jobs:
1818
gen_chunks:
1919
if: |
2020
contains(github.event.pull_request.labels.*.name, 'hil_test') ||
21+
contains(github.event.pull_request.labels.*.name, 'perf_test') ||
2122
(github.event_name == 'schedule' && github.repository == 'espressif/arduino-esp32')
2223
name: Generate Chunks matrix
2324
runs-on: ubuntu-latest
2425
outputs:
2526
chunks: ${{ steps.gen-chunks.outputs.chunks }}
27+
test_folder: ${{ steps.gen-chunks.outputs.test_folder }}
28+
test_type: ${{ steps.gen-chunks.outputs.test_type }}
2629
steps:
2730
- name: Checkout Repository
2831
uses: actions/checkout@v4
@@ -31,15 +34,29 @@ jobs:
3134
id: gen-chunks
3235
run: |
3336
set +e
34-
.github/scripts/sketch_utils.sh count tests
37+
if [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "true" ] && \
38+
[ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "false" ]; then
39+
test_folder="tests/validation"
40+
test_type="validation"
41+
elif [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "false" ] && \
42+
[ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "true" ]; then
43+
test_folder="tests/performance"
44+
test_type="performance"
45+
else
46+
test_folder="tests"
47+
test_type="all"
48+
fi
49+
.github/scripts/sketch_utils.sh count $test_folder
3550
sketches=$?
3651
if [[ $sketches -ge ${{env.MAX_CHUNKS}} ]]; then
3752
$sketches=${{env.MAX_CHUNKS}}
3853
fi
3954
set -e
4055
rm sketches.txt
4156
CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $((sketches - 1))`)
42-
echo "chunks=${CHUNKS}" >>$GITHUB_OUTPUT
57+
echo "chunks=${CHUNKS}" >> $GITHUB_OUTPUT
58+
echo "test_folder=${test_folder}" >> $GITHUB_OUTPUT
59+
echo "test_type=${test_type}" >> $GITHUB_OUTPUT
4360
4461
Build:
4562
needs: gen_chunks
@@ -52,17 +69,19 @@ jobs:
5269
steps:
5370
- name: Checkout Repository
5471
uses: actions/checkout@v4
72+
5573
- name: Build sketches
5674
run: |
57-
bash .github/scripts/tests_build.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}
75+
bash .github/scripts/tests_build.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}
76+
5877
- name: Upload ${{matrix.chip}}-${{matrix.chunks}} artifacts
5978
uses: actions/upload-artifact@v4
6079
with:
6180
name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
6281
path: |
63-
~/.arduino/tests/*/build*.tmp/*.bin
64-
~/.arduino/tests/*/build*.tmp/*.json
65-
if-no-files-found: error
82+
~/.arduino/tests/**/build*.tmp/*.bin
83+
~/.arduino/tests/**/build*.tmp/*.json
84+
6685
Test:
6786
needs: [gen_chunks, Build]
6887
name: ${{matrix.chip}}-Test#${{matrix.chunks}}
@@ -77,36 +96,48 @@ jobs:
7796
options: --privileged
7897

7998
steps:
80-
- name: Checkout repository
81-
uses: actions/checkout@v4
99+
- name: Checkout repository
100+
uses: actions/checkout@v4
82101

83-
- name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
84-
uses: actions/download-artifact@v4
85-
with:
86-
name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
87-
path: ~/.arduino/tests/
102+
- name: Check if test artifacts are available
103+
id: check-artifact
104+
uses: LIT-Protocol/artifact-exists-action@v0
105+
with:
106+
name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
107+
108+
- name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
109+
uses: actions/download-artifact@v4
110+
if: ${{ steps.check-artifact.outputs.exists == 'true' }}
111+
with:
112+
name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
113+
path: ~/.arduino/tests/
88114

89-
- name: Install dependencies
90-
run: |
91-
pip install -U pip
92-
pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
93-
apt update && apt install -y -qq jq
115+
- name: Install dependencies
116+
if: ${{ steps.check-artifact.outputs.exists == 'true' }}
117+
run: |
118+
pip install -U pip
119+
pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
120+
apt update && apt install -y -qq jq
94121
95-
- name: Run Tests
96-
run: |
97-
bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e
122+
- name: Run Tests
123+
if: ${{ steps.check-artifact.outputs.exists == 'true' }}
124+
run: |
125+
bash .github/scripts/tests_run.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e
98126
99-
- name: Upload test result artifacts
100-
uses: actions/upload-artifact@v4
101-
if: always()
102-
with:
103-
name: test_results-${{matrix.chip}}-${{matrix.chunks}}
104-
path: tests/*/*.xml
127+
- name: Upload test result artifacts
128+
uses: actions/upload-artifact@v4
129+
if: ${{ always() && steps.check-artifact.outputs.exists == 'true' }}
130+
with:
131+
name: test_results-${{matrix.chip}}-${{matrix.chunks}}
132+
path: |
133+
tests/**/*.xml
134+
tests/**/result_*.json
105135
106136
event_file:
107137
name: "Event File"
108138
if: |
109139
contains(github.event.pull_request.labels.*.name, 'hil_test') ||
140+
contains(github.event.pull_request.labels.*.name, 'perf_test') ||
110141
github.event_name == 'schedule'
111142
needs: Test
112143
runs-on: ubuntu-latest

‎.pre-commit-config.yaml

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
exclude: ".github/.*"
1+
exclude: |
2+
(?x)(
3+
^\.github\/|
4+
^tests\/performance\/coremark\/.*\.[ch]$
5+
)
26
37
default_language_version:
48
# force all unspecified python hooks to run python3

‎tests/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
build*/
22
__pycache__/
33
*.xml
4+
result_*.json

0 commit comments

Comments
 (0)