ci(performance): Add performance tests to CI

Author: lucasssvaz

.github/scripts/tests_build.sh

@@ -2,19 +2,20 @@
 
 USAGE="
 USAGE:
-    ${0} -c <chunk_build_opts>
-       Example: ${0} -c -t esp32 -i 0 -m 15
+    ${0} -c -type <test_type> <chunk_build_opts>
+       Example: ${0} -c -type validation -t esp32 -i 0 -m 15
     ${0} -s sketch_name <build_opts>
        Example: ${0} -s hello_world -t esp32
     ${0} -clean
        Remove build and test generated files
 "
 
 function clean(){
-    rm -rf tests/*/build*/
     rm -rf tests/.pytest_cache
-    rm -rf tests/*/__pycache__/
-    rm -rf tests/*/*.xml
+    find tests/ -type d -name 'build*' -exec rm -rf "{}" \+
+    find tests/ -type d -name '__pycache__' -exec rm -rf "{}" \+
+    find tests/ -name '*.xml' -exec rm -rf "{}" \+
+    find tests/ -name 'result_*.json' -exec rm -rf "{}" \+
 }
 
 SCRIPTS_DIR="./.github/scripts"
@@ -35,6 +36,10 @@ while [ ! -z "$1" ]; do
         echo "$USAGE"
         exit 0
         ;;
+    -type )
+        shift
+        test_type=$1
+        ;;
     -clean )
         clean
         exit 0
@@ -52,12 +57,25 @@ source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
 args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH"
 
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+    if [ -n "$sketch" ]; then
+        tmp_sketch_path=$(find tests -name $sketch.ino)
+        test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+        echo "Sketch $sketch test type: $test_type"
+        test_folder="$PWD/tests/$test_type"
+    else
+      test_folder="$PWD/tests"
+    fi
+else
+    test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_build -eq 1 ]; then
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
-    args+=" -p $PWD/tests"
+    args+=" -p $test_folder"
 else
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
-    args+=" -s $PWD/tests/$sketch"
+    args+=" -s $test_folder/$sketch"
 fi
 
 ${BUILD_CMD} ${args} $*

.github/scripts/tests_run.sh

@@ -15,9 +15,9 @@ function run_test() {
     fi
 
     if [ $len -eq 1 ]; then
-      # build_dir="tests/$sketchname/build"
+      # build_dir="$sketchdir/build"
       build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
-      report_file="tests/$sketchname/$sketchname.xml"
+      report_file="$sketchdir/$sketchname.xml"
     fi
 
     for i in `seq 0 $(($len - 1))`
@@ -28,9 +28,9 @@ function run_test() {
         fi
 
         if [ $len -ne 1 ]; then
-            # build_dir="tests/$sketchname/build$i"
+            # build_dir="$sketchdir/build$i"
             build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
-            report_file="tests/$sketchname/$sketchname$i.xml"
+            report_file="$sketchdir/$sketchname$i.xml"
         fi
 
         pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file
@@ -79,6 +79,10 @@ while [ ! -z "$1" ]; do
         echo "$USAGE"
         exit 0
         ;;
+    -type )
+        shift
+        test_type=$1
+        ;;
     * )
       break
       ;;
@@ -88,8 +92,26 @@ done
 
 source ${SCRIPTS_DIR}/install-arduino-ide.sh
 
+# If sketch is provided and test type is not, test type is inferred from the sketch path
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+    if [ -n "$sketch" ]; then
+        tmp_sketch_path=$(find tests -name $sketch.ino)
+        test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+        echo "Sketch $sketch test type: $test_type"
+        test_folder="$PWD/tests/$test_type"
+    else
+      test_folder="$PWD/tests"
+    fi
+else
+    test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_run -eq 0 ]; then
-    run_test $target $PWD/tests/$sketch/$sketch.ino $options $erase
+    if [ -z $sketch ]; then
+        echo "ERROR: Sketch name is required for single test run"
+        return 1
+    fi
+    run_test $target $test_folder/$sketch/$sketch.ino $options $erase
 else
   if [ "$chunk_max" -le 0 ]; then
       echo "ERROR: Chunks count must be positive number"
@@ -102,7 +124,7 @@ else
   fi
 
   set +e
-  ${COUNT_SKETCHES} $PWD/tests $target
+  ${COUNT_SKETCHES} $test_folder $target
   sketchcount=$?
   set -e
   sketches=$(cat sketches.txt)

.github/workflows/hil.yml

@@ -18,11 +18,14 @@ jobs:
   gen_chunks:
     if: |
       contains(github.event.pull_request.labels.*.name, 'hil_test') ||
+      contains(github.event.pull_request.labels.*.name, 'perf_test') ||
       (github.event_name == 'schedule' && github.repository == 'espressif/arduino-esp32')
     name: Generate Chunks matrix
     runs-on: ubuntu-latest
     outputs:
       chunks: ${{ steps.gen-chunks.outputs.chunks }}
+      test_folder: ${{ steps.gen-chunks.outputs.test_folder }}
+      test_type: ${{ steps.gen-chunks.outputs.test_type }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -31,15 +34,29 @@ jobs:
         id: gen-chunks
         run: |
           set +e
-          .github/scripts/sketch_utils.sh count tests
+          if [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "true" ] && \
+             [ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "false" ]; then
+            test_folder="tests/validation"
+            test_type="validation"
+          elif [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "false" ] && \
+               [ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "true" ]; then
+            test_folder="tests/performance"
+            test_type="performance"
+          else
+            test_folder="tests"
+            test_type="all"
+          fi
+          .github/scripts/sketch_utils.sh count $test_folder
           sketches=$?
           if [[ $sketches -ge ${{env.MAX_CHUNKS}} ]]; then
             $sketches=${{env.MAX_CHUNKS}}
           fi
           set -e
           rm sketches.txt
           CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $((sketches - 1))`)
-          echo "chunks=${CHUNKS}" >>$GITHUB_OUTPUT
+          echo "chunks=${CHUNKS}" >> $GITHUB_OUTPUT
+          echo "test_folder=${test_folder}" >> $GITHUB_OUTPUT
+          echo "test_type=${test_type}" >> $GITHUB_OUTPUT
 
   Build:
     needs: gen_chunks
@@ -54,14 +71,14 @@ jobs:
         uses: actions/checkout@v4
       - name: Build sketches
         run: |
-          bash .github/scripts/tests_build.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}
+          bash .github/scripts/tests_build.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}
       - name: Upload ${{matrix.chip}}-${{matrix.chunks}} artifacts
         uses: actions/upload-artifact@v4
         with:
           name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
           path: |
-             ~/.arduino/tests/*/build*.tmp/*.bin
-             ~/.arduino/tests/*/build*.tmp/*.json
+             ~/.arduino/tests/**/build*.tmp/*.bin
+             ~/.arduino/tests/**/build*.tmp/*.json
           if-no-files-found: error
   Test:
     needs: [gen_chunks, Build]
@@ -94,19 +111,22 @@ jobs:
 
        - name: Run Tests
          run: |
-           bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e
+           bash .github/scripts/tests_run.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e
 
        - name: Upload test result artifacts
          uses: actions/upload-artifact@v4
          if: always()
          with:
            name: test_results-${{matrix.chip}}-${{matrix.chunks}}
-           path: tests/*/*.xml
+           path: |
+             tests/**/*.xml
+             tests/**/result_*.json
 
   event_file:
     name: "Event File"
     if: |
       contains(github.event.pull_request.labels.*.name, 'hil_test') ||
+      contains(github.event.pull_request.labels.*.name, 'perf_test') ||
       github.event_name == 'schedule'
     needs: Test
     runs-on: ubuntu-latest

tests/.gitignore

@@ -1,3 +1,4 @@
 build*/
 __pycache__/
 *.xml
+result_*.json

tests/performance/fibonacci/fibonacci.ino

@@ -0,0 +1,44 @@
+#include <Arduino.h>
+
+#define N_RUNS 3 // Number of runs to average
+
+// Fibonacci number to calculate. Keep between 35 and 45.
+#define FIB_N 40
+
+uint32_t times[N_RUNS];
+
+uint64_t fib(uint32_t n) {
+  if (n < 2) return n;
+  return fib(n - 1) + fib(n - 2);
+}
+
+void setup(){
+  uint64_t fibonacci;
+
+  Serial.begin(115200);
+  while (!Serial) delay(10);
+
+  log_d("Starting fibonacci calculation");
+  Serial.printf("Runs: %d\n", N_RUNS);
+  for (int i = 0; i < N_RUNS; i++) {
+    log_d("Run %d", i);
+    unsigned long start = millis();
+    fibonacci = fib(FIB_N);
+    unsigned long end = millis();
+    times[i] = end - start;
+    log_d("Run %d: %lu.%03lu s\n", i, times[i]/1000, times[i]%1000);
+  }
+
+  Serial.printf("N: %d\n", FIB_N);
+  Serial.printf("Fibonacci(N): %llu\n", fibonacci);
+  uint32_t sum = 0;
+  for (int i = 0; i < N_RUNS; i++) {
+    sum += times[i];
+  }
+  uint32_t avg = sum / N_RUNS;
+  Serial.printf("Average time: %lu.%03lu s\n", avg/1000, avg%1000);
+}
+
+void loop(){
+  vTaskDelete(NULL);
+}

tests/performance/fibonacci/test_fibonacci.py

@@ -0,0 +1,51 @@
+import json
+import logging
+import os
+
+LOGGER = logging.getLogger(__name__)
+
+# Fibonacci results starting from fib(35) to fib(45)
+fib_results = [9227465, 14930352, 24157817, 39088169, 63245986, 102334155, 165580141, 267914296, 433494437, 701408733]
+
+def test_fibonacci(dut, request):
+    # Match "Runs: %d"
+    res = dut.expect(r"Runs: (\d+)", timeout=60)
+    runs = int(res.group(0).decode("utf-8").split(" ")[1])
+    LOGGER.info("Number of runs: {}".format(runs))
+
+    # Match "N: %d"
+    res = dut.expect(r"N: (\d+)", timeout=300)
+    fib_n = int(res.group(0).decode("utf-8").split(" ")[1])
+    LOGGER.info("Calculating Fibonacci({})".format(fib_n))
+
+    # Match "Fibonacci(N): %llu"
+    res = dut.expect(r"Fibonacci\(N\): (\d+)", timeout=300)
+    fib_result = int(res.group(0).decode("utf-8").split(" ")[1])
+    LOGGER.info("Fibonacci({}) = {}".format(fib_n, fib_result))
+
+    # Check if the result is correct
+    assert fib_result == fib_results[fib_n - 35]
+
+    # Match "Average time: %lu.%03lu s"
+    res = dut.expect(r"Average time: (\d+)\.(\d+) s", timeout=300)
+    avg_time = float(res.group(0).decode("utf-8").split(" ")[2])
+    LOGGER.info("Average time on {} runs: {} s".format(runs, avg_time))
+
+    # Create JSON with results and write it to file
+    # Always create a JSON with the format (so it can be merged later on):
+    # { TEST_NAME_STR: TEST_RESULTS_DICT }
+    results = {
+        "fibonacci": {
+            "runs": runs,
+            "fib_n": fib_n,
+            "fib_result": fib_result,
+            "avg_time": avg_time
+        }
+    }
+
+    current_folder = os.path.dirname(request.path)
+    with open(current_folder + "/result_fibonacci.json", "w") as f:
+        try:
+            f.write(json.dumps(results))
+        except Exception as e:
+            LOGGER.warning("Failed to write results to file: {}".format(e))