Merge pull request #1716 from esp8266/host_tests

Initial batch of host testing mocks and some filesystem tests

- CATCH unit testing framework
- some mocks
- filesystem tests
- test coverage calculation (gcov)
- codecov.io integration
- add tools/build.py — a wrapper around arduino-builder
- use arduino-builder instead of full IDE to build sketches (reduces build time from 23 minutes to 16 minutes)

Author: igrr

.travis.yml

@@ -3,7 +3,20 @@ language: bash
 os:
   - linux
 
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-4.8
+
 script:
+  - set -e
+  - export CXX="g++-4.8" CC="gcc-4.8" GCOV="gcov-4.8"
+  - pushd $TRAVIS_BUILD_DIR/tests/host
+  - make
+  - make clean-objects
+  - popd
   - wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
   - tar xf arduino.tar.xz
   - mv arduino-nightly $HOME/arduino_ide
@@ -20,10 +33,12 @@ script:
   - which arduino
   - cd $TRAVIS_BUILD_DIR
   - source tests/common.sh
-  - arduino --board esp8266com:esp8266:generic --save-prefs
-  - arduino --get-pref sketchbook.path
   - install_libraries
-  - build_sketches arduino $TRAVIS_BUILD_DIR
+  - build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR "python tools/build.py -l $HOME/Arduino/libraries -b generic -v"
+
+after_success:
+    - pushd $TRAVIS_BUILD_DIR/tests/host
+    - bash <(curl -s https://codecov.io/bash) -X gcov
 
 notifications:
   email:

tests/common.sh

@@ -4,28 +4,42 @@ function build_sketches()
 {
     local arduino=$1
     local srcpath=$2
+    local build_cmd=$3
+    echo $build_cmd
     local sketches=$(find $srcpath -name *.ino)
+    export ARDUINO_IDE_PATH=$arduino
     for sketch in $sketches; do
         local sketchdir=$(dirname $sketch)
+        local sketchdirname=$(basename $sketchdir)
+        local sketchname=$(basename $sketch)
+        if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
+            echo "Skipping $sketch, beacause it is not the main sketch file";
+            continue
+        fi;
         if [[ -f "$sketchdir/.test.skip" ]]; then
-            echo -e "\n\n ------------ Skipping $sketch ------------ \n\n";
+            echo -e "\n ------------ Skipping $sketch ------------ \n";
             continue
         fi
-        echo -e "\n\n ------------ Building $sketch ------------ \n\n";
-        $arduino --verify $sketch;
+        echo -e "\n ------------ Building $sketch ------------ \n";
+        # $arduino --verify $sketch;
+        echo "$build_cmd $sketch"
+        time ($build_cmd $sketch >build.log)
         local result=$?
         if [ $result -ne 0 ]; then
             echo "Build failed ($1)"
+            echo "Build log:"
+            cat build.log
             return $result
         fi
+        rm build.log
     done
 }
 
 function install_libraries()
 {
     mkdir -p $HOME/Arduino/libraries
     pushd $HOME/Arduino/libraries
-    
+
     # install ArduinoJson library
     wget https://github.com/bblanchon/ArduinoJson/releases/download/v4.6.1/ArduinoJson-v4.6.1.zip && unzip ArduinoJson-v4.6.1.zip
 

tests/host/Makefile

@@ -0,0 +1,97 @@
+OBJECT_DIRECTORY := obj
+BINARY_DIRECTORY := bin
+OUTPUT_BINARY := $(BINARY_DIRECTORY)/host_tests
+CORE_PATH := ../../cores/esp8266
+
+# I wasn't able to build with clang when -coverage flag is enabled, forcing GCC on OS X
+ifeq ($(shell uname -s),Darwin)
+CC  := gcc
+CXX := g++
+endif
+GCOV ?= gcov
+
+CORE_CPP_FILES := $(addprefix $(CORE_PATH)/,\
+	StreamString.cpp \
+	Stream.cpp \
+	WString.cpp \
+	Print.cpp \
+	FS.cpp \
+	spiffs_api.cpp \
+)
+
+CORE_C_FILES := $(addprefix $(CORE_PATH)/,\
+	core_esp8266_noniso.c \
+	spiffs/spiffs_cache.c \
+	spiffs/spiffs_check.c \
+	spiffs/spiffs_gc.c \
+	spiffs/spiffs_hydrogen.c \
+	spiffs/spiffs_nucleus.c \
+)
+
+MOCK_CPP_FILES := $(addprefix common/,\
+	Arduino.cpp \
+	spiffs_mock.cpp \
+	WMath.cpp \
+)
+
+INC_PATHS += $(addprefix -I, \
+	common \
+	$(CORE_PATH) \
+)
+
+TEST_CPP_FILES := \
+	fs/test_fs.cpp \
+
+CXXFLAGS += -std=c++11 -Wall -coverage -O0
+CFLAGS += -std=c99 -Wall -coverage -O0
+LDFLAGS += -coverage -O0
+
+remduplicates = $(strip $(if $1,$(firstword $1) $(call remduplicates,$(filter-out $(firstword $1),$1))))
+
+C_SOURCE_FILES = $(CORE_C_FILES)
+CPP_SOURCE_FILES = $(MOCK_CPP_FILES) $(CORE_CPP_FILES) $(TEST_CPP_FILES)
+C_OBJECTS = $(C_SOURCE_FILES:.c=.c.o)
+
+CPP_OBJECTS = $(CPP_SOURCE_FILES:.cpp=.cpp.o)
+
+OBJECTS = $(C_OBJECTS) $(CPP_OBJECTS)
+COVERAGE_FILES = $(OBJECTS:.o=.gc*)
+
+all: build-info $(OUTPUT_BINARY) test gcov
+
+test: $(OUTPUT_BINARY)
+	$(OUTPUT_BINARY)
+
+clean: clean-objects clean-coverage
+	rm -rf $(BINARY_DIRECTORY)
+
+clean-objects:
+	rm -rf $(OBJECTS)
+
+clean-coverage:
+	rm -rf $(COVERAGE_FILES) *.gcov
+
+gcov: test
+	find $(CORE_PATH) -name "*.gcno" -exec $(GCOV) -r -pb {} +
+
+build-info:
+	echo "-------- build tools info --------"
+	echo "CC: " $(CC)
+	$(CC) -v
+	echo "CXX: " $(CXX)
+	$(CXX) -v
+	echo "GCOV: " $(GCOV)
+	$(GCOV) -v
+	echo "----------------------------------"
+
+$(BINARY_DIRECTORY):
+	mkdir -p $@
+
+$(C_OBJECTS): %.c.o: %.c
+	$(CC) $(CFLAGS) $(INC_PATHS) -c -o $@ $<
+
+$(CPP_OBJECTS): %.cpp.o: %.cpp
+	$(CXX) $(CXXFLAGS) $(INC_PATHS) -c -o $@ $<
+
+$(OUTPUT_BINARY): $(BINARY_DIRECTORY) $(OBJECTS)
+	$(CXX) $(LDFLAGS) $(OBJECTS) $(LIBS) -o $(OUTPUT_BINARY)

tests/host/common/Arduino.cpp

@@ -0,0 +1,37 @@
+/*
+ Arduino.cpp - Mocks for common Arduino APIs
+ Copyright © 2016 Ivan Grokhotkov
+ 
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ 
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+*/
+
+#define CATCH_CONFIG_MAIN
+#include <catch.hpp>
+#include <sys/time.h>
+#include "Arduino.h"
+
+
+extern "C" unsigned long millis()
+{
+    timeval time;
+    gettimeofday(&time, NULL);
+    return (time.tv_sec * 1000) + (time.tv_usec / 1000);
+}
+
+
+extern "C" void yield()
+{
+}
+
+
+extern "C" void __panic_func(const char* file, int line, const char* func) {
+    abort();
+}