feat: purge build cache (#2033)

Author: Bikappa

.gitignore

@@ -28,3 +28,6 @@ venv
 /docsgen/arduino-cli.exe
 /docs/rpc/*.md
 /docs/commands/*.md
+
+# Delve debugger binary file
+__debug_bin

arduino/sketch/sketch.go

@@ -294,7 +294,7 @@ func CheckForPdeFiles(sketch *paths.Path) []*paths.Path {
 // DefaultBuildPath generates the default build directory for a given sketch.
 // The build path is in a temporary directory and is unique for each sketch.
 func (s *Sketch) DefaultBuildPath() *paths.Path {
-	return paths.TempDir().Join("arduino", "sketch-"+s.Hash())
+	return paths.TempDir().Join("arduino", "sketches", s.Hash())
 }
 
 // Hash generate a unique hash for the given sketch.

arduino/sketch/sketch_test.go

@@ -286,7 +286,7 @@ func TestNewSketchFolderSymlink(t *testing.T) {
 }
 
 func TestGenBuildPath(t *testing.T) {
-	want := paths.TempDir().Join("arduino", "sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
+	want := paths.TempDir().Join("arduino", "sketches", "ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
 	assert.True(t, (&Sketch{FullPath: paths.New("foo")}).DefaultBuildPath().EquivalentTo(want))
 	assert.Equal(t, "ACBD18DB4CC2F85CEDEF654FCCC4A4D8", (&Sketch{FullPath: paths.New("foo")}).Hash())
 }

buildcache/build_cache.go

@@ -0,0 +1,113 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package buildcache
+
+import (
+	"time"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	createDirErrCode = 1
+	fileWriteErrCode = 2
+)
+
+type cacheError struct {
+	Code       int
+	wrappedErr error
+}
+
+func (e cacheError) Error() string {
+	return e.wrappedErr.Error()
+}
+
+func (e cacheError) Unwrap() error {
+	return e.wrappedErr
+}
+
+func (e cacheError) Is(target error) bool {
+	te, ok := target.(cacheError)
+	return ok && te.Code == e.Code
+}
+
+var (
+	// CreateDirErr error occurred when creating the cache directory
+	CreateDirErr = cacheError{Code: createDirErrCode}
+	// FileWriteErr error occurred when writing the placeholder file
+	FileWriteErr = cacheError{Code: fileWriteErrCode}
+)
+
+const lastUsedFileName = ".last-used"
+
+// BuildCache represents a cache of built files (sketches and cores), it's designed
+// to work on directories. Given a directory as "base" it handles direct subdirectories as
+// keys
+type BuildCache struct {
+	baseDir *paths.Path
+}
+
+// GetOrCreate retrieves or creates the cache directory at the given path
+// If the cache already exists the lifetime of the cache is extended.
+func (bc *BuildCache) GetOrCreate(key string) (*paths.Path, error) {
+	keyDir := bc.baseDir.Join(key)
+	if err := keyDir.MkdirAll(); err != nil {
+		return nil, cacheError{createDirErrCode, err}
+	}
+
+	if err := keyDir.Join(lastUsedFileName).WriteFile([]byte{}); err != nil {
+		return nil, cacheError{fileWriteErrCode, err}
+	}
+	return keyDir, nil
+}
+
+// Purge removes all cache directories within baseDir that have expired
+// To know how long ago a directory has been last used
+// it checks into the .last-used file.
+func (bc *BuildCache) Purge(ttl time.Duration) {
+	files, err := bc.baseDir.ReadDir()
+	if err != nil {
+		return
+	}
+	for _, file := range files {
+		if file.IsDir() {
+			removeIfExpired(file, ttl)
+		}
+	}
+}
+
+// New instantiates a build cache
+func New(baseDir *paths.Path) *BuildCache {
+	return &BuildCache{baseDir}
+}
+
+func removeIfExpired(dir *paths.Path, ttl time.Duration) {
+	fileInfo, err := dir.Join(lastUsedFileName).Stat()
+	if err != nil {
+		return
+	}
+	lifeExpectancy := ttl - time.Since(fileInfo.ModTime())
+	if lifeExpectancy > 0 {
+		return
+	}
+	logrus.Tracef(`Purging cache directory "%s". Expired by %s`, dir, lifeExpectancy.Abs())
+	err = dir.RemoveAll()
+	if err != nil {
+		logrus.Tracef(`Error while pruning cache directory "%s": %s`, dir, errors.WithStack(err))
+	}
+}

buildcache/build_cache_test.go

@@ -0,0 +1,78 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package buildcache
+
+import (
+	"testing"
+	"time"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_UpdateLastUsedFileNotExisting(t *testing.T) {
+	testBuildDir := paths.New(t.TempDir(), "sketches", "xxx")
+	require.NoError(t, testBuildDir.MkdirAll())
+	timeBeforeUpdating := time.Unix(0, 0)
+	requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating)
+}
+
+func Test_UpdateLastUsedFileExisting(t *testing.T) {
+	testBuildDir := paths.New(t.TempDir(), "sketches", "xxx")
+	require.NoError(t, testBuildDir.MkdirAll())
+
+	// create the file
+	preExistingFile := testBuildDir.Join(lastUsedFileName)
+	require.NoError(t, preExistingFile.WriteFile([]byte{}))
+	timeBeforeUpdating := time.Now().Add(-time.Second)
+	preExistingFile.Chtimes(time.Now(), timeBeforeUpdating)
+	requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating)
+}
+
+func requireCorrectUpdate(t *testing.T, dir *paths.Path, prevModTime time.Time) {
+	_, err := New(dir.Parent()).GetOrCreate(dir.Base())
+	require.NoError(t, err)
+	expectedFile := dir.Join(lastUsedFileName)
+	fileInfo, err := expectedFile.Stat()
+	require.Nil(t, err)
+	require.Greater(t, fileInfo.ModTime(), prevModTime)
+}
+
+func TestPurge(t *testing.T) {
+	ttl := time.Minute
+
+	dirToPurge := paths.New(t.TempDir(), "root")
+
+	lastUsedTimesByDirPath := map[*paths.Path]time.Time{
+		(dirToPurge.Join("old")):   time.Now().Add(-ttl - time.Hour),
+		(dirToPurge.Join("fresh")): time.Now().Add(-ttl + time.Minute),
+	}
+
+	// create the metadata files
+	for dirPath, lastUsedTime := range lastUsedTimesByDirPath {
+		require.NoError(t, dirPath.MkdirAll())
+		infoFilePath := dirPath.Join(lastUsedFileName).Canonical()
+		require.NoError(t, infoFilePath.WriteFile([]byte{}))
+		// make sure access time does not matter
+		accesstime := time.Now()
+		require.NoError(t, infoFilePath.Chtimes(accesstime, lastUsedTime))
+	}
+
+	New(dirToPurge).Purge(ttl)
+
+	require.False(t, dirToPurge.Join("old").Exist())
+	require.True(t, dirToPurge.Join("fresh").Exist())
+}

commands/compile/compile.go

@@ -27,9 +27,11 @@ import (
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
 	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/arduino/arduino-cli/buildcache"
 	"github.com/arduino/arduino-cli/commands"
 	"github.com/arduino/arduino-cli/configuration"
 	"github.com/arduino/arduino-cli/i18n"
+	"github.com/arduino/arduino-cli/inventory"
 	"github.com/arduino/arduino-cli/legacy/builder"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
@@ -135,6 +137,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	if err = builderCtx.BuildPath.MkdirAll(); err != nil {
 		return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build directory"), Cause: err}
 	}
+
+	buildcache.New(builderCtx.BuildPath.Parent()).GetOrCreate(builderCtx.BuildPath.Base())
+	// cache is purged after compilation to not remove entries that might be required
+	defer maybePurgeBuildCache()
+
 	builderCtx.CompilationDatabase = bldr.NewCompilationDatabase(
 		builderCtx.BuildPath.Join("compile_commands.json"),
 	)
@@ -153,7 +160,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	builderCtx.CustomBuildProperties = append(req.GetBuildProperties(), securityKeysOverride...)
 
 	if req.GetBuildCachePath() == "" {
-		builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "core-cache")
+		builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
 	} else {
 		buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs()
 		if err != nil {
@@ -287,3 +294,24 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	return r, nil
 }
+
+// maybePurgeBuildCache runs the build files cache purge if the policy conditions are met.
+func maybePurgeBuildCache() {
+
+	compilationsBeforePurge := configuration.Settings.GetUint("build_cache.compilations_before_purge")
+	// 0 means never purge
+	if compilationsBeforePurge == 0 {
+		return
+	}
+	compilationSinceLastPurge := inventory.Store.GetUint("build_cache.compilation_count_since_last_purge")
+	compilationSinceLastPurge++
+	inventory.Store.Set("build_cache.compilation_count_since_last_purge", compilationSinceLastPurge)
+	defer inventory.WriteStore()
+	if compilationsBeforePurge == 0 || compilationSinceLastPurge < compilationsBeforePurge {
+		return
+	}
+	inventory.Store.Set("build_cache.compilation_count_since_last_purge", 0)
+	cacheTTL := configuration.Settings.GetDuration("build_cache.ttl").Abs()
+	buildcache.New(paths.TempDir().Join("arduino", "cores")).Purge(cacheTTL)
+	buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL)
+}

configuration/defaults.go

@@ -18,6 +18,7 @@ package configuration
 import (
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/spf13/viper"
 )
@@ -41,6 +42,8 @@ func SetDefaults(settings *viper.Viper) {
 
 	// Sketch compilation
 	settings.SetDefault("sketch.always_export_binaries", false)
+	settings.SetDefault("build_cache.ttl", time.Hour*24*30)
+	settings.SetDefault("build_cache.compilations_before_purge", 10)
 
 	// daemon settings
 	settings.SetDefault("daemon.port", "50051")

docs/configuration.md

@@ -33,6 +33,12 @@
     to the sketch folder. This is the equivalent of using the [`--export-binaries`][arduino-cli compile options] flag.
 - `updater` - configuration options related to Arduino CLI updates
   - `enable_notification` - set to `false` to disable notifications of new Arduino CLI releases, defaults to `true`
+- `build_cache` configuration options related to the compilation cache
+  - `compilations_before_purge` - interval, in number of compilations, at which the cache is purged, defaults to `10`.
+    When `0` the cache is never purged.
+  - `ttl` - cache expiration time of build folders. If the cache is hit by a compilation the corresponding build files
+    lifetime is renewed. The value format must be a valid input for
+    [time.ParseDuration()](https://pkg.go.dev/time#ParseDuration), defaults to `720h` (30 days).
 
 ## Configuration methods
 

internal/integrationtest/arduino-cli.go

@@ -112,6 +112,7 @@ func NewArduinoCliWithinEnvironment(env *Environment, config *ArduinoCLIConfig)
 		"ARDUINO_DATA_DIR":       cli.dataDir.String(),
 		"ARDUINO_DOWNLOADS_DIR":  cli.stagingDir.String(),
 		"ARDUINO_SKETCHBOOK_DIR": cli.sketchbookDir.String(),
+		"ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE": "0",
 	}
 	env.RegisterCleanUpCallback(cli.CleanUp)
 	return cli