Merge branch 'master' into feat/purge-build-cache

Author: Luca Bianconi

arduino/libraries/librariesmanager/install.go

@@ -64,6 +64,7 @@ func (lm *LibrariesManager) InstallPrerequisiteCheck(name string, version *semve
 		return nil, err
 	}
 
+	lm.RescanLibraries()
 	libs := lm.FindByReference(&librariesindex.Reference{Name: name}, installLocation)
 
 	if len(libs) > 1 {

arduino/libraries/librariesmanager/librariesmanager.go

@@ -132,6 +132,7 @@ func (lm *LibrariesManager) AddPlatformReleaseLibrariesDir(plaftormRelease *core
 
 // RescanLibraries reload all installed libraries in the system.
 func (lm *LibrariesManager) RescanLibraries() []*status.Status {
+	lm.clearLibraries()
 	statuses := []*status.Status{}
 	for _, dir := range lm.LibrariesDir {
 		if errs := lm.LoadLibrariesFromDir(dir); len(errs) > 0 {
@@ -217,3 +218,9 @@ func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference, in
 	}
 	return alternatives.FilterByVersionAndInstallLocation(libRef.Version, installLocation)
 }
+
+func (lm *LibrariesManager) clearLibraries() {
+	for k := range lm.Libraries {
+		delete(lm.Libraries, k)
+	}
+}

arduino/libraries/librariesmanager/librariesmanager_test.go

@@ -0,0 +1,32 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+package librariesmanager
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/libraries"
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_RescanLibrariesCallClear(t *testing.T) {
+	baseDir := paths.New(t.TempDir())
+	lm := NewLibraryManager(baseDir.Join("index_dir"), baseDir.Join("downloads_dir"))
+	lm.Libraries["testLibA"] = libraries.List{}
+	lm.Libraries["testLibB"] = libraries.List{}
+	lm.RescanLibraries()
+	require.Len(t, lm.Libraries, 0)
+}

arduino/sketch/sketch.go

@@ -32,8 +32,7 @@ import (
 type Sketch struct {
 	Name             string
 	MainFile         *paths.Path
-	FullPath         *paths.Path // FullPath is the path to the Sketch folder
-	BuildPath        *paths.Path
+	FullPath         *paths.Path    // FullPath is the path to the Sketch folder
 	OtherSketchFiles paths.PathList // Sketch files that end in .ino other than main file
 	AdditionalFiles  paths.PathList
 	RootFolderFiles  paths.PathList // All files that are in the Sketch root
@@ -81,7 +80,6 @@ func New(path *paths.Path) (*Sketch, error) {
 		Name:             path.Base(),
 		MainFile:         mainFile,
 		FullPath:         path,
-		BuildPath:        GenBuildPath(path),
 		OtherSketchFiles: paths.PathList{},
 		AdditionalFiles:  paths.PathList{},
 		RootFolderFiles:  paths.PathList{},
@@ -293,19 +291,15 @@ func CheckForPdeFiles(sketch *paths.Path) []*paths.Path {
 	return files
 }
 
-// GenBuildPath generates a suitable name for the build folder.
-// The sketchPath, if not nil, is also used to furhter differentiate build paths.
-func GenBuildPath(sketchPath *paths.Path) *paths.Path {
-	path := ""
-	if sketchPath != nil {
-		path = sketchPath.String()
-	}
-	md5SumBytes := md5.Sum([]byte(path))
-	md5Sum := strings.ToUpper(hex.EncodeToString(md5SumBytes[:]))
-
-	return getSketchesCacheDir().Join(md5Sum)
+// DefaultBuildPath generates the default build directory for a given sketch.
+// The build path is in a temporary directory and is unique for each sketch.
+func (s *Sketch) DefaultBuildPath() *paths.Path {
+	return paths.TempDir().Join("arduino", "sketches", s.Hash())
 }
 
-func getSketchesCacheDir() *paths.Path {
-	return paths.TempDir().Join("arduino", "sketches").Canonical()
+// Hash generate a unique hash for the given sketch.
+func (s *Sketch) Hash() string {
+	path := s.FullPath.String()
+	md5SumBytes := md5.Sum([]byte(path))
+	return strings.ToUpper(hex.EncodeToString(md5SumBytes[:]))
 }

arduino/sketch/sketch_test.go

@@ -287,10 +287,8 @@ func TestNewSketchFolderSymlink(t *testing.T) {
 
 func TestGenBuildPath(t *testing.T) {
 	want := paths.TempDir().Join("arduino", "sketches", "ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
-	assert.True(t, GenBuildPath(paths.New("foo")).EquivalentTo(want))
-
-	want = paths.TempDir().Join("arduino", "sketches", "D41D8CD98F00B204E9800998ECF8427E")
-	assert.True(t, GenBuildPath(nil).EquivalentTo(want))
+	assert.True(t, (&Sketch{FullPath: paths.New("foo")}).DefaultBuildPath().EquivalentTo(want))
+	assert.Equal(t, "ACBD18DB4CC2F85CEDEF654FCCC4A4D8", (&Sketch{FullPath: paths.New("foo")}).Hash())
 }
 
 func TestCheckForPdeFiles(t *testing.T) {

commands/compile/compile.go

@@ -118,7 +118,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	}
 	builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery()
 	builderCtx.FQBN = fqbn
-	builderCtx.SketchLocation = sk.FullPath
+	builderCtx.Sketch = sk
 	builderCtx.ProgressCB = progressCB
 
 	// FIXME: This will be redundant when arduino-builder will be part of the cli
@@ -130,7 +130,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	builderCtx.OtherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings))
 	builderCtx.LibraryDirs = paths.NewPathList(req.Library...)
 	if req.GetBuildPath() == "" {
-		builderCtx.BuildPath = sk.BuildPath
+		builderCtx.BuildPath = sk.DefaultBuildPath()
 	} else {
 		builderCtx.BuildPath = paths.New(req.GetBuildPath()).Canonical()
 	}
@@ -150,7 +150,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	// Optimize for debug
 	builderCtx.OptimizeForDebug = req.GetOptimizeForDebug()
-	builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
 
 	builderCtx.Jobs = int(req.GetJobs())
 
@@ -160,12 +159,17 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	builderCtx.CustomBuildProperties = append(req.GetBuildProperties(), "build.warn_data_percentage=75")
 	builderCtx.CustomBuildProperties = append(req.GetBuildProperties(), securityKeysOverride...)
 
-	if req.GetBuildCachePath() != "" {
-		builderCtx.BuildCachePath = paths.New(req.GetBuildCachePath())
-		err = builderCtx.BuildCachePath.MkdirAll()
+	if req.GetBuildCachePath() == "" {
+		builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
+	} else {
+		buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs()
 		if err != nil {
 			return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err}
 		}
+		if err := buildCachePath.MkdirAll(); err != nil {
+			return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err}
+		}
+		builderCtx.CoreBuildCachePath = buildCachePath.Join("core")
 	}
 
 	builderCtx.BuiltInLibrariesDirs = configuration.IDEBuiltinLibrariesDir(configuration.Settings)

commands/daemon/daemon.go

@@ -343,7 +343,6 @@ func (s *ArduinoCoreServerImpl) LibraryDownload(req *rpc.LibraryDownloadRequest,
 	resp, err := lib.LibraryDownload(
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryDownloadResponse{Progress: p}) },
-		"",
 	)
 	if err != nil {
 		return convertErrorToRPCStatus(err)
@@ -357,7 +356,6 @@ func (s *ArduinoCoreServerImpl) LibraryInstall(req *rpc.LibraryInstallRequest, s
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryInstallResponse{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryInstallResponse{TaskProgress: p}) },
-		"",
 	)
 	if err != nil {
 		return convertErrorToRPCStatus(err)

commands/debug/debug_info.go

@@ -115,9 +115,11 @@ func getDebugProperties(req *debug.DebugConfigRequest, pme *packagemanager.Explo
 		}
 	}
 
-	importPath := sk.BuildPath
+	var importPath *paths.Path
 	if importDir := req.GetImportDir(); importDir != "" {
 		importPath = paths.New(importDir)
+	} else {
+		importPath = sk.DefaultBuildPath()
 	}
 	if !importPath.Exist() {
 		return nil, &arduino.NotFoundError{Message: tr("Compiled sketch not found in %s", importPath)}

commands/lib/download.go

@@ -32,8 +32,7 @@ var tr = i18n.Tr
 
 // LibraryDownload executes the download of the library.
 // A DownloadProgressCB callback function must be passed to monitor download progress.
-// queryParameter is passed for analysis purposes.
-func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downloadCB rpc.DownloadProgressCB, queryParameter string) (*rpc.LibraryDownloadResponse, error) {
+func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downloadCB rpc.DownloadProgressCB) (*rpc.LibraryDownloadResponse, error) {
 	logrus.Info("Executing `arduino-cli lib download`")
 
 	lm := commands.GetLibraryManager(req)
@@ -48,7 +47,7 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downl
 		return nil, err
 	}
 
-	if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}, queryParameter); err != nil {
+	if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}, "download"); err != nil {
 		return nil, err
 	}
 

commands/lib/install.go

@@ -31,8 +31,7 @@ import (
 )
 
 // LibraryInstall resolves the library dependencies, then downloads and installs the libraries into the install location.
-// queryParameter is passed for analysis purposes and forwarded to the called functions. It is set to "depends" when a dependency is installed
-func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB, queryParameter string) error {
+func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error {
 	lm := commands.GetLibraryManager(req)
 	if lm == nil {
 		return &arduino.InvalidInstanceError{}
@@ -98,21 +97,19 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa
 
 	for libRelease, installTask := range libReleasesToInstall {
 		// Checks if libRelease is the requested library and not a dependency
+		downloadReason := "depends"
 		if libRelease.GetName() == req.Name {
-			if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, queryParameter); err != nil {
-				return err
-			}
-			if err := installLibrary(lm, libRelease, installTask, taskCB); err != nil {
-				return err
-			}
-		} else {
-			if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, "depends"); err != nil {
-				return err
-			}
-			if err := installLibrary(lm, libRelease, installTask, taskCB); err != nil {
-				return err
+			downloadReason = "install"
+			if installTask.ReplacedLib != nil {
+				downloadReason = "upgrade"
 			}
 		}
+		if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, downloadReason); err != nil {
+			return err
+		}
+		if err := installLibrary(lm, libRelease, installTask, taskCB); err != nil {
+			return err
+		}
 	}
 
 	if err := commands.Init(&rpc.InitRequest{Instance: req.Instance}, nil); err != nil {

commands/lib/upgrade.go

@@ -73,7 +73,7 @@ func upgrade(instance *rpc.Instance, libs []*installedLib, downloadCB rpc.Downlo
 			NoDeps:      false,
 			NoOverwrite: false,
 		}
-		err := LibraryInstall(context.Background(), libInstallReq, downloadCB, taskCB, "upgrade")
+		err := LibraryInstall(context.Background(), libInstallReq, downloadCB, taskCB)
 		if err != nil {
 			return err
 		}

commands/upload/upload.go

@@ -575,7 +575,7 @@ func determineBuildPathAndSketchName(importFile, importDir string, sk *sketch.Sk
 
 	// Case 4: only sketch specified. In this case we use the generated build path
 	// and the given sketch name.
-	return sk.BuildPath, sk.Name + sk.MainFile.Ext(), nil
+	return sk.DefaultBuildPath(), sk.Name + sk.MainFile.Ext(), nil
 }
 
 func detectSketchNameFromBuildPath(buildPath *paths.Path) (string, error) {

commands/upload/upload_test.go

@@ -79,7 +79,7 @@ func TestDetermineBuildPathAndSketchName(t *testing.T) {
 		// 03: error: used both importPath and importFile
 		{"testdata/build_path_2/Blink.ino.hex", "testdata/build_path_2", nil, nil, "<nil>", ""},
 		// 04: only sketch without FQBN
-		{"", "", blonk, nil, sketch.GenBuildPath(blonk.FullPath).String(), "Blonk.ino"},
+		{"", "", blonk, nil, blonk.DefaultBuildPath().String(), "Blonk.ino"},
 		// 05: use importFile to detect build.path and project_name, sketch is ignored.
 		{"testdata/build_path_2/Blink.ino.hex", "", blonk, nil, "testdata/build_path_2", "Blink.ino"},
 		// 06: use importPath as build.path and Blink as project name, ignore the sketch Blonk
@@ -95,7 +95,7 @@ func TestDetermineBuildPathAndSketchName(t *testing.T) {
 		// 11: error: used both importPath and importFile
 		{"testdata/build_path_2/Blink.ino.hex", "testdata/build_path_2", nil, fqbn, "<nil>", ""},
 		// 12: use sketch to determine project name and sketch+fqbn to determine build path
-		{"", "", blonk, fqbn, sketch.GenBuildPath(blonk.FullPath).String(), "Blonk.ino"},
+		{"", "", blonk, fqbn, blonk.DefaultBuildPath().String(), "Blonk.ino"},
 		// 13: use importFile to detect build.path and project_name, sketch+fqbn is ignored.
 		{"testdata/build_path_2/Blink.ino.hex", "", blonk, fqbn, "testdata/build_path_2", "Blink.ino"},
 		// 14: use importPath as build.path and Blink as project name, ignore the sketch Blonk, ignore fqbn

docs/UPGRADING.md

@@ -4,6 +4,11 @@ Here you can find a list of migration guides to handle breaking changes between
 
 ## 0.30.0
 
+### `daemon` CLI command's `--ip` flag removal
+
+The `daemon` CLI command no longer allows to set a custom ip for the gRPC communication. Currently there is not enough
+bandwith to support this feature. For this reason, the `--ip` flag has been removed.
+
 ### `board attach` CLI command changed behaviour
 
 The `board attach` CLI command has changed behaviour: now it just pick whatever port and FQBN is passed as parameter and

i18n/data/ar.po

@@ -871,7 +871,7 @@ msgid "Error installing tool %s"
 msgstr "خطأ اثناء تثبيت الاداة %s"
 
 #: cli/lib/list.go:79
-msgid "Error listing Libraries: %v"
+msgid "Error listing libraries: %v"
 msgstr "خطا اثناء انشاء قائمة المكتبات : %v"
 
 #: cli/board/listall.go:64
@@ -953,7 +953,7 @@ msgid "Error searching boards: %v"
 msgstr "خطا اثناء البحث عن اللوحات : %v"
 
 #: cli/lib/search.go:78
-msgid "Error searching for Libraries: %v"
+msgid "Error searching for libraries: %v"
 msgstr ""
 
 #: cli/core/search.go:87

i18n/data/de.po

@@ -863,7 +863,7 @@ msgid "Error installing tool %s"
 msgstr "Fehler beim Installieren des Werkzeugs %s"
 
 #: cli/lib/list.go:79
-msgid "Error listing Libraries: %v"
+msgid "Error listing libraries: %v"
 msgstr "Fehler beim Auflisten von Bibliotheken: %v"
 
 #: cli/board/listall.go:64
@@ -944,7 +944,7 @@ msgid "Error searching boards: %v"
 msgstr "Fehler bei der Suche nach Platinen: %v"
 
 #: cli/lib/search.go:78
-msgid "Error searching for Libraries: %v"
+msgid "Error searching for libraries: %v"
 msgstr ""
 
 #: cli/core/search.go:87

i18n/data/en.po

@@ -814,8 +814,8 @@ msgid "Error installing tool %s"
 msgstr "Error installing tool %s"
 
 #: cli/lib/list.go:76
-msgid "Error listing Libraries: %v"
-msgstr "Error listing Libraries: %v"
+msgid "Error listing libraries: %v"
+msgstr "Error listing libraries: %v"
 
 #: cli/board/listall.go:64
 msgid "Error listing boards: %v"

i18n/data/es.po

@@ -869,7 +869,7 @@ msgid "Error installing tool %s"
 msgstr "Error instalando la herramienta %s"
 
 #: cli/lib/list.go:79
-msgid "Error listing Libraries: %v"
+msgid "Error listing libraries: %v"
 msgstr "Error listando las librerías: %v"
 
 #: cli/board/listall.go:64
@@ -950,7 +950,7 @@ msgid "Error searching boards: %v"
 msgstr "Error en la búsqueda de placas: %v"
 
 #: cli/lib/search.go:78
-msgid "Error searching for Libraries: %v"
+msgid "Error searching for libraries: %v"
 msgstr ""
 
 #: cli/core/search.go:87

i18n/data/fr.po

@@ -849,7 +849,7 @@ msgid "Error installing tool %s"
 msgstr "Erreur lors de l’installation de l'outil %s."
 
 #: cli/lib/list.go:79
-msgid "Error listing Libraries: %v"
+msgid "Error listing libraries: %v"
 msgstr "Erreur lors du listage des librairies : %v"
 
 #: cli/board/listall.go:64
@@ -930,7 +930,7 @@ msgid "Error searching boards: %v"
 msgstr ""
 
 #: cli/lib/search.go:78
-msgid "Error searching for Libraries: %v"
+msgid "Error searching for libraries: %v"
 msgstr ""
 
 #: cli/core/search.go:87