add diagnsotics to preprocessing step

Author: alessio-perugini

internal/arduino/builder/builder.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/compilation"
@@ -220,12 +221,6 @@ func NewBuilder(
 		targetPlatform:                targetPlatform,
 		actualPlatform:                actualPlatform,
 		toolEnv:                       toolEnv,
-		libsDetector: detector.NewSketchLibrariesDetector(
-			libsManager, libsResolver,
-			useCachedLibrariesResolution,
-			onlyUpdateCompilationDatabase,
-			logger,
-		),
 		buildOptions: newBuildOptions(
 			hardwareDirs, otherLibrariesDirs,
 			builtInLibrariesDirs, buildPath,
@@ -256,6 +251,14 @@ func NewBuilder(
 		b.compilerDiagnostics = append(b.compilerDiagnostics, diags...)
 	}
 
+	b.libsDetector = detector.NewSketchLibrariesDetector(
+		libsManager, libsResolver,
+		useCachedLibrariesResolution,
+		onlyUpdateCompilationDatabase,
+		logger,
+		b.compilerOutputParser,
+	)
+
 	return b, nil
 }
 
@@ -281,7 +284,18 @@ func (b *Builder) ImportedLibraries() libraries.List {
 
 // CompilerDiagnostics returns the parsed compiler diagnostics
 func (b *Builder) CompilerDiagnostics() diagnostics.Diagnostics {
-	return b.compilerDiagnostics
+	// When producing the preprocessing diagnostics, we might have duplicates
+	// caused by the run of the libraries detector and the sketch preprocessing.
+	return slices.CompactFunc(b.compilerDiagnostics, func(d1, d2 *diagnostics.Diagnostic) bool {
+		if d1 == nil || d2 == nil {
+			return false
+		}
+		return d1.Column == d2.Column &&
+			d1.File == d2.File &&
+			d1.Line == d2.Line &&
+			d1.Message == d2.Message &&
+			d1.Severity == d2.Severity
+	})
 }
 
 // Preprocess fixdoc

internal/arduino/builder/internal/detector/detector.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/diagnostics"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/logger"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/preprocessor"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils"
@@ -57,6 +58,7 @@ type SketchLibrariesDetector struct {
 	librariesResolutionResults    map[string]libraryResolutionResult
 	includeFolders                paths.PathList
 	logger                        *logger.BuilderLogger
+	compilerOutputParser          diagnostics.CompilerOutputParserCB
 }
 
 // NewSketchLibrariesDetector todo
@@ -66,6 +68,7 @@ func NewSketchLibrariesDetector(
 	useCachedLibrariesResolution bool,
 	onlyUpdateCompilationDatabase bool,
 	logger *logger.BuilderLogger,
+	compilerOutputParser diagnostics.CompilerOutputParserCB,
 ) *SketchLibrariesDetector {
 	return &SketchLibrariesDetector{
 		librariesManager:              lm,
@@ -76,6 +79,7 @@ func NewSketchLibrariesDetector(
 		includeFolders:                paths.PathList{},
 		onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase,
 		logger:                        logger,
+		compilerOutputParser:          compilerOutputParser,
 	}
 }
 
@@ -347,7 +351,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 			}
 		} else {
 			var preprocStdout []byte
-			preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties)
+			preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties, nil)
 			if l.logger.Verbose() {
 				l.logger.WriteStdout(preprocStdout)
 			}
@@ -379,7 +383,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 			if preprocErr == nil || preprocStderr == nil {
 				// Filename came from cache, so run preprocessor to obtain error to show
 				var preprocStdout []byte
-				preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties)
+				preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties, l.compilerOutputParser)
 				if l.logger.Verbose() {
 					l.logger.WriteStdout(preprocStdout)
 				}

internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go

@@ -22,6 +22,7 @@ import (
 	"path/filepath"
 	"runtime"
 
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/diagnostics"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils"
 	"github.com/arduino/arduino-cli/internal/arduino/sketch"
 	"github.com/arduino/go-paths-helper"
@@ -30,7 +31,11 @@ import (
 
 // PreprocessSketchWithArduinoPreprocessor performs preprocessing of the arduino sketch
 // using arduino-preprocessor (https://github.com/arduino/arduino-preprocessor).
-func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) {
+func PreprocessSketchWithArduinoPreprocessor(
+	sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList,
+	lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool,
+	compilerOutputParserCB diagnostics.CompilerOutputParserCB,
+) ([]byte, []byte, error) {
 	verboseOut := &bytes.Buffer{}
 	normalOut := &bytes.Buffer{}
 	if err := buildPath.Join("preproc").MkdirAll(); err != nil {
@@ -39,7 +44,7 @@ func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths
 
 	sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp")
 	targetFile := buildPath.Join("preproc", "sketch_merged.cpp")
-	gccStdout, gccStderr, err := GCC(sourceFile, targetFile, includeFolders, buildProperties)
+	gccStdout, gccStderr, err := GCC(sourceFile, targetFile, includeFolders, buildProperties, compilerOutputParserCB)
 	verboseOut.Write(gccStdout)
 	verboseOut.Write(gccStderr)
 	if err != nil {

internal/arduino/builder/internal/preprocessor/ctags.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 
 	"github.com/arduino/arduino-cli/internal/arduino/builder/cpp"
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/diagnostics"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/preprocessor/internal/ctags"
 	"github.com/arduino/arduino-cli/internal/arduino/sketch"
 	"github.com/arduino/arduino-cli/internal/i18n"
@@ -40,7 +41,11 @@ var tr = i18n.Tr
 var DebugPreprocessor bool
 
 // PreprocessSketchWithCtags performs preprocessing of the arduino sketch using CTags.
-func PreprocessSketchWithCtags(sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) {
+func PreprocessSketchWithCtags(
+	sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList,
+	lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool,
+	compilerOutputParserCB diagnostics.CompilerOutputParserCB,
+) ([]byte, []byte, error) {
 	// Create a temporary working directory
 	tmpDir, err := paths.MkTempDir("", "")
 	if err != nil {
@@ -54,7 +59,7 @@ func PreprocessSketchWithCtags(sketch *sketch.Sketch, buildPath *paths.Path, inc
 
 	// Run GCC preprocessor
 	sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp")
-	gccStdout, gccStderr, err := GCC(sourceFile, ctagsTarget, includes, buildProperties)
+	gccStdout, gccStderr, err := GCC(sourceFile, ctagsTarget, includes, buildProperties, compilerOutputParserCB)
 	verboseOutput.Write(gccStdout)
 	verboseOutput.Write(gccStderr)
 	normalOutput.Write(gccStderr)

internal/arduino/builder/internal/preprocessor/gcc.go

@@ -23,13 +23,18 @@ import (
 
 	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/cpp"
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/diagnostics"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 )
 
 // GCC performs a run of the gcc preprocess (macro/includes expansion). The function outputs the result
 // to targetFilePath. Returns the stdout/stderr of gcc if any.
-func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map) ([]byte, []byte, error) {
+func GCC(
+	sourceFilePath, targetFilePath *paths.Path,
+	includes paths.PathList, buildProperties *properties.Map,
+	compilerOutputParserCB diagnostics.CompilerOutputParserCB,
+) ([]byte, []byte, error) {
 	gccBuildProperties := properties.NewMap()
 	gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
 	gccBuildProperties.Merge(buildProperties)
@@ -74,6 +79,10 @@ func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.
 	}
 	stdout, stderr, err := proc.RunAndCaptureOutput(context.Background())
 
+	if compilerOutputParserCB != nil {
+		compilerOutputParserCB(proc.GetArgs(), stderr)
+	}
+
 	// Append gcc arguments to stdout
 	stdout = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stdout...)
 

internal/arduino/builder/preprocess_sketch.go

@@ -26,6 +26,7 @@ func (b *Builder) preprocessSketch(includes paths.PathList) error {
 	normalOutput, verboseOutput, err := preprocessor.PreprocessSketchWithCtags(
 		b.sketch, b.buildPath, includes, b.lineOffset,
 		b.buildProperties, b.onlyUpdateCompilationDatabase,
+		b.compilerOutputParser,
 	)
 	if b.logger.Verbose() {
 		b.logger.WriteStdout(verboseOutput)