[lldb][test] Toolchain detection rewrite in Python

lldb/packages/Python/lldbsuite/test/builders/builder.py

@@ -1,10 +1,12 @@
 import os
+import pathlib
 import platform
 import subprocess
 import sys
 import itertools
 
 import lldbsuite.test.lldbtest as lldbtest
+import lldbsuite.test.lldbplatformutil as lldbplatformutil
 import lldbsuite.test.lldbutil as lldbutil
 from lldbsuite.test import configuration
 from lldbsuite.test_event import build_exception
@@ -96,17 +98,105 @@ def getArchSpec(self, architecture):
         """
         return ["ARCH=" + architecture] if architecture else []
 
-    def getCCSpec(self, compiler):
+    def getToolchainSpec(self, compiler):
         """
-        Helper function to return the key-value string to specify the compiler
+        Helper function to return the key-value strings to specify the toolchain
         used for the make system.
         """
         cc = compiler if compiler else None
         if not cc and configuration.compiler:
             cc = configuration.compiler
-        if cc:
-            return ['CC="%s"' % cc]
-        return []
+
+        if not cc:
+            return []
+
+        exe_ext = ""
+        if lldbplatformutil.getHostPlatform() == "windows":
+            exe_ext = ".exe"
+
+        cc = cc.strip()
+        cc_path = pathlib.Path(cc)
+
+        # We can get CC compiler string in the following formats:
+        #  [<tool>] <compiler>    - such as 'xrun clang', 'xrun /usr/bin/clang' & etc
+        #
+        # Where <compiler> could contain the following parts:
+        #   <simple-name>[.<exe-ext>]                           - sucn as 'clang', 'clang.exe' ('clang-cl.exe'?)
+        #   <target-triple>-<simple-name>[.<exe-ext>]           - such as 'armv7-linux-gnueabi-gcc'
+        #   <path>/<simple-name>[.<exe-ext>]                    - such as '/usr/bin/clang', 'c:\path\to\compiler\clang,exe'
+        #   <path>/<target-triple>-<simple-name>[.<exe-ext>]    - such as '/usr/bin/clang', 'c:\path\to\compiler\clang,exe'
+
+        cc_ext = cc_path.suffix
+        # Compiler name without extension
+        cc_name = cc_path.stem.split(" ")[-1]
+
+        # A kind of compiler (canonical name): clang, gcc, cc & etc.
+        cc_type = cc_name
+        # A triple prefix of compiler name: <armv7-none-linux-gnu->gcc
+        cc_prefix = ""
+        if not "clang-cl" in cc_name and not "llvm-gcc" in cc_name:
+            cc_name_parts = cc_name.split("-")
+            cc_type = cc_name_parts[-1]
+            if len(cc_name_parts) > 1:
+                cc_prefix = "-".join(cc_name_parts[:-1]) + "-"
+
+        # A kind of C++ compiler.
+        cxx_types = {
+            "icc": "icpc",
+            "llvm-gcc": "llvm-g++",
+            "gcc": "g++",
+            "cc": "c++",
+            "clang": "clang++",
+        }
+        cxx_type = cxx_types.get(cc_type, cc_type)
+
+        cc_dir = cc_path.parent
+
+        def getToolchainUtil(util_name):
+            return cc_dir / (cc_prefix + util_name + cc_ext)
+
+        cxx = getToolchainUtil(cxx_type)
+
+        util_names = {
+            "OBJCOPY": "objcopy",
+            "STRIP": "strip",
+            "ARCHIVER": "ar",
+            "DWP": "dwp",
+        }
+        utils = []
+
+        if not lldbplatformutil.platformIsDarwin():
+            if cc_type in ["clang", "cc", "gcc"]:
+                util_paths = {}
+                # Assembly a toolchain side tool cmd based on passed CC.
+                for var, name in util_names.items():
+                    # Do not override explicity specified tool from the cmd line.
+                    if not os.getenv(var):
+                        util_paths[var] = getToolchainUtil(name)
+                    else:
+                        util_paths[var] = os.getenv(var)
+                utils.extend(["AR=%s" % util_paths["ARCHIVER"]])
+
+                # Look for llvm-dwp or gnu dwp
+                if not lldbutil.which(util_paths["DWP"]):
+                    util_paths["DWP"] = getToolchainUtil("llvm-dwp")
+                if not lldbutil.which(util_paths["DWP"]):
+                    util_paths["DWP"] = lldbutil.which("llvm-dwp")
+                if not util_paths["DWP"]:
+                    util_paths["DWP"] = lldbutil.which("dwp")
+                    if not util_paths["DWP"]:
+                        del util_paths["DWP"]
+
+                for var, path in util_paths.items():
+                    utils.append("%s=%s" % (var, path))
+        else:
+            utils.extend(["AR=%slibtool" % os.getenv("CROSS_COMPILE", "")])
+
+        return [
+            "CC=%s" % cc,
+            "CC_TYPE=%s" % cc_type,
+            "CXX=%s" % cxx,
+        ] + utils
 
     def getSDKRootSpec(self):
         """
@@ -178,7 +268,7 @@ def getBuildCommand(
             make_targets,
             self.getArchCFlags(architecture),
             self.getArchSpec(architecture),
-            self.getCCSpec(compiler),
+            self.getToolchainSpec(compiler),
             self.getExtraMakeArgs(),
             self.getSDKRootSpec(),
             self.getModuleCacheSpec(),

lldb/packages/Python/lldbsuite/test/make/Makefile.rules

@@ -102,15 +102,22 @@ endif
 # If you change the defaults of CC, be sure to also change it in the file
 # test/builders/builder_base.py, which provides a Python way to return the
 # value of the make variable CC -- getCompiler().
-#
-# See also these functions:
-#   o cxx_compiler
-#   o cxx_linker
 #----------------------------------------------------------------------
 ifeq "$(CC)" ""
 $(error "C compiler is not specified. Please run tests through lldb-dotest or lit")
 endif
 
+# Always override the linker. Assign already normalized CC.
+override LD := $(CC)
+# A kind of linker. It always gets retrieved from CC.
+override LDC := $(CC_TYPE)
+
+ifeq "$(HOST_OS)" "Windows_NT"
+       # This function enframes the full path with the platform specific quotes. This is necessary to run the c++ executable
+       # properly under 'sh' on Windows host (prevent the path breakage because of Windows style path separators).
+       override CXX := $(QUOTE)$(CXX)$(QUOTE)
+endif
+
 #----------------------------------------------------------------------
 # Handle SDKROOT for the cross platform builds.
 #----------------------------------------------------------------------
@@ -147,10 +154,8 @@ ifeq "$(OS)" "Darwin"
 	DS := $(DSYMUTIL)
 	DSFLAGS := $(DSFLAGS_EXTRAS)
 	DSYM = $(EXE).dSYM
-	AR := $(CROSS_COMPILE)libtool
 	ARFLAGS := -static -o
 else
-	AR := $(CROSS_COMPILE)ar
 	# On non-Apple platforms, -arch becomes -m
 	ARCHFLAG := -m
 
@@ -213,7 +218,7 @@ endif
 LIMIT_DEBUG_INFO_FLAGS =
 NO_LIMIT_DEBUG_INFO_FLAGS =
 MODULE_DEBUG_INFO_FLAGS =
-ifneq (,$(findstring clang,$(CC)))
+ifeq ($(CC_TYPE), clang)
    LIMIT_DEBUG_INFO_FLAGS += -flimit-debug-info
    NO_LIMIT_DEBUG_INFO_FLAGS += -fno-limit-debug-info
    MODULE_DEBUG_INFO_FLAGS += -gmodules
@@ -279,7 +284,6 @@ endif
 
 CFLAGS += $(CFLAGS_EXTRAS)
 CXXFLAGS += -std=c++11 $(CFLAGS) $(ARCH_CXXFLAGS)
-LD = $(CC)
 # Copy common options to the linker flags (dwarf, arch. & etc).
 # Note: we get some 'garbage' options for linker here (such as -I, --isystem & etc).
 LDFLAGS += $(CFLAGS)
@@ -312,61 +316,6 @@ ifneq "$(DYLIB_NAME)" ""
 	endif
 endif
 
-# Function that returns the counterpart C++ compiler, given $(CC) as arg.
-cxx_compiler_notdir = $(if $(findstring icc,$(1)), \
-			$(subst icc,icpc,$(1)), \
-			$(if $(findstring llvm-gcc,$(1)), \
-				$(subst llvm-gcc,llvm-g++,$(1)), \
-				$(if $(findstring gcc,$(1)), \
-					$(subst gcc,g++,$(1)), \
-					$(subst cc,c++,$(1)))))
-cxx_compiler = $(if $(findstring /,$(1)),$(join $(dir $(1)), $(call cxx_compiler_notdir,$(notdir $(1)))),$(call cxx_compiler_notdir,$(1)))
-
-# Function that returns the C++ linker, given $(CC) as arg.
-cxx_linker_notdir = $(if $(findstring icc,$(1)), \
-			$(subst icc,icpc,$(1)), \
-			$(if $(findstring llvm-gcc,$(1)), \
-				$(subst llvm-gcc,llvm-g++,$(1)), \
-				$(if $(findstring gcc,$(1)), \
-					$(subst gcc,g++,$(1)), \
-					$(subst cc,c++,$(1)))))
-cxx_linker = $(if $(findstring /,$(1)),$(join $(dir $(1)), $(call cxx_linker_notdir,$(notdir $(1)))),$(call cxx_linker_notdir,$(1)))
-
-ifneq "$(OS)" "Darwin"
-	CLANG_OR_GCC := $(strip $(if $(findstring clang,$(CC)), \
-				$(findstring clang,$(CC)), \
-				$(if $(findstring gcc,$(CC)), \
-					$(findstring gcc,$(CC)), \
-					cc)))
-
-	CC_LASTWORD := $(strip $(lastword $(subst -, ,$(CC))))
-
-	replace_with = $(strip $(if $(findstring $(3),$(CC_LASTWORD)), \
-			$(subst $(3),$(1),$(2)), \
-			$(subst $(3),$(1),$(subst -$(CC_LASTWORD),,$(2)))))
-
-	ifeq "$(notdir $(CC))" "$(CC)"
-		replace_cc_with = $(call replace_with,$(1),$(CC),$(CLANG_OR_GCC))
-	else
-		replace_cc_with = $(join $(dir $(CC)),$(call replace_with,$(1),$(notdir $(CC)),$(CLANG_OR_GCC)))
-	endif
-
-	OBJCOPY ?= $(call replace_cc_with,objcopy)
-	ARCHIVER ?= $(call replace_cc_with,ar)
-	# Look for llvm-dwp or gnu dwp
-	DWP ?= $(call replace_cc_with,llvm-dwp)
-	ifeq ($(wildcard $(DWP)),)
-		DWP = $(call replace_cc_with,dwp)
-		ifeq ($(wildcard $(DWP)),)
-			DWP = $(shell command -v llvm-dwp 2> /dev/null)
-			ifeq ($(wildcard $(DWP)),)
-				DWP = $(shell command -v dwp 2> /dev/null)
-			endif
-		endif
-	endif
-	override AR = $(ARCHIVER)
-endif
-
 ifdef PIE
 	LDFLAGS += -pie
 endif
@@ -375,7 +324,7 @@ endif
 # Windows specific options
 #----------------------------------------------------------------------
 ifeq "$(OS)" "Windows_NT"
-	ifneq (,$(findstring clang,$(CC)))
+	ifeq ($(CC_TYPE), clang)
 		# Clang for Windows doesn't support C++ Exceptions
 		CXXFLAGS += -fno-exceptions
 		CXXFLAGS += -D_HAS_EXCEPTIONS=0
@@ -420,7 +369,7 @@ endif
 
 ifeq (1,$(USE_LIBSTDCPP))
 	# Clang requires an extra flag: -stdlib=libstdc++
-	ifneq (,$(findstring clang,$(CC)))
+	ifeq ($(CC_TYPE), clang)
 		# Force clang looking for the gcc's headers at specific rootfs folder.
 		CXXFLAGS += -stdlib=libstdc++ $(GCC_TOOLCHAIN_FLAGS)
 		LDFLAGS += -stdlib=libstdc++ $(GCC_TOOLCHAIN_FLAGS)
@@ -458,7 +407,7 @@ ifeq (1, $(USE_SYSTEM_STDLIB))
         CXXFLAGS += -nostdlib++ -nostdinc++ -cxx-isystem $(SDKROOT)/usr/include/c++/v1
         LDFLAGS += -L$(SDKROOT)/usr/lib -Wl,-rpath,$(SDKROOT)/usr/lib -lc++
     else
-        ifneq (,$(findstring clang,$(CC)))
+        ifeq ($(CC_TYPE),clang)
             # Force clang looking for the gcc's headers at specific rootfs folder.
             CXXFLAGS += $(GCC_TOOLCHAIN_FLAGS)
             LDFLAGS += $(GCC_TOOLCHAIN_FLAGS)
@@ -485,8 +434,6 @@ DYLIB_OBJECTS +=$(strip $(DYLIB_C_SOURCES:.c=.o))
 DYLIB_OBJECTS +=$(strip $(DYLIB_OBJC_SOURCES:.m=.o))
 ifneq "$(strip $(DYLIB_CXX_SOURCES))" ""
 	DYLIB_OBJECTS +=$(strip $(patsubst %.mm, %.o, $(DYLIB_CXX_SOURCES:.cpp=.o)))
-	CXX = $(call cxx_compiler,$(CC))
-	LD = $(call cxx_linker,$(CC))
 endif
 
 #----------------------------------------------------------------------
@@ -509,8 +456,6 @@ endif
 #----------------------------------------------------------------------
 ifneq "$(strip $(CXX_SOURCES))" ""
 	OBJECTS +=$(strip $(CXX_SOURCES:.cpp=.o))
-	CXX = $(call cxx_compiler,$(CC))
-	LD = $(call cxx_linker,$(CC))
 endif
 
 #----------------------------------------------------------------------
@@ -526,19 +471,18 @@ endif
 #----------------------------------------------------------------------
 ifneq "$(strip $(OBJCXX_SOURCES))" ""
 	OBJECTS +=$(strip $(OBJCXX_SOURCES:.mm=.o))
-	CXX = $(call cxx_compiler,$(CC))
-	LD = $(call cxx_linker,$(CC))
 	ifeq "$(findstring lobjc,$(LDFLAGS))" ""
 		LDFLAGS +=-lobjc
 	endif
 endif
 
-ifeq ($(findstring clang, $(CXX)), clang)
+ifeq ($(CC_TYPE), clang)
 	CXXFLAGS += --driver-mode=g++
 endif
 
 ifneq "$(CXX)" ""
-	ifeq ($(findstring clang, $(LD)), clang)
+	# Specify the driver mode parameter if we use clang as the linker.
+	ifeq ($(LDC), clang)
 		LDFLAGS += --driver-mode=g++
 	endif
 endif

lldb/test/API/functionalities/breakpoint/breakpoint_ids/Makefile

@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CC_TYPE), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/breakpoint_locations/Makefile

@@ -1,6 +1,6 @@
 C_SOURCES := main.c
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CC_TYPE), icc)
     CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/Makefile

@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CC_TYPE), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/cpp/Makefile

@@ -1,7 +1,7 @@
 CXX_SOURCES := main.cpp
 CXXFLAGS_EXTRAS := -std=c++14
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CC_TYPE), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/dummy_target_breakpoints/Makefile

@@ -1,6 +1,6 @@
 C_SOURCES := main.c
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CC_TYPE), icc)
     CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/Makefile

@@ -1,6 +1,6 @@
 C_SOURCES := main.c
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CC_TYPE), icc)
     CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/step_over_breakpoint/Makefile

@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CC_TYPE), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/Makefile

@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-ifneq (,$(findstring icc,$(CC)))
+ifeq ($(CC_TYPE), icc)
     CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 

lldb/test/API/functionalities/data-formatter/data-formatter-objc/ObjCDataFormatterTestCase.py

@@ -16,12 +16,12 @@ def appkit_tester_impl(self, commands, use_constant_classes):
             self.build()
         else:
             disable_constant_classes = {
-                "CC": "xcrun clang",  # FIXME: Remove when flags are available upstream.
                 "CFLAGS_EXTRAS": "-fno-constant-nsnumber-literals "
                 + "-fno-constant-nsarray-literals "
                 + "-fno-constant-nsdictionary-literals",
             }
-            self.build(dictionary=disable_constant_classes)
+            # FIXME: Remove compiler when flags are available upstream.
+            self.build(dictionary=disable_constant_classes, compiler="xcrun clang")
         self.appkit_common_data_formatters_command()
         commands()