tapi & clang cherrypicks

clang/lib/InstallAPI/DirectoryScanner.cpp

@@ -130,7 +130,8 @@ Error DirectoryScanner::scanHeaders(StringRef Path, Library &Lib,
   if (ParentPath.empty())
     ParentPath = Path;
   for (const StringRef Dir : SubDirectories)
-    return scanHeaders(Dir, Lib, Type, BasePath, ParentPath);
+    if (Error Err = scanHeaders(Dir, Lib, Type, BasePath, ParentPath))
+      return Err;
 
   return Error::success();
 }
@@ -276,7 +277,8 @@ llvm::Error DirectoryScanner::scanForFrameworks(StringRef Directory) {
   // Expect a certain directory structure and naming convention to find
   // frameworks.
   static const char *SubDirectories[] = {"System/Library/Frameworks/",
-                                         "System/Library/PrivateFrameworks/"};
+                                         "System/Library/PrivateFrameworks/",
+                                         "System/Library/SubFrameworks"};
 
   // Check if the directory is already a framework.
   if (isFramework(Directory)) {

clang/lib/Lex/InitHeaderSearch.cpp

@@ -346,6 +346,7 @@ void InitHeaderSearch::AddDefaultIncludePaths(
         AddPath("/System/DriverKit/System/Library/Frameworks", System, true);
       } else {
         AddPath("/System/Library/Frameworks", System, true);
+        AddPath("/System/Library/SubFrameworks", System, true);
         AddPath("/Library/Frameworks", System, true);
       }
     }

clang/test/Driver/darwin-subframeworks.c

@@ -0,0 +1,18 @@
+// UNSUPPORTED: system-windows
+//   Windows is unsupported because we use the Unix path separator `\`.
+
+// Add default directories before running clang to check default 
+// search paths. 
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: cp -R %S/Inputs/MacOSX15.1.sdk %t/
+// RUN: mkdir -p %t/MacOSX15.1.sdk/System/Library/Frameworks
+// RUN: mkdir -p %t/MacOSX15.1.sdk/System/Library/SubFrameworks
+// RUN: mkdir -p %t/MacOSX15.1.sdk/usr/include
+
+// RUN: %clang %s -target arm64-apple-darwin13.0 -isysroot %t/MacOSX15.1.sdk -E -v 2>&1 | FileCheck %s
+
+// CHECK:    -isysroot [[PATH:[^ ]*/MacOSX15.1.sdk]]
+// CHECK:    #include <...> search starts here:
+// CHECK:    [[PATH]]/usr/include
+// CHECK:    [[PATH]]/System/Library/Frameworks (framework directory)
+// CHECK:    [[PATH]]/System/Library/SubFrameworks (framework directory)

clang/test/InstallAPI/directory-scanning-subdirectories.test

@@ -0,0 +1,61 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t
+; RUN: mkdir -p %t/DstRoot/
+; RUN: cp -r %S/Inputs/LibFoo/* %t/DstRoot/
+
+; RUN: clang-installapi \
+; RUN: -target arm64-apple-macos12 -install_name @rpath/libfoo.dylib \
+; RUN: -current_version 1 -compatibility_version 1 \
+; RUN: -I%t/DstRoot/usr/include -dynamiclib \
+; RUN: -exclude-public-header %t/DstRoot/usr/include/public.h \
+; RUN: %t/DstRoot -o %t/output.tbd 2>&1 | FileCheck %s --allow-empty \
+; RUN: --implicit-check-not=error --implicit-check-not=warning 
+; RUN: llvm-readtapi --compare %t/output.tbd %t/expected.tbd 
+
+
+;--- DstRoot/usr/include/extra/extra.h
+int extra(void);
+
+;--- DstRoot/usr/include/extra/additional/additional.h
+int additional(void);
+
+;--- DstRoot/usr/include/more/more.h
+int more(void);
+
+;--- DstRoot/usr/include/another/another.h
+int another(void);
+
+;--- expected.tbd
+{
+  "main_library": {
+    "exported_symbols": [
+      {
+        "text": {
+          "global": [
+            "_foo", "_additional", "_more",
+            "_another", "_extra"
+          ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": [
+          "not_app_extension_safe"
+        ]
+      }
+    ],
+    "install_names": [
+      {
+        "name": "@rpath/libfoo.dylib"
+      }
+    ],
+    "target_info": [
+      {
+        "min_deployment": "12",
+        "target": "arm64-macos"
+      }
+    ]
+  },
+  "tapi_tbd_version": 5
+}

compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake

@@ -136,14 +136,13 @@ function(darwin_test_archs os valid_archs)
 
   # The simple program will build for x86_64h on the simulator because it is
   # compatible with x86_64 libraries (mostly), but since x86_64h isn't actually
-  # a valid or useful architecture for the iOS simulator we should drop it.
+  # a valid or useful architecture for the simulators. We should drop it.
   if(${os} MATCHES "^(iossim|tvossim|watchossim)$")
     list(REMOVE_ITEM archs "x86_64h")
-  endif()
-
-  if(${os} MATCHES "iossim")
-    message(STATUS "Disabling i386 slice for iossim")
-    list(REMOVE_ITEM archs "i386")
+    if ("i386" IN_LIST archs)
+      list(REMOVE_ITEM archs "i386")
+      message(STATUS "Disabling i386 slice for simulator")
+    endif()
   endif()
 
   if(${os} MATCHES "^ios$")

llvm/lib/TextAPI/InterfaceFile.cpp

@@ -87,6 +87,9 @@ void InterfaceFile::addDocument(std::shared_ptr<InterfaceFile> &&Document) {
                                   const std::shared_ptr<InterfaceFile> &RHS) {
                                  return LHS->InstallName < RHS->InstallName;
                                });
+  assert((Pos == Documents.end() ||
+          (*Pos)->InstallName != Document->InstallName) &&
+         "Unexpected duplicate document added");
   Document->Parent = this;
   Documents.insert(Pos, Document);
 }
@@ -169,6 +172,7 @@ InterfaceFile::merge(const InterfaceFile *O) const {
 
   IF->setTwoLevelNamespace(isTwoLevelNamespace());
   IF->setApplicationExtensionSafe(isApplicationExtensionSafe());
+  IF->setOSLibNotForSharedCache(isOSLibNotForSharedCache());
 
   for (const auto &It : umbrellas()) {
     if (!It.second.empty())
@@ -235,6 +239,8 @@ InterfaceFile::remove(Architecture Arch) const {
       return make_error<TextAPIError>(TextAPIErrorCode::NoSuchArchitecture);
   }
 
+  // FIXME: Figure out how to keep these attributes in sync when new ones are
+  // added.
   std::unique_ptr<InterfaceFile> IF(new InterfaceFile());
   IF->setFileType(getFileType());
   IF->setPath(getPath());
@@ -245,6 +251,7 @@ InterfaceFile::remove(Architecture Arch) const {
   IF->setSwiftABIVersion(getSwiftABIVersion());
   IF->setTwoLevelNamespace(isTwoLevelNamespace());
   IF->setApplicationExtensionSafe(isApplicationExtensionSafe());
+  IF->setOSLibNotForSharedCache(isOSLibNotForSharedCache());
   for (const auto &It : umbrellas())
     if (It.first.Arch != Arch)
       IF->addParentUmbrella(It.first, It.second);
@@ -313,6 +320,7 @@ InterfaceFile::extract(Architecture Arch) const {
   IF->setSwiftABIVersion(getSwiftABIVersion());
   IF->setTwoLevelNamespace(isTwoLevelNamespace());
   IF->setApplicationExtensionSafe(isApplicationExtensionSafe());
+  IF->setOSLibNotForSharedCache(isOSLibNotForSharedCache());
   for (const auto &It : umbrellas())
     if (It.first.Arch == Arch)
       IF->addParentUmbrella(It.first, It.second);

llvm/lib/TextAPI/Utils.cpp

@@ -120,6 +120,9 @@ bool llvm::MachO::isPrivateLibrary(StringRef Path, bool IsSymLink) {
   if (Path.starts_with("/System/Library/PrivateFrameworks"))
     return true;
 
+  if (Path.starts_with("/System/Library/SubFrameworks"))
+    return true;
+
   // Everything in /usr/lib/swift (including sub-directories) are considered
   // public.
   if (Path.consume_front("/usr/lib/swift/"))

llvm/test/tools/llvm-readtapi/compare-ignore-archs.test

@@ -0,0 +1,68 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t  
+
+; RUN: llvm-readtapi --compare %t/all_archs.tbd  %t/missing_archs.tbd --ignore-arch armv7 --ignore-arch armv7s 2>&1 | FileCheck %s --allow-empty --implicit-check-not warning: --implicit-check-not error:
+; RUN: not llvm-readtapi --compare %t/all_archs.tbd  %t/missing_archs.tbd --ignore-arch armv7s 2>&1 | FileCheck %s --check-prefix ARMV7
+
+all_archs.tbd
+; ARMV7: < {{.*}}all_archs.tbd
+; ARMV7: > {{.*}}missing_archs.tbd
+
+; ARMV7:      Reexported Libraries
+; ARMV7-NEXT: 	    armv7-apple-ios
+; ARMV7:      Symbols
+; ARMV7-NEXT: 	    armv7-apple-ios
+; ARMV7:      Inlined Reexported Frameworks/Libraries
+; ARMV7-NEXT: 	/System/Library/Frameworks/FooCore.framework/FooCore
+; ARMV7: 		  Symbols
+; ARMV7-NEXT: 			armv7-apple-ios
+
+;--- all_archs.tbd
+--- !tapi-tbd
+tbd-version:     4
+targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+install-name:    '/System/Library/Frameworks/Foo.framework/Foo'
+current-version: 1986.34.9
+reexported-libraries:
+  - targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+    libraries:       [ '/System/Library/Frameworks/FooCore.framework/FooCore' ]
+exports:
+  - targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+    symbols:         [ _AllRequestsKeyPathFragment, _AnalyticsLoggingSubsystem, _AnyRequestKeyPathFragment, 
+                       _bar_getBarPointSize_ints, _bar_newBarMessage, _bar_serialize ]
+  - targets:         [ arm64-ios, arm64e-ios ]
+    symbols:         [ __ZN3lingo11MapEdgeRoad6lengthEv, 
+                       __ZTVN3lingo11MapEdgeRoadE, __ZTVN3lingo7MapNodeE, __ZTVN5bar19GeometryPathElementE ]
+--- !tapi-tbd
+tbd-version:     4
+targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+install-name:    '/System/Library/Frameworks/FooCore.framework/FooCore'
+current-version: 1986.34.9
+exports:
+  - targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+    symbols:         [ _sym, _workgroupsym, _taskgroup_sim, meta_sim ]
+...
+
+;--- missing_archs.tbd
+--- !tapi-tbd
+tbd-version:     4
+targets:         [ arm64-ios, arm64e-ios ]
+install-name:    '/System/Library/Frameworks/Foo.framework/Foo'
+current-version: 1986.34.9
+reexported-libraries:
+  - targets:         [ arm64-ios, arm64e-ios ]
+    libraries:       [ '/System/Library/Frameworks/FooCore.framework/FooCore' ]
+exports:
+  - targets:         [ arm64-ios, arm64e-ios ]
+    symbols:         [ _AllRequestsKeyPathFragment, _AnalyticsLoggingSubsystem, _AnyRequestKeyPathFragment, 
+                       _bar_getBarPointSize_ints, _bar_newBarMessage, _bar_serialize, __ZN3lingo11MapEdgeRoad6lengthEv, 
+                       __ZTVN3lingo11MapEdgeRoadE, __ZTVN3lingo7MapNodeE, __ZTVN5bar19GeometryPathElementE ]
+--- !tapi-tbd
+tbd-version:     4
+targets:         [ arm64-ios, arm64e-ios ]
+install-name:    '/System/Library/Frameworks/FooCore.framework/FooCore'
+current-version: 1986.34.9
+exports:
+  - targets:         [ arm64-ios, arm64e-ios ]
+    symbols:         [ _sym, _workgroupsym, _taskgroup_sim, meta_sim ]
+...

llvm/test/tools/llvm-readtapi/stubify-delete.test

@@ -2,17 +2,20 @@
 # Setup a mix of public and private libraries that resemble apple sdk.
 ; RUN: mkdir -p %t/sysroot/usr/local/lib/ %t/sysroot/usr/lib/
 ; RUN: mkdir -p %t/sysroot/System/Library/Frameworks/System.framework %t/sysroot/System/Library/PrivateFrameworks/Fat.framework
+; RUN: mkdir -p %t/sysroot/System/Library/SubFrameworks/Fat.framework/Headers
 ; RUN: yaml2obj %S/Inputs/libSystem.1.yaml -o %t/sysroot/System/Library/Frameworks/System.framework/System
 ; RUN: yaml2obj %S/Inputs/objc.yaml -o %t/sysroot/usr/lib/libobjc.dylib
 ; RUN: cp %t/sysroot/usr/lib/libobjc.dylib %t/sysroot/usr/local/lib/libobjc-unstable.dylib
 ; RUN: yaml2obj %S/Inputs/universal.yaml -o %t/sysroot/System/Library/PrivateFrameworks/Fat.framework/Fat
+; RUN: cp %t/sysroot/System/Library/PrivateFrameworks/Fat.framework/Fat %t/sysroot/System/Library/SubFrameworks/Fat.framework/Fat
+; RUN: touch %t/sysroot/System/Library/SubFrameworks/Fat.framework/Headers/Fat.h
 ; RUN: llvm-readtapi -stubify %t/sysroot --delete-input --delete-private-libraries 2>&1 | FileCheck %s --allow-empty  --implicit-check-not warning: --implicit-check-not error:
 # Validate expected files are removed.
 ; RUN: not test -f %t/sysroot/System/Library/PrivateFrameworks
 ; RUN: not test -f %t/sysroot/usr/local
 ; RUN: not test -f %t/sysroot/usr/lib/libobjc.dylib
 ; RUN: not test -f %t/sysroot/System/Library/Frameworks/System.framework/System
+; RUN: not test -f %t/sysroot/System/Library/SubFrameworks/Fat.framework/Fat
 ; RUN: test -f %t/sysroot/System/Library/Frameworks/System.framework/System.tbd
 ; RUN: test -f %t/sysroot/usr/lib/libobjc.tbd
-
-
+; RUN: test -f %t/sysroot/System/Library/SubFrameworks/Fat.framework/Headers/Fat.h

llvm/tools/llvm-readtapi/TapiOpts.td

@@ -35,3 +35,8 @@ def delete_input : FF<"delete-input", "delete and replace input file on success"
 def delete_private_libraries : FF<"delete-private-libraries", "delete private system dynamic libraries and frameworks">;
 def t: FF<"t", "logs each library loaded, useful for debugging problems with search paths where the wrong library is loaded">;
 
+
+//
+// Compare options
+//
+defm ignore_arch : JS<"ignore-arch", "<architecture> slice to ignore for comparison", "<architecture>">;

llvm/tools/llvm-readtapi/llvm-readtapi.cpp

@@ -71,9 +71,14 @@ struct StubOptions {
   bool TraceLibs = false;
 };
 
+struct CompareOptions {
+  ArchitectureSet ArchsToIgnore;
+};
+
 struct Context {
   std::vector<std::string> Inputs;
   StubOptions StubOpt;
+  CompareOptions CmpOpt;
   std::unique_ptr<llvm::raw_fd_stream> OutStream;
   FileType WriteFT = FileType::TBD_V5;
   bool Compact = false;
@@ -160,6 +165,28 @@ static bool handleCompareAction(const Context &Ctx) {
   auto LeftIF = getInterfaceFile(Ctx.Inputs.front());
   auto RightIF = getInterfaceFile(Ctx.Inputs.at(1));
 
+  // Remove all architectures to ignore before running comparison.
+  auto removeArchFromIF = [](auto &IF, const ArchitectureSet &ArchSet,
+                             const Architecture ArchToRemove) {
+    if (!ArchSet.has(ArchToRemove))
+      return;
+    if (ArchSet.count() == 1)
+      return;
+    auto OutIF = IF->remove(ArchToRemove);
+    if (!OutIF)
+      ExitOnErr(OutIF.takeError());
+    IF = std::move(*OutIF);
+  };
+
+  if (!Ctx.CmpOpt.ArchsToIgnore.empty()) {
+    const ArchitectureSet LeftArchs = LeftIF->getArchitectures();
+    const ArchitectureSet RightArchs = RightIF->getArchitectures();
+    for (const auto Arch : Ctx.CmpOpt.ArchsToIgnore) {
+      removeArchFromIF(LeftIF, LeftArchs, Arch);
+      removeArchFromIF(RightIF, RightArchs, Arch);
+    }
+  }
+
   raw_ostream &OS = Ctx.OutStream ? *Ctx.OutStream : outs();
   return DiffEngine(LeftIF.get(), RightIF.get()).compareFiles(OS);
 }
@@ -228,16 +255,21 @@ static void stubifyDirectory(const StringRef InputPath, Context &Ctx) {
     if (EC)
       reportError(IT->path() + ": " + EC.message());
 
-    // Skip header directories (include/Headers/PrivateHeaders) and module
-    // files.
+    // Skip header directories (include/Headers/PrivateHeaders).
     StringRef Path = IT->path();
-    if (Path.ends_with("/include") || Path.ends_with("/Headers") ||
-        Path.ends_with("/PrivateHeaders") || Path.ends_with("/Modules") ||
-        Path.ends_with(".map") || Path.ends_with(".modulemap")) {
-      IT.no_push();
-      continue;
+    if (sys::fs::is_directory(Path)) {
+      const StringRef Stem = sys::path::stem(Path);
+      if ((Stem == "include") || (Stem == "Headers") ||
+          (Stem == "PrivateHeaders") || (Stem == "Modules")) {
+        IT.no_push();
+        continue;
+      }
     }
 
+    // Skip module files too.
+    if (Path.ends_with(".map") || Path.ends_with(".modulemap"))
+      continue;
+
     // Check if the entry is a symlink. We don't follow symlinks but we record
     // their content.
     bool IsSymLink;
@@ -497,12 +529,20 @@ int main(int Argc, char **Argv) {
       reportError("unsupported filetype '" + FT + "'");
   }
 
-  if (opt::Arg *A = Args.getLastArg(OPT_arch_EQ)) {
-    StringRef Arch = A->getValue();
-    Ctx.Arch = getArchitectureFromName(Arch);
-    if (Ctx.Arch == AK_unknown)
-      reportError("unsupported architecture '" + Arch);
-  }
+  auto SanitizeArch = [&](opt::Arg *A) {
+    StringRef ArchStr = A->getValue();
+    auto Arch = getArchitectureFromName(ArchStr);
+    if (Arch == AK_unknown)
+      reportError("unsupported architecture '" + ArchStr);
+    return Arch;
+  };
+
+  if (opt::Arg *A = Args.getLastArg(OPT_arch_EQ))
+    Ctx.Arch = SanitizeArch(A);
+
+  for (opt::Arg *A : Args.filtered(OPT_ignore_arch_EQ))
+    Ctx.CmpOpt.ArchsToIgnore.set(SanitizeArch(A));
+
   // Handle top level and exclusive operation.
   SmallVector<opt::Arg *, 1> ActionArgs(Args.filtered(OPT_action_group));