reland "[lld-link] implement -start-lib and -end-lib"

Summary:
This is a re-land of r370487 with a fix for the use-after-free bug
that rev contained.

This implements -start-lib and -end-lib flags for lld-link, analogous
to the similarly named options in ld.lld. Object files after
-start-lib are included in the link only when needed to resolve
undefined symbols. The -end-lib flag goes back to the normal behavior
of always including object files in the link. This mimics the
semantics of static libraries, but without needing to actually create
the archive file.

Reviewers: ruiu, smeenai, MaskRay

Reviewed By: ruiu, MaskRay

Subscribers: akhuang, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D66848

llvm-svn: 370816

Author: inglorion

lld/COFF/DebugTypes.cpp

@@ -231,7 +231,7 @@ void TypeServerSource::enqueue(const ObjFile *dependentFile,
   if (!it.second)
     return; // another OBJ already scheduled this PDB for load
 
-  driver->enqueuePath(*p, false);
+  driver->enqueuePath(*p, false, false);
 }
 
 // Create an instance of TypeServerSource or an error string if the PDB couldn't

lld/COFF/Driver.cpp

@@ -170,7 +170,7 @@ MemoryBufferRef LinkerDriver::takeBuffer(std::unique_ptr<MemoryBuffer> mb) {
 }
 
 void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> mb,
-                             bool wholeArchive) {
+                             bool wholeArchive, bool lazy) {
   StringRef filename = mb->getBufferIdentifier();
 
   MemoryBufferRef mbref = takeBuffer(std::move(mb));
@@ -195,11 +195,17 @@ void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> mb,
     symtab->addFile(make<ArchiveFile>(mbref));
     break;
   case file_magic::bitcode:
-    symtab->addFile(make<BitcodeFile>(mbref, "", 0));
+    if (lazy)
+      symtab->addFile(make<LazyObjFile>(mbref));
+    else
+      symtab->addFile(make<BitcodeFile>(mbref, "", 0));
     break;
   case file_magic::coff_object:
   case file_magic::coff_import_library:
-    symtab->addFile(make<ObjFile>(mbref));
+    if (lazy)
+      symtab->addFile(make<LazyObjFile>(mbref));
+    else
+      symtab->addFile(make<ObjFile>(mbref));
     break;
   case file_magic::pdb:
     loadTypeServerSource(mbref);
@@ -220,7 +226,7 @@ void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> mb,
   }
 }
 
-void LinkerDriver::enqueuePath(StringRef path, bool wholeArchive) {
+void LinkerDriver::enqueuePath(StringRef path, bool wholeArchive, bool lazy) {
   auto future =
       std::make_shared<std::future<MBErrPair>>(createFutureForFile(path));
   std::string pathStr = path;
@@ -240,7 +246,7 @@ void LinkerDriver::enqueuePath(StringRef path, bool wholeArchive) {
       else
         error(msg + "; did you mean '" + nearest + "'");
     } else
-      driver->addBuffer(std::move(mbOrErr.first), wholeArchive);
+      driver->addBuffer(std::move(mbOrErr.first), wholeArchive, lazy);
   });
 }
 
@@ -359,7 +365,7 @@ void LinkerDriver::parseDirectives(InputFile *file) {
       break;
     case OPT_defaultlib:
       if (Optional<StringRef> path = findLib(arg->getValue()))
-        enqueuePath(*path, false);
+        enqueuePath(*path, false, false);
       break;
     case OPT_entry:
       config->entry = addUndefined(mangle(arg->getValue()));
@@ -1553,19 +1559,45 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
     return false;
   };
 
-  // Create a list of input files. Files can be given as arguments
-  // for /defaultlib option.
-  for (auto *arg : args.filtered(OPT_INPUT, OPT_wholearchive_file))
-    if (Optional<StringRef> path = findFile(arg->getValue()))
-      enqueuePath(*path, isWholeArchive(*path));
+  // Create a list of input files. These can be given as OPT_INPUT options
+  // and OPT_wholearchive_file options, and we also need to track OPT_start_lib
+  // and OPT_end_lib.
+  bool inLib = false;
+  for (auto *arg : args) {
+    switch (arg->getOption().getID()) {
+    case OPT_end_lib:
+      if (!inLib)
+        error("stray " + arg->getSpelling());
+      inLib = false;
+      break;
+    case OPT_start_lib:
+      if (inLib)
+        error("nested " + arg->getSpelling());
+      inLib = true;
+      break;
+    case OPT_wholearchive_file:
+      if (Optional<StringRef> path = findFile(arg->getValue()))
+        enqueuePath(*path, true, inLib);
+      break;
+    case OPT_INPUT:
+      if (Optional<StringRef> path = findFile(arg->getValue()))
+        enqueuePath(*path, isWholeArchive(*path), inLib);
+      break;
+    default:
+      // Ignore other options.
+      break;
+    }
+  }
 
+  // Process files specified as /defaultlib. These should be enequeued after
+  // other files, which is why they are in a separate loop.
   for (auto *arg : args.filtered(OPT_defaultlib))
     if (Optional<StringRef> path = findLib(arg->getValue()))
-      enqueuePath(*path, false);
+      enqueuePath(*path, false, false);
 
   // Windows specific -- Create a resource file containing a manifest file.
   if (config->manifest == Configuration::Embed)
-    addBuffer(createManifestRes(), false);
+    addBuffer(createManifestRes(), false, false);
 
   // Read all input files given via the command line.
   run();
@@ -1782,7 +1814,7 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
   if (args.hasArg(OPT_include_optional)) {
     // Handle /includeoptional
     for (auto *arg : args.filtered(OPT_include_optional))
-      if (dyn_cast_or_null<Lazy>(symtab->find(arg->getValue())))
+      if (dyn_cast_or_null<LazyArchive>(symtab->find(arg->getValue())))
         addUndefined(arg->getValue());
     while (run());
   }

lld/COFF/Driver.h

@@ -77,7 +77,7 @@ class LinkerDriver {
 
   MemoryBufferRef takeBuffer(std::unique_ptr<MemoryBuffer> mb);
 
-  void enqueuePath(StringRef path, bool wholeArchive);
+  void enqueuePath(StringRef path, bool wholeArchive, bool lazy);
 
 private:
   std::unique_ptr<llvm::TarWriter> tar; // for /linkrepro
@@ -124,7 +124,8 @@ class LinkerDriver {
   StringRef findDefaultEntry();
   WindowsSubsystem inferSubsystem();
 
-  void addBuffer(std::unique_ptr<MemoryBuffer> mb, bool wholeArchive);
+  void addBuffer(std::unique_ptr<MemoryBuffer> mb, bool wholeArchive,
+                 bool lazy);
   void addArchiveBuffer(MemoryBufferRef mbref, StringRef symName,
                         StringRef parentName, uint64_t offsetInArchive);
 

lld/COFF/InputFiles.cpp

@@ -73,6 +73,10 @@ static void checkAndSetWeakAlias(SymbolTable *symtab, InputFile *f,
   }
 }
 
+static bool ignoredSymbolName(StringRef name) {
+  return name == "@feat.00" || name == "@comp.id";
+}
+
 ArchiveFile::ArchiveFile(MemoryBufferRef m) : InputFile(ArchiveKind, m) {}
 
 void ArchiveFile::parse() {
@@ -81,7 +85,7 @@ void ArchiveFile::parse() {
 
   // Read the symbol table to construct Lazy objects.
   for (const Archive::Symbol &sym : file->symbols())
-    symtab->addLazy(this, sym);
+    symtab->addLazyArchive(this, sym);
 }
 
 // Returns a buffer pointing to a member file containing a given symbol.
@@ -116,6 +120,49 @@ std::vector<MemoryBufferRef> getArchiveMembers(Archive *file) {
   return v;
 }
 
+void LazyObjFile::fetch() {
+  if (mb.getBuffer().empty())
+    return;
+
+  InputFile *file;
+  if (isBitcode(mb))
+    file = make<BitcodeFile>(mb, "", 0, std::move(symbols));
+  else
+    file = make<ObjFile>(mb, std::move(symbols));
+  mb = {};
+  symtab->addFile(file);
+}
+
+void LazyObjFile::parse() {
+  if (isBitcode(this->mb)) {
+    // Bitcode file.
+    std::unique_ptr<lto::InputFile> obj =
+        CHECK(lto::InputFile::create(this->mb), this);
+    for (const lto::InputFile::Symbol &sym : obj->symbols()) {
+      if (!sym.isUndefined())
+        symtab->addLazyObject(this, sym.getName());
+    }
+    return;
+  }
+
+  // Native object file.
+  std::unique_ptr<Binary> coffObjPtr = CHECK(createBinary(mb), this);
+  COFFObjectFile *coffObj = cast<COFFObjectFile>(coffObjPtr.get());
+  uint32_t numSymbols = coffObj->getNumberOfSymbols();
+  for (uint32_t i = 0; i < numSymbols; ++i) {
+    COFFSymbolRef coffSym = check(coffObj->getSymbol(i));
+    if (coffSym.isUndefined() || !coffSym.isExternal() ||
+        coffSym.isWeakExternal())
+      continue;
+    StringRef name;
+    coffObj->getSymbolName(coffSym, name);
+    if (coffSym.isAbsolute() && ignoredSymbolName(name))
+      continue;
+    symtab->addLazyObject(this, name);
+    i += coffSym.getNumberOfAuxSymbols();
+  }
+}
+
 void ObjFile::parse() {
   // Parse a memory buffer as a COFF file.
   std::unique_ptr<Binary> bin = CHECK(createBinary(mb), this);
@@ -526,13 +573,11 @@ Optional<Symbol *> ObjFile::createDefined(
   if (sym.isAbsolute()) {
     StringRef name = getName();
 
-    // Skip special symbols.
-    if (name == "@comp.id")
-      return nullptr;
-    if (name == "@feat.00") {
+    if (name == "@feat.00")
       feat00Flags = sym.getValue();
+    // Skip special symbols.
+    if (ignoredSymbolName(name))
       return nullptr;
-    }
 
     if (sym.isExternal())
       return symtab->addAbsolute(name, sym);
@@ -782,8 +827,9 @@ void ImportFile::parse() {
 }
 
 BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
-                         uint64_t offsetInArchive)
-    : InputFile(BitcodeKind, mb) {
+                         uint64_t offsetInArchive,
+                         std::vector<Symbol *> &&symbols)
+    : InputFile(BitcodeKind, mb), symbols(std::move(symbols)) {
   std::string path = mb.getBufferIdentifier().str();
   if (config->thinLTOIndexOnly)
     path = replaceThinLTOSuffix(mb.getBufferIdentifier());

lld/COFF/InputFiles.h

@@ -14,6 +14,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/BinaryFormat/Magic.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/Archive.h"
@@ -55,7 +56,13 @@ class TpiSource;
 // The root class of input files.
 class InputFile {
 public:
-  enum Kind { ArchiveKind, ObjectKind, ImportKind, BitcodeKind };
+  enum Kind {
+    ArchiveKind,
+    ObjectKind,
+    LazyObjectKind,
+    ImportKind,
+    BitcodeKind
+  };
   Kind kind() const { return fileKind; }
   virtual ~InputFile() {}
 
@@ -102,10 +109,28 @@ class ArchiveFile : public InputFile {
   llvm::DenseSet<uint64_t> seen;
 };
 
+// .obj or .o file between -start-lib and -end-lib.
+class LazyObjFile : public InputFile {
+public:
+  explicit LazyObjFile(MemoryBufferRef m) : InputFile(LazyObjectKind, m) {}
+  static bool classof(const InputFile *f) {
+    return f->kind() == LazyObjectKind;
+  }
+  // Makes this object file part of the link.
+  void fetch();
+  // Adds the symbols in this file to the symbol table as LazyObject symbols.
+  void parse() override;
+
+private:
+  std::vector<Symbol *> symbols;
+};
+
 // .obj or .o file. This may be a member of an archive file.
 class ObjFile : public InputFile {
 public:
   explicit ObjFile(MemoryBufferRef m) : InputFile(ObjectKind, m) {}
+  explicit ObjFile(MemoryBufferRef m, std::vector<Symbol *> &&symbols)
+      : InputFile(ObjectKind, m), symbols(std::move(symbols)) {}
   static bool classof(const InputFile *f) { return f->kind() == ObjectKind; }
   void parse() override;
   MachineTypes getMachineType() override;
@@ -301,7 +326,11 @@ class ImportFile : public InputFile {
 class BitcodeFile : public InputFile {
 public:
   BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
-              uint64_t offsetInArchive);
+              uint64_t offsetInArchive)
+      : BitcodeFile(mb, archiveName, offsetInArchive, {}) {}
+  explicit BitcodeFile(MemoryBufferRef m, StringRef archiveName,
+                       uint64_t offsetInArchive,
+                       std::vector<Symbol *> &&symbols);
   static bool classof(const InputFile *f) { return f->kind() == BitcodeKind; }
   ArrayRef<Symbol *> getSymbols() { return symbols; }
   MachineTypes getMachineType() override;
@@ -314,6 +343,10 @@ class BitcodeFile : public InputFile {
   std::vector<Symbol *> symbols;
 };
 
+inline bool isBitcode(MemoryBufferRef mb) {
+  return identify_magic(mb.getBuffer()) == llvm::file_magic::bitcode;
+}
+
 std::string replaceThinLTOSuffix(StringRef path);
 } // namespace coff
 

lld/COFF/Options.td

@@ -162,6 +162,8 @@ def help : F<"help">;
 def help_q : Flag<["/??", "-??", "/?", "-?"], "">, Alias<help>;
 
 // LLD extensions
+def end_lib : F<"end-lib">,
+  HelpText<"End a grouping of objects that should be treated as if they were together in an archive">;
 def exclude_all_symbols : F<"exclude-all-symbols">;
 def export_all_symbols : F<"export-all-symbols">;
 defm demangle : B<"demangle",
@@ -176,6 +178,8 @@ def pdb_source_path : P<"pdbsourcepath",
                         "Base path used to make relative source file path absolute in PDB">;
 def rsp_quoting : Joined<["--"], "rsp-quoting=">,
   HelpText<"Quoting style for response files, 'windows' (default) or 'posix'">;
+def start_lib : F<"start-lib">,
+  HelpText<"Start a grouping of objects that should be treated as if they were together in an archive">;
 def thinlto_emit_imports_files :
     F<"thinlto-emit-imports-files">,
     HelpText<"Emit .imports files with -thinlto-index-only">;