Bump to green commit from Nov. 14

bolt/CMakeLists.txt

@@ -55,6 +55,7 @@ if (BOLT_ENABLE_RUNTIME)
                -DCMAKE_BUILD_TYPE=Release
                -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
                -DCMAKE_INSTALL_PREFIX=${LLVM_BINARY_DIR}
+               -DLLVM_LIBDIR_SUFFIX=${LLVM_LIBDIR_SUFFIX}
     BUILD_ALWAYS True
     )
   install(CODE "execute_process\(COMMAND \${CMAKE_COMMAND} -DCMAKE_INSTALL_PREFIX=\${CMAKE_INSTALL_PREFIX} -P ${CMAKE_CURRENT_BINARY_DIR}/bolt_rt-bins/cmake_install.cmake \)"
@@ -87,3 +88,6 @@ option(BOLT_INCLUDE_DOCS "Generate build targets for the BOLT docs."
 if (BOLT_INCLUDE_DOCS)
   add_subdirectory(docs)
 endif()
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/bolt/RuntimeLibs/RuntimeLibraryVariables.inc.in
+               ${CMAKE_CURRENT_BINARY_DIR}/include/bolt/RuntimeLibs/RuntimeLibraryVariables.inc @ONLY)

bolt/include/bolt/Passes/Hugify.h

@@ -0,0 +1,29 @@
+//===- bolt/Passes/Hugify.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_HUGIFY_H
+#define BOLT_PASSES_HUGIFY_H
+
+#include "bolt/Passes/BinaryPasses.h"
+
+namespace llvm {
+namespace bolt {
+
+class HugePage : public BinaryFunctionPass {
+public:
+  HugePage(const cl::opt<bool> &PrintPass) : BinaryFunctionPass(PrintPass) {}
+
+  void runOnFunctions(BinaryContext &BC) override;
+
+  const char *getName() const override { return "HugePage"; }
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/RuntimeLibs/HugifyRuntimeLibrary.h

@@ -22,13 +22,11 @@ class HugifyRuntimeLibrary : public RuntimeLibrary {
 public:
   /// Add custom section names generated by the runtime libraries to \p
   /// SecNames.
-  void addRuntimeLibSections(std::vector<std::string> &SecNames) const final {
-    SecNames.push_back(".bolt.hugify.entries");
-  }
+  void addRuntimeLibSections(std::vector<std::string> &SecNames) const final {}
 
   void adjustCommandLineOptions(const BinaryContext &BC) const final;
 
-  void emitBinary(BinaryContext &BC, MCStreamer &Streamer) final;
+  void emitBinary(BinaryContext &BC, MCStreamer &Streamer) final {}
 
   void link(BinaryContext &BC, StringRef ToolPath, RuntimeDyld &RTDyld,
             std::function<void(RuntimeDyld &)> OnLoad) final;

bolt/include/bolt/RuntimeLibs/RuntimeLibraryVariables.inc.in

@@ -0,0 +1,17 @@
+//===-- RuntimeLibraryVariables.inc.in - bolt build variables -*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is configured by the build system to define the runtime library
+// location.
+//
+// The variant of this file not ending with .in has been autogenerated by the
+// LLVM build. Do not edit!
+//
+//===----------------------------------------------------------------------===//
+
+#define LLVM_LIBDIR_SUFFIX "@LLVM_LIBDIR_SUFFIX@"

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -44,6 +44,7 @@ extern llvm::cl::opt<unsigned long long> HeatmapMinAddress;
 extern llvm::cl::opt<bool> HotData;
 extern llvm::cl::opt<bool> HotFunctionsAtEnd;
 extern llvm::cl::opt<bool> HotText;
+extern llvm::cl::opt<bool> Hugify;
 extern llvm::cl::opt<bool> Instrument;
 extern llvm::cl::opt<std::string> OutputFilename;
 extern llvm::cl::opt<std::string> PerfData;

bolt/lib/Core/BinaryEmitter.cpp

@@ -740,10 +740,12 @@ void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
 
   for (auto &JTI : BF.jumpTables()) {
     JumpTable &JT = *JTI.second;
+    // Only emit shared jump tables once, when processing the first parent
+    if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
+      continue;
     if (opts::PrintJumpTables)
       JT.print(outs());
-    if ((opts::JumpTables == JTS_BASIC || !BF.isSimple()) &&
-        BC.HasRelocations) {
+    if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
       JT.updateOriginal();
     } else {
       MCSection *HotSection, *ColdSection;

bolt/lib/Passes/CMakeLists.txt

@@ -15,6 +15,7 @@ add_llvm_library(LLVMBOLTPasses
   FrameOptimizer.cpp
   HFSort.cpp
   HFSortPlus.cpp
+  Hugify.cpp
   IdenticalCodeFolding.cpp
   IndirectCallPromotion.cpp
   Inliner.cpp

bolt/lib/Passes/Hugify.cpp

@@ -0,0 +1,50 @@
+//===--- bolt/Passes/Hugify.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Passes/Hugify.h"
+#include "llvm/Support/CommandLine.h"
+
+#define DEBUG_TYPE "bolt-hugify"
+
+using namespace llvm;
+
+namespace llvm {
+namespace bolt {
+
+void HugePage::runOnFunctions(BinaryContext &BC) {
+  auto *RtLibrary = BC.getRuntimeLibrary();
+  if (!RtLibrary || !BC.isELF() || !BC.StartFunctionAddress) {
+    return;
+  }
+
+  auto createSimpleFunction =
+      [&](std::string Title, std::vector<MCInst> Instrs) -> BinaryFunction * {
+    BinaryFunction *Func = BC.createInjectedBinaryFunction(Title);
+
+    std::vector<std::unique_ptr<BinaryBasicBlock>> BBs;
+    BBs.emplace_back(Func->createBasicBlock(nullptr));
+    BBs.back()->addInstructions(Instrs.begin(), Instrs.end());
+    BBs.back()->setCFIState(0);
+    BBs.back()->setOffset(BinaryBasicBlock::INVALID_OFFSET);
+
+    Func->insertBasicBlocks(nullptr, std::move(BBs),
+                            /*UpdateLayout=*/true,
+                            /*UpdateCFIState=*/false);
+    Func->updateState(BinaryFunction::State::CFG_Finalized);
+    return Func;
+  };
+
+  const BinaryFunction *const Start =
+      BC.getBinaryFunctionAtAddress(*BC.StartFunctionAddress);
+  assert(Start && "Entry point function not found");
+  const MCSymbol *StartSym = Start->getSymbol();
+  createSimpleFunction("__bolt_hugify_start_program",
+                       BC.MIB->createSymbolTrampoline(StartSym, BC.Ctx.get()));
+}
+} // namespace bolt
+} // namespace llvm

bolt/lib/Rewrite/BinaryPassManager.cpp

@@ -13,6 +13,7 @@
 #include "bolt/Passes/AsmDump.h"
 #include "bolt/Passes/CMOVConversion.h"
 #include "bolt/Passes/FrameOptimizer.h"
+#include "bolt/Passes/Hugify.h"
 #include "bolt/Passes/IdenticalCodeFolding.h"
 #include "bolt/Passes/IndirectCallPromotion.h"
 #include "bolt/Passes/Inliner.h"
@@ -333,6 +334,8 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
 
   if (opts::Instrument)
     Manager.registerPass(std::make_unique<Instrumentation>(NeverPrint));
+  else if (opts::Hugify)
+    Manager.registerPass(std::make_unique<HugePage>(NeverPrint));
 
   Manager.registerPass(std::make_unique<ShortenInstructions>(NeverPrint));
 

bolt/lib/Rewrite/RewriteInstance.cpp

@@ -479,6 +479,11 @@ Error RewriteInstance::discoverStorage() {
   NextAvailableAddress = alignTo(NextAvailableAddress, BC->PageAlign);
   NextAvailableOffset = alignTo(NextAvailableOffset, BC->PageAlign);
 
+  // Hugify: Additional huge page from left side due to
+  // weird ASLR mapping addresses (4KB aligned)
+  if (opts::Hugify && !BC->HasFixedLoadAddress)
+    NextAvailableAddress += BC->PageAlign;
+
   if (!opts::UseGnuStack) {
     // This is where the black magic happens. Creating PHDR table in a segment
     // other than that containing ELF header is tricky. Some loaders and/or
@@ -3719,6 +3724,12 @@ void RewriteInstance::mapCodeSections(RuntimeDyld &RTDyld) {
         Address = alignTo(Address, Section->getAlignment());
         Section->setOutputAddress(Address);
         Address += Section->getOutputSize();
+
+        // Hugify: Additional huge page from right side due to
+        // weird ASLR mapping addresses (4KB aligned)
+        if (opts::Hugify && !BC->HasFixedLoadAddress &&
+            Section->getName() == BC->getMainCodeSectionName())
+          Address = alignTo(Address, Section->getAlignment());
       }
 
       // Make sure we allocate enough space for huge pages.

bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp

@@ -60,35 +60,6 @@ void HugifyRuntimeLibrary::adjustCommandLineOptions(
   }
 }
 
-void HugifyRuntimeLibrary::emitBinary(BinaryContext &BC, MCStreamer &Streamer) {
-  const BinaryFunction *StartFunction =
-      BC.getBinaryFunctionAtAddress(*(BC.StartFunctionAddress));
-  assert(!StartFunction->isFragment() && "expected main function fragment");
-  if (!StartFunction) {
-    errs() << "BOLT-ERROR: failed to locate function at binary start address\n";
-    exit(1);
-  }
-
-  const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
-                                             /*IsText=*/false,
-                                             /*IsAllocatable=*/true);
-  MCSectionELF *Section =
-      BC.Ctx->getELFSection(".bolt.hugify.entries", ELF::SHT_PROGBITS, Flags);
-
-  // __bolt_hugify_init_ptr stores the poiter the hugify library needs to
-  // jump to after finishing the init code.
-  MCSymbol *InitPtr = BC.Ctx->getOrCreateSymbol("__bolt_hugify_init_ptr");
-
-  Section->setAlignment(llvm::Align(BC.RegularPageSize));
-  Streamer.switchSection(Section);
-
-  Streamer.emitLabel(InitPtr);
-  Streamer.emitSymbolAttribute(InitPtr, MCSymbolAttr::MCSA_Global);
-  Streamer.emitValue(
-      MCSymbolRefExpr::create(StartFunction->getSymbol(), *(BC.Ctx)),
-      /*Size=*/8);
-}
-
 void HugifyRuntimeLibrary::link(BinaryContext &BC, StringRef ToolPath,
                                 RuntimeDyld &RTDyld,
                                 std::function<void(RuntimeDyld &)> OnLoad) {

bolt/lib/RuntimeLibs/RuntimeLibrary.cpp

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "bolt/RuntimeLibs/RuntimeLibrary.h"
+#include "bolt/RuntimeLibs/RuntimeLibraryVariables.inc"
 #include "bolt/Utils/Utils.h"
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
@@ -28,12 +29,12 @@ std::string RuntimeLibrary::getLibPath(StringRef ToolPath,
                                        StringRef LibFileName) {
   StringRef Dir = llvm::sys::path::parent_path(ToolPath);
   SmallString<128> LibPath = llvm::sys::path::parent_path(Dir);
-  llvm::sys::path::append(LibPath, "lib");
+  llvm::sys::path::append(LibPath, "lib" LLVM_LIBDIR_SUFFIX);
   if (!llvm::sys::fs::exists(LibPath)) {
     // In some cases we install bolt binary into one level deeper in bin/,
     // we need to go back one more level to find lib directory.
     LibPath = llvm::sys::path::parent_path(llvm::sys::path::parent_path(Dir));
-    llvm::sys::path::append(LibPath, "lib");
+    llvm::sys::path::append(LibPath, "lib" LLVM_LIBDIR_SUFFIX);
   }
   llvm::sys::path::append(LibPath, LibFileName);
   if (!llvm::sys::fs::exists(LibPath)) {

bolt/runtime/CMakeLists.txt

@@ -15,35 +15,39 @@ add_library(bolt_rt_instr STATIC
   instr.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/config.h
   )
+set_target_properties(bolt_rt_instr PROPERTIES LIBRARY_OUTPUT_DIRECTORY "lib${LLVM_LIBDIR_SUFFIX}")
 add_library(bolt_rt_hugify STATIC
   hugify.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/config.h
   )
+set_target_properties(bolt_rt_hugify PROPERTIES LIBRARY_OUTPUT_DIRECTORY "lib${LLVM_LIBDIR_SUFFIX}")
 
 set(BOLT_RT_FLAGS
   -ffreestanding
   -fno-exceptions
   -fno-rtti
   -fno-stack-protector
-  -mno-sse)
+  -mno-sse
+  -fPIE)
 
 # Don't let the compiler think it can create calls to standard libs
-target_compile_options(bolt_rt_instr PRIVATE ${BOLT_RT_FLAGS} -fPIE)
+target_compile_options(bolt_rt_instr PRIVATE ${BOLT_RT_FLAGS})
 target_include_directories(bolt_rt_instr PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 target_compile_options(bolt_rt_hugify PRIVATE ${BOLT_RT_FLAGS})
 target_include_directories(bolt_rt_hugify PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
-install(TARGETS bolt_rt_instr DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-install(TARGETS bolt_rt_hugify DESTINATION "${CMAKE_INSTALL_LIBDIR}")
+install(TARGETS bolt_rt_instr DESTINATION "lib${LLVM_LIBDIR_SUFFIX}")
+install(TARGETS bolt_rt_hugify DESTINATION "lib${LLVM_LIBDIR_SUFFIX}")
 
 if (CMAKE_CXX_COMPILER_ID MATCHES ".*Clang.*")
   add_library(bolt_rt_instr_osx STATIC
     instr.cpp
     ${CMAKE_CURRENT_BINARY_DIR}/config.h
   )
+  set_target_properties(bolt_rt_instr_osx PROPERTIES LIBRARY_OUTPUT_DIRECTORY "lib${LLVM_LIBDIR_SUFFIX}")
   target_include_directories(bolt_rt_instr_osx PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
   target_compile_options(bolt_rt_instr_osx PRIVATE
     -target x86_64-apple-darwin19.6.0
     ${BOLT_RT_FLAGS})
-  install(TARGETS bolt_rt_instr_osx DESTINATION "${CMAKE_INSTALL_LIBDIR}")
+  install(TARGETS bolt_rt_instr_osx DESTINATION "lib${LLVM_LIBDIR_SUFFIX}")
 endif()