[BOLT] Match blocks with pseudo probes (#99891)

Match inline trees first between profile and the binary: by GUID,
checksum, parent, and inline site for inlined functions. Map profile
probes to binary probes via matched inline tree nodes. Each binary probe
has an associated binary basic block. If all probes from one profile
basic block map to the same binary basic block, it’s an exact match,
otherwise the block is determined by majority vote and reported as loose
match.

Pseudo probe matching happens between exact hash matching and call/loose
matching.

Introduce ProbeMatchSpec - a mechanism to match probes belonging to
another binary function. For example, given functions foo and bar:
```
void foo() {
  bar();
}
```
profiled binary: bar is not inlined => have top-level function bar
new binary where the profile is applied to: bar is inlined into foo.

Currently, BOLT does 1:1 matching between profile functions and binary
functions based on the name. #100446 will extend this to N:M where
multiple profiles can be matched to one binary function (as in the
example above where binary function foo would use profiles for foo and
bar), and one profile can be matched to multiple binary functions (e.g.
if bar was inlined into multiple functions).

In this diff, ProbeMatchSpecs would only have one BinaryFunctionProfile
(existing name-based matching). 

Test Plan: Added match-blocks-with-pseudo-probes.test

Performance test:
- Setup:
  - Baseline no-BOLT: Clang with pseudo probes, ThinLTO + CSSPGO
  (#79942)
  - BOLT fresh: BOLTed Clang using fresh profile,
  - BOLT stale (hash): BOLTed Clang using stale profile (collected on
    Clang 10K commits back), `-infer-stale-profile` (hash+call block
    matching)
  - BOLT stale (+probe): BOLTed Clang using stale profile,
    `-infer-stale-profile` with `-stale-matching-with-pseudo-probes`
    (hash+call+pseudo probe block matching)
  - 2S Intel SKX Xeon 6138 with 40C/80T and 256GB RAM, using 20C/40T for
    build,
  - BOLT profiles are collected on Clang compiling large preprocessed
    C++ file.
- Benchmark: building Clang (average of 5 runs), see driver in
  aaupov/llvm-devmtg-2022
- Results, wall time, lower is better:
  - Baseline no-BOLT: 429.52 +- 2.61s,
  - BOLT stale (hash): 413.21 +- 2.19s,
  - BOLT stale (+probe): 409.69 +- 1.41s,
  - BOLT fresh: 384.50 +- 1.80s.

---------

Co-authored-by: Amir Ayupov <[email protected]>

Author: shawbyoung

bolt/include/bolt/Core/BinaryContext.h

@@ -723,12 +723,28 @@ class BinaryContext {
     /// Stats for stale profile matching:
     ///   the total number of basic blocks in the profile
     uint32_t NumStaleBlocks{0};
-    ///   the number of matched basic blocks
-    uint32_t NumMatchedBlocks{0};
+    ///   the number of exactly matched basic blocks
+    uint32_t NumExactMatchedBlocks{0};
+    ///   the number of loosely matched basic blocks
+    uint32_t NumLooseMatchedBlocks{0};
+    ///   the number of exactly pseudo probe matched basic blocks
+    uint32_t NumPseudoProbeExactMatchedBlocks{0};
+    ///   the number of loosely pseudo probe matched basic blocks
+    uint32_t NumPseudoProbeLooseMatchedBlocks{0};
+    ///   the number of call matched basic blocks
+    uint32_t NumCallMatchedBlocks{0};
     ///   the total count of samples in the profile
     uint64_t StaleSampleCount{0};
-    ///   the count of matched samples
-    uint64_t MatchedSampleCount{0};
+    ///   the count of exactly matched samples
+    uint64_t ExactMatchedSampleCount{0};
+    ///   the count of loosely matched samples
+    uint64_t LooseMatchedSampleCount{0};
+    ///   the count of exactly pseudo probe matched samples
+    uint64_t PseudoProbeExactMatchedSampleCount{0};
+    ///   the count of loosely pseudo probe matched samples
+    uint64_t PseudoProbeLooseMatchedSampleCount{0};
+    ///   the count of call matched samples
+    uint64_t CallMatchedSampleCount{0};
     ///   the number of stale functions that have matching number of blocks in
     ///   the profile
     uint64_t NumStaleFuncsWithEqualBlockCount{0};

bolt/include/bolt/Profile/ProfileYAMLMapping.h

@@ -174,6 +174,9 @@ struct InlineTreeNode {
   uint32_t CallSiteProbe;
   // Index in PseudoProbeDesc.GUID, UINT32_MAX for same as previous (omitted)
   uint32_t GUIDIndex;
+  // Decoded contents, ParentIndexDelta becomes absolute value.
+  uint64_t GUID;
+  uint64_t Hash;
   bool operator==(const InlineTreeNode &) const { return false; }
 };
 } // end namespace bolt

bolt/include/bolt/Profile/YAMLProfileReader.h

@@ -14,6 +14,8 @@
 #include <unordered_set>
 
 namespace llvm {
+class MCDecodedPseudoProbeInlineTree;
+
 namespace bolt {
 
 class YAMLProfileReader : public ProfileReaderBase {
@@ -43,6 +45,9 @@ class YAMLProfileReader : public ProfileReaderBase {
   using ProfileLookupMap =
       DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *>;
 
+  using GUIDInlineTreeMap =
+      std::unordered_map<uint64_t, const MCDecodedPseudoProbeInlineTree *>;
+
   /// A class for matching binary functions in functions in the YAML profile.
   /// First, a call graph is constructed for both profiled and binary functions.
   /// Then functions are hashed based on the names of their callee/caller
@@ -96,6 +101,61 @@ class YAMLProfileReader : public ProfileReaderBase {
         YamlBFAdjacencyMap;
   };
 
+  // A class for matching inline tree nodes between profile and binary.
+  // Provides the mapping from profile inline tree node id to a
+  // corresponding binary MCDecodedPseudoProbeInlineTree node.
+  //
+  // The whole mapping process is the following:
+  //
+  //     (profile)                             (binary)
+  //      | blocks                                ^
+  //      v                                       |
+  // yaml::bolt::BinaryBasicBlockProfile ~= FlowBlock
+  //     ||| probes                               ^  (majority vote)
+  //      v                                      ||| BBPseudoProbeToBlock
+  // yaml::bolt::PseudoProbeInfo            MCDecodedPseudoProbe
+  //      | InlineTreeIndex                       ^
+  //      v                                       | probe id
+  // [ profile node id (uint32_t)    ->     MCDecodedPseudoProbeInlineTree *]
+  //                     InlineTreeNodeMapTy
+  class InlineTreeNodeMapTy {
+    DenseMap<uint32_t, const MCDecodedPseudoProbeInlineTree *> Map;
+
+    void mapInlineTreeNode(uint32_t ProfileNodeIdx,
+                           const MCDecodedPseudoProbeInlineTree *BinaryNode) {
+      auto Res = Map.try_emplace(ProfileNodeIdx, BinaryNode);
+      assert(Res.second &&
+             "Duplicate mapping from profile node index to binary inline tree");
+      (void)Res;
+    }
+
+  public:
+    /// Returns matched InlineTree * for a given profile inline_tree_id.
+    const MCDecodedPseudoProbeInlineTree *
+    getInlineTreeNode(uint32_t ProfileInlineTreeNodeId) const {
+      auto It = Map.find(ProfileInlineTreeNodeId);
+      if (It == Map.end())
+        return nullptr;
+      return It->second;
+    }
+
+    // Match up \p YamlInlineTree with binary inline tree rooted at \p Root.
+    // Return the number of matched nodes.
+    //
+    // This function populates the mapping from profile inline tree node id to a
+    // corresponding binary MCDecodedPseudoProbeInlineTree node.
+    size_t matchInlineTrees(
+        const MCPseudoProbeDecoder &Decoder,
+        const std::vector<yaml::bolt::InlineTreeNode> &YamlInlineTree,
+        const MCDecodedPseudoProbeInlineTree *Root);
+  };
+
+  // Partial probe matching specification: matched inline tree and corresponding
+  // BinaryFunctionProfile
+  using ProbeMatchSpec =
+      std::pair<InlineTreeNodeMapTy,
+                std::reference_wrapper<yaml::bolt::BinaryFunctionProfile>>;
+
 private:
   /// Adjustments for basic samples profiles (without LBR).
   bool NormalizeByInsnCount{false};
@@ -129,6 +189,13 @@ class YAMLProfileReader : public ProfileReaderBase {
   /// BinaryFunction pointers indexed by YamlBP functions.
   std::vector<BinaryFunction *> ProfileBFs;
 
+  // Pseudo probe function GUID to inline tree node
+  GUIDInlineTreeMap TopLevelGUIDToInlineTree;
+
+  // Mapping from a binary function to its partial match specification
+  // (YAML profile and its inline tree mapping to binary).
+  DenseMap<BinaryFunction *, std::vector<ProbeMatchSpec>> BFToProbeMatchSpecs;
+
   /// Populate \p Function profile with the one supplied in YAML format.
   bool parseFunctionProfile(BinaryFunction &Function,
                             const yaml::bolt::BinaryFunctionProfile &YamlBF);
@@ -139,7 +206,8 @@ class YAMLProfileReader : public ProfileReaderBase {
 
   /// Infer function profile from stale data (collected on older binaries).
   bool inferStaleProfile(BinaryFunction &Function,
-                         const yaml::bolt::BinaryFunctionProfile &YamlBF);
+                         const yaml::bolt::BinaryFunctionProfile &YamlBF,
+                         const ArrayRef<ProbeMatchSpec> ProbeMatchSpecs);
 
   /// Initialize maps for profile matching.
   void buildNameMaps(BinaryContext &BC);
@@ -156,6 +224,10 @@ class YAMLProfileReader : public ProfileReaderBase {
   /// Matches functions using the call graph.
   size_t matchWithCallGraph(BinaryContext &BC);
 
+  /// Matches functions using the call graph.
+  /// Populates BF->partial probe match spec map.
+  size_t matchWithPseudoProbes(BinaryContext &BC);
+
   /// Matches functions with similarly named profiled functions.
   size_t matchWithNameSimilarity(BinaryContext &BC);
 

bolt/lib/Passes/BinaryPasses.cpp

@@ -1549,10 +1549,48 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
         "BOLT-INFO: inference found an exact match for %.2f%% of basic blocks"
         " (%zu out of %zu stale) responsible for %.2f%% samples"
         " (%zu out of %zu stale)\n",
-        100.0 * BC.Stats.NumMatchedBlocks / BC.Stats.NumStaleBlocks,
-        BC.Stats.NumMatchedBlocks, BC.Stats.NumStaleBlocks,
-        100.0 * BC.Stats.MatchedSampleCount / BC.Stats.StaleSampleCount,
-        BC.Stats.MatchedSampleCount, BC.Stats.StaleSampleCount);
+        100.0 * BC.Stats.NumExactMatchedBlocks / BC.Stats.NumStaleBlocks,
+        BC.Stats.NumExactMatchedBlocks, BC.Stats.NumStaleBlocks,
+        100.0 * BC.Stats.ExactMatchedSampleCount / BC.Stats.StaleSampleCount,
+        BC.Stats.ExactMatchedSampleCount, BC.Stats.StaleSampleCount);
+    BC.outs() << format(
+        "BOLT-INFO: inference found an exact pseudo probe match for %.2f%% of "
+        "basic blocks (%zu out of %zu stale) responsible for %.2f%% samples"
+        " (%zu out of %zu stale)\n",
+        100.0 * BC.Stats.NumPseudoProbeExactMatchedBlocks /
+            BC.Stats.NumStaleBlocks,
+        BC.Stats.NumPseudoProbeExactMatchedBlocks, BC.Stats.NumStaleBlocks,
+        100.0 * BC.Stats.PseudoProbeExactMatchedSampleCount /
+            BC.Stats.StaleSampleCount,
+        BC.Stats.PseudoProbeExactMatchedSampleCount, BC.Stats.StaleSampleCount);
+    BC.outs() << format(
+        "BOLT-INFO: inference found a loose pseudo probe match for %.2f%% of "
+        "basic blocks (%zu out of %zu stale) responsible for %.2f%% samples"
+        " (%zu out of %zu stale)\n",
+        100.0 * BC.Stats.NumPseudoProbeLooseMatchedBlocks /
+            BC.Stats.NumStaleBlocks,
+        BC.Stats.NumPseudoProbeLooseMatchedBlocks, BC.Stats.NumStaleBlocks,
+        100.0 * BC.Stats.PseudoProbeLooseMatchedSampleCount /
+            BC.Stats.StaleSampleCount,
+        BC.Stats.PseudoProbeLooseMatchedSampleCount, BC.Stats.StaleSampleCount);
+    BC.outs() << format(
+        "BOLT-INFO: inference found a call match for %.2f%% of basic "
+        "blocks"
+        " (%zu out of %zu stale) responsible for %.2f%% samples"
+        " (%zu out of %zu stale)\n",
+        100.0 * BC.Stats.NumCallMatchedBlocks / BC.Stats.NumStaleBlocks,
+        BC.Stats.NumCallMatchedBlocks, BC.Stats.NumStaleBlocks,
+        100.0 * BC.Stats.CallMatchedSampleCount / BC.Stats.StaleSampleCount,
+        BC.Stats.CallMatchedSampleCount, BC.Stats.StaleSampleCount);
+    BC.outs() << format(
+        "BOLT-INFO: inference found a loose match for %.2f%% of basic "
+        "blocks"
+        " (%zu out of %zu stale) responsible for %.2f%% samples"
+        " (%zu out of %zu stale)\n",
+        100.0 * BC.Stats.NumLooseMatchedBlocks / BC.Stats.NumStaleBlocks,
+        BC.Stats.NumLooseMatchedBlocks, BC.Stats.NumStaleBlocks,
+        100.0 * BC.Stats.LooseMatchedSampleCount / BC.Stats.StaleSampleCount,
+        BC.Stats.LooseMatchedSampleCount, BC.Stats.StaleSampleCount);
   }
 
   if (const uint64_t NumUnusedObjects = BC.getNumUnusedProfiledObjects()) {