SIL function can be serialized with different kinds: [serialized] or

[serialized_for_package] if Package CMO is enabled. The latter kind
allows a function to be serialized even if it contains loadable types,
if Package CMO is enabled.

The tri-state serialization kind requires validating inlinability
depending on the serialization kinds of callee vs caller; e.g. if the
callee is [serialized_for_package], the caller must be _not_ [serialized].

Package CMO sets [serialized_for_package] to SIL function, global var,
v-table, and witness-table, if inlinable.

Renamed IsSerialized_t as SerializedKind_t and added IsSerializedForPackage.
Updated various call sites with getSerializedKind().
SILFunction::canBeSerializedIntoCaller handles inlinability checks b/t caller and callee.

Resolves rdar://128406520

Author: elsh

include/swift/SIL/GenericSpecializationMangler.h

@@ -37,7 +37,7 @@ class SpecializationMangler : public Mangle::ASTMangler {
   /// The specialization pass.
   SpecializationPass Pass;
 
-  IsSerialized_t Serialized;
+  swift::SerializedKind_t Serialized;
 
   /// The original function which is specialized.
   SILFunction *Function;
@@ -50,12 +50,12 @@ class SpecializationMangler : public Mangle::ASTMangler {
   PossibleEffects RemovedEffects;
 
 protected:
-  SpecializationMangler(SpecializationPass P, IsSerialized_t Serialized,
+  SpecializationMangler(SpecializationPass P, swift::SerializedKind_t Serialized,
                         SILFunction *F)
       : Pass(P), Serialized(Serialized), Function(F),
         ArgOpBuffer(ArgOpStorage) {}
 
-  SpecializationMangler(SpecializationPass P, IsSerialized_t Serialized,
+  SpecializationMangler(SpecializationPass P, swift::SerializedKind_t Serialized,
                         std::string functionName)
       : Pass(P), Serialized(Serialized), Function(nullptr),
         FunctionName(functionName), ArgOpBuffer(ArgOpStorage) {}
@@ -88,7 +88,7 @@ class GenericSpecializationMangler : public SpecializationMangler {
                                       SubstitutionMap subs);
 
 public:
-  GenericSpecializationMangler(SILFunction *F, IsSerialized_t Serialized)
+  GenericSpecializationMangler(SILFunction *F, swift::SerializedKind_t Serialized)
       : SpecializationMangler(SpecializationPass::GenericSpecializer,
                               Serialized, F) {}
 

include/swift/SIL/SILDeclRef.h

@@ -46,7 +46,7 @@ namespace swift {
   class EffectsAttr;
   class FileUnit;
   class SILFunctionType;
-  enum IsSerialized_t : unsigned char;
+  enum SerializedKind_t : uint8_t;
   enum class SubclassScope : unsigned char;
   class SILModule;
   class SILLocation;
@@ -384,7 +384,11 @@ struct SILDeclRef {
   /// True if the function should be treated as transparent.
   bool isTransparent() const;
   /// True if the function should have its body serialized.
-  IsSerialized_t isSerialized() const;
+  bool isSerialized() const;
+  /// True if this function is neither [serialized] or [serialized_for_package].
+  bool isNotSerialized() const;
+  /// Returns IsNotSerialized, IsSerializedForPackage, or IsSerialized.
+  SerializedKind_t getSerializedKind() const;
   /// True if the function has noinline attribute.
   bool isNoinline() const;
   /// True if the function has __always inline attribute.

include/swift/SIL/SILFunction.h

@@ -364,11 +364,7 @@ class SILFunction
   unsigned Transparent : 1;
 
   /// The function's serialized attribute.
-  bool Serialized : 1;
-
-  /// [serialized_for_package] attribute if package serialization
-  /// is enabled.
-  bool SerializedForPackage : 1;
+  unsigned SerializedKind : 2;
 
   /// Specifies if this function is a thunk or a reabstraction thunk.
   ///
@@ -508,7 +504,7 @@ class SILFunction
   SILFunction(SILModule &module, SILLinkage linkage, StringRef mangledName,
               CanSILFunctionType loweredType, GenericEnvironment *genericEnv,
               IsBare_t isBareSILFunction, IsTransparent_t isTrans,
-              IsSerialized_t isSerialized, ProfileCounter entryCount,
+              SerializedKind_t serializedKind, ProfileCounter entryCount,
               IsThunk_t isThunk, SubclassScope classSubclassScope,
               Inline_t inlineStrategy, EffectsKind E,
               const SILDebugScope *debugScope,
@@ -521,7 +517,7 @@ class SILFunction
   create(SILModule &M, SILLinkage linkage, StringRef name,
          CanSILFunctionType loweredType, GenericEnvironment *genericEnv,
          std::optional<SILLocation> loc, IsBare_t isBareSILFunction,
-         IsTransparent_t isTrans, IsSerialized_t isSerialized,
+         IsTransparent_t isTrans, SerializedKind_t serializedKind,
          ProfileCounter entryCount, IsDynamicallyReplaceable_t isDynamic,
          IsDistributed_t isDistributed,
          IsRuntimeAccessible_t isRuntimeAccessible,
@@ -534,7 +530,7 @@ class SILFunction
 
   void init(SILLinkage Linkage, StringRef Name, CanSILFunctionType LoweredType,
             GenericEnvironment *genericEnv, IsBare_t isBareSILFunction,
-            IsTransparent_t isTrans, IsSerialized_t isSerialized,
+            IsTransparent_t isTrans, SerializedKind_t serializedKind,
             ProfileCounter entryCount, IsThunk_t isThunk,
             SubclassScope classSubclassScope, Inline_t inlineStrategy,
             EffectsKind E, const SILDebugScope *DebugScope,
@@ -879,11 +875,20 @@ class SILFunction
 
   /// Returns true if this function can be inlined into a fragile function
   /// body.
-  bool hasValidLinkageForFragileInline() const { return isSerialized(); }
+  /// If the caller is [serialized] due to explicit `@inlinable` but
+  /// the callee was [serialized_for_package] due to package-cmo, the
+  /// callee is [serialized] and might contain loadable types, which
+  /// is normally disallowed. This check prevents such loadable types
+  /// from being inlined.
+  bool canBeSerializedIntoCaller(
+      std::optional<SerializedKind_t> callerSerializedKind =
+                                 std::nullopt) const;
 
   /// Returns true if this function can be referenced from a fragile function
   /// body.
-  bool hasValidLinkageForFragileRef() const;
+  bool hasValidLinkageForFragileRef(
+      std::optional<SerializedKind_t> callerSerializedKind =
+          std::nullopt) const;
 
   /// Get's the effective linkage which is used to derive the llvm linkage.
   /// Usually this is the same as getLinkage(), except in one case: if this
@@ -1137,28 +1142,24 @@ class SILFunction
     assert(!Transparent || !IsDynamicReplaceable);
   }
 
-  /// Get this function's serialized attribute.
-  IsSerialized_t isSerialized() const { return IsSerialized_t(Serialized); }
-  void setSerialized(IsSerialized_t isSerialized) {
-    Serialized = isSerialized;
-    assert(this->isSerialized() == isSerialized &&
-           "too few bits for Serialized storage");
+  bool isSerialized() const {
+    return SerializedKind_t(SerializedKind) == IsSerialized;
+  }
+  bool isSerializedForPackage() const {
+    return SerializedKind_t(SerializedKind) == IsSerializedForPackage;
+  }
+  bool isNotSerialized() const {
+    return SerializedKind_t(SerializedKind) == IsNotSerialized;
   }
 
-  /// A [serialized_for_package] attribute is used to indicate that a function
-  /// is [serialized] because of package-cmo optimization.
-  /// Package-cmo allows serializing a function containing a loadable type in
-  /// a resiliently built module, which is normally illegal. During SIL deserialization,
-  /// this attribute can be used to check whether a loaded function that was serialized
-  /// can be allowed to have loadable types. This attribute is also used to determine
-  /// if a callee can be inlined into a caller that's serialized without package-cmo, for
-  /// example, by explicitly annotating the caller decl with `@inlinable`.
-  IsSerializedForPackage_t isSerializedForPackage() const {
-    return IsSerializedForPackage_t(SerializedForPackage);
+  /// Get this function's serialized attribute.
+  SerializedKind_t getSerializedKind() const {
+    return SerializedKind_t(SerializedKind);
   }
-  void
-  setSerializedForPackage(IsSerializedForPackage_t isSerializedForPackage) {
-    SerializedForPackage = isSerializedForPackage;
+  void setSerializedKind(SerializedKind_t serializedKind) {
+    SerializedKind = serializedKind;
+    assert(this->getSerializedKind() == serializedKind &&
+           "too few bits for Serialized storage");
   }
 
   /// Get this function's thunk attribute.

include/swift/SIL/SILFunctionBuilder.h

@@ -60,24 +60,19 @@ class SILFunctionBuilder {
 
   /// Return the declaration of a utility function that can, but needn't, be
   /// shared between different parts of a program.
-  SILFunction *getOrCreateSharedFunction(SILLocation loc, StringRef name,
-                                         CanSILFunctionType type,
-                                         IsBare_t isBareSILFunction,
-                                         IsTransparent_t isTransparent,
-                                         IsSerialized_t isSerialized,
-                                         ProfileCounter entryCount,
-                                         IsThunk_t isThunk,
-                                         IsDynamicallyReplaceable_t isDynamic,
-                                         IsDistributed_t isDistributed,
-                                         IsRuntimeAccessible_t isRuntimeAccessible);
+  SILFunction *getOrCreateSharedFunction(
+      SILLocation loc, StringRef name, CanSILFunctionType type,
+      IsBare_t isBareSILFunction, IsTransparent_t isTransparent,
+      SerializedKind_t serializedKind, ProfileCounter entryCount,
+      IsThunk_t isThunk, IsDynamicallyReplaceable_t isDynamic,
+      IsDistributed_t isDistributed, IsRuntimeAccessible_t isRuntimeAccessible);
 
   /// Return the declaration of a function, or create it if it doesn't exist.
   SILFunction *getOrCreateFunction(
       SILLocation loc, StringRef name, SILLinkage linkage,
       CanSILFunctionType type, IsBare_t isBareSILFunction,
-      IsTransparent_t isTransparent, IsSerialized_t isSerialized,
-      IsDynamicallyReplaceable_t isDynamic,
-      IsDistributed_t isDistributed,
+      IsTransparent_t isTransparent, SerializedKind_t serializedKind,
+      IsDynamicallyReplaceable_t isDynamic, IsDistributed_t isDistributed,
       IsRuntimeAccessible_t isRuntimeAccessible,
       ProfileCounter entryCount = ProfileCounter(),
       IsThunk_t isThunk = IsNotThunk,
@@ -104,7 +99,7 @@ class SILFunctionBuilder {
       SILLinkage linkage, StringRef name, CanSILFunctionType loweredType,
       GenericEnvironment *genericEnv, std::optional<SILLocation> loc,
       IsBare_t isBareSILFunction, IsTransparent_t isTrans,
-      IsSerialized_t isSerialized, IsDynamicallyReplaceable_t isDynamic,
+      SerializedKind_t serializedKind, IsDynamicallyReplaceable_t isDynamic,
       IsDistributed_t isDistributed, IsRuntimeAccessible_t isRuntimeAccessible,
       ProfileCounter entryCount = ProfileCounter(),
       IsThunk_t isThunk = IsNotThunk,

include/swift/SIL/SILGlobalVariable.h

@@ -69,7 +69,7 @@ class SILGlobalVariable
   /// The global variable's serialized attribute.
   /// Serialized means that the variable can be "inlined" into another module.
   /// Currently this flag is set for all global variables in the stdlib.
-  unsigned Serialized : 1;
+  unsigned Serialized : 2;
 
   /// Whether this is a 'let' property, which can only be initialized
   /// once (either in its declaration, or once later), making it immutable.
@@ -97,7 +97,7 @@ class SILGlobalVariable
   SILBasicBlock StaticInitializerBlock;
 
   SILGlobalVariable(SILModule &M, SILLinkage linkage,
-                    IsSerialized_t IsSerialized, StringRef mangledName,
+                    SerializedKind_t serializedKind, StringRef mangledName,
                     SILType loweredType, std::optional<SILLocation> loc,
                     VarDecl *decl);
 
@@ -107,7 +107,7 @@ class SILGlobalVariable
   }
 
   static SILGlobalVariable *
-  create(SILModule &Module, SILLinkage Linkage, IsSerialized_t IsSerialized,
+  create(SILModule &Module, SILLinkage Linkage, SerializedKind_t serializedKind,
          StringRef MangledName, SILType LoweredType,
          std::optional<SILLocation> Loc = std::nullopt,
          VarDecl *Decl = nullptr);
@@ -151,10 +151,18 @@ class SILGlobalVariable
   /// potentially be inspected by the debugger.
   bool shouldBePreservedForDebugger() const;
 
+  /// Check if this global variable is [serialized]. This does not check
+  /// if it's [serialized_for_package].
+  bool isSerialized() const;
+  /// Check if this global variable is [serialized_for_package].
+  bool isSerializedForPackage() const;
+  /// Checks whether this global var is neither [serialized] nor
+  /// [serialized_for_package].
+  bool isNotSerialized() const;
   /// Get this global variable's serialized attribute.
-  IsSerialized_t isSerialized() const;
-  void setSerialized(IsSerialized_t isSerialized);
-  
+  SerializedKind_t getSerializedKind() const;
+  void setSerializedKind(SerializedKind_t isSerialized);
+
   /// Is this an immutable 'let' property?
   bool isLet() const { return IsLet; }
   void setLet(bool isLet) { IsLet = isLet; }

include/swift/SIL/SILLinkage.h

@@ -141,7 +141,7 @@ enum {
 /// After the swiftmodule file is written, the IsSerialized flag is cleared from
 /// all functions. This means that optimizations after the serialization point
 /// are not limited anymore regarding serialized functions.
-enum IsSerialized_t : unsigned char {
+enum SerializedKind_t : uint8_t {
 
   /// The function is not inlinable and will not be serialized.
   IsNotSerialized,
@@ -151,19 +151,29 @@ enum IsSerialized_t : unsigned char {
   /// This flag is only valid for Public, PublicNonABI, PublicExternal,
   /// HiddenExternal and Shared functions.
   /// Functions with external linkage (PublicExternal, HiddenExternal) will not
-  /// be serialized, because they are available in a different module (from which
-  /// they were de-serialized).
+  /// be serialized, because they are available in a different module (from
+  /// which they were de-serialized).
   ///
-  /// Functions with Shared linkage will only be serialized if they are referenced
-  /// from another serialized function (or table).
+  /// Functions with Shared linkage will only be serialized if they are
+  /// referenced from another serialized function (or table).
   ///
   /// This flag is removed from all functions after the serialization point in
   /// the optimizer pipeline.
-  IsSerialized
-};
+  IsSerialized,
 
-enum IsSerializedForPackage_t : unsigned char {
-  IsNotSerializedForPackage,
+  /// This flag is valid for all linkages applicable to IsSerialized as well as
+  /// Package, PackageNonABI, and PackageExternal, if package-wide
+  /// serialization is enabled with Package-CMO optimization.
+  ///
+  /// The [serialized_for_package] attribute is used to indicate that a function
+  /// is serialized because of Package CMO, which allows loadable types in a
+  /// serialized function in a resiliently built module, which is otherwise illegal.
+  /// It's also used to determine during SIL deserialization whether loadable
+  /// types in a serialized function can be allowed in the client module that
+  /// imports the module built with Package CMO. If the client contains a [serialized]
+  /// function due to `@inlinable`, funtions with [serialized_for_package] from
+  /// the imported module are not allowed being inlined into the client function,
+  /// which is the correct behavior.
   IsSerializedForPackage
 };
 

include/swift/SIL/SILMoveOnlyDeinit.h

@@ -29,7 +29,7 @@
 
 namespace swift {
 
-enum IsSerialized_t : unsigned char;
+enum SerializedKind_t : uint8_t;
 class SILFunction;
 class SILModule;
 
@@ -45,18 +45,18 @@ class SILMoveOnlyDeinit final : public SILAllocated<SILMoveOnlyDeinit> {
   /// Whether or not this deinit table is serialized. If a deinit is not
   /// serialized, then other modules can not consume directly a move only type
   /// since the deinit can not be called directly.
-  bool serialized : 1;
+  unsigned serialized : 2;
 
   SILMoveOnlyDeinit()
-      : nominalDecl(nullptr), funcImpl(nullptr), serialized(false) {}
+      : nominalDecl(nullptr), funcImpl(nullptr), serialized(unsigned(IsNotSerialized)) {}
 
   SILMoveOnlyDeinit(NominalTypeDecl *nominaldecl, SILFunction *implementation,
-                    bool serialized);
+                    unsigned serialized);
   ~SILMoveOnlyDeinit();
 
 public:
   static SILMoveOnlyDeinit *create(SILModule &mod, NominalTypeDecl *nominalDecl,
-                                   IsSerialized_t serialized,
+                                   SerializedKind_t serialized,
                                    SILFunction *funcImpl);
 
   NominalTypeDecl *getNominalDecl() const { return nominalDecl; }
@@ -66,11 +66,14 @@ class SILMoveOnlyDeinit final : public SILAllocated<SILMoveOnlyDeinit> {
     return funcImpl;
   }
 
-  IsSerialized_t isSerialized() const {
-    return serialized ? IsSerialized : IsNotSerialized;
+  bool isNotSerialized() const {
+    return SerializedKind_t(serialized) == IsNotSerialized;
   }
-  void setSerialized(IsSerialized_t inputSerialized) {
-    serialized = inputSerialized ? 1 : 0;
+  SerializedKind_t getSerializedKind() const {
+    return SerializedKind_t(serialized);
+  }
+  void setSerializedKind(SerializedKind_t inputSerialized) {
+    serialized = unsigned(inputSerialized);
   }
 
   void print(llvm::raw_ostream &os, bool verbose) const;

include/swift/SIL/SILProperty.h

@@ -36,25 +36,28 @@ class SILProperty : public llvm::ilist_node<SILProperty>,
 {
 private:
   /// True if serialized.
-  bool Serialized;
+  unsigned Serialized : 2;
 
   /// The declaration the descriptor represents.
   AbstractStorageDecl *Decl;
 
   /// The key path component that represents its implementation.
   std::optional<KeyPathPatternComponent> Component;
 
-  SILProperty(bool Serialized, AbstractStorageDecl *Decl,
+  SILProperty(unsigned Serialized, AbstractStorageDecl *Decl,
               std::optional<KeyPathPatternComponent> Component)
       : Serialized(Serialized), Decl(Decl), Component(Component) {}
 
 public:
-  static SILProperty *create(SILModule &M, bool Serialized,
+  static SILProperty *create(SILModule &M, unsigned Serialized,
                              AbstractStorageDecl *Decl,
                              std::optional<KeyPathPatternComponent> Component);
 
-  bool isSerialized() const { return Serialized; }
-  
+  bool isNotSerialized() const { return SerializedKind_t(Serialized) == IsNotSerialized; }
+  SerializedKind_t getSerializedKind() const {
+    return SerializedKind_t(Serialized);
+  }
+
   AbstractStorageDecl *getDecl() const { return Decl; }
 
   bool isTrivial() const {