SILOptimizer: Fix to deal with casts involving dynamic Self

The cast optimizer was too eager to fold casts where the source and
target lowered types were the same, even though the formal types
might be different. Move the classifyFeasibility() check to deal
with this case.

Also in dead code elimination we have to mark all operands of a
cast branch instruction as live, not just the first operand,
otherwise we miss the special 'type dependent' self metadata
operand and replace it with 'undef'.

Author: slavapestov

lib/SILOptimizer/Transforms/DeadCodeElimination.cpp

@@ -400,22 +400,18 @@ void DCE::propagateLiveness(SILInstruction *I) {
   case TermKind::SwitchEnumInst:
   case TermKind::SwitchEnumAddrInst:
   case TermKind::DynamicMethodBranchInst:
-  case TermKind::CheckedCastBranchInst:
-  case TermKind::CheckedCastValueBranchInst:
     markValueLive(I->getOperand(0));
     return;
 
+  case TermKind::CheckedCastBranchInst:
+  case TermKind::CheckedCastValueBranchInst:
+  case TermKind::CheckedCastAddrBranchInst:
   case TermKind::TryApplyInst:
   case TermKind::SwitchValueInst:
   case TermKind::YieldInst:
     for (auto &O : I->getAllOperands())
       markValueLive(O.get());
     return;
-
-  case TermKind::CheckedCastAddrBranchInst:
-    markValueLive(I->getOperand(0));
-    markValueLive(I->getOperand(1));
-    return;
   }
   llvm_unreachable("corrupt instruction!");
 }

lib/SILOptimizer/Utils/CastOptimizer.cpp

@@ -997,6 +997,9 @@ CastOptimizer::simplifyCheckedCastBranchInst(CheckedCastBranchInst *Inst) {
   // Check if we can statically predict the outcome of the cast.
   auto Feasibility =
       dynamicCast.classifyFeasibility(false /*allow whole module*/);
+  if (Feasibility == DynamicCastFeasibility::MaySucceed) {
+    return nullptr;
+  }
 
   SILBuilderWithScope Builder(Inst, builderContext);
   if (Feasibility == DynamicCastFeasibility::WillFail) {
@@ -1009,6 +1012,8 @@ CastOptimizer::simplifyCheckedCastBranchInst(CheckedCastBranchInst *Inst) {
     return NewI;
   }
 
+  assert(Feasibility == DynamicCastFeasibility::WillSucceed);
+
   bool ResultNotUsed = SuccessBB->getArgument(0)->use_empty();
   SILValue CastedValue;
   if (Op->getType() != TargetLoweredType) {
@@ -1022,14 +1027,6 @@ CastOptimizer::simplifyCheckedCastBranchInst(CheckedCastBranchInst *Inst) {
       llvm_unreachable(
           "Bridged casts cannot be expressed by checked_cast_br yet");
     } else {
-      // If the cast may succeed or fail and can't be turned into a bridging
-      // call, then let it be.
-      if (Feasibility == DynamicCastFeasibility::MaySucceed) {
-        return nullptr;
-      }
-
-      assert(Feasibility == DynamicCastFeasibility::WillSucceed);
-
       // Replace by unconditional_cast, followed by a branch.
       // The unconditional_cast can be skipped, if the result of a cast
       // is not used afterwards.

test/SILOptimizer/dynamic_self_cast.sil

@@ -0,0 +1,46 @@
+// RUN: %target-sil-opt -enable-sil-verify-all -O %s | %FileCheck %s
+
+sil_stage raw
+
+import Swift
+
+public class SelfCasts {
+}
+
+// CHECK-LABEL: sil @$dynamic_self_cast_1 : $@convention(method) (@guaranteed SelfCasts, @thick SelfCasts.Type) -> @owned Optional<SelfCasts>
+sil @$dynamic_self_cast_1 : $@convention(method) (@guaranteed SelfCasts, @thick SelfCasts.Type) -> @owned Optional<SelfCasts> {
+bb0(%0 : $SelfCasts, %1 : $@thick SelfCasts.Type):
+  strong_retain %0 : $SelfCasts
+  // CHECK: checked_cast_br
+  checked_cast_br %0 : $SelfCasts to @dynamic_self SelfCasts, bb1, bb2
+bb1(%7 : $SelfCasts):
+  %8 = enum $Optional<SelfCasts>, #Optional.some!enumelt.1, %7 : $SelfCasts
+  br bb3(%8 : $Optional<SelfCasts>)
+
+bb2:
+  strong_release %0 : $SelfCasts
+  %11 = enum $Optional<SelfCasts>, #Optional.none!enumelt
+  br bb3(%11 : $Optional<SelfCasts>)
+
+bb3(%13 : $Optional<SelfCasts>):
+  return %13 : $Optional<SelfCasts>
+}
+
+// CHECK-LABEL: sil @$dynamic_self_cast_2 : $@convention(method) (@guaranteed SelfCasts, @thick SelfCasts.Type) -> @owned Optional<SelfCasts>
+sil @$dynamic_self_cast_2 : $@convention(method) (@guaranteed SelfCasts, @thick SelfCasts.Type) -> @owned Optional<SelfCasts> {
+bb0(%0 : $SelfCasts, %1 : $@thick SelfCasts.Type):
+  strong_retain %0 : $SelfCasts
+  // CHECK-NOT: checked_cast_br
+  checked_cast_br %0 : $SelfCasts to SelfCasts, bb1, bb2
+bb1(%7 : $SelfCasts):
+  %8 = enum $Optional<SelfCasts>, #Optional.some!enumelt.1, %7 : $SelfCasts
+  br bb3(%8 : $Optional<SelfCasts>)
+
+bb2:
+  strong_release %0 : $SelfCasts
+  %11 = enum $Optional<SelfCasts>, #Optional.none!enumelt
+  br bb3(%11 : $Optional<SelfCasts>)
+
+bb3(%13 : $Optional<SelfCasts>):
+  return %13 : $Optional<SelfCasts>
+}