Tweak lambda/tvar cleanup, to avoid mis-instantiating

Author: dwijnand

compiler/src/dotty/tools/dotc/core/ConstraintHandling.scala

@@ -436,17 +436,8 @@ trait ConstraintHandling {
 
     val level1 = nestingLevel(p1)
     val level2 = nestingLevel(p2)
-    val p1Wins = if level1 == level2 then
-      // If the nesting levels match, then we would prefer to unify to the outer most parameter.
-      // For instance in pos/i21981, while running `normalizedCompatible` against `.map2`,
-      // we want to unify to B over K, to allow easily removing K by just instantiating it.
-      def preferP1(ctx: Context): Boolean =
-        val c = ctx.typerState.constraint
-        !c.contains(p2) || c.contains(p1) && preferP1(ctx.outer)
-      preferP1(ctx)
-    else level1 <= level2
-    val pKept    = if p1Wins then p1 else p2
-    val pRemoved = if p1Wins then p2 else p1
+    val pKept    = if level1 <= level2 then p1 else p2
+    val pRemoved = if level1 <= level2 then p2 else p1
 
     val down = constraint.exclusiveLower(p2, p1)
     val up = constraint.exclusiveUpper(p1, p2)

compiler/src/dotty/tools/dotc/typer/ProtoTypes.scala

@@ -72,10 +72,20 @@ object ProtoTypes {
                    |constraint now: ${newctx.typerState.constraint}""")
             if result && (ctx.typerState.constraint ne newctx.typerState.constraint) then
               // Remove all type lambdas and tvars introduced by testCompat
-              for tvar <- newctx.typerState.ownedVars do
-                inContext(newctx):
-                  if !tvar.isInstantiated then
-                    tvar.instantiate(fromBelow = false) // any direction
+              inContext(newctx):
+                val tvars = ctx.typerState.ownedVars
+                val tvars1 = for tvar <- tvars.toList yield
+                  // Filter out any tvar that instantiating would further constrain the current constraint
+                  val aboveOK = !ctx.typerState.constraint.dependsOn(tvar, tvars, co = true)
+                  val belowOK = !ctx.typerState.constraint.dependsOn(tvar, tvars, co = false)
+                  val fromBelow: Boolean | Null =
+                    if aboveOK then true
+                    else if belowOK then false
+                    else null // don't instantiate
+                  (tvar, fromBelow)
+                for (tvar, fromBelow) <- tvars1 do
+                  if fromBelow != null && !tvar.isInstantiated then
+                    tvar.instantiate(fromBelow.uncheckedNN)
 
               // commit any remaining changes in typer state
               newctx.typerState.commit()

tests/pos/i21981.alt.scala

@@ -0,0 +1,28 @@
+trait Ops[F[_], A]:
+  def map0[B](f0: A => B): F[B] = ???
+
+trait Functor1[G[_]]
+
+trait Functor2[H[_]]:
+  extension [C1, C2](hc: H[(C1, C2)])
+    def map2[D](f1: (C1, C2) => D): H[D]
+
+trait Ref[I[_], +E]
+
+final class Cov[+F]
+
+class Test:
+  given [J[_]](using J: Functor1[J]): Functor2[J] with
+    extension [K1, K2](jk: J[(K1, K2)])
+      def map2[L](f2: (K1, K2) => L): J[L] = ???
+
+  def t1[
+    M[_[t]],
+    N[_],
+  ](using N: Functor1[N]): Unit =
+
+    val x3: Ops[N, M[[t] =>> Ref[N, t]]] = ???
+
+    val x2: N[(M[N], M[[t] =>> Ref[N, t]])] = x3
+      .map0 { refs             => (???, refs) }
+      .map2 { case (not, refs) => (???, refs) }