[indylambda] Support lambda {de}serialization

To support serialization, we use the alternative lambda metafactory
that lets us specify that our anonymous functions should extend the
marker interface `scala.Serializable`. They will also have a
`writeObject` method added that implements the serialization proxy
pattern using `j.l.invoke.SerializedLamba`.

To support deserialization, we synthesize a `$deserializeLamba$`
method in each class with lambdas. This will be called reflectively by
`SerializedLambda#readResolve`. This method in turn delegates to
`LambdaDeserializer`, currently defined [1] in `scala-java8-compat`,
that uses `LambdaMetafactory` to spin up the anonymous class and
instantiate it with the deserialized environment.

Note: `LambdaDeserializer` can reuses the anonymous class on subsequent
deserializations of a given lambda, in the same spirit as an
invokedynamic call site only spins up the class on the first time
it is run. But first we'll need to host a cache in a static field
of each lambda hosting class. This is noted as a TODO and a failing
test, and will be updated in the next commit.

`LambdaDeserializer` will be moved into our standard library in
the 2.12.x branch, where we can introduce dependencies on the
Java 8 standard library.

The enclosed test cases must be manually run with indylambda enabled.
Once we enable indylambda by default on 2.12.x, the test will
actually test the new feature.

```
% echo $INDYLAMBDA
-Ydelambdafy:method -Ybackend:GenBCode -target:jvm-1.8 -classpath .:scala-java8-compat_2.11-0.5.0-SNAPSHOT.jar
% qscala $INDYLAMBDA -e "println((() => 42).getClass)"
class Main$$anon$1$$Lambda$1/1183231938
% qscala $INDYLAMBDA -e "assert(classOf[scala.Serializable].isInstance(() => 42))"
% qscalac $INDYLAMBDA test/files/run/lambda-serialization.scala && qscala $INDYLAMBDA Test
```

This commit contains a few minor refactorings to the code that
generates the invokedynamic instruction to use more meaningful
names and to reuse Java signature generation code in ASM rather
than the DIY approach.

[1] scala/scala-java8-compat#37

Author: retronym

src/compiler/scala/tools/nsc/backend/jvm/BCodeBodyBuilder.scala

@@ -33,7 +33,6 @@ abstract class BCodeBodyBuilder extends BCodeSkelBuilder {
    * Functionality to build the body of ASM MethodNode, except for `synchronized` and `try` expressions.
    */
   abstract class PlainBodyBuilder(cunit: CompilationUnit) extends PlainSkelBuilder(cunit) {
-
     import icodes.TestOp
     import icodes.opcodes.InvokeStyle
 
@@ -1287,38 +1286,42 @@ abstract class BCodeBodyBuilder extends BCodeSkelBuilder {
 
     def genInvokeDynamicLambda(lambdaTarget: Symbol, arity: Int, functionalInterface: Symbol) {
       val isStaticMethod = lambdaTarget.hasFlag(Flags.STATIC)
+      def asmType(sym: Symbol) = classBTypeFromSymbol(sym).toASMType
 
-      val targetHandle =
+      val implMethodHandle =
         new asm.Handle(if (lambdaTarget.hasFlag(Flags.STATIC)) asm.Opcodes.H_INVOKESTATIC else asm.Opcodes.H_INVOKEVIRTUAL,
           classBTypeFromSymbol(lambdaTarget.owner).internalName,
           lambdaTarget.name.toString,
           asmMethodType(lambdaTarget).descriptor)
-      val receiver = if (isStaticMethod) None else Some(lambdaTarget.owner)
+      val receiver = if (isStaticMethod) Nil else lambdaTarget.owner :: Nil
       val (capturedParams, lambdaParams) = lambdaTarget.paramss.head.splitAt(lambdaTarget.paramss.head.length - arity)
       // Requires https://github.com/scala/scala-java8-compat on the runtime classpath
-      val returnUnit = lambdaTarget.info.resultType.typeSymbol == UnitClass
-      val functionalInterfaceDesc: String = classBTypeFromSymbol(functionalInterface).descriptor
-      val desc = (receiver.toList ::: capturedParams).map(sym => toTypeKind(sym.info)).mkString(("("), "", ")") + functionalInterfaceDesc
+      val invokedType = asm.Type.getMethodDescriptor(asmType(functionalInterface), (receiver ::: capturedParams).map(sym => toTypeKind(sym.info).toASMType): _*)
 
-      // TODO specialization
       val constrainedType = new MethodBType(lambdaParams.map(p => toTypeKind(p.tpe)), toTypeKind(lambdaTarget.tpe.resultType)).toASMType
-      val abstractMethod = functionalInterface.info.decls.find(_.isDeferred).getOrElse(functionalInterface.info.member(nme.apply))
-      val methodName = abstractMethod.name.toString
-      val applyN = {
-        val mt = asmMethodType(abstractMethod)
-        mt.toASMType
-      }
-
-      bc.jmethod.visitInvokeDynamicInsn(methodName, desc, lambdaMetaFactoryBootstrapHandle,
-          // boostrap args
-        applyN, targetHandle, constrainedType
+      val sam = functionalInterface.info.decls.find(_.isDeferred).getOrElse(functionalInterface.info.member(nme.apply))
+      val samName = sam.name.toString
+      val samMethodType = asmMethodType(sam).toASMType
+
+      val flags = 3 // TODO 2.12.x Replace with LambdaMetafactory.FLAG_SERIALIZABLE | LambdaMetafactory.FLAG_MARKERS
+
+      val ScalaSerializable = classBTypeFromSymbol(definitions.SerializableClass).toASMType
+      bc.jmethod.visitInvokeDynamicInsn(samName, invokedType, lambdaMetaFactoryBootstrapHandle,
+        /* samMethodType          = */ samMethodType,
+        /* implMethod             = */ implMethodHandle,
+        /* instantiatedMethodType = */ constrainedType,
+        /* flags                  = */ flags.asInstanceOf[AnyRef],
+        /* markerInterfaceCount   = */ 1.asInstanceOf[AnyRef],
+        /* markerInterfaces[0]    = */ ScalaSerializable,
+        /* bridgeCount            = */ 0.asInstanceOf[AnyRef]
       )
+      indyLambdaHosts += this.claszSymbol
     }
   }
 
-  val lambdaMetaFactoryBootstrapHandle =
+  lazy val lambdaMetaFactoryBootstrapHandle =
     new asm.Handle(asm.Opcodes.H_INVOKESTATIC,
-      "java/lang/invoke/LambdaMetafactory", "metafactory",
-      "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodHandle;Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/CallSite;")
+      definitions.LambdaMetaFactory.fullName('/'), sn.AltMetafactory.toString,
+      "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite;")
 
 }

src/compiler/scala/tools/nsc/backend/jvm/BCodeHelpers.scala

@@ -682,6 +682,33 @@ abstract class BCodeHelpers extends BCodeIdiomatic with BytecodeWriters {
         new java.lang.Long(id)
       ).visitEnd()
     }
+
+    /**
+     * Add:
+     *
+     * private static Object $deserializeLambda$(SerializedLambda l) {
+     *   return scala.compat.java8.runtime.LambdaDeserializer.deserializeLambda(MethodHandles.lookup(), null, l);
+     * }
+     * @param jclass
+     */
+    // TODO add a static cache field to the class, and pass that as the second argument to `deserializeLambda`.
+    // This will make the test at run/lambda-serialization.scala:15 work
+    def addLambdaDeserialize(jclass: asm.ClassVisitor): Unit = {
+      val cw = jclass
+      import scala.tools.asm.Opcodes._
+      cw.visitInnerClass("java/lang/invoke/MethodHandles$Lookup", "java/lang/invoke/MethodHandles", "Lookup", ACC_PUBLIC + ACC_FINAL + ACC_STATIC)
+
+      {
+        val mv = cw.visitMethod(ACC_PRIVATE + ACC_STATIC + ACC_SYNTHETIC, "$deserializeLambda$", "(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;", null, null)
+        mv.visitCode()
+        mv.visitMethodInsn(INVOKESTATIC, "java/lang/invoke/MethodHandles", "lookup", "()Ljava/lang/invoke/MethodHandles$Lookup;", false)
+        mv.visitInsn(asm.Opcodes.ACONST_NULL)
+        mv.visitVarInsn(ALOAD, 0)
+        mv.visitMethodInsn(INVOKESTATIC, "scala/compat/java8/runtime/LambdaDeserializer", "deserializeLambda", "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/util/Map;Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;", false)
+        mv.visitInsn(ARETURN)
+        mv.visitEnd()
+      }
+    }
   } // end of trait BCClassGen
 
   /* functionality for building plain and mirror classes */

src/compiler/scala/tools/nsc/backend/jvm/BCodeSkelBuilder.scala

@@ -68,6 +68,8 @@ abstract class BCodeSkelBuilder extends BCodeHelpers {
     var isCZStaticModule           = false
     var isCZRemote                 = false
 
+    protected val indyLambdaHosts = collection.mutable.Set[Symbol]()
+
     /* ---------------- idiomatic way to ask questions to typer ---------------- */
 
     def paramTKs(app: Apply): List[BType] = {
@@ -121,6 +123,16 @@ abstract class BCodeSkelBuilder extends BCodeHelpers {
 
       innerClassBufferASM ++= classBType.info.get.nestedClasses
       gen(cd.impl)
+
+
+      val shouldAddLambdaDeserialize = (
+        settings.target.value == "jvm-1.8"
+          && settings.Ydelambdafy.value == "method"
+          && indyLambdaHosts.contains(claszSymbol))
+
+      if (shouldAddLambdaDeserialize)
+        addLambdaDeserialize(cnode)
+
       addInnerClassesASM(cnode, innerClassBufferASM.toList)
 
       cnode.visitAttribute(classBType.inlineInfoAttribute.get)

src/compiler/scala/tools/nsc/transform/Delambdafy.scala

@@ -146,6 +146,9 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
       val isStatic = target.hasFlag(STATIC)
 
       def createBoxingBridgeMethod(functionParamTypes: List[Type], functionResultType: Type): Tree = {
+        // Note: we bail out of this method and return EmptyTree if we find there is no adaptation required.
+        // If we need to improve performance, we could check the types first before creating the
+        // method and parameter symbols.
         val methSym = oldClass.newMethod(target.name.append("$adapted").toTermName, target.pos, target.flags | FINAL | ARTIFACT)
         var neededAdaptation = false
         def boxedType(tpe: Type): Type = {

src/reflect/scala/reflect/internal/Definitions.scala

@@ -514,6 +514,7 @@ trait Definitions extends api.StandardDefinitions {
     lazy val ScalaSignatureAnnotation = requiredClass[scala.reflect.ScalaSignature]
     lazy val ScalaLongSignatureAnnotation = requiredClass[scala.reflect.ScalaLongSignature]
 
+    lazy val LambdaMetaFactory = getClassIfDefined("java.lang.invoke.LambdaMetafactory")
     lazy val MethodHandle = getClassIfDefined("java.lang.invoke.MethodHandle")
 
     // Option classes

src/reflect/scala/reflect/internal/StdNames.scala

@@ -1167,6 +1167,8 @@ trait StdNames {
     final val Invoke: TermName           = newTermName("invoke")
     final val InvokeExact: TermName      = newTermName("invokeExact")
 
+    final val AltMetafactory: TermName      = newTermName("altMetafactory")
+
     val Boxed = immutable.Map[TypeName, TypeName](
       tpnme.Boolean -> BoxedBoolean,
       tpnme.Byte    -> BoxedByte,

src/reflect/scala/reflect/internal/transform/PostErasure.scala

@@ -9,8 +9,7 @@ trait PostErasure {
   object elimErasedValueType extends TypeMap {
     def apply(tp: Type) = tp match {
       case ConstantType(Constant(tp: Type)) => ConstantType(Constant(apply(tp)))
-      case ErasedValueType(_, underlying)   =>
-        underlying
+      case ErasedValueType(_, underlying)   => underlying
       case _                                => mapOver(tp)
     }
   }

src/reflect/scala/reflect/runtime/JavaUniverseForce.scala

@@ -310,6 +310,7 @@ trait JavaUniverseForce { self: runtime.JavaUniverse  =>
     definitions.QuasiquoteClass_api_unapply
     definitions.ScalaSignatureAnnotation
     definitions.ScalaLongSignatureAnnotation
+    definitions.LambdaMetaFactory
     definitions.MethodHandle
     definitions.OptionClass
     definitions.OptionModule

test/files/run/lambda-serialization.scala

@@ -0,0 +1,35 @@
+import java.io.{ByteArrayInputStream, ObjectInputStream, ObjectOutputStream, ByteArrayOutputStream}
+
+object Test {
+  def main(args: Array[String]): Unit = {
+    roundTrip
+  }
+
+  def roundTrip(): Unit = {
+    val c = new Capture("Capture")
+    val lambda = (p: Param) => ("a", p, c)
+    val reconstituted1 = serializeDeserialize(lambda).asInstanceOf[Object => Any]
+    val p = new Param
+    assert(reconstituted1.apply(p) == ("a", p, c))
+    val reconstituted2 = serializeDeserialize(lambda).asInstanceOf[Object => Any]
+    assert(reconstituted1.getClass == reconstituted2.getClass)
+
+    val reconstituted3 = serializeDeserialize(reconstituted1)
+    assert(reconstituted3.apply(p) == ("a", p, c))
+
+    val specializedLambda = (p: Int) => List(p, c).length
+    assert(serializeDeserialize(specializedLambda).apply(42) == 2)
+    assert(serializeDeserialize(serializeDeserialize(specializedLambda)).apply(42) == 2)
+  }
+
+  def serializeDeserialize[T <: AnyRef](obj: T) = {
+    val buffer = new ByteArrayOutputStream
+    val out = new ObjectOutputStream(buffer)
+    out.writeObject(obj)
+    val in = new ObjectInputStream(new ByteArrayInputStream(buffer.toByteArray))
+    in.readObject.asInstanceOf[T]
+  }
+}
+
+case class Capture(s: String) extends Serializable
+class Param