[indylambda] Support lambda {de}serialization

To support serialization, we use the alternative lambda metafactory
that lets us specify that our anonymous functions should extend the
marker interface `scala.Serializable`. They will also have a
`writeObject` method added that implements the serialization proxy
pattern using `j.l.invoke.SerializedLamba`.

To support deserialization, we synthesize a `$deserializeLamba$`
method in each class. This will be called reflectively by
`SerializedLambda#readResolve`. This method in turn delegates to
`LambdaDeserializer`, currently defined [1] in `scala-java8-compat`,
that uses `LambdaMetafactory` to spin up the anonymous class and
instantiate it with the deserialized environment.

Note: `LambdaDeserializer` reuses the anonymous class on subsequent
deserializations of a given lambda, in the same spirit as an
invokedynamic call site only spins up the class on the first time
it is run.

`LambdaDeserializer` will be moved into our standard library in
the 2.12.x branch, where we can introduce dependencies on the
Java 8 standard library.

The enclosed test cases must be manually run with indylambda enabled.
Once we enable indylambda by default on 2.12.x, the test will
actually test the new feature.

```
% echo $INDYLAMBDA
-Ydelambdafy:method -Ybackend:GenBCode -target:jvm-1.8 -classpath .:scala-java8-compat_2.11-0.5.0-SNAPSHOT.jar
% $INDYLAMBDA -e "println((() => 42).getClass)"
class Main$$anon$1$$Lambda$1/1183231938
% qscala $INDYLAMBDA -e "assert(classOf[scala.Serializable].isInstance(() => 42))"
% qscalac $INDYLAMBDA test/files/run/lambda-serialization.scala && qscala $INDYLAMBDA Test
```

[1] scala/scala-java8-compat#37

Author: retronym

src/compiler/scala/tools/nsc/backend/jvm/BCodeBodyBuilder.scala

@@ -1287,38 +1287,41 @@ abstract class BCodeBodyBuilder extends BCodeSkelBuilder {
 
     def genInvokeDynamicLambda(lambdaTarget: Symbol, arity: Int, functionalInterface: Symbol) {
       val isStaticMethod = lambdaTarget.hasFlag(Flags.STATIC)
+      def asmType(sym: Symbol) = classBTypeFromSymbol(sym).toASMType
 
-      val targetHandle =
+      val implMethodHandle =
         new asm.Handle(if (lambdaTarget.hasFlag(Flags.STATIC)) asm.Opcodes.H_INVOKESTATIC else asm.Opcodes.H_INVOKEVIRTUAL,
           classBTypeFromSymbol(lambdaTarget.owner).internalName,
           lambdaTarget.name.toString,
           asmMethodType(lambdaTarget).descriptor)
       val receiver = if (isStaticMethod) None else Some(lambdaTarget.owner)
       val (capturedParams, lambdaParams) = lambdaTarget.paramss.head.splitAt(lambdaTarget.paramss.head.length - arity)
       // Requires https://github.com/scala/scala-java8-compat on the runtime classpath
-      val returnUnit = lambdaTarget.info.resultType.typeSymbol == UnitClass
-      val functionalInterfaceDesc: String = classBTypeFromSymbol(functionalInterface).descriptor
-      val desc = (receiver.toList ::: capturedParams).map(sym => toTypeKind(sym.info)).mkString(("("), "", ")") + functionalInterfaceDesc
+      val invokedType = asm.Type.getMethodDescriptor(asmType(functionalInterface), (receiver.toList ::: capturedParams).map(sym => toTypeKind(sym.info).toASMType): _*)
 
-      // TODO specialization
       val constrainedType = new MethodBType(lambdaParams.map(p => toTypeKind(p.tpe)), toTypeKind(lambdaTarget.tpe.resultType)).toASMType
-      val abstractMethod = functionalInterface.info.decls.find(_.isDeferred).getOrElse(functionalInterface.info.member(nme.apply))
-      val methodName = abstractMethod.name.toString
-      val applyN = {
-        val mt = asmMethodType(abstractMethod)
-        mt.toASMType
-      }
-
-      bc.jmethod.visitInvokeDynamicInsn(methodName, desc, lambdaMetaFactoryBootstrapHandle,
-          // boostrap args
-        applyN, targetHandle, constrainedType
+      val sam = functionalInterface.info.decls.find(_.isDeferred).getOrElse(functionalInterface.info.member(nme.apply))
+      val samName = sam.name.toString
+      val samMethodType = asmMethodType(sam).toASMType
+
+      val flags = 3 // TODO 2.12.x Replace with LambdaMetafactory.FLAG_SERIALIZABLE | LambdaMetafactory.FLAG_MARKERS
+
+      val ScalaSerializable = classBTypeFromSymbol(definitions.SerializableClass).toASMType
+      bc.jmethod.visitInvokeDynamicInsn(samName, invokedType, lambdaMetaFactoryBootstrapHandle,
+        /* samMethodType          = */ samMethodType,
+        /* implMethod             = */ implMethodHandle,
+        /* instantiatedMethodType = */ constrainedType,
+        /* flags                  = */ flags.asInstanceOf[AnyRef],
+        /* markerInterfaceCount   = */ 1.asInstanceOf[AnyRef],
+        /* markerInterfaces[0]    = */ ScalaSerializable,
+        /* bridgeCount            = */ 0.asInstanceOf[AnyRef]
       )
     }
   }
 
   val lambdaMetaFactoryBootstrapHandle =
     new asm.Handle(asm.Opcodes.H_INVOKESTATIC,
-      "java/lang/invoke/LambdaMetafactory", "metafactory",
-      "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodHandle;Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/CallSite;")
+      definitions.LambdaMetaFactory.fullName('/'), sn.AltMetafactory.toString,
+      "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite;")
 
 }

src/compiler/scala/tools/nsc/backend/jvm/BCodeHelpers.scala

@@ -682,6 +682,32 @@ abstract class BCodeHelpers extends BCodeIdiomatic with BytecodeWriters {
         new java.lang.Long(id)
       ).visitEnd()
     }
+
+    /**
+     * Add:
+     *
+     * private static Object $deserializeLambda$(SerializedLambda l) {
+     *   return scala.compat.java8.runtime.LambdaDeserializer.deserializeLambda(MethodHandles.lookup(), l);
+     * }
+     * @param jclass
+     */
+    def addLambdaDeserialize(jclass: asm.ClassVisitor): Unit = {
+      val cw = jclass
+      import scala.tools.asm.Opcodes._
+      cw.visitInnerClass("java/lang/invoke/MethodHandles$Lookup", "java/lang/invoke/MethodHandles", "Lookup", ACC_PUBLIC + ACC_FINAL + ACC_STATIC)
+
+      {
+        val mv = cw.visitMethod(ACC_PRIVATE + ACC_STATIC + ACC_SYNTHETIC, "$deserializeLambda$", "(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;", null, null)
+        mv.visitCode()
+        mv.visitMethodInsn(INVOKESTATIC, "java/lang/invoke/MethodHandles", "lookup", "()Ljava/lang/invoke/MethodHandles$Lookup;", false)
+        mv.visitVarInsn(ALOAD, 0)
+        mv.visitMethodInsn(INVOKESTATIC, "scala/compat/java8/runtime/LambdaDeserializer", "deserializeLambda", "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;", false)
+        mv.visitInsn(ARETURN)
+        mv.visitMaxs(2, 1)
+        mv.visitEnd()
+      }
+      cw.visitEnd()
+    }
   } // end of trait BCClassGen
 
   /* functionality for building plain and mirror classes */
@@ -695,7 +721,7 @@ abstract class BCodeHelpers extends BCodeIdiomatic with BytecodeWriters {
   class JMirrorBuilder extends JCommonBuilder {
 
     private var cunit: CompilationUnit = _
-    def getCurrentCUnit(): CompilationUnit = cunit;
+    def getCurrentCUnit(): CompilationUnit = cunit
 
     /* Generate a mirror class for a top-level module. A mirror class is a class
      *  containing only static methods that forward to the corresponding method
@@ -781,8 +807,8 @@ abstract class BCodeHelpers extends BCodeIdiomatic with BytecodeWriters {
 
       for (f <- fieldSymbols if f.hasGetter;
                  g = f.getterIn(cls);
-                 s = f.setterIn(cls);
-	         if g.isPublic && !(f.name startsWith "$")
+                 s = f.setterIn(cls)
+           if g.isPublic && !(f.name startsWith "$")
           ) {
              // inserting $outer breaks the bean
              fieldList = javaSimpleName(f) :: javaSimpleName(g) :: (if (s != NoSymbol) javaSimpleName(s) else null) :: fieldList

src/compiler/scala/tools/nsc/backend/jvm/BCodeSkelBuilder.scala

@@ -116,6 +116,9 @@ abstract class BCodeSkelBuilder extends BCodeHelpers {
 
       val optSerial: Option[Long] = serialVUID(claszSymbol)
       if (optSerial.isDefined) { addSerialVUID(optSerial.get, cnode)}
+      val shouldAddLambdaDeserialize = settings.target.value == "jvm-1.8" && settings.Ydelambdafy.value == "method"
+      if (shouldAddLambdaDeserialize)
+        addLambdaDeserialize(cnode)
 
       addClassFields()
 

src/compiler/scala/tools/nsc/transform/Delambdafy.scala

@@ -146,6 +146,9 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
       val isStatic = target.hasFlag(STATIC)
 
       def createBoxingBridgeMethod(functionParamTypes: List[Type], functionResultType: Type): Tree = {
+        // Note: we bail out of this method and return EmptyTree if we find there is no adaptation required.
+        // If we need to improve performance, we could check the types first before creating the
+        // method and parameter symbols.
         val methSym = oldClass.newMethod(target.name.append("$adapted").toTermName, target.pos, target.flags | FINAL | ARTIFACT)
         var neededAdaptation = false
         def boxedType(tpe: Type): Type = {

src/reflect/scala/reflect/internal/Definitions.scala

@@ -514,6 +514,7 @@ trait Definitions extends api.StandardDefinitions {
     lazy val ScalaSignatureAnnotation = requiredClass[scala.reflect.ScalaSignature]
     lazy val ScalaLongSignatureAnnotation = requiredClass[scala.reflect.ScalaLongSignature]
 
+    lazy val LambdaMetaFactory = getClassIfDefined("java.lang.invoke.LambdaMetafactory")
     lazy val MethodHandle = getClassIfDefined("java.lang.invoke.MethodHandle")
 
     // Option classes

src/reflect/scala/reflect/internal/StdNames.scala

@@ -1167,6 +1167,9 @@ trait StdNames {
     final val Invoke: TermName           = newTermName("invoke")
     final val InvokeExact: TermName      = newTermName("invokeExact")
 
+    final val Metafactory: TermName      = newTermName("metafactory")
+    final val AltMetafactory: TermName      = newTermName("altMetafactory")
+
     val Boxed = immutable.Map[TypeName, TypeName](
       tpnme.Boolean -> BoxedBoolean,
       tpnme.Byte    -> BoxedByte,

test/files/run/lambda-serialization.scala

@@ -0,0 +1,35 @@
+import java.io.{ByteArrayInputStream, ObjectInputStream, ObjectOutputStream, ByteArrayOutputStream}
+
+object Test {
+  def main(args: Array[String]): Unit = {
+    roundTrip
+  }
+
+  def roundTrip(): Unit = {
+    val c = new Capture("Capture")
+    val lambda = (p: Param) => ("a", p, c)
+    val reconstituted1 = serializeDeserialize(lambda).asInstanceOf[Object => Any]
+    val p = new Param
+    assert(reconstituted1.apply(p) == ("a", p, c))
+    val reconstituted2 = serializeDeserialize(lambda).asInstanceOf[Object => Any]
+    assert(reconstituted1.getClass == reconstituted2.getClass)
+
+    val reconstituted3 = serializeDeserialize(reconstituted1)
+    assert(reconstituted3.apply(p) == ("a", p, c))
+
+    val specializedLambda = (p: Int) => List(p, c).length
+    assert(serializeDeserialize(specializedLambda).apply(42) == 2)
+    assert(serializeDeserialize(serializeDeserialize(specializedLambda)).apply(42) == 2)
+  }
+
+  def serializeDeserialize[T <: AnyRef](obj: T) = {
+    val buffer = new ByteArrayOutputStream
+    val out = new ObjectOutputStream(buffer)
+    out.writeObject(obj)
+    val in = new ObjectInputStream(new ByteArrayInputStream(buffer.toByteArray))
+    in.readObject.asInstanceOf[T]
+  }
+}
+
+case class Capture(s: String) extends Serializable
+class Param