Merge pull request #14273 from dotty-staging/mb/compiletime-ops-benchmarks

Add compiletime benchmarks and fix performance regression

Author: mbovel

.gitignore

@@ -58,6 +58,9 @@ tests/partest-generated/
 tests/locks/
 /test-classes/
 
+# Benchmarks
+bench/tests-generated
+
 # Ignore output files but keep the directory
 out/
 build/

bench/profiles/compiletime.yml

@@ -0,0 +1,51 @@
+charts:
+
+    - name: "Compile-time sums of constant integer types (generated)"
+      url: https://github.com/lampepfl/dotty/blob/master/bench/src/main/scala/generateBenchmarks.scala
+      lines:
+        - key: compiletime-sum-constants
+          label: bootstrapped
+
+    - name: "Compile-time sums of term reference types (generated)"
+      url: https://github.com/lampepfl/dotty/blob/master/bench/src/main/scala/generateBenchmarks.scala
+      lines:
+        - key: compiletime-sum-termrefs
+          label: bootstrapped
+
+    - name: "Sums of term references, result type inferred (generated)"
+      url: https://github.com/lampepfl/dotty/blob/master/bench/src/main/scala/generateBenchmarks.scala
+      lines:
+        - key: compiletime-sum-termrefs-terms
+          label: bootstrapped
+
+    - name: "Compile-time sums of type applications (generated)"
+      url: https://github.com/lampepfl/dotty/blob/master/bench/src/main/scala/generateBenchmarks.scala
+      lines:
+        - key: compiletime-sum-applications
+          label: bootstrapped
+
+    - name: "Compile-time additions inside multiplications (generated)"
+      url: https://github.com/lampepfl/dotty/blob/master/bench/src/main/scala/generateBenchmarks.scala
+      lines:
+        - key: compiletime-distribute
+          label: bootstrapped
+
+scripts:
+
+    compiletime-sum-constants:
+        - measure 6 6 7 1 $PROG_HOME/dotty/bench/tests-generated/compiletime-ops/sum-constants.scala
+
+    compiletime-sum-termrefs:
+        - measure 6 6 7 1 $PROG_HOME/dotty/bench/tests-generated/compiletime-ops/sum-termrefs.scala
+
+    compiletime-sum-termrefs-terms:
+        - measure 6 6 7 1 $PROG_HOME/dotty/bench/tests-generated/compiletime-ops/sum-termrefs-terms.scala
+
+    compiletime-sum-applications:
+        - measure 6 6 7 1 $PROG_HOME/dotty/bench/tests-generated/compiletime-ops/sum-applications.scala
+
+    compiletime-distribute:
+        - measure 6 6 7 1 $PROG_HOME/dotty/bench/tests-generated/compiletime-ops/distribute.scala
+
+config:
+    pr_base_url: "https://github.com/lampepfl/dotty/pull/"

bench/profiles/default.yml

@@ -7,6 +7,7 @@ includes:
     - empty.yml
     - quotes.yml
     - tuples.yml
+    - compiletime.yml
 
 
 config:

bench/src/main/scala/Benchmarks.scala

@@ -8,6 +8,8 @@ import reporting._
 import org.openjdk.jmh.results.RunResult
 import org.openjdk.jmh.runner.Runner
 import org.openjdk.jmh.runner.options.OptionsBuilder
+import org.openjdk.jmh.runner.options.TimeValue
+//import org.openjdk.jmh.results.format.ResultFormatType
 import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.results.format._
 import java.util.concurrent.TimeUnit
@@ -21,8 +23,11 @@ import dotty.tools.io.AbstractFile
 
 object Bench {
   val COMPILE_OPTS_FILE = "compile.txt"
+  val GENERATED_BENCHMARKS_DIR = "tests-generated"
 
   def main(args: Array[String]): Unit = {
+    generateBenchmarks(GENERATED_BENCHMARKS_DIR)
+
     if (args.isEmpty) {
       println("Missing <args>")
       return
@@ -32,7 +37,7 @@ object Bench {
     val warmup = if (intArgs.length > 0) intArgs(0).toInt else 30
     val iterations = if (intArgs.length > 1) intArgs(1).toInt else 20
     val forks = if (intArgs.length > 2) intArgs(2).toInt else 1
-
+    val measurementTime = if (intArgs.length > 3) intArgs(3).toInt else 1
 
     import File.{ separator => sep }
 
@@ -48,7 +53,13 @@ object Bench {
                .mode(Mode.AverageTime)
                .timeUnit(TimeUnit.MILLISECONDS)
                .warmupIterations(warmup)
+               .warmupTime(TimeValue.seconds(measurementTime))
                .measurementIterations(iterations)
+               .measurementTime(TimeValue.seconds(measurementTime))
+               // To output results to bench/results.json, uncomment the 2
+               // following lines and the ResultFormatType import.
+               //.result("results.json")
+               //.resultFormat(ResultFormatType.JSON)
                .forks(forks)
                .build
 

bench/src/main/scala/generateBenchmarks.scala

@@ -0,0 +1,155 @@
+package dotty.tools.benchmarks
+
+import java.nio.file.{Files, Paths, Path}
+import java.util.Random
+
+/** Generates benchmarks in `genDirName`.
+ *
+ *  Called automatically by the benchmarks runner ([[Bench.main]]).
+ */
+def generateBenchmarks(genDirName: String) =
+  val thisFile = Paths.get("src/main/scala/generateBenchmarks.scala")
+  val genDir = Paths.get(genDirName)
+
+  def generateBenchmark(subDirName: String, fileName: String, make: () => String) =
+    val outputDir = genDir.resolve(Paths.get(subDirName))
+    Files.createDirectories(outputDir)
+    val file = outputDir.resolve(Paths.get(fileName))
+    if !Files.exists(file) ||
+        Files.getLastModifiedTime(file).toMillis() <
+        Files.getLastModifiedTime(thisFile).toMillis() then
+      println(f"Generate benchmark $file")
+      Files.write(file, make().getBytes())
+
+  // Big compile-time sums of constant integer types: (1.type + 2.type + …).
+  // This should ideally have a linear complexity.
+  generateBenchmark("compiletime-ops", "sum-constants.scala", () =>
+    val innerSum = (1 to 50) // Limited to 50 to avoid stackoverflows in the compiler.
+      .map(i => f"$i")
+      .mkString(" + ")
+    val outerSum = (1 to 50)
+      .map(_ => f"($innerSum)")
+      .mkString(" + ")
+    val vals = (1 to 50)
+      .map(i => f"val v$i: $outerSum = ???")
+      .mkString("\n\n  ")
+
+    f"""
+import scala.compiletime.ops.int.*
+
+object Test:
+  val one: 1 = ???
+  val n: Int = ???
+  val m: Int = ???
+
+  $vals
+    """
+  )
+
+  // Big compile-time sums of term reference types: (one.type + m.type + n.type
+  // + one.type + m.type + n.type + …). This big type is normalized to (8000 +
+  // 8000 * m.type + 8000 * n.type).
+  generateBenchmark("compiletime-ops", "sum-termrefs.scala", () =>
+    val innerSum = (1 to 40)
+      .map(_ => "one.type + m.type + n.type")
+      .mkString(" + ")
+    val outerSum = (1 to 20)
+      .map(_ => f"($innerSum)")
+      .mkString(" + ")
+    val vals = (1 to 4)
+      .map(i => f"val v$i: $outerSum = ???")
+      .mkString("\n\n  ")
+
+    f"""
+import scala.compiletime.ops.int.*
+
+object Test:
+  val one: 1 = ???
+  val n: Int = ???
+  val m: Int = ???
+
+  $vals
+    """
+  )
+
+  // Big compile-time sums of term references: (n + m + …). The result type is
+  // inferred. The goal of this benchmark is to measure the performance cost of
+  // inferring precise types for arithmetic operations.
+  generateBenchmark("compiletime-ops", "sum-termrefs-terms.scala", () =>
+    val innerSum = (1 to 40)
+      .map(_ => "one + m + n")
+      .mkString(" + ")
+    val outerSum = (1 to 20)
+      .map(_ => f"($innerSum)")
+      .mkString(" + ")
+    val vals = (1 to 4)
+      .map(i => f"val v$i = $outerSum")
+      .mkString("\n\n  ")
+
+    f"""
+import scala.compiletime.ops.int.*
+
+object Test:
+  val one: 1 = ???
+  val n: Int = ???
+  val m: Int = ???
+
+  $vals
+    """
+  )
+
+  // Big compile-time product of sums of term references: (one + n + m) * (one +
+  // n + m) * …. The goal of this benchmark is to measure the performance impact
+  // of distributing addition over multiplication during compile-time operations
+  // normalization.
+  generateBenchmark("compiletime-ops", "distribute.scala", () =>
+    val product = (1 to 18)
+      .map(_ => "(one.type + m.type + n.type)")
+      .mkString(" * ")
+    val vals = (1 to 50)
+      .map(i => f"val v$i: $product = ???")
+      .mkString("\n\n  ")
+
+    f"""
+import scala.compiletime.ops.int.*
+
+object Test:
+  val one: 1 = ???
+  val n: Int = ???
+  val m: Int = ???
+
+  $vals
+    """
+  )
+
+  def applicationCount = 14
+  def applicationDepth = 10
+  def applicationVals = 2
+
+  // Compile-time sums of big applications: Op[Op[…], Op[…]] + Op[Op[…], Op[…]]
+  // + …. Applications are deep balanced binary trees only differing in their
+  // very last (top-right) leafs. These applications are compared pairwise in
+  // order to sort the terms of the sum.
+  generateBenchmark("compiletime-ops", "sum-applications.scala", () =>
+    def makeOp(depth: Int, last: Boolean, k: Int): String =
+      if depth == 0 then f"Op[one.type, ${if last then k.toString else "n.type"}]"
+      else f"Op[${makeOp(depth - 1, false, k)}, ${makeOp(depth - 1, last, k)}]"
+    val sum = (applicationCount to 1 by -1)
+      .map(k => makeOp(applicationDepth, true, k))
+      .mkString(" + ")
+    val vals = (1 to applicationVals)
+      .map(i => f"val v$i: $sum = ???")
+      .mkString("\n\n  ")
+
+    f"""
+import scala.compiletime.ops.int.*
+
+object Test:
+  val one: 1 = ???
+  val n: Int = ???
+  type SInt = Int & Singleton
+  type Op[A <: SInt, B <: SInt] <:SInt
+
+  $vals
+    """
+  )

bench/tests/compiletime-ops/empty.scala

@@ -0,0 +1,6 @@
+import scala.compiletime.ops.int.*
+
+object Test:
+  val one: 1 = ???
+  val n: Int = ???
+  val m: Int = ???

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -4248,15 +4248,6 @@ object Types {
           case _ => None
         }
 
-        val opsSet = Set(
-          defn.CompiletimeOpsAnyModuleClass,
-          defn.CompiletimeOpsIntModuleClass,
-          defn.CompiletimeOpsLongModuleClass,
-          defn.CompiletimeOpsFloatModuleClass,
-          defn.CompiletimeOpsBooleanModuleClass,
-          defn.CompiletimeOpsStringModuleClass
-        )
-
         // Returns Some(true) if the type is a constant.
         // Returns Some(false) if the type is not a constant.
         // Returns None if there is not enough information to determine if the type is a constant.
@@ -4272,7 +4263,7 @@ object Types {
           // constant if the term is constant
           case t: TermRef => isConst(t.underlying)
           // an operation type => recursively check all argument compositions
-          case applied: AppliedType if opsSet.contains(applied.typeSymbol.owner) =>
+          case applied: AppliedType if defn.isCompiletimeAppliedType(applied.typeSymbol) =>
             val argsConst = applied.args.map(isConst)
             if (argsConst.exists(_.isEmpty)) None
             else Some(argsConst.forall(_.get))