Merge pull request github#99 from github/kotlin-build-versions

tamasvajk · web-flow · commit 13a179907df2 · 2021-11-18T13:30:24.000+01:00
Change build script to build multiple versions of the plugin
diff --git a/java/kotlin-extractor/build.gradle b/java/kotlin-extractor/build.gradle
@@ -18,6 +18,19 @@ repositories {
 tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
     kotlinOptions {
         jvmTarget = "1.8"
+        freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"
+        // enable the below for building with kotlinVersion=1.4.32:
+        // languageVersion = "1.5"
+    }
+}
+
+sourceSets {
+    main {
+        kotlin {
+            excludes = ["utils/versions/v_1_4/*.kt"]
+            // change the above line to the below for building with kotlinVersion=1.4.32:
+            //excludes = ["utils/versions/default/*.kt"]
+        }
     }
 }
 
diff --git a/java/kotlin-extractor/build.py b/java/kotlin-extractor/build.py
@@ -1,32 +1,50 @@
 #!/usr/bin/env python3
 
+import kotlin_plugin_versions
 import glob
 import re
 import subprocess
 import shutil
 import os
 import os.path
 import sys
+import shlex
 
 kotlinc = 'kotlinc'
 javac = 'javac'
 
+kotlin_dependency_folder = '../../../resources/kotlin-dependencies'
+if (len(sys.argv) > 1):
+    kotlin_dependency_folder = sys.argv[1]
+
+
+def run_process(cmd):
+    try:
+        # print("Running command: " + shlex.join(cmd))
+        return subprocess.run(cmd, check=True, capture_output=True)
+    except subprocess.CalledProcessError as e:
+        print("Command failed: " + shlex.join(cmd), file=sys.stderr)
+        print("Output: " + e.stderr.decode(encoding='UTF-8',
+              errors='strict'), file=sys.stderr)
+        raise e
+
 
 def compile_to_dir(srcs, classpath, java_classpath, output):
     # Use kotlinc to compile .kt files:
-    subprocess.run([kotlinc,
-                    # kotlinc can default to 256M, which isn't enough when we are extracting the build
-                    '-J-Xmx2G',
-                    '-d', output,
-                    '-module-name', 'codeql-kotlin-extractor',
-                    '-no-reflect',
-                    '-jvm-target', '1.8',
-                    '-classpath', classpath] + srcs, check=True)
+    run_process([kotlinc,
+                 # kotlinc can default to 256M, which isn't enough when we are extracting the build
+                 '-J-Xmx2G',
+                 '-d', output,
+                 '-module-name', 'codeql-kotlin-extractor',
+                 '-no-reflect', '-no-stdlib',
+                 '-jvm-target', '1.8',
+                 '-classpath', classpath] + srcs)
+
     # Use javac to compile .java files, referencing the Kotlin class files:
-    subprocess.run([javac,
-                    '-d', output,
-                    '-source', '8', '-target', '8',
-                    '-classpath', "%s:%s:%s" % (output, classpath, java_classpath)] + [s for s in srcs if s.endswith(".java")], check=True)
+    run_process([javac,
+                 '-d', output,
+                 '-source', '8', '-target', '8',
+                 '-classpath', "%s:%s:%s" % (output, classpath, java_classpath)] + [s for s in srcs if s.endswith(".java")])
 
 
 def compile_to_jar(srcs, classpath, java_classpath, output):
@@ -39,9 +57,9 @@ def compile_to_jar(srcs, classpath, java_classpath, output):
 
         compile_to_dir(srcs, classpath, java_classpath, builddir)
 
-        subprocess.run(['jar', '-c', '-f', output,
-                        '-C', builddir, '.',
-                        '-C', 'src/main/resources', 'META-INF'], check=True)
+        run_process(['jar', '-c', '-f', output,
+                     '-C', builddir, '.',
+                     '-C', 'src/main/resources', 'META-INF'])
     finally:
         if os.path.exists(builddir):
             shutil.rmtree(builddir)
@@ -51,34 +69,35 @@ def find_sources(path):
     return glob.glob(path + '/**/*.kt', recursive=True) + glob.glob(path + '/**/*.java', recursive=True)
 
 
-def jarnames_to_classpath(path, jars):
-    return ":".join(os.path.join(path, jar) + ".jar" for jar in jars)
-
-
-def compile_standalone():
-    srcs = find_sources("src")
-    jars = ['kotlin-compiler']
-    java_jars = ['kotlin-stdlib']
-
-    x = subprocess.run([kotlinc, '-version', '-verbose'],
-                       check=True, capture_output=True)
+def get_kotlin_lib_folder():
+    x = run_process([kotlinc, '-version', '-verbose'])
     output = x.stderr.decode(encoding='UTF-8', errors='strict')
     m = re.match(
         r'.*\nlogging: using Kotlin home directory ([^\n]+)\n.*', output)
     if m is None:
         raise Exception('Cannot determine kotlinc home directory')
     kotlin_home = m.group(1)
-    kotlin_lib = kotlin_home + '/lib'
-    classpath = jarnames_to_classpath(kotlin_lib, jars)
-    java_classpath = jarnames_to_classpath(kotlin_lib, java_jars)
+    print("Kotlin home directory: " + kotlin_home)
+    return kotlin_home + '/lib'
+
 
-    compile_to_jar(srcs, classpath, java_classpath, 'codeql-extractor-kotlin-standalone.jar')
+def get_gradle_lib_folder():
+    x = run_process(['gradle', 'getHomeDir'])
+    output = x.stdout.decode(encoding='UTF-8', errors='strict')
+    m = re.search(r'(?m)^> Task :getHomeDir\n([^\n]+)$', output)
+    if m is None:
+        print("gradle getHomeDir output:\n" + output, file=sys.stderr)
+        raise Exception('Cannot determine gradle home directory')
+    gradle_home = m.group(1)
+    print("Gradle home directory: " + gradle_home)
+    return gradle_home + '/lib'
 
 
 def find_jar(path, pattern):
     result = glob.glob(path + '/' + pattern + '*.jar')
     if len(result) == 0:
-        raise Exception('Cannot find jar file %s under path %s' % (pattern, path))
+        raise Exception('Cannot find jar file %s under path %s' %
+                        (pattern, path))
     return result
 
 
@@ -89,42 +108,61 @@ def patterns_to_classpath(path, patterns):
     return ':'.join(result)
 
 
-def compile_embeddable():
-    x = subprocess.run(['gradle', 'getHomeDir'],
-                       check=True, capture_output=True)
-    output = x.stdout.decode(encoding='UTF-8', errors='strict')
-    m = re.search(r'(?m)^> Task :getHomeDir\n([^\n]+)$', output)
-    if m is None:
-        print("gradle getHomeDir output:\n" + output, file = sys.stderr)
-        raise Exception('Cannot determine gradle home directory')
-    gradle_home = m.group(1)
+def transform_to_embeddable(srcs):
+    # replace imports in files:
+    for src in srcs:
+        with open(src, 'r') as f:
+            content = f.read()
+        content = content.replace('import com.intellij',
+                                  'import org.jetbrains.kotlin.com.intellij')
+        with open(src, 'w') as f:
+            f.write(content)
+
 
-    gradle_lib = gradle_home + '/lib'
-    jar_patterns = ['kotlin-compiler-embeddable']
-    java_jar_patterns = ['kotlin-stdlib']
-    classpath = patterns_to_classpath(gradle_lib, jar_patterns)
-    java_classpath = patterns_to_classpath(gradle_lib, java_jar_patterns)
+def compile(jars, java_jars, dependency_folder, transform_to_embeddable, output, tmp_dir, version):
+    classpath = patterns_to_classpath(dependency_folder, jars)
+    java_classpath = patterns_to_classpath(dependency_folder, java_jars)
 
     try:
-        if os.path.exists('build/temp_src'):
-            shutil.rmtree('build/temp_src')
-        shutil.copytree('src', 'build/temp_src')
-        srcs = find_sources('build/temp_src')
-
-        # replace imports in files:
-        for src in srcs:
-            with open(src, 'r') as f:
-                content = f.read()
-            content = content.replace('import com.intellij',
-                                      'import org.jetbrains.kotlin.com.intellij')
-            with open(src, 'w') as f:
-                f.write(content)
-
-        compile_to_jar(srcs, classpath, java_classpath, 'codeql-extractor-kotlin-embeddable.jar')
+        if os.path.exists(tmp_dir):
+            shutil.rmtree(tmp_dir)
+        shutil.copytree('src', tmp_dir)
+
+        if version.startswith('1.4'):
+            shutil.rmtree(tmp_dir + '/main/kotlin/utils/versions/default')
+        else:
+            shutil.rmtree(tmp_dir + '/main/kotlin/utils/versions/v_1_4')
+
+        srcs = find_sources(tmp_dir)
+
+        transform_to_embeddable(srcs)
+
+        compile_to_jar(srcs, classpath, java_classpath, output)
     finally:
-        if os.path.exists('build/temp_src'):
-            shutil.rmtree('build/temp_src')
+        if os.path.exists(tmp_dir):
+            shutil.rmtree(tmp_dir)
+
+
+def compile_embeddable(version):
+    compile(['kotlin-stdlib-' + version, 'kotlin-compiler-embeddable-' + version],
+            ['kotlin-stdlib-' + version],
+            kotlin_dependency_folder,
+            transform_to_embeddable,
+            'codeql-extractor-kotlin-embeddable-%s.jar' % (version),
+            'build/temp_src',
+            version)
+
+
+def compile_standalone(version):
+    compile(['kotlin-stdlib-' + version, 'kotlin-compiler-' + version],
+            ['kotlin-stdlib-' + version],
+            kotlin_dependency_folder,
+            lambda srcs: None,
+            'codeql-extractor-kotlin-standalone-%s.jar' % (version),
+            'build/temp_src',
+            version)
 
 
-compile_standalone()
-compile_embeddable()
+for version in kotlin_plugin_versions.versions:
+    compile_standalone(version)
+    compile_embeddable(version)
diff --git a/java/kotlin-extractor/kotlin_plugin_versions.py b/java/kotlin-extractor/kotlin_plugin_versions.py
@@ -0,0 +1 @@
+versions = [ '1.4.32', '1.5.31', '1.6.0-RC2' ]
diff --git a/java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt b/java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt
@@ -243,6 +243,7 @@ class KotlinSourceFileExtractor(
         CommentExtractor(this).extract()
     }
 
+  @OptIn(kotlin.ExperimentalStdlibApi::class) // Annotation required by kotlin versions < 1.5
   fun extractFileClass(f: IrFile): Label<out DbClass> {
       val fileName = f.fileEntry.name
       val pkg = f.fqName.asString()
diff --git a/java/kotlin-extractor/src/main/kotlin/TrapWriter.kt b/java/kotlin-extractor/src/main/kotlin/TrapWriter.kt
@@ -1,9 +1,9 @@
 package com.github.codeql
 
+import com.github.codeql.utils.versions.FileEntry
 import java.io.BufferedWriter
 import java.io.File
 import org.jetbrains.kotlin.ir.IrElement
-import org.jetbrains.kotlin.ir.IrFileEntry
 import org.jetbrains.kotlin.ir.declarations.path
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrVariable
@@ -82,7 +82,7 @@ open class TrapWriter (val lm: TrapLabelManager, val bw: BufferedWriter) {
      * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but with the given
      * default file used in getLocation etc.
      */
-    fun withTargetFile(filePath: String, fileEntry: IrFileEntry?, populateFileTables: Boolean = true) =
+    fun withTargetFile(filePath: String, fileEntry: FileEntry?, populateFileTables: Boolean = true) =
         FileTrapWriter(lm, bw, filePath, fileEntry, populateFileTables)
 }
 
@@ -104,7 +104,7 @@ open class FileTrapWriter (
     lm: TrapLabelManager,
     bw: BufferedWriter,
     val filePath: String,
-    val sourceFileEntry: IrFileEntry?,
+    val sourceFileEntry: FileEntry?,
     populateFileTables: Boolean = true
 ): TrapWriter (lm, bw) {
     val populateFile = PopulateFile(this)
diff --git a/java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt b/java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt
@@ -2,9 +2,9 @@ package com.github.codeql.comments
 
 import com.github.codeql.*
 import com.github.codeql.utils.IrVisitorLookup
+import com.github.codeql.utils.versions.Psi2Ir
 import com.intellij.psi.PsiComment
 import com.intellij.psi.PsiElement
-import org.jetbrains.kotlin.backend.jvm.ir.getKtFile
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.path
 import org.jetbrains.kotlin.kdoc.psi.api.KDoc
@@ -17,7 +17,7 @@ class CommentExtractor(private val fileExtractor: KotlinSourceFileExtractor) {
     private val file = fileExtractor.file
     private val tw = fileExtractor.tw
     private val logger = fileExtractor.logger
-    private val ktFile = file.getKtFile()
+    private val ktFile = Psi2Ir().getKtFile(file)
 
     init {
         if (ktFile == null) {
diff --git a/java/kotlin-extractor/src/main/kotlin/utils/IrVisitorLookup.kt b/java/kotlin-extractor/src/main/kotlin/utils/IrVisitorLookup.kt
@@ -1,7 +1,7 @@
 package com.github.codeql.utils
 
+import com.github.codeql.utils.versions.Psi2Ir
 import com.intellij.psi.PsiElement
-import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
@@ -19,7 +19,7 @@ class IrVisitorLookup(private val psi: PsiElement, private val file: IrFile) :
         }
 
         if (location.contains(elementLocation)) {
-            val psiElement = PsiSourceManager.findPsiElement(element, file)
+            val psiElement = Psi2Ir().findPsiElement(element, file)
             if (psiElement == psi) {
                 // There can be multiple IrElements that match the same PSI element.
                 data.add(element)
diff --git a/java/kotlin-extractor/src/main/kotlin/utils/versions/Psi2IrFacade.kt b/java/kotlin-extractor/src/main/kotlin/utils/versions/Psi2IrFacade.kt
@@ -0,0 +1,11 @@
+package com.github.codeql.utils.versions
+
+import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.psi.KtFile
+
+interface Psi2IrFacade {
+    fun getKtFile(irFile: IrFile): KtFile?
+    fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement?
+}
diff --git a/java/kotlin-extractor/src/main/kotlin/utils/versions/default/FileEntry.kt b/java/kotlin-extractor/src/main/kotlin/utils/versions/default/FileEntry.kt
@@ -0,0 +1,5 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.IrFileEntry
+
+typealias FileEntry = IrFileEntry
diff --git a/java/kotlin-extractor/src/main/kotlin/utils/versions/default/Psi2Ir.kt b/java/kotlin-extractor/src/main/kotlin/utils/versions/default/Psi2Ir.kt
@@ -0,0 +1,18 @@
+package com.github.codeql.utils.versions
+
+import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager
+import org.jetbrains.kotlin.backend.jvm.ir.getKtFile
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.psi.KtFile
+
+class Psi2Ir: Psi2IrFacade {
+    override fun getKtFile(irFile: IrFile): KtFile? {
+        return irFile.getKtFile()
+    }
+
+    override fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? {
+        return PsiSourceManager.findPsiElement(irElement, irFile)
+    }
+}
diff --git a/java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4/FileEntry.kt b/java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4/FileEntry.kt
@@ -0,0 +1,5 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.SourceManager
+
+typealias FileEntry = SourceManager.FileEntry
diff --git a/java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4/Psi2Ir.kt b/java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4/Psi2Ir.kt
@@ -0,0 +1,21 @@
+package com.github.codeql.utils.versions
+
+import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi2ir.PsiSourceManager
+
+class Psi2Ir : Psi2IrFacade {
+    companion object {
+        val psiManager = PsiSourceManager()
+    }
+
+    override fun getKtFile(irFile: IrFile): KtFile? {
+        return psiManager.getKtFile(irFile)
+    }
+
+    override fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? {
+        return psiManager.findPsiElement(irElement, irFile)
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+versions = [ '1.4.32', '1.5.31', '1.6.0-RC2' ]`
Original file line number	Diff line number	Diff line change
`@@ -243,6 +243,7 @@ class KotlinSourceFileExtractor(`
`243`	`243`	`CommentExtractor(this).extract()`
`244`	`244`	`}`
`245`	`245`
	`246`	`+ @OptIn(kotlin.ExperimentalStdlibApi::class) // Annotation required by kotlin versions < 1.5`
`246`	`247`	`fun extractFileClass(f: IrFile): Label<out DbClass> {`
`247`	`248`	`val fileName = f.fileEntry.name`
`248`	`249`	`val pkg = f.fqName.asString()`