Merge pull request #16546 from erik-krogh/ts-big-file-fix

JS: fix that very large TypeScript files would crash the extractor

Author: erik-krogh

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -735,6 +735,7 @@ private CompletableFuture<?> extractSource() throws IOException {
          .collect(Collectors.toList());
 
     filesToExtract = filesToExtract.stream()
+        .filter(p -> !isFileTooLarge(p))
         .sorted(PATH_ORDERING)
         .collect(Collectors.toCollection(() -> new LinkedHashSet<>()));
 
@@ -1010,6 +1011,15 @@ private ExtractorConfig mkExtractorConfig() {
     return config;
   }
 
+  private boolean isFileTooLarge(Path f) {
+    long fileSize = f.toFile().length();
+    if (fileSize > 1_000_000L * this.maximumFileSizeInMegabytes) {
+      warn("Skipping " + f + " because it is too large (" + StringUtil.printFloat(fileSize / 1_000_000.0) + " MB). The limit is " + this.maximumFileSizeInMegabytes + " MB.");
+      return true;
+    }
+    return false;
+  }
+
   private Set<Path> extractTypeScript(
       Set<Path> files,
       Set<Path> extractedFiles,
@@ -1051,9 +1061,10 @@ private Set<Path> extractTypeScript(
             // compiler can parse them for us.
             continue;
           }
-          if (!extractedFiles.contains(sourcePath)) {
-            typeScriptFiles.add(sourcePath);
+          if (extractedFiles.contains(sourcePath)) {
+            continue;
           }
+          typeScriptFiles.add(sourcePath);
         }
         typeScriptFiles.sort(PATH_ORDERING);
         extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors);
@@ -1236,11 +1247,6 @@ private void doExtract(FileExtractor extractor, Path file, ExtractorState state)
       warn("Skipping " + file + ", which does not exist.");
       return;
     }
-    long fileSize = f.length();
-    if (fileSize > 1_000_000L * this.maximumFileSizeInMegabytes) {
-      warn("Skipping " + file + " because it is too large (" + StringUtil.printFloat(fileSize / 1_000_000.0) + " MB). The limit is " + this.maximumFileSizeInMegabytes + " MB.");
-      return;
-    }
 
     try {
       long start = logBeginProcess("Extracting " + file);

javascript/ql/lib/change-notes/2024-05-21-big-ts-files.md

@@ -0,0 +1,4 @@
+---
+category: fix
+---
+* Fixed a bug where very large TypeScript files would cause database creation to crash. Large files over 10MB were already excluded from analysis, but the file size check was not applied to TypeScript files.