Merge branch 'change/adjust-extracted-files-diagnostics' of https://github.com/sidshank/codeql into change/adjust-extracted-files-diagnostics

Author: sidshank

.gitattributes

@@ -71,3 +71,6 @@ go/extractor/opencsv/CSVReader.java -text
 # `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
 javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
 javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
+
+# Auto-generated modeling for Python
+python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true

.github/workflows/mad_modelDiff.yml

@@ -12,6 +12,7 @@ on:
       - main
     paths:
       - "java/ql/src/utils/modelgenerator/**/*.*"
+      - "misc/scripts/models-as-data/*.*"
       - ".github/workflows/mad_modelDiff.yml"
 
 permissions:
@@ -61,8 +62,9 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE ${SHORTNAME}.temp.model.yml
-            mv java/ql/lib/ext/generated/${SHORTNAME}.temp.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
+            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
+            mkdir -p $MODELS/$SHORTNAME
+            mv java/ql/lib/ext/generated/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME
             cd ..
           }
 
@@ -85,16 +87,16 @@ jobs:
           set -x
           MODELS=`pwd`/tmp-models
           ls -1 tmp-models/
-          for m in $MODELS/*_main.model.yml ; do
+          for m in $MODELS/*/main/*.model.yml ; do
             t="${m/main/"pr"}"
             basename=`basename $m`
-            name="diff_${basename/_main.model.yml/""}"
+            name="diff_${basename/.model.yml/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
       - uses: actions/upload-artifact@v3
         with:
           name: models
-          path: tmp-models/*.model.yml
+          path: tmp-models/**/**/*.model.yml
           retention-days: 20
       - uses: actions/upload-artifact@v3
         with:

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties

@@ -1,5 +1,6 @@
 description: Support C++17 if and switch initializers
 compatibility: partial
+constexpr_if_initialization.rel: delete
 if_initialization.rel: delete
 switch_initialization.rel: delete
 exprparents.rel: run exprparents.qlo

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 0.12.3
+
+### Deprecated APIs
+
+* The `isUserInput`, `userInputArgument`, and `userInputReturned` predicates from `SecurityOptions` have been deprecated. Use `FlowSource` instead.
+
+### New Features
+
+* `UserDefineLiteral` and `DeductionGuide` classes have been added, representing C++11 user defined literals and C++17 deduction guides.
+
+### Minor Analysis Improvements
+
+* Changed the output of `Node.toString` to better reflect how many indirections a given dataflow node has.
+* Added a new predicate `Node.asDefinition` on `DataFlow::Node`s for selecting the dataflow node corresponding to a particular definition.
+* The deprecated `DefaultTaintTracking` library has been removed.
+* The `Guards` library has been replaced with the API-compatible `IRGuards` implementation, which has better precision in some cases.
+
+### Bug Fixes
+
+* Under certain circumstances a function declaration that is not also a definition could be associated with a `Function` that did not have the definition as a `FunctionDeclarationEntry`. This is now fixed when only one definition exists, and a unique `Function` will exist that has both the declaration and the definition as a `FunctionDeclarationEntry`.
+
 ## 0.12.2
 
 No user-facing changes.

cpp/ql/lib/change-notes/2022-11-21-ir-guards-replacement.md

@@ -0,0 +1,20 @@
+## 0.12.3
+
+### Deprecated APIs
+
+* The `isUserInput`, `userInputArgument`, and `userInputReturned` predicates from `SecurityOptions` have been deprecated. Use `FlowSource` instead.
+
+### New Features
+
+* `UserDefineLiteral` and `DeductionGuide` classes have been added, representing C++11 user defined literals and C++17 deduction guides.
+
+### Minor Analysis Improvements
+
+* Changed the output of `Node.toString` to better reflect how many indirections a given dataflow node has.
+* Added a new predicate `Node.asDefinition` on `DataFlow::Node`s for selecting the dataflow node corresponding to a particular definition.
+* The deprecated `DefaultTaintTracking` library has been removed.
+* The `Guards` library has been replaced with the API-compatible `IRGuards` implementation, which has better precision in some cases.
+
+### Bug Fixes
+
+* Under certain circumstances a function declaration that is not also a definition could be associated with a `Function` that did not have the definition as a `FunctionDeclarationEntry`. This is now fixed when only one definition exists, and a unique `Function` will exist that has both the declaration and the definition as a `FunctionDeclarationEntry`.

cpp/ql/lib/change-notes/2023-11-25-default-taint-tracking-removal.md

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.12.2
+lastReleaseVersion: 0.12.3

cpp/ql/lib/change-notes/2023-11-30-as-definition.md

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.12.3-dev
+version: 0.12.4-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/change-notes/2023-12-08-ususerinput-deprecation.md

@@ -7,7 +7,6 @@ import semmle.code.cpp.Location
 private import semmle.code.cpp.Enclosing
 private import semmle.code.cpp.internal.ResolveClass
 private import semmle.code.cpp.internal.ResolveGlobalVariable
-private import semmle.code.cpp.internal.ResolveFunction
 
 /**
  * Get the `Element` that represents this `@element`.
@@ -31,14 +30,11 @@ pragma[inline]
 @element unresolveElement(Element e) {
   not result instanceof @usertype and
   not result instanceof @variable and
-  not result instanceof @function and
   result = e
   or
   e = resolveClass(result)
   or
   e = resolveGlobalVariable(result)
-  or
-  e = resolveFunction(result)
 }
 
 /**

cpp/ql/lib/change-notes/2023-12-14-dataflow-tostring.md

@@ -9,7 +9,6 @@ import semmle.code.cpp.exprs.Call
 import semmle.code.cpp.metrics.MetricFunction
 import semmle.code.cpp.Linkage
 private import semmle.code.cpp.internal.ResolveClass
-private import semmle.code.cpp.internal.ResolveFunction
 
 /**
  * A C/C++ function [N4140 8.3.5]. Both member functions and non-member
@@ -26,8 +25,6 @@ private import semmle.code.cpp.internal.ResolveFunction
  * in more detail in `Declaration.qll`.
  */
 class Function extends Declaration, ControlFlowNode, AccessHolder, @function {
-  Function() { isFunction(underlyingElement(this)) }
-
   override string getName() { functions(underlyingElement(this), result, _) }
 
   /**

cpp/ql/lib/change-notes/2023-12-22-unique-function.md

@@ -0,0 +1,101 @@
+/**
+ * A library for detecting general string concatenations.
+ */
+
+import cpp
+import semmle.code.cpp.models.implementations.Strcat
+import semmle.code.cpp.models.interfaces.FormattingFunction
+private import semmle.code.cpp.dataflow.new.DataFlow
+
+/**
+ * A call that performs a string concatenation. A string can be either a C
+ * string (i.e., a value of type `char*`), or a C++ string (i.e., a value of
+ * type `std::string`).
+ */
+class StringConcatenation extends Call {
+  StringConcatenation() {
+    // sprintf-like functions, i.e., concat through formatting
+    this instanceof FormattingFunctionCall
+    or
+    this.getTarget() instanceof StrcatFunction
+    or
+    this.getTarget() instanceof StrlcatFunction
+    or
+    // operator+ and ostream (<<) concat
+    exists(Call call, Operator op |
+      call.getTarget() = op and
+      op.hasQualifiedName(["std", "bsl"], ["operator+", "operator<<"]) and
+      op.getType()
+          .stripType()
+          .(UserType)
+          .hasQualifiedName(["std", "bsl"], ["basic_string", "basic_ostream"]) and
+      this = call
+    )
+  }
+
+  /**
+   * Gets an operand of this concatenation (one of the string operands being
+   * concatenated).
+   * Will not return out param for sprintf-like functions, but will consider the format string
+   * to be part of the operands.
+   */
+  Expr getAnOperand() {
+    // The result is an argument of 'this' (a call)
+    result = this.getAnArgument() and
+    // addresses odd behavior with overloaded operators
+    // i.e., "call to operator+" appearing as an operand
+    // occurs in cases like `string s = s1 + s2 + s3`, which is represented as
+    // `string s = (s1.operator+(s2)).operator+(s3);`
+    // By limiting to non-calls we get the leaf operands (the variables or raw strings)
+    // also, by not enumerating allowed types (variables and strings) we avoid issues
+    // with missed corner cases or extensions/changes to CodeQL in the future which might
+    // invalidate that approach.
+    not result instanceof Call and
+    // Limit the result type to string
+    (
+      result.getUnderlyingType().stripType().getName() = "char"
+      or
+      result
+          .getType()
+          .getUnspecifiedType()
+          .(UserType)
+          .hasQualifiedName(["std", "bsl"], "basic_string")
+    ) and
+    // when 'this' is a `FormattingFunctionCall` the result must be the format string argument
+    // or one of the formatting arguments
+    (
+      this instanceof FormattingFunctionCall
+      implies
+      (
+        result = this.(FormattingFunctionCall).getFormat()
+        or
+        exists(int n |
+          result = this.getArgument(n) and
+          n >= this.(FormattingFunctionCall).getTarget().getFirstFormatArgumentIndex()
+        )
+      )
+    )
+  }
+
+  /**
+   * Gets the data flow node representing the concatenation result.
+   */
+  DataFlow::Node getResultNode() {
+    if this.getTarget() instanceof StrcatFunction
+    then
+      result.asDefiningArgument() =
+        this.getArgument(this.getTarget().(StrcatFunction).getParamDest())
+      or
+      // Hardcoding it is also the return
+      result.asExpr() = this.(Call)
+    else
+      if this.getTarget() instanceof StrlcatFunction
+      then (
+        result.asDefiningArgument() =
+          this.getArgument(this.getTarget().(StrlcatFunction).getParamDest())
+      ) else
+        if this instanceof FormattingFunctionCall
+        then result.asDefiningArgument() = this.(FormattingFunctionCall).getOutputArgument(_)
+        else result.asExpr() = this.(Call)
+  }
+}

cpp/ql/lib/change-notes/2024-01-02-function-types.md

@@ -54,18 +54,6 @@ private predicate functionSignature(Function f, string qualifiedName, int nparam
   not f.isStatic()
 }
 
-/**
- * Holds if the set of viable implementations that can be called by `call`
- * might be improved by knowing the call context.
- */
-predicate mayBenefitFromCallContext(DataFlowCall call, Function f) { none() }
-
-/**
- * Gets a viable dispatch target of `call` in the context `ctx`. This is
- * restricted to those `call`s for which a context might make a difference.
- */
-Function viableImplInCallContext(DataFlowCall call, DataFlowCall ctx) { none() }
-
 /** A parameter position represented by an integer. */
 class ParameterPosition extends int {
   ParameterPosition() { any(ParameterNode p).isParameterOf(_, this) }

cpp/ql/lib/change-notes/released/0.12.3.md

@@ -249,9 +249,7 @@ private predicate functionSignature(Function f, string qualifiedName, int nparam
  * Holds if the set of viable implementations that can be called by `call`
  * might be improved by knowing the call context.
  */
-predicate mayBenefitFromCallContext(DataFlowCall call, DataFlowCallable f) {
-  mayBenefitFromCallContext(call, f, _)
-}
+predicate mayBenefitFromCallContext(DataFlowCall call) { mayBenefitFromCallContext(call, _, _) }
 
 /**
  * Holds if `call` is a call through a function pointer, and the pointer

cpp/ql/lib/codeql-pack.release.yml

@@ -22,4 +22,8 @@ module CppDataFlow implements InputSig {
   predicate getAdditionalFlowIntoCallNodeTerm = Private::getAdditionalFlowIntoCallNodeTerm/2;
 
   predicate validParameterAliasStep = Private::validParameterAliasStep/2;
+
+  predicate mayBenefitFromCallContext = Private::mayBenefitFromCallContext/1;
+
+  predicate viableImplInCallContext = Private::viableImplInCallContext/2;
 }