Skip to content

Commit c70a2be

Browse files
erik-krogherik-krogh
authored
Merge pull request #7410 from erik-krogh/erik-krogh/publish-ql-for-ql
Add QL for QL
2 parents 3adc0b5 + 2626b0b commit c70a2be

File tree

145 files changed

+40880
-1
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

145 files changed

+40880
-1
lines changed

‎.codeqlmanifest.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
"misc/legacy-support/*/qlpack.yml",
1212
"misc/suite-helpers/qlpack.yml",
1313
"ruby/extractor-pack/codeql-extractor.yml",
14-
"ruby/ql/consistency-queries/qlpack.yml"
14+
"ruby/ql/consistency-queries/qlpack.yml",
15+
"ql/ql/consistency-queries/qlpack.yml",
16+
"ql/extractor-pack/codeql-extractor.yml"
1517
],
1618
"versionPolicies": {
1719
"default": {

‎.github/workflows/ql-for-ql-build.yml

+152
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
name: Run QL for QL
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
9+
env:
10+
CARGO_TERM_COLOR: always
11+
12+
jobs:
13+
queries:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v2
17+
- name: Find codeql
18+
id: find-codeql
19+
uses: github/codeql-action/init@esbena/ql
20+
with:
21+
languages: javascript # does not matter
22+
- name: Get CodeQL version
23+
id: get-codeql-version
24+
run: |
25+
echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
26+
shell: bash
27+
env:
28+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
29+
- name: Cache queries
30+
id: cache-queries
31+
uses: actions/cache@v2
32+
with:
33+
path: ${{ runner.temp }}/query-pack.zip
34+
key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
35+
- name: Build query pack
36+
if: steps.cache-queries.outputs.cache-hit != 'true'
37+
run: |
38+
cd ql/ql/src
39+
"${CODEQL}" pack create
40+
cd .codeql/pack/codeql/ql-all/0.0.0
41+
zip "${PACKZIP}" -r .
42+
env:
43+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
44+
PACKZIP: ${{ runner.temp }}/query-pack.zip
45+
- name: Upload query pack
46+
uses: actions/upload-artifact@v2
47+
with:
48+
name: query-pack-zip
49+
path: ${{ runner.temp }}/query-pack.zip
50+
51+
extractors:
52+
strategy:
53+
fail-fast: false
54+
55+
runs-on: ubuntu-latest
56+
57+
steps:
58+
- uses: actions/checkout@v2
59+
- uses: actions/cache@v2
60+
with:
61+
path: |
62+
~/.cargo/registry
63+
~/.cargo/git
64+
ql/target
65+
key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
66+
- name: Check formatting
67+
run: cd ql; cargo fmt --all -- --check
68+
- name: Build
69+
run: cd ql; cargo build --verbose
70+
- name: Run tests
71+
run: cd ql; cargo test --verbose
72+
- name: Release build
73+
run: cd ql; cargo build --release
74+
- name: Generate dbscheme
75+
run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
76+
- uses: actions/upload-artifact@v2
77+
with:
78+
name: extractor-ubuntu-latest
79+
path: |
80+
ql/target/release/ql-extractor
81+
ql/target/release/ql-extractor.exe
82+
retention-days: 1
83+
package:
84+
runs-on: ubuntu-latest
85+
86+
needs:
87+
- extractors
88+
- queries
89+
90+
steps:
91+
- uses: actions/checkout@v2
92+
- uses: actions/download-artifact@v2
93+
with:
94+
name: query-pack-zip
95+
path: query-pack-zip
96+
- uses: actions/download-artifact@v2
97+
with:
98+
name: extractor-ubuntu-latest
99+
path: linux64
100+
- run: |
101+
unzip query-pack-zip/*.zip -d pack
102+
cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
103+
mkdir -p pack/tools/linux64
104+
if [[ -f linux64/ql-extractor ]]; then
105+
cp linux64/ql-extractor pack/tools/linux64/extractor
106+
chmod +x pack/tools/linux64/extractor
107+
fi
108+
cd pack
109+
zip -rq ../codeql-ql.zip .
110+
- uses: actions/upload-artifact@v2
111+
with:
112+
name: codeql-ql-pack
113+
path: codeql-ql.zip
114+
retention-days: 1
115+
analyze:
116+
runs-on: ubuntu-latest
117+
118+
needs:
119+
- package
120+
121+
steps:
122+
- name: Download pack
123+
uses: actions/download-artifact@v2
124+
with:
125+
name: codeql-ql-pack
126+
path: ${{ runner.temp }}/codeql-ql-pack-artifact
127+
128+
- name: Prepare pack
129+
run: |
130+
unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
131+
env:
132+
PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
133+
PACK: ${{ runner.temp }}/pack
134+
- name: Hack codeql-action options
135+
run: |
136+
JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
137+
echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
138+
env:
139+
PACK: ${{ runner.temp }}/pack
140+
141+
- name: Checkout repository
142+
uses: actions/checkout@v2
143+
144+
- name: Initialize CodeQL
145+
uses: github/codeql-action/init@esbena/ql
146+
with:
147+
languages: ql
148+
db-location: ${{ runner.temp }}/db
149+
150+
- name: Perform CodeQL Analysis
151+
uses: github/codeql-action/analyze@esbena/ql
152+

‎.github/workflows/ql-for-ql-dataset_measure.yml

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: Collect database stats for QL for QL
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- ql/ql/src/ql.dbscheme
8+
pull_request:
9+
branches: [main]
10+
paths:
11+
- ql/ql/src/ql.dbscheme
12+
workflow_dispatch:
13+
14+
jobs:
15+
measure:
16+
env:
17+
CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
18+
strategy:
19+
matrix:
20+
repo:
21+
- github/codeql
22+
- github/codeql-go
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v2
26+
27+
- name: Find codeql
28+
id: find-codeql
29+
uses: github/codeql-action/init@esbena/ql
30+
with:
31+
languages: javascript # does not matter
32+
- uses: actions/cache@v2
33+
with:
34+
path: |
35+
~/.cargo/registry
36+
~/.cargo/git
37+
ql/target
38+
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
39+
- name: Build Extractor
40+
run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
41+
env:
42+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
43+
- name: Checkout ${{ matrix.repo }}
44+
uses: actions/checkout@v2
45+
with:
46+
repository: ${{ matrix.repo }}
47+
path: ${{ github.workspace }}/repo
48+
- name: Create database
49+
run: |
50+
"${CODEQL}" database create \
51+
--search-path "ql/extractor-pack" \
52+
--threads 4 \
53+
--language ql --source-root "${{ github.workspace }}/repo" \
54+
"${{ runner.temp }}/database"
55+
env:
56+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
57+
- name: Measure database
58+
run: |
59+
mkdir -p "stats/${{ matrix.repo }}"
60+
"${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
61+
env:
62+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
63+
- uses: actions/upload-artifact@v2
64+
with:
65+
name: measurements
66+
path: stats
67+
retention-days: 1
68+
69+
merge:
70+
runs-on: ubuntu-latest
71+
needs: measure
72+
steps:
73+
- uses: actions/checkout@v2
74+
- uses: actions/download-artifact@v2
75+
with:
76+
name: measurements
77+
path: stats
78+
- run: |
79+
python -m pip install --user lxml
80+
find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ql/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
81+
- uses: actions/upload-artifact@v2
82+
with:
83+
name: ql.dbscheme.stats
84+
path: ql/ql/src/ql.dbscheme.stats

‎.github/workflows/ql-for-ql-tests.yml

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Run QL for QL Tests
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- ql/*
8+
pull_request:
9+
branches: [main]
10+
paths:
11+
- ql/*
12+
13+
env:
14+
CARGO_TERM_COLOR: always
15+
16+
jobs:
17+
qltest:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v2
21+
- name: Find codeql
22+
id: find-codeql
23+
uses: github/codeql-action/init@esbena/ql
24+
with:
25+
languages: javascript # does not matter
26+
- uses: actions/cache@v2
27+
with:
28+
path: |
29+
~/.cargo/registry
30+
~/.cargo/git
31+
ql/target
32+
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
33+
- name: Build extractor
34+
run: |
35+
cd ql;
36+
codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
37+
env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
38+
- name: Run QL tests
39+
run: |
40+
"${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
41+
env:
42+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
43+
- name: Check QL formatting
44+
run: |
45+
find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
46+
env:
47+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
48+
- name: Check QL compilation
49+
run: |
50+
"${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
51+
env:
52+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

‎CODEOWNERS

+3
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,6 @@
2525
/docs/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
2626
/docs/ql-language-reference/ @github/codeql-frontend-reviewers
2727
/docs/query-*-style-guide.md @github/codeql-analysis-reviewers
28+
29+
# QL for QL reviewers
30+
/ql/ @erik-krogh @tausbn

‎ql/.gitattributes

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Cargo.lock -diff -whitespace

‎ql/.gitignore

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
target
2+
extractor-pack
3+
.vscode/launch.json
4+
.cache
5+
ql/test/**/*.testproj
6+
ql/test/**/*.actual
7+
ql/test/**/CONSISTENCY
8+
work

‎ql/.vscode/tasks.json

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"version": "2.0.0",
3+
"tasks": [
4+
{
5+
"type": "cargo",
6+
"subcommand": "build",
7+
"problemMatcher": [
8+
"$rustc"
9+
],
10+
"group": "build",
11+
"label": "Rust: cargo build"
12+
}
13+
]
14+
}

0 commit comments

Comments
 (0)