Skip to content

Commit faf730a

Browse files
njhalenjhale
committed
test: add smoke test github workflow
1 parent 9ee00d4 commit faf730a

File tree

2 files changed

+184
-1
lines changed

2 files changed

+184
-1
lines changed

‎.github/workflows/smoke.yaml

+180
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
name: test
2+
3+
on:
4+
pull_request_target:
5+
types: [opened, synchronize, reopened]
6+
branches:
7+
- main
8+
push:
9+
branches:
10+
- main
11+
- test/smoke
12+
paths-ignore:
13+
- docs/**
14+
workflow_dispatch:
15+
16+
jobs:
17+
check-label:
18+
runs-on: ubuntu-22.04
19+
outputs:
20+
run_smoke_tests: ${{ steps.check.outputs.run_smoke_tests }}
21+
steps:
22+
- name: Check if PR author is a member of the organization or has the run-smoke label
23+
id: check
24+
run: |
25+
case "${{ github.event_name }}" in
26+
push)
27+
# Run smoke tests for push to base repo
28+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
29+
exit 0
30+
;;
31+
workflow_dispatch)
32+
# Run smoke tests for manual runs against base branch
33+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
34+
exit 0
35+
;;
36+
pull_request_target)
37+
ORG="gptscript-ai"
38+
AUTHOR="${{ github.event.pull_request.user.login }}"
39+
40+
# Check for org membership
41+
MEMBERSHIP_RESPONSE_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
42+
"https://api.github.com/orgs/$ORG/members/$AUTHOR")
43+
44+
if [ "$MEMBERSHIP_RESPONSE_CODE" -eq 204 ]; then
45+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
46+
exit 0
47+
fi
48+
49+
# Check for "run-smoke" label
50+
LABELS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
51+
"https://api.github.com/repos/${{ github.repository_owner }}/${{ github.event.repository.name }}/issues/${{ github.event.pull_request.number }}/labels" | jq -r '.[].name')
52+
if echo "$LABELS" | grep -q "run-smoke"; then
53+
# Run smoke tests for PR with the "run-smoke" label
54+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
55+
exit 0
56+
fi
57+
58+
;;
59+
esac
60+
61+
echo "run_smoke_tests=false" >> $GITHUB_OUTPUT
62+
63+
smoke-gpt-4o-2024-05-13:
64+
needs: check-label
65+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
66+
runs-on: ubuntu-22.04
67+
steps:
68+
- name: Checkout base repository
69+
uses: actions/checkout@v4
70+
with:
71+
fetch-depth: 1
72+
- name: Checkout PR code if running for a PR
73+
if: ${{ github.event_name == 'pull_request_target' }}
74+
uses: actions/checkout@v4
75+
with:
76+
fetch-depth: 1
77+
repository: ${{ github.event.pull_request.head.repo.full_name }}
78+
ref: ${{ github.event.pull_request.head.ref }}
79+
- uses: actions/setup-go@v5
80+
with:
81+
cache: false
82+
go-version: "1.21"
83+
- env:
84+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
85+
GPTSCRIPT_DEFAULT_MODEL: gpt-4o-2024-05-13
86+
name: Run smoke test for gpt-4o-2024-05-13
87+
run: |
88+
echo "Running smoke test for model gpt-4o-2024-05-13"
89+
export PATH="$(pwd)/bin:${PATH}"
90+
make smoke
91+
92+
smoke-gpt-4-turbo-2024-04-09:
93+
needs: check-label
94+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
95+
runs-on: ubuntu-22.04
96+
steps:
97+
- name: Checkout base repository
98+
uses: actions/checkout@v4
99+
with:
100+
fetch-depth: 1
101+
- name: Checkout PR code if running for a PR
102+
if: ${{ github.event_name == 'pull_request_target' }}
103+
uses: actions/checkout@v4
104+
with:
105+
fetch-depth: 1
106+
repository: ${{ github.event.pull_request.head.repo.full_name }}
107+
ref: ${{ github.event.pull_request.head.ref }}
108+
- uses: actions/setup-go@v5
109+
with:
110+
cache: false
111+
go-version: "1.21"
112+
- env:
113+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
114+
GPTSCRIPT_DEFAULT_MODEL: gpt-4-turbo-2024-04-09
115+
name: Run smoke test for gpt-4-turbo-2024-04-09
116+
run: |
117+
echo "Running smoke test for model gpt-4-turbo-2024-04-09"
118+
export PATH="$(pwd)/bin:${PATH}"
119+
make smoke
120+
121+
smoke-claude-3-opus-20240229:
122+
needs: check-label
123+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
124+
runs-on: ubuntu-22.04
125+
steps:
126+
- name: Checkout base repository
127+
uses: actions/checkout@v4
128+
with:
129+
fetch-depth: 1
130+
- name: Checkout PR code if running for a PR
131+
if: ${{ github.event_name == 'pull_request_target' }}
132+
uses: actions/checkout@v4
133+
with:
134+
fetch-depth: 1
135+
repository: ${{ github.event.pull_request.head.repo.full_name }}
136+
ref: ${{ github.event.pull_request.head.ref }}
137+
- uses: actions/setup-go@v5
138+
with:
139+
cache: false
140+
go-version: "1.21"
141+
- env:
142+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
143+
GPTSCRIPT_DEFAULT_MODEL: claude-3-opus-20240229 from github.com/gptscript-ai/claude3-anthropic-provider@tool-beta
144+
ANTHROPIC_API_KEY: ${{ secrets.SMOKE_ANTHROPIC_API_KEY }}
145+
name: Run smoke test for claude-3-opus-20240229
146+
run: |
147+
echo "Running smoke test for model claude-3-opus-20240229"
148+
export PATH="$(pwd)/bin:${PATH}"
149+
make smoke
150+
151+
smoke-mistral-large-2402:
152+
needs: check-label
153+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
154+
runs-on: ubuntu-22.04
155+
steps:
156+
- name: Checkout base repository
157+
uses: actions/checkout@v4
158+
with:
159+
fetch-depth: 1
160+
- name: Checkout PR code if running for a PR
161+
if: ${{ github.event_name == 'pull_request_target' }}
162+
uses: actions/checkout@v4
163+
with:
164+
fetch-depth: 1
165+
repository: ${{ github.event.pull_request.head.repo.full_name }}
166+
ref: ${{ github.event.pull_request.head.ref }}
167+
- uses: actions/setup-go@v5
168+
with:
169+
cache: false
170+
go-version: "1.21"
171+
- env:
172+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
173+
GPTSCRIPT_DEFAULT_MODEL: mistral-large-2402 from https://api.mistral.ai/v1
174+
GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY: ${{ secrets.SMOKE_GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY }}
175+
name: Run smoke test for mistral-large-2402
176+
run: |
177+
echo "Running smoke test for model mistral-large-2402"
178+
export PATH="$(pwd)/bin:${PATH}"
179+
make smoke
180+

‎Makefile

+4-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ tidy:
1414
test:
1515
go test -v ./...
1616

17+
smoke:
18+
go test -v -tags='smoke' ./pkg/tests/smoke/...
19+
1720
GOLANGCI_LINT_VERSION ?= v1.59.0
1821
lint:
1922
if ! command -v golangci-lint &> /dev/null; then \
@@ -52,4 +55,4 @@ validate-docs:
5255
echo "Encountered dirty repo!"; \
5356
git diff; \
5457
exit 1 \
55-
;fi
58+
;fi

0 commit comments

Comments
 (0)