Skip to content

Commit a875d29

Browse files
njhalenjhale
committed
test: add smoke test github workflow
1 parent 02a3cee commit a875d29

File tree

2 files changed

+168
-1
lines changed

2 files changed

+168
-1
lines changed

‎.github/workflows/smoke.yaml

+164
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
name: test
2+
3+
on:
4+
pull_request_target:
5+
types: [opened, synchronize, reopened]
6+
branches:
7+
- main
8+
- test/smoke
9+
push:
10+
branches:
11+
- main
12+
- test/smoke
13+
paths-ignore:
14+
- docs/**
15+
workflow_dispatch:
16+
17+
jobs:
18+
check-label:
19+
runs-on: ubuntu-22.04
20+
outputs:
21+
run_smoke_tests: ${{ steps.check.outputs.run_smoke_tests }}
22+
steps:
23+
- name: Check if PR author is a member of the organization or has the run-smoke label
24+
id: check
25+
run: |
26+
if [ "${{ github.event_name }}" == "push" ]; then
27+
echo "run_smoke_tests=true" >> $GITHUB_ENV
28+
exit 0
29+
fi
30+
31+
ORG="gptscript-ai"
32+
USERNAME="${{ github.event.pull_request.user.login }}"
33+
RESPONSE=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
34+
"https://api.github.com/orgs/$ORG/members/$USERNAME")
35+
36+
if [ "$RESPONSE" != "{}" ]; then
37+
echo "run_smoke_tests=true" >> $GITHUB_ENV
38+
exit 0
39+
fi
40+
41+
LABELS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
42+
"https://api.github.com/repos/${{ github.repository_owner }}/${{ github.event.repository.name }}/issues/${{ github.event.pull_request.number }}/labels" | jq -r '.[].name')
43+
44+
if echo "$LABELS" | grep -q "run-smoke"; then
45+
echo "run_smoke_tests=true" >> $GITHUB_ENV
46+
exit 0
47+
fi
48+
49+
echo "run_smoke_tests=false" >> $GITHUB_ENV
50+
51+
smoke-gpt-4o-2024-05-13:
52+
needs: check-label
53+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
54+
runs-on: ubuntu-22.04
55+
steps:
56+
- name: Checkout base repository
57+
uses: actions/checkout@v4
58+
with:
59+
fetch-depth: 1
60+
- name: Checkout PR code if running for a PR
61+
if: ${{ github.event_name == 'pull_request_target' }}
62+
uses: actions/checkout@v4
63+
with:
64+
fetch-depth: 1
65+
repository: ${{ github.event.pull_request.head.repo.full_name }}
66+
ref: ${{ github.event.pull_request.head.ref }}
67+
- uses: actions/setup-go@v5
68+
with:
69+
cache: false
70+
go-version: "1.21"
71+
- env:
72+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
73+
GPTSCRIPT_DEFAULT_MODEL: gpt-4o-2024-05-13
74+
name: Run smoke test for gpt-4o-2024-05-13
75+
run: |
76+
echo "Running smoke test for model gpt-4o-2024-05-13"
77+
make smoke
78+
79+
smoke-gpt-4-turbo-2024-04-09:
80+
needs: check-label
81+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
82+
runs-on: ubuntu-22.04
83+
steps:
84+
- name: Checkout base repository
85+
uses: actions/checkout@v4
86+
with:
87+
fetch-depth: 1
88+
- name: Checkout PR code if running for a PR
89+
if: ${{ github.event_name == 'pull_request_target' }}
90+
uses: actions/checkout@v4
91+
with:
92+
fetch-depth: 1
93+
repository: ${{ github.event.pull_request.head.repo.full_name }}
94+
ref: ${{ github.event.pull_request.head.ref }}
95+
- uses: actions/setup-go@v5
96+
with:
97+
cache: false
98+
go-version: "1.21"
99+
- env:
100+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
101+
GPTSCRIPT_DEFAULT_MODEL: gpt-4-turbo-2024-04-09
102+
name: Run smoke test for gpt-4-turbo-2024-04-09
103+
run: |
104+
echo "Running smoke test for model gpt-4-turbo-2024-04-09"
105+
make smoke
106+
107+
smoke-claude-3-opus-20240229:
108+
needs: check-label
109+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
110+
runs-on: ubuntu-22.04
111+
steps:
112+
- name: Checkout base repository
113+
uses: actions/checkout@v4
114+
with:
115+
fetch-depth: 1
116+
- name: Checkout PR code if running for a PR
117+
if: ${{ github.event_name == 'pull_request_target' }}
118+
uses: actions/checkout@v4
119+
with:
120+
fetch-depth: 1
121+
repository: ${{ github.event.pull_request.head.repo.full_name }}
122+
ref: ${{ github.event.pull_request.head.ref }}
123+
- uses: actions/setup-go@v5
124+
with:
125+
cache: false
126+
go-version: "1.21"
127+
- env:
128+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
129+
GPTSCRIPT_SMOKE_DEFAULT_MODEL: claude-3-opus-20240229 from github.com/gptscript-ai/claude3-anthropic-provider@tool-beta
130+
ANTHROPIC_API_KEY: ${{ secrets.SMOKE_ANTHROPIC_API_KEY }}
131+
name: Run smoke test for claude-3-opus-20240229
132+
run: |
133+
echo "Running smoke test for model claude-3-opus-20240229"
134+
make smoke
135+
136+
smoke-mistral-large-2402:
137+
needs: check-label
138+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
139+
runs-on: ubuntu-22.04
140+
steps:
141+
- name: Checkout base repository
142+
uses: actions/checkout@v4
143+
with:
144+
fetch-depth: 1
145+
- name: Checkout PR code if running for a PR
146+
if: ${{ github.event_name == 'pull_request_target' }}
147+
uses: actions/checkout@v4
148+
with:
149+
fetch-depth: 1
150+
repository: ${{ github.event.pull_request.head.repo.full_name }}
151+
ref: ${{ github.event.pull_request.head.ref }}
152+
- uses: actions/setup-go@v5
153+
with:
154+
cache: false
155+
go-version: "1.21"
156+
- env:
157+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
158+
GPTSCRIPT_DEFAULT_MODEL: mistral-large-2402 from https://api.mistral.ai/v1
159+
GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY: ${{ secrets.SMOKE_GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY }}
160+
name: Run smoke test for mistral-large-2402
161+
run: |
162+
echo "Running smoke test for model mistral-large-2402"
163+
make smoke
164+

‎Makefile

+4-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ tidy:
1414
test:
1515
go test -v ./...
1616

17+
smoke:
18+
go test -v -tags='smoke' ./pkg/tests/smoke/...
19+
1720
GOLANGCI_LINT_VERSION ?= v1.59.0
1821
lint:
1922
if ! command -v golangci-lint &> /dev/null; then \
@@ -52,4 +55,4 @@ validate-docs:
5255
echo "Encountered dirty repo!"; \
5356
git diff; \
5457
exit 1 \
55-
;fi
58+
;fi

0 commit comments

Comments
 (0)