Skip to content

Commit ae18d6a

Browse files
njhalenjhale
committed
test: add smoke test github workflow
1 parent 64d0748 commit ae18d6a

File tree

2 files changed

+172
-1
lines changed

2 files changed

+172
-1
lines changed

‎.github/workflows/smoke.yaml

+168
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
name: test
2+
3+
on:
4+
pull_request_target:
5+
types: [opened, synchronize, reopened]
6+
branches:
7+
- main
8+
- test/smoke
9+
push:
10+
branches:
11+
- main
12+
- test/smoke
13+
paths-ignore:
14+
- docs/**
15+
workflow_dispatch:
16+
17+
jobs:
18+
check-label:
19+
runs-on: ubuntu-22.04
20+
outputs:
21+
run_smoke_tests: ${{ steps.check.outputs.run_smoke_tests }}
22+
steps:
23+
- name: Check if PR author is a member of the organization or has the run-smoke label
24+
id: check
25+
run: |
26+
if [ "${{ github.event_name }}" == "push" ]; then
27+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
28+
exit 0
29+
fi
30+
31+
ORG="gptscript-ai"
32+
USERNAME="${{ github.event.pull_request.user.login }}"
33+
RESPONSE=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
34+
"https://api.github.com/orgs/$ORG/members/$USERNAME")
35+
36+
if [ "$RESPONSE" != "{}" ]; then
37+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
38+
exit 0
39+
fi
40+
41+
LABELS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
42+
"https://api.github.com/repos/${{ github.repository_owner }}/${{ github.event.repository.name }}/issues/${{ github.event.pull_request.number }}/labels" | jq -r '.[].name')
43+
44+
if echo "$LABELS" | grep -q "run-smoke"; then
45+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
46+
exit 0
47+
fi
48+
49+
echo "run_smoke_tests=false" >> $GITHUB_OUTPUT
50+
51+
smoke-gpt-4o-2024-05-13:
52+
needs: check-label
53+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
54+
runs-on: ubuntu-22.04
55+
steps:
56+
- name: Checkout base repository
57+
uses: actions/checkout@v4
58+
with:
59+
fetch-depth: 1
60+
- name: Checkout PR code if running for a PR
61+
if: ${{ github.event_name == 'pull_request_target' }}
62+
uses: actions/checkout@v4
63+
with:
64+
fetch-depth: 1
65+
repository: ${{ github.event.pull_request.head.repo.full_name }}
66+
ref: ${{ github.event.pull_request.head.ref }}
67+
- uses: actions/setup-go@v5
68+
with:
69+
cache: false
70+
go-version: "1.21"
71+
- env:
72+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
73+
GPTSCRIPT_DEFAULT_MODEL: gpt-4o-2024-05-13
74+
name: Run smoke test for gpt-4o-2024-05-13
75+
run: |
76+
echo "Running smoke test for model gpt-4o-2024-05-13"
77+
export PATH="$(pwd)/bin:${PATH}"
78+
make smoke
79+
80+
smoke-gpt-4-turbo-2024-04-09:
81+
needs: check-label
82+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
83+
runs-on: ubuntu-22.04
84+
steps:
85+
- name: Checkout base repository
86+
uses: actions/checkout@v4
87+
with:
88+
fetch-depth: 1
89+
- name: Checkout PR code if running for a PR
90+
if: ${{ github.event_name == 'pull_request_target' }}
91+
uses: actions/checkout@v4
92+
with:
93+
fetch-depth: 1
94+
repository: ${{ github.event.pull_request.head.repo.full_name }}
95+
ref: ${{ github.event.pull_request.head.ref }}
96+
- uses: actions/setup-go@v5
97+
with:
98+
cache: false
99+
go-version: "1.21"
100+
- env:
101+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
102+
GPTSCRIPT_DEFAULT_MODEL: gpt-4-turbo-2024-04-09
103+
name: Run smoke test for gpt-4-turbo-2024-04-09
104+
run: |
105+
echo "Running smoke test for model gpt-4-turbo-2024-04-09"
106+
export PATH="$(pwd)/bin:${PATH}"
107+
make smoke
108+
109+
smoke-claude-3-opus-20240229:
110+
needs: check-label
111+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
112+
runs-on: ubuntu-22.04
113+
steps:
114+
- name: Checkout base repository
115+
uses: actions/checkout@v4
116+
with:
117+
fetch-depth: 1
118+
- name: Checkout PR code if running for a PR
119+
if: ${{ github.event_name == 'pull_request_target' }}
120+
uses: actions/checkout@v4
121+
with:
122+
fetch-depth: 1
123+
repository: ${{ github.event.pull_request.head.repo.full_name }}
124+
ref: ${{ github.event.pull_request.head.ref }}
125+
- uses: actions/setup-go@v5
126+
with:
127+
cache: false
128+
go-version: "1.21"
129+
- env:
130+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
131+
GPTSCRIPT_DEFAULT_MODEL: claude-3-opus-20240229 from github.com/gptscript-ai/claude3-anthropic-provider@tool-beta
132+
ANTHROPIC_API_KEY: ${{ secrets.SMOKE_ANTHROPIC_API_KEY }}
133+
name: Run smoke test for claude-3-opus-20240229
134+
run: |
135+
echo "Running smoke test for model claude-3-opus-20240229"
136+
export PATH="$(pwd)/bin:${PATH}"
137+
make smoke
138+
139+
smoke-mistral-large-2402:
140+
needs: check-label
141+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
142+
runs-on: ubuntu-22.04
143+
steps:
144+
- name: Checkout base repository
145+
uses: actions/checkout@v4
146+
with:
147+
fetch-depth: 1
148+
- name: Checkout PR code if running for a PR
149+
if: ${{ github.event_name == 'pull_request_target' }}
150+
uses: actions/checkout@v4
151+
with:
152+
fetch-depth: 1
153+
repository: ${{ github.event.pull_request.head.repo.full_name }}
154+
ref: ${{ github.event.pull_request.head.ref }}
155+
- uses: actions/setup-go@v5
156+
with:
157+
cache: false
158+
go-version: "1.21"
159+
- env:
160+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
161+
GPTSCRIPT_DEFAULT_MODEL: mistral-large-2402 from https://api.mistral.ai/v1
162+
GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY: ${{ secrets.SMOKE_GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY }}
163+
name: Run smoke test for mistral-large-2402
164+
run: |
165+
echo "Running smoke test for model mistral-large-2402"
166+
export PATH="$(pwd)/bin:${PATH}"
167+
make smoke
168+

‎Makefile

+4-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ tidy:
1414
test:
1515
go test -v ./...
1616

17+
smoke:
18+
go test -v -tags='smoke' ./pkg/tests/smoke/...
19+
1720
GOLANGCI_LINT_VERSION ?= v1.59.0
1821
lint:
1922
if ! command -v golangci-lint &> /dev/null; then \
@@ -52,4 +55,4 @@ validate-docs:
5255
echo "Encountered dirty repo!"; \
5356
git diff; \
5457
exit 1 \
55-
;fi
58+
;fi

0 commit comments

Comments
 (0)