test: add smoke test runner and initial test cases

Signed-off-by: Nick Hale <[email protected]>

Author: njhale

go.mod

@@ -32,6 +32,7 @@ require (
 	golang.org/x/sync v0.7.0
 	golang.org/x/term v0.20.0
 	gopkg.in/yaml.v3 v3.0.1
+	gotest.tools/v3 v3.5.1
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -107,6 +108,5 @@ require (
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
-	gotest.tools/v3 v3.5.1 // indirect
 	mvdan.cc/gofumpt v0.6.0 // indirect
 )

pkg/tests/judge/judge.go

@@ -0,0 +1,127 @@
+package judge
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/getkin/kin-openapi/openapi3gen"
+	openai "github.com/gptscript-ai/chat-completion-client"
+)
+
+const instructions = `When given JSON objects that conform to the following JSONSchema:
+
+%s
+
+Determine if "actual" is equal to "expected" based on the comparison constraints described by "criteria".
+"actual" is considered equal to "expected" if and only if the all of the constraints described by "criteria" are satisfied.
+
+After making a determination, respond with a JSON object that conforms to the following JSONSchema:
+
+{
+  "name": "ruling",
+  "type": "object",
+  "properties": {
+    "equal": {
+      "type": "boolean",
+        "description": "Set to true if and only if actual is considered equal to expected."
+      },
+    "reasoning": {
+      "type": "string",
+      "description": "The reasoning used to come to the determination, that points out all instances where the given criteria was violated"
+    }
+  },
+  "required": [
+    "equal",
+    "reasoning"
+  ]
+}
+
+Your responses are concise and include only the json object described above.
+`
+
+type Judge[T any] struct {
+	client       *openai.Client
+	instructions string
+}
+
+type comparison[T any] struct {
+	Expected T      `json:"expected"`
+	Actual   T      `json:"actual"`
+	Criteria string `json:"criteria"`
+}
+
+type ruling struct {
+	Equal     bool   `json:"equal"`
+	Reasoning string `json:"reasoning"`
+}
+
+func New[T any](client *openai.Client) (*Judge[T], error) {
+	schema, err := openapi3gen.NewSchemaRefForValue(
+		new(comparison[T]),
+		nil,
+		openapi3gen.CreateComponentSchemas(
+			openapi3gen.ExportComponentSchemasOptions{
+				ExportComponentSchemas: true,
+				ExportGenerics:         false,
+			}),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate JSONSchema for %T: %w", new(T), err)
+	}
+
+	schemaJSON, err := json.MarshalIndent(schema, "", "    ")
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal JSONSchema for %T: %w", new(T), err)
+	}
+
+	return &Judge[T]{
+		client:       client,
+		instructions: fmt.Sprintf(instructions, schemaJSON),
+	}, nil
+}
+
+func (j *Judge[T]) Equal(ctx context.Context, expected, actual T, criteria string) (equal bool, reasoning string, err error) {
+	comparisonJSON, err := json.MarshalIndent(&comparison[T]{
+		Expected: expected,
+		Actual:   actual,
+		Criteria: criteria,
+	}, "", "    ")
+	if err != nil {
+		return false, "", fmt.Errorf("failed to marshal judge testcase JSON: %w", err)
+	}
+
+	request := openai.ChatCompletionRequest{
+		Model:       openai.GPT4o,
+		Temperature: new(float32),
+		N:           1,
+		ResponseFormat: &openai.ChatCompletionResponseFormat{
+			Type: openai.ChatCompletionResponseFormatTypeJSONObject,
+		},
+		Messages: []openai.ChatCompletionMessage{
+			{
+				Role:    openai.ChatMessageRoleSystem,
+				Content: j.instructions,
+			},
+			{
+				Role:    openai.ChatMessageRoleUser,
+				Content: string(comparisonJSON),
+			},
+		},
+	}
+	response, err := j.client.CreateChatCompletion(ctx, request)
+	if err != nil {
+		return false, "", fmt.Errorf("failed to make judge chat completion request: %w", err)
+	}
+
+	if len(response.Choices) < 1 {
+		return false, "", fmt.Errorf("judge chat completion request returned no choices")
+	}
+
+	var equality ruling
+	if err := json.Unmarshal([]byte(response.Choices[0].Message.Content), &equality); err != nil {
+		return false, "", fmt.Errorf("failed to unmarshal judge ruling: %w", err)
+	}
+
+	return equality.Equal, equality.Reasoning, nil
+}

pkg/tests/smoke/smoke_test.go

@@ -0,0 +1,181 @@
+//go:build smoke
+
+package smoke
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	openai "github.com/gptscript-ai/chat-completion-client"
+	"github.com/gptscript-ai/gptscript/pkg/runner"
+	"github.com/gptscript-ai/gptscript/pkg/tests/judge"
+	"github.com/gptscript-ai/gptscript/pkg/types"
+	"github.com/samber/lo"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gotest.tools/v3/icmd"
+)
+
+const defaultModelEnvVar = "GPTSCRIPT_DEFAULT_MODEL"
+
+func TestSmoke(t *testing.T) {
+	client := openai.NewClient(os.Getenv("OPENAI_API_KEY"))
+	smokeJudge, err := judge.New[[]event](client)
+	require.NoError(t, err, "error initializing smoke test judge")
+
+	for _, tc := range getTestcases(t) {
+		t.Run(tc.name, func(t *testing.T) {
+			cmd := icmd.Command(
+				"gptscript",
+				"--color=false",
+				"--disable-cache",
+				"--events-stream-to",
+				tc.actualEventsFile,
+				"--default-model",
+				tc.defaultModel,
+				tc.gptFile,
+			)
+
+			result := icmd.RunCmd(cmd)
+			defer func() {
+				t.Helper()
+				assert.NoError(t, os.Remove(tc.actualEventsFile))
+			}()
+
+			require.NoError(t, result.Error, "stderr: %q", result.Stderr())
+			require.Zero(t, result.ExitCode)
+
+			var (
+				actualEvents   = getActualEvents(t, tc.actualEventsFile)
+				expectedEvents = make([]event, 0)
+			)
+			f, err := os.Open(tc.expectedEventsFile)
+			if os.IsNotExist(err) {
+				// No expected events found, store the results of the latest call as the golden file for future tests runs
+				f, err := os.Create(tc.expectedEventsFile)
+				require.NoError(t, err)
+				defer f.Close()
+
+				encoder := json.NewEncoder(f)
+				encoder.SetIndent("", "    ")
+				require.NoError(t, encoder.Encode(actualEvents))
+				t.Skipf("Generated initial golden file %q, skipping test", tc.expectedEventsFile)
+			} else {
+				require.NoError(t, err)
+				defer f.Close()
+
+				decoder := json.NewDecoder(f)
+				require.NoError(t, decoder.Decode(&expectedEvents))
+			}
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			equal, reasoning, err := smokeJudge.Equal(
+				ctx,
+				expectedEvents,
+				actualEvents,
+				`Actual and expected must have semantically equivalent elements.
+Ignore fields with timestamp values, except in json strings.
+The final elements must have semantically equivalent output values.
+`,
+			)
+			require.NoError(t, err, "error getting judge ruling on output")
+			require.True(t, equal, reasoning)
+			t.Logf("reasoning: %q", reasoning)
+		})
+	}
+}
+
+type testcase struct {
+	name               string
+	dir                string
+	gptFile            string
+	defaultModel       string
+	modelName          string
+	env                []string
+	actualEventsFile   string
+	expectedEventsFile string
+}
+
+func getTestcases(t *testing.T) []testcase {
+	t.Helper()
+
+	defaultModel := os.Getenv(defaultModelEnvVar)
+	modelName := strings.Split(defaultModel, " ")[0]
+
+	var testcases []testcase
+	for _, d := range lo.Must(os.ReadDir("testdata")) {
+		if !d.IsDir() {
+			continue
+		}
+		var (
+			dirName = d.Name()
+			dir     = filepath.Join("testdata", dirName)
+		)
+
+		files, err := os.ReadDir(dir)
+		require.NoError(t, err, "failed to get testdata dir %q", dir)
+
+		for _, f := range files {
+			if f.IsDir() || filepath.Ext(f.Name()) != ".gpt" {
+				continue
+			}
+
+			testcases = append(testcases, testcase{
+				name:               dirName,
+				dir:                dir,
+				gptFile:            filepath.Join(dir, f.Name()),
+				defaultModel:       defaultModel,
+				modelName:          modelName,
+				expectedEventsFile: filepath.Join(dir, fmt.Sprintf("%s-expected.json", modelName)),
+				actualEventsFile:   filepath.Join(dir, fmt.Sprintf("%s.json", modelName)),
+			})
+
+			// Only take the first .gpt file in each testcase directory
+			break
+		}
+	}
+
+	return testcases
+}
+
+type event struct {
+	runner.Event
+	ChatRequest  *openai.ChatCompletionRequest `json:"chatRequest,omitempty"`
+	ChatResponse *types.CompletionMessage      `json:"chatResponse,omitempty"`
+}
+
+func getActualEvents(t *testing.T, eventsFile string) []event {
+	t.Helper()
+
+	f, err := os.Open(eventsFile)
+	require.NoError(t, err)
+	defer f.Close()
+
+	var (
+		events  []event
+		scanner = bufio.NewScanner(f)
+	)
+	for scanner.Scan() {
+		line := scanner.Text()
+		// Skip blank lines
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+
+		var e event
+		require.NoError(t, json.Unmarshal([]byte(line), &e))
+		events = append(events, e)
+	}
+
+	require.NoError(t, scanner.Err())
+
+	return events
+}

pkg/tests/smoke/testdata/.gitignore

@@ -0,0 +1,3 @@
+# Ignore intermediate event stream JSON files
+*-events.json
+