Skip to content

Commit cb6d8b8

Browse files
ibuildthecloudibuildthecloud
committed
feat: manage node/python runtime for local files in dev
1 parent a383d4f commit cb6d8b8

17 files changed

+558
-24
lines changed

‎pkg/repos/get.go

+16-2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ type Runtime interface {
2828
ID() string
2929
Supports(tool types.Tool, cmd []string) bool
3030
Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error)
31+
GetHash(tool types.Tool) (string, error)
3132
}
3233

3334
type noopRuntime struct {
@@ -37,6 +38,10 @@ func (n noopRuntime) ID() string {
3738
return "none"
3839
}
3940

41+
func (n noopRuntime) GetHash(_ types.Tool) (string, error) {
42+
return "", nil
43+
}
44+
4045
func (n noopRuntime) Supports(_ types.Tool, _ []string) bool {
4146
return false
4247
}
@@ -183,8 +188,13 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
183188
locker.Lock(tool.ID)
184189
defer locker.Unlock(tool.ID)
185190

191+
runtimeHash, err := runtime.GetHash(tool)
192+
if err != nil {
193+
return "", nil, err
194+
}
195+
186196
target := filepath.Join(m.storageDir, tool.Source.Repo.Revision, tool.Source.Repo.Path, tool.Source.Repo.Name, runtime.ID())
187-
targetFinal := filepath.Join(target, tool.Source.Repo.Path)
197+
targetFinal := filepath.Join(target, tool.Source.Repo.Path+runtimeHash)
188198
doneFile := targetFinal + ".done"
189199
envData, err := os.ReadFile(doneFile)
190200
if err == nil {
@@ -251,7 +261,11 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st
251261
for _, runtime := range m.runtimes {
252262
if runtime.Supports(tool, cmd) {
253263
log.Debugf("Runtime %s supports %v", runtime.ID(), cmd)
254-
return m.setup(ctx, runtime, tool, env)
264+
wd, env, err := m.setup(ctx, runtime, tool, env)
265+
if isLocal {
266+
wd = tool.WorkingDir
267+
}
268+
return wd, env, err
255269
}
256270
}
257271

‎pkg/repos/runtimes/busybox/busybox.go

+4
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ func (r *Runtime) ID() string {
3333
return "busybox"
3434
}
3535

36+
func (r *Runtime) GetHash(_ types.Tool) (string, error) {
37+
return "", nil
38+
}
39+
3640
func (r *Runtime) Supports(_ types.Tool, cmd []string) bool {
3741
if runtime.GOOS != "windows" {
3842
return false

‎pkg/repos/runtimes/golang/golang.go

+4
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ func (r *Runtime) ID() string {
3535
return "go" + r.Version
3636
}
3737

38+
func (r *Runtime) GetHash(_ types.Tool) (string, error) {
39+
return "", nil
40+
}
41+
3842
func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
3943
return tool.Source.IsGit() &&
4044
len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool"

‎pkg/repos/runtimes/node/node.go

+22-4
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,7 @@ func (r *Runtime) ID() string {
3939
return "node" + r.Version
4040
}
4141

42-
func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
43-
if _, hasPackageJSON := tool.MetaData[packageJSON]; !hasPackageJSON && !tool.Source.IsGit() {
44-
return false
45-
}
42+
func (r *Runtime) Supports(_ types.Tool, cmd []string) bool {
4643
for _, testCmd := range []string{"node", "npx", "npm"} {
4744
if r.supports(testCmd, cmd) {
4845
return true
@@ -61,6 +58,15 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool {
6158
return runtimeEnv.Matches(cmd, testCmd)
6259
}
6360

61+
func (r *Runtime) GetHash(tool types.Tool) (string, error) {
62+
if !tool.Source.IsGit() && tool.WorkingDir != "" {
63+
if s, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); err == nil {
64+
return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil
65+
}
66+
}
67+
return "", nil
68+
}
69+
6470
func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) {
6571
binPath, err := r.getRuntime(ctx, dataRoot)
6672
if err != nil {
@@ -74,6 +80,8 @@ func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSour
7480

7581
if _, ok := tool.MetaData[packageJSON]; ok {
7682
newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+toolSource)
83+
} else if !tool.Source.IsGit() && tool.WorkingDir != "" {
84+
newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+tool.WorkingDir, "GPTSCRIPT_RUNTIME_DEV=true")
7785
}
7886

7987
return newEnv, nil
@@ -120,6 +128,16 @@ func (r *Runtime) runNPM(ctx context.Context, tool types.Tool, toolSource, binDi
120128
if err := os.WriteFile(filepath.Join(toolSource, packageJSON), []byte(contents+"\n"), 0644); err != nil {
121129
return err
122130
}
131+
} else if !tool.Source.IsGit() {
132+
if tool.WorkingDir == "" {
133+
return nil
134+
}
135+
if _, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); errors.Is(fs.ErrNotExist, err) {
136+
return nil
137+
} else if err != nil {
138+
return err
139+
}
140+
cmd.Dir = tool.WorkingDir
123141
}
124142
return cmd.Run()
125143
}

‎pkg/repos/runtimes/python/python.go

+36-18
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@ import (
2424
var releasesData []byte
2525

2626
const (
27-
uvVersion = "uv==0.2.33"
28-
requirementsTxt = "requirements.txt"
27+
uvVersion = "uv==0.2.33"
28+
requirementsTxt = "requirements.txt"
29+
gptscriptRequirementsTxt = "requirements-gptscript.txt"
2930
)
3031

3132
type Release struct {
@@ -47,10 +48,7 @@ func (r *Runtime) ID() string {
4748
return "python" + r.Version
4849
}
4950

50-
func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
51-
if _, hasRequirements := tool.MetaData[requirementsTxt]; !hasRequirements && !tool.Source.IsGit() {
52-
return false
53-
}
51+
func (r *Runtime) Supports(_ types.Tool, cmd []string) bool {
5452
if runtimeEnv.Matches(cmd, r.ID()) {
5553
return true
5654
}
@@ -152,11 +150,7 @@ func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSour
152150
}
153151
}
154152

155-
if err := r.runPip(ctx, tool, toolSource, binPath, append(env, newEnv...)); err != nil {
156-
return nil, err
157-
}
158-
159-
return newEnv, nil
153+
return r.runPip(ctx, tool, toolSource, binPath, append(env, newEnv...))
160154
}
161155

162156
func readRelease() (result []Release) {
@@ -177,28 +171,52 @@ func (r *Runtime) getReleaseAndDigest() (string, string, error) {
177171
return "", "", fmt.Errorf("failed to find an python runtime for %s", r.Version)
178172
}
179173

180-
func (r *Runtime) runPip(ctx context.Context, tool types.Tool, toolSource, binDir string, env []string) error {
174+
func (r *Runtime) GetHash(tool types.Tool) (string, error) {
175+
if !tool.Source.IsGit() && tool.WorkingDir != "" {
176+
if _, ok := tool.MetaData[requirementsTxt]; ok {
177+
return "", nil
178+
}
179+
for _, req := range []string{gptscriptRequirementsTxt, requirementsTxt} {
180+
reqFile := filepath.Join(tool.WorkingDir, req)
181+
if s, err := os.Stat(reqFile); err == nil && !s.IsDir() {
182+
return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil
183+
}
184+
}
185+
}
186+
187+
return "", nil
188+
}
189+
190+
func (r *Runtime) runPip(ctx context.Context, tool types.Tool, toolSource, binDir string, env []string) ([]string, error) {
181191
log.InfofCtx(ctx, "Running pip in %s", toolSource)
182192
if content, ok := tool.MetaData[requirementsTxt]; ok {
183193
reqFile := filepath.Join(toolSource, requirementsTxt)
184194
if err := os.WriteFile(reqFile, []byte(content+"\n"), 0644); err != nil {
185-
return err
195+
return nil, err
186196
}
187197
cmd := debugcmd.New(ctx, uvBin(binDir), "pip", "install", "-r", reqFile)
188198
cmd.Env = env
189-
return cmd.Run()
199+
return env, cmd.Run()
190200
}
191201

192-
for _, req := range []string{"requirements-gptscript.txt", requirementsTxt} {
193-
reqFile := filepath.Join(toolSource, req)
202+
reqPath := toolSource
203+
if !tool.Source.IsGit() {
204+
if tool.WorkingDir == "" {
205+
return env, nil
206+
}
207+
reqPath = tool.WorkingDir
208+
}
209+
210+
for _, req := range []string{gptscriptRequirementsTxt, requirementsTxt} {
211+
reqFile := filepath.Join(reqPath, req)
194212
if s, err := os.Stat(reqFile); err == nil && !s.IsDir() {
195213
cmd := debugcmd.New(ctx, uvBin(binDir), "pip", "install", "-r", reqFile)
196214
cmd.Env = env
197-
return cmd.Run()
215+
return env, cmd.Run()
198216
}
199217
}
200218

201-
return nil
219+
return env, nil
202220
}
203221

204222
func (r *Runtime) setupUV(ctx context.Context, tmp string) error {

‎pkg/tests/runner_test.go

+23
Original file line numberDiff line numberDiff line change
@@ -1018,3 +1018,26 @@ func TestRuntimes(t *testing.T) {
10181018
})
10191019
r.RunDefault()
10201020
}
1021+
1022+
func TestRuntimesLocalDev(t *testing.T) {
1023+
r := tester.NewRunner(t)
1024+
r.RespondWith(tester.Result{
1025+
Func: types.CompletionFunctionCall{
1026+
Name: "py",
1027+
Arguments: "{}",
1028+
},
1029+
}, tester.Result{
1030+
Func: types.CompletionFunctionCall{
1031+
Name: "node",
1032+
Arguments: "{}",
1033+
},
1034+
}, tester.Result{
1035+
Func: types.CompletionFunctionCall{
1036+
Name: "bash",
1037+
Arguments: "{}",
1038+
},
1039+
})
1040+
r.RunDefault()
1041+
_ = os.RemoveAll("testdata/TestRuntimesLocalDev/node_modules")
1042+
_ = os.RemoveAll("testdata/TestRuntimesLocalDev/package-lock.json")
1043+
}

‎pkg/tests/testdata/TestRuntimesLocalDev/call1-resp.golden

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
`{
2+
"role": "assistant",
3+
"content": [
4+
{
5+
"toolCall": {
6+
"index": 0,
7+
"id": "call_1",
8+
"function": {
9+
"name": "py",
10+
"arguments": "{}"
11+
}
12+
}
13+
}
14+
],
15+
"usage": {}
16+
}`

‎pkg/tests/testdata/TestRuntimesLocalDev/call1.golden

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
`{
2+
"model": "gpt-4o",
3+
"tools": [
4+
{
5+
"function": {
6+
"toolID": "testdata/TestRuntimesLocalDev/test.gpt:py",
7+
"name": "py",
8+
"parameters": null
9+
}
10+
},
11+
{
12+
"function": {
13+
"toolID": "testdata/TestRuntimesLocalDev/test.gpt:node",
14+
"name": "node",
15+
"parameters": null
16+
}
17+
},
18+
{
19+
"function": {
20+
"toolID": "testdata/TestRuntimesLocalDev/test.gpt:bash",
21+
"name": "bash",
22+
"parameters": null
23+
}
24+
}
25+
],
26+
"messages": [
27+
{
28+
"role": "system",
29+
"content": [
30+
{
31+
"text": "Dummy"
32+
}
33+
],
34+
"usage": {}
35+
}
36+
]
37+
}`

‎pkg/tests/testdata/TestRuntimesLocalDev/call2-resp.golden

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
`{
2+
"role": "assistant",
3+
"content": [
4+
{
5+
"toolCall": {
6+
"index": 1,
7+
"id": "call_2",
8+
"function": {
9+
"name": "node",
10+
"arguments": "{}"
11+
}
12+
}
13+
}
14+
],
15+
"usage": {}
16+
}`

‎pkg/tests/testdata/TestRuntimesLocalDev/call2.golden

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
`{
2+
"model": "gpt-4o",
3+
"tools": [
4+
{
5+
"function": {
6+
"toolID": "testdata/TestRuntimesLocalDev/test.gpt:py",
7+
"name": "py",
8+
"parameters": null
9+
}
10+
},
11+
{
12+
"function": {
13+
"toolID": "testdata/TestRuntimesLocalDev/test.gpt:node",
14+
"name": "node",
15+
"parameters": null
16+
}
17+
},
18+
{
19+
"function": {
20+
"toolID": "testdata/TestRuntimesLocalDev/test.gpt:bash",
21+
"name": "bash",
22+
"parameters": null
23+
}
24+
}
25+
],
26+
"messages": [
27+
{
28+
"role": "system",
29+
"content": [
30+
{
31+
"text": "Dummy"
32+
}
33+
],
34+
"usage": {}
35+
},
36+
{
37+
"role": "assistant",
38+
"content": [
39+
{
40+
"toolCall": {
41+
"index": 0,
42+
"id": "call_1",
43+
"function": {
44+
"name": "py",
45+
"arguments": "{}"
46+
}
47+
}
48+
}
49+
],
50+
"usage": {}
51+
},
52+
{
53+
"role": "tool",
54+
"content": [
55+
{
56+
"text": "py worked\r\n"
57+
}
58+
],
59+
"toolCall": {
60+
"index": 0,
61+
"id": "call_1",
62+
"function": {
63+
"name": "py",
64+
"arguments": "{}"
65+
}
66+
},
67+
"usage": {}
68+
}
69+
]
70+
}`

0 commit comments

Comments
 (0)