Skip to content

Commit f5c7d4c

Browse files
authored
Reduce unnecessary DB queries for Actions tasks (#25199)
Close #24544 Changes: - Create `action_tasks_version` table to store the latest version of each scope (global, org and repo). - When a job with the status of `waiting` is created, the tasks version of the scopes it belongs to will increase. - When the status of a job already in the database is updated to `waiting`, the tasks version of the scopes it belongs to will increase. - On Gitea side, in `FeatchTask()`, will try to query the `action_tasks_version` record of the scope of the runner that call `FetchTask()`. If the record does not exist, will insert a row. Then, Gitea will compare the version passed from runner to Gitea with the version in database, if inconsistent, try pick task. Gitea always returns the latest version from database to the runner. Related: - Protocol: https://gitea.com/gitea/actions-proto-def/pulls/10 - Runner: https://gitea.com/gitea/act_runner/pulls/219
1 parent 674df05 commit f5c7d4c

File tree

7 files changed

+174
-9
lines changed

7 files changed

+174
-9
lines changed

models/actions/run.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ func InsertRun(ctx context.Context, run *ActionRun, jobs []*jobparser.SingleWork
195195
}
196196

197197
runJobs := make([]*ActionRunJob, 0, len(jobs))
198+
var hasWaiting bool
198199
for _, v := range jobs {
199200
id, job := v.Job()
200201
needs := job.Needs()
@@ -205,6 +206,8 @@ func InsertRun(ctx context.Context, run *ActionRun, jobs []*jobparser.SingleWork
205206
status := StatusWaiting
206207
if len(needs) > 0 || run.NeedApproval {
207208
status = StatusBlocked
209+
} else {
210+
hasWaiting = true
208211
}
209212
job.Name, _ = util.SplitStringAtByteN(job.Name, 255)
210213
runJobs = append(runJobs, &ActionRunJob{
@@ -225,6 +228,13 @@ func InsertRun(ctx context.Context, run *ActionRun, jobs []*jobparser.SingleWork
225228
return err
226229
}
227230

231+
// if there is a job in the waiting status, increase tasks version.
232+
if hasWaiting {
233+
if err := IncreaseTaskVersion(ctx, run.OwnerID, run.RepoID); err != nil {
234+
return err
235+
}
236+
}
237+
228238
return commiter.Commit()
229239
}
230240

models/actions/run_job.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,13 @@ func UpdateRunJob(ctx context.Context, job *ActionRunJob, cond builder.Cond, col
111111
return affected, nil
112112
}
113113

114+
if affected != 0 && util.SliceContains(cols, "status") && job.Status.IsWaiting() {
115+
// if the status of job changes to waiting again, increase tasks version.
116+
if err := IncreaseTaskVersion(ctx, job.OwnerID, job.RepoID); err != nil {
117+
return affected, err
118+
}
119+
}
120+
114121
if job.RunID == 0 {
115122
var err error
116123
if job, err = GetRunJobByID(ctx, job.ID); err != nil {

models/actions/task.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,12 +215,11 @@ func GetRunningTaskByToken(ctx context.Context, token string) (*ActionTask, erro
215215
}
216216

217217
func CreateTaskForRunner(ctx context.Context, runner *ActionRunner) (*ActionTask, bool, error) {
218-
dbCtx, commiter, err := db.TxContext(ctx)
218+
ctx, commiter, err := db.TxContext(ctx)
219219
if err != nil {
220220
return nil, false, err
221221
}
222222
defer commiter.Close()
223-
ctx = dbCtx.WithContext(ctx)
224223

225224
e := db.GetEngine(ctx)
226225

models/actions/tasks_version.go

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// Copyright 2023 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package actions
5+
6+
import (
7+
"context"
8+
9+
"code.gitea.io/gitea/models/db"
10+
"code.gitea.io/gitea/modules/log"
11+
"code.gitea.io/gitea/modules/timeutil"
12+
)
13+
14+
// ActionTasksVersion
15+
// If both ownerID and repoID is zero, its scope is global.
16+
// If ownerID is not zero and repoID is zero, its scope is org (there is no user-level runner currrently).
17+
// If ownerID is zero and repoID is not zero, its scope is repo.
18+
type ActionTasksVersion struct {
19+
ID int64 `xorm:"pk autoincr"`
20+
OwnerID int64 `xorm:"UNIQUE(owner_repo)"`
21+
RepoID int64 `xorm:"INDEX UNIQUE(owner_repo)"`
22+
Version int64
23+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
24+
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
25+
}
26+
27+
func init() {
28+
db.RegisterModel(new(ActionTasksVersion))
29+
}
30+
31+
func GetTasksVersionByScope(ctx context.Context, ownerID, repoID int64) (int64, error) {
32+
var tasksVersion ActionTasksVersion
33+
has, err := db.GetEngine(ctx).Where("owner_id = ? AND repo_id = ?", ownerID, repoID).Get(&tasksVersion)
34+
if err != nil {
35+
return 0, err
36+
} else if !has {
37+
return 0, nil
38+
}
39+
return tasksVersion.Version, err
40+
}
41+
42+
func insertTasksVersion(ctx context.Context, ownerID, repoID int64) (*ActionTasksVersion, error) {
43+
tasksVersion := &ActionTasksVersion{
44+
OwnerID: ownerID,
45+
RepoID: repoID,
46+
Version: 1,
47+
}
48+
if _, err := db.GetEngine(ctx).Insert(tasksVersion); err != nil {
49+
return nil, err
50+
}
51+
return tasksVersion, nil
52+
}
53+
54+
func increaseTasksVersionByScope(ctx context.Context, ownerID, repoID int64) error {
55+
result, err := db.GetEngine(ctx).Exec("UPDATE action_tasks_version SET version = version + 1 WHERE owner_id = ? AND repo_id = ?", ownerID, repoID)
56+
if err != nil {
57+
return err
58+
}
59+
affected, err := result.RowsAffected()
60+
if err != nil {
61+
return err
62+
}
63+
64+
if affected == 0 {
65+
// if update sql does not affect any rows, the database may be broken,
66+
// so re-insert the row of version data here.
67+
if _, err := insertTasksVersion(ctx, ownerID, repoID); err != nil {
68+
return err
69+
}
70+
}
71+
72+
return nil
73+
}
74+
75+
func IncreaseTaskVersion(ctx context.Context, ownerID, repoID int64) error {
76+
ctx, commiter, err := db.TxContext(ctx)
77+
if err != nil {
78+
return err
79+
}
80+
defer commiter.Close()
81+
82+
// 1. increase global
83+
if err := increaseTasksVersionByScope(ctx, 0, 0); err != nil {
84+
log.Error("IncreaseTasksVersionByScope(Global): %v", err)
85+
return err
86+
}
87+
88+
// 2. increase owner
89+
if ownerID > 0 {
90+
if err := increaseTasksVersionByScope(ctx, ownerID, 0); err != nil {
91+
log.Error("IncreaseTasksVersionByScope(Owner): %v", err)
92+
return err
93+
}
94+
}
95+
96+
// 3. increase repo
97+
if repoID > 0 {
98+
if err := increaseTasksVersionByScope(ctx, 0, repoID); err != nil {
99+
log.Error("IncreaseTasksVersionByScope(Repo): %v", err)
100+
return err
101+
}
102+
}
103+
104+
return commiter.Commit()
105+
}

models/migrations/migrations.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,8 @@ var migrations = []Migration{
515515
NewMigration("Alter Actions Artifact table", v1_21.AlterActionArtifactTable),
516516
// v266 -> v267
517517
NewMigration("Reduce commit status", v1_21.ReduceCommitStatus),
518+
// v267 -> v268
519+
NewMigration("Add action_tasks_version table", v1_21.CreateActionTasksVersionTable),
518520
}
519521

520522
// GetCurrentDBVersion returns the current db version

models/migrations/v1_21/v267.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright 2023 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package v1_21 //nolint
5+
6+
import (
7+
"code.gitea.io/gitea/modules/timeutil"
8+
9+
"xorm.io/xorm"
10+
)
11+
12+
func CreateActionTasksVersionTable(x *xorm.Engine) error {
13+
type ActionTasksVersion struct {
14+
ID int64 `xorm:"pk autoincr"`
15+
OwnerID int64 `xorm:"UNIQUE(owner_repo)"`
16+
RepoID int64 `xorm:"INDEX UNIQUE(owner_repo)"`
17+
Version int64
18+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
19+
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
20+
}
21+
22+
return x.Sync(new(ActionTasksVersion))
23+
}

routers/api/actions/runner/runner.go

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -127,20 +127,39 @@ func (s *Service) Declare(
127127
// FetchTask assigns a task to the runner
128128
func (s *Service) FetchTask(
129129
ctx context.Context,
130-
_ *connect.Request[runnerv1.FetchTaskRequest],
130+
req *connect.Request[runnerv1.FetchTaskRequest],
131131
) (*connect.Response[runnerv1.FetchTaskResponse], error) {
132132
runner := GetRunner(ctx)
133133

134134
var task *runnerv1.Task
135-
if t, ok, err := pickTask(ctx, runner); err != nil {
136-
log.Error("pick task failed: %v", err)
137-
return nil, status.Errorf(codes.Internal, "pick task: %v", err)
138-
} else if ok {
139-
task = t
135+
tasksVersion := req.Msg.TasksVersion // task version from runner
136+
latestVersion, err := actions_model.GetTasksVersionByScope(ctx, runner.OwnerID, runner.RepoID)
137+
if err != nil {
138+
return nil, status.Errorf(codes.Internal, "query tasks version failed: %v", err)
139+
} else if latestVersion == 0 {
140+
if err := actions_model.IncreaseTaskVersion(ctx, runner.OwnerID, runner.RepoID); err != nil {
141+
return nil, status.Errorf(codes.Internal, "fail to increase task version: %v", err)
142+
}
143+
// if we don't increase the value of `latestVersion` here,
144+
// the response of FetchTask will return tasksVersion as zero.
145+
// and the runner will treat it as an old version of Gitea.
146+
latestVersion++
140147
}
141148

149+
if tasksVersion != latestVersion {
150+
// if the task version in request is not equal to the version in db,
151+
// it means there may still be some tasks not be assgined.
152+
// try to pick a task for the runner that send the request.
153+
if t, ok, err := pickTask(ctx, runner); err != nil {
154+
log.Error("pick task failed: %v", err)
155+
return nil, status.Errorf(codes.Internal, "pick task: %v", err)
156+
} else if ok {
157+
task = t
158+
}
159+
}
142160
res := connect.NewResponse(&runnerv1.FetchTaskResponse{
143-
Task: task,
161+
Task: task,
162+
TasksVersion: latestVersion,
144163
})
145164
return res, nil
146165
}

0 commit comments

Comments
 (0)