Skip to content

Commit fc34481

Browse files
lunnywolfogre
lunny
and
wolfogre
authored
Add commit status summary table to reduce query from commit status table (#30223)
This PR adds a new table named commit status summary to reduce queries from the commit status table. After this change, commit status summary table will be used for the final result, commit status table will be for details. --------- Co-authored-by: Jason Song <[email protected]>
1 parent 26ee663 commit fc34481

File tree

7 files changed

+170
-31
lines changed

7 files changed

+170
-31
lines changed

‎models/git/commit_status.go

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -292,30 +292,27 @@ func GetLatestCommitStatus(ctx context.Context, repoID int64, sha string, listOp
292292
}
293293

294294
// GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
295-
func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHAs map[int64]string, listOptions db.ListOptions) (map[int64][]*CommitStatus, error) {
295+
func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map[int64][]*CommitStatus, error) {
296296
type result struct {
297297
Index int64
298298
RepoID int64
299+
SHA string
299300
}
300301

301-
results := make([]result, 0, len(repoIDsToLatestCommitSHAs))
302+
results := make([]result, 0, len(repoSHAs))
302303

303304
getBase := func() *xorm.Session {
304305
return db.GetEngine(ctx).Table(&CommitStatus{})
305306
}
306307

307308
// Create a disjunction of conditions for each repoID and SHA pair
308-
conds := make([]builder.Cond, 0, len(repoIDsToLatestCommitSHAs))
309-
for repoID, sha := range repoIDsToLatestCommitSHAs {
310-
conds = append(conds, builder.Eq{"repo_id": repoID, "sha": sha})
309+
conds := make([]builder.Cond, 0, len(repoSHAs))
310+
for _, repoSHA := range repoSHAs {
311+
conds = append(conds, builder.Eq{"repo_id": repoSHA.RepoID, "sha": repoSHA.SHA})
311312
}
312313
sess := getBase().Where(builder.Or(conds...)).
313-
Select("max( `index` ) as `index`, repo_id").
314-
GroupBy("context_hash, repo_id").OrderBy("max( `index` ) desc")
315-
316-
if !listOptions.IsListAll() {
317-
sess = db.SetSessionPagination(sess, &listOptions)
318-
}
314+
Select("max( `index` ) as `index`, repo_id, sha").
315+
GroupBy("context_hash, repo_id, sha").OrderBy("max( `index` ) desc")
319316

320317
err := sess.Find(&results)
321318
if err != nil {
@@ -332,7 +329,7 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHA
332329
cond := builder.Eq{
333330
"`index`": result.Index,
334331
"repo_id": result.RepoID,
335-
"sha": repoIDsToLatestCommitSHAs[result.RepoID],
332+
"sha": result.SHA,
336333
}
337334
conds = append(conds, cond)
338335
}

‎models/git/commit_status_summary.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright 2024 Gitea. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package git
5+
6+
import (
7+
"context"
8+
9+
"code.gitea.io/gitea/models/db"
10+
"code.gitea.io/gitea/modules/setting"
11+
api "code.gitea.io/gitea/modules/structs"
12+
13+
"xorm.io/builder"
14+
)
15+
16+
// CommitStatusSummary holds the latest commit Status of a single Commit
17+
type CommitStatusSummary struct {
18+
ID int64 `xorm:"pk autoincr"`
19+
RepoID int64 `xorm:"INDEX UNIQUE(repo_id_sha)"`
20+
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
21+
State api.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
22+
}
23+
24+
func init() {
25+
db.RegisterModel(new(CommitStatusSummary))
26+
}
27+
28+
type RepoSHA struct {
29+
RepoID int64
30+
SHA string
31+
}
32+
33+
func GetLatestCommitStatusForRepoAndSHAs(ctx context.Context, repoSHAs []RepoSHA) ([]*CommitStatus, error) {
34+
cond := builder.NewCond()
35+
for _, rs := range repoSHAs {
36+
cond = cond.Or(builder.Eq{"repo_id": rs.RepoID, "sha": rs.SHA})
37+
}
38+
39+
var summaries []CommitStatusSummary
40+
if err := db.GetEngine(ctx).Where(cond).Find(&summaries); err != nil {
41+
return nil, err
42+
}
43+
44+
commitStatuses := make([]*CommitStatus, 0, len(repoSHAs))
45+
for _, summary := range summaries {
46+
commitStatuses = append(commitStatuses, &CommitStatus{
47+
RepoID: summary.RepoID,
48+
SHA: summary.SHA,
49+
State: summary.State,
50+
})
51+
}
52+
return commitStatuses, nil
53+
}
54+
55+
func UpdateCommitStatusSummary(ctx context.Context, repoID int64, sha string) error {
56+
commitStatuses, _, err := GetLatestCommitStatus(ctx, repoID, sha, db.ListOptionsAll)
57+
if err != nil {
58+
return err
59+
}
60+
state := CalcCommitStatus(commitStatuses)
61+
// mysql will return 0 when update a record which state hasn't been changed which behaviour is different from other database,
62+
// so we need to use insert in on duplicate
63+
if setting.Database.Type.IsMySQL() {
64+
_, err := db.GetEngine(ctx).Exec("INSERT INTO commit_status_summary (repo_id,sha,state) VALUES (?,?,?) ON DUPLICATE KEY UPDATE state=?",
65+
repoID, sha, state.State, state.State)
66+
return err
67+
}
68+
69+
if cnt, err := db.GetEngine(ctx).Where("repo_id=? AND sha=?", repoID, sha).
70+
Cols("state").
71+
Update(&CommitStatusSummary{
72+
State: state.State,
73+
}); err != nil {
74+
return err
75+
} else if cnt == 0 {
76+
_, err = db.GetEngine(ctx).Insert(&CommitStatusSummary{
77+
RepoID: repoID,
78+
SHA: sha,
79+
State: state.State,
80+
})
81+
return err
82+
}
83+
return nil
84+
}

‎models/migrations/migrations.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,10 @@ var migrations = []Migration{
576576

577577
// Gitea 1.22.0 ends at 294
578578

579+
// v294 -> v295
579580
NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue),
581+
// v295 -> v296
582+
NewMigration("Add commit status summary table", v1_23.AddCommitStatusSummary),
580583
}
581584

582585
// GetCurrentDBVersion returns the current db version

‎models/migrations/v1_23/v295.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package v1_23 //nolint
5+
6+
import "xorm.io/xorm"
7+
8+
func AddCommitStatusSummary(x *xorm.Engine) error {
9+
type CommitStatusSummary struct {
10+
ID int64 `xorm:"pk autoincr"`
11+
RepoID int64 `xorm:"INDEX UNIQUE(repo_id_sha)"`
12+
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
13+
State string `xorm:"VARCHAR(7) NOT NULL"`
14+
}
15+
// there is no migrations because if there is no data on this table, it will fall back to get data
16+
// from commit status
17+
return x.Sync2(new(CommitStatusSummary))
18+
}

‎services/actions/commit_status.go

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"code.gitea.io/gitea/modules/log"
1717
api "code.gitea.io/gitea/modules/structs"
1818
webhook_module "code.gitea.io/gitea/modules/webhook"
19+
commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus"
1920

2021
"github.com/nektos/act/pkg/jobparser"
2122
)
@@ -122,18 +123,13 @@ func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) er
122123
if err != nil {
123124
return fmt.Errorf("HashTypeInterfaceFromHashString: %w", err)
124125
}
125-
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
126-
Repo: repo,
127-
SHA: commitID,
128-
Creator: creator,
129-
CommitStatus: &git_model.CommitStatus{
130-
SHA: sha,
131-
TargetURL: fmt.Sprintf("%s/jobs/%d", run.Link(), index),
132-
Description: description,
133-
Context: ctxname,
134-
CreatorID: creator.ID,
135-
State: state,
136-
},
126+
if err := commitstatus_service.CreateCommitStatus(ctx, repo, creator, commitID.String(), &git_model.CommitStatus{
127+
SHA: sha,
128+
TargetURL: fmt.Sprintf("%s/jobs/%d", run.Link(), index),
129+
Description: description,
130+
Context: ctxname,
131+
CreatorID: creator.ID,
132+
State: state,
137133
}); err != nil {
138134
return fmt.Errorf("NewCommitStatus: %w", err)
139135
}

‎services/repository/commitstatus/commitstatus.go

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"context"
88
"crypto/sha256"
99
"fmt"
10+
"slices"
1011

1112
"code.gitea.io/gitea/models/db"
1213
git_model "code.gitea.io/gitea/models/git"
@@ -59,13 +60,19 @@ func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creato
5960
sha = commit.ID.String()
6061
}
6162

62-
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
63-
Repo: repo,
64-
Creator: creator,
65-
SHA: commit.ID,
66-
CommitStatus: status,
63+
if err := db.WithTx(ctx, func(ctx context.Context) error {
64+
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
65+
Repo: repo,
66+
Creator: creator,
67+
SHA: commit.ID,
68+
CommitStatus: status,
69+
}); err != nil {
70+
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
71+
}
72+
73+
return git_model.UpdateCommitStatusSummary(ctx, repo.ID, commit.ID.String())
6774
}); err != nil {
68-
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
75+
return err
6976
}
7077

7178
defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
@@ -114,8 +121,35 @@ func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Rep
114121
return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err)
115122
}
116123

124+
var repoSHAs []git_model.RepoSHA
125+
for id, sha := range repoIDsToLatestCommitSHAs {
126+
repoSHAs = append(repoSHAs, git_model.RepoSHA{RepoID: id, SHA: sha})
127+
}
128+
129+
summaryResults, err := git_model.GetLatestCommitStatusForRepoAndSHAs(ctx, repoSHAs)
130+
if err != nil {
131+
return nil, fmt.Errorf("GetLatestCommitStatusForRepoAndSHAs: %v", err)
132+
}
133+
134+
for _, summary := range summaryResults {
135+
for i, repo := range repos {
136+
if repo.ID == summary.RepoID {
137+
results[i] = summary
138+
_ = slices.DeleteFunc(repoSHAs, func(repoSHA git_model.RepoSHA) bool {
139+
return repoSHA.RepoID == repo.ID
140+
})
141+
if results[i].State != "" {
142+
if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil {
143+
log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
144+
}
145+
}
146+
break
147+
}
148+
}
149+
}
150+
117151
// call the database O(1) times to get the commit statuses for all repos
118-
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll)
152+
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoSHAs)
119153
if err != nil {
120154
return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err)
121155
}

‎tests/integration/pull_status_test.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import (
1212
"testing"
1313

1414
auth_model "code.gitea.io/gitea/models/auth"
15+
git_model "code.gitea.io/gitea/models/git"
16+
repo_model "code.gitea.io/gitea/models/repo"
17+
"code.gitea.io/gitea/models/unittest"
1518
api "code.gitea.io/gitea/modules/structs"
1619

1720
"github.com/stretchr/testify/assert"
@@ -90,6 +93,10 @@ func TestPullCreate_CommitStatus(t *testing.T) {
9093
assert.True(t, ok)
9194
assert.Contains(t, cls, statesIcons[status])
9295
}
96+
97+
repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user1", Name: "repo1"})
98+
css := unittest.AssertExistsAndLoadBean(t, &git_model.CommitStatusSummary{RepoID: repo1.ID, SHA: commitID})
99+
assert.EqualValues(t, api.CommitStatusWarning, css.State)
93100
})
94101
}
95102

0 commit comments

Comments
 (0)