Skip to content

Commit 0046e0e

Browse files
committed
Cache repository default branch commit status to reduce query on commit status table (go-gitea#29444)
After repository commit status has been introduced on dashaboard, the most top SQL comes from `GetLatestCommitStatusForPairs`. This PR adds a cache for the repository's default branch's latest combined commit status. When a new commit status updated, the cache will be marked as invalid. <img width="998" alt="image" src="https://github.com/go-gitea/gitea/assets/81045/76759de7-3a83-4d54-8571-278f5422aed3">
1 parent 76b6754 commit 0046e0e

File tree

3 files changed

+142
-22
lines changed

3 files changed

+142
-22
lines changed

routers/api/v1/repo/status.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"code.gitea.io/gitea/modules/web"
1414
"code.gitea.io/gitea/routers/api/v1/utils"
1515
"code.gitea.io/gitea/services/convert"
16-
files_service "code.gitea.io/gitea/services/repository/files"
16+
commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus"
1717
)
1818

1919
// NewCommitStatus creates a new CommitStatus
@@ -63,7 +63,7 @@ func NewCommitStatus(ctx *context.APIContext) {
6363
Description: form.Description,
6464
Context: form.Context,
6565
}
66-
if err := files_service.CreateCommitStatus(ctx, ctx.Repo.Repository, ctx.Doer, sha, status); err != nil {
66+
if err := commitstatus_service.CreateCommitStatus(ctx, ctx.Repo.Repository, ctx.Doer, sha, status); err != nil {
6767
ctx.Error(http.StatusInternalServerError, "CreateCommitStatus", err)
6868
return
6969
}

routers/web/repo/repo.go

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"code.gitea.io/gitea/services/forms"
3535
repo_service "code.gitea.io/gitea/services/repository"
3636
archiver_service "code.gitea.io/gitea/services/repository/archiver"
37+
commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus"
3738
)
3839

3940
const (
@@ -592,30 +593,14 @@ func SearchRepo(ctx *context.Context) {
592593
return
593594
}
594595

595-
// collect the latest commit of each repo
596-
// at most there are dozens of repos (limited by MaxResponseItems), so it's not a big problem at the moment
597-
repoBranchNames := make(map[int64]string, len(repos))
598-
for _, repo := range repos {
599-
repoBranchNames[repo.ID] = repo.DefaultBranch
600-
}
601-
602-
repoIDsToLatestCommitSHAs, err := git_model.FindBranchesByRepoAndBranchName(ctx, repoBranchNames)
596+
latestCommitStatuses, err := commitstatus_service.FindReposLastestCommitStatuses(ctx, repos)
603597
if err != nil {
604-
log.Error("FindBranchesByRepoAndBranchName: %v", err)
605-
return
606-
}
607-
608-
// call the database O(1) times to get the commit statuses for all repos
609-
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptions{})
610-
if err != nil {
611-
log.Error("GetLatestCommitStatusForPairs: %v", err)
598+
log.Error("FindReposLastestCommitStatuses: %v", err)
612599
return
613600
}
614601

615602
results := make([]*repo_service.WebSearchRepository, len(repos))
616603
for i, repo := range repos {
617-
latestCommitStatus := git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID])
618-
619604
results[i] = &repo_service.WebSearchRepository{
620605
Repository: &api.Repository{
621606
ID: repo.ID,
@@ -629,8 +614,11 @@ func SearchRepo(ctx *context.Context) {
629614
Link: repo.Link(),
630615
Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate,
631616
},
632-
LatestCommitStatus: latestCommitStatus,
633-
LocaleLatestCommitStatus: latestCommitStatus.LocaleString(ctx.Locale),
617+
}
618+
619+
if latestCommitStatuses[i] != nil {
620+
results[i].LatestCommitStatus = latestCommitStatuses[i]
621+
results[i].LocaleLatestCommitStatus = latestCommitStatuses[i].LocaleString(ctx.Locale)
634622
}
635623
}
636624

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package commitstatus
5+
6+
import (
7+
"context"
8+
"crypto/sha256"
9+
"fmt"
10+
11+
"code.gitea.io/gitea/models/db"
12+
git_model "code.gitea.io/gitea/models/git"
13+
repo_model "code.gitea.io/gitea/models/repo"
14+
user_model "code.gitea.io/gitea/models/user"
15+
"code.gitea.io/gitea/modules/cache"
16+
"code.gitea.io/gitea/modules/git"
17+
"code.gitea.io/gitea/modules/log"
18+
api "code.gitea.io/gitea/modules/structs"
19+
"code.gitea.io/gitea/services/automerge"
20+
)
21+
22+
func getCacheKey(repoID int64, brancheName string) string {
23+
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%d:%s", repoID, brancheName)))
24+
return fmt.Sprintf("commit_status:%x", hashBytes)
25+
}
26+
27+
func updateCommitStatusCache(ctx context.Context, repoID int64, branchName string, status api.CommitStatusState) error {
28+
c := cache.GetCache()
29+
return c.Put(getCacheKey(repoID, branchName), string(status), 3*24*60)
30+
}
31+
32+
func deleteCommitStatusCache(ctx context.Context, repoID int64, branchName string) error {
33+
c := cache.GetCache()
34+
return c.Delete(getCacheKey(repoID, branchName))
35+
}
36+
37+
// CreateCommitStatus creates a new CommitStatus given a bunch of parameters
38+
// NOTE: All text-values will be trimmed from whitespaces.
39+
// Requires: Repo, Creator, SHA
40+
func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creator *user_model.User, sha string, status *git_model.CommitStatus) error {
41+
repoPath := repo.RepoPath()
42+
43+
// confirm that commit is exist
44+
gitRepo, closer, err := git.RepositoryFromContextOrOpen(ctx, repoPath)
45+
if err != nil {
46+
return fmt.Errorf("OpenRepository[%s]: %w", repoPath, err)
47+
}
48+
defer closer.Close()
49+
50+
commit, err := gitRepo.GetCommit(sha)
51+
if err != nil {
52+
return fmt.Errorf("GetCommit[%s]: %w", sha, err)
53+
}
54+
if len(sha) != git.SHAFullLength {
55+
// use complete commit sha
56+
sha = commit.ID.String()
57+
}
58+
59+
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
60+
Repo: repo,
61+
Creator: creator,
62+
SHA: sha,
63+
CommitStatus: status,
64+
}); err != nil {
65+
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
66+
}
67+
68+
defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
69+
if err != nil {
70+
return fmt.Errorf("GetBranchCommit[%s]: %w", repo.DefaultBranch, err)
71+
}
72+
73+
if commit.ID.String() == defaultBranchCommit.ID.String() { // since one commit status updated, the combined commit status should be invalid
74+
if err := deleteCommitStatusCache(ctx, repo.ID, repo.DefaultBranch); err != nil {
75+
log.Error("deleteCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
76+
}
77+
}
78+
79+
if status.State.IsSuccess() {
80+
if err := automerge.MergeScheduledPullRequest(ctx, sha, repo); err != nil {
81+
return fmt.Errorf("MergeScheduledPullRequest[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
82+
}
83+
}
84+
85+
return nil
86+
}
87+
88+
// FindReposLastestCommitStatuses loading repository default branch latest combinded commit status with cache
89+
func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Repository) ([]*git_model.CommitStatus, error) {
90+
results := make([]*git_model.CommitStatus, len(repos))
91+
c := cache.GetCache()
92+
93+
for i, repo := range repos {
94+
status, ok := c.Get(getCacheKey(repo.ID, repo.DefaultBranch)).(string)
95+
if ok && status != "" {
96+
results[i] = &git_model.CommitStatus{State: api.CommitStatusState(status)}
97+
}
98+
}
99+
100+
// collect the latest commit of each repo
101+
// at most there are dozens of repos (limited by MaxResponseItems), so it's not a big problem at the moment
102+
repoBranchNames := make(map[int64]string, len(repos))
103+
for i, repo := range repos {
104+
if results[i] == nil {
105+
repoBranchNames[repo.ID] = repo.DefaultBranch
106+
}
107+
}
108+
109+
repoIDsToLatestCommitSHAs, err := git_model.FindBranchesByRepoAndBranchName(ctx, repoBranchNames)
110+
if err != nil {
111+
return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err)
112+
}
113+
114+
// call the database O(1) times to get the commit statuses for all repos
115+
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptions{ListAll: true})
116+
if err != nil {
117+
return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err)
118+
}
119+
120+
for i, repo := range repos {
121+
if results[i] == nil {
122+
results[i] = git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID])
123+
if results[i].State != "" {
124+
if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil {
125+
log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
126+
}
127+
}
128+
}
129+
}
130+
131+
return results, nil
132+
}

0 commit comments

Comments
 (0)