Skip to content

Commit b75ad7b

Browse files
authored
Improve mirror iterator (#18928)
* Improve mirror iterator * fix test
1 parent 59959ab commit b75ad7b

File tree

4 files changed

+19
-19
lines changed

4 files changed

+19
-19
lines changed

models/repo/mirror.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,11 +120,12 @@ func DeleteMirrorByRepoID(repoID int64) error {
120120
}
121121

122122
// MirrorsIterate iterates all mirror repositories.
123-
func MirrorsIterate(f func(idx int, bean interface{}) error) error {
123+
func MirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
124124
return db.GetEngine(db.DefaultContext).
125125
Where("next_update_unix<=?", time.Now().Unix()).
126126
And("next_update_unix!=0").
127127
OrderBy("updated_unix ASC").
128+
Limit(limit).
128129
Iterate(new(Mirror), f)
129130
}
130131

models/repo/pushmirror.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,11 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
101101
}
102102

103103
// PushMirrorsIterate iterates all push-mirror repositories.
104-
func PushMirrorsIterate(f func(idx int, bean interface{}) error) error {
104+
func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
105105
return db.GetEngine(db.DefaultContext).
106106
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
107107
And("`interval` != 0").
108108
OrderBy("last_update ASC").
109+
Limit(limit).
109110
Iterate(new(PushMirror), f)
110111
}

models/repo/pushmirror_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func TestPushMirrorsIterate(t *testing.T) {
4040

4141
time.Sleep(1 * time.Millisecond)
4242

43-
PushMirrorsIterate(func(idx int, bean interface{}) error {
43+
PushMirrorsIterate(1, func(idx int, bean interface{}) error {
4444
m, ok := bean.(*PushMirror)
4545
assert.True(t, ok)
4646
assert.Equal(t, "test-1", m.RemoteName)

services/mirror/mirror.go

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,7 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
5555
}
5656
log.Trace("Doing: Update")
5757

58-
requested := 0
59-
60-
handler := func(idx int, bean interface{}, limit int) error {
58+
handler := func(idx int, bean interface{}) error {
6159
var item SyncRequest
6260
var repo *repo_model.Repository
6361
if m, ok := bean.(*repo_model.Mirror); ok {
@@ -104,35 +102,35 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
104102
}
105103
return err
106104
}
107-
108-
requested++
109-
if limit > 0 && requested > limit {
110-
return errLimit
111-
}
112105
return nil
113106
}
114107

115108
pullMirrorsRequested := 0
116109
if pullLimit != 0 {
117-
requested = 0
118-
if err := repo_model.MirrorsIterate(func(idx int, bean interface{}) error {
119-
return handler(idx, bean, pullLimit)
110+
if err := repo_model.MirrorsIterate(pullLimit, func(idx int, bean interface{}) error {
111+
if err := handler(idx, bean); err != nil {
112+
return err
113+
}
114+
pullMirrorsRequested++
115+
return nil
120116
}); err != nil && err != errLimit {
121117
log.Error("MirrorsIterate: %v", err)
122118
return err
123119
}
124-
pullMirrorsRequested, requested = requested, 0
125120
}
121+
126122
pushMirrorsRequested := 0
127123
if pushLimit != 0 {
128-
requested = 0
129-
if err := repo_model.PushMirrorsIterate(func(idx int, bean interface{}) error {
130-
return handler(idx, bean, pushLimit)
124+
if err := repo_model.PushMirrorsIterate(pushLimit, func(idx int, bean interface{}) error {
125+
if err := handler(idx, bean); err != nil {
126+
return err
127+
}
128+
pushMirrorsRequested++
129+
return nil
131130
}); err != nil && err != errLimit {
132131
log.Error("PushMirrorsIterate: %v", err)
133132
return err
134133
}
135-
pushMirrorsRequested, requested = requested, 0
136134
}
137135
log.Trace("Finished: Update: %d pull mirrors and %d push mirrors queued", pullMirrorsRequested, pushMirrorsRequested)
138136
return nil

0 commit comments

Comments
 (0)