Skip to content

Commit 86e5268

Browse files
KN4CK3R6543lunnylafrikswxiaoguang
authored
Add Docker /v2/_catalog endpoint (#20469)
* Added properties for packages. * Fixed authenticate header format. * Added _catalog endpoint. * Check owner visibility. * Extracted condition. * Added test for _catalog. Co-authored-by: 6543 <[email protected]> Co-authored-by: Lunny Xiao <[email protected]> Co-authored-by: Lauris BH <[email protected]> Co-authored-by: wxiaoguang <[email protected]>
1 parent 4604048 commit 86e5268

File tree

21 files changed

+341
-78
lines changed

21 files changed

+341
-78
lines changed

‎integrations/api_packages_container_test.go

Lines changed: 70 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727

2828
func TestPackageContainer(t *testing.T) {
2929
defer prepareTestEnv(t)()
30+
3031
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User)
3132

3233
has := func(l packages_model.PackagePropertyList, name string) bool {
@@ -37,6 +38,15 @@ func TestPackageContainer(t *testing.T) {
3738
}
3839
return false
3940
}
41+
getAllByName := func(l packages_model.PackagePropertyList, name string) []string {
42+
values := make([]string, 0, len(l))
43+
for _, pp := range l {
44+
if pp.Name == name {
45+
values = append(values, pp.Value)
46+
}
47+
}
48+
return values
49+
}
4050

4151
images := []string{"test", "te/st"}
4252
tags := []string{"latest", "main"}
@@ -67,7 +77,7 @@ func TestPackageContainer(t *testing.T) {
6777
Token string `json:"token"`
6878
}
6979

70-
authenticate := []string{`Bearer realm="` + setting.AppURL + `v2/token"`}
80+
authenticate := []string{`Bearer realm="` + setting.AppURL + `v2/token",service="container_registry",scope="*"`}
7181

7282
t.Run("Anonymous", func(t *testing.T) {
7383
defer PrintCurrentTest(t)()
@@ -237,7 +247,8 @@ func TestPackageContainer(t *testing.T) {
237247
assert.Nil(t, pd.SemVer)
238248
assert.Equal(t, image, pd.Package.Name)
239249
assert.Equal(t, tag, pd.Version.Version)
240-
assert.True(t, has(pd.Properties, container_module.PropertyManifestTagged))
250+
assert.ElementsMatch(t, []string{strings.ToLower(user.LowerName + "/" + image)}, getAllByName(pd.PackageProperties, container_module.PropertyRepository))
251+
assert.True(t, has(pd.VersionProperties, container_module.PropertyManifestTagged))
241252

242253
assert.IsType(t, &container_module.Metadata{}, pd.Metadata)
243254
metadata := pd.Metadata.(*container_module.Metadata)
@@ -331,7 +342,8 @@ func TestPackageContainer(t *testing.T) {
331342
assert.Nil(t, pd.SemVer)
332343
assert.Equal(t, image, pd.Package.Name)
333344
assert.Equal(t, untaggedManifestDigest, pd.Version.Version)
334-
assert.False(t, has(pd.Properties, container_module.PropertyManifestTagged))
345+
assert.ElementsMatch(t, []string{strings.ToLower(user.LowerName + "/" + image)}, getAllByName(pd.PackageProperties, container_module.PropertyRepository))
346+
assert.False(t, has(pd.VersionProperties, container_module.PropertyManifestTagged))
335347

336348
assert.IsType(t, &container_module.Metadata{}, pd.Metadata)
337349

@@ -363,18 +375,10 @@ func TestPackageContainer(t *testing.T) {
363375
assert.Nil(t, pd.SemVer)
364376
assert.Equal(t, image, pd.Package.Name)
365377
assert.Equal(t, multiTag, pd.Version.Version)
366-
assert.True(t, has(pd.Properties, container_module.PropertyManifestTagged))
378+
assert.ElementsMatch(t, []string{strings.ToLower(user.LowerName + "/" + image)}, getAllByName(pd.PackageProperties, container_module.PropertyRepository))
379+
assert.True(t, has(pd.VersionProperties, container_module.PropertyManifestTagged))
367380

368-
getAllByName := func(l packages_model.PackagePropertyList, name string) []string {
369-
values := make([]string, 0, len(l))
370-
for _, pp := range l {
371-
if pp.Name == name {
372-
values = append(values, pp.Value)
373-
}
374-
}
375-
return values
376-
}
377-
assert.ElementsMatch(t, []string{manifestDigest, untaggedManifestDigest}, getAllByName(pd.Properties, container_module.PropertyManifestReference))
381+
assert.ElementsMatch(t, []string{manifestDigest, untaggedManifestDigest}, getAllByName(pd.VersionProperties, container_module.PropertyManifestReference))
378382

379383
assert.IsType(t, &container_module.Metadata{}, pd.Metadata)
380384
metadata := pd.Metadata.(*container_module.Metadata)
@@ -536,4 +540,56 @@ func TestPackageContainer(t *testing.T) {
536540
})
537541
})
538542
}
543+
544+
t.Run("OwnerNameChange", func(t *testing.T) {
545+
defer PrintCurrentTest(t)()
546+
547+
checkCatalog := func(owner string) func(t *testing.T) {
548+
return func(t *testing.T) {
549+
defer PrintCurrentTest(t)()
550+
551+
req := NewRequest(t, "GET", fmt.Sprintf("%sv2/_catalog", setting.AppURL))
552+
addTokenAuthHeader(req, userToken)
553+
resp := MakeRequest(t, req, http.StatusOK)
554+
555+
type RepositoryList struct {
556+
Repositories []string `json:"repositories"`
557+
}
558+
559+
repoList := &RepositoryList{}
560+
DecodeJSON(t, resp, &repoList)
561+
562+
assert.Len(t, repoList.Repositories, len(images))
563+
names := make([]string, 0, len(images))
564+
for _, image := range images {
565+
names = append(names, strings.ToLower(owner+"/"+image))
566+
}
567+
assert.ElementsMatch(t, names, repoList.Repositories)
568+
}
569+
}
570+
571+
t.Run(fmt.Sprintf("Catalog[%s]", user.LowerName), checkCatalog(user.LowerName))
572+
573+
session := loginUser(t, user.Name)
574+
575+
newOwnerName := "newUsername"
576+
577+
req := NewRequestWithValues(t, "POST", "/user/settings", map[string]string{
578+
"_csrf": GetCSRF(t, session, "/user/settings"),
579+
"name": newOwnerName,
580+
"email": "[email protected]",
581+
"language": "en-US",
582+
})
583+
session.MakeRequest(t, req, http.StatusSeeOther)
584+
585+
t.Run(fmt.Sprintf("Catalog[%s]", newOwnerName), checkCatalog(newOwnerName))
586+
587+
req = NewRequestWithValues(t, "POST", "/user/settings", map[string]string{
588+
"_csrf": GetCSRF(t, session, "/user/settings"),
589+
"name": user.Name,
590+
"email": "[email protected]",
591+
"language": "en-US",
592+
})
593+
session.MakeRequest(t, req, http.StatusSeeOther)
594+
})
539595
}

‎integrations/api_packages_npm_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,9 @@ func TestPackageNpm(t *testing.T) {
8585
assert.IsType(t, &npm.Metadata{}, pd.Metadata)
8686
assert.Equal(t, packageName, pd.Package.Name)
8787
assert.Equal(t, packageVersion, pd.Version.Version)
88-
assert.Len(t, pd.Properties, 1)
89-
assert.Equal(t, npm.TagProperty, pd.Properties[0].Name)
90-
assert.Equal(t, packageTag, pd.Properties[0].Value)
88+
assert.Len(t, pd.VersionProperties, 1)
89+
assert.Equal(t, npm.TagProperty, pd.VersionProperties[0].Name)
90+
assert.Equal(t, packageTag, pd.VersionProperties[0].Value)
9191

9292
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
9393
assert.NoError(t, err)

‎models/migrations/migrations.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,8 @@ var migrations = []Migration{
398398
NewMigration("Improve Action table indices v2", improveActionTableIndices),
399399
// v219 -> v220
400400
NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnCommitColForPushMirror),
401+
// v220 -> v221
402+
NewMigration("Add container repository property", addContainerRepositoryProperty),
401403
}
402404

403405
// GetCurrentDBVersion returns the current db version

‎models/migrations/v220.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package migrations
6+
7+
import (
8+
packages_model "code.gitea.io/gitea/models/packages"
9+
container_module "code.gitea.io/gitea/modules/packages/container"
10+
11+
"xorm.io/xorm"
12+
"xorm.io/xorm/schemas"
13+
)
14+
15+
func addContainerRepositoryProperty(x *xorm.Engine) error {
16+
switch x.Dialect().URI().DBType {
17+
case schemas.SQLITE:
18+
_, err := x.Exec("INSERT INTO package_property (ref_type, ref_id, name, value) SELECT ?, p.id, ?, u.lower_name || '/' || p.lower_name FROM package p JOIN `user` u ON p.owner_id = u.id WHERE p.type = ?", packages_model.PropertyTypePackage, container_module.PropertyRepository, packages_model.TypeContainer)
19+
if err != nil {
20+
return err
21+
}
22+
default:
23+
_, err := x.Exec("INSERT INTO package_property (ref_type, ref_id, name, value) SELECT ?, p.id, ?, CONCAT(u.lower_name, '/', p.lower_name) FROM package p JOIN `user` u ON p.owner_id = u.id WHERE p.type = ?", packages_model.PropertyTypePackage, container_module.PropertyRepository, packages_model.TypeContainer)
24+
if err != nil {
25+
return err
26+
}
27+
}
28+
return nil
29+
}

‎models/packages/container/search.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"code.gitea.io/gitea/models/db"
1414
"code.gitea.io/gitea/models/packages"
15+
user_model "code.gitea.io/gitea/models/user"
1516
container_module "code.gitea.io/gitea/modules/packages/container"
1617

1718
"xorm.io/builder"
@@ -210,6 +211,7 @@ func SearchImageTags(ctx context.Context, opts *ImageTagsSearchOptions) ([]*pack
210211
return pvs, count, err
211212
}
212213

214+
// SearchExpiredUploadedBlobs gets all uploaded blobs which are older than specified
213215
func SearchExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) ([]*packages.PackageFile, error) {
214216
var cond builder.Cond = builder.Eq{
215217
"package_version.is_internal": true,
@@ -225,3 +227,37 @@ func SearchExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) ([
225227
Where(cond).
226228
Find(&pfs)
227229
}
230+
231+
// GetRepositories gets a sorted list of all repositories
232+
func GetRepositories(ctx context.Context, actor *user_model.User, n int, last string) ([]string, error) {
233+
var cond builder.Cond = builder.Eq{
234+
"package.type": packages.TypeContainer,
235+
"package_property.ref_type": packages.PropertyTypePackage,
236+
"package_property.name": container_module.PropertyRepository,
237+
}
238+
239+
cond = cond.And(builder.Exists(
240+
builder.
241+
Select("package_version.id").
242+
Where(builder.Eq{"package_version.is_internal": false}.And(builder.Expr("package.id = package_version.package_id"))).
243+
From("package_version"),
244+
))
245+
246+
if last != "" {
247+
cond = cond.And(builder.Gt{"package_property.value": strings.ToLower(last)})
248+
}
249+
250+
cond = cond.And(user_model.BuildCanSeeUserCondition(actor))
251+
252+
sess := db.GetEngine(ctx).
253+
Table("package").
254+
Select("package_property.value").
255+
Join("INNER", "user", "`user`.id = package.owner_id").
256+
Join("INNER", "package_property", "package_property.ref_id = package.id").
257+
Where(cond).
258+
Asc("package_property.value").
259+
Limit(n)
260+
261+
repositories := make([]string, 0, n)
262+
return repositories, sess.Find(&repositories)
263+
}

‎models/packages/descriptor.go

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,16 @@ func (l PackagePropertyList) GetByName(name string) string {
4040

4141
// PackageDescriptor describes a package
4242
type PackageDescriptor struct {
43-
Package *Package
44-
Owner *user_model.User
45-
Repository *repo_model.Repository
46-
Version *PackageVersion
47-
SemVer *version.Version
48-
Creator *user_model.User
49-
Properties PackagePropertyList
50-
Metadata interface{}
51-
Files []*PackageFileDescriptor
43+
Package *Package
44+
Owner *user_model.User
45+
Repository *repo_model.Repository
46+
Version *PackageVersion
47+
SemVer *version.Version
48+
Creator *user_model.User
49+
PackageProperties PackagePropertyList
50+
VersionProperties PackagePropertyList
51+
Metadata interface{}
52+
Files []*PackageFileDescriptor
5253
}
5354

5455
// PackageFileDescriptor describes a package file
@@ -102,6 +103,10 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
102103
return nil, err
103104
}
104105
}
106+
pps, err := GetProperties(ctx, PropertyTypePackage, p.ID)
107+
if err != nil {
108+
return nil, err
109+
}
105110
pvps, err := GetProperties(ctx, PropertyTypeVersion, pv.ID)
106111
if err != nil {
107112
return nil, err
@@ -152,15 +157,16 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
152157
}
153158

154159
return &PackageDescriptor{
155-
Package: p,
156-
Owner: o,
157-
Repository: repository,
158-
Version: pv,
159-
SemVer: semVer,
160-
Creator: creator,
161-
Properties: PackagePropertyList(pvps),
162-
Metadata: metadata,
163-
Files: pfds,
160+
Package: p,
161+
Owner: o,
162+
Repository: repository,
163+
Version: pv,
164+
SemVer: semVer,
165+
Creator: creator,
166+
PackageProperties: PackagePropertyList(pps),
167+
VersionProperties: PackagePropertyList(pvps),
168+
Metadata: metadata,
169+
Files: pfds,
164170
}, nil
165171
}
166172

‎models/packages/package.go

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,12 @@ func TryInsertPackage(ctx context.Context, p *Package) (*Package, error) {
131131
return p, nil
132132
}
133133

134+
// DeletePackageByID deletes a package by id
135+
func DeletePackageByID(ctx context.Context, packageID int64) error {
136+
_, err := db.GetEngine(ctx).ID(packageID).Delete(&Package{})
137+
return err
138+
}
139+
134140
// SetRepositoryLink sets the linked repository
135141
func SetRepositoryLink(ctx context.Context, packageID, repoID int64) error {
136142
_, err := db.GetEngine(ctx).ID(packageID).Cols("repo_id").Update(&Package{RepoID: repoID})
@@ -192,21 +198,20 @@ func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([]
192198
Find(&ps)
193199
}
194200

195-
// DeletePackagesIfUnreferenced deletes a package if there are no associated versions
196-
func DeletePackagesIfUnreferenced(ctx context.Context) error {
201+
// FindUnreferencedPackages gets all packages without associated versions
202+
func FindUnreferencedPackages(ctx context.Context) ([]*Package, error) {
197203
in := builder.
198204
Select("package.id").
199205
From("package").
200206
LeftJoin("package_version", "package_version.package_id = package.id").
201207
Where(builder.Expr("package_version.id IS NULL"))
202208

203-
_, err := db.GetEngine(ctx).
209+
ps := make([]*Package, 0, 10)
210+
return ps, db.GetEngine(ctx).
204211
// double select workaround for MySQL
205212
// https://stackoverflow.com/questions/4471277/mysql-delete-from-with-subquery-as-condition
206213
Where(builder.In("package.id", builder.Select("id").From(in, "temp"))).
207-
Delete(&Package{})
208-
209-
return err
214+
Find(&ps)
210215
}
211216

212217
// HasOwnerPackages tests if a user/org has packages

‎models/packages/package_property.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@ const (
2121
PropertyTypeVersion PropertyType = iota // 0
2222
// PropertyTypeFile means the reference is a package file
2323
PropertyTypeFile // 1
24+
// PropertyTypePackage means the reference is a package
25+
PropertyTypePackage // 2
2426
)
2527

26-
// PackageProperty represents a property of a package version or file
28+
// PackageProperty represents a property of a package, version or file
2729
type PackageProperty struct {
2830
ID int64 `xorm:"pk autoincr"`
2931
RefType PropertyType `xorm:"INDEX NOT NULL"`
@@ -68,3 +70,9 @@ func DeletePropertyByID(ctx context.Context, propertyID int64) error {
6870
_, err := db.GetEngine(ctx).ID(propertyID).Delete(&PackageProperty{})
6971
return err
7072
}
73+
74+
// DeletePropertyByName deletes properties by name
75+
func DeletePropertyByName(ctx context.Context, refType PropertyType, refID int64, name string) error {
76+
_, err := db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Delete(&PackageProperty{})
77+
return err
78+
}

0 commit comments

Comments
 (0)