Skip to content

Commit bf77e21

Browse files
KN4CK3Rawkwardbunnytechknowlogick
authored
Add Debian package registry (#22854)
Co-authored-by: @awkwardbunny This PR adds a Debian package registry. You can follow [this tutorial](https://www.baeldung.com/linux/create-debian-package) to build a *.deb package for testing. Source packages are not supported at the moment and I did not find documentation of the architecture "all" and how these packages should be treated. --------- Co-authored-by: Brian Hong <[email protected]> Co-authored-by: techknowlogick <[email protected]>
1 parent bc4e061 commit bf77e21

File tree

57 files changed

+1995
-96
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+1995
-96
lines changed

assets/go-licenses.json

+5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

cmd/migrate_storage_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ func TestMigratePackages(t *testing.T) {
2525
creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
2626

2727
content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
28-
buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
28+
buf, err := packages_module.CreateHashedBufferFromReaderWithSize(strings.NewReader(content), 1024)
2929
assert.NoError(t, err)
3030
defer buf.Close()
3131

custom/conf/app.example.ini

+2
Original file line numberDiff line numberDiff line change
@@ -2501,6 +2501,8 @@ ROUTER = console
25012501
;LIMIT_SIZE_CONDA = -1
25022502
;; Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
25032503
;LIMIT_SIZE_CONTAINER = -1
2504+
;; Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
2505+
;LIMIT_SIZE_DEBIAN = -1
25042506
;; Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
25052507
;LIMIT_SIZE_GENERIC = -1
25062508
;; Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

docs/content/doc/administration/config-cheat-sheet.en-us.md

+1
Original file line numberDiff line numberDiff line change
@@ -1252,6 +1252,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
12521252
- `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
12531253
- `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
12541254
- `LIMIT_SIZE_CONTAINER`: **-1**: Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
1255+
- `LIMIT_SIZE_DEBIAN`: **-1**: Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
12551256
- `LIMIT_SIZE_GENERIC`: **-1**: Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
12561257
- `LIMIT_SIZE_HELM`: **-1**: Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
12571258
- `LIMIT_SIZE_MAVEN`: **-1**: Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
---
2+
date: "2023-01-07T00:00:00+00:00"
3+
title: "Debian Packages Repository"
4+
slug: "packages/debian"
5+
draft: false
6+
toc: false
7+
menu:
8+
sidebar:
9+
parent: "packages"
10+
name: "Debian"
11+
weight: 35
12+
identifier: "debian"
13+
---
14+
15+
# Debian Packages Repository
16+
17+
Publish [Debian](https://www.debian.org/distrib/packages) packages for your user or organization.
18+
19+
**Table of Contents**
20+
21+
{{< toc >}}
22+
23+
## Requirements
24+
25+
To work with the Debian registry, you need to use a HTTP client like `curl` to upload and a package manager like `apt` to consume packages.
26+
27+
The following examples use `apt`.
28+
29+
## Configuring the package registry
30+
31+
To register the Debian registry add the url to the list of known apt sources:
32+
33+
```shell
34+
echo "deb https://gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list
35+
```
36+
37+
| Placeholder | Description |
38+
| -------------- | ----------- |
39+
| `owner` | The owner of the package. |
40+
| `distribution` | The distribution to use. |
41+
| `component` | The component to use. |
42+
43+
If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}):
44+
45+
```shell
46+
echo "deb https://{username}:{your_password_or_token}@gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list
47+
```
48+
49+
The Debian registry files are signed with a PGP key which must be known to apt:
50+
51+
```shell
52+
sudo curl https://gitea.example.com/api/packages/{owner}/debian/repository.key -o /etc/apt/trusted.gpg.d/gitea-{owner}.asc
53+
```
54+
55+
Afterwards update the local package index:
56+
57+
```shell
58+
apt update
59+
```
60+
61+
## Publish a package
62+
63+
To publish a Debian package (`*.deb`), perform a HTTP PUT operation with the package content in the request body.
64+
65+
```
66+
PUT https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/upload
67+
```
68+
69+
| Parameter | Description |
70+
| -------------- | ----------- |
71+
| `owner` | The owner of the package. |
72+
| `distribution` | The distribution may match the release name of the OS, ex: `bionic`. |
73+
| `component` | The component can be used to group packages or just `main` or similar. |
74+
75+
Example request using HTTP Basic authentication:
76+
77+
```shell
78+
curl --user your_username:your_password_or_token \
79+
--upload-file path/to/file.deb \
80+
https://gitea.example.com/api/packages/testuser/debian/pool/bionic/main/upload
81+
```
82+
83+
If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password.
84+
You cannot publish a file with the same name twice to a package. You must delete the existing package version first.
85+
86+
The server reponds with the following HTTP Status codes.
87+
88+
| HTTP Status Code | Meaning |
89+
| ----------------- | ------- |
90+
| `201 Created` | The package has been published. |
91+
| `400 Bad Request` | The package name, version, distribution, component or architecture are invalid. |
92+
| `409 Conflict` | A package file with the same combination of parameters exist already in the package. |
93+
94+
## Delete a package
95+
96+
To delete a Debian package perform a HTTP DELETE operation. This will delete the package version too if there is no file left.
97+
98+
```
99+
DELETE https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/{package_name}/{package_version}/{architecture}
100+
```
101+
102+
| Parameter | Description |
103+
| ----------------- | ----------- |
104+
| `owner` | The owner of the package. |
105+
| `package_name` | The package name. |
106+
| `package_version` | The package version. |
107+
| `distribution` | The package distribution. |
108+
| `component` | The package component. |
109+
| `architecture` | The package architecture. |
110+
111+
Example request using HTTP Basic authentication:
112+
113+
```shell
114+
curl --user your_username:your_token_or_password -X DELETE \
115+
https://gitea.example.com/api/packages/testuser/debian/pools/bionic/main/test-package/1.0.0/amd64
116+
```
117+
118+
The server reponds with the following HTTP Status codes.
119+
120+
| HTTP Status Code | Meaning |
121+
| ----------------- | ------- |
122+
| `204 No Content` | Success |
123+
| `404 Not Found` | The package or file was not found. |
124+
125+
## Install a package
126+
127+
To install a package from the Debian registry, execute the following commands:
128+
129+
```shell
130+
# use latest version
131+
apt install {package_name}
132+
# use specific version
133+
apt install {package_name}={package_version}
134+
```

docs/content/doc/usage/packages/overview.en-us.md

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ The following package managers are currently supported:
3333
| [Conan]({{< relref "doc/usage/packages/conan.en-us.md" >}}) | C++ | `conan` |
3434
| [Conda]({{< relref "doc/usage/packages/conda.en-us.md" >}}) | - | `conda` |
3535
| [Container]({{< relref "doc/usage/packages/container.en-us.md" >}}) | - | any OCI compliant client |
36+
| [Debian]({{< relref "doc/usage/packages/debian.en-us.md" >}}) | - | `apt` |
3637
| [Generic]({{< relref "doc/usage/packages/generic.en-us.md" >}}) | - | any HTTP client |
3738
| [Helm]({{< relref "doc/usage/packages/helm.en-us.md" >}}) | - | any HTTP client, `cm-push` |
3839
| [Maven]({{< relref "doc/usage/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` |

go.mod

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ require (
1717
github.com/NYTimes/gziphandler v1.1.1
1818
github.com/PuerkitoBio/goquery v1.8.0
1919
github.com/alecthomas/chroma/v2 v2.5.0
20+
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
2021
github.com/blevesearch/bleve/v2 v2.3.6
2122
github.com/bufbuild/connect-go v1.3.1
2223
github.com/buildkite/terminal-to-html/v3 v3.7.0
@@ -96,6 +97,7 @@ require (
9697
github.com/stretchr/testify v1.8.1
9798
github.com/syndtr/goleveldb v1.0.0
9899
github.com/tstranex/u2f v1.0.0
100+
github.com/ulikunitz/xz v0.5.11
99101
github.com/urfave/cli v1.22.12
100102
github.com/xanzy/go-gitlab v0.80.2
101103
github.com/xeipuuv/gojsonschema v1.2.0
@@ -260,7 +262,6 @@ require (
260262
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
261263
github.com/subosito/gotenv v1.4.1 // indirect
262264
github.com/toqueteos/webbrowser v1.2.0 // indirect
263-
github.com/ulikunitz/xz v0.5.11 // indirect
264265
github.com/unknwon/com v1.0.1 // indirect
265266
github.com/valyala/bytebufferpool v1.0.0 // indirect
266267
github.com/valyala/fasthttp v1.44.0 // indirect

go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,8 @@ github.com/bits-and-blooms/bitset v1.1.10/go.mod h1:w0XsmFg8qg6cmpTtJ0z3pKgjTDBM
162162
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
163163
github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8=
164164
github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
165+
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
166+
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
165167
github.com/blevesearch/bleve/v2 v2.0.5/go.mod h1:ZjWibgnbRX33c+vBRgla9QhPb4QOjD6fdVJ+R1Bk8LM=
166168
github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4=
167169
github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4=

models/migrations/migrations.go

+2
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,8 @@ var migrations = []Migration{
489489
NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable),
490490
// v255 -> v256
491491
NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository),
492+
// v256 -> v257
493+
NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage),
492494
}
493495

494496
// GetCurrentDBVersion returns the current db version

models/migrations/v1_20/v256.go

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright 2023 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package v1_20 //nolint
5+
6+
import (
7+
"xorm.io/xorm"
8+
)
9+
10+
func AddIsInternalColumnToPackage(x *xorm.Engine) error {
11+
type Package struct {
12+
ID int64 `xorm:"pk autoincr"`
13+
OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
14+
RepoID int64 `xorm:"INDEX"`
15+
Type string `xorm:"UNIQUE(s) INDEX NOT NULL"`
16+
Name string `xorm:"NOT NULL"`
17+
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
18+
SemverCompatible bool `xorm:"NOT NULL DEFAULT false"`
19+
IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"`
20+
}
21+
22+
return x.Sync2(new(Package))
23+
}

models/packages/container/search.go

+1-10
Original file line numberDiff line numberDiff line change
@@ -101,16 +101,7 @@ func getContainerBlobsLimit(ctx context.Context, opts *BlobSearchOptions, limit
101101
return nil, err
102102
}
103103

104-
pfds := make([]*packages.PackageFileDescriptor, 0, len(pfs))
105-
for _, pf := range pfs {
106-
pfd, err := packages.GetPackageFileDescriptor(ctx, pf)
107-
if err != nil {
108-
return nil, err
109-
}
110-
pfds = append(pfds, pfd)
111-
}
112-
113-
return pfds, nil
104+
return packages.GetPackageFileDescriptors(ctx, pfs)
114105
}
115106

116107
// GetManifestVersions gets all package versions representing the matching manifest

models/packages/debian/search.go

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
// Copyright 2023 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package debian
5+
6+
import (
7+
"context"
8+
"strconv"
9+
10+
"code.gitea.io/gitea/models/db"
11+
"code.gitea.io/gitea/models/packages"
12+
debian_module "code.gitea.io/gitea/modules/packages/debian"
13+
14+
"xorm.io/builder"
15+
)
16+
17+
type PackageSearchOptions struct {
18+
OwnerID int64
19+
Distribution string
20+
Component string
21+
Architecture string
22+
}
23+
24+
// SearchLatestPackages gets the latest packages matching the search options
25+
func SearchLatestPackages(ctx context.Context, opts *PackageSearchOptions) ([]*packages.PackageFileDescriptor, error) {
26+
var cond builder.Cond = builder.Eq{
27+
"package_file.is_lead": true,
28+
"package.type": packages.TypeDebian,
29+
"package.owner_id": opts.OwnerID,
30+
"package.is_internal": false,
31+
"package_version.is_internal": false,
32+
}
33+
34+
props := make(map[string]string)
35+
if opts.Distribution != "" {
36+
props[debian_module.PropertyDistribution] = opts.Distribution
37+
}
38+
if opts.Component != "" {
39+
props[debian_module.PropertyComponent] = opts.Component
40+
}
41+
if opts.Architecture != "" {
42+
props[debian_module.PropertyArchitecture] = opts.Architecture
43+
}
44+
45+
if len(props) > 0 {
46+
var propsCond builder.Cond = builder.Eq{
47+
"package_property.ref_type": packages.PropertyTypeFile,
48+
}
49+
propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id"))
50+
51+
propsCondBlock := builder.NewCond()
52+
for name, value := range props {
53+
propsCondBlock = propsCondBlock.Or(builder.Eq{
54+
"package_property.name": name,
55+
"package_property.value": value,
56+
})
57+
}
58+
propsCond = propsCond.And(propsCondBlock)
59+
60+
cond = cond.And(builder.Eq{
61+
strconv.Itoa(len(props)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"),
62+
})
63+
}
64+
65+
cond = cond.
66+
And(builder.Expr("pv2.id IS NULL"))
67+
68+
joinCond := builder.
69+
Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))").
70+
And(builder.Eq{"pv2.is_internal": false})
71+
72+
pfs := make([]*packages.PackageFile, 0, 10)
73+
err := db.GetEngine(ctx).
74+
Table("package_file").
75+
Select("package_file.*").
76+
Join("INNER", "package_version", "package_version.id = package_file.version_id").
77+
Join("LEFT", "package_version pv2", joinCond).
78+
Join("INNER", "package", "package.id = package_version.package_id").
79+
Where(cond).
80+
Desc("package_version.created_unix").
81+
Find(&pfs)
82+
if err != nil {
83+
return nil, err
84+
}
85+
86+
return packages.GetPackageFileDescriptors(ctx, pfs)
87+
}
88+
89+
// GetDistributions gets all available distributions
90+
func GetDistributions(ctx context.Context, ownerID int64) ([]string, error) {
91+
return getDistinctPropertyValues(ctx, ownerID, "", debian_module.PropertyDistribution)
92+
}
93+
94+
// GetComponents gets all available components for the given distribution
95+
func GetComponents(ctx context.Context, ownerID int64, distribution string) ([]string, error) {
96+
return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyComponent)
97+
}
98+
99+
// GetArchitectures gets all available architectures for the given distribution
100+
func GetArchitectures(ctx context.Context, ownerID int64, distribution string) ([]string, error) {
101+
return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyArchitecture)
102+
}
103+
104+
func getDistinctPropertyValues(ctx context.Context, ownerID int64, distribution, propName string) ([]string, error) {
105+
var cond builder.Cond = builder.Eq{
106+
"package_property.ref_type": packages.PropertyTypeFile,
107+
"package_property.name": propName,
108+
"package.type": packages.TypeDebian,
109+
"package.owner_id": ownerID,
110+
}
111+
if distribution != "" {
112+
innerCond := builder.
113+
Expr("pp.ref_id = package_property.ref_id").
114+
And(builder.Eq{
115+
"pp.ref_type": packages.PropertyTypeFile,
116+
"pp.name": debian_module.PropertyDistribution,
117+
"pp.value": distribution,
118+
})
119+
cond = cond.And(builder.Exists(builder.Select("pp.ref_id").From("package_property pp").Where(innerCond)))
120+
}
121+
122+
values := make([]string, 0, 5)
123+
return values, db.GetEngine(ctx).
124+
Table("package_property").
125+
Distinct("package_property.value").
126+
Join("INNER", "package_file", "package_file.id = package_property.ref_id").
127+
Join("INNER", "package_version", "package_version.id = package_file.version_id").
128+
Join("INNER", "package", "package.id = package_version.package_id").
129+
Where(cond).
130+
Find(&values)
131+
}

0 commit comments

Comments
 (0)