feat(nuget): support nuspec manifest download

viceice · viceice · commit 5a54d476c4bf · 2024-04-05T10:25:30.000+02:00
diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go
@@ -48,10 +48,11 @@ const maxNuspecFileSize = 3 * 1024 * 1024
 
 // Package represents a Nuget package
 type Package struct {
-	PackageType PackageType
-	ID          string
-	Version     string
-	Metadata    *Metadata
+	PackageType   PackageType
+	ID            string
+	Version       string
+	Metadata      *Metadata
+	NuspecContent *bytes.Buffer
 }
 
 // Metadata represents the metadata of a Nuget package
@@ -130,8 +131,9 @@ func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) {
 
 // ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package
 func ParseNuspecMetaData(r io.Reader) (*Package, error) {
+	var nuspecBuf bytes.Buffer
 	var p nuspecPackage
-	if err := xml.NewDecoder(r).Decode(&p); err != nil {
+	if err := xml.NewDecoder(io.TeeReader(r, &nuspecBuf)).Decode(&p); err != nil {
 		return nil, err
 	}
 
@@ -182,10 +184,11 @@ func ParseNuspecMetaData(r io.Reader) (*Package, error) {
 		}
 	}
 	return &Package{
-		PackageType: packageType,
-		ID:          p.Metadata.ID,
-		Version:     toNormalizedVersion(v),
-		Metadata:    m,
+		PackageType:   packageType,
+		ID:            p.Metadata.ID,
+		Version:       toNormalizedVersion(v),
+		Metadata:      m,
+		NuspecContent: &nuspecBuf,
 	}, nil
 }
 
diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go
@@ -388,7 +388,8 @@ func EnumeratePackageVersionsV3(ctx *context.Context) {
 	ctx.JSON(http.StatusOK, resp)
 }
 
-// https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg
+// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec
+// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg
 func DownloadPackageFile(ctx *context.Context) {
 	packageName := ctx.Params("id")
 	packageVersion := ctx.Params("version")
@@ -431,7 +432,7 @@ func UploadPackage(ctx *context.Context) {
 		return
 	}
 
-	_, _, err := packages_service.CreatePackageAndAddFile(
+	pv, _, err := packages_service.CreatePackageAndAddFile(
 		ctx,
 		&packages_service.PackageCreationInfo{
 			PackageInfo: packages_service.PackageInfo{
@@ -465,6 +466,35 @@ func UploadPackage(ctx *context.Context) {
 		return
 	}
 
+	nuspecBuf, err := packages_module.CreateHashedBufferFromReaderWithSize(np.NuspecContent, np.NuspecContent.Len())
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer nuspecBuf.Close()
+
+	_, err = packages_service.AddFileToPackageVersionInternal(
+		ctx,
+		pv,
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename: strings.ToLower(fmt.Sprintf("%s.nuspec", np.ID)),
+			},
+			Creator: ctx.Doer,
+			Data:    nuspecBuf,
+			IsLead:  false,
+		},
+	)
+	if err != nil {
+		switch err {
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
 	ctx.Status(http.StatusCreated)
 }
 
diff --git a/services/doctor/doctor.go b/services/doctor/doctor.go
@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 )
 
 // Check represents a Doctor check
@@ -25,6 +26,7 @@ type Check struct {
 	AbortIfFailed              bool
 	SkipDatabaseInitialization bool
 	Priority                   int
+	InitStorage                bool
 }
 
 func initDBSkipLogger(ctx context.Context) error {
@@ -84,6 +86,7 @@ func RunChecks(ctx context.Context, colorize, autofix bool, checks []*Check) err
 	logger := log.BaseLoggerToGeneralLogger(&doctorCheckLogger{colorize: colorize})
 	loggerStep := log.BaseLoggerToGeneralLogger(&doctorCheckStepLogger{colorize: colorize})
 	dbIsInit := false
+	storageIsInit := false
 	for i, check := range checks {
 		if !dbIsInit && !check.SkipDatabaseInitialization {
 			// Only open database after the most basic configuration check
@@ -94,6 +97,14 @@ func RunChecks(ctx context.Context, colorize, autofix bool, checks []*Check) err
 			}
 			dbIsInit = true
 		}
+		if !storageIsInit && check.InitStorage {
+			if err := storage.Init(); err != nil {
+				logger.Error("Error whilst initializing the storage: %v", err)
+				logger.Error("Check if you are using the right config file. You can use a --config directive to specify one.")
+				return nil
+			}
+			storageIsInit = true
+		}
 		logger.Info("\n[%d] %s", i+1, check.Title)
 		if err := check.Run(ctx, loggerStep, autofix); err != nil {
 			if check.AbortIfFailed {
diff --git a/services/doctor/packages_nuget.go b/services/doctor/packages_nuget.go
@@ -0,0 +1,172 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	nuget_module "code.gitea.io/gitea/modules/packages/nuget"
+	packages_service "code.gitea.io/gitea/services/packages"
+)
+
+func init() {
+	Register(&Check{
+		Title:       "Extract Nuget Nuspec Files to content store",
+		Name:        "packages-nuget-nuspec",
+		IsDefault:   false,
+		Run:         PackagesNugetNuspecCheck,
+		Priority:    15,
+		InitStorage: true,
+	})
+}
+
+// getAllUsers returns a slice of all users and organizations found in DB.
+func getAllUsers(ctx context.Context) ([]*user.User, error) {
+	users := make([]*user.User, 0)
+	return users, db.GetEngine(ctx).OrderBy("id").Find(&users)
+}
+
+func PackagesNugetNuspecCheck(ctx context.Context, logger log.Logger, autofix bool) error {
+	users, err := getAllUsers(ctx)
+	userMap := make(map[int64]*user.User, len(users))
+
+	for _, u := range users {
+		userMap[u.ID] = u
+	}
+
+	if err != nil {
+		logger.Error("Failed to get users: %v", err)
+		return err
+	}
+
+	logger.Info("Found %d users", len(users))
+
+	fixed := 0
+	errors := 0
+
+	for _, user := range users {
+		pkgs, err := packages.GetPackagesByType(ctx, user.ID, packages.TypeNuGet)
+		if err != nil {
+			logger.Error("Failed to get NuGet packages for owner %s: %v", user.Name, err)
+			continue
+		}
+
+		logger.Info("Found %d NuGet packages for owner %s", len(pkgs), user.Name)
+
+		for _, pkg := range pkgs {
+			pvs, _, err := packages.SearchVersions(ctx, &packages.PackageSearchOptions{
+				Type:      packages.TypeNuGet,
+				PackageID: pkg.ID,
+			})
+			if err != nil {
+				// Should never happen
+				logger.Error("Failed to search for versions for package %s: %v", pkg.Name, err)
+				continue
+			}
+
+			logger.Info("Found %d versions for package %s", pkg.Name, user.Name)
+
+			for _, pv := range pvs {
+
+				pfs, err := packages.GetFilesByVersionID(ctx, pv.ID)
+				if err != nil {
+					logger.Error("Failed to get files for package version %s %s: %v", pkg.Name, pv.Version, err)
+					errors++
+					continue
+				}
+
+				if slices.IndexFunc(pfs, func(pf *packages.PackageFile) bool { return strings.HasSuffix(pf.LowerName, ".nuspec") }) >= 0 {
+					logger.Debug("Nuspec file already exists for %s %s", pkg.Name, pv.Version)
+					continue
+				}
+
+				nuspecIdx := slices.IndexFunc(pfs, func(pf *packages.PackageFile) bool { return pf.IsLead })
+
+				if nuspecIdx < 0 {
+					logger.Error("Missing nupkg file for %s %s", pkg.Name, pv.Version)
+					errors++
+					continue
+				}
+
+				pf := pfs[nuspecIdx]
+
+				creator, ok := userMap[pv.CreatorID]
+				if !ok {
+					logger.Warn("Failed to find creator for %s %s", pkg.Name, pv.Version)
+					creator = user
+				}
+
+				logger.Info("Missing nuspec file found for %s %s", pkg.Name, pv.Version)
+				fixed++
+
+				if autofix {
+					s, _, _, err := packages_service.GetPackageFileStream(ctx, pf)
+					if err != nil {
+						logger.Error("Failed to get file stream for %s %s: %v", pkg.Name, pv.Version, err)
+						errors++
+						continue
+					}
+					defer s.Close()
+
+					buf, err := packages_module.CreateHashedBufferFromReader(s)
+					if err != nil {
+						logger.Error("Failed to create hashed buffer for nupkg from reader for %s %s: %v", pkg.Name, pv.Version, err)
+						errors++
+						continue
+					}
+					defer buf.Close()
+
+					np, err := nuget_module.ParsePackageMetaData(buf, buf.Size())
+					if err != nil {
+						logger.Error("Failed to parse package metadata for %s %s: %v", pkg.Name, pv.Version, err)
+						errors++
+						continue
+					}
+
+					nuspecBuf, err := packages_module.CreateHashedBufferFromReaderWithSize(np.NuspecContent, np.NuspecContent.Len())
+					if err != nil {
+						logger.Error("Failed to create hashed buffer for nuspec from reader for %s %s: %v", pkg.Name, pv.Version, err)
+						errors++
+						continue
+					}
+					defer nuspecBuf.Close()
+
+					_, err = packages_service.AddFileToPackageVersionInternal(
+						ctx,
+						pv,
+						&packages_service.PackageFileCreationInfo{
+							PackageFileInfo: packages_service.PackageFileInfo{
+								Filename: fmt.Sprintf("%s.nuspec", pkg.LowerName),
+							},
+							Creator: creator,
+							Data:    nuspecBuf,
+							IsLead:  false,
+						},
+					)
+					if err != nil {
+						logger.Error("Failed to add nuspec file for %s %s: %v", pkg.Name, pv.Version, err)
+						errors++
+					}
+				}
+			}
+		}
+	}
+
+	if fixed > 0 {
+		logger.Info("Fixed %d NuGet packages by extracting nuspec files", fixed)
+	}
+	if errors > 0 {
+		logger.Info("Failed to fix %d nuspec files", errors)
+	}
+
+	return nil
+}
diff --git a/tests/integration/api_packages_nuget_test.go b/tests/integration/api_packages_nuget_test.go
@@ -112,6 +112,20 @@ func TestPackageNuGet(t *testing.T) {
 		return &buf
 	}
 
+	nuspec := `<?xml version="1.0" encoding="utf-8"?>
+		<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+			<metadata>
+				<id>` + packageName + `</id>
+				<version>` + packageVersion + `</version>
+				<authors>` + packageAuthors + `</authors>
+				<description>` + packageDescription + `</description>
+				<dependencies>
+					<group targetFramework=".NETStandard2.0">
+						<dependency id="Microsoft.CSharp" version="4.5.0" />
+					</group>
+				</dependencies>
+			</metadata>
+		</package>`
 	content, _ := io.ReadAll(createPackage(packageName, packageVersion))
 
 	url := fmt.Sprintf("/api/packages/%s/nuget", user.Name)
@@ -224,7 +238,7 @@ func TestPackageNuGet(t *testing.T) {
 
 			pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet)
 			assert.NoError(t, err)
-			assert.Len(t, pvs, 1)
+			assert.Len(t, pvs, 1, "Should have one version")
 
 			pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0])
 			assert.NoError(t, err)
@@ -235,7 +249,7 @@ func TestPackageNuGet(t *testing.T) {
 
 			pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
 			assert.NoError(t, err)
-			assert.Len(t, pfs, 1)
+			assert.Len(t, pfs, 2, "Should have 2 files: nuget and nuspec")
 			assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name)
 			assert.True(t, pfs[0].IsLead)
 
@@ -302,16 +316,27 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`)
 
 			pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
 			assert.NoError(t, err)
-			assert.Len(t, pfs, 3)
+			assert.Len(t, pfs, 4, "Should have 4 files: nupkg, snupkg, nuspec and pdb")
 			for _, pf := range pfs {
 				switch pf.Name {
 				case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion):
+					assert.True(t, pf.IsLead)
+
+					pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID)
+					assert.NoError(t, err)
+					assert.Equal(t, int64(414), pb.Size)
 				case fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion):
 					assert.False(t, pf.IsLead)
 
 					pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID)
 					assert.NoError(t, err)
 					assert.Equal(t, int64(616), pb.Size)
+				case fmt.Sprintf("%s.nuspec", packageName):
+					assert.False(t, pf.IsLead)
+
+					pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID)
+					assert.NoError(t, err)
+					assert.Equal(t, int64(453), pb.Size)
 				case symbolFilename:
 					assert.False(t, pf.IsLead)
 
@@ -353,6 +378,12 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`)
 
 		assert.Equal(t, content, resp.Body.Bytes())
 
+		req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.nuspec", url, packageName, packageVersion, packageName)).
+			AddBasicAuth(user.Name)
+		resp = MakeRequest(t, req, http.StatusOK)
+
+		assert.Equal(t, nuspec, resp.Body.String())
+
 		checkDownloadCount(1)
 
 		req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)).
diff --git a/tests/integration/doctor_packages_nuget_test.go b/tests/integration/doctor_packages_nuget_test.go
