Artifacts retention and auto clean up

models/actions/artifact.go

@@ -9,6 +9,7 @@ package actions
 import (
 	"context"
 	"errors"
+	"time"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -22,6 +23,8 @@ const (
 	ArtifactStatusUploadConfirmed = 2
 	// ArtifactStatusUploadError is the status of an artifact upload that is errored
 	ArtifactStatusUploadError = 3
+	// ArtifactStatusExpired is the status of an artifact that is expired
+	ArtifactStatusExpired = 4
 )
 
 func init() {
@@ -45,9 +48,10 @@ type ActionArtifact struct {
 	Status             int64              `xorm:"index"`                         // The status of the artifact, uploading, expired or need-delete
 	CreatedUnix        timeutil.TimeStamp `xorm:"created"`
 	UpdatedUnix        timeutil.TimeStamp `xorm:"updated index"`
+	ExpiredUnix        timeutil.TimeStamp `xorm:"index"` // The time when the artifact will be expired
 }
 
-func CreateArtifact(ctx context.Context, t *ActionTask, artifactName, artifactPath string) (*ActionArtifact, error) {
+func CreateArtifact(ctx context.Context, t *ActionTask, artifactName, artifactPath string, expiredDays int64) (*ActionArtifact, error) {
 	if err := t.LoadJob(ctx); err != nil {
 		return nil, err
 	}
@@ -62,6 +66,7 @@ func CreateArtifact(ctx context.Context, t *ActionTask, artifactName, artifactPa
 			OwnerID:      t.OwnerID,
 			CommitSHA:    t.CommitSHA,
 			Status:       ArtifactStatusUploadPending,
+			ExpiredUnix:  timeutil.TimeStamp(time.Now().Unix() + 3600*24*expiredDays),
 		}
 		if _, err := db.GetEngine(ctx).Insert(artifact); err != nil {
 			return nil, err
@@ -126,15 +131,16 @@ func ListUploadedArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionAr
 type ActionArtifactMeta struct {
 	ArtifactName string
 	FileSize     int64
+	Status       int64
 }
 
 // ListUploadedArtifactsMeta returns all uploaded artifacts meta of a run
 func ListUploadedArtifactsMeta(ctx context.Context, runID int64) ([]*ActionArtifactMeta, error) {
 	arts := make([]*ActionArtifactMeta, 0, 10)
 	return arts, db.GetEngine(ctx).Table("action_artifact").
-		Where("run_id=? AND status=?", runID, ArtifactStatusUploadConfirmed).
+		Where("run_id=? AND (status=? OR status=?)", runID, ArtifactStatusUploadConfirmed, ArtifactStatusExpired).
 		GroupBy("artifact_name").
-		Select("artifact_name, sum(file_size) as file_size").
+		Select("artifact_name, sum(file_size) as file_size, max(status) as status").
 		Find(&arts)
 }
 
@@ -149,3 +155,16 @@ func ListArtifactsByRunIDAndName(ctx context.Context, runID int64, name string)
 	arts := make([]*ActionArtifact, 0, 10)
 	return arts, db.GetEngine(ctx).Where("run_id=? AND artifact_name=?", runID, name).Find(&arts)
 }
+
+// ListExpiredArtifacts returns all expired artifacts but not deleted
+func ListExpiredArtifacts(ctx context.Context) ([]*ActionArtifact, error) {
+	arts := make([]*ActionArtifact, 0, 10)
+	return arts, db.GetEngine(ctx).
+		Where("expired_unix < ? AND status = ?", timeutil.TimeStamp(time.Now().Unix()), ArtifactStatusUploadConfirmed).Find(&arts)
+}
+
+// SetArtifactExpired sets an artifact to expired
+func SetArtifactExpired(ctx context.Context, artifactID int64) error {
+	_, err := db.GetEngine(ctx).Where("id=?", artifactID).Cols("status").Update(&ActionArtifact{Status: ArtifactStatusExpired})
+	return err
+}

modules/setting/actions.go

@@ -13,10 +13,11 @@ import (
 // Actions settings
 var (
 	Actions = struct {
-		LogStorage        *Storage // how the created logs should be stored
-		ArtifactStorage   *Storage // how the created artifacts should be stored
-		Enabled           bool
-		DefaultActionsURL defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"`
+		LogStorage            *Storage // how the created logs should be stored
+		ArtifactStorage       *Storage // how the created artifacts should be stored
+		ArtifactRetentionDays int64    `ini:"ARTIFACT_RETENTION_DAYS"`
+		Enabled               bool
+		DefaultActionsURL     defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"`
 	}{
 		Enabled:           false,
 		DefaultActionsURL: defaultActionsURLGitHub,
@@ -76,5 +77,10 @@ func loadActionsFrom(rootCfg ConfigProvider) error {
 
 	Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec)
 
+	// default to 90 days in Github Actions
+	if Actions.ArtifactRetentionDays <= 0 {
+		Actions.ArtifactRetentionDays = 90
+	}
+
 	return err
 }

options/locale/locale_en-US.ini

@@ -2704,6 +2704,7 @@ dashboard.reinit_missing_repos = Reinitialize all missing Git repositories for w
 dashboard.sync_external_users = Synchronize external user data
 dashboard.cleanup_hook_task_table = Cleanup hook_task table
 dashboard.cleanup_packages = Cleanup expired packages
+dashboard.cleanup_actions = Cleanup actions expired logs and artifacts
 dashboard.server_uptime = Server Uptime
 dashboard.current_goroutine = Current Goroutines
 dashboard.current_memory_usage = Current Memory Usage

routers/api/actions/artifacts.go

@@ -170,8 +170,9 @@ func (ar artifactRoutes) buildArtifactURL(runID int64, artifactHash, suffix stri
 }
 
 type getUploadArtifactRequest struct {
-	Type string
-	Name string
+	Type          string
+	Name          string
+	RetentionDays int64
 }
 
 type getUploadArtifactResponse struct {
@@ -192,10 +193,16 @@ func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) {
 		return
 	}
 
+	// set retention days
+	retentionQuery := ""
+	if req.RetentionDays > 0 {
+		retentionQuery = fmt.Sprintf("?retentionDays=%d", req.RetentionDays)
+	}
+
 	// use md5(artifact_name) to create upload url
 	artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(req.Name)))
 	resp := getUploadArtifactResponse{
-		FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"),
+		FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"+retentionQuery),
 	}
 	log.Debug("[artifact] get upload url: %s", resp.FileContainerResourceURL)
 	ctx.JSON(http.StatusOK, resp)
@@ -219,8 +226,21 @@ func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) {
 		return
 	}
 
+	// get artifact retention days
+	expiredDays := setting.Actions.ArtifactRetentionDays
+	if queryRetentionDays := ctx.Req.URL.Query().Get("retentionDays"); queryRetentionDays != "" {
+		expiredDays, err = strconv.ParseInt(queryRetentionDays, 10, 64)
+		if err != nil {
+			log.Error("Error parse retention days: %v", err)
+			ctx.Error(http.StatusBadRequest, "Error parse retention days")
+			return
+		}
+	}
+	log.Debug("[artifact] upload chunk, name: %s, path: %s, size: %d, retention days: %d",
+		artifactName, artifactPath, fileRealTotalSize, expiredDays)
+
 	// create or get artifact with name and path
-	artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath)
+	artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays)
 	if err != nil {
 		log.Error("Error create or get artifact: %v", err)
 		ctx.Error(http.StatusInternalServerError, "Error create or get artifact")

routers/web/repo/actions/view.go

@@ -481,8 +481,9 @@ type ArtifactsViewResponse struct {
 }
 
 type ArtifactsViewItem struct {
-	Name string `json:"name"`
-	Size int64  `json:"size"`
+	Name   string `json:"name"`
+	Size   int64  `json:"size"`
+	Status string `json:"status"`
 }
 
 func ArtifactsView(ctx *context_module.Context) {
@@ -505,9 +506,14 @@ func ArtifactsView(ctx *context_module.Context) {
 		Artifacts: make([]*ArtifactsViewItem, 0, len(artifacts)),
 	}
 	for _, art := range artifacts {
+		status := "completed"
+		if art.Status == actions_model.ArtifactStatusExpired {
+			status = "expired"
+		}
 		artifactsResponse.Artifacts = append(artifactsResponse.Artifacts, &ArtifactsViewItem{
-			Name: art.ArtifactName,
-			Size: art.FileSize,
+			Name:   art.ArtifactName,
+			Size:   art.FileSize,
+			Status: status,
 		})
 	}
 	ctx.JSON(http.StatusOK, artifactsResponse)

services/actions/cleanup.go

@@ -0,0 +1,42 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"context"
+	"time"
+
+	"code.gitea.io/gitea/models/actions"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/storage"
+)
+
+// Cleanup removes expired actions logs, data and artifacts
+func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
+	// TODO: clean up expired actions logs
+
+	// clean up expired artifacts
+	return CleanupArtifacts(taskCtx)
+}
+
+// CleanupArtifacts removes expired artifacts and set records expired status
+func CleanupArtifacts(taskCtx context.Context) error {
+	artifacts, err := actions.ListExpiredArtifacts(taskCtx)
+	if err != nil {
+		return err
+	}
+	log.Info("Found %d expired artifacts", len(artifacts))
+	for _, artifact := range artifacts {
+		if err := storage.ActionsArtifacts.Delete(artifact.StoragePath); err != nil {
+			log.Error("Cannot delete artifact %d: %v", artifact.ID, err)
+			continue
+		}
+		if err := actions.SetArtifactExpired(taskCtx, artifact.ID); err != nil {
+			log.Error("Cannot set artifact %d expired: %v", artifact.ID, err)
+			continue
+		}
+		log.Info("Artifact %d set expired", artifact.ID)
+	}
+	return nil
+}

services/cron/tasks_basic.go

@@ -13,6 +13,7 @@ import (
 	"code.gitea.io/gitea/models/webhook"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/actions"
 	"code.gitea.io/gitea/services/auth"
 	"code.gitea.io/gitea/services/migrations"
 	mirror_service "code.gitea.io/gitea/services/mirror"
@@ -156,6 +157,20 @@ func registerCleanupPackages() {
 	})
 }
 
+func registerActionsCleanup() {
+	RegisterTaskFatal("cleanup_actions", &OlderThanConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    true,
+			RunAtStart: true,
+			Schedule:   "@midnight",
+		},
+		OlderThan: 24 * time.Hour,
+	}, func(ctx context.Context, _ *user_model.User, config Config) error {
+		realConfig := config.(*OlderThanConfig)
+		return actions.Cleanup(ctx, realConfig.OlderThan)
+	})
+}
+
 func initBasicTasks() {
 	if setting.Mirror.Enabled {
 		registerUpdateMirrorTask()
@@ -172,4 +187,7 @@ func initBasicTasks() {
 	if setting.Packages.Enabled {
 		registerCleanupPackages()
 	}
+	if setting.Actions.Enabled {
+		registerActionsCleanup()
+	}
 }

tests/integration/api_actions_artifact_test.go

@@ -18,8 +18,9 @@ type uploadArtifactResponse struct {
 }
 
 type getUploadArtifactRequest struct {
-	Type string
-	Name string
+	Type          string
+	Name          string
+	RetentionDays int64
 }
 
 func TestActionsArtifactUploadSingleFile(t *testing.T) {
@@ -252,3 +253,40 @@ func TestActionsArtifactDownloadMultiFiles(t *testing.T) {
 		assert.Equal(t, resp.Body.String(), body)
 	}
 }
+
+func TestActionsArtifactUploadWithRetentionDays(t *testing.T) {
+	defer tests.PrepareTestEnv(t)()
+
+	// acquire artifact upload url
+	req := NewRequestWithJSON(t, "POST", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts", getUploadArtifactRequest{
+		Type:          "actions_storage",
+		Name:          "artifact-retention-days",
+		RetentionDays: 9,
+	})
+	req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
+	resp := MakeRequest(t, req, http.StatusOK)
+	var uploadResp uploadArtifactResponse
+	DecodeJSON(t, resp, &uploadResp)
+	assert.Contains(t, uploadResp.FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts")
+	assert.Contains(t, uploadResp.FileContainerResourceURL, "?retentionDays=9")
+
+	// get upload url
+	idx := strings.Index(uploadResp.FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/")
+	url := uploadResp.FileContainerResourceURL[idx:] + "&itemPath=artifact-retention-days/abc.txt"
+
+	// upload artifact chunk
+	body := strings.Repeat("A", 1024)
+	req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
+	req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
+	req.Header.Add("Content-Range", "bytes 0-1023/1024")
+	req.Header.Add("x-tfs-filelength", "1024")
+	req.Header.Add("x-actions-results-md5", "1HsSe8LeLWh93ILaw1TEFQ==") // base64(md5(body))
+	MakeRequest(t, req, http.StatusOK)
+
+	t.Logf("Create artifact confirm")
+
+	// confirm artifact upload
+	req = NewRequest(t, "PATCH", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts?artifactName=artifact-retention-days")
+	req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
+	MakeRequest(t, req, http.StatusOK)
+}

web_src/js/components/RepoActionView.vue

@@ -50,9 +50,12 @@
           </div>
           <ul class="job-artifacts-list">
             <li class="job-artifacts-item" v-for="artifact in artifacts" :key="artifact.name">
-              <a class="job-artifacts-link" target="_blank" :href="run.link+'/artifacts/'+artifact.name">
+              <a v-if="artifact.status!=='expired'" class="job-artifacts-link" target="_blank" :href="run.link+'/artifacts/'+artifact.name">
                 <SvgIcon name="octicon-file" class="ui text black job-artifacts-icon"/>{{ artifact.name }}
               </a>
+              <span v-if="artifact.status==='expired'">
+                <SvgIcon name="octicon-file" class="ui text black job-artifacts-icon"/>{{ artifact.name }}
+              </span>
             </li>
           </ul>
         </div>