Merge branch 'main' into k8s-client-app

Author: kripanshdevtron

App.go

@@ -30,9 +30,9 @@ import (
 
 	"github.com/casbin/casbin"
 	authMiddleware "github.com/devtron-labs/authenticator/middleware"
+	pubsub "github.com/devtron-labs/common-lib/pubsub-lib"
 	"github.com/devtron-labs/devtron/api/router"
 	"github.com/devtron-labs/devtron/api/sse"
-	"github.com/devtron-labs/devtron/client/pubsub"
 	"github.com/devtron-labs/devtron/internal/middleware"
 	"github.com/devtron-labs/devtron/pkg/user"
 	"github.com/go-pg/pg"
@@ -48,7 +48,7 @@ type App struct {
 	Enforcer      *casbin.SyncedEnforcer
 	server        *http.Server
 	db            *pg.DB
-	pubsubClient  *pubsub.PubSubClient
+	pubsubClient  *pubsub.PubSubClientServiceImpl
 	posthogClient *telemetry.PosthogClient
 	// used for local dev only
 	serveTls           bool
@@ -61,7 +61,7 @@ func NewApp(router *router.MuxRouter,
 	sse *sse.SSE,
 	enforcer *casbin.SyncedEnforcer,
 	db *pg.DB,
-	pubsubClient *pubsub.PubSubClient,
+	pubsubClient *pubsub.PubSubClientServiceImpl,
 	sessionManager2 *authMiddleware.SessionManager,
 	posthogClient *telemetry.PosthogClient,
 ) *App {
@@ -145,7 +145,6 @@ func (app *App) Stop() {
 		app.Logger.Errorw("error in closing db connection", "err", err)
 	}
 	//Close not needed if you Drain.
-	err = app.pubsubClient.Conn.Drain()
 
 	if err != nil {
 		app.Logger.Errorw("Error in draining nats connection", "error", err)

CHANGELOG/release-notes-v0.6.11.md

@@ -0,0 +1,47 @@
+## v0.6.11
+
+## Bugs
+- fix: Nats server version upgrade (#2837)
+- fix: K8s side resources error handled some unable to fetch some kinds in cluster (#2896)
+- fix: helm and gitOps pipeline feature gap changes (#2891)
+- fix: Adding /Deleting git material reflection in workflow editor and build & deploy (#2744)
+- fix: issue resolved for notification on deployment trigger and telemetry summary event (#2893)
+- fix: helm manifest output in deployment history (#2822)
+- fix: sending appName in the response of getAppWorkflow API (#2879)
+- fix: cluster not reachable handling (#2872)
+## Enhancements
+- feat: Support of reuse container image in containers section in reference chart 4-17 (#2675)
+- feat: Added Rollout chart 4.17 and Deployment chart 1.1 with canary using flagger and Istio support (#2910)
+- feat: Codacy plugin integration (#2892)
+- feat: Semgrep plugin integration (#2877)
+- perf: Tracing observability tool integration (#2782)
+## Documentation
+- docs: removed command to install a particular version of Devtron (#2888)
+- docs: Updated README.md file for Devtron chart (#2887)
+- docs: added manage tags (#2907)
+- docs: updated target platform (#2857)
+## Others
+- task: added new fields for eso secrets (#2899)
+- task: updated deployment status timeline api for helm based pipelines (#2897)
+- task: updated logic for adding health status timeline (#2870)
+
+
+## v0.6.11-rc.0
+
+## Bugs
+- fix: bitbucket gitops config fixes (#2768)
+- fix: updated sonarqube plugin script (#2855)
+- fix: updated the description in sonarqube migration (#2853)
+- fix: External Ci Trigger - Won't show images for manual Cd pipelines (#2802)
+- fix: app create api bug fix if chart ref is different (#2851)
+- fix: Dockerfile git repo delete tooltip view to build config (#2792)
+## Enhancements
+- feat: Option for changing project in Helm apps and project assignment to CLI helm apps. (#2780)
+- feat: Audit logs for resource deletion from devtron UI  (#2791)
+- feat: verify token function scope made public (#2856)
+- feat: Sonarqube polling script update (#2846)
+- feat: deployment status restructuring (#2825)
+## Documentation
+- docs: updated aws external secret doc (#2781)
+- docs: updated broken links and minor updates (resource browser) (#2844)
+- docs: added cluster, k8s resource browser, k8s resource permission (#2787)

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18  AS build-env
+FROM golang:1.19  AS build-env
 
 RUN echo $GOPATH
 RUN apt update

DockerfileEA

@@ -1,4 +1,4 @@
-FROM golang:1.18  AS build-env
+FROM golang:1.19  AS build-env
 
 RUN echo $GOPATH
 RUN apt update

Wire.go

@@ -22,6 +22,7 @@ package main
 
 import (
 	"github.com/devtron-labs/authenticator/middleware"
+	pubsub1 "github.com/devtron-labs/common-lib/pubsub-lib"
 	"github.com/devtron-labs/devtron/api/apiToken"
 	appStoreRestHandler "github.com/devtron-labs/devtron/api/appStore"
 	appStoreDeployment "github.com/devtron-labs/devtron/api/appStore/deployment"
@@ -58,10 +59,10 @@ import (
 	"github.com/devtron-labs/devtron/client/grafana"
 	jClient "github.com/devtron-labs/devtron/client/jira"
 	"github.com/devtron-labs/devtron/client/lens"
-	pubsub2 "github.com/devtron-labs/devtron/client/pubsub"
 	"github.com/devtron-labs/devtron/client/telemetry"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
 	app2 "github.com/devtron-labs/devtron/internal/sql/repository/app"
+	appStatusRepo "github.com/devtron-labs/devtron/internal/sql/repository/appStatus"
 	appWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow"
 	"github.com/devtron-labs/devtron/internal/sql/repository/bulkUpdate"
 	"github.com/devtron-labs/devtron/internal/sql/repository/chartConfig"
@@ -74,6 +75,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/app"
 	"github.com/devtron-labs/devtron/pkg/appClone"
 	"github.com/devtron-labs/devtron/pkg/appClone/batch"
+	"github.com/devtron-labs/devtron/pkg/appStatus"
 	appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean"
 	appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode"
 	repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository"
@@ -397,7 +399,7 @@ func InitializeApp() (*App, error) {
 		pipeline.NewCiLogServiceImpl,
 		wire.Bind(new(pipeline.CiLogService), new(*pipeline.CiLogServiceImpl)),
 
-		pubsub2.NewPubSubClient,
+		pubsub1.NewPubSubClientServiceImpl,
 
 		pubsub.NewGitWebhookHandler,
 		wire.Bind(new(pubsub.GitWebhookHandler), new(*pubsub.GitWebhookHandlerImpl)),
@@ -518,8 +520,6 @@ func InitializeApp() (*App, error) {
 		wire.Bind(new(repository.DeploymentGroupAppRepository), new(*repository.DeploymentGroupAppRepositoryImpl)),
 		restHandler.NewPubSubClientRestHandlerImpl,
 		wire.Bind(new(restHandler.PubSubClientRestHandler), new(*restHandler.PubSubClientRestHandlerImpl)),
-		pubsub2.NewNatsPublishClientImpl,
-		wire.Bind(new(pubsub2.NatsPublishClient), new(*pubsub2.NatsPublishClientImpl)),
 
 		//Batch actions
 		batch.NewWorkflowActionImpl,
@@ -753,6 +753,13 @@ func InitializeApp() (*App, error) {
 		cron.NewCdApplicationStatusUpdateHandlerImpl,
 		wire.Bind(new(cron.CdApplicationStatusUpdateHandler), new(*cron.CdApplicationStatusUpdateHandlerImpl)),
 
+		//app_status
+		appStatusRepo.NewAppStatusRepositoryImpl,
+		wire.Bind(new(appStatusRepo.AppStatusRepository), new(*appStatusRepo.AppStatusRepositoryImpl)),
+		appStatus.NewAppStatusServiceImpl,
+		wire.Bind(new(appStatus.AppStatusService), new(*appStatus.AppStatusServiceImpl)),
+		//app_status ends
+
 		cron.GetCiWorkflowStatusUpdateConfig,
 		cron.NewCiStatusUpdateCronImpl,
 		wire.Bind(new(cron.CiStatusUpdateCron), new(*cron.CiStatusUpdateCronImpl)),

api/appStore/InstalledAppRestHandler.go

@@ -143,6 +143,12 @@ func (handler InstalledAppRestHandlerImpl) GetAllInstalledApp(w http.ResponseWri
 			}
 		}
 	}
+
+	var appStatuses []string
+	appStatusesStr := v.Get("appStatuses")
+	if len(appStatusesStr) > 0 {
+		appStatuses = strings.Split(appStatusesStr, ",")
+	}
 	appStoreName := v.Get("appStoreName")
 	appName := v.Get("appName")
 	offset := 0
@@ -155,7 +161,15 @@ func (handler InstalledAppRestHandlerImpl) GetAllInstalledApp(w http.ResponseWri
 	if len(sizeStr) > 0 {
 		size, _ = strconv.Atoi(sizeStr)
 	}
-	filter := &appStoreBean.AppStoreFilter{OnlyDeprecated: onlyDeprecated, ChartRepoId: chartRepoIds, AppStoreName: appStoreName, EnvIds: envIds, AppName: appName, ClusterIds: clusterIds}
+	filter := &appStoreBean.AppStoreFilter{
+		OnlyDeprecated: onlyDeprecated,
+		ChartRepoId:    chartRepoIds,
+		AppStoreName:   appStoreName,
+		EnvIds:         envIds,
+		AppName:        appName,
+		ClusterIds:     clusterIds,
+		AppStatuses:    appStatuses,
+	}
 	if size > 0 {
 		filter.Size = size
 		filter.Offset = offset

api/bean/AppView.go

@@ -62,6 +62,7 @@ type AppEnvironmentContainer struct {
 	DeploymentCounter           int                       `json:"deploymentCounter,omitempty"`
 	InstanceCounter             int                       `json:"instanceCounter,omitempty"`
 	Status                      string                    `json:"status"`
+	AppStatus                   string                    `json:"appStatus"`
 	CdStageStatus               *string                   `json:"cdStageStatus"`
 	PreStageStatus              *string                   `json:"preStageStatus"`
 	PostStageStatus             *string                   `json:"postStageStatus"`
@@ -123,6 +124,7 @@ type AppDetailContainer struct {
 }
 
 type Environment struct {
+	AppStatus       string `json:"appStatus"` //this is not the status of environment , this make sense with a specific app only
 	EnvironmentId   int    `json:"environmentId"`
 	EnvironmentName string `json:"environmentName"`
 	AppMetrics      *bool  `json:"appMetrics"`

api/helm-app/openapiClient/model_helm_app.go

@@ -33,6 +33,7 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/app"
+	"github.com/devtron-labs/devtron/pkg/appStatus"
 	service1 "github.com/devtron-labs/devtron/pkg/appStore/deployment/service"
 	"github.com/devtron-labs/devtron/pkg/cluster"
 	"github.com/devtron-labs/devtron/pkg/deploymentGroup"
@@ -85,6 +86,7 @@ type AppListingRestHandlerImpl struct {
 	installedAppService              service1.InstalledAppService
 	cdApplicationStatusUpdateHandler cron.CdApplicationStatusUpdateHandler
 	pipelineRepository               pipelineConfig.PipelineRepository
+	appStatusService                 appStatus.AppStatusService
 }
 
 type AppStatus struct {
@@ -105,7 +107,8 @@ func NewAppListingRestHandlerImpl(application application.ServiceClient,
 	helmAppClient client.HelmAppClient, clusterService cluster.ClusterService, helmAppService client.HelmAppService,
 	argoUserService argo.ArgoUserService, k8sApplicationService k8s.K8sApplicationService, installedAppService service1.InstalledAppService,
 	cdApplicationStatusUpdateHandler cron.CdApplicationStatusUpdateHandler,
-	pipelineRepository pipelineConfig.PipelineRepository) *AppListingRestHandlerImpl {
+	pipelineRepository pipelineConfig.PipelineRepository,
+	appStatusService appStatus.AppStatusService) *AppListingRestHandlerImpl {
 	appListingHandler := &AppListingRestHandlerImpl{
 		application:                      application,
 		appListingService:                appListingService,
@@ -124,6 +127,7 @@ func NewAppListingRestHandlerImpl(application application.ServiceClient,
 		installedAppService:              installedAppService,
 		cdApplicationStatusUpdateHandler: cdApplicationStatusUpdateHandler,
 		pipelineRepository:               pipelineRepository,
+		appStatusService:                 appStatusService,
 	}
 	return appListingHandler
 }
@@ -806,6 +810,12 @@ func (handler AppListingRestHandlerImpl) fetchResourceTree(w http.ResponseWriter
 				handler.logger.Errorw("error in syncing pipeline status", "err", err)
 			}
 		}
+		//updating app_status table here
+		err = handler.appStatusService.UpdateStatusWithAppIdEnvId(appDetail.AppId, appDetail.EnvironmentId, resp.Status)
+		if err != nil {
+			handler.logger.Warnw("error in updating app status", "err", err, "appId", appDetail.AppId, "envId", appDetail.EnvironmentId)
+		}
+
 	} else if len(appDetail.AppName) > 0 && len(appDetail.EnvironmentName) > 0 && util.IsHelmApp(appDetail.DeploymentAppType) {
 		config, err := handler.helmAppService.GetClusterConf(appDetail.ClusterId)
 		if err != nil {

api/restHandler/AppListingRestHandler.go

@@ -22,8 +22,8 @@ import (
 	"net/http"
 	"strings"
 
+	pubsub "github.com/devtron-labs/common-lib/pubsub-lib"
 	"github.com/devtron-labs/devtron/api/restHandler/common"
-	"github.com/devtron-labs/devtron/client/pubsub"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
 	"go.uber.org/zap"
 )
@@ -33,22 +33,27 @@ type PubSubClientRestHandler interface {
 }
 
 type PubSubClientRestHandlerImpl struct {
-	natsPublishClient pubsub.NatsPublishClient
-	logger            *zap.SugaredLogger
-	cdConfig          *pipeline.CdConfig
+	pubsubClient *pubsub.PubSubClientServiceImpl
+	logger       *zap.SugaredLogger
+	cdConfig     *pipeline.CdConfig
 }
 
-func NewPubSubClientRestHandlerImpl(natsPublishClient pubsub.NatsPublishClient, logger *zap.SugaredLogger, cdConfig *pipeline.CdConfig) *PubSubClientRestHandlerImpl {
+type PublishRequest struct {
+	Topic   string          `json:"topic"`
+	Payload json.RawMessage `json:"payload"`
+}
+
+func NewPubSubClientRestHandlerImpl(pubsubClient *pubsub.PubSubClientServiceImpl, logger *zap.SugaredLogger, cdConfig *pipeline.CdConfig) *PubSubClientRestHandlerImpl {
 	return &PubSubClientRestHandlerImpl{
-		natsPublishClient: natsPublishClient,
-		logger:            logger,
-		cdConfig:          cdConfig,
+		pubsubClient: pubsubClient,
+		logger:       logger,
+		cdConfig:     cdConfig,
 	}
 }
 
 func (impl *PubSubClientRestHandlerImpl) PublishEventsToNats(w http.ResponseWriter, r *http.Request) {
 	decoder := json.NewDecoder(r.Body)
-	var publishRequest pubsub.PublishRequest
+	var publishRequest PublishRequest
 	err := decoder.Decode(&publishRequest)
 	if err != nil {
 		impl.logger.Errorw("request err, HandleExternalCiWebhook", "err", err, "payload", publishRequest)
@@ -68,8 +73,13 @@ func (impl *PubSubClientRestHandlerImpl) PublishEventsToNats(w http.ResponseWrit
 		common.WriteJsonResp(w, err, "Unauthorized req", http.StatusUnauthorized)
 		return
 	}
-
-	err = impl.natsPublishClient.Publish(&publishRequest)
+	data, err := json.Marshal(publishRequest.Payload)
+	if err != nil {
+		impl.logger.Errorw("error occurred in un-marshaling publishResquest in publishEvents to Nats", "err", err)
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	err = impl.pubsubClient.Publish(publishRequest.Topic, string(data))
 	if err != nil {
 		impl.logger.Errorw("service err, HandleExternalCiWebhook", "err", err, "payload", publishRequest)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)