Skip to content

Commit 15ccd00

Browse files
bjee19bjee19
committed
Collect telemetry for control data plane split
1 parent e2115c3 commit 15ccd00

File tree

7 files changed

+158
-56
lines changed

7 files changed

+158
-56
lines changed

‎internal/mode/static/telemetry/collector.go

+54-16
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@ import (
1212
appsv1 "k8s.io/api/apps/v1"
1313
v1 "k8s.io/api/core/v1"
1414
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/labels"
1516
"k8s.io/apimachinery/pkg/types"
1617
k8sversion "k8s.io/apimachinery/pkg/util/version"
1718
"sigs.k8s.io/controller-runtime/pkg/client"
1819

1920
ngfAPI "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1"
21+
"github.com/nginx/nginx-gateway-fabric/internal/framework/controller"
2022
"github.com/nginx/nginx-gateway-fabric/internal/framework/kinds"
2123
"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
2224
"github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/dataplane"
@@ -60,8 +62,10 @@ type Data struct {
6062
// then lastly by directive string.
6163
SnippetsFiltersDirectivesCount []int64
6264
NGFResourceCounts // embedding is required by the generator.
63-
// NGFReplicaCount is the number of replicas of the NGF Pod.
64-
NGFReplicaCount int64
65+
// NginxPodCount is the total number of Nginx data plane Pods.
66+
NginxPodCount int64
67+
// ControlPlanePodCount is the total number of NGF control plane Pods.
68+
ControlPlanePodCount int64
6569
}
6670

6771
// NGFResourceCounts stores the counts of all relevant resources that NGF processes and generates configuration from.
@@ -99,6 +103,8 @@ type NGFResourceCounts struct {
99103
SnippetsFilterCount int64
100104
// UpstreamSettingsPolicyCount is the number of UpstreamSettingsPolicies.
101105
UpstreamSettingsPolicyCount int64
106+
// GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway.
107+
GatewayAttachedNpCount int64
102108
}
103109

104110
// DataCollectorConfig holds configuration parameters for DataCollectorImpl.
@@ -152,18 +158,17 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) {
152158
return Data{}, fmt.Errorf("failed to get replica set for pod %v: %w", c.cfg.PodNSName, err)
153159
}
154160

155-
replicaCount, err := getReplicas(replicaSet)
156-
if err != nil {
157-
return Data{}, fmt.Errorf("failed to collect NGF replica count: %w", err)
158-
}
159-
160161
deploymentID, err := getDeploymentID(replicaSet)
161162
if err != nil {
162163
return Data{}, fmt.Errorf("failed to get NGF deploymentID: %w", err)
163164
}
164165

165166
snippetsFiltersDirectives, snippetsFiltersDirectivesCount := collectSnippetsFilterDirectives(g)
166167

168+
nginxPodCount := getNginxPodCount(g)
169+
170+
controlPlanePodCount := getControlPlanePodCount(ctx, c.cfg.K8sClientReader)
171+
167172
data := Data{
168173
Data: tel.Data{
169174
ProjectName: "NGF",
@@ -179,9 +184,10 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) {
179184
ImageSource: c.cfg.ImageSource,
180185
FlagNames: c.cfg.Flags.Names,
181186
FlagValues: c.cfg.Flags.Values,
182-
NGFReplicaCount: int64(replicaCount),
183187
SnippetsFiltersDirectives: snippetsFiltersDirectives,
184188
SnippetsFiltersDirectivesCount: snippetsFiltersDirectivesCount,
189+
NginxPodCount: nginxPodCount,
190+
ControlPlanePodCount: controlPlanePodCount,
185191
}
186192

187193
return data, nil
@@ -241,6 +247,18 @@ func collectGraphResourceCount(
241247
ngfResourceCounts.NginxProxyCount = int64(len(g.ReferencedNginxProxies))
242248
ngfResourceCounts.SnippetsFilterCount = int64(len(g.SnippetsFilters))
243249

250+
var gatewayAttachedNPCount int64
251+
if g.GatewayClass != nil && g.GatewayClass.NginxProxy != nil {
252+
gatewayClassNP := g.GatewayClass.NginxProxy
253+
for _, np := range g.ReferencedNginxProxies {
254+
if np != gatewayClassNP {
255+
gatewayAttachedNPCount++
256+
}
257+
}
258+
}
259+
260+
ngfResourceCounts.GatewayAttachedNpCount = gatewayAttachedNPCount
261+
244262
return ngfResourceCounts
245263
}
246264

@@ -309,14 +327,6 @@ func getPodReplicaSet(
309327
return &replicaSet, nil
310328
}
311329

312-
func getReplicas(replicaSet *appsv1.ReplicaSet) (int, error) {
313-
if replicaSet.Spec.Replicas == nil {
314-
return 0, errors.New("replica set replicas was nil")
315-
}
316-
317-
return int(*replicaSet.Spec.Replicas), nil
318-
}
319-
320330
// getDeploymentID gets the deployment ID of the provided ReplicaSet.
321331
func getDeploymentID(replicaSet *appsv1.ReplicaSet) (string, error) {
322332
replicaOwnerRefs := replicaSet.GetOwnerReferences()
@@ -495,3 +505,31 @@ func parseDirectiveContextMapIntoLists(directiveContextMap map[sfDirectiveContex
495505

496506
return directiveContextList, countList
497507
}
508+
509+
func getNginxPodCount(g *graph.Graph) int64 {
510+
var count int64
511+
for _, gateway := range g.Gateways {
512+
np := gateway.EffectiveNginxProxy
513+
if np != nil &&
514+
np.Kubernetes != nil &&
515+
np.Kubernetes.Deployment != nil &&
516+
np.Kubernetes.Deployment.Replicas != nil {
517+
count += int64(*np.Kubernetes.Deployment.Replicas)
518+
}
519+
}
520+
521+
return count
522+
}
523+
524+
func getControlPlanePodCount(ctx context.Context, k8sClient client.Reader) int64 {
525+
var podList v1.PodList
526+
opts := &client.ListOptions{
527+
LabelSelector: labels.SelectorFromSet(labels.Set{controller.AppNameLabel: "nginx-gateway-fabric"}),
528+
}
529+
530+
if err := k8sClient.List(ctx, &podList, opts); err != nil {
531+
return 0
532+
}
533+
534+
return int64(len(podList.Items))
535+
}

‎internal/mode/static/telemetry/collector_test.go

+81-33
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ import (
1818
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
1919

2020
ngfAPI "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1"
21+
"github.com/nginx/nginx-gateway-fabric/apis/v1alpha2"
22+
"github.com/nginx/nginx-gateway-fabric/internal/framework/controller"
23+
"github.com/nginx/nginx-gateway-fabric/internal/framework/helpers"
2124
"github.com/nginx/nginx-gateway-fabric/internal/framework/kinds"
2225
"github.com/nginx/nginx-gateway-fabric/internal/framework/kubernetes/kubernetesfakes"
2326
"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
@@ -35,9 +38,7 @@ type listCallsFunc = func(
3538
) error
3639

3740
func createListCallsFunc(objects ...client.ObjectList) listCallsFunc {
38-
return func(_ context.Context, object client.ObjectList, option ...client.ListOption) error {
39-
Expect(option).To(BeEmpty())
40-
41+
return func(_ context.Context, object client.ObjectList, _ ...client.ListOption) error {
4142
for _, obj := range objects {
4243
if reflect.TypeOf(obj) == reflect.TypeOf(object) {
4344
reflect.ValueOf(object).Elem().Set(reflect.ValueOf(obj).Elem())
@@ -87,6 +88,7 @@ var _ = Describe("Collector", Ordered, func() {
8788
baseListCalls listCallsFunc
8889
flags config.Flags
8990
nodeList *v1.NodeList
91+
podList *v1.PodList
9092
)
9193

9294
BeforeAll(func() {
@@ -155,6 +157,17 @@ var _ = Describe("Collector", Ordered, func() {
155157
},
156158
},
157159
}
160+
161+
podList = &v1.PodList{
162+
Items: []v1.Pod{
163+
{
164+
ObjectMeta: metav1.ObjectMeta{
165+
Name: "ngf-pod-1",
166+
Labels: map[string]string{controller.AppNameLabel: "nginx-gateway-fabric"},
167+
},
168+
},
169+
},
170+
}
158171
})
159172

160173
BeforeEach(func() {
@@ -170,7 +183,7 @@ var _ = Describe("Collector", Ordered, func() {
170183
ClusterNodeCount: 1,
171184
},
172185
NGFResourceCounts: telemetry.NGFResourceCounts{},
173-
NGFReplicaCount: 1,
186+
ControlPlanePodCount: 1,
174187
ImageSource: "local",
175188
FlagNames: flags.Names,
176189
FlagValues: flags.Values,
@@ -198,7 +211,7 @@ var _ = Describe("Collector", Ordered, func() {
198211
baseGetCalls = createGetCallsFunc(ngfPod, ngfReplicaSet, kubeNamespace)
199212
k8sClientReader.GetCalls(baseGetCalls)
200213

201-
baseListCalls = createListCallsFunc(nodeList)
214+
baseListCalls = createListCallsFunc(nodeList, podList)
202215
k8sClientReader.ListCalls(baseListCalls)
203216
})
204217

@@ -260,7 +273,24 @@ var _ = Describe("Collector", Ordered, func() {
260273
},
261274
}
262275

263-
k8sClientReader.ListCalls(createListCallsFunc(nodes))
276+
podList := &v1.PodList{
277+
Items: []v1.Pod{
278+
{
279+
ObjectMeta: metav1.ObjectMeta{
280+
Name: "ngf-pod-1",
281+
Labels: map[string]string{controller.AppNameLabel: "nginx-gateway-fabric"},
282+
},
283+
},
284+
{
285+
ObjectMeta: metav1.ObjectMeta{
286+
Name: "ngf-pod-2",
287+
Labels: map[string]string{controller.AppNameLabel: "nginx-gateway-fabric"},
288+
},
289+
},
290+
},
291+
}
292+
293+
k8sClientReader.ListCalls(createListCallsFunc(nodes, podList))
264294

265295
secret1 := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret1"}}
266296
secret2 := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret2"}}
@@ -270,11 +300,33 @@ var _ = Describe("Collector", Ordered, func() {
270300
svc2 := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc2"}}
271301
nilsvc := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nilsvc"}}
272302

303+
gcNP := graph.NginxProxy{
304+
Source: nil,
305+
ErrMsgs: nil,
306+
Valid: false,
307+
}
308+
273309
graph := &graph.Graph{
274-
GatewayClass: &graph.GatewayClass{},
310+
GatewayClass: &graph.GatewayClass{NginxProxy: &gcNP},
275311
Gateways: map[types.NamespacedName]*graph.Gateway{
276-
{Name: "gateway1"}: {},
277-
{Name: "gateway2"}: {},
312+
{Name: "gateway1"}: {
313+
EffectiveNginxProxy: &graph.EffectiveNginxProxy{
314+
Kubernetes: &v1alpha2.KubernetesSpec{
315+
Deployment: &v1alpha2.DeploymentSpec{
316+
Replicas: helpers.GetPointer(int32(1)),
317+
},
318+
},
319+
},
320+
},
321+
{Name: "gateway2"}: {
322+
EffectiveNginxProxy: &graph.EffectiveNginxProxy{
323+
Kubernetes: &v1alpha2.KubernetesSpec{
324+
Deployment: &v1alpha2.DeploymentSpec{
325+
Replicas: helpers.GetPointer(int32(3)),
326+
},
327+
},
328+
},
329+
},
278330
{Name: "gateway3"}: {},
279331
},
280332
IgnoredGatewayClasses: map[types.NamespacedName]*gatewayv1.GatewayClass{
@@ -335,9 +387,11 @@ var _ = Describe("Collector", Ordered, func() {
335387
}: {},
336388
},
337389
ReferencedNginxProxies: map[types.NamespacedName]*graph.NginxProxy{
338-
{Namespace: "test", Name: "NginxProxy-1"}: {},
339-
{Namespace: "test", Name: "NginxProxy-2"}: {},
340-
}, SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
390+
{Namespace: "test", Name: "NginxProxy-1"}: &gcNP,
391+
{Namespace: "test", Name: "NginxProxy-2"}: {Valid: true},
392+
{Namespace: "test", Name: "NginxProxy-3"}: {Valid: true},
393+
},
394+
SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
341395
{Namespace: "test", Name: "sf-1"}: {
342396
Snippets: map[ngfAPI.NginxContext]string{
343397
ngfAPI.NginxContextMain: "worker_priority 0;",
@@ -432,9 +486,10 @@ var _ = Describe("Collector", Ordered, func() {
432486
GatewayAttachedClientSettingsPolicyCount: 1,
433487
RouteAttachedClientSettingsPolicyCount: 2,
434488
ObservabilityPolicyCount: 1,
435-
NginxProxyCount: 2,
489+
NginxProxyCount: 3,
436490
SnippetsFilterCount: 3,
437491
UpstreamSettingsPolicyCount: 1,
492+
GatewayAttachedNpCount: 2,
438493
}
439494
expData.ClusterVersion = "1.29.2"
440495
expData.ClusterPlatform = "kind"
@@ -462,6 +517,9 @@ var _ = Describe("Collector", Ordered, func() {
462517
1,
463518
}
464519

520+
expData.NginxPodCount = int64(4)
521+
expData.ControlPlanePodCount = int64(2)
522+
465523
data, err := dataCollector.Collect(ctx)
466524
Expect(err).ToNot(HaveOccurred())
467525

@@ -527,7 +585,7 @@ var _ = Describe("Collector", Ordered, func() {
527585
},
528586
}
529587

530-
k8sClientReader.ListCalls(createListCallsFunc(nodes))
588+
k8sClientReader.ListCalls(createListCallsFunc(nodes, podList))
531589
expData.ClusterVersion = "unknown"
532590
expData.ClusterPlatform = "k3s"
533591

@@ -543,7 +601,7 @@ var _ = Describe("Collector", Ordered, func() {
543601
Describe("node count collector", func() {
544602
When("collecting node count data", func() {
545603
It("collects correct data for one node", func(ctx SpecContext) {
546-
k8sClientReader.ListCalls(createListCallsFunc(nodeList))
604+
k8sClientReader.ListCalls(createListCallsFunc(nodeList, podList))
547605

548606
expData.ClusterNodeCount = 1
549607

@@ -593,7 +651,7 @@ var _ = Describe("Collector", Ordered, func() {
593651
svc := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc1"}}
594652

595653
graph1 = &graph.Graph{
596-
GatewayClass: &graph.GatewayClass{},
654+
GatewayClass: &graph.GatewayClass{NginxProxy: &graph.NginxProxy{Valid: true}},
597655
Gateways: map[types.NamespacedName]*graph.Gateway{
598656
{Name: "gateway1"}: {},
599657
},
@@ -634,12 +692,14 @@ var _ = Describe("Collector", Ordered, func() {
634692
}: {},
635693
},
636694
ReferencedNginxProxies: map[types.NamespacedName]*graph.NginxProxy{
637-
{Namespace: "test", Name: "NginxProxy-1"}: {},
638-
{Namespace: "test", Name: "NginxProxy-2"}: {},
695+
{Namespace: "test", Name: "NginxProxy-1"}: {Valid: true},
639696
},
640697
SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
641698
{Namespace: "test", Name: "sf-1"}: {},
642699
},
700+
BackendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{
701+
{Namespace: "test", Name: "BackendTLSPolicy-1"}: {},
702+
},
643703
}
644704

645705
config1 = []*dataplane.Configuration{
@@ -716,9 +776,11 @@ var _ = Describe("Collector", Ordered, func() {
716776
GatewayAttachedClientSettingsPolicyCount: 1,
717777
RouteAttachedClientSettingsPolicyCount: 1,
718778
ObservabilityPolicyCount: 1,
719-
NginxProxyCount: 2,
779+
NginxProxyCount: 1,
720780
SnippetsFilterCount: 1,
721781
UpstreamSettingsPolicyCount: 1,
782+
GatewayAttachedNpCount: 1,
783+
BackendTLSPolicyCount: 1,
722784
}
723785

724786
data, err := dataCollector.Collect(ctx)
@@ -834,20 +896,6 @@ var _ = Describe("Collector", Ordered, func() {
834896
Expect(err).To(MatchError(expectedErr))
835897
})
836898

837-
It("should error if the replica set's replicas is nil", func(ctx SpecContext) {
838-
expectedErr := errors.New("replica set replicas was nil")
839-
k8sClientReader.GetCalls(mergeGetCallsWithBase(createGetCallsFunc(
840-
&appsv1.ReplicaSet{
841-
Spec: appsv1.ReplicaSetSpec{
842-
Replicas: nil,
843-
},
844-
},
845-
)))
846-
847-
_, err := dataCollector.Collect(ctx)
848-
Expect(err).To(MatchError(expectedErr))
849-
})
850-
851899
It("should error if the kubernetes client errored when getting the ReplicaSet", func(ctx SpecContext) {
852900
expectedErr := errors.New("there was an error getting the ReplicaSet")
853901
k8sClientReader.GetCalls(mergeGetCallsWithBase(

‎internal/mode/static/telemetry/data.avdl

+8-2
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,14 @@ attached at the Gateway level. */
102102
/** UpstreamSettingsPolicyCount is the number of UpstreamSettingsPolicies. */
103103
long? UpstreamSettingsPolicyCount = null;
104104

105-
/** NGFReplicaCount is the number of replicas of the NGF Pod. */
106-
long? NGFReplicaCount = null;
105+
/** GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway. */
106+
long? GatewayAttachedNpCount = null;
107+
108+
/** NginxPodCount is the total number of Nginx data plane Pods. */
109+
long? NginxPodCount = null;
110+
111+
/** ControlPlanePodCount is the total number of NGF control plane Pods. */
112+
long? ControlPlanePodCount = null;
107113

108114
}
109115
}

0 commit comments

Comments
 (0)