Skip to content

Commit 80e202d

Browse files
zifterRobert Bailey
zifter
and
Robert Bailey
authored
Prometheus metrics: Use ServiceMonitor instead of deprecated annotation mechanism (agones-dev#2290)
* Prometheus metrics: Allow to use ServiceMonitor * 2262 fix review notes * 2262 rename service2 to serviceInternal * Fix review notes * Remove unnecessary header * Fix review notes Co-authored-by: Robert Bailey <robertbailey@google.com>
1 parent deb5764 commit 80e202d

13 files changed

Lines changed: 269 additions & 13 deletions

File tree

‎build/Makefile‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -319,10 +319,11 @@ install: CRD_CLEANUP := true
319319
install: LOG_LEVEL := "debug"
320320
install: EXTERNAL_IP ?= $(shell $(DOCKER_RUN) kubectl get services agones-allocator -n agones-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
321321
install: FEATURE_GATES ?= $(ALPHA_FEATURE_GATES)
322+
install: SERVICE_MONITOR := false
322323
install: HELM_ARGS ?=
323324
install: $(ensure-build-image) install-custom-pull-secret
324325
$(DOCKER_RUN) \
325-
helm upgrade --install --atomic --namespace=agones-system \
326+
helm upgrade --install --atomic --wait --namespace=agones-system \
326327
--create-namespace \
327328
--set agones.image.tag=$(VERSION),agones.image.registry=$(REGISTRY) \
328329
--set agones.image.controller.pullPolicy=$(IMAGE_PULL_POLICY),agones.image.sdk.alwaysPull=$(ALWAYS_PULL_SIDECAR) \
@@ -333,6 +334,7 @@ install: $(ensure-build-image) install-custom-pull-secret
333334
--set agones.crds.cleanupOnDelete=$(CRD_CLEANUP) \
334335
--set agones.featureGates=$(FEATURE_GATES) \
335336
--set agones.allocator.service.loadBalancerIP=$(EXTERNAL_IP) \
337+
--set agones.metrics.serviceMonitor.enabled=$(SERVICE_MONITOR) \
336338
$(HELM_ARGS) \
337339
agones $(mount_path)/install/helm/agones/
338340

@@ -553,6 +555,26 @@ uninstall-grafana: $(ensure-build-image)
553555
$(DOCKER_RUN) helm uninstall grafana --namespace=metrics
554556
$(DOCKER_RUN) kubectl delete -f $(mount_path)/build/grafana/
555557

558+
# setup prometheus-stack in the current cluster by default Persistent Volume Claims are requested.
559+
setup-prometheus-stack: PVC ?= true
560+
setup-prometheus-stack: PV_SIZE ?= 64Gi
561+
setup-prometheus-stack: SCRAPE_INTERVAL=30s
562+
setup-prometheus-stack: PASSWORD ?= admin
563+
setup-prometheus-stack:
564+
$(DOCKER_RUN) helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
565+
$(DOCKER_RUN) helm repo update
566+
$(DOCKER_RUN) helm upgrade prometheus-stack prometheus-community/kube-prometheus-stack --install --wait --version 19.0.2 \
567+
--namespace monitoring --create-namespace \
568+
--set prometheus.server.global.scrape_interval=$(SCRAPE_INTERVAL) \
569+
--set prometheus.server.persistentVolume.enabled=$(PVC) \
570+
--set prometheus.server.persistentVolume.size=$(PV_SIZE) \
571+
--set grafana.adminPassword=$(PASSWORD) \
572+
$(HELM_ARGS) -f $(mount_path)/build/prometheus-stack.yaml
573+
574+
uninstall-prometheus-stack: $(ensure-build-image)
575+
$(DOCKER_RUN) \
576+
helm uninstall prometheus-stack --namespace=monitoring
577+
556578
helm-repo-update:
557579
$(DOCKER_RUN) helm repo update
558580

‎build/README.md‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ Table of Contents
4646
* [make test-e2e-failure](#make-test-e2e-failure)
4747
* [make setup-prometheus](#make-setup-prometheus)
4848
* [make setup-grafana](#make-setup-grafana)
49+
* [make setup-prometheus-stack](#make-setup-prometheus-stack)
4950
* [make prometheus-portforward](#make-prometheus-portforward)
5051
* [make grafana-portforward](#make-grafana-portforward)
5152
* [make controller-portforward](#make-controller-portforward)
@@ -76,6 +77,7 @@ Table of Contents
7677
* [make minikube-install](#make-minikube-install)
7778
* [make minikube-setup-prometheus](#make-minikube-setup-prometheus)
7879
* [make minikube-setup-grafana](#make-minikube-setup-grafana)
80+
* [make minikube-setup-prometheus-stack](#make-minikube-setup-prometheus)
7981
* [make minikube-prometheus-portforward](#make-minikube-prometheus-portforward)
8082
* [make minikube-grafana-portforward](#make-minikube-grafana-portforward)
8183
* [make minikube-test-e2e](#make-minikube-test-e2e)
@@ -88,6 +90,7 @@ Table of Contents
8890
* [make kind-install](#make-kind-install)
8991
* [make kind-setup-prometheus](#make-kind-setup-prometheus)
9092
* [make kind-setup-grafana](#make-kind-setup-grafana)
93+
* [make kind-setup-prometheus-stack](#make-kind-setup-prometheus-stack)
9194
* [make kind-prometheus-portforward](#make-kind-prometheus-portforward)
9295
* [make kind-grafana-portforward](#make-kind-grafana-portforward)
9396
* [make kind-test-e2e](#make-kind-test-e2e)
@@ -485,7 +488,7 @@ Run controller failure portion of the end-to-end tests.
485488

486489
#### `make setup-prometheus`
487490

488-
Install Prometheus server using [Prometheus Community](https://prometheus-community.github.io/helm-charts)
491+
Install Prometheus server using [Prometheus Community](https://prometheus-community.github.io/helm-charts)
489492
chart into the current cluster.
490493

491494
By default all exporters and alertmanager is disabled.
@@ -500,13 +503,23 @@ Run helm repo update to get the mose recent charts.
500503

501504
#### `make setup-grafana`
502505

503-
Install Grafana server using [grafana community](https://grafana.github.io/helm-charts) chart into
506+
Install Grafana server using [grafana community](https://grafana.github.io/helm-charts) chart into
504507
the current cluster and setup [Agones dashboards with Prometheus datasource](./grafana/).
505508

506509
You can set your own password using the `PASSWORD` environment variable.
507510

508511
See [`make minikube-setup-grafana`](#make-minikube-setup-grafana) and [`make kind-setup-grafana`](#make-kind-setup-grafana) to run the installation on Minikube or Kind.
509512

513+
#### `make setup-prometheus-stack`
514+
515+
Install Prometheus-stack using [Prometheus Community](https://prometheus-community.github.io/helm-charts) chart into the current cluster.
516+
517+
By default only prometheus and grafana will installed, all exporters and alertmanager is disabled.
518+
519+
You can use this to collect Agones [Metrics](../site/content/en/docs/Guides/metrics.md) using ServiceMonitor.
520+
521+
See [`make minikube-setup-prometheus-stack`](#make-minikube-setup-prometheus-stack) and [`make kind-setup-prometheus-stack`](#make-kind-setup-prometheus-stack) to run the installation on Minikube or Kind.
522+
510523
#### `make prometheus-portforward`
511524

512525
Sets up port forwarding to the
@@ -671,6 +684,11 @@ Use this instead of `make setup-prometheus`, as it disables Persistent Volume Cl
671684
Installs grafana into the Kubernetes cluster.
672685
Use this instead of `make setup-grafana`, as it disables Persistent Volume Claim.
673686

687+
#### `make minikube-setup-prometheus-stack`
688+
689+
Installs prometheus-stack into the Kubernetes cluster.
690+
Use this instead of `make setup-prometheus-stack`, as it disables Persistent Volume Claim.
691+
674692
#### `make minikube-prometheus-portforward`
675693

676694
The minikube version of [`make prometheus-portforward`](#make-prometheus-portforward) to setup
@@ -722,14 +740,19 @@ Use this instead of `make setup-prometheus`, as it disables Persistent Volume Cl
722740
Installs grafana into the Kubernetes cluster.
723741
Use this instead of `make setup-grafana`, as it disables Persistent Volume Claim.
724742

743+
#### `make kind-setup-prometheus-stack`
744+
745+
Installs prometheus-stack into the Kubernetes cluster.
746+
Use this instead of `make setup-prometheus-stack`, as it disables Persistent Volume Claim.
747+
725748
#### `make kind-prometheus-portforward`
726749

727-
The minikube version of [`make prometheus-portforward`](#make-prometheus-portforward) to setup
750+
The kind version of [`make prometheus-portforward`](#make-prometheus-portforward) to setup
728751
port forwarding to the prometheus deployment.
729752

730753
#### `make kind-grafana-portforward`
731754

732-
The minikube version of [`make grafana-portforward`](#make-grafana-portforward) to setup
755+
The kind version of [`make grafana-portforward`](#make-grafana-portforward) to setup
733756
port forwarding to the grafana deployment.
734757

735758

‎build/includes/kind.mk‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ kind-setup-prometheus:
6666
kind-setup-grafana:
6767
$(MAKE) setup-grafana DOCKER_RUN_ARGS="--network=host" PVC=false
6868

69+
kind-setup-prometheus-stack:
70+
$(MAKE) setup-prometheus-stack DOCKER_RUN_ARGS="--network=host" PVC=false \
71+
HELM_ARGS="--set prometheus.server.resources.requests.cpu=0,prometheus.server.resources.requests.memory=0"
72+
6973
# kind port forwarding controller web
7074
kind-controller-portforward:
7175
$(MAKE) controller-portforward DOCKER_RUN_ARGS="--network=host"

‎build/includes/minikube.mk‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,13 +67,19 @@ minikube-setup-prometheus:
6767
DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)" \
6868
PVC=false HELM_ARGS="--set server.resources.requests.cpu=0,server.resources.requests.memory=0"
6969

70-
7170
# grafana on minkube with dashboards and prometheus datasource installed.
7271
# we have to disable PVC as it's not supported on minkube.
7372
minikube-setup-grafana:
7473
$(MAKE) setup-grafana \
7574
DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)"
7675

76+
# prometheus-stack on minkube
77+
# we have to disable PVC as it's not supported on minkube.
78+
minikube-setup-prometheus-stack:
79+
$(MAKE) setup-prometheus-stack \
80+
DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)" \
81+
PVC=false HELM_ARGS="--set prometheus.server.resources.requests.cpu=0,prometheus.server.resources.requests.memory=0"
82+
7783
# minikube port forwarding
7884
minikube-controller-portforward:
7985
$(MAKE) controller-portforward DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)"

‎build/prometheus-stack.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# disabled all not necessary chart
2+
defaultRules:
3+
enabled: false
4+
alertmanager:
5+
enabled: false
6+
kubeApiServer:
7+
enabled: false
8+
kubelet:
9+
enabled: false
10+
kubeControllerManager:
11+
enabled: false
12+
coreDns:
13+
enabled: false
14+
kubeDns:
15+
enabled: false
16+
kubeEtcd:
17+
enabled: false
18+
kubeScheduler:
19+
enabled: false
20+
kubeProxy:
21+
enabled: false
22+
kubeStateMetrics:
23+
enabled: false
24+
nodeExporter:
25+
enabled: false
26+
27+
prometheusOperator:
28+
enabled: true
29+
prometheus:
30+
enabled: true
31+
prometheusSpec:
32+
# allow discovery of third party service monitor
33+
serviceMonitorSelectorNilUsesHelmValues: false
34+
grafana:
35+
enabled: true
36+
adminUser: admin
37+
adminPassword: admin

‎install/helm/agones/templates/controller.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ spec:
4242
{{- end }}
4343
{{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.prometheusEnabled) }}
4444
prometheus.io/scrape: "true"
45-
prometheus.io/port: {{ .Values.agones.controller.http.port | quote }}
45+
prometheus.io/port: "8080"
4646
prometheus.io/path: "/metrics"
4747
{{- end }}
4848
{{- if .Values.agones.controller.annotations }}
@@ -129,10 +129,15 @@ spec:
129129
fieldPath: metadata.namespace
130130
- name: CONTAINER_NAME
131131
value: "agones-controller"
132+
ports:
133+
- name: webhooks
134+
containerPort: 8081
135+
- name: http
136+
containerPort: 8080
132137
livenessProbe:
133138
httpGet:
134139
path: /live
135-
port: {{ .Values.agones.controller.http.port }}
140+
port: http
136141
initialDelaySeconds: {{ .Values.agones.controller.healthCheck.initialDelaySeconds }}
137142
periodSeconds: {{ .Values.agones.controller.healthCheck.periodSeconds }}
138143
failureThreshold: {{ .Values.agones.controller.healthCheck.failureThreshold }}

‎install/helm/agones/templates/service-monitor.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
{{- if and (.Values.agones.metrics.prometheusEnabled) (.Values.agones.metrics.serviceMonitor.enabled) }}
2+
# Copyright 2018 Google LLC All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
apiVersion: monitoring.coreos.com/v1
17+
kind: ServiceMonitor
18+
metadata:
19+
name: agones-controller-monitor
20+
namespace: {{ .Release.Namespace }}
21+
labels:
22+
agones.dev/role: controller
23+
app: {{ template "agones.name" . }}
24+
chart: {{ template "agones.chart" . }}
25+
release: {{ .Release.Name }}
26+
heritage: {{ .Release.Service }}
27+
spec:
28+
selector:
29+
matchLabels:
30+
agones.dev/role: controller
31+
endpoints:
32+
- port: web
33+
path: /metrics
34+
interval: {{ .Values.agones.metrics.serviceMonitor.interval }}
35+
relabelings:
36+
- sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
37+
action: keep
38+
regex: "true"
39+
- sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
40+
action: replace
41+
targetLabel: __metrics_path__
42+
regex: (.+)
43+
- sourceLabels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
44+
action: replace
45+
regex: ([^:]+)(?::\d+)?;(\d+)
46+
replacement: $1:$2
47+
targetLabel: __address__
48+
- action: labelmap
49+
regex: __meta_kubernetes_pod_label_(.+)
50+
- sourceLabels: [__meta_kubernetes_namespace]
51+
action: replace
52+
targetLabel: kubernetes_namespace
53+
- sourceLabels: [__meta_kubernetes_pod_name]
54+
action: replace
55+
targetLabel: kubernetes_pod_name
56+
{{- end }}

‎install/helm/agones/templates/service.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ spec:
2929
ports:
3030
- name: webhooks
3131
port: 443
32-
targetPort: 8081
32+
targetPort: webhooks
3333
- name: web
34-
port: {{ .Values.agones.controller.http.port }}
34+
port: {{ .Values.agones.controller.http.port }}
35+
targetPort: http

‎install/helm/agones/templates/service/allocation.yaml‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,52 @@ spec:
6363
{{ toYaml .Values.agones.allocator.service.loadBalancerSourceRanges | indent 4 }}
6464
{{- end }}
6565
{{- end }}
66-
66+
---
67+
apiVersion: v1
68+
kind: Service
69+
metadata:
70+
name: {{ .Values.agones.allocator.serviceMetrics.name }}
71+
namespace: {{ .Release.Namespace }}
72+
labels:
73+
multicluster.agones.dev/role: allocator
74+
app: {{ template "agones.name" . }}
75+
chart: {{ template "agones.chart" . }}
76+
release: {{ .Release.Name }}
77+
heritage: {{ .Release.Service }}
78+
{{- if .Values.agones.allocator.serviceMetrics.annotations }}
79+
annotations:
80+
{{ toYaml .Values.agones.allocator.serviceMetrics.annotations | indent 4 }}
81+
{{- end }}
82+
spec:
83+
selector:
84+
multicluster.agones.dev/role: allocator
85+
ports:
86+
- port: {{ .Values.agones.allocator.serviceMetrics.http.port }}
87+
name: {{ .Values.agones.allocator.serviceMetrics.http.portName }}
88+
targetPort: 8080
89+
protocol: TCP
90+
{{- if and (.Values.agones.metrics.prometheusEnabled) (.Values.agones.metrics.serviceMonitor.enabled) }}
91+
---
92+
apiVersion: monitoring.coreos.com/v1
93+
kind: ServiceMonitor
94+
metadata:
95+
name: agones-allocator-monitor
96+
namespace: {{ .Release.Namespace }}
97+
labels:
98+
multicluster.agones.dev/role: allocator
99+
app: {{ template "agones.name" . }}
100+
chart: {{ template "agones.chart" . }}
101+
release: {{ .Release.Name }}
102+
heritage: {{ .Release.Service }}
103+
spec:
104+
selector:
105+
matchLabels:
106+
multicluster.agones.dev/role: allocator
107+
endpoints:
108+
- port: {{ .Values.agones.allocator.serviceMetrics.http.portName }}
109+
path: /metrics
110+
interval: {{ .Values.agones.metrics.serviceMonitor.interval }}
111+
{{- end }}
67112
---
68113
# Deploy pods to run the agones-allocator code
69114
apiVersion: apps/v1

‎install/helm/agones/values.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ agones:
2222
stackdriverEnabled: false
2323
stackdriverProjectID: ""
2424
stackdriverLabels: ""
25+
serviceMonitor:
26+
enabled: false
27+
interval: 30s
2528
rbacEnabled: true
2629
registerServiceAccounts: true
2730
registerWebhooks: true
@@ -164,6 +167,13 @@ agones:
164167
port: 443
165168
portName: grpc
166169
targetPort: 8443
170+
serviceMetrics:
171+
name: agones-allocator-metrics-service
172+
annotations: {}
173+
http:
174+
enabled: true
175+
port: 8080
176+
portName: http
167177
disableSecretCreation: false
168178
generateTLS: true
169179
generateClientTLS: true

0 commit comments

Comments
 (0)