Skip to content

Commit 61ae05f

Browse files
Cyril TOVENAmarkmandel
Cyril TOVENA
authored and
markmandel
committed
Adds OpenCensus metrics integration.
- First set of metrics added using informers cache. - Added flag for switching off metrics via helm and env vars. - Added a documentation section about metrics and usage.
1 parent b492850 commit 61ae05f

317 files changed

Lines changed: 38581 additions & 2700 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Gopkg.lock‎

Lines changed: 60 additions & 139 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Gopkg.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,11 @@
8383
[[constraint]]
8484
branch = "master"
8585
name = "github.com/joonix/log"
86+
87+
[[constraint]]
88+
name = "go.opencensus.io"
89+
version = "0.18.0"
90+
91+
[[override]]
92+
name = "github.com/prometheus/client_golang"
93+
version = "0.9.2"

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ Documentation and usage guides on how to develop and host dedicated game servers
6161
- [Game Server Specification](./docs/gameserver_spec.md)
6262
- [Fleet Specification](./docs/fleet_spec.md)
6363
- [Fleet Autoscaler Specification](./docs/fleetautoscaler_spec.md)
64+
- [Metrics](./docs/metrics.md)
6465

6566
### Examples
6667
- [Full GameServer Configuration](./examples/gameserver.yaml)

‎build/Makefile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,7 @@ push-build-image:
348348

349349
# port forward the agones controller.
350350
# useful for pprof and stats viewing, etc
351-
controller-portforward: PORT ?= 6060
351+
controller-portforward: PORT ?= 8080
352352
controller-portforward:
353353
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) \
354354
-e "KUBECONFIG=/root/.kube/$(kubeconfig_file)" -p $(PORT):$(PORT) $(build_tag) \
@@ -395,6 +395,7 @@ do-release:
395395
git push -u upstream release-$(RELEASE_VERSION)
396396
@echo "Now go make the $(RELEASE_VERSION) release on Github!"
397397

398+
setup-test-cluster: DOCKER_RUN_ARGS+=--network=host
398399
setup-test-cluster: $(ensure-build-image)
399400
$(DOCKER_RUN) kubectl apply -f $(mount_path)/build/helm.yaml
400401
$(DOCKER_RUN) helm init --wait --service-account helm
@@ -570,7 +571,7 @@ kind-delete-cluster:
570571
kind delete cluster --name $(KIND_PROFILE)
571572

572573
# start an interactive shell with kubectl configured to target the kind cluster
573-
kind-shell: $(ensure-build-image)
574+
kind-shell: $(ensure-build-image)
574575
$(MAKE) shell KUBECONFIG="$(shell kind get kubeconfig-path --name="$(KIND_PROFILE)")" \
575576
DOCKER_RUN_ARGS="--network=host $(DOCKER_RUN_ARGS)"
576577

‎build/helm.yaml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,23 @@ roleRef:
3131
subjects:
3232
- kind: ServiceAccount
3333
name: helm
34-
namespace: kube-system
34+
namespace: kube-system
35+
---
36+
apiVersion: rbac.authorization.k8s.io/v1
37+
kind: ClusterRole
38+
metadata:
39+
creationTimestamp: null
40+
name: cluster-admin
41+
annotations:
42+
rbac.authorization.kubernetes.io/autoupdate: "true"
43+
rules:
44+
- apiGroups:
45+
- '*'
46+
resources:
47+
- '*'
48+
verbs:
49+
- '*'
50+
- nonResourceURLs:
51+
- '*'
52+
verbs:
53+
- '*'

‎cmd/controller/main.go‎

Lines changed: 49 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,21 +32,24 @@ import (
3232
"agones.dev/agones/pkg/fleets"
3333
"agones.dev/agones/pkg/gameservers"
3434
"agones.dev/agones/pkg/gameserversets"
35+
"agones.dev/agones/pkg/metrics"
3536
"agones.dev/agones/pkg/util/runtime"
3637
"agones.dev/agones/pkg/util/signals"
3738
"agones.dev/agones/pkg/util/webhooks"
3839
"github.com/heptiolabs/healthcheck"
3940
"github.com/pkg/errors"
41+
prom "github.com/prometheus/client_golang/prometheus"
4042
"github.com/spf13/pflag"
4143
"github.com/spf13/viper"
4244
extclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
4345
"k8s.io/apimachinery/pkg/api/resource"
4446
"k8s.io/client-go/informers"
4547
"k8s.io/client-go/kubernetes"
46-
"k8s.io/client-go/rest"
48+
"k8s.io/client-go/tools/clientcmd"
4749
)
4850

4951
const (
52+
enableMetricsFlag = "metrics"
5053
sidecarImageFlag = "sidecar-image"
5154
sidecarCPURequestFlag = "sidecar-cpu-request"
5255
sidecarCPULimitFlag = "sidecar-cpu-limit"
@@ -55,6 +58,7 @@ const (
5558
maxPortFlag = "max-port"
5659
certFileFlag = "cert-file"
5760
keyFileFlag = "key-file"
61+
kubeconfigFlag = "kubeconfig"
5862
workers = 2
5963
defaultResync = 30 * time.Second
6064
)
@@ -73,7 +77,8 @@ func main() {
7377
logger.WithError(err).Fatal("Could not create controller from environment or flags")
7478
}
7579

76-
clientConf, err := rest.InClusterConfig()
80+
// if the kubeconfig fails BuildConfigFromFlags will try in cluster config
81+
clientConf, err := clientcmd.BuildConfigFromFlags("", ctlConf.KubeConfig)
7782
if err != nil {
7883
logger.WithError(err).Fatal("Could not create in cluster config")
7984
}
@@ -93,11 +98,30 @@ func main() {
9398
logger.WithError(err).Fatal("Could not create the agones api clientset")
9499
}
95100

96-
health := healthcheck.NewHandler()
97101
wh := webhooks.NewWebHook(ctlConf.CertFile, ctlConf.KeyFile)
98102
agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync)
99103
kubeInformationFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync)
100104

105+
server := &httpServer{}
106+
var health healthcheck.Handler
107+
var metricsController *metrics.Controller
108+
109+
if ctlConf.Metrics {
110+
registry := prom.NewRegistry()
111+
metricHandler, err := metrics.RegisterPrometheusExporter(registry)
112+
if err != nil {
113+
logger.WithError(err).Fatal("Could not create register prometheus exporter")
114+
}
115+
server.Handle("/metrics", metricHandler)
116+
health = healthcheck.NewMetricsHandler(registry, "agones")
117+
metricsController = metrics.NewController(kubeClient, agonesClient, agonesInformerFactory)
118+
119+
} else {
120+
health = healthcheck.NewHandler()
121+
}
122+
123+
server.Handle("/", health)
124+
101125
allocationMutex := &sync.Mutex{}
102126

103127
gsController := gameservers.NewController(wh, health, allocationMutex,
@@ -112,15 +136,19 @@ func main() {
112136
fasController := fleetautoscalers.NewController(wh, health,
113137
kubeClient, extClient, agonesClient, agonesInformerFactory)
114138

139+
rs := []runner{
140+
wh, gsController, gsSetController, fleetController, faController, fasController, metricsController, server,
141+
}
142+
115143
stop := signals.NewStopChannel()
116144

117145
kubeInformationFactory.Start(stop)
118146
agonesInformerFactory.Start(stop)
119147

120-
rs := []runner{
121-
wh, gsController, gsSetController, fleetController, faController, fasController, healthServer{handler: health},
122-
}
123148
for _, r := range rs {
149+
if r == nil {
150+
continue
151+
}
124152
go func(rr runner) {
125153
if runErr := rr.Run(workers, stop); runErr != nil {
126154
logger.WithError(runErr).Fatalf("could not start runner: %s", reflect.TypeOf(rr))
@@ -145,6 +173,7 @@ func parseEnvFlags() config {
145173
viper.SetDefault(pullSidecarFlag, false)
146174
viper.SetDefault(certFileFlag, filepath.Join(base, "certs/server.crt"))
147175
viper.SetDefault(keyFileFlag, filepath.Join(base, "certs/server.key"))
176+
viper.SetDefault(enableMetricsFlag, true)
148177

149178
pflag.String(sidecarImageFlag, viper.GetString(sidecarImageFlag), "Flag to overwrite the GameServer sidecar image that is used. Can also use SIDECAR env variable")
150179
pflag.String(sidecarCPULimitFlag, viper.GetString(sidecarCPULimitFlag), "Flag to overwrite the GameServer sidecar container's cpu limit. Can also use SIDECAR_CPU_LIMIT env variable")
@@ -154,6 +183,8 @@ func parseEnvFlags() config {
154183
pflag.Int32(maxPortFlag, 0, "Required. The maximum port that that a GameServer can be allocated to. Can also use MAX_PORT env variable")
155184
pflag.String(keyFileFlag, viper.GetString(keyFileFlag), "Optional. Path to the key file")
156185
pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file")
186+
pflag.String(kubeconfigFlag, viper.GetString(kubeconfigFlag), "Optional. kubeconfig to run the controller out of the cluster. Only use it for debugging as webhook won't works.")
187+
pflag.Bool(enableMetricsFlag, viper.GetBool(enableMetricsFlag), "Flag to activate metrics of Agones. Can also use METRICS env variable.")
157188
pflag.Parse()
158189

159190
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -165,6 +196,8 @@ func parseEnvFlags() config {
165196
runtime.Must(viper.BindEnv(maxPortFlag))
166197
runtime.Must(viper.BindEnv(keyFileFlag))
167198
runtime.Must(viper.BindEnv(certFileFlag))
199+
runtime.Must(viper.BindEnv(kubeconfigFlag))
200+
runtime.Must(viper.BindEnv(enableMetricsFlag))
168201
runtime.Must(viper.BindPFlags(pflag.CommandLine))
169202

170203
request, err := resource.ParseQuantity(viper.GetString(sidecarCPURequestFlag))
@@ -186,6 +219,8 @@ func parseEnvFlags() config {
186219
AlwaysPullSidecar: viper.GetBool(pullSidecarFlag),
187220
KeyFile: viper.GetString(keyFileFlag),
188221
CertFile: viper.GetString(certFileFlag),
222+
KubeConfig: viper.GetString(kubeconfigFlag),
223+
Metrics: viper.GetBool(enableMetricsFlag),
189224
}
190225
}
191226

@@ -197,8 +232,10 @@ type config struct {
197232
SidecarCPURequest resource.Quantity
198233
SidecarCPULimit resource.Quantity
199234
AlwaysPullSidecar bool
235+
Metrics bool
200236
KeyFile string
201237
CertFile string
238+
KubeConfig string
202239
}
203240

204241
// validate ensures the ctlConfig data is valid.
@@ -216,21 +253,21 @@ type runner interface {
216253
Run(workers int, stop <-chan struct{}) error
217254
}
218255

219-
type healthServer struct {
220-
handler http.Handler
256+
type httpServer struct {
257+
http.ServeMux
221258
}
222259

223-
func (h healthServer) Run(workers int, stop <-chan struct{}) error {
224-
logger.Info("Starting health check...")
260+
func (h *httpServer) Run(workers int, stop <-chan struct{}) error {
261+
logger.Info("Starting http server...")
225262
srv := &http.Server{
226263
Addr: ":8080",
227-
Handler: h.handler,
264+
Handler: h,
228265
}
229266
defer srv.Close() // nolint: errcheck
230267

231268
if err := srv.ListenAndServe(); err != nil {
232269
if err == http.ErrServerClosed {
233-
logger.WithError(err).Info("health check: http server closed")
270+
logger.WithError(err).Info("http server closed")
234271
} else {
235272
wrappedErr := errors.Wrap(err, "Could not listen on :8080")
236273
runtime.HandleError(logger.WithError(wrappedErr), wrappedErr)

‎docs/metrics.md‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Metrics
2+
3+
Agones controller exposes metrics via [OpenCensus](https://opencensus.io/). OpenCensus is a single distribution of libraries that collect metrics and distributed traces from your services, we only use it for metrics but it will allow us to support multiple exporters in the future.
4+
5+
We choose to start with Prometheus as this is the most popular with Kubernetes but it is also compatible with Stackdriver.
6+
If you need another exporter, check the [list of supported](https://opencensus.io/exporters/supported-exporters/go/) exporters. It should be pretty straightforward to register a new one.(Github PR are more than welcomed)
7+
8+
We plan to support multiple exporters in the future via environement variables and helm flags.
9+
10+
## Backend integrations
11+
12+
### Prometheus
13+
14+
If you are running a [Prometheus](https://prometheus.io/) intance you just need to ensure that metrics and kubernetes service discovery are enabled. (helm chart values `agones.metrics.enabled` and `agones.metrics.prometheusServiceDiscovery`). This will automatically add annotations required by Prometheus to discover Agones metrics and start collecting them. (see [example](https://github.com/prometheus/prometheus/tree/master/documentation/examples/kubernetes-rabbitmq))
15+
16+
### Prometheus Operator
17+
18+
If you have [Prometheus operator](https://github.com/coreos/prometheus-operator) installed in your cluster, make sure to add a [`ServiceMonitor`](https://github.com/coreos/prometheus-operator/blob/v0.17.0/Documentation/api.md#servicemonitorspec) to discover Agones metrics as shown below:
19+
20+
```yaml
21+
apiVersion: monitoring.coreos.com/v1
22+
kind: ServiceMonitor
23+
metadata:
24+
name: agones
25+
labels:
26+
app: agones
27+
spec:
28+
selector:
29+
matchLabels:
30+
stable.agones.dev/role: controller
31+
endpoints:
32+
- port: web
33+
```
34+
35+
Finally include that `ServiceMonitor` in your [Prometheus instance CRD](https://github.com/coreos/prometheus-operator/blob/v0.17.0/Documentation/user-guides/getting-started.md#include-servicemonitors), this is usually done by adding a label to the `ServiceMonitor` above that is matched by the prometheus instance of your choice.
36+
37+
### Stackdriver
38+
39+
We don't yet support the [OpenCensus Stackdriver exporter](https://opencensus.io/exporters/supported-exporters/go/stackdriver/) but you can still use the Prometheus Stackdriver integration by following these [instructions](https://cloud.google.com/monitoring/kubernetes-engine/prometheus).
40+
Annotations required by this integration can be activated by setting the `agones.metrics.prometheusServiceDiscovery` to true (default) via the [helm chart value](../install/helm/README.md#configuration).
41+
42+
## Metrics available
43+
44+
| Name | Description | Type |
45+
|-------------------------------------------------|---------------------------------------------------------------------|---------|
46+
| agones_gameservers_count | The number of gameservers per fleet and status | gauge |
47+
| agones_fleet_allocations_count | The number of fleet allocations per fleet | gauge |
48+
| agones_gameservers_total | The total of gameservers per fleet and status | counter |
49+
| agones_fleet_allocations_total | The total of fleet allocations per fleet | counter |
50+
| agones_fleets_replicas_count | The number of replicas per fleet (total, desired, ready, allocated) | gauge |
51+
| agones_fleet_autoscalers_able_to_scale | The fleet autoscaler can access the fleet to scale | gauge |
52+
| agones_fleet_autoscalers_buffer_limits | he limits of buffer based fleet autoscalers (min, max) | gauge |
53+
| agones_fleet_autoscalers_buffer_size | The buffer size of fleet autoscalers (count or percentage) | gauge |
54+
| agones_fleet_autoscalers_current_replicas_count | The current replicas count as seen by autoscalers | gauge |
55+
| agones_fleet_autoscalers_desired_replicas_count | The desired replicas count as seen by autoscalers | gauge |
56+
| agones_fleet_autoscalers_limited | The fleet autoscaler is capped (1) | gauge |
57+
58+
## Dashboard
59+
60+
Grafana and Stackdriver - Coming Soon
61+
62+
## Adding more metrics
63+
64+
If you want to contribute and add more metrics we recommend to use shared informers (cache) as it is currently implemented in the [metrics controller](../pkg/metrics/controller.go). Using shared informers allows to keep metrics code in one place and doesn't overload the Kubernetes API.
65+
66+
However there is some cases where you will have to add code inside your ressource controller (eg. latency metrics), you should minize metrics code in your controller by adding specific functions in the metrics packages as shown below.
67+
68+
```golang
69+
package metrics
70+
71+
import "go.opencensus.io/stats"
72+
73+
...
74+
75+
func RecordSomeLatency(latency int64,ressourceName string) {
76+
stats.RecordWithTags(....)
77+
}
78+
```

‎install/helm/README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ The following tables lists the configurable parameters of the Agones chart and t
8686
| `agones.rbacEnabled` | Creates RBAC resources. Must be set for any cluster configured with RBAC | `true` |
8787
| `agones.crds.install` | Install the CRDs with this chart. Useful to disable if you want to subchart (since crd-install hook is broken), so you can copy the CRDs into your own chart. | `true` |
8888
| `agones.crds.cleanupOnDelete` | Run the pre-delete hook to delete all GameServers and their backing Pods when deleting the helm chart, so that all CRDs can be removed on chart deletion | `true` |
89+
| `agones.metrics.enabled` | Enables controller metrics on port `8080` and path `/metrics` | `true` |
90+
| `agones.metrics.prometheusServiceDiscovery` | Adds annotations for Prometheus ServiceDiscovery (and also Strackdriver) | `true` |
8991
| `agones.serviceaccount.controller` | Service account name for the controller | `agones-controller` |
9092
| `agones.serviceaccount.sdk` | Service account name for the sdk | `agones-sdk` |
9193
| `agones.image.registry` | Global image registry for all images | `gcr.io/agones-images` |
@@ -99,7 +101,7 @@ The following tables lists the configurable parameters of the Agones chart and t
99101
| `agones.image.sdk.alwaysPull` | Tells if the sdk image should always be pulled | `false` |
100102
| `agones.image.ping.name` | ( ⚠️ development feature ⚠️ ) Image name for the ping service | `agones-ping` |
101103
| `agones.image.ping.pullPolicy` | ( ⚠️ development feature ⚠️ ) Image pull policy for the ping service | `IfNotPresent` |
102-
| `agones.controller.healthCheck.http.port` | Port to use for liveness probe service | `8080` |
104+
| `agones.controller.http.port` | Port to use for liveness probe service and metrics | `8080` |
103105
| `agones.controller.healthCheck.initialDelaySeconds` | Initial delay before performing the first probe (in seconds) | `3` |
104106
| `agones.controller.healthCheck.periodSeconds` | Seconds between every liveness probe (in seconds) | `3` |
105107
| `agones.controller.healthCheck.failureThreshold` | Number of times before giving up (in seconds) | `3` |

‎install/helm/agones/templates/controller.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ spec:
3939
cluster-autoscaler.kubernetes.io/safe-to-evict: {{ .Values.agones.controller.safeToEvict | quote }}
4040
{{- if .Values.agones.controller.generateTLS }}
4141
revision/tls-cert: {{ .Release.Revision | quote }}
42+
{{- end }}
43+
{{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.enabled) }}
44+
prometheus.io/scrape: "true"
45+
prometheus.io/port: {{ .Values.agones.controller.http.port | quote }}
46+
prometheus.io/path: "/metrics"
4247
{{- end }}
4348
labels:
4449
stable.agones.dev/role: controller
@@ -64,12 +69,14 @@ spec:
6469
value: {{ .Values.agones.image.sdk.alwaysPull | quote }}
6570
- name: SIDECAR_CPU_REQUEST
6671
value: {{ .Values.agones.image.sdk.cpuRequest | quote }}
72+
- name: METRICS
73+
value: {{ .Values.agones.metrics.enabled | quote }}
6774
- name: SIDECAR_CPU_LIMIT
6875
value: {{ .Values.agones.image.sdk.cpuLimit | quote }}
6976
livenessProbe:
7077
httpGet:
7178
path: /live
72-
port: {{ .Values.agones.controller.healthCheck.http.port }}
79+
port: {{ .Values.agones.controller.http.port }}
7380
initialDelaySeconds: {{ .Values.agones.controller.healthCheck.initialDelaySeconds }}
7481
periodSeconds: {{ .Values.agones.controller.healthCheck.periodSeconds }}
7582
failureThreshold: {{ .Values.agones.controller.healthCheck.failureThreshold }}

‎install/helm/agones/templates/service.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ metadata:
1818
name: agones-controller-service
1919
namespace: {{ .Release.Namespace }}
2020
labels:
21-
component: controller
21+
stable.agones.dev/role: controller
2222
app: {{ template "agones.name" . }}
2323
chart: {{ template "agones.chart" . }}
2424
release: {{ .Release.Name }}
@@ -27,5 +27,8 @@ spec:
2727
selector:
2828
stable.agones.dev/role: controller
2929
ports:
30-
- port: 443
31-
targetPort: 8081
30+
- name: webhooks
31+
port: 443
32+
targetPort: 8081
33+
- name: web
34+
port: {{ .Values.agones.controller.http.port }}

0 commit comments

Comments
 (0)