Skip to content

Commit 992b1bb

Browse files
jkowalskimarkmandel
jkowalski
authored and
markmandel
committed
Emit stress test metrics in Fortio format.
1 parent 5f6d61a commit 992b1bb

77 files changed

Lines changed: 12657 additions & 104 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Gopkg.lock‎

Lines changed: 27 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Gopkg.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,7 @@
9999
[[constraint]]
100100
name = "contrib.go.opencensus.io/exporter/stackdriver"
101101
version = "v0.8.0"
102+
103+
[[constraint]]
104+
name = "fortio.org/fortio"
105+
version = "1.3.1"

‎build/Makefile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ endif
138138
GO_TEST=$(DOCKER_RUN) go test $(RACE_DETECTOR_ARGS)
139139
GO_E2E_TEST_ARGS=--kubeconfig /root/.kube/$(kubeconfig_file)
140140

141+
PERF_OUTPUT_DIR=$(mount_path)/build/.perf
142+
141143
go_build_base_path=$(mount_path)
142144

143145
ifdef LOCAL_GO
@@ -148,6 +150,7 @@ ifdef LOCAL_GO
148150
GO_BUILD_DARWIN_AMD64=GOOS=darwin GOARCH=amd64 go build
149151
GO_TEST=go test -v $(RACE_DETECTOR_ARGS)
150152
GO_E2E_TEST_ARGS=
153+
PERF_OUTPUT_DIR=$(build_path)/.perf
151154
go_build_base_path=$(agones_path)
152155
endif
153156

@@ -229,7 +232,8 @@ stress-test-e2e: $(ensure-build-image)
229232
-run '.*StressTest.*' \
230233
--gameserver-image=$(GS_TEST_IMAGE) \
231234
--pullsecret=$(IMAGE_PULL_SECRET) \
232-
--stress $(STRESS_TEST_LEVEL)
235+
--stress $(STRESS_TEST_LEVEL) \
236+
--perf-output $(PERF_OUTPUT_DIR)
233237

234238
# Run test on install yaml - make sure there is no change
235239
# mostly this is for CI

‎test/e2e/fleet_test.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -670,8 +670,11 @@ func TestScaleUpAndDownInParallelStressTest(t *testing.T) {
670670

671671
var fleets []*v1alpha1.Fleet
672672

673-
var scaleUpResults e2e.PerfResults
674-
var scaleDownResults e2e.PerfResults
673+
scaleUpStats := framework.NewStatsCollector(fmt.Sprintf("fleet_%v_scale_up", fleetSize))
674+
scaleDownStats := framework.NewStatsCollector(fmt.Sprintf("fleet_%v_scale_down", fleetSize))
675+
676+
defer scaleUpStats.Report()
677+
defer scaleDownStats.Report()
675678

676679
for fleetNumber := 0; fleetNumber < fleetCount; fleetNumber++ {
677680
flt := defaultFleet()
@@ -713,22 +716,19 @@ func TestScaleUpAndDownInParallelStressTest(t *testing.T) {
713716
}()
714717

715718
if fleetNumber%2 == 0 {
716-
scaleDownResults.AddSample(scaleAndWait(t, flt, 0))
719+
scaleDownStats.ReportDuration(scaleAndWait(t, flt, 0), nil)
717720
}
718721
for i := 0; i < repeatCount; i++ {
719722
if time.Now().After(deadline) {
720723
break
721724
}
722-
scaleUpResults.AddSample(scaleAndWait(t, flt, fleetSize))
723-
scaleDownResults.AddSample(scaleAndWait(t, flt, 0))
725+
scaleUpStats.ReportDuration(scaleAndWait(t, flt, fleetSize), nil)
726+
scaleDownStats.ReportDuration(scaleAndWait(t, flt, 0), nil)
724727
}
725728
}(fleetNumber, flt)
726729
}
727730

728731
wg.Wait()
729-
730-
scaleUpResults.Report(fmt.Sprintf("scale up 0 to %v with %v fleets", fleetSize, fleetCount))
731-
scaleDownResults.Report(fmt.Sprintf("scale down %v to 0 with %v fleets", fleetSize, fleetCount))
732732
}
733733

734734
func scaleAndWait(t *testing.T, flt *v1alpha1.Fleet, fleetSize int32) time.Duration {

‎test/e2e/framework/framework.go‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,11 @@ type Framework struct {
4949
GameServerImage string
5050
PullSecret string
5151
StressTestLevel int
52+
PerfOutputDir string
5253
}
5354

5455
// New setups a testing framework using a kubeconfig path and the game server image to use for testing.
55-
func New(kubeconfig, gsimage string, pullSecret string, stressTestLevel int) (*Framework, error) {
56+
func New(kubeconfig string) (*Framework, error) {
5657
config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
5758
if err != nil {
5859
return nil, errors.Wrap(err, "build config from flags failed")
@@ -69,11 +70,8 @@ func New(kubeconfig, gsimage string, pullSecret string, stressTestLevel int) (*F
6970
}
7071

7172
return &Framework{
72-
KubeClient: kubeClient,
73-
AgonesClient: agonesClient,
74-
GameServerImage: gsimage,
75-
PullSecret: pullSecret,
76-
StressTestLevel: stressTestLevel,
73+
KubeClient: kubeClient,
74+
AgonesClient: agonesClient,
7775
}, nil
7876
}
7977

@@ -201,6 +199,15 @@ func (f *Framework) WaitForFleetGameServersCondition(flt *v1alpha1.Fleet, cond f
201199
})
202200
}
203201

202+
// NewStatsCollector returns new instance of statistics collector,
203+
// which can be used to emit performance statistics for load tests and stress tests.
204+
func (f *Framework) NewStatsCollector(name string) *StatsCollector {
205+
if f.StressTestLevel > 0 {
206+
name = fmt.Sprintf("stress_%v_%v", f.StressTestLevel, name)
207+
}
208+
return &StatsCollector{name: name, outputDir: f.PerfOutputDir}
209+
}
210+
204211
// CleanUp Delete all Agones resources in a given namespace.
205212
func (f *Framework) CleanUp(ns string) error {
206213
logrus.Info("Cleaning up now.")

‎test/e2e/framework/perf.go‎

Lines changed: 78 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,35 @@
11
package framework
22

33
import (
4-
"sort"
4+
"encoding/json"
5+
"net/http"
6+
"os"
7+
"path/filepath"
58
"sync"
69
"time"
710

11+
"fortio.org/fortio/fhttp"
12+
"fortio.org/fortio/stats"
813
"github.com/sirupsen/logrus"
14+
15+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
916
)
1017

11-
// PerfResults aggregates performance test results.
12-
// The AddSample() method is safe for concurrent use by multiple goroutines.
13-
type PerfResults struct {
14-
mu sync.Mutex
15-
samples []time.Duration
18+
// StatsCollector collects latency and throughput counters.
19+
// The ReportDuration() method is safe for concurrent use by multiple goroutines.
20+
type StatsCollector struct {
21+
name string
22+
outputDir string
1623

24+
mu sync.Mutex
25+
samples []time.Duration
26+
statusCounts map[int]int64
1727
firstSampleTime time.Time
1828
lastSampleTime time.Time
1929
}
2030

21-
// AddSample adds a single time measurement.
22-
func (p *PerfResults) AddSample(d time.Duration) {
31+
// ReportDuration adds a single time measurement.
32+
func (p *StatsCollector) ReportDuration(d time.Duration, err error) {
2333
p.mu.Lock()
2434
defer p.mu.Unlock()
2535

@@ -29,37 +39,77 @@ func (p *PerfResults) AddSample(d time.Duration) {
2939
}
3040
p.lastSampleTime = n
3141
p.samples = append(p.samples, d)
42+
if p.statusCounts == nil {
43+
p.statusCounts = map[int]int64{}
44+
}
45+
p.statusCounts[errToHTTPStatusCode(err)]++
46+
}
47+
48+
func errToHTTPStatusCode(err error) int {
49+
// crude translation from 'err' to HTTP status code.
50+
switch {
51+
case err == nil:
52+
return http.StatusOK
53+
case k8serrors.IsNotFound(err):
54+
return http.StatusNotFound
55+
case k8serrors.IsConflict(err):
56+
return http.StatusConflict
57+
case k8serrors.IsUnauthorized(err):
58+
return http.StatusUnauthorized
59+
case k8serrors.IsServiceUnavailable(err):
60+
return http.StatusServiceUnavailable
61+
default:
62+
return http.StatusInternalServerError
63+
}
3264
}
3365

3466
// Report outputs performance report to log.
35-
func (p *PerfResults) Report(name string) {
67+
func (p *StatsCollector) Report() {
3668
if len(p.samples) == 0 {
3769
return
3870
}
3971

40-
sort.Slice(p.samples, func(i, j int) bool {
41-
return p.samples[i] < p.samples[j]
42-
})
43-
44-
var sum time.Duration
72+
h := stats.NewHistogram(0, 1)
4573
for _, s := range p.samples {
46-
sum += s
74+
h.Record(s.Seconds())
4775
}
4876

49-
avg := time.Duration(int64(sum) / int64(len(p.samples)))
77+
var rr fhttp.HTTPRunnerResults
78+
rr.RunType = "HTTP"
79+
rr.Labels = "Agones " + p.name
80+
rr.StartTime = time.Now()
81+
rr.ActualDuration = p.lastSampleTime.Sub(p.firstSampleTime)
82+
rr.DurationHistogram = h.Export()
83+
rr.DurationHistogram.CalcPercentiles([]float64{50, 90, 95, 99, 99.9})
84+
rr.RetCodes = map[int]int64{}
85+
rr.ActualQPS = float64(len(p.samples)) / rr.ActualDuration.Seconds()
86+
5087
logrus.
51-
WithField("avg", avg).
52-
WithField("count", len(p.samples)).
53-
WithField("min", p.samples[0].Seconds()).
54-
WithField("max", p.samples[len(p.samples)-1].Seconds()).
55-
WithField("p50", p.samples[len(p.samples)*500/1001].Seconds()).
56-
WithField("p90", p.samples[len(p.samples)*900/1001].Seconds()).
57-
WithField("p95", p.samples[len(p.samples)*950/1001].Seconds()).
58-
WithField("p99", p.samples[len(p.samples)*990/1001].Seconds()).
59-
WithField("p999", p.samples[len(p.samples)*999/1001].Seconds()).
88+
WithField("avg", rr.DurationHistogram.Avg).
89+
WithField("count", rr.DurationHistogram.Count).
90+
WithField("min", rr.DurationHistogram.Min).
91+
WithField("max", rr.DurationHistogram.Max).
92+
WithField("p50", rr.DurationHistogram.CalcPercentile(50)).
93+
WithField("p90", rr.DurationHistogram.CalcPercentile(90)).
94+
WithField("p95", rr.DurationHistogram.CalcPercentile(95)).
95+
WithField("p99", rr.DurationHistogram.CalcPercentile(99)).
96+
WithField("p999", rr.DurationHistogram.CalcPercentile(99.9)).
6097
WithField("duration", p.lastSampleTime.Sub(p.firstSampleTime).Seconds()).
61-
Info(name)
98+
Info(p.name)
99+
100+
if p.outputDir != "" {
101+
os.MkdirAll(p.outputDir, 0755) //nolint:errcheck
62102

63-
// TODO - use something like Fortio ("fortio.org/fortio/stats") to
64-
// generate histogram for long-term storage and analysis.
103+
fname := filepath.Join(p.outputDir, p.name+"_"+rr.StartTime.UTC().Format("2006-01-02_1504")+".json")
104+
f, err := os.Create(fname)
105+
if err != nil {
106+
logrus.WithError(err).Error("unable to create performance log")
107+
return
108+
}
109+
defer f.Close() //nolint:errcheck
110+
111+
e := json.NewEncoder(f)
112+
e.SetIndent("", " ")
113+
e.Encode(rr) //nolint:errcheck
114+
}
65115
}

‎test/e2e/main_test.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ func TestMain(m *testing.M) {
3939
pullSecret := flag.String("pullsecret", "",
4040
"optional secret to be used for pulling the gameserver and/or Agones SDK sidecar images")
4141
stressTestLevel := flag.Int("stress", 0, "enable stress test at given level 0-100")
42+
perfOutputDir := flag.String("perf-output", "", "write performance statistics to the specified directrory")
4243

4344
flag.Parse()
4445

@@ -53,11 +54,16 @@ func TestMain(m *testing.M) {
5354
exitCode int
5455
)
5556

56-
if framework, err = e2eframework.New(*kubeconfig, *gsimage, *pullSecret, *stressTestLevel); err != nil {
57+
if framework, err = e2eframework.New(*kubeconfig); err != nil {
5758
log.Printf("failed to setup framework: %v\n", err)
5859
os.Exit(1)
5960
}
6061

62+
framework.GameServerImage = *gsimage
63+
framework.PullSecret = *pullSecret
64+
framework.StressTestLevel = *stressTestLevel
65+
framework.PerfOutputDir = *perfOutputDir
66+
6167
// run cleanup before tests, to ensure no resources from previous runs exist.
6268
err = framework.CleanUp(defaultNs)
6369
if err != nil {

‎vendor/fortio.org/fortio/.gitignore‎

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/fortio.org/fortio/CONTRIBUTING.md‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)