Skip to content

Commit 296fbb7

Browse files
markmandelmarkmandel
authored
Hope to reduce e2e flakiness (agones-dev#2348)
A few things added to this PR to help combat/diagnose the e2e test flakines: * Enhanced logging. Await for Fleet condition functions now log which test they are part of, and do dumps of gameserver state and events if a Fleet does not match the criteria. * New logging methods, such as the ability to dump all an object's events. * Retry on UDP message. Didn't seem to need this before (or only very rarely), but apparently do now. 🤞🏻 this makes a difference!
1 parent fc6e787 commit 296fbb7

6 files changed

Lines changed: 133 additions & 95 deletions

File tree

‎test/e2e/controller/crash_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020
"time"
2121

2222
agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
23-
"github.com/sirupsen/logrus"
23+
e2eframework "agones.dev/agones/test/e2e/framework"
2424
"github.com/stretchr/testify/assert"
2525
corev1 "k8s.io/api/core/v1"
2626
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -29,11 +29,11 @@ import (
2929
)
3030

3131
func TestGameServerUnhealthyAfterDeletingPodWhileControllerDown(t *testing.T) {
32-
logger := logrus.WithField("test", t.Name())
32+
logger := e2eframework.TestLogger(t)
3333
gs := framework.DefaultGameServer(defaultNs)
3434
ctx := context.Background()
3535

36-
readyGs, err := framework.CreateGameServerAndWaitUntilReady(defaultNs, gs)
36+
readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, defaultNs, gs)
3737
if err != nil {
3838
t.Fatalf("Could not get a GameServer ready: %v", err)
3939
}
@@ -52,7 +52,7 @@ func TestGameServerUnhealthyAfterDeletingPodWhileControllerDown(t *testing.T) {
5252
err = podClient.Delete(ctx, pod.ObjectMeta.Name, metav1.DeleteOptions{})
5353
assert.NoError(t, err)
5454

55-
_, err = framework.WaitForGameServerState(readyGs, agonesv1.GameServerStateUnhealthy, 3*time.Minute)
55+
_, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateUnhealthy, 3*time.Minute)
5656
assert.NoError(t, err)
5757
logger.Info("waiting for Agones controller to come back to running")
5858
assert.NoError(t, waitForAgonesControllerRunning(ctx))

‎test/e2e/fleet_test.go‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ func TestFleetScaleUpEditAndScaleDown(t *testing.T) {
160160
framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(targetScale))
161161
gsa := framework.CreateAndApplyAllocation(t, flt)
162162

163-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
163+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
164164
return fleet.Status.AllocatedReplicas == 1
165165
})
166166

@@ -210,7 +210,7 @@ func TestFleetScaleUpEditAndScaleDown(t *testing.T) {
210210

211211
framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(1))
212212

213-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
213+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
214214
return fleet.Status.AllocatedReplicas == 0
215215
})
216216
})
@@ -334,7 +334,7 @@ func TestFleetRollingUpdate(t *testing.T) {
334334

335335
framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(1))
336336

337-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
337+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
338338
return fleet.Status.AllocatedReplicas == 0
339339
})
340340
})
@@ -432,7 +432,7 @@ func TestScaleFleetUpAndDownWithGameServerAllocation(t *testing.T) {
432432
gsa, err = framework.AgonesClient.AllocationV1().GameServerAllocations(framework.Namespace).Create(ctx, gsa, metav1.CreateOptions{})
433433
assert.Nil(t, err)
434434
assert.Equal(t, allocationv1.GameServerAllocationAllocated, gsa.Status.State)
435-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
435+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
436436
return fleet.Status.AllocatedReplicas == 1
437437
})
438438

@@ -452,7 +452,7 @@ func TestScaleFleetUpAndDownWithGameServerAllocation(t *testing.T) {
452452
assert.Nil(t, err)
453453
framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(1))
454454

455-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
455+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
456456
return fleet.Status.AllocatedReplicas == 0
457457
})
458458
})
@@ -553,7 +553,7 @@ func TestUpdateGameServerConfigurationInFleet(t *testing.T) {
553553
gsa, err = framework.AgonesClient.AllocationV1().GameServerAllocations(framework.Namespace).Create(ctx, gsa, metav1.CreateOptions{})
554554
assert.Nil(t, err, "cloud not create gameserver allocation")
555555
assert.Equal(t, allocationv1.GameServerAllocationAllocated, gsa.Status.State)
556-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
556+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
557557
return fleet.Status.AllocatedReplicas == 1
558558
})
559559

@@ -604,7 +604,7 @@ func TestReservedGameServerInFleet(t *testing.T) {
604604
assert.NoError(t, err)
605605

606606
// make sure counts are correct
607-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
607+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
608608
return fleet.Status.ReadyReplicas == 2 && fleet.Status.ReservedReplicas == 1
609609
})
610610

@@ -613,9 +613,9 @@ func TestReservedGameServerInFleet(t *testing.T) {
613613
framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(0))
614614

615615
// one should be left behind
616-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
616+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
617617
result := fleet.Status.ReservedReplicas == 1
618-
logrus.WithField("reserved", fleet.Status.ReservedReplicas).WithField("result", result).Info("waiting for 1 reserved replica")
618+
log.WithField("reserved", fleet.Status.ReservedReplicas).WithField("result", result).Info("waiting for 1 reserved replica")
619619
return result
620620
})
621621

@@ -1114,7 +1114,7 @@ func TestFleetWithLongLabelsAnnotations(t *testing.T) {
11141114
assert.Error(t, err)
11151115
statusErr, ok = err.(*k8serrors.StatusError)
11161116
assert.True(t, ok)
1117-
assert.Len(t, statusErr.Status().Details.Causes, 1)
1117+
require.Len(t, statusErr.Status().Details.Causes, 1)
11181118
assert.Equal(t, "annotations", statusErr.Status().Details.Causes[0].Field)
11191119
assert.Equal(t, metav1.CauseTypeFieldValueInvalid, statusErr.Status().Details.Causes[0].Type)
11201120

@@ -1309,13 +1309,13 @@ func TestFleetAggregatedPlayerStatus(t *testing.T) {
13091309
flt, err := client.Fleets(framework.Namespace).Create(ctx, flt.DeepCopy(), metav1.CreateOptions{})
13101310
assert.NoError(t, err)
13111311

1312-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
1312+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
13131313
if fleet.Status.Players == nil {
1314-
logrus.WithField("status", fleet.Status).Info("No Players")
1314+
log.WithField("status", fleet.Status).Info("No Players")
13151315
return false
13161316
}
13171317

1318-
logrus.WithField("status", fleet.Status).Info("Checking Capacity")
1318+
log.WithField("status", fleet.Status).Info("Checking Capacity")
13191319
return fleet.Status.Players.Capacity == 30
13201320
})
13211321

@@ -1356,8 +1356,8 @@ func TestFleetAggregatedPlayerStatus(t *testing.T) {
13561356
}
13571357
}
13581358

1359-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
1360-
logrus.WithField("players", fleet.Status.Players).WithField("totalCapacity", totalCapacity).
1359+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
1360+
log.WithField("players", fleet.Status.Players).WithField("totalCapacity", totalCapacity).
13611361
WithField("totalPlayers", totalPlayers).Info("Checking Capacity")
13621362
// since UDP packets might fail, we might get an extra player, so we'll check for that.
13631363
return (fleet.Status.Players.Capacity == int64(totalCapacity)) && (fleet.Status.Players.Count >= int64(totalPlayers))

‎test/e2e/fleetautoscaler_test.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ func TestAutoscalerBasicFunctions(t *testing.T) {
9696

9797
// do an allocation and watch the fleet scale up
9898
gsa := framework.CreateAndApplyAllocation(t, flt)
99-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
99+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
100100
return fleet.Status.AllocatedReplicas == 1
101101
})
102102

@@ -115,7 +115,7 @@ func TestAutoscalerBasicFunctions(t *testing.T) {
115115
assert.True(t, fas.Status.AbleToScale, "Could not get AbleToScale status")
116116

117117
// check that we are able to scale
118-
framework.WaitForFleetAutoScalerCondition(t, fas, func(fas *autoscalingv1.FleetAutoscaler) bool {
118+
framework.WaitForFleetAutoScalerCondition(t, fas, func(log *logrus.Entry, fas *autoscalingv1.FleetAutoscaler) bool {
119119
return !fas.Status.ScalingLimited
120120
})
121121

@@ -124,15 +124,15 @@ func TestAutoscalerBasicFunctions(t *testing.T) {
124124
assert.Nil(t, err, "could not patch fleetautoscaler")
125125

126126
// check that we are not able to scale
127-
framework.WaitForFleetAutoScalerCondition(t, fas, func(fas *autoscalingv1.FleetAutoscaler) bool {
127+
framework.WaitForFleetAutoScalerCondition(t, fas, func(log *logrus.Entry, fas *autoscalingv1.FleetAutoscaler) bool {
128128
return fas.Status.ScalingLimited
129129
})
130130

131131
// delete the allocated GameServer and watch the fleet scale down
132132
gp := int64(1)
133133
err = stable.GameServers(framework.Namespace).Delete(ctx, gsa.Status.GameServerName, metav1.DeleteOptions{GracePeriodSeconds: &gp})
134134
assert.Nil(t, err)
135-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
135+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
136136
return fleet.Status.AllocatedReplicas == 0 &&
137137
fleet.Status.ReadyReplicas == 1 &&
138138
fleet.Status.Replicas == 1
@@ -236,7 +236,7 @@ func TestFleetAutoScalerRollingUpdate(t *testing.T) {
236236
assert.True(t, fas.Status.AbleToScale, "Could not get AbleToScale status")
237237

238238
// check that we are able to scale
239-
framework.WaitForFleetAutoScalerCondition(t, fas, func(fas *autoscalingv1.FleetAutoscaler) bool {
239+
framework.WaitForFleetAutoScalerCondition(t, fas, func(log *logrus.Entry, fas *autoscalingv1.FleetAutoscaler) bool {
240240
return !fas.Status.ScalingLimited
241241
})
242242

@@ -471,11 +471,11 @@ func TestAutoscalerWebhook(t *testing.T) {
471471
assert.FailNow(t, "Failed creating autoscaler, aborting TestAutoscalerWebhook")
472472
}
473473
framework.CreateAndApplyAllocation(t, flt)
474-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
474+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
475475
return fleet.Status.AllocatedReplicas == 1
476476
})
477477

478-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
478+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
479479
return fleet.Status.Replicas > initialReplicasCount
480480
})
481481

@@ -701,11 +701,11 @@ func TestFleetAutoscalerTLSWebhook(t *testing.T) {
701701
assert.FailNow(t, "Failed creating autoscaler, aborting TestTlsWebhook")
702702
}
703703
framework.CreateAndApplyAllocation(t, flt)
704-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
704+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
705705
return fleet.Status.AllocatedReplicas == 1
706706
})
707707

708-
framework.AssertFleetCondition(t, flt, func(fleet *agonesv1.Fleet) bool {
708+
framework.AssertFleetCondition(t, flt, func(log *logrus.Entry, fleet *agonesv1.Fleet) bool {
709709
return fleet.Status.Replicas > initialReplicasCount
710710
})
711711
}

0 commit comments

Comments
 (0)