Skip to content

Commit 9d78307

Browse files
olavloiterahul2393
olavloite
and
rahul2393
authored
feat(spanner): add settings for gRCP keep-alive (#13643)
Adds options for setting a gRPC keep-alive and keep-alive-timeout for the Spanner client. --------- Co-authored-by: Rahul Yadav <rahulyadavsep92@gmail.com>
1 parent bb65627 commit 9d78307

10 files changed

Lines changed: 558 additions & 5 deletions

File tree

‎java-spanner/google-cloud-spanner/pom.xml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
</parent>
1616
<properties>
1717
<site.installationModule>google-cloud-spanner</site.installationModule>
18+
<awaitility.version>4.3.0</awaitility.version>
1819
<opencensus.version>0.31.1</opencensus.version>
1920
<google.cloud.monitoring.version>3.85.0</google.cloud.monitoring.version>
2021
<spanner.testenv.config.class>com.google.cloud.spanner.GceTestEnvConfig</spanner.testenv.config.class>
@@ -435,6 +436,13 @@
435436
<scope>test</scope>
436437
</dependency>
437438

439+
<dependency>
440+
<groupId>org.awaitility</groupId>
441+
<artifactId>awaitility</artifactId>
442+
<version>${awaitility.version}</version>
443+
<scope>test</scope>
444+
</dependency>
445+
438446
<dependency>
439447
<groupId>com.google.api.grpc</groupId>
440448
<artifactId>proto-google-cloud-spanner-executor-v1</artifactId>

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,8 @@ static GcpChannelPoolOptions mergeWithDefaultChannelPoolOptions(
268268
private final InstanceAdminStubSettings instanceAdminStubSettings;
269269
private final DatabaseAdminStubSettings databaseAdminStubSettings;
270270
private final Duration partitionedDmlTimeout;
271+
private final Duration grpcKeepAliveTime;
272+
private final Duration grpcKeepAliveTimeout;
271273
private final boolean grpcGcpExtensionEnabled;
272274
private final GcpManagedChannelOptions grpcGcpOptions;
273275
private final boolean dynamicChannelPoolEnabled;
@@ -939,6 +941,8 @@ protected SpannerOptions(Builder builder) {
939941
throw SpannerExceptionFactory.newSpannerException(e);
940942
}
941943
partitionedDmlTimeout = builder.partitionedDmlTimeout;
944+
grpcKeepAliveTime = builder.grpcKeepAliveTime;
945+
grpcKeepAliveTimeout = builder.grpcKeepAliveTimeout;
942946
grpcGcpExtensionEnabled = builder.grpcGcpExtensionEnabled;
943947
grpcGcpOptions = builder.grpcGcpOptions;
944948

@@ -1317,6 +1321,8 @@ private static Builder prepareBuilder(Builder builder) {
13171321
private Map<DatabaseId, QueryOptions> defaultQueryOptions = new HashMap<>();
13181322
private boolean enableGrpcGcpOtelMetrics =
13191323
SpannerOptions.environment.isEnableGrpcGcpOtelMetrics();
1324+
private Duration grpcKeepAliveTime = Duration.ofSeconds(120);
1325+
private Duration grpcKeepAliveTimeout = Duration.ofSeconds(20);
13201326
private CallCredentialsProvider callCredentialsProvider;
13211327
private CloseableExecutorProvider asyncExecutorProvider;
13221328
private String compressorName;
@@ -1430,6 +1436,8 @@ protected Builder() {
14301436
this.enableGrpcGcpOtelMetrics = options.enableGrpcGcpOtelMetrics;
14311437
this.defaultQueryOptions = options.defaultQueryOptions;
14321438
this.callCredentialsProvider = options.callCredentialsProvider;
1439+
this.grpcKeepAliveTime = options.grpcKeepAliveTime;
1440+
this.grpcKeepAliveTimeout = options.grpcKeepAliveTimeout;
14331441
this.asyncExecutorProvider = options.asyncExecutorProvider;
14341442
this.compressorName = options.compressorName;
14351443
this.channelProvider = options.channelProvider;
@@ -1692,6 +1700,32 @@ public Builder setPartitionedDmlTimeoutDuration(Duration timeout) {
16921700
return this;
16931701
}
16941702

1703+
/**
1704+
* Sets the keep-alive time for gRPC connections. The default is 120 seconds. Note that the
1705+
* client-side keepalive time is clamped to a minimum of 10 seconds by gRPC.
1706+
*/
1707+
public Builder setGrpcKeepAliveTime(Duration grpcKeepAliveTime) {
1708+
Preconditions.checkNotNull(grpcKeepAliveTime, "grpcKeepAliveTime cannot be null");
1709+
Preconditions.checkArgument(
1710+
!grpcKeepAliveTime.isNegative() && !grpcKeepAliveTime.isZero(),
1711+
"grpcKeepAliveTime must be positive");
1712+
this.grpcKeepAliveTime = grpcKeepAliveTime;
1713+
return this;
1714+
}
1715+
1716+
/**
1717+
* Sets the keep-alive timeout for gRPC connections. The default is 20 seconds. Note that the
1718+
* client-side keepalive timeout is clamped to a minimum of 20 milliseconds by gRPC.
1719+
*/
1720+
public Builder setGrpcKeepAliveTimeout(Duration grpcKeepAliveTimeout) {
1721+
Preconditions.checkNotNull(grpcKeepAliveTimeout, "grpcKeepAliveTimeout cannot be null");
1722+
Preconditions.checkArgument(
1723+
!grpcKeepAliveTimeout.isNegative() && !grpcKeepAliveTimeout.isZero(),
1724+
"grpcKeepAliveTimeout must be positive");
1725+
this.grpcKeepAliveTimeout = grpcKeepAliveTimeout;
1726+
return this;
1727+
}
1728+
16951729
/**
16961730
* Instructs the client library to automatically throttle the number of administrative requests
16971731
* if the rate of administrative requests generated by this {@link Spanner} instance will exceed
@@ -2493,6 +2527,14 @@ public Duration getPartitionedDmlTimeoutDuration() {
24932527
return partitionedDmlTimeout;
24942528
}
24952529

2530+
public Duration getGrpcKeepAliveTime() {
2531+
return grpcKeepAliveTime;
2532+
}
2533+
2534+
public Duration getGrpcKeepAliveTimeout() {
2535+
return grpcKeepAliveTimeout;
2536+
}
2537+
24962538
public boolean isGrpcGcpExtensionEnabled() {
24972539
return grpcGcpExtensionEnabled;
24982540
}

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
import static com.google.cloud.spanner.connection.ConnectionProperties.ENCODED_CREDENTIALS;
4343
import static com.google.cloud.spanner.connection.ConnectionProperties.ENDPOINT;
4444
import static com.google.cloud.spanner.connection.ConnectionProperties.GRPC_INTERCEPTOR_PROVIDER;
45+
import static com.google.cloud.spanner.connection.ConnectionProperties.GRPC_KEEPALIVE_TIME;
46+
import static com.google.cloud.spanner.connection.ConnectionProperties.GRPC_KEEPALIVE_TIMEOUT;
4547
import static com.google.cloud.spanner.connection.ConnectionProperties.IS_EXPERIMENTAL_HOST;
4648
import static com.google.cloud.spanner.connection.ConnectionProperties.LENIENT;
4749
import static com.google.cloud.spanner.connection.ConnectionProperties.MAX_COMMIT_DELAY;
@@ -279,6 +281,12 @@ public class ConnectionOptions {
279281
*/
280282
public static final String OAUTH_TOKEN_PROPERTY_NAME = "oauthToken";
281283

284+
/** Name of the 'grpcKeepAliveTime' connection property. */
285+
public static final String GRPC_KEEPALIVE_TIME_PROPERTY_NAME = "grpcKeepAliveTime";
286+
287+
/** Name of the 'grpcKeepAliveTimeout' connection property. */
288+
public static final String GRPC_KEEPALIVE_TIMEOUT_PROPERTY_NAME = "grpcKeepAliveTimeout";
289+
282290
/** Name of the 'minSessions' connection property. */
283291
public static final String MIN_SESSIONS_PROPERTY_NAME = "minSessions";
284292

@@ -1083,6 +1091,16 @@ public Integer getNumChannels() {
10831091
return getInitialConnectionPropertyValue(NUM_CHANNELS);
10841092
}
10851093

1094+
/** The gRPC keepalive time for this connection. */
1095+
public Duration getGrpcKeepAliveTime() {
1096+
return getInitialConnectionPropertyValue(GRPC_KEEPALIVE_TIME);
1097+
}
1098+
1099+
/** The gRPC keepalive timeout for this connection. */
1100+
public Duration getGrpcKeepAliveTimeout() {
1101+
return getInitialConnectionPropertyValue(GRPC_KEEPALIVE_TIMEOUT);
1102+
}
1103+
10861104
/** Whether dynamic channel pooling is enabled for this connection. */
10871105
public Boolean isEnableDynamicChannelPool() {
10881106
return getInitialConnectionPropertyValue(ENABLE_DYNAMIC_CHANNEL_POOL);

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@
9797
import static com.google.cloud.spanner.connection.ConnectionOptions.ENABLE_GRPC_INTERCEPTOR_PROVIDER_SYSTEM_PROPERTY;
9898
import static com.google.cloud.spanner.connection.ConnectionOptions.ENCODED_CREDENTIALS_PROPERTY_NAME;
9999
import static com.google.cloud.spanner.connection.ConnectionOptions.ENDPOINT_PROPERTY_NAME;
100+
import static com.google.cloud.spanner.connection.ConnectionOptions.GRPC_KEEPALIVE_TIMEOUT_PROPERTY_NAME;
101+
import static com.google.cloud.spanner.connection.ConnectionOptions.GRPC_KEEPALIVE_TIME_PROPERTY_NAME;
100102
import static com.google.cloud.spanner.connection.ConnectionOptions.IS_EXPERIMENTAL_HOST_PROPERTY_NAME;
101103
import static com.google.cloud.spanner.connection.ConnectionOptions.KEEP_TRANSACTION_ALIVE_PROPERTY_NAME;
102104
import static com.google.cloud.spanner.connection.ConnectionOptions.LENIENT_PROPERTY_NAME;
@@ -253,6 +255,24 @@ public class ConnectionProperties {
253255
BOOLEANS,
254256
BooleanConverter.INSTANCE,
255257
Context.STARTUP);
258+
static final ConnectionProperty<Duration> GRPC_KEEPALIVE_TIME =
259+
create(
260+
GRPC_KEEPALIVE_TIME_PROPERTY_NAME,
261+
"The keepalive time for gRPC connections (e.g. '120s', '20s'). "
262+
+ "Setting a lower keep-alive time (minimum 10s enforced by the gRPC library) "
263+
+ "helps detect disconnected connections faster.",
264+
null,
265+
DurationConverter.INSTANCE,
266+
Context.STARTUP);
267+
static final ConnectionProperty<Duration> GRPC_KEEPALIVE_TIMEOUT =
268+
create(
269+
GRPC_KEEPALIVE_TIMEOUT_PROPERTY_NAME,
270+
"The keepalive timeout for gRPC connections (e.g. '20s', '5s'). "
271+
+ "This determines how long the client waits for a keep-alive ping response before terminating "
272+
+ "the connection. A lower timeout helps speed up recovery during network failures.",
273+
null,
274+
DurationConverter.INSTANCE,
275+
Context.STARTUP);
256276

257277
/**
258278
* @deprecated Use {@link #TYPE} with value "omni" instead.

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import io.grpc.ManagedChannelBuilder;
3333
import io.opentelemetry.api.OpenTelemetry;
3434
import java.io.IOException;
35+
import java.time.Duration;
3536
import java.util.ArrayList;
3637
import java.util.HashMap;
3738
import java.util.List;
@@ -151,6 +152,8 @@ public boolean equals(Object o) {
151152
static class SpannerPoolKey {
152153
private final String host;
153154
private final String projectId;
155+
private final Duration grpcKeepAliveTime;
156+
private final Duration grpcKeepAliveTimeout;
154157
private final CredentialsKey credentialsKey;
155158
private final SessionPoolOptions sessionPoolOptions;
156159
private final Integer numChannels;
@@ -222,6 +225,8 @@ private SpannerPoolKey(ConnectionOptions options) throws IOException {
222225
this.enableDirectAccess = options.isEnableDirectAccess();
223226
this.universeDomain = options.getUniverseDomain();
224227
this.grpcInterceptorProvider = options.getGrpcInterceptorProviderName();
228+
this.grpcKeepAliveTime = options.getGrpcKeepAliveTime();
229+
this.grpcKeepAliveTimeout = options.getGrpcKeepAliveTimeout();
225230
}
226231

227232
@Override
@@ -259,7 +264,9 @@ public boolean equals(Object o) {
259264
&& Objects.equals(this.instanceType, other.instanceType)
260265
&& Objects.equals(this.enableDirectAccess, other.enableDirectAccess)
261266
&& Objects.equals(this.universeDomain, other.universeDomain)
262-
&& Objects.equals(this.grpcInterceptorProvider, other.grpcInterceptorProvider);
267+
&& Objects.equals(this.grpcInterceptorProvider, other.grpcInterceptorProvider)
268+
&& Objects.equals(this.grpcKeepAliveTime, other.grpcKeepAliveTime)
269+
&& Objects.equals(this.grpcKeepAliveTimeout, other.grpcKeepAliveTimeout);
263270
}
264271

265272
@Override
@@ -292,7 +299,9 @@ public int hashCode() {
292299
this.instanceType,
293300
this.enableDirectAccess,
294301
this.universeDomain,
295-
this.grpcInterceptorProvider);
302+
this.grpcInterceptorProvider,
303+
this.grpcKeepAliveTime,
304+
this.grpcKeepAliveTimeout);
296305
}
297306
}
298307

@@ -510,6 +519,12 @@ Spanner createSpanner(SpannerPoolKey key, ConnectionOptions options) {
510519
if (options.getChannelProvider() != null) {
511520
builder.setChannelProvider(options.getChannelProvider());
512521
}
522+
if (key.grpcKeepAliveTime != null) {
523+
builder.setGrpcKeepAliveTime(key.grpcKeepAliveTime);
524+
}
525+
if (key.grpcKeepAliveTimeout != null) {
526+
builder.setGrpcKeepAliveTimeout(key.grpcKeepAliveTimeout);
527+
}
513528
if (!options.isRouteToLeader()) {
514529
builder.disableLeaderAwareRouting();
515530
}

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,6 @@ public class GapicSpannerRpc implements SpannerRpc {
251251
"com.google.cloud.spanner.watchdogPeriodSeconds";
252252
private static final int DEFAULT_TIMEOUT_SECONDS = 30 * 60;
253253
private static final int DEFAULT_PERIOD_SECONDS = 10;
254-
private static final int GRPC_KEEPALIVE_SECONDS = 2 * 60;
255254
private static final String USER_AGENT_KEY = "user-agent";
256255
private static final String CLIENT_LIBRARY_LANGUAGE = "spanner-java";
257256
public static final String DEFAULT_USER_AGENT =
@@ -740,9 +739,10 @@ private InstantiatingGrpcChannelProvider.Builder createBaseChannelProviderBuilde
740739
.setMaxInboundMetadataSize(MAX_METADATA_SIZE)
741740
.setPoolSize(options.getNumChannels())
742741

743-
// Set a keepalive time of 120 seconds to help long running
742+
// Set a keepalive time to help long running
744743
// commit GRPC calls succeed
745-
.setKeepAliveTimeDuration(Duration.ofSeconds(GRPC_KEEPALIVE_SECONDS))
744+
.setKeepAliveTimeDuration(options.getGrpcKeepAliveTime())
745+
.setKeepAliveTimeoutDuration(options.getGrpcKeepAliveTimeout())
746746

747747
// Then check if SpannerOptions provides an InterceptorProvider. Create a default
748748
// SpannerInterceptorProvider if none is provided

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTest.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1452,4 +1452,69 @@ public void testLogin() {
14521452
SpannerOptions options = builder.build();
14531453
assertTrue(options.getCredentials() instanceof SpannerOmniCredentials);
14541454
}
1455+
1456+
@Test
1457+
public void testGrpcKeepAliveTime() {
1458+
SpannerOptions defaultOptions =
1459+
SpannerOptions.newBuilder()
1460+
.setProjectId("test-project")
1461+
.setCredentials(NoCredentials.getInstance())
1462+
.build();
1463+
assertEquals(Duration.ofSeconds(120), defaultOptions.getGrpcKeepAliveTime());
1464+
1465+
SpannerOptions customOptions =
1466+
SpannerOptions.newBuilder()
1467+
.setProjectId("test-project")
1468+
.setCredentials(NoCredentials.getInstance())
1469+
.setGrpcKeepAliveTime(Duration.ofSeconds(20))
1470+
.build();
1471+
assertEquals(Duration.ofSeconds(20), customOptions.getGrpcKeepAliveTime());
1472+
1473+
SpannerOptions optionsFromBuilder = customOptions.toBuilder().build();
1474+
assertEquals(Duration.ofSeconds(20), optionsFromBuilder.getGrpcKeepAliveTime());
1475+
1476+
assertThrows(
1477+
NullPointerException.class, () -> SpannerOptions.newBuilder().setGrpcKeepAliveTime(null));
1478+
1479+
assertThrows(
1480+
IllegalArgumentException.class,
1481+
() -> SpannerOptions.newBuilder().setGrpcKeepAliveTime(Duration.ZERO));
1482+
1483+
assertThrows(
1484+
IllegalArgumentException.class,
1485+
() -> SpannerOptions.newBuilder().setGrpcKeepAliveTime(Duration.ofSeconds(-10)));
1486+
}
1487+
1488+
@Test
1489+
public void testGrpcKeepAliveTimeout() {
1490+
SpannerOptions defaultOptions =
1491+
SpannerOptions.newBuilder()
1492+
.setProjectId("test-project")
1493+
.setCredentials(NoCredentials.getInstance())
1494+
.build();
1495+
assertEquals(Duration.ofSeconds(20), defaultOptions.getGrpcKeepAliveTimeout());
1496+
1497+
SpannerOptions customOptions =
1498+
SpannerOptions.newBuilder()
1499+
.setProjectId("test-project")
1500+
.setCredentials(NoCredentials.getInstance())
1501+
.setGrpcKeepAliveTimeout(Duration.ofSeconds(5))
1502+
.build();
1503+
assertEquals(Duration.ofSeconds(5), customOptions.getGrpcKeepAliveTimeout());
1504+
1505+
SpannerOptions optionsFromBuilder = customOptions.toBuilder().build();
1506+
assertEquals(Duration.ofSeconds(5), optionsFromBuilder.getGrpcKeepAliveTimeout());
1507+
1508+
assertThrows(
1509+
NullPointerException.class,
1510+
() -> SpannerOptions.newBuilder().setGrpcKeepAliveTimeout(null));
1511+
1512+
assertThrows(
1513+
IllegalArgumentException.class,
1514+
() -> SpannerOptions.newBuilder().setGrpcKeepAliveTimeout(Duration.ZERO));
1515+
1516+
assertThrows(
1517+
IllegalArgumentException.class,
1518+
() -> SpannerOptions.newBuilder().setGrpcKeepAliveTimeout(Duration.ofSeconds(-10)));
1519+
}
14551520
}

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/AbstractMockServerTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,8 @@ public void getOperation(
191191
.addService(mockDatabaseAdmin)
192192
.addService(mockOperations)
193193
.intercept(interceptor)
194+
.permitKeepAliveTime(10, TimeUnit.MILLISECONDS)
195+
.permitKeepAliveWithoutCalls(true)
194196
.build()
195197
.start();
196198
mockSpanner.putStatementResult(

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1597,4 +1597,44 @@ public void testDcpWithAllOptions() {
15971597
assertEquals(Integer.valueOf(15), options.getDcpMaxChannels());
15981598
assertEquals(Integer.valueOf(5), options.getDcpInitialChannels());
15991599
}
1600+
1601+
@Test
1602+
public void testGrpcKeepAliveTimeOption() {
1603+
ConnectionOptions options =
1604+
ConnectionOptions.newBuilder()
1605+
.setUri(
1606+
"cloudspanner:/projects/test-project-123/instances/test-instance/databases/test-database"
1607+
+ "?grpcKeepAliveTime='20s'")
1608+
.setCredentials(NoCredentials.getInstance())
1609+
.build();
1610+
assertEquals(Duration.ofSeconds(20), options.getGrpcKeepAliveTime());
1611+
1612+
ConnectionOptions defaultOptions =
1613+
ConnectionOptions.newBuilder()
1614+
.setUri(
1615+
"cloudspanner:/projects/test-project-123/instances/test-instance/databases/test-database")
1616+
.setCredentials(NoCredentials.getInstance())
1617+
.build();
1618+
assertNull(defaultOptions.getGrpcKeepAliveTime());
1619+
}
1620+
1621+
@Test
1622+
public void testGrpcKeepAliveTimeoutOption() {
1623+
ConnectionOptions options =
1624+
ConnectionOptions.newBuilder()
1625+
.setUri(
1626+
"cloudspanner:/projects/test-project-123/instances/test-instance/databases/test-database"
1627+
+ "?grpcKeepAliveTimeout='15s'")
1628+
.setCredentials(NoCredentials.getInstance())
1629+
.build();
1630+
assertEquals(Duration.ofSeconds(15), options.getGrpcKeepAliveTimeout());
1631+
1632+
ConnectionOptions defaultOptions =
1633+
ConnectionOptions.newBuilder()
1634+
.setUri(
1635+
"cloudspanner:/projects/test-project-123/instances/test-instance/databases/test-database")
1636+
.setCredentials(NoCredentials.getInstance())
1637+
.build();
1638+
assertNull(defaultOptions.getGrpcKeepAliveTimeout());
1639+
}
16001640
}

0 commit comments

Comments
 (0)