Skip to content

Commit 704497d

Browse files
oyeliseievtdcmeehan
oyeliseiev
authored and
tdcmeehan
committed
Add SingleStore workflow and docs
1 parent cab7808 commit 704497d

9 files changed

Lines changed: 182 additions & 8 deletions

File tree

‎.github/workflows/singlestore-tests.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: singlestore tests
2+
3+
on:
4+
pull_request:
5+
6+
env:
7+
# An envar that signals to tests we are executing in the CI environment
8+
CONTINUOUS_INTEGRATION: true
9+
MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError"
10+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
11+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
12+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
13+
RETRY: .github/bin/retry
14+
15+
jobs:
16+
changes:
17+
runs-on: ubuntu-latest
18+
# Required permissions
19+
permissions:
20+
pull-requests: read
21+
# Set job outputs to values from filter step
22+
outputs:
23+
codechange: ${{ steps.filter.outputs.codechange }}
24+
steps:
25+
# For pull requests it's not necessary to checkout the code
26+
- uses: dorny/paths-filter@v2
27+
id: filter
28+
with:
29+
filters: |
30+
codechange:
31+
- '!presto-docs/**'
32+
singlestore-dockerized-tests:
33+
runs-on: ubuntu-latest
34+
needs: changes
35+
if: needs.changes.outputs.codechange == 'true'
36+
37+
timeout-minutes: 30
38+
concurrency:
39+
group: ${{ github.workflow }}-singlestore-dockerized-tests-${{ github.event.pull_request.number }}
40+
cancel-in-progress: true
41+
steps:
42+
- uses: actions/checkout@v2
43+
- uses: actions/setup-java@v1
44+
with:
45+
java-version: 8
46+
- name: Cache local Maven repository
47+
id: cache-maven
48+
uses: actions/cache@v2
49+
with:
50+
path: ~/.m2/repository
51+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
52+
restore-keys: |
53+
${{ runner.os }}-maven-2-
54+
- name: Populate maven cache
55+
if: steps.cache-maven.outputs.cache-hit != 'true'
56+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
57+
- name: Install SingleStore Module
58+
run: |
59+
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
60+
./mvnw install ${MAVEN_FAST_INSTALL} -am -pl :presto-singlestore
61+
- name: Run SingleStore Dockerized Tests
62+
env:
63+
SINGLESTORE_LICENSE: ${{ secrets.SINGLESTORE_LICENSE }}
64+
run: ./mvnw test ${MAVEN_TEST} -pl :presto-singlestore

‎.github/workflows/test-other-modules.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ env:
1111
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
1212
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
1313
RETRY: .github/bin/retry
14-
SINGLESTORE_LICENSE: ${{ secrets.SINGLESTORE_LICENSE }}
1514

1615
jobs:
1716
changes:
@@ -80,4 +79,5 @@ jobs:
8079
!presto-thrift-connector,
8180
!presto-native-execution,
8281
!presto-test-coverage,
83-
!presto-iceberg'
82+
!presto-iceberg,
83+
!presto-singlestore'

‎presto-docs/src/main/sphinx/connector.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ from different data sources.
3636
connector/redis
3737
connector/redshift
3838
connector/scylladb
39+
connector/singlestore
3940
connector/sqlserver
4041
connector/system
4142
connector/thrift

‎presto-docs/src/main/sphinx/connector/singlestore.rst‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
===============
2+
SingleStore Connector
3+
===============
4+
5+
The SingleStore connector allows querying and creating tables in an external
6+
SingleStore database. This can be used to join data between different
7+
systems like SingleStore and Hive, or between two different SingleStore instances.
8+
9+
Configuration
10+
-------------
11+
12+
To configure the SingleStore connector, create a catalog properties file
13+
in ``etc/catalog`` named, for example, ``singlestore.properties``, to
14+
mount the SingleStore connector as the ``singlestore`` catalog.
15+
Create the file with the following contents, replacing the
16+
connection properties as appropriate for your setup:
17+
18+
.. code-block:: none
19+
20+
connector.name=singlestore
21+
connection-url=jdbc:singlestore://localhost:3306
22+
connection-user=root
23+
connection-password=LbRootPass1
24+
25+
The ``connection-url`` defines the connection information and parameters to pass
26+
to the SingleStore JDBC driver. The supported parameters for the URL are
27+
available in the `SingleStore Connection String Parameters
28+
<https://docs.singlestore.com/cloud/developer-resources/connect-with-application-development-tools/connect-with-java-jdbc/the-singlestore-jdbc-driver/#connection-string-parameters>`_.
29+
30+
The ``connection-user`` and ``connection-password`` are typically required and
31+
determine the user credentials for the connection, often a service user.
32+
33+
Multiple SingleStore Servers
34+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
35+
36+
You can have as many catalogs as you need, so if you have additional
37+
SingleStore servers, simply add another properties file to ``etc/catalog``
38+
with a different name (making sure it ends in ``.properties``). For
39+
example, if you name the property file ``sales.properties``, Presto
40+
will create a catalog named ``sales`` using the configured connector.
41+
42+
Querying SingleStore
43+
--------------------
44+
45+
The SingleStore connector provides a schema for every SingleStore *database*.
46+
You can see the available SingleStore databases by running ``SHOW SCHEMAS``::
47+
48+
SHOW SCHEMAS FROM singlestore;
49+
50+
If you have a SingleStore database named ``web``, you can view the tables
51+
in this database by running ``SHOW TABLES``::
52+
53+
SHOW TABLES FROM singlestore.web;
54+
55+
You can see a list of the columns in the ``clicks`` table in the ``web`` database
56+
using either of the following::
57+
58+
DESCRIBE singlestore.web.clicks;
59+
SHOW COLUMNS FROM singlestore.web.clicks;
60+
61+
Finally, you can access the ``clicks`` table in the ``web`` database::
62+
63+
SELECT * FROM singlestore.web.clicks;
64+
65+
If you used a different name for your catalog properties file, use
66+
that catalog name instead of ``singlestore`` in the above examples.
67+
68+
SingleStore Connector Limitations
69+
---------------------------
70+
71+
The following SQL statements are not yet supported:
72+
73+
* :doc:`/sql/create-table` with ``rowstore`` table type (supports only ``columnstore`` table type, set by default `docs <https://docs.singlestore.com/cloud/reference/sql-reference/data-definition-language-ddl/create-table/>`_.)
74+
* :doc:`/sql/alter-schema`
75+
* :doc:`/sql/analyze`
76+
* :doc:`/sql/create-role`
77+
* :doc:`/sql/create-schema`
78+
* :doc:`/sql/create-view`
79+
* :doc:`/sql/delete`
80+
* :doc:`/sql/drop-role`
81+
* :doc:`/sql/grant`
82+
* :doc:`/sql/revoke`
83+
* :doc:`/sql/show-grants`
84+
* :doc:`/sql/show-roles`
85+
* :doc:`/sql/show-role-grants`
86+
* :doc:`/sql/set-role`

‎presto-singlestore/README.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
SingleStore connector.
2+
3+
Please see the website for SingleSore:
4+
https://www.singlestore.com/
5+
6+
Test execution:
7+
8+
To execute unit tests you need to set up the environment variable 'SINGLESTORE_LICENSE' with SingleStore license value.
9+
To get a free-tier license you need to create a free SingleStore account.
10+
More details:
11+
https://www.singlestore.com/blog/announcing-memsql-free-tier/

‎presto-singlestore/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@
2929
<artifactId>configuration</artifactId>
3030
</dependency>
3131

32+
<dependency>
33+
<groupId>com.facebook.airlift</groupId>
34+
<artifactId>log</artifactId>
35+
</dependency>
36+
3237
<dependency>
3338
<groupId>com.google.guava</groupId>
3439
<artifactId>guava</artifactId>

‎presto-singlestore/src/test/java/com/facebook/presto/plugin/singlestore/DockerizedSingleStoreServer.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
package com.facebook.presto.plugin.singlestore;
1616

17+
import com.facebook.airlift.log.Logger;
1718
import com.facebook.presto.testing.docker.DockerContainer;
1819
import com.google.common.collect.ImmutableList;
1920
import com.google.common.collect.ImmutableMap;
@@ -25,11 +26,12 @@
2526
import java.sql.Statement;
2627

2728
import static java.lang.String.format;
28-
import static org.testng.Assert.assertNotNull;
29+
import static org.testng.Assert.assertTrue;
2930

3031
public class DockerizedSingleStoreServer
3132
implements Closeable
3233
{
34+
private static final Logger LOG = Logger.get(DockerizedSingleStoreServer.class);
3335
private static final int SINGLESTORE_PORT = 3306;
3436
private static final String SINGLESTORE_USER = "root";
3537
private static final String ROOT_PASSWORD = "LbRootPass1";
@@ -39,8 +41,12 @@ public class DockerizedSingleStoreServer
3941

4042
public DockerizedSingleStoreServer()
4143
{
42-
final String license = System.getenv(SINGLESTORE_LICENSE_PROPERTY);
43-
assertNotNull(license, "Missed environment variable '" + SINGLESTORE_LICENSE_PROPERTY + "'");
44+
String license = System.getenv(SINGLESTORE_LICENSE_PROPERTY);
45+
if (license == null || license.isEmpty()) {
46+
LOG.info("Missed environment variable: '" + SINGLESTORE_LICENSE_PROPERTY + "'");
47+
license = "BGNkNGNlYzFlYWYwYjQ0Yjc5ZGQ5ZTUwM2NjZTc3OWMxAAAAAAAAAAAEAAAAAAAAACgwNQIYK8UIk2TShKVDLkN3bRKbH/JMFGURGStoAhkAtCoka/omJzS5DWPltpVhJMjqh4ZV2bYEAA==";
48+
}
49+
assertTrue(license != null && !license.isEmpty(), "Missed environment variable: '" + SINGLESTORE_LICENSE_PROPERTY + "'");
4450
this.dockerContainer = new DockerContainer(
4551
"ghcr.io/singlestore-labs/singlestoredb-dev:latest",
4652
ImmutableList.of(SINGLESTORE_PORT),

‎presto-singlestore/src/test/java/com/facebook/presto/plugin/singlestore/SingleStoreQueryRunner.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ private SingleStoreQueryRunner()
4040
}
4141

4242
private static final String TPCH_SCHEMA = "tpch";
43+
private static final String SINGLE_STORE = "singlestore";
4344

4445
public static QueryRunner createSingleStoreQueryRunner(DockerizedSingleStoreServer server, TpchTable<?>... tables)
4546
throws Exception
@@ -64,7 +65,7 @@ public static QueryRunner createSingleStoreQueryRunner(DockerizedSingleStoreServ
6465
createSchema(server.getJdbcUrl(), TPCH_SCHEMA);
6566

6667
queryRunner.installPlugin(new SingleStorePlugin());
67-
queryRunner.createCatalog("singlestore", "singlestore", connectorProperties);
68+
queryRunner.createCatalog(SINGLE_STORE, SINGLE_STORE, connectorProperties);
6869

6970
copyTpchTables(queryRunner, TPCH_SCHEMA, TINY_SCHEMA_NAME, createSession(), tables);
7071

@@ -88,7 +89,7 @@ private static void createSchema(String url, String schema)
8889
public static Session createSession()
8990
{
9091
return testSessionBuilder()
91-
.setCatalog("singlestore")
92+
.setCatalog(SINGLE_STORE)
9293
.setSchema(TPCH_SCHEMA)
9394
.build();
9495
}

‎presto-testing-docker/src/main/java/com/facebook/presto/testing/docker/DockerContainer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ private void startContainer(List<Integer> ports, CheckedConsumer<HostPortProvide
8080
throws Exception
8181
{
8282
dockerClient = DefaultDockerClient.fromEnv().build();
83-
if (dockerClient.listImages(DockerClient.ListImagesParam.byName(image)).isEmpty()) {
83+
if (dockerClient.listImages().stream().noneMatch(i -> i.repoTags().contains(image))) {
8484
checkState(!image.endsWith("-SNAPSHOT"), "Unavailable snapshot image %s, please build before running tests", image);
8585
LOG.info("Pulling image %s...", image);
8686
dockerClient.pull(image);

0 commit comments

Comments
 (0)