Skip to content

Commit 0bc29df

Browse files
Cyril Tovenamarkmandel
Cyril Tovena
authored and
markmandel
committed
Removes the sdk generation tooling from our main build image.
Each supported SDK language has its own Dockerfile. We support 3 commands (gen,build,test) for each SDK. Add documentation on how to add a new SDK.
1 parent bafcfea commit 0bc29df

23 files changed

Lines changed: 542 additions & 116 deletions

File tree

‎build/Makefile‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ include ./includes/google-cloud.mk
177177
include ./includes/minikube.mk
178178
include ./includes/kind.mk
179179
include ./includes/website.mk
180+
include ./includes/sdk.mk
180181

181182
# personal includes, excluded from the git repository
182183
-include ./local-includes/*.mk
@@ -220,21 +221,14 @@ push-release-chart: $(ensure-build-image)
220221
mv /tmp/agones/install/helm/agones $(agones_path)/install/.helm-$(RELEASE_VERSION)/
221222
CHART_DIR=install/.helm-$(RELEASE_VERSION)/agones $(MAKE) push-chart
222223

223-
#build all the sdks
224-
build-sdks: build-sdk-cpp
225-
226224
# Run all tests
227-
test: $(ensure-build-image) test-go test-nodejs test-install-yaml site-test
225+
test: $(ensure-build-image) test-go test-sdks test-install-yaml site-test
228226

229227
# Run go tests
230228
test-go:
231229
$(GO_TEST) $(agones_package)/pkg/... \
232230
$(agones_package)/sdks/... $(agones_package)/cmd/...
233231

234-
# Run Node.js SDK tests
235-
test-nodejs:
236-
docker run --rm $(common_mounts) -w $(workdir_path)/sdks/nodejs $(build_tag) npm install-test
237-
238232
# Runs end-to-end tests on the current configured cluster
239233
# For minikube user the minikube-test-e2e targets
240234
test-e2e: $(ensure-build-image)
@@ -353,10 +347,6 @@ push-ping-image: $(ensure-build-image)
353347
build-ping-image: $(ensure-build-image) build-ping-binary build-licenses build-required-src-dist
354348
docker build $(agones_path)/cmd/ping/ --tag=$(ping_tag) $(DOCKER_BUILD_ARGS)
355349

356-
# Build the cpp sdk linux archive
357-
build-sdk-cpp: $(ensure-build-image)
358-
docker run --rm $(common_mounts) -w $(workdir_path)/sdks/cpp $(build_tag) make build install archive VERSION=$(VERSION)
359-
360350
# push the gameservers sidecar image
361351
push-agones-sdk-image: $(ensure-build-image)
362352
docker push $(sidecar_tag)
@@ -369,13 +359,6 @@ gen-install: $(ensure-build-image)
369359
--set agones.crds.cleanupOnDelete=false \
370360
> $(mount_path)/install/yaml/install.yaml'
371361

372-
# Generate the SDK gRPC server and client code
373-
gen-gameservers-sdk-grpc: $(ensure-build-image)
374-
docker run --rm $(common_mounts) -w $(workdir_path) $(build_tag) /root/gen-grpc-go.sh
375-
docker run --rm $(common_mounts) -w $(workdir_path) $(build_tag) /root/gen-grpc-cpp.sh
376-
docker run --rm $(common_mounts) -w $(workdir_path) $(build_tag) /root/gen-grpc-rust.sh
377-
docker run --rm $(common_mounts) -w $(workdir_path) $(build_tag) /root/gen-grpc-nodejs.sh
378-
379362
# Generate the client for our CustomResourceDefinition
380363
gen-crd-client: $(ensure-build-image)
381364
docker run --rm $(common_mounts) -w $(workdir_path) $(build_tag) /root/gen-crd-client.sh

‎build/build-image/Dockerfile‎

Lines changed: 5 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -12,40 +12,15 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
# ForceUpdate 6 -- change here if you need to force a rebuild
15+
# ForceUpdate 7 -- change here if you need to force a rebuild
1616

17-
# compiling proto + grpc takes an exceptionally long time
18-
# so we'll use a base from `base` - which is manually built using the below tag.
19-
FROM gcr.io/agones-images/grpc-cxx:1.16.1
17+
FROM debian:stretch
2018

2119
RUN apt-get update && \
22-
apt-get install -y wget psmisc rsync make python bash-completion zip nano jq graphviz \
23-
gettext-base plantuml && \
20+
apt-get install -y build-essential gnupg curl git wget psmisc rsync make python bash-completion \
21+
zip nano jq graphviz gettext-base plantuml && \
2422
apt-get clean
2523

26-
# _
27-
# | | __ _ _ __ __ _ _ _ __ _ __ _ ___ ___
28-
# | | / _` | '_ \ / _` | | | |/ _` |/ _` |/ _ \ __|
29-
# | |___ (_| | | | | (_| | |_| | (_| | (_| | __\__ \
30-
# |_____\__,_|_| |_|\__, |\__,_|\__,_|\__, |\___|___/
31-
# |___/ |___/
32-
33-
# install rust
34-
ENV RUSTUP_HOME=/usr/local/rustup \
35-
CARGO_HOME=/usr/local/cargo \
36-
PATH=/usr/local/cargo/bin:$PATH \
37-
RUST_VERSION=1.26.1
38-
ENV RUST_ARCH=x86_64-unknown-linux-gnu \
39-
RUSTUP_SHA256=c9837990bce0faab4f6f52604311a19bb8d2cde989bea6a7b605c8e526db6f02
40-
RUN wget -q https://static.rust-lang.org/rustup/archive/1.11.0/${RUST_ARCH}/rustup-init && \
41-
echo "${RUSTUP_SHA256} *rustup-init" | sha256sum -c - && \
42-
chmod +x rustup-init && \
43-
./rustup-init -y --no-modify-path --default-toolchain $RUST_VERSION && \
44-
rm rustup-init && \
45-
rustup --version; \
46-
cargo --version; \
47-
rustc --version;
48-
4924
# install go
5025
WORKDIR /usr/local
5126
ENV GO_VERSION=1.12
@@ -83,23 +58,6 @@ RUN curl -L ${HELM_URL} > /tmp/helm.tar.gz \
8358
&& rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64
8459
RUN echo "source <(helm completion bash)" >> /root/.bashrc
8560

86-
# ____ ____ ____ _____ _ _
87-
# __ _| _ \| _ \ / ___| |_ _|__ ___ | (_)_ __ __ _
88-
# / _` | |_) | |_) | | | |/ _ \ / _ \| | | '_ \ / _` |
89-
# | (_| | _ <| __/| |___ | | (_) | (_) | | | | | | (_| |
90-
# \__, |_| \_\_| \____| |_|\___/ \___/|_|_|_| |_|\__, |
91-
# |___/ |___/
92-
93-
# install protoc-gen-go 1.2.0
94-
RUN go get github.com/golang/protobuf/protoc-gen-go@v1.2.0
95-
96-
# install rust tooling for SDK generation
97-
RUN cargo install protobuf-codegen --vers 2.0.2
98-
RUN cargo install grpcio-compiler --vers 0.3.0
99-
100-
# install go tooling for development, building and testing
101-
RUN go get -u golang.org/x/tools/cmd/goimports
102-
10361
# install golang-ci linter
10462
RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $GOPATH/bin v1.15.0
10563

@@ -120,7 +78,7 @@ RUN mkdir /tmp/hugo && \
12078
mv /tmp/hugo/hugo /usr/local/bin/ && \
12179
rm -r /tmp/hugo
12280

123-
RUN curl -sL https://deb.nodesource.com/setup_11.x | bash - && \
81+
RUN curl -sL https://deb.nodesource.com/setup_11.x | bash - && \
12482
apt-get install -y nodejs
12583

12684
# install API reference docs generator

‎build/build-sdk-images/README.md‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Agones SDKs build
2+
3+
As we seen before GameServers can communicate their current state back to Agones controller using a side car. This sidecar runs a GRPC server handling the communication to Agones, the GameServer connects using a SDK which a thin wrapper around GRPC client code.
4+
5+
By using GRPC, adding more languages should be pretty straigthforward as it supports code generation for many of them.
6+
7+
> if your language doesn't support GRPC you can still create a SDK by connecting to the HTTP GRPC gateway.
8+
9+
This guide explains how to build, test and generate GRPC code for our SDKs but also how to add a new language.
10+
11+
## How it works
12+
13+
Our build system heavily rely on Docker and make, but that's all you need. The rest of the GRPC tooling and language specific dependencies are installed inside Docker images via Dockerfile.
14+
15+
We separated each language specific tooling in their on Docker image, this way building them will be faster and easier to maintain.
16+
17+
A base GRPC image with protoc is provided, every other images inherit from it via the `FROM` Dockerfile syntax. This way SDKs grpc code is generated from the same version of our SDK sidecar.
18+
19+
## Targets
20+
21+
We currently support 3 commands per SDK:
22+
23+
- `gen` to generate GRPC required by SDKs.
24+
- `test` to run SDKs tests.
25+
- `build` to build SDKs binaries.
26+
27+
> All commands might not be required for all SDKs. (e.g. build is only used for our cpp SDK)
28+
29+
SDKs build scripts and Dockerfile are stored in a folder in this directory.
30+
31+
To run tests for a single SDK use with the `SDK_FOLDER` of your choice:
32+
33+
```bash
34+
make test-sdk SDK_FOLDER=go
35+
```
36+
37+
You can also run all SDKs tests by using `make test-sdks`.
38+
39+
To generate GRPC code and build binaries you can respectively use:
40+
41+
```bash
42+
make gen-sdk-grpc SDK_FOLDER=go
43+
make build-sdk SDK_FOLDER=go
44+
45+
# for all SDKs
46+
make gen-all-sdk-grpc
47+
make build-sdks
48+
```
49+
50+
## Adding support for a new language
51+
52+
Makefile targets run docker containers built from Dockerfile in each folder found in this directory. This means you don't need to change our Makefile
53+
54+
Simply create a new directory with the name of your SDK. Then copy our [template](./tool/template) folder content you SDK folder.
55+
56+
Edit the Dockerfile to install all dependencies you need to generate, build and test your SDK. (you should not need to change the base image)
57+
58+
> As explained in our [SDK documentation](https://agones.dev/site/docs/guides/client-sdks/) you have 2 options HTTP or GRPC. If you are using HTTP you don't need to use our base SDK image so feel free to use the distribution of your choice.
59+
60+
Then add your steps in `build.sh`, `test.sh` and `gen.sh` script files.
61+
62+
You should now be able to use your `SDK_FOLDER` with our [Makefile targets](#targets).
63+
64+
Each targets will ensure that your Dockerfile is built and then run the image with a pre-defined command. The Agones code source repository is mounted in the working directory inside the container and you can also access the current Agones version via the environment variable `VERSION`.

‎build/build-sdk-images/cpp/Dockerfile‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2018 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
ARG BASE_IMAGE=agones-build-sdk-base:latest
15+
FROM $BASE_IMAGE
16+
17+
RUN apt-get update && \
18+
apt-get install -y zip wget && \
19+
apt-get clean
20+
21+
ADD https://cmake.org/files/v3.14/cmake-3.14.1-Linux-x86_64.sh /cmake-3.14.1-Linux-x86_64.sh
22+
RUN mkdir /opt/cmake
23+
RUN sh /cmake-3.14.1-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
24+
RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
25+
RUN cmake --version
26+
27+
WORKDIR /usr/local
28+
ENV GO_VERSION=1.12
29+
ENV GO111MODULE=on
30+
ENV GOPATH /go
31+
RUN wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \
32+
tar -xzf go${GO_VERSION}.linux-amd64.tar.gz && rm go${GO_VERSION}.linux-amd64.tar.gz && mkdir -p ${GOPATH}
33+
34+
WORKDIR /go/src/agones.dev/agones
35+
36+
ENV PATH /usr/local/go/bin:/go/bin:$PATH
37+
38+
# code generation scripts
39+
COPY *.sh /root/
40+
RUN chmod +x /root/*.sh

‎build/build-sdk-images/cpp/build.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2019 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -ex
18+
19+
cd ./sdks/cpp
20+
make build install archive VERSION=$VERSION

‎build/build-sdk-images/cpp/gen.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2019 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -ex
18+
19+
header() {
20+
cat /go/src/agones.dev/agones/build/boilerplate.go.txt ./$1 >> $2/$1
21+
}
22+
23+
googleapis=/go/src/agones.dev/agones/vendor/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis
24+
protoc_intermediate=/go/src/agones.dev/agones/sdks/cpp/.generated
25+
protoc_destination=/go/src/agones.dev/agones/sdks/cpp
26+
27+
mkdir -p ${protoc_intermediate}
28+
mkdir -p ${protoc_destination}/src/agones
29+
mkdir -p ${protoc_destination}/src/google
30+
mkdir -p ${protoc_destination}/include/agones
31+
mkdir -p ${protoc_destination}/include/google/api
32+
33+
cd /go/src/agones.dev/agones/sdks/cpp
34+
find -name '*.pb.*' -delete
35+
cd /go/src/agones.dev/agones
36+
protoc -I ${googleapis} -I . --grpc_out=${protoc_intermediate} --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` sdk.proto
37+
protoc -I ${googleapis} -I . --cpp_out=dllexport_decl=AGONES_EXPORT:${protoc_intermediate} sdk.proto ${googleapis}/google/api/annotations.proto ${googleapis}/google/api/http.proto
38+
39+
cd ${protoc_intermediate}
40+
header sdk.grpc.pb.cc ${protoc_destination}/src/agones
41+
header sdk.pb.cc ${protoc_destination}/src/agones
42+
header sdk.grpc.pb.h ${protoc_destination}/include/agones
43+
header sdk.pb.h ${protoc_destination}/include/agones
44+
45+
cd ${protoc_intermediate}/google/api
46+
header annotations.pb.cc ${protoc_destination}/src/google
47+
header http.pb.cc ${protoc_destination}/src/google
48+
header annotations.pb.h ${protoc_destination}/include/google/api
49+
header http.pb.h ${protoc_destination}/include/google/api
50+
51+
rm -r ${protoc_intermediate}

‎build/build-sdk-images/go/Dockerfile‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Copyright 2019 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
ARG BASE_IMAGE=agones-build-sdk-base:latest
15+
FROM $BASE_IMAGE
16+
17+
RUN apt-get update && \
18+
apt-get install -y wget jq && \
19+
apt-get clean
20+
21+
# install go
22+
WORKDIR /usr/local
23+
ENV GO_VERSION=1.12
24+
ENV GO111MODULE=on
25+
ENV GOPATH /go
26+
RUN wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \
27+
tar -xzf go${GO_VERSION}.linux-amd64.tar.gz && rm go${GO_VERSION}.linux-amd64.tar.gz && mkdir -p ${GOPATH}
28+
29+
ENV PATH /usr/local/go/bin:/go/bin:$PATH
30+
31+
# install go-proto-gen 1.1
32+
RUN go get github.com/golang/protobuf/protoc-gen-go@v1.2.0 && \
33+
go get -u golang.org/x/tools/cmd/goimports
34+
35+
36+
# code generation scripts
37+
COPY *.sh /root/
38+
RUN chmod +x /root/*.sh

build/build-image/gen-grpc-go.sh renamed to build/build-sdk-images/go/gen.sh

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env bash
22

3-
# Copyright 2017 Google LLC All Rights Reserved.
3+
# Copyright 2019 Google Inc. All Rights Reserved.
44
#
55
# Licensed under the Apache License, Version 2.0 (the "License");
66
# you may not use this file except in compliance with the License.
@@ -14,19 +14,20 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
set -x
17+
set -ex
1818

1919
export GO111MODULE=on
2020

2121
mkdir -p /go/src/
2222
cp -r /go/src/agones.dev/agones/vendor/* /go/src/
2323

24+
cd /go/src/agones.dev/agones
2425
go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
2526
go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
2627

2728
googleapis=/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis
2829

29-
cd /go/src/agones.dev/agones
30+
3031
protoc -I ${googleapis} -I . sdk.proto --go_out=plugins=grpc:pkg/sdk
3132
protoc -I ${googleapis} -I . sdk.proto --grpc-gateway_out=logtostderr=true:pkg/sdk
3233
protoc -I ${googleapis} -I . sdk.proto --swagger_out=logtostderr=true:.

0 commit comments

Comments
 (0)