Skip to content

Commit 21cefbe

Browse files
author
Etienne Laurin
committed
Python driver: do not build C++ extension by default
Review 1024 by @Tryneus Closes rethinkdb#1394
1 parent 4a6aef0 commit 21cefbe

7 files changed

Lines changed: 98 additions & 107 deletions

File tree

‎drivers/python/MANIFEST.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
recursive-include rethinkdb *.pb.h
1+
include ql2.proto

‎drivers/python/Makefile‎

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,24 @@
1-
PROTOC=protoc
21
RETHINKDB_HOME=../..
3-
PROTO_FILE_DIR=$(RETHINKDB_HOME)/src/rdb_protocol
4-
PROTO_BASE=ql2
5-
PROTO_FILE=$(PROTO_FILE_DIR)/$(PROTO_BASE).proto
2+
PROTO_FILE_SRC=$(RETHINKDB_HOME)/src/rdb_protocol/ql2.proto
63

7-
PYTHON_SRC=rethinkdb
8-
PBCPP_SRC=rethinkdb
9-
PYTHON_PB_FILE=$(PYTHON_SRC)/$(PROTO_BASE)_pb2.py
10-
CPP_PB_FILE=$(PBCPP_SRC)/$(PROTO_BASE).pb.cc
11-
PBCPP=rethinkdb_pbcpp.so
12-
PBCPP_BUILT=./build/lib.linux-x86_64-2.7/rethinkdb_pbcpp.so
13-
PYTHON_DOCS=$(PYTHON_SRC)/docs.py
4+
PYTHON_PB_FILE=rethinkdb/ql2_pb2.py
5+
PROTO_FILE=ql2.proto
6+
PYTHON_DOCS=rethinkdb/docs.py
147

15-
all: $(PYTHON_PB_FILE) $(PBCPP) $(PYTHON_DOCS)
8+
all: $(PYTHON_PB_FILE) $(PYTHON_DOCS) $(PROTO_FILE)
169

17-
$(PYTHON_DOCS): ../../docs/rql/py_docs.json
10+
$(PYTHON_DOCS): $(RETHINKDB_HOME)/docs/rql/py_docs.json
1811
python gendocs.py > $@
1912

2013
$(PYTHON_PB_FILE): $(PROTO_FILE)
21-
$(PROTOC) --python_out=$(PYTHON_SRC) -I$(PROTO_FILE_DIR) $(PROTO_FILE)
14+
protoc --python_out=rethinkdb $(PROTO_FILE)
2215

23-
$(CPP_PB_FILE): $(PROTO_FILE)
24-
$(PROTOC) --cpp_out=$(PBCPP_SRC) -I$(PROTO_FILE_DIR) $(PROTO_FILE)
25-
26-
$(PBCPP): $(PBCPP_BUILT)
27-
test ! -e $< || cp $< $@
28-
29-
$(PBCPP_BUILT): $(CPP_PB_FILE)
30-
python setup.py build
16+
$(PROTO_FILE): $(PROTO_FILE_SRC)
17+
cp $< $@
3118

3219
clean:
3320
rm -f $(PYTHON_PB_FILE)
34-
rm -f $(CPP_PB_FILE)
35-
rm -f $(PBCPP)
21+
rm -f $(PROTO_FILE)
3622
rm -f $(PYTHON_DOCS)
3723
rm -rf ./build
3824
rm -rf ./dist
@@ -41,18 +27,20 @@ clean:
4127

4228
PY_PKG_DIR=$(RETHINKDB_HOME)/build/packages/python
4329

44-
sdist: $(PYTHON_PB_FILE) $(CPP_PB_FILE) $(PYTHON_DOCS)
30+
sdist: $(PYTHON_PB_FILE) $(PYTHON_DOCS) $(PROTO_FILE)
31+
rm -rf $(PY_PKG_DIR)
4532
mkdir -p $(PY_PKG_DIR)
4633
cp setup.py $(PY_PKG_DIR)
4734
cp MANIFEST.in $(PY_PKG_DIR)
4835
cp -r rethinkdb $(PY_PKG_DIR)
4936
cp $(PYTHON_PB_FILE) $(PY_PKG_DIR)/rethinkdb
37+
cp $(PROTO_FILE) $(PY_PKG_DIR)/$(PROTO_FILE)
5038
cd $(PY_PKG_DIR) && python setup.py sdist
5139

5240
publish: sdist
5341
cd $(PY_PKG_DIR) && python setup.py register upload
5442

5543
install: sdist
56-
cd $(PY_PKG_DIR) && python setup.py install
44+
cd $(PY_PKG_DIR) && PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp python setup.py install
5745

5846
.PHONY: all clean publish sdist install

‎drivers/python/rethinkdb/_pbcpp.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// This is an empty python module. It gets linked to ql2.pb.o, whose functions get
2+
// exposed and used by the C++ implementation of the google.protobuf package.
3+
4+
#include <python2.7/Python.h>
5+
6+
static PyMethodDef PbMethods[] = {
7+
{NULL, NULL, 0, NULL}
8+
};
9+
10+
PyMODINIT_FUNC init_pbcpp() {
11+
PyObject *m;
12+
m = Py_InitModule("rethinkdb._pbcpp", PbMethods);
13+
if (m == NULL)
14+
return;
15+
}

‎drivers/python/rethinkdb/net.py‎

Lines changed: 12 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,23 @@
11
# Copyright 2010-2012 RethinkDB, all rights reserved.
22

3-
__all__ = ['connect', 'Connection', 'Cursor','protobuf_implementation']
3+
__all__ = ['connect', 'Connection', 'Cursor', 'protobuf_implementation']
44

55
import errno
66
import socket
77
import struct
88
from os import environ
99

10-
if 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION' in environ:
11-
protobuf_implementation = environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION']
12-
if environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] == 'cpp':
13-
import rethinkdb_pbcpp
14-
else:
15-
try:
16-
# Set an environment variable telling the protobuf library
17-
# to use the fast C++ based serializer implementation
18-
# over the pure python one if it is available.
19-
environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
20-
21-
# The cpp_message module could change between versions of the
22-
# protobuf module
23-
from google.protobuf.internal import cpp_message
24-
import rethinkdb_pbcpp
25-
protobuf_implementation = 'cpp'
26-
except ImportError as e:
27-
# Default to using the python implementation of protobuf
28-
environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'
29-
protobuf_implementation = 'python'
30-
31-
from . import ql2_pb2 as p
32-
33-
from . import repl # For the repl connection
34-
from .errors import *
35-
from .ast import Datum, DB, expr
10+
try:
11+
import rethinkdb.pbcpp
12+
protobuf_implementation = 'cpp'
13+
except ImportError:
14+
protobuf_implementation = 'python'
15+
16+
from rethinkdb import ql2_pb2 as p
17+
18+
from rethinkdb import repl # For the repl connection
19+
from rethinkdb.errors import *
20+
from rethinkdb.ast import Datum, DB, expr
3621

3722
class Cursor(object):
3823
def __init__(self, conn, query, term, opts):

‎drivers/python/rethinkdb/pbcpp.cpp‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎drivers/python/rethinkdb/pbcpp.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Load the C++ protobuf backend
2+
3+
from os import environ
4+
5+
# This rethinkdb-specific library is required to fully benefit from
6+
# the C++ backend
7+
import rethinkdb._pbcpp
8+
9+
# The google.protobuf package will activate the C++ backend only if this
10+
# variable is set to 'cpp'
11+
environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
12+
13+
try:
14+
import google.protobuf
15+
16+
# In protobuf 2.4.* and 2.5.0, this module imports correctly only if
17+
# the C++ backend is active in the google.protobuf package
18+
from google.protobuf.internal import cpp_message
19+
20+
except ImportError:
21+
del environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION']
22+
raise

‎drivers/python/setup.py‎

Lines changed: 34 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,44 @@
1-
# Copyright 2010-2012 RethinkDB, all rights reserved.
1+
# Copyright 2010-2013 RethinkDB, all rights reserved.
22

33
from setuptools import setup, Extension
44
from distutils.command.build_ext import build_ext
5-
from distutils.errors import DistutilsPlatformError, CCompilerError, DistutilsExecError
6-
import sys
5+
import os
6+
from subprocess import check_call
77

8-
class build_ext_nofail(build_ext):
9-
# This class can replace the build_ext command with one that does not fail
10-
# when the extension fails to build.
11-
12-
def run(self):
13-
try:
14-
build_ext.run(self)
15-
except DistutilsPlatformError as e:
16-
self._failed(e)
8+
class build_ext_genproto(build_ext):
9+
# This class replaces the build_ext command with one that
10+
# first generates the ql2.pb.{cpp,h} files if the correct
11+
# environment variable is set.
1712

1813
def build_extension(self, ext):
19-
try:
14+
if os.environ.get('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION') == 'cpp':
15+
print "Calling protoc to generate ql2.pb.cc and ql2.pb.h"
16+
check_call(['protoc', 'ql2.proto', '--cpp_out=.'])
17+
ext.sources = ['./ql2.pb.cc'] + ext.sources
2018
build_ext.build_extension(self, ext)
21-
except (CCompilerError, DistutilsExecError) as e:
22-
self._failed(e)
2319
else:
24-
try:
25-
import google.protobuf.internal.cpp_message
26-
except ImportError:
27-
sys.stderr.write("*** WARNING: The installed protobuf library does not seem to include the C++ extension\n")
28-
sys.stderr.write("*** WARNING: The RethinkDB driver will fallback to using the pure python implementation\n")
29-
30-
def _failed(self, e):
31-
sys.stderr.write("*** WARNING: Unable to compile the C++ extension\n")
32-
sys.stderr.write(str(e) + "\n")
33-
sys.stderr.write("*** WARNING: Defaulting to the python implementation\n")
20+
print "* * * * * * * * * * * * * * * * *"
21+
print "* WARNING: The faster C++ protobuf backend is not enabled."
22+
print "* WARNING: To enable it, run `export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp' and reinstall the rethinkdb package."
23+
print "* WARNING: See http://rethinkdb.com/docs/driver-performance/ for more information."
24+
print "* * * * * * * * * * * * * * * * *"
3425

3526
setup(name="rethinkdb"
36-
,version="1.10.0-0"
37-
,description="This package provides the Python driver library for the RethinkDB database server."
38-
,url="http://rethinkdb.com"
39-
,maintainer="RethinkDB Inc."
40-
,maintainer_email="bugs@rethinkdb.com"
41-
,packages=['rethinkdb']
42-
,install_requires=['protobuf']
43-
,ext_modules=[Extension('rethinkdb_pbcpp', sources=['./rethinkdb/pbcpp.cpp', './rethinkdb/ql2.pb.cc'],
44-
include_dirs=['./rethinkdb'], libraries=['protobuf'])]
45-
,cmdclass={"build_ext":build_ext_nofail}
46-
,entry_points={'console_scripts': [
47-
'rethinkdb-import = rethinkdb._import:main',
48-
'rethinkdb-dump = rethinkdb._dump:main',
49-
'rethinkdb-export = rethinkdb._export:main',
50-
'rethinkdb-restore = rethinkdb._restore:main']}
51-
)
27+
,version = "1.10.0-999"
28+
,description = "This package provides the Python driver library for the RethinkDB database server."
29+
,url = "http://rethinkdb.com"
30+
,maintainer = "RethinkDB Inc."
31+
,maintainer_email = "bugs@rethinkdb.com"
32+
,packages = ['rethinkdb']
33+
,install_requires = ['protobuf']
34+
,entry_points = {'console_scripts': [
35+
'rethinkdb-import = rethinkdb._import:main',
36+
'rethinkdb-dump = rethinkdb._dump:main',
37+
'rethinkdb-export = rethinkdb._export:main',
38+
'rethinkdb-restore = rethinkdb._restore:main']}
39+
,cmdclass = {"build_ext":build_ext_genproto}
40+
,ext_modules = [Extension(
41+
'rethinkdb/_pbcpp',
42+
sources=['./rethinkdb/_pbcpp.cpp'],
43+
include_dirs=['./'],
44+
libraries=['protobuf'])])

0 commit comments

Comments
 (0)