Skip to content

Commit 7d72391

Browse files
heitorgo1heitorgo1
committed
Move grpcio content to grpc
1 parent 17ccd49 commit 7d72391

10 files changed

Lines changed: 120 additions & 1936 deletions

File tree

‎pythonforandroid/recipes/grpc/Dockerfile‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -292,9 +292,30 @@ ENV HOME_DIR="/home/${USER}"
292292
ENV WORK_DIR="${HOME_DIR}" \
293293
PATH="${HOME_DIR}/.local/bin:${PATH}"
294294

295+
# Install python 3.7 dependencies
296+
RUN ${RETRY} apt -y install -qq --no-install-recommends \
297+
build-essential zlib1g-dev \
298+
libncurses5-dev libgdbm-dev \
299+
libnss3-dev libssl-dev \
300+
libreadline-dev libffi-dev wget \
301+
&& apt -y autoremove
302+
303+
# Install python 3.7
304+
RUN wget https://www.python.org/ftp/python/3.7.3/Python-3.7.3.tar.xz \
305+
&& tar -xf Python-3.7.3.tar.xz && cd Python-3.7.3 \
306+
&& ./configure --enable-optimizations \
307+
&& make -j "${N_JOBS}" \
308+
&& make altinstall
309+
310+
RUN python3.7 --version
311+
RUN pip --version
312+
313+
# Install virtualenv
314+
RUN pip install --upgrade virtualenv
315+
316+
295317
# install system dependencies
296318
RUN ${RETRY} apt -y install -qq --no-install-recommends \
297-
python3 virtualenv python3-pip python3-venv \
298319
wget lbzip2 patch sudo \
299320
&& apt -y autoremove
300321

@@ -303,7 +324,7 @@ RUN ${RETRY} apt -y install -qq --no-install-recommends \
303324
RUN dpkg --add-architecture i386 \
304325
&& ${RETRY} apt -y update -qq \
305326
&& ${RETRY} apt -y install -qq --no-install-recommends \
306-
build-essential ccache git python3 python3-dev \
327+
build-essential ccache git \
307328
libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
308329
libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \
309330
zip zlib1g-dev zlib1g:i386 \
@@ -323,7 +344,8 @@ RUN usermod -append --groups sudo ${USER}
323344
RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
324345

325346

326-
RUN pip3 install --upgrade cython==0.29.7
347+
RUN pip install --upgrade --ignore-installed setuptools
348+
RUN pip install --upgrade cython==0.28.3
327349

328350
WORKDIR ${WORK_DIR}
329351
COPY --chown=user:user . ${WORK_DIR}
@@ -333,5 +355,4 @@ USER ${USER}
333355
# install python-for-android from current branch
334356
RUN virtualenv --python=python3 venv \
335357
&& . venv/bin/activate \
336-
&& pip3 install -e .
337-
358+
&& pip install -e .

‎pythonforandroid/recipes/grpc/README.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,5 @@ docker build --tag p4a .
2929

3030
``` bash
3131
cd /home/user/python_project
32-
docker run -it --rm -v $(pwd):/home/user/project p4a /bin/sh -c '. venv/bin/activate && cd /home/user/project && p4a apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/android-ndk --android_api 27 --private . --package=com.android.project --name "TEST P4A Grpc" --version 0.1.0 --bootstrap=sdl2 --requirements=hostpython3,python3,googleapis-common-protos,hbmqtt,pyyaml,pyjwt,pytz,librt,grpc,grpcio --permission INTERNET'
32+
docker run -it --rm -v $(pwd):/home/user/project p4a /bin/sh -c '. venv/bin/activate && cd /home/user/project && p4a apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/android-ndk --android_api 27 --private . --package=com.android.project --name "TEST P4A Grpc" --version 0.1.0 --bootstrap=sdl2 --requirements=hostpython3,python3,googleapis-common-protos,hbmqtt,pyyaml,pyjwt,pytz,librt,grpcio --permission INTERNET'
3333
```
34-

‎pythonforandroid/recipes/grpc/__init__.py‎

Lines changed: 93 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -1,145 +1,118 @@
11
from os.path import join, isdir, isfile
22
import sh
3-
from pythonforandroid.recipe import NDKRecipe
4-
from pythonforandroid.toolchain import (
5-
current_directory,
6-
shprint,
7-
)
3+
from pythonforandroid.recipe import NDKRecipe, CythonRecipe, Recipe
4+
from pythonforandroid.toolchain import current_directory, shprint
85
from pythonforandroid.logger import info
96
from multiprocessing import cpu_count
7+
import glob
108

119

12-
class GRPCRecipe(NDKRecipe):
13-
version = 'v1.20.1'
14-
#url = 'https://github.com/grpc/grpc/archive/{version}.zip'
10+
class GRPCRecipe(CythonRecipe):
11+
name = "grpc"
12+
version = "v1.20.1"
13+
# url = 'https://github.com/grpc/grpc/archive/{version}.zip'
1514
url = None
16-
port_git = 'https://github.com/grpc/grpc.git'
17-
generated_libraries = [
18-
'libgrpc++_cronet.so',
19-
'libgrpc_csharp_ext.so',
20-
'libgrpc++_unsecure.so',
21-
'libgrpc_plugin_support.so',
22-
'libgrpc++.so',
23-
'libgrpc_cronet.so',
24-
'libgrpc_unsecure.so',
25-
'libgrpc.so',
26-
'libaddress_sorting.a',
27-
'libbenchmark.a',
28-
'libbenchmark_main.a',
29-
'libcares.so',
30-
'libcrypto.a',
31-
'libgflags.a',
32-
'libgflags_nothreads.a',
33-
'libglog.a',
34-
'libgpr.a',
35-
'libgrpc.a',
36-
'libgrpc++.a',
37-
'libgrpc_cronet.a',
38-
'libgrpc++_cronet.a',
39-
'libgrpc_csharp_ext.so',
40-
'libgrpc_plugin_support.a',
41-
'libgrpc_unsecure.a',
42-
'libgrpc++_unsecure.a',
43-
'libprotobuf.a',
44-
'libprotobuf-lite.a',
45-
'libprotoc.a',
46-
'libssl.a',
47-
'libz.a',
48-
'libz.so'
49-
]
50-
51-
def get_lib_dir(self, arch):
52-
return join(self.get_build_dir(arch.arch), 'build', 'lib', arch.arch)
15+
port_git = "https://github.com/grpc/grpc.git"
16+
depends = ["setuptools"]
17+
site_packages_name = "grpcio"
18+
cython_args = ["src/python/grpcio/grpc/_cython"]
19+
# patches = ["fix_cares.patch"]
5320

5421
def get_recipe_env(self, arch):
5522
env = super(GRPCRecipe, self).get_recipe_env(arch)
56-
env['ANDROID_NDK'] = self.ctx.ndk_dir
57-
env['ANDROID_SDK'] = self.ctx.sdk_dir
23+
build_dir = self.get_build_dir(arch.arch)
24+
third_party_dir = join(build_dir, "third_party")
25+
boringssl_dir = join(third_party_dir, "boringssl")
26+
env["CC"] = "arm-linux-androideabi-gcc "
27+
# -fno-rtti avoids ImportError: dlopen failed: cannot locate symbol "_ZTVN10__cxxabiv117__class_type_infoE" referenced by "/data/data/com.admobilize.admp/files/app/_python_bundle/site-packages/grpc/_cython/cygrpc.so"
28+
# This overwrites the CFLAGS used by grpc
29+
env[
30+
"GRPC_PYTHON_CFLAGS"
31+
] = " -I{} -I/opt/android/android-ndk/sources/cxx-stl/gnu-libstdc++/4.9/include -I/opt/android/android-ndk/sources/cxx-stl/gnu-libstdc++/4.9/libs/armeabi-v7a/include -std=c++11 -std=c99 -fvisibility=hidden -fno-wrapv -fno-exceptions -fpermissive -fno-rtti -w ".format(
32+
join(boringssl_dir, "include")
33+
)
34+
# -llog avoids ImportError: dlopen failed: cannot locate symbol "__android_log_print" referenced by "/data/data/com.admobilize.admp/files/app/_python_bundle/site-packages/grpc/_cython/cygrpc.so"
35+
# This overwrites the LDFLAGS used by grpc
36+
env["GRPC_PYTHON_LDFLAGS"] = " -Wl,-wrap,memcpy -static-libgcc -llog -ldl -lm"
5837
return env
5938

6039
def prebuild_arch(self, arch):
6140
build_dir = self.get_build_dir(arch.arch)
62-
source_dir = join(build_dir, 'grpc')
63-
if not isfile(join(source_dir, 'setup.py')):
41+
source_dir = join(build_dir, "grpc")
42+
if not isfile(join(build_dir, "setup.py")) and not isfile(
43+
join(source_dir, "setup.py")
44+
):
6445
info("clone GRPC sources from {}".format(self.port_git))
65-
shprint(sh.git, 'clone', '--recursive', self.port_git, source_dir, _tail=20, _critical=True)
46+
shprint(
47+
sh.git,
48+
"clone",
49+
"--branch",
50+
self.version,
51+
"--single-branch",
52+
"--recursive",
53+
self.port_git,
54+
source_dir,
55+
_tail=20,
56+
_critical=True,
57+
)
58+
59+
if isfile(join(source_dir, "setup.py")):
60+
shprint(sh.mv, *glob.glob(join(source_dir, "*")), build_dir)
61+
shprint(sh.rm, "-rf", source_dir)
62+
# TODO Fix this hardcoded path
63+
self.apply_patch(
64+
"/home/user/admp/p4a-recipes/grpcio/fix_cares.patch", arch.arch
65+
)
6666

6767
def build_arch(self, arch):
68-
build_dir = self.get_build_dir(arch.arch)
69-
source_dir = join(build_dir, 'grpc')
70-
build_dir = join(source_dir, 'build')
71-
shprint(sh.rm, '-rf', build_dir)
72-
shprint(sh.mkdir, '-p', build_dir)
73-
with current_directory(build_dir):
74-
env = self.get_recipe_env(arch)
68+
Recipe.build_arch(self, arch)
7569

76-
python_major = self.ctx.python_recipe.version[0]
77-
python_include_root = self.ctx.python_recipe.include_root(arch.arch)
78-
python_site_packages = self.ctx.get_site_packages_dir()
79-
python_link_root = self.ctx.python_recipe.link_root(arch.arch)
80-
python_link_version = self.ctx.python_recipe.major_minor_version_string
81-
if 'python3' in self.ctx.python_recipe.name:
82-
python_link_version += 'm'
83-
python_library = join(python_link_root,
84-
'libpython{}.so'.format(python_link_version))
85-
python_include_numpy = join(python_site_packages,
86-
'numpy', 'core', 'include')
70+
self.build_cython_components(arch)
71+
self.install_python_package(arch) # this is the same as in a PythonRecipe
8772

88-
shprint(sh.cmake,
89-
'-DP4A=ON',
90-
'-DANDROID_ABI={}'.format(arch.arch),
91-
'-DANDROID_STANDALONE_TOOLCHAIN={}'.format(self.ctx.ndk_dir),
92-
'-DANDROID_NATIVE_API_LEVEL={}'.format(self.ctx.ndk_api),
93-
'-DANDROID_EXECUTABLE={}/tools/android'.format(env['ANDROID_SDK']),
94-
'-DCMAKE_TOOLCHAIN_FILE={}'.format(
95-
join(self.ctx.ndk_dir, 'build', 'cmake',
96-
'android.toolchain.cmake')),
97-
'-DBUILD_WITH_STANDALONE_TOOLCHAIN=ON',
98-
'-DBUILD_SHARED_LIBS=ON',
99-
'-DBUILD_STATIC_LIBS=OFF',
100-
'-Dprotobuf_BUILD_PROTOC_BINARIES=OFF',
101-
'-DCMAKE_INSTALL_PREFIX="/opt/install"',
102-
'-DCMAKE_FIND_ROOT_PATH="/opt/install"',
103-
'-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH',
104-
'-DCMAKE_SHARED_LINKER_FLAGS="-llog"',
105-
'-DCMAKE_EXE_LINKER_FLAGS="-llog"',
106-
'-DProtobuf_PROTOC_EXECUTABLE=/usr/local/bin/protoc',
107-
'-DProtobuf_LIBRARIES=/opt/install/lib/libprotobuf.a',
108-
'-DProtobuf_PROTOC_LIBRARY=/opt/install/lib/libprotoc.a',
109-
'-DProtobuf_INCLUDE_DIR=/opt/install/include',
110-
'-DHAVE_THREAD_SAFETY_ATTRIBUTES=ON',
111-
'-DHAVE_GNU_POSIX_REGEX=ON',
112-
'-DHAVE_STD_REGEX=ON',
113-
'-DRUN_HAVE_STD_REGEX=ON',
114-
'-DHAVE_POSIX_REGEX=1',
115-
'-DRUN_HAVE_POSIX_REGEX=ON',
116-
'-DHAVE_STEADY_CLOCK=ON',
117-
'-DgRPC_BUILD_TESTS=OFF',
118-
'-DCMAKE_CROSSCOMPILING=1',
119-
'-DOPENSSL_SSL_LIBRARY=/usr/local/lib/libssl.so',
120-
'-DOPENSSL_CRYPTO_LIBRARY=/usr/local/lib/libcrypto.so',
121-
'-DOPENSSL_INCLUDE_DIR=/usr/local/include',
122-
'-Dgflags_DIR=/opt/install/lib/cmake/gflags',
123-
'-DgRPC_PROTOBUF_PROVIDER=package',
124-
'-DgRPC_ZLIB_PROVIDER=package',
125-
'-DgRPC_CARES_PROVIDER=package',
126-
'-DgRPC_SSL_PROVIDER=package',
127-
'-DgRPC_GFLAGS_PROVIDER=package',
128-
'-DgRPC_BUILD_CODEGEN=OFF',
73+
def build_cython_components(self, arch):
74+
env = self.get_recipe_env(arch)
75+
build_dir = self.get_build_dir(arch.arch)
76+
with current_directory(build_dir):
77+
hostpython = sh.Command(self.ctx.hostpython)
12978

130-
source_dir,
131-
_env=env)
132-
shprint(sh.make, '-j' + str(cpu_count()))
133-
# Copy third party shared libs that we need in our final apk
134-
#sh.cp('-a', sh.glob('./lib/{}/lib*.a'.format(arch.arch)),
135-
# self.ctx.get_libs_dir(arch.arch))
79+
# We manually run cython from the user's system
80+
# note the --cplus flag, used to build cygrpc.cpp
81+
shprint(
82+
sh.find,
83+
join(build_dir, self.cython_args[0]),
84+
"-iname",
85+
"*.pyx",
86+
"-exec",
87+
self.ctx.cython,
88+
"--cplus",
89+
"{}",
90+
";",
91+
_env=env,
92+
)
13693

137-
# copy static libs to libs collection
138-
for lib in sh.glob(join(build_dir, '*.a')):
139-
shprint(sh.cp, '-L', lib, self.ctx.libs_dir)
94+
# Now that cython has been run, the build works
95+
try:
96+
shprint(hostpython, "setup.py", "build_ext", "-v", _env=env)
97+
except sh.ErrorReturnCode as e:
98+
for line in str(e.stdout).split("\\n"):
99+
print(line)
100+
raise
140101

141-
for lib in sh.glob(join(build_dir, '*.so')):
142-
shprint(sh.cp, '-L', lib, self.ctx.libs_dir)
102+
# stripping debug symbols lowers the file size a lot
103+
build_lib = glob.glob("./python_build/lib*")
104+
info("{}".format(build_lib))
105+
shprint(
106+
sh.find,
107+
"./python_build/",
108+
"-name",
109+
"*.o",
110+
"-exec",
111+
env["STRIP"],
112+
"{}",
113+
";",
114+
_env=env,
115+
)
143116

144117

145118
recipe = GRPCRecipe()

pythonforandroid/recipes/grpcio/fix_cares.patch renamed to pythonforandroid/recipes/grpc/fix_cares.patch

File renamed without changes.

0 commit comments

Comments
 (0)