Skip to content

Commit 2f107b1

Browse files
T-DynamosT-Dynamos
authored
Add support for prebuilt wheels (kivy#3280)
Add support for prebuilt wheels
1 parent d15f056 commit 2f107b1

5 files changed

Lines changed: 219 additions & 23 deletions

File tree

‎pythonforandroid/build.py‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import re
1212
import shutil
1313
import subprocess
14+
import sys
1415

1516
import sh
1617

@@ -21,7 +22,7 @@
2122
from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64
2223
from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint, Out_Style, Out_Fore)
2324
from pythonforandroid.pythonpackage import get_package_name
24-
from pythonforandroid.recipe import CythonRecipe, Recipe
25+
from pythonforandroid.recipe import CythonRecipe, Recipe, PyProjectRecipe
2526
from pythonforandroid.recommendations import (
2627
check_ndk_version, check_target_api, check_ndk_api,
2728
RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
@@ -101,6 +102,14 @@ class Context:
101102

102103
java_build_tool = 'auto'
103104

105+
skip_prebuilt = False
106+
107+
extra_index_urls = []
108+
109+
use_prebuilt_version_for = []
110+
111+
save_wheel_dir = ''
112+
104113
@property
105114
def packages_path(self):
106115
'''Where packages are downloaded before being unpacked'''
@@ -503,6 +512,10 @@ def build_recipes(build_order, python_modules, ctx, project_dir,
503512
recipe.prepare_build_dir(arch.arch)
504513

505514
info_main('# Prebuilding recipes')
515+
# ensure we have `ctx.python_recipe` and `ctx.hostpython`
516+
Recipe.get_recipe("python3", ctx).prebuild_arch(arch)
517+
ctx.hostpython = Recipe.get_recipe("hostpython3", ctx).python_exe
518+
506519
# 2) prebuild packages
507520
for recipe in recipes:
508521
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
@@ -667,7 +680,17 @@ def is_wheel_platform_independent(whl_name):
667680
return all(tag.platform == "any" for tag in tags)
668681

669682

670-
def process_python_modules(ctx, modules):
683+
def is_wheel_compatible(whl_name, arch, ctx):
684+
name, version, build, tags = parse_wheel_filename(whl_name)
685+
supported_tags = PyProjectRecipe.get_wheel_platform_tags(arch.arch, ctx)
686+
supported_tags.append("any")
687+
result = all(tag.platform in supported_tags for tag in tags)
688+
if not result:
689+
warning(f"Incompatible module : {whl_name}")
690+
return result
691+
692+
693+
def process_python_modules(ctx, modules, arch):
671694
"""Use pip --dry-run to resolve dependencies and filter for pure-Python packages
672695
"""
673696
modules = list(modules)
@@ -702,6 +725,7 @@ def process_python_modules(ctx, modules):
702725

703726
# setup hostpython recipe
704727
env = environ.copy()
728+
host_recipe = None
705729
try:
706730
host_recipe = Recipe.get_recipe("hostpython3", ctx)
707731
_python_path = host_recipe.get_path_to_python()
@@ -710,14 +734,32 @@ def process_python_modules(ctx, modules):
710734
_python_path, "Modules") + ":" + (libdir[0] if libdir else "")
711735
pip = host_recipe.pip
712736
except Exception:
713-
# hostpython3 non available so we use system pip (like in tests)
737+
# hostpython3 is unavailable, so fall back to system pip
714738
pip = sh.Command("pip")
715739

740+
# add platform tags
741+
platforms = []
742+
tags = PyProjectRecipe.get_wheel_platform_tags(arch.arch, ctx)
743+
for tag in tags:
744+
platforms.append(f"--platform={tag}")
745+
746+
if host_recipe is not None:
747+
platforms.extend(["--python-version", host_recipe.version])
748+
else:
749+
# use the version of the currently running Python interpreter
750+
current_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
751+
platforms.extend(["--python-version", current_version])
752+
753+
indices = []
754+
# add extra index urls
755+
for index in ctx.extra_index_urls:
756+
indices.extend(["--extra-index-url", index])
716757
try:
717758
shprint(
718759
pip, 'install', *modules,
719760
'--dry-run', '--break-system-packages', '--ignore-installed',
720-
'--report', path, '-q', _env=env
761+
'--disable-pip-version-check', '--only-binary=:all:',
762+
'--report', path, '-q', *platforms, *indices, _env=env
721763
)
722764
except Exception as e:
723765
warning(f"Auto module resolution failed: {e}")
@@ -751,7 +793,9 @@ def process_python_modules(ctx, modules):
751793
filename = basename(module["download_info"]["url"])
752794
pure_python = True
753795

754-
if (filename.endswith(".whl") and not is_wheel_platform_independent(filename)):
796+
if (
797+
filename.endswith(".whl") and not is_wheel_compatible(filename, arch, ctx)
798+
):
755799
any_not_pure_python = True
756800
pure_python = False
757801

@@ -769,7 +813,7 @@ def process_python_modules(ctx, modules):
769813
)
770814

771815
if pure_python:
772-
processed_modules.append(f"{mname}=={mver}")
816+
processed_modules.append(module["download_info"]["url"])
773817
info(" ")
774818

775819
if any_not_pure_python:
@@ -793,7 +837,7 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None,
793837

794838
info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch))
795839

796-
modules = process_python_modules(ctx, modules)
840+
modules = process_python_modules(ctx, modules, arch)
797841

798842
modules = [m for m in modules if ctx.not_has_package(m, arch)]
799843

‎pythonforandroid/recipe.py‎

Lines changed: 89 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -923,8 +923,7 @@ def real_hostpython_location(self):
923923
if host_name == 'hostpython3':
924924
return self._host_recipe.python_exe
925925
else:
926-
python_recipe = self.ctx.python_recipe
927-
return 'python{}'.format(python_recipe.version)
926+
return 'python{}'.format(self.ctx.python_recipe.version)
928927

929928
@property
930929
def hostpython_location(self):
@@ -1248,6 +1247,59 @@ class PyProjectRecipe(PythonRecipe):
12481247
extra_build_args = []
12491248
call_hostpython_via_targetpython = False
12501249

1250+
def get_pip_name(self):
1251+
name_str = self.name
1252+
if self.name not in self.ctx.use_prebuilt_version_for and self.version is not None:
1253+
# Like: v2.3.0 -> 2.3.0
1254+
cleaned_version = self.version.lstrip("v")
1255+
name_str += f"=={cleaned_version}"
1256+
return name_str
1257+
1258+
def get_pip_install_args(self, arch):
1259+
python_recipe = Recipe.get_recipe("python3", self.ctx)
1260+
opts = [
1261+
"install",
1262+
self.get_pip_name(),
1263+
"--ignore-installed",
1264+
"--disable-pip-version-check",
1265+
"--python-version",
1266+
python_recipe.version,
1267+
"--only-binary=:all:",
1268+
"--no-deps",
1269+
]
1270+
# add platform tags
1271+
tags = PyProjectRecipe.get_wheel_platform_tags(arch.arch, self.ctx)
1272+
for tag in tags:
1273+
opts.append(f"--platform={tag}")
1274+
1275+
# add extra index urls
1276+
for index in self.ctx.extra_index_urls:
1277+
opts.extend(["--extra-index-url", index])
1278+
1279+
return opts
1280+
1281+
def lookup_prebuilt(self, arch):
1282+
pip_options = self.get_pip_install_args(arch)
1283+
# do not install
1284+
pip_options.extend(["--dry-run", "-q"])
1285+
pip_env = self.get_hostrecipe_env()
1286+
try:
1287+
shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
1288+
except Exception:
1289+
return False
1290+
return True
1291+
1292+
def check_prebuilt(self, arch, msg=""):
1293+
if self.ctx.skip_prebuilt:
1294+
return False
1295+
1296+
if self.lookup_prebuilt(arch):
1297+
if msg != "":
1298+
info(f"Prebuilt pip wheel found, {msg}")
1299+
return True
1300+
1301+
return False
1302+
12511303
def get_recipe_env(self, arch, **kwargs):
12521304
# Custom hostpython
12531305
self.ctx.python_recipe.python_exe = join(
@@ -1259,24 +1311,42 @@ def get_recipe_env(self, arch, **kwargs):
12591311

12601312
with open(build_opts, "w") as file:
12611313
file.write("[bdist_wheel]\nplat_name={}".format(
1262-
self.get_wheel_platform_tag(arch)
1314+
self.get_wheel_platform_tag(arch.arch)
12631315
))
12641316
file.close()
12651317

12661318
env["DIST_EXTRA_CONFIG"] = build_opts
12671319
return env
12681320

1269-
def get_wheel_platform_tag(self, arch):
1321+
@staticmethod
1322+
def get_wheel_platform_tags(arch, ctx):
12701323
# https://peps.python.org/pep-0738/#packaging
12711324
# official python only supports 64 bit:
12721325
# android_21_arm64_v8a
12731326
# android_21_x86_64
1274-
return f"android_{self.ctx.ndk_api}_" + {
1275-
"arm64-v8a": "arm64_v8a",
1276-
"x86_64": "x86_64",
1277-
"armeabi-v7a": "arm",
1278-
"x86": "i686",
1279-
}[arch.arch]
1327+
_suffix = {
1328+
"arm64-v8a": ["arm64_v8a", "aarch64"],
1329+
"x86_64": ["x86_64"],
1330+
"armeabi-v7a": ["arm"],
1331+
"x86": ["i686"],
1332+
}[arch]
1333+
return [f"android_{ctx.ndk_api}_" + _ for _ in _suffix]
1334+
1335+
def get_wheel_platform_tag(self, arch):
1336+
return PyProjectRecipe.get_wheel_platform_tags(arch, self.ctx)[0]
1337+
1338+
def install_prebuilt_wheel(self, arch):
1339+
info("Installing prebuilt wheel")
1340+
destination = self.ctx.get_python_install_dir(arch.arch)
1341+
pip_options = self.get_pip_install_args(arch)
1342+
pip_options.extend(["--target", destination])
1343+
pip_options.append("--upgrade")
1344+
pip_env = self.get_hostrecipe_env()
1345+
try:
1346+
shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
1347+
except Exception:
1348+
return False
1349+
return True
12801350

12811351
def install_wheel(self, arch, built_wheels):
12821352
with patch_wheel_setuptools_logging():
@@ -1287,16 +1357,18 @@ def install_wheel(self, arch, built_wheels):
12871357
# Fix wheel platform tag
12881358
wheel_tag = wheel_tags(
12891359
_wheel,
1290-
platform_tags=self.get_wheel_platform_tag(arch),
1360+
platform_tags=self.get_wheel_platform_tag(arch.arch),
12911361
remove=True,
12921362
)
12931363
selected_wheel = join(built_wheel_dir, wheel_tag)
1294-
12951364
_dev_wheel_dir = environ.get("P4A_WHEEL_DIR", False)
12961365
if _dev_wheel_dir:
12971366
ensure_dir(_dev_wheel_dir)
12981367
shprint(sh.cp, selected_wheel, _dev_wheel_dir)
12991368

1369+
if exists(self.ctx.save_wheel_dir):
1370+
shprint(sh.cp, selected_wheel, self.ctx.save_wheel_dir)
1371+
13001372
info(f"Installing built wheel: {wheel_tag}")
13011373
destination = self.ctx.get_python_install_dir(arch.arch)
13021374
with WheelFile(selected_wheel) as wf:
@@ -1305,6 +1377,11 @@ def install_wheel(self, arch, built_wheels):
13051377
wf.close()
13061378

13071379
def build_arch(self, arch):
1380+
if self.check_prebuilt(arch, "skipping build_arch"):
1381+
result = self.install_prebuilt_wheel(arch)
1382+
if result:
1383+
return
1384+
warning("Failed to install prebuilt wheel, falling back to build_arch")
13081385

13091386
build_dir = self.get_build_dir(arch.arch)
13101387
if not (isfile(join(build_dir, "pyproject.toml")) or isfile(join(build_dir, "setup.py"))):

‎pythonforandroid/recipes/hostpython3/__init__.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,15 @@ class HostPython3Recipe(Recipe):
4848

4949
patches = ["fix_ensurepip.patch"]
5050

51+
# apply version guard
52+
def download(self):
53+
python_recipe = Recipe.get_recipe("python3", self.ctx)
54+
if python_recipe.version != self.version:
55+
raise BuildInterruptingException(
56+
f"python3 should have same version as hostpython3, {python_recipe.version} != {self.version}"
57+
)
58+
super().download()
59+
5160
@property
5261
def _exe_name(self):
5362
'''

‎pythonforandroid/toolchain.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ class NoAbbrevParser(argparse.ArgumentParser):
188188
This subclass alternative is follows the suggestion at
189189
https://bugs.python.org/issue14910.
190190
"""
191+
191192
def _get_option_tuples(self, option_string):
192193
return []
193194

@@ -267,6 +268,44 @@ def __init__(self):
267268
'--arch', help='The archs to build for.',
268269
action='append', default=[])
269270

271+
generic_parser.add_argument(
272+
'--extra-index-url',
273+
help=(
274+
'Extra package indexes to look for prebuilt Android wheels. '
275+
'Can be used multiple times.'
276+
),
277+
action='append',
278+
default=[],
279+
dest="extra_index_urls",
280+
)
281+
282+
generic_parser.add_argument(
283+
'--skip-prebuilt',
284+
help='Always build from source; do not use prebuilt wheels.',
285+
action='store_true',
286+
default=False,
287+
dest="skip_prebuilt",
288+
)
289+
290+
generic_parser.add_argument(
291+
'--use-prebuilt-version-for',
292+
help=(
293+
'For these packages, ignore pinned versions and use the latest '
294+
'prebuilt version from the extra index if available. '
295+
'Only applies to packages with a recipe.'
296+
),
297+
action='append',
298+
default=[],
299+
dest="use_prebuilt_version_for",
300+
)
301+
302+
generic_parser.add_argument(
303+
'--save-wheel-dir',
304+
dest='save_wheel_dir',
305+
default='',
306+
help='Directory to store wheels built by PyProjectRecipe.',
307+
)
308+
270309
# Options for specifying the Distribution
271310
generic_parser.add_argument(
272311
'--dist-name', '--dist_name',
@@ -672,6 +711,11 @@ def add_parser(subparsers, *args, **kwargs):
672711
self.ctx.activity_class_name = args.activity_class_name
673712
self.ctx.service_class_name = args.service_class_name
674713

714+
self.ctx.extra_index_urls = args.extra_index_urls
715+
self.ctx.skip_prebuilt = args.skip_prebuilt
716+
self.ctx.use_prebuilt_version_for = args.use_prebuilt_version_for
717+
self.ctx.save_wheel_dir = args.save_wheel_dir
718+
675719
# Each subparser corresponds to a method
676720
command = args.subparser_name.replace('-', '_')
677721
getattr(self, command)(args)

0 commit comments

Comments
 (0)