Skip to content

Commit ea461ff

Browse files
opacamjoergbrech
opacam
authored and
joergbrech
committed
[LIBS - PART II] Part II of NDK r19 migration - Initial STL lib migration (kivy#1947)
* [recipe-stl] Add android's STL lib support to `Recipe` To allow us to refactor some common operations that we use in our recipes that depends on android's STL library. Note: This commit will allow us to begin the migration to `c++_shared`. This is a must when we move to android's NDK r19+, because as for android NDK >= 18 is the only one supported STL library. * [recipe-stl] Make CppCompiledComponentsPythonRecipe use Recipe's STL support * [recipe-stl] Make icu a library recipe with STL support (rework) Also done here:   - Remove hardcoded version in url   - Disable versioned shared libraries   - Make it to be build as a shared libraries (instead of static)   - Disable the build of static libraries (because we build them as shared ones, so we avoid to link with them without our consents)   - Shorten long lines to be pep8's friendly   - Remove icu from ci/constants   - Remove `RuntimeError` about the need to use NDK api <= 19 (because that is not the case anymore)   - consider host's number of cpu to perform the build * [recipe-stl] Rework pyicu recipe to match latest icu changes Also done here:   - Remove icu.patch because now we don't have the version in our icu libraries   - Shorten long lines to be pep8's friendly * [tests] Add tests for recipe with STL support * [tests] Add tests for icu recipe * [tests] Add tests for pyicu recipe
1 parent a1310fc commit ea461ff

File tree

9 files changed

+563
-168
lines changed

9 files changed

+563
-168
lines changed

‎ci/constants.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,6 @@ class TargetPython(Enum):
6565
# IndexError: list index out of range
6666
'secp256k1',
6767
'ffpyplayer',
68-
'icu',
6968
# requires `libpq-dev` system dependency e.g. for `pg_config` binary
7069
'psycopg2',
7170
'protobuf_cpp',

‎pythonforandroid/recipe.py

Lines changed: 60 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,46 @@ class Recipe(with_metaclass(RecipeMeta)):
125125
path: `'.', None or ''`
126126
"""
127127

128+
need_stl_shared = False
129+
'''Some libraries or python packages may need to be linked with android's
130+
stl. We can automatically do this for any recipe if we set this property to
131+
`True`'''
132+
133+
stl_lib_name = 'c++_shared'
134+
'''
135+
The default STL shared lib to use: `c++_shared`.
136+
137+
.. note:: Android NDK version > 17 only supports 'c++_shared', because
138+
starting from NDK r18 the `gnustl_shared` lib has been deprecated.
139+
'''
140+
141+
stl_lib_source = '{ctx.ndk_dir}/sources/cxx-stl/llvm-libc++'
142+
'''
143+
The source directory of the selected stl lib, defined in property
144+
`stl_lib_name`
145+
'''
146+
147+
@property
148+
def stl_include_dir(self):
149+
return join(self.stl_lib_source.format(ctx=self.ctx), 'include')
150+
151+
def get_stl_lib_dir(self, arch):
152+
return join(
153+
self.stl_lib_source.format(ctx=self.ctx), 'libs', arch.arch
154+
)
155+
156+
def get_stl_library(self, arch):
157+
return join(
158+
self.get_stl_lib_dir(arch),
159+
'lib{name}.so'.format(name=self.stl_lib_name),
160+
)
161+
162+
def install_stl_lib(self, arch):
163+
if not self.ctx.has_lib(
164+
arch.arch, 'lib{name}.so'.format(name=self.stl_lib_name)
165+
):
166+
self.install_libs(arch, self.get_stl_library(arch))
167+
128168
@property
129169
def version(self):
130170
key = 'VERSION_' + self.name
@@ -454,7 +494,22 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
454494
"""
455495
if arch is None:
456496
arch = self.filtered_archs[0]
457-
return arch.get_env(with_flags_in_cc=with_flags_in_cc)
497+
env = arch.get_env(with_flags_in_cc=with_flags_in_cc)
498+
499+
if self.need_stl_shared:
500+
env['CPPFLAGS'] = env.get('CPPFLAGS', '')
501+
env['CPPFLAGS'] += ' -I{}'.format(self.stl_include_dir)
502+
503+
env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions'
504+
505+
if with_flags_in_cc:
506+
env['CXX'] += ' -frtti -fexceptions'
507+
508+
env['LDFLAGS'] += ' -L{}'.format(self.get_stl_lib_dir(arch))
509+
env['LIBS'] = env.get('LIBS', '') + " -l{}".format(
510+
self.stl_lib_name
511+
)
512+
return env
458513

459514
def prebuild_arch(self, arch):
460515
'''Run any pre-build tasks for the Recipe. By default, this checks if
@@ -538,6 +593,9 @@ def postbuild_arch(self, arch):
538593
if hasattr(self, postbuild):
539594
getattr(self, postbuild)()
540595

596+
if self.need_stl_shared:
597+
self.install_stl_lib(arch)
598+
541599
def prepare_build_dir(self, arch):
542600
'''Copies the recipe data into a build dir for the given arch. By
543601
default, this unpacks a downloaded recipe. You should override
@@ -982,35 +1040,7 @@ def rebuild_compiled_components(self, arch, env):
9821040
class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe):
9831041
""" Extensions that require the cxx-stl """
9841042
call_hostpython_via_targetpython = False
985-
986-
def get_recipe_env(self, arch):
987-
env = super(CppCompiledComponentsPythonRecipe, self).get_recipe_env(arch)
988-
keys = dict(
989-
ctx=self.ctx,
990-
arch=arch,
991-
arch_noeabi=arch.arch.replace('eabi', '')
992-
)
993-
env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions'
994-
env['CFLAGS'] += (
995-
" -I{ctx.ndk_dir}/platforms/android-{ctx.android_api}/arch-{arch_noeabi}/usr/include" +
996-
" -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/include" +
997-
" -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/include").format(**keys)
998-
env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions'
999-
env['LDFLAGS'] += (
1000-
" -L{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}" +
1001-
" -lgnustl_shared").format(**keys)
1002-
1003-
return env
1004-
1005-
def build_compiled_components(self, arch):
1006-
super(CppCompiledComponentsPythonRecipe, self).build_compiled_components(arch)
1007-
1008-
# Copy libgnustl_shared.so
1009-
with current_directory(self.get_build_dir(arch.arch)):
1010-
sh.cp(
1011-
"{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx, arch=arch),
1012-
self.ctx.get_libs_dir(arch.arch)
1013-
)
1043+
need_stl_shared = True
10141044

10151045

10161046
class CythonRecipe(PythonRecipe):

‎pythonforandroid/recipes/icu/__init__.py

Lines changed: 47 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,54 @@
11
import sh
22
import os
3-
from os.path import join, isdir
4-
from pythonforandroid.recipe import NDKRecipe
3+
from os.path import join, isdir, exists
4+
from multiprocessing import cpu_count
5+
from pythonforandroid.recipe import Recipe
56
from pythonforandroid.toolchain import shprint
67
from pythonforandroid.util import current_directory, ensure_dir
78

89

9-
class ICURecipe(NDKRecipe):
10+
class ICURecipe(Recipe):
1011
name = 'icu4c'
1112
version = '57.1'
12-
url = 'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz'
13+
major_version = version.split('.')[0]
14+
url = ('http://download.icu-project.org/files/icu4c/'
15+
'{version}/icu4c-{version_underscore}-src.tgz')
1316

1417
depends = [('hostpython2', 'hostpython3')] # installs in python
15-
generated_libraries = [
16-
'libicui18n.so', 'libicuuc.so', 'libicudata.so', 'libicule.so']
17-
18-
def get_lib_dir(self, arch):
19-
lib_dir = join(self.ctx.get_python_install_dir(), "lib")
20-
ensure_dir(lib_dir)
21-
return lib_dir
22-
23-
def prepare_build_dir(self, arch):
24-
if self.ctx.android_api > 19:
25-
# greater versions do not have /usr/include/sys/exec_elf.h
26-
raise RuntimeError("icu needs an android api <= 19")
27-
28-
super(ICURecipe, self).prepare_build_dir(arch)
29-
30-
def build_arch(self, arch, *extra_args):
18+
patches = ['disable-libs-version.patch']
19+
20+
built_libraries = {
21+
'libicui18n{}.so'.format(major_version): 'build_icu_android/lib',
22+
'libicuuc{}.so'.format(major_version): 'build_icu_android/lib',
23+
'libicudata{}.so'.format(major_version): 'build_icu_android/lib',
24+
'libicule{}.so'.format(major_version): 'build_icu_android/lib',
25+
'libicuio{}.so'.format(major_version): 'build_icu_android/lib',
26+
'libicutu{}.so'.format(major_version): 'build_icu_android/lib',
27+
'libiculx{}.so'.format(major_version): 'build_icu_android/lib',
28+
}
29+
need_stl_shared = True
30+
31+
@property
32+
def versioned_url(self):
33+
if self.url is None:
34+
return None
35+
return self.url.format(
36+
version=self.version,
37+
version_underscore=self.version.replace('.', '_'))
38+
39+
def get_recipe_dir(self):
40+
"""
41+
.. note:: We need to overwrite `Recipe.get_recipe_dir` due to the
42+
mismatch name between the recipe's folder (icu) and the value
43+
of `ICURecipe.name` (icu4c).
44+
"""
45+
if self.ctx.local_recipes is not None:
46+
local_recipe_dir = join(self.ctx.local_recipes, 'icu')
47+
if exists(local_recipe_dir):
48+
return local_recipe_dir
49+
return join(self.ctx.root_dir, 'recipes', 'icu')
50+
51+
def build_arch(self, arch):
3152
env = self.get_recipe_env(arch).copy()
3253
build_root = self.get_build_dir(arch.arch)
3354

@@ -60,74 +81,35 @@ def make_build_dest(dest):
6081
"--prefix="+icu_build,
6182
"--enable-extras=no",
6283
"--enable-strict=no",
63-
"--enable-static",
84+
"--enable-static=no",
6485
"--enable-tests=no",
6586
"--enable-samples=no",
6687
_env=host_env)
67-
shprint(sh.make, "-j5", _env=host_env)
88+
shprint(sh.make, "-j", str(cpu_count()), _env=host_env)
6889
shprint(sh.make, "install", _env=host_env)
6990

7091
build_android, exists = make_build_dest("build_icu_android")
7192
if not exists:
7293

7394
configure = sh.Command(join(build_root, "source", "configure"))
7495

75-
include = (
76-
" -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/include/"
77-
" -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/"
78-
"{arch}/include")
79-
include = include.format(ndk=self.ctx.ndk_dir,
80-
version=env["TOOLCHAIN_VERSION"],
81-
arch=arch.arch)
82-
env["CPPFLAGS"] = env["CXXFLAGS"] + " "
83-
env["CPPFLAGS"] += host_env["CPPFLAGS"]
84-
env["CPPFLAGS"] += include
85-
86-
lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
87-
lib = lib.format(ndk=self.ctx.ndk_dir,
88-
version=env["TOOLCHAIN_VERSION"],
89-
arch=arch.arch)
90-
env["LDFLAGS"] += " -lgnustl_shared -L"+lib
91-
92-
env.pop("CFLAGS", None)
93-
env.pop("CXXFLAGS", None)
94-
9596
with current_directory(build_android):
9697
shprint(
9798
configure,
9899
"--with-cross-build="+build_linux,
99100
"--enable-extras=no",
100101
"--enable-strict=no",
101-
"--enable-static",
102+
"--enable-static=no",
102103
"--enable-tests=no",
103104
"--enable-samples=no",
104105
"--host="+env["TOOLCHAIN_PREFIX"],
105106
"--prefix="+icu_build,
106107
_env=env)
107-
shprint(sh.make, "-j5", _env=env)
108+
shprint(sh.make, "-j", str(cpu_count()), _env=env)
108109
shprint(sh.make, "install", _env=env)
109110

110-
self.copy_files(arch)
111-
112-
def copy_files(self, arch):
113-
env = self.get_recipe_env(arch)
114-
115-
lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
116-
lib = lib.format(ndk=self.ctx.ndk_dir,
117-
version=env["TOOLCHAIN_VERSION"],
118-
arch=arch.arch)
119-
stl_lib = join(lib, "libgnustl_shared.so")
120-
dst_dir = join(self.ctx.get_site_packages_dir(), "..", "lib-dynload")
121-
shprint(sh.cp, stl_lib, dst_dir)
122-
123-
src_lib = join(self.get_build_dir(arch.arch), "icu_build", "lib")
124-
dst_lib = self.get_lib_dir(arch)
125-
126-
src_suffix = "." + self.version
127-
dst_suffix = "." + self.version.split(".")[0] # main version
128-
for lib in self.generated_libraries:
129-
shprint(sh.cp, join(src_lib, lib+src_suffix),
130-
join(dst_lib, lib+dst_suffix))
111+
def install_libraries(self, arch):
112+
super(ICURecipe, self).install_libraries(arch)
131113

132114
src_include = join(
133115
self.get_build_dir(arch.arch), "icu_build", "include")
@@ -137,16 +119,5 @@ def copy_files(self, arch):
137119
shprint(sh.cp, "-r", join(src_include, "layout"), dst_include)
138120
shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include)
139121

140-
# copy stl library
141-
lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
142-
lib = lib.format(ndk=self.ctx.ndk_dir,
143-
version=env["TOOLCHAIN_VERSION"],
144-
arch=arch.arch)
145-
stl_lib = join(lib, "libgnustl_shared.so")
146-
147-
dst_dir = join(self.ctx.get_python_install_dir(), "lib")
148-
ensure_dir(dst_dir)
149-
shprint(sh.cp, stl_lib, dst_dir)
150-
151122

152123
recipe = ICURecipe()

‎pythonforandroid/recipes/icu/disable-libs-version.patch

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
diff -aur icu4c-org/source/config/Makefile.inc.in icu4c/source/config/Makefile.inc.in
2+
--- icu/source/config/Makefile.inc.in.orig 2016-03-23 21:50:50.000000000 +0100
3+
+++ icu/source/config/Makefile.inc.in 2019-02-15 17:59:28.331749766 +0100
4+
@@ -142,8 +142,8 @@
5+
LDLIBRARYPATH_ENVVAR = LD_LIBRARY_PATH
6+
7+
# Versioned target for a shared library
8+
-FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION)
9+
-MIDDLE_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION_MAJOR)
10+
+FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION)
11+
+MIDDLE_SO_TARGET = $(SO_TARGET)
12+
13+
# Access to important ICU tools.
14+
# Use as follows: $(INVOKE) $(GENRB) arguments ..
15+
diff -aur icu4c-org/source/config/mh-linux icu4c/source/config/mh-linux
16+
--- icu4c-org/source/config/mh-linux 2017-07-05 13:23:06.000000000 +0200
17+
+++ icu4c/source/config/mh-linux 2017-07-06 14:02:52.275016858 +0200
18+
@@ -24,9 +24,17 @@
19+
20+
## Compiler switch to embed a library name
21+
# The initial tab in the next line is to prevent icu-config from reading it.
22+
- LD_SONAME = -Wl,-soname -Wl,$(notdir $(MIDDLE_SO_TARGET))
23+
+ LD_SONAME = -Wl,-soname -Wl,$(notdir $(SO_TARGET))
24+
+ DATA_STUBNAME = data$(SO_TARGET_VERSION_MAJOR)
25+
+ COMMON_STUBNAME = uc$(SO_TARGET_VERSION_MAJOR)
26+
+ I18N_STUBNAME = i18n$(SO_TARGET_VERSION_MAJOR)
27+
+ LAYOUT_STUBNAME = le$(SO_TARGET_VERSION_MAJOR)
28+
+ LAYOUTEX_STUBNAME = lx$(SO_TARGET_VERSION_MAJOR)
29+
+ IO_STUBNAME = io$(SO_TARGET_VERSION_MAJOR)
30+
+ TOOLUTIL_STUBNAME = tu$(SO_TARGET_VERSION_MAJOR)
31+
+ CTESTFW_STUBNAME = test$(SO_TARGET_VERSION_MAJOR)
32+
#SH# # We can't depend on MIDDLE_SO_TARGET being set.
33+
-#SH# LD_SONAME=
34+
+#SH# LD_SONAME=$(SO_TARGET)
35+
36+
## Shared library options
37+
LD_SOOPTIONS= -Wl,-Bsymbolic
38+
@@ -64,10 +64,10 @@
39+
40+
## Versioned libraries rules
41+
42+
-%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION)
43+
- $(RM) $@ && ln -s ${<F} $@
44+
-%.$(SO): %.$(SO).$(SO_TARGET_VERSION_MAJOR)
45+
- $(RM) $@ && ln -s ${*F}.$(SO).$(SO_TARGET_VERSION) $@
46+
+%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION)
47+
+ $(RM) $@ && ln -s ${<F} $@
48+
+%.$(SO): %.$(SO).$(SO_TARGET_VERSION_MAJOR)
49+
+ $(RM) $@ && ln -s ${*F}.$(SO).$(SO_TARGET_VERSION) $@
50+
51+
## Bind internal references
52+
53+
diff -aur icu4c-org/source/data/Makefile.in icu4c/source/data/Makefile.in
54+
--- icu4c-org/source/data/Makefile.in 2017-07-05 13:23:06.000000000 +0200
55+
+++ icu4c/source/data/Makefile.in 2017-07-06 14:05:31.607995855 +0200
56+
@@ -24,9 +24,9 @@
57+
ifeq ($(PKGDATA_OPTS),)
58+
PKGDATA_OPTS = -O $(top_builddir)/data/icupkg.inc
59+
endif
60+
-ifeq ($(PKGDATA_VERSIONING),)
61+
-PKGDATA_VERSIONING = -r $(SO_TARGET_VERSION)
62+
-endif
63+
+#ifeq ($(PKGDATA_VERSIONING),)
64+
+#PKGDATA_VERSIONING = -r $(SO_TARGET_VERSION)
65+
+#endif
66+

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy