From dd89ea45050df5efb10b96db8f07e000d04edb94 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Mon, 11 Sep 2023 22:12:23 +0200 Subject: [PATCH 01/30] android/AndroidManifest.xml: raise minSdkVersion to 24 This is needed to build libFLAC which uses ftello(). --- NEWS | 2 ++ android/AndroidManifest.xml | 2 +- android/build.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 50e0c3125..49b8a485e 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,8 @@ ver 0.23.14 (not yet released) - wasapi: fix problem setting volume * more libfmt 10 fixes * fix auto-detected systemd unit directory +* Android + - require Android 7 or newer ver 0.23.13 (2023/05/22) * input diff --git a/android/AndroidManifest.xml b/android/AndroidManifest.xml index cbcc5f893..cbe293389 100644 --- a/android/AndroidManifest.xml +++ b/android/AndroidManifest.xml @@ -5,7 +5,7 @@ android:versionCode="72" android:versionName="0.23.14"> - + diff --git a/android/build.py b/android/build.py index 509cf85a4..e01c1c22f 100755 --- a/android/build.py +++ b/android/build.py @@ -78,7 +78,7 @@ class AndroidNdkToolchain: self.build_path = build_path ndk_arch = abi_info['ndk_arch'] - android_api_level = '21' + android_api_level = '24' install_prefix = os.path.join(arch_path, 'root') From 3f2016e552d4101dab3930f14e9da8558b77cf77 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Sat, 16 Sep 2023 23:23:18 +0200 Subject: [PATCH 02/30] python: add type hints --- python/build/autotools.py | 20 +++++++++++--------- python/build/cmake.py | 19 +++++++++++-------- python/build/makeproject.py | 15 ++++++++------- python/build/meson.py | 12 +++++++----- python/build/openssl.py | 9 +++++---- python/build/project.py | 20 +++++++++++--------- python/build/quilt.py | 5 +++-- python/build/verify.py | 13 +++++++------ python/build/zlib.py | 9 +++++---- 9 files changed, 68 insertions(+), 54 deletions(-) diff --git a/python/build/autotools.py b/python/build/autotools.py index 46fbd273b..d121bf6bf 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -1,15 +1,17 @@ import os.path, subprocess, sys +from typing import Collection, Iterable, Optional from build.makeproject import MakeProject class AutotoolsProject(MakeProject): - def __init__(self, url, md5, installed, configure_args=[], - autogen=False, - autoreconf=False, - cppflags='', - ldflags='', - libs='', - subdirs=None, + def __init__(self, url: str, md5: str, installed: str, + configure_args: Iterable[str]=[], + autogen: bool=False, + autoreconf: bool=False, + cppflags: str='', + ldflags: str='', + libs: str='', + subdirs: Optional[Collection[str]]=None, **kwargs): MakeProject.__init__(self, url, md5, installed, **kwargs) self.configure_args = configure_args @@ -20,7 +22,7 @@ class AutotoolsProject(MakeProject): self.libs = libs self.subdirs = subdirs - def configure(self, toolchain): + def configure(self, toolchain) -> str: src = self.unpack(toolchain) if self.autogen: if sys.platform == 'darwin': @@ -68,7 +70,7 @@ class AutotoolsProject(MakeProject): return build - def _build(self, toolchain): + def _build(self, toolchain) -> None: build = self.configure(toolchain) if self.subdirs is not None: for subdir in self.subdirs: diff --git a/python/build/cmake.py b/python/build/cmake.py index b41d14698..def58c65d 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -1,17 +1,19 @@ import os import re import subprocess +from typing import Optional, TextIO +from collections.abc import Mapping from build.project import Project -def __write_cmake_compiler(f, language, compiler): +def __write_cmake_compiler(f: TextIO, language: str, compiler: str) -> None: s = compiler.split(' ', 1) if len(s) == 2: print(f'set(CMAKE_{language}_COMPILER_LAUNCHER {s[0]})', file=f) compiler = s[1] print(f'set(CMAKE_{language}_COMPILER {compiler})', file=f) -def __write_cmake_toolchain_file(f, toolchain): +def __write_cmake_toolchain_file(f: TextIO, toolchain) -> None: if '-darwin' in toolchain.actual_arch: cmake_system_name = 'Darwin' elif toolchain.is_windows: @@ -52,7 +54,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) """) -def configure(toolchain, src, build, args=(), env=None): +def configure(toolchain, src: str, build: str, args: list[str]=[], env: Optional[Mapping[str, str]]=None) -> None: cross_args = [] if toolchain.is_windows: @@ -91,16 +93,17 @@ def configure(toolchain, src, build, args=(), env=None): subprocess.check_call(configure, env=env, cwd=build) class CmakeProject(Project): - def __init__(self, url, md5, installed, configure_args=[], - windows_configure_args=[], - env=None, + def __init__(self, url: str, md5: str, installed: str, + configure_args: list[str]=[], + windows_configure_args: list[str]=[], + env: Optional[Mapping[str, str]]=None, **kwargs): Project.__init__(self, url, md5, installed, **kwargs) self.configure_args = configure_args self.windows_configure_args = windows_configure_args self.env = env - def configure(self, toolchain): + def configure(self, toolchain) -> str: src = self.unpack(toolchain) build = self.make_build_path(toolchain) configure_args = self.configure_args @@ -109,7 +112,7 @@ class CmakeProject(Project): configure(toolchain, src, build, configure_args, self.env) return build - def _build(self, toolchain): + def _build(self, toolchain) -> None: build = self.configure(toolchain) subprocess.check_call(['ninja', '-v', 'install'], cwd=build, env=toolchain.env) diff --git a/python/build/makeproject.py b/python/build/makeproject.py index 9d498cfbd..c9a2b6eb9 100644 --- a/python/build/makeproject.py +++ b/python/build/makeproject.py @@ -1,15 +1,16 @@ import subprocess, multiprocessing +from typing import Optional from build.project import Project class MakeProject(Project): - def __init__(self, url, md5, installed, - install_target='install', + def __init__(self, url: str, md5: str, installed: str, + install_target: str='install', **kwargs): Project.__init__(self, url, md5, installed, **kwargs) self.install_target = install_target - def get_simultaneous_jobs(self): + def get_simultaneous_jobs(self) -> int: try: # use twice as many simultaneous jobs as we have CPU cores return multiprocessing.cpu_count() * 2 @@ -17,17 +18,17 @@ class MakeProject(Project): # default to 12, if multiprocessing.cpu_count() is not implemented return 12 - def get_make_args(self, toolchain): + def get_make_args(self, toolchain) -> list[str]: return ['--quiet', '-j' + str(self.get_simultaneous_jobs())] - def get_make_install_args(self, toolchain): + def get_make_install_args(self, toolchain) -> list[str]: return ['--quiet', self.install_target] - def make(self, toolchain, wd, args): + def make(self, toolchain, wd: str, args: list[str]) -> None: subprocess.check_call(['make'] + args, cwd=wd, env=toolchain.env) - def build_make(self, toolchain, wd, install=True): + def build_make(self, toolchain, wd: str, install: bool=True) -> None: self.make(toolchain, wd, self.get_make_args(toolchain)) if install: self.make(toolchain, wd, self.get_make_install_args(toolchain)) diff --git a/python/build/meson.py b/python/build/meson.py index 084e36610..e585800f1 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -1,10 +1,11 @@ import os import subprocess import platform +from typing import Optional from build.project import Project -def make_cross_file(toolchain): +def make_cross_file(toolchain) -> str: if toolchain.is_windows: system = 'windows' windres = "windres = '%s'" % toolchain.windres @@ -80,7 +81,7 @@ endian = '{endian}' """) return path -def configure(toolchain, src, build, args=()): +def configure(toolchain, src: str, build: str, args: list[str]=[]) -> None: cross_file = make_cross_file(toolchain) configure = [ 'meson', 'setup', @@ -103,18 +104,19 @@ def configure(toolchain, src, build, args=()): subprocess.check_call(configure, env=env) class MesonProject(Project): - def __init__(self, url, md5, installed, configure_args=[], + def __init__(self, url: str, md5: str, installed: str, + configure_args: list[str]=[], **kwargs): Project.__init__(self, url, md5, installed, **kwargs) self.configure_args = configure_args - def configure(self, toolchain): + def configure(self, toolchain) -> str: src = self.unpack(toolchain) build = self.make_build_path(toolchain) configure(toolchain, src, build, self.configure_args) return build - def _build(self, toolchain): + def _build(self, toolchain) -> None: build = self.configure(toolchain) subprocess.check_call(['ninja', '-v', 'install'], cwd=build, env=toolchain.env) diff --git a/python/build/openssl.py b/python/build/openssl.py index dd02941b5..053fe97b8 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -1,13 +1,14 @@ import subprocess +from typing import Optional from build.makeproject import MakeProject class OpenSSLProject(MakeProject): - def __init__(self, url, md5, installed, + def __init__(self, url: str, md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, install_target='install_dev', **kwargs) - def get_make_args(self, toolchain): + def get_make_args(self, toolchain) -> list[str]: return MakeProject.get_make_args(self, toolchain) + [ 'CC=' + toolchain.cc, 'CFLAGS=' + toolchain.cflags, @@ -17,13 +18,13 @@ class OpenSSLProject(MakeProject): 'build_libs', ] - def get_make_install_args(self, toolchain): + def get_make_install_args(self, toolchain) -> list[str]: # OpenSSL's Makefile runs "ranlib" during installation return MakeProject.get_make_install_args(self, toolchain) + [ 'RANLIB=' + toolchain.ranlib, ] - def _build(self, toolchain): + def _build(self, toolchain) -> None: src = self.unpack(toolchain, out_of_tree=False) # OpenSSL has a weird target architecture scheme with lots of diff --git a/python/build/project.py b/python/build/project.py index bf787cf43..3607d4942 100644 --- a/python/build/project.py +++ b/python/build/project.py @@ -1,16 +1,18 @@ import os, shutil import re +from typing import cast, BinaryIO, Optional from build.download import download_and_verify from build.tar import untar from build.quilt import push_all class Project: - def __init__(self, url, md5, installed, name=None, version=None, - base=None, - patches=None, + def __init__(self, url: str, md5: str, installed: str, + name: Optional[str]=None, version: Optional[str]=None, + base: Optional[str]=None, + patches: Optional[str]=None, edits=None, - use_cxx=False): + use_cxx: bool=False): if base is None: basename = os.path.basename(url) m = re.match(r'^(.+)\.(tar(\.(gz|bz2|xz|lzma))?|zip)$', basename) @@ -39,10 +41,10 @@ class Project: self.edits = edits self.use_cxx = use_cxx - def download(self, toolchain): + def download(self, toolchain) -> str: return download_and_verify(self.url, self.md5, toolchain.tarball_path) - def is_installed(self, toolchain): + def is_installed(self, toolchain) -> bool: tarball = self.download(toolchain) installed = os.path.join(toolchain.install_prefix, self.installed) tarball_mtime = os.path.getmtime(tarball) @@ -51,7 +53,7 @@ class Project: except FileNotFoundError: return False - def unpack(self, toolchain, out_of_tree=True): + def unpack(self, toolchain, out_of_tree: bool=True) -> str: if out_of_tree: parent_path = toolchain.src_path else: @@ -72,7 +74,7 @@ class Project: return path - def make_build_path(self, toolchain, lazy=False): + def make_build_path(self, toolchain, lazy: bool=False) -> str: path = os.path.join(toolchain.build_path, self.base) if lazy and os.path.isdir(path): return path @@ -83,5 +85,5 @@ class Project: os.makedirs(path, exist_ok=True) return path - def build(self, toolchain): + def build(self, toolchain) -> None: self._build(toolchain) diff --git a/python/build/quilt.py b/python/build/quilt.py index 876453d2b..8751404db 100644 --- a/python/build/quilt.py +++ b/python/build/quilt.py @@ -1,9 +1,10 @@ import subprocess +from typing import Union -def run_quilt(toolchain, cwd, patches_path, *args): +def run_quilt(toolchain, cwd: str, patches_path: str, *args: str) -> None: env = dict(toolchain.env) env['QUILT_PATCHES'] = patches_path subprocess.check_call(['quilt'] + list(args), cwd=cwd, env=env) -def push_all(toolchain, src_path, patches_path): +def push_all(toolchain, src_path: str, patches_path: str) -> None: run_quilt(toolchain, src_path, patches_path, 'push', '-a') diff --git a/python/build/verify.py b/python/build/verify.py index e22ff274d..763f224ce 100644 --- a/python/build/verify.py +++ b/python/build/verify.py @@ -1,6 +1,7 @@ import hashlib +from typing import cast, Any, BinaryIO -def feed_file(h, f): +def feed_file(h: Any, f: BinaryIO) -> None: """Feed data read from an open file into the hashlib instance.""" while True: @@ -10,20 +11,20 @@ def feed_file(h, f): break h.update(data) -def feed_file_path(h, path): +def feed_file_path(h: Any, path: str) -> None: """Feed data read from a file (to be opened by this function) into the hashlib instance.""" with open(path, 'rb') as f: feed_file(h, f) -def file_digest(algorithm, path): +def file_digest(algorithm: Any, path: str) -> str: """Calculate the digest of a file and return it in hexadecimal notation.""" h = algorithm() feed_file_path(h, path) - return h.hexdigest() + return cast(str, h.hexdigest()) -def guess_digest_algorithm(digest): +def guess_digest_algorithm(digest: str) -> Any: l = len(digest) if l == 32: return hashlib.md5 @@ -36,7 +37,7 @@ def guess_digest_algorithm(digest): else: return None -def verify_file_digest(path, expected_digest): +def verify_file_digest(path: str, expected_digest: str) -> bool: """Verify the digest of a file, and return True if the digest matches with the given expected digest.""" algorithm = guess_digest_algorithm(expected_digest) diff --git a/python/build/zlib.py b/python/build/zlib.py index c5aeb8c30..98eede580 100644 --- a/python/build/zlib.py +++ b/python/build/zlib.py @@ -1,13 +1,14 @@ import subprocess +from typing import Optional from build.makeproject import MakeProject class ZlibProject(MakeProject): - def __init__(self, url, md5, installed, + def __init__(self, url: str, md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, **kwargs) - def get_make_args(self, toolchain): + def get_make_args(self, toolchain) -> list[str]: return MakeProject.get_make_args(self, toolchain) + [ 'CC=' + toolchain.cc + ' ' + toolchain.cppflags + ' ' + toolchain.cflags, 'CPP=' + toolchain.cc + ' -E ' + toolchain.cppflags, @@ -18,13 +19,13 @@ class ZlibProject(MakeProject): 'libz.a' ] - def get_make_install_args(self, toolchain): + def get_make_install_args(self, toolchain) -> list[str]: return [ 'RANLIB=' + toolchain.ranlib, self.install_target ] - def _build(self, toolchain): + def _build(self, toolchain) -> None: src = self.unpack(toolchain, out_of_tree=False) subprocess.check_call(['./configure', '--prefix=' + toolchain.install_prefix, '--static'], From 100e471b499faef920753e0f3cbbe1389230909c Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 12:11:49 +0200 Subject: [PATCH 03/30] android/build.py: remove duplicate import --- android/build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/android/build.py b/android/build.py index e01c1c22f..30056c8b2 100755 --- a/android/build.py +++ b/android/build.py @@ -60,7 +60,6 @@ sys.path[0] = os.path.join(mpd_path, 'python') # output directories from build.dirs import lib_path, tarball_path, src_path -from build.meson import configure as run_meson arch_path = os.path.join(lib_path, arch) build_path = os.path.join(arch_path, 'build') From 719333e16e80ab7e80b31001ad31a4794d1e6c2b Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 12:09:21 +0200 Subject: [PATCH 04/30] android/build.py: move code to class AndroidNdkToolchain --- android/build.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/android/build.py b/android/build.py index 30056c8b2..05fc5322f 100755 --- a/android/build.py +++ b/android/build.py @@ -50,10 +50,6 @@ android_abis = { }, } -# select the NDK target -abi_info = android_abis[android_abi] -arch = abi_info['arch'] - # the path to the MPD sources mpd_path = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]) or '.', '..')) sys.path[0] = os.path.join(mpd_path, 'python') @@ -61,20 +57,25 @@ sys.path[0] = os.path.join(mpd_path, 'python') # output directories from build.dirs import lib_path, tarball_path, src_path -arch_path = os.path.join(lib_path, arch) -build_path = os.path.join(arch_path, 'build') - -# build host configuration -build_arch = 'linux-x86_64' - # set up the NDK toolchain class AndroidNdkToolchain: - def __init__(self, tarball_path, src_path, build_path, + def __init__(self, top_path: str, lib_path: str, + tarball_path: str, src_path: str, + ndk_path: str, android_abi: str, use_cxx): + # build host configuration + build_arch = 'linux-x86_64' + + # select the NDK target + abi_info = android_abis[android_abi] + arch = abi_info['arch'] + + arch_path = os.path.join(lib_path, arch) + self.tarball_path = tarball_path self.src_path = src_path - self.build_path = build_path + self.build_path = os.path.join(arch_path, 'build') ndk_arch = abi_info['ndk_arch'] android_api_level = '24' @@ -141,7 +142,7 @@ class AndroidNdkToolchain: import shutil bin_dir = os.path.join(install_prefix, 'bin') os.makedirs(bin_dir, exist_ok=True) - self.pkg_config = shutil.copy(os.path.join(mpd_path, 'build', 'pkg-config.sh'), + self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), os.path.join(bin_dir, 'pkg-config')) self.env['PKG_CONFIG'] = self.pkg_config @@ -165,13 +166,17 @@ thirdparty_libs = [ # build the third-party libraries for x in thirdparty_libs: - toolchain = AndroidNdkToolchain(tarball_path, src_path, build_path, + toolchain = AndroidNdkToolchain(mpd_path, lib_path, + tarball_path, src_path, + ndk_path, android_abi, use_cxx=x.use_cxx) if not x.is_installed(toolchain): x.build(toolchain) # configure and build MPD -toolchain = AndroidNdkToolchain(tarball_path, src_path, build_path, +toolchain = AndroidNdkToolchain(mpd_path, lib_path, + tarball_path, src_path, + ndk_path, android_abi, use_cxx=True) configure_args += [ From a7213b78d6034294fffedc5fff2b79b409cd1730 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 12:19:10 +0200 Subject: [PATCH 05/30] win32/build.py: move code to class CrossGccToolchain --- win32/build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/win32/build.py b/win32/build.py index 494969792..ee79b90b6 100755 --- a/win32/build.py +++ b/win32/build.py @@ -35,7 +35,8 @@ build_path = os.path.join(arch_path, 'build') root_path = os.path.join(arch_path, 'root') class CrossGccToolchain: - def __init__(self, toolchain_path, arch, + def __init__(self, top_path: str, + toolchain_path, arch, x64: bool, tarball_path, src_path, build_path, install_prefix): self.arch = arch self.actual_arch = arch @@ -85,7 +86,7 @@ class CrossGccToolchain: import shutil bin_dir = os.path.join(install_prefix, 'bin') os.makedirs(bin_dir, exist_ok=True) - self.pkg_config = shutil.copy(os.path.join(mpd_path, 'build', 'pkg-config.sh'), + self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), os.path.join(bin_dir, 'pkg-config')) self.env['PKG_CONFIG'] = self.pkg_config @@ -111,7 +112,8 @@ thirdparty_libs = [ ] # build the third-party libraries -toolchain = CrossGccToolchain('/usr', host_arch, +toolchain = CrossGccToolchain(mpd_path, + '/usr', host_arch, x64, tarball_path, src_path, build_path, root_path) for x in thirdparty_libs: From 4c90f8870482fb47b6a295c25d6d71faa4b8529a Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 12:46:04 +0200 Subject: [PATCH 06/30] win32: rename CrossGccToolchain to MingwToolchain --- win32/build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/win32/build.py b/win32/build.py index ee79b90b6..35e6852ed 100755 --- a/win32/build.py +++ b/win32/build.py @@ -34,7 +34,7 @@ arch_path = os.path.join(lib_path, host_arch) build_path = os.path.join(arch_path, 'build') root_path = os.path.join(arch_path, 'root') -class CrossGccToolchain: +class MingwToolchain: def __init__(self, top_path: str, toolchain_path, arch, x64: bool, tarball_path, src_path, build_path, install_prefix): @@ -112,9 +112,9 @@ thirdparty_libs = [ ] # build the third-party libraries -toolchain = CrossGccToolchain(mpd_path, - '/usr', host_arch, x64, - tarball_path, src_path, build_path, root_path) +toolchain = MingwToolchain(mpd_path, + '/usr', host_arch, x64, + tarball_path, src_path, build_path, root_path) for x in thirdparty_libs: if not x.is_installed(toolchain): From 4669f7e2b9ea9fea8295be3ec32cc247d79ad3b5 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 12:07:48 +0200 Subject: [PATCH 07/30] {android,win32}/build.py: move Toolchain classes to python/build/toolchain.py --- android/build.py | 120 +------------------------- python/build/toolchain.py | 174 ++++++++++++++++++++++++++++++++++++++ win32/build.py | 57 +------------ 3 files changed, 176 insertions(+), 175 deletions(-) create mode 100644 python/build/toolchain.py diff --git a/android/build.py b/android/build.py index 05fc5322f..8164682e1 100755 --- a/android/build.py +++ b/android/build.py @@ -20,131 +20,13 @@ if not os.path.isdir(ndk_path): print("NDK not found in", ndk_path, file=sys.stderr) sys.exit(1) -android_abis = { - 'armeabi-v7a': { - 'arch': 'arm-linux-androideabi', - 'ndk_arch': 'arm', - 'llvm_triple': 'armv7-linux-androideabi', - 'cflags': '-fpic -mfpu=neon -mfloat-abi=softfp', - }, - - 'arm64-v8a': { - 'arch': 'aarch64-linux-android', - 'ndk_arch': 'arm64', - 'llvm_triple': 'aarch64-linux-android', - 'cflags': '-fpic', - }, - - 'x86': { - 'arch': 'i686-linux-android', - 'ndk_arch': 'x86', - 'llvm_triple': 'i686-linux-android', - 'cflags': '-fPIC -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32', - }, - - 'x86_64': { - 'arch': 'x86_64-linux-android', - 'ndk_arch': 'x86_64', - 'llvm_triple': 'x86_64-linux-android', - 'cflags': '-fPIC -m64', - }, -} - # the path to the MPD sources mpd_path = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]) or '.', '..')) sys.path[0] = os.path.join(mpd_path, 'python') # output directories from build.dirs import lib_path, tarball_path, src_path - -# set up the NDK toolchain - -class AndroidNdkToolchain: - def __init__(self, top_path: str, lib_path: str, - tarball_path: str, src_path: str, - ndk_path: str, android_abi: str, - use_cxx): - # build host configuration - build_arch = 'linux-x86_64' - - # select the NDK target - abi_info = android_abis[android_abi] - arch = abi_info['arch'] - - arch_path = os.path.join(lib_path, arch) - - self.tarball_path = tarball_path - self.src_path = src_path - self.build_path = os.path.join(arch_path, 'build') - - ndk_arch = abi_info['ndk_arch'] - android_api_level = '24' - - install_prefix = os.path.join(arch_path, 'root') - - self.arch = arch - self.actual_arch = arch - self.install_prefix = install_prefix - - llvm_path = os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', build_arch) - llvm_triple = abi_info['llvm_triple'] + android_api_level - - common_flags = '-Os -g' - common_flags += ' ' + abi_info['cflags'] - - llvm_bin = os.path.join(llvm_path, 'bin') - self.cc = os.path.join(llvm_bin, 'clang') - self.cxx = os.path.join(llvm_bin, 'clang++') - common_flags += ' -target ' + llvm_triple - - common_flags += ' -fvisibility=hidden -fdata-sections -ffunction-sections' - - self.ar = os.path.join(llvm_bin, 'llvm-ar') - self.arflags = 'rcs' - self.ranlib = os.path.join(llvm_bin, 'llvm-ranlib') - self.nm = os.path.join(llvm_bin, 'llvm-nm') - self.strip = os.path.join(llvm_bin, 'llvm-strip') - - self.cflags = common_flags - self.cxxflags = common_flags - self.cppflags = ' -isystem ' + os.path.join(install_prefix, 'include') - self.ldflags = ' -L' + os.path.join(install_prefix, 'lib') + \ - ' -Wl,--exclude-libs=ALL' + \ - ' ' + common_flags - self.ldflags = common_flags - self.libs = '' - - self.is_arm = ndk_arch == 'arm' - self.is_armv7 = self.is_arm and 'armv7' in self.cflags - self.is_aarch64 = ndk_arch == 'arm64' - self.is_windows = False - - libstdcxx_flags = '' - libstdcxx_cxxflags = '' - libstdcxx_ldflags = '' - libstdcxx_libs = '-static-libstdc++' - - if self.is_armv7: - # On 32 bit ARM, clang generates no ".eh_frame" section; - # instead, the LLVM unwinder library is used for unwinding - # the stack after a C++ exception was thrown - libstdcxx_libs += ' -lunwind' - - if use_cxx: - self.cxxflags += ' ' + libstdcxx_cxxflags - self.ldflags += ' ' + libstdcxx_ldflags - self.libs += ' ' + libstdcxx_libs - - self.env = dict(os.environ) - - # redirect pkg-config to use our root directory instead of the - # default one on the build host - import shutil - bin_dir = os.path.join(install_prefix, 'bin') - os.makedirs(bin_dir, exist_ok=True) - self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), - os.path.join(bin_dir, 'pkg-config')) - self.env['PKG_CONFIG'] = self.pkg_config +from build.toolchain import AndroidNdkToolchain # a list of third-party libraries to be used by MPD on Android from build.libs import * diff --git a/python/build/toolchain.py b/python/build/toolchain.py new file mode 100644 index 000000000..78d4954c7 --- /dev/null +++ b/python/build/toolchain.py @@ -0,0 +1,174 @@ +import os.path +import shutil + +android_abis = { + 'armeabi-v7a': { + 'arch': 'arm-linux-androideabi', + 'ndk_arch': 'arm', + 'llvm_triple': 'armv7-linux-androideabi', + 'cflags': '-fpic -mfpu=neon -mfloat-abi=softfp', + }, + + 'arm64-v8a': { + 'arch': 'aarch64-linux-android', + 'ndk_arch': 'arm64', + 'llvm_triple': 'aarch64-linux-android', + 'cflags': '-fpic', + }, + + 'x86': { + 'arch': 'i686-linux-android', + 'ndk_arch': 'x86', + 'llvm_triple': 'i686-linux-android', + 'cflags': '-fPIC -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32', + }, + + 'x86_64': { + 'arch': 'x86_64-linux-android', + 'ndk_arch': 'x86_64', + 'llvm_triple': 'x86_64-linux-android', + 'cflags': '-fPIC -m64', + }, +} + +class AndroidNdkToolchain: + def __init__(self, top_path: str, lib_path: str, + tarball_path: str, src_path: str, + ndk_path: str, android_abi: str, + use_cxx): + # build host configuration + build_arch = 'linux-x86_64' + + # select the NDK target + abi_info = android_abis[android_abi] + arch = abi_info['arch'] + + arch_path = os.path.join(lib_path, arch) + + self.tarball_path = tarball_path + self.src_path = src_path + self.build_path = os.path.join(arch_path, 'build') + + ndk_arch = abi_info['ndk_arch'] + android_api_level = '24' + + install_prefix = os.path.join(arch_path, 'root') + + self.arch = arch + self.actual_arch = arch + self.install_prefix = install_prefix + + llvm_path = os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', build_arch) + llvm_triple = abi_info['llvm_triple'] + android_api_level + + common_flags = '-Os -g' + common_flags += ' ' + abi_info['cflags'] + + llvm_bin = os.path.join(llvm_path, 'bin') + self.cc = os.path.join(llvm_bin, 'clang') + self.cxx = os.path.join(llvm_bin, 'clang++') + common_flags += ' -target ' + llvm_triple + + common_flags += ' -fvisibility=hidden -fdata-sections -ffunction-sections' + + self.ar = os.path.join(llvm_bin, 'llvm-ar') + self.arflags = 'rcs' + self.ranlib = os.path.join(llvm_bin, 'llvm-ranlib') + self.nm = os.path.join(llvm_bin, 'llvm-nm') + self.strip = os.path.join(llvm_bin, 'llvm-strip') + + self.cflags = common_flags + self.cxxflags = common_flags + self.cppflags = ' -isystem ' + os.path.join(install_prefix, 'include') + self.ldflags = ' -L' + os.path.join(install_prefix, 'lib') + \ + ' -Wl,--exclude-libs=ALL' + \ + ' ' + common_flags + self.ldflags = common_flags + self.libs = '' + + self.is_arm = ndk_arch == 'arm' + self.is_armv7 = self.is_arm and 'armv7' in self.cflags + self.is_aarch64 = ndk_arch == 'arm64' + self.is_windows = False + + libstdcxx_flags = '' + libstdcxx_cxxflags = '' + libstdcxx_ldflags = '' + libstdcxx_libs = '-static-libstdc++' + + if self.is_armv7: + # On 32 bit ARM, clang generates no ".eh_frame" section; + # instead, the LLVM unwinder library is used for unwinding + # the stack after a C++ exception was thrown + libstdcxx_libs += ' -lunwind' + + if use_cxx: + self.cxxflags += ' ' + libstdcxx_cxxflags + self.ldflags += ' ' + libstdcxx_ldflags + self.libs += ' ' + libstdcxx_libs + + self.env = dict(os.environ) + + # redirect pkg-config to use our root directory instead of the + # default one on the build host + bin_dir = os.path.join(install_prefix, 'bin') + os.makedirs(bin_dir, exist_ok=True) + self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), + os.path.join(bin_dir, 'pkg-config')) + self.env['PKG_CONFIG'] = self.pkg_config + +class MingwToolchain: + def __init__(self, top_path: str, + toolchain_path, arch, x64: bool, + tarball_path, src_path, build_path, install_prefix): + self.arch = arch + self.actual_arch = arch + self.tarball_path = tarball_path + self.src_path = src_path + self.build_path = build_path + self.install_prefix = install_prefix + + toolchain_bin = os.path.join(toolchain_path, 'bin') + self.cc = os.path.join(toolchain_bin, arch + '-gcc') + self.cxx = os.path.join(toolchain_bin, arch + '-g++') + self.ar = os.path.join(toolchain_bin, arch + '-ar') + self.arflags = 'rcs' + self.ranlib = os.path.join(toolchain_bin, arch + '-ranlib') + self.nm = os.path.join(toolchain_bin, arch + '-nm') + self.strip = os.path.join(toolchain_bin, arch + '-strip') + self.windres = os.path.join(toolchain_bin, arch + '-windres') + + common_flags = '-O2 -g' + + if not x64: + # enable SSE support which is required for LAME + common_flags += ' -march=pentium3' + + self.cflags = common_flags + self.cxxflags = common_flags + self.cppflags = '-isystem ' + os.path.join(install_prefix, 'include') + \ + ' -DWINVER=0x0600 -D_WIN32_WINNT=0x0600' + self.ldflags = '-L' + os.path.join(install_prefix, 'lib') + \ + ' -static-libstdc++ -static-libgcc' + self.libs = '' + + # Explicitly disable _FORTIFY_SOURCE because it is broken with + # mingw. This prevents some libraries such as libFLAC to + # enable it. + self.cppflags += ' -D_FORTIFY_SOURCE=0' + + self.is_arm = arch.startswith('arm') + self.is_armv7 = self.is_arm and 'armv7' in self.cflags + self.is_aarch64 = arch == 'aarch64' + self.is_windows = 'mingw32' in arch + + self.env = dict(os.environ) + + # redirect pkg-config to use our root directory instead of the + # default one on the build host + import shutil + bin_dir = os.path.join(install_prefix, 'bin') + os.makedirs(bin_dir, exist_ok=True) + self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), + os.path.join(bin_dir, 'pkg-config')) + self.env['PKG_CONFIG'] = self.pkg_config diff --git a/win32/build.py b/win32/build.py index 35e6852ed..7a51298ad 100755 --- a/win32/build.py +++ b/win32/build.py @@ -29,67 +29,12 @@ sys.path[0] = os.path.join(mpd_path, 'python') # output directories from build.dirs import lib_path, tarball_path, src_path +from build.toolchain import MingwToolchain arch_path = os.path.join(lib_path, host_arch) build_path = os.path.join(arch_path, 'build') root_path = os.path.join(arch_path, 'root') -class MingwToolchain: - def __init__(self, top_path: str, - toolchain_path, arch, x64: bool, - tarball_path, src_path, build_path, install_prefix): - self.arch = arch - self.actual_arch = arch - self.tarball_path = tarball_path - self.src_path = src_path - self.build_path = build_path - self.install_prefix = install_prefix - - toolchain_bin = os.path.join(toolchain_path, 'bin') - self.cc = os.path.join(toolchain_bin, arch + '-gcc') - self.cxx = os.path.join(toolchain_bin, arch + '-g++') - self.ar = os.path.join(toolchain_bin, arch + '-ar') - self.arflags = 'rcs' - self.ranlib = os.path.join(toolchain_bin, arch + '-ranlib') - self.nm = os.path.join(toolchain_bin, arch + '-nm') - self.strip = os.path.join(toolchain_bin, arch + '-strip') - self.windres = os.path.join(toolchain_bin, arch + '-windres') - - common_flags = '-O2 -g' - - if not x64: - # enable SSE support which is required for LAME - common_flags += ' -march=pentium3' - - self.cflags = common_flags - self.cxxflags = common_flags - self.cppflags = '-isystem ' + os.path.join(install_prefix, 'include') + \ - ' -DWINVER=0x0600 -D_WIN32_WINNT=0x0600' - self.ldflags = '-L' + os.path.join(install_prefix, 'lib') + \ - ' -static-libstdc++ -static-libgcc' - self.libs = '' - - # Explicitly disable _FORTIFY_SOURCE because it is broken with - # mingw. This prevents some libraries such as libFLAC to - # enable it. - self.cppflags += ' -D_FORTIFY_SOURCE=0' - - self.is_arm = arch.startswith('arm') - self.is_armv7 = self.is_arm and 'armv7' in self.cflags - self.is_aarch64 = arch == 'aarch64' - self.is_windows = 'mingw32' in arch - - self.env = dict(os.environ) - - # redirect pkg-config to use our root directory instead of the - # default one on the build host - import shutil - bin_dir = os.path.join(install_prefix, 'bin') - os.makedirs(bin_dir, exist_ok=True) - self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), - os.path.join(bin_dir, 'pkg-config')) - self.env['PKG_CONFIG'] = self.pkg_config - # a list of third-party libraries to be used by MPD on Android from build.libs import * thirdparty_libs = [ From 5f253e66f6b9bddae2a44431844059e8975fde2c Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 12:47:44 +0200 Subject: [PATCH 08/30] python/build/toolchain.py: add AnyToolchain for type hints --- python/build/autotools.py | 5 +++-- python/build/cmake.py | 9 +++++---- python/build/makeproject.py | 9 +++++---- python/build/meson.py | 9 +++++---- python/build/openssl.py | 7 ++++--- python/build/project.py | 11 ++++++----- python/build/quilt.py | 6 ++++-- python/build/toolchain.py | 3 +++ python/build/zlib.py | 7 ++++--- 9 files changed, 39 insertions(+), 27 deletions(-) diff --git a/python/build/autotools.py b/python/build/autotools.py index d121bf6bf..f8c4f42c4 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -2,6 +2,7 @@ import os.path, subprocess, sys from typing import Collection, Iterable, Optional from build.makeproject import MakeProject +from .toolchain import AnyToolchain class AutotoolsProject(MakeProject): def __init__(self, url: str, md5: str, installed: str, @@ -22,7 +23,7 @@ class AutotoolsProject(MakeProject): self.libs = libs self.subdirs = subdirs - def configure(self, toolchain) -> str: + def configure(self, toolchain: AnyToolchain) -> str: src = self.unpack(toolchain) if self.autogen: if sys.platform == 'darwin': @@ -70,7 +71,7 @@ class AutotoolsProject(MakeProject): return build - def _build(self, toolchain) -> None: + def _build(self, toolchain: AnyToolchain) -> None: build = self.configure(toolchain) if self.subdirs is not None: for subdir in self.subdirs: diff --git a/python/build/cmake.py b/python/build/cmake.py index def58c65d..a26d7d6f8 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -5,6 +5,7 @@ from typing import Optional, TextIO from collections.abc import Mapping from build.project import Project +from .toolchain import AnyToolchain def __write_cmake_compiler(f: TextIO, language: str, compiler: str) -> None: s = compiler.split(' ', 1) @@ -13,7 +14,7 @@ def __write_cmake_compiler(f: TextIO, language: str, compiler: str) -> None: compiler = s[1] print(f'set(CMAKE_{language}_COMPILER {compiler})', file=f) -def __write_cmake_toolchain_file(f: TextIO, toolchain) -> None: +def __write_cmake_toolchain_file(f: TextIO, toolchain: AnyToolchain) -> None: if '-darwin' in toolchain.actual_arch: cmake_system_name = 'Darwin' elif toolchain.is_windows: @@ -54,7 +55,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) """) -def configure(toolchain, src: str, build: str, args: list[str]=[], env: Optional[Mapping[str, str]]=None) -> None: +def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[], env: Optional[Mapping[str, str]]=None) -> None: cross_args = [] if toolchain.is_windows: @@ -103,7 +104,7 @@ class CmakeProject(Project): self.windows_configure_args = windows_configure_args self.env = env - def configure(self, toolchain) -> str: + def configure(self, toolchain: AnyToolchain) -> str: src = self.unpack(toolchain) build = self.make_build_path(toolchain) configure_args = self.configure_args @@ -112,7 +113,7 @@ class CmakeProject(Project): configure(toolchain, src, build, configure_args, self.env) return build - def _build(self, toolchain) -> None: + def _build(self, toolchain: AnyToolchain) -> None: build = self.configure(toolchain) subprocess.check_call(['ninja', '-v', 'install'], cwd=build, env=toolchain.env) diff --git a/python/build/makeproject.py b/python/build/makeproject.py index c9a2b6eb9..d8d0910ac 100644 --- a/python/build/makeproject.py +++ b/python/build/makeproject.py @@ -2,6 +2,7 @@ import subprocess, multiprocessing from typing import Optional from build.project import Project +from .toolchain import AnyToolchain class MakeProject(Project): def __init__(self, url: str, md5: str, installed: str, @@ -18,17 +19,17 @@ class MakeProject(Project): # default to 12, if multiprocessing.cpu_count() is not implemented return 12 - def get_make_args(self, toolchain) -> list[str]: + def get_make_args(self, toolchain: AnyToolchain) -> list[str]: return ['--quiet', '-j' + str(self.get_simultaneous_jobs())] - def get_make_install_args(self, toolchain) -> list[str]: + def get_make_install_args(self, toolchain: AnyToolchain) -> list[str]: return ['--quiet', self.install_target] - def make(self, toolchain, wd: str, args: list[str]) -> None: + def make(self, toolchain: AnyToolchain, wd: str, args: list[str]) -> None: subprocess.check_call(['make'] + args, cwd=wd, env=toolchain.env) - def build_make(self, toolchain, wd: str, install: bool=True) -> None: + def build_make(self, toolchain: AnyToolchain, wd: str, install: bool=True) -> None: self.make(toolchain, wd, self.get_make_args(toolchain)) if install: self.make(toolchain, wd, self.get_make_install_args(toolchain)) diff --git a/python/build/meson.py b/python/build/meson.py index e585800f1..ffcb5d38a 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -4,8 +4,9 @@ import platform from typing import Optional from build.project import Project +from .toolchain import AnyToolchain -def make_cross_file(toolchain) -> str: +def make_cross_file(toolchain: AnyToolchain) -> str: if toolchain.is_windows: system = 'windows' windres = "windres = '%s'" % toolchain.windres @@ -81,7 +82,7 @@ endian = '{endian}' """) return path -def configure(toolchain, src: str, build: str, args: list[str]=[]) -> None: +def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[]) -> None: cross_file = make_cross_file(toolchain) configure = [ 'meson', 'setup', @@ -110,13 +111,13 @@ class MesonProject(Project): Project.__init__(self, url, md5, installed, **kwargs) self.configure_args = configure_args - def configure(self, toolchain) -> str: + def configure(self, toolchain: AnyToolchain) -> str: src = self.unpack(toolchain) build = self.make_build_path(toolchain) configure(toolchain, src, build, self.configure_args) return build - def _build(self, toolchain) -> None: + def _build(self, toolchain: AnyToolchain) -> None: build = self.configure(toolchain) subprocess.check_call(['ninja', '-v', 'install'], cwd=build, env=toolchain.env) diff --git a/python/build/openssl.py b/python/build/openssl.py index 053fe97b8..df63f2738 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -2,13 +2,14 @@ import subprocess from typing import Optional from build.makeproject import MakeProject +from .toolchain import AnyToolchain class OpenSSLProject(MakeProject): def __init__(self, url: str, md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, install_target='install_dev', **kwargs) - def get_make_args(self, toolchain) -> list[str]: + def get_make_args(self, toolchain: AnyToolchain) -> list[str]: return MakeProject.get_make_args(self, toolchain) + [ 'CC=' + toolchain.cc, 'CFLAGS=' + toolchain.cflags, @@ -18,13 +19,13 @@ class OpenSSLProject(MakeProject): 'build_libs', ] - def get_make_install_args(self, toolchain) -> list[str]: + def get_make_install_args(self, toolchain: AnyToolchain) -> list[str]: # OpenSSL's Makefile runs "ranlib" during installation return MakeProject.get_make_install_args(self, toolchain) + [ 'RANLIB=' + toolchain.ranlib, ] - def _build(self, toolchain) -> None: + def _build(self, toolchain: AnyToolchain) -> None: src = self.unpack(toolchain, out_of_tree=False) # OpenSSL has a weird target architecture scheme with lots of diff --git a/python/build/project.py b/python/build/project.py index 3607d4942..12791a05a 100644 --- a/python/build/project.py +++ b/python/build/project.py @@ -5,6 +5,7 @@ from typing import cast, BinaryIO, Optional from build.download import download_and_verify from build.tar import untar from build.quilt import push_all +from .toolchain import AnyToolchain class Project: def __init__(self, url: str, md5: str, installed: str, @@ -41,10 +42,10 @@ class Project: self.edits = edits self.use_cxx = use_cxx - def download(self, toolchain) -> str: + def download(self, toolchain: AnyToolchain) -> str: return download_and_verify(self.url, self.md5, toolchain.tarball_path) - def is_installed(self, toolchain) -> bool: + def is_installed(self, toolchain: AnyToolchain) -> bool: tarball = self.download(toolchain) installed = os.path.join(toolchain.install_prefix, self.installed) tarball_mtime = os.path.getmtime(tarball) @@ -53,7 +54,7 @@ class Project: except FileNotFoundError: return False - def unpack(self, toolchain, out_of_tree: bool=True) -> str: + def unpack(self, toolchain: AnyToolchain, out_of_tree: bool=True) -> str: if out_of_tree: parent_path = toolchain.src_path else: @@ -74,7 +75,7 @@ class Project: return path - def make_build_path(self, toolchain, lazy: bool=False) -> str: + def make_build_path(self, toolchain: AnyToolchain, lazy: bool=False) -> str: path = os.path.join(toolchain.build_path, self.base) if lazy and os.path.isdir(path): return path @@ -85,5 +86,5 @@ class Project: os.makedirs(path, exist_ok=True) return path - def build(self, toolchain) -> None: + def build(self, toolchain: AnyToolchain) -> None: self._build(toolchain) diff --git a/python/build/quilt.py b/python/build/quilt.py index 8751404db..03adf0e42 100644 --- a/python/build/quilt.py +++ b/python/build/quilt.py @@ -1,10 +1,12 @@ import subprocess from typing import Union -def run_quilt(toolchain, cwd: str, patches_path: str, *args: str) -> None: +from .toolchain import AnyToolchain + +def run_quilt(toolchain: AnyToolchain, cwd: str, patches_path: str, *args: str) -> None: env = dict(toolchain.env) env['QUILT_PATCHES'] = patches_path subprocess.check_call(['quilt'] + list(args), cwd=cwd, env=env) -def push_all(toolchain, src_path: str, patches_path: str) -> None: +def push_all(toolchain: AnyToolchain, src_path: str, patches_path: str) -> None: run_quilt(toolchain, src_path, patches_path, 'push', '-a') diff --git a/python/build/toolchain.py b/python/build/toolchain.py index 78d4954c7..15a29ca20 100644 --- a/python/build/toolchain.py +++ b/python/build/toolchain.py @@ -1,5 +1,6 @@ import os.path import shutil +from typing import Union android_abis = { 'armeabi-v7a': { @@ -172,3 +173,5 @@ class MingwToolchain: self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), os.path.join(bin_dir, 'pkg-config')) self.env['PKG_CONFIG'] = self.pkg_config + +AnyToolchain = Union[AndroidNdkToolchain, MingwToolchain] diff --git a/python/build/zlib.py b/python/build/zlib.py index 98eede580..2007c6c6d 100644 --- a/python/build/zlib.py +++ b/python/build/zlib.py @@ -2,13 +2,14 @@ import subprocess from typing import Optional from build.makeproject import MakeProject +from .toolchain import AnyToolchain class ZlibProject(MakeProject): def __init__(self, url: str, md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, **kwargs) - def get_make_args(self, toolchain) -> list[str]: + def get_make_args(self, toolchain: AnyToolchain) -> list[str]: return MakeProject.get_make_args(self, toolchain) + [ 'CC=' + toolchain.cc + ' ' + toolchain.cppflags + ' ' + toolchain.cflags, 'CPP=' + toolchain.cc + ' -E ' + toolchain.cppflags, @@ -19,13 +20,13 @@ class ZlibProject(MakeProject): 'libz.a' ] - def get_make_install_args(self, toolchain) -> list[str]: + def get_make_install_args(self, toolchain: AnyToolchain) -> list[str]: return [ 'RANLIB=' + toolchain.ranlib, self.install_target ] - def _build(self, toolchain) -> None: + def _build(self, toolchain: AnyToolchain) -> None: src = self.unpack(toolchain, out_of_tree=False) subprocess.check_call(['./configure', '--prefix=' + toolchain.install_prefix, '--static'], From a56a709406ee929ec2b1676ff37cb8090e7ab897 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:14:50 +0200 Subject: [PATCH 09/30] python/build/download: relative imports --- python/build/download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/build/download.py b/python/build/download.py index 9cc710c8f..3cc491e11 100644 --- a/python/build/download.py +++ b/python/build/download.py @@ -1,7 +1,8 @@ -from build.verify import verify_file_digest import os import urllib.request +from .verify import verify_file_digest + def download_and_verify(url, md5, parent_path): """Download a file, verify its MD5 checksum and return the local path.""" From f6d73555a64146cbd5ac70061484065a94dd49d4 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:21:25 +0200 Subject: [PATCH 10/30] python/build/libs: update OpenSSL toi 3.1.3 --- python/build/libs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/build/libs.py b/python/build/libs.py index 8a049045c..54f444d9f 100644 --- a/python/build/libs.py +++ b/python/build/libs.py @@ -599,8 +599,8 @@ ffmpeg = FfmpegProject( ) openssl = OpenSSLProject( - 'https://www.openssl.org/source/openssl-3.1.2.tar.gz', - 'a0ce69b8b97ea6a35b96875235aa453b966ba3cba8af2de23657d8b6767d6539', + 'https://www.openssl.org/source/openssl-3.1.3.tar.gz', + 'f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6', 'include/openssl/ossl_typ.h', ) From eb23788fec8e4724a138544a36e1dc331911c35c Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:14:33 +0200 Subject: [PATCH 11/30] python/build: add support for fallback download URLs --- python/build/autotools.py | 4 ++-- python/build/cmake.py | 4 ++-- python/build/download.py | 48 ++++++++++++++++++++++++++++++------- python/build/libs.py | 9 ++++--- python/build/makeproject.py | 4 ++-- python/build/meson.py | 4 ++-- python/build/openssl.py | 4 ++-- python/build/project.py | 8 +++---- python/build/zlib.py | 4 ++-- 9 files changed, 62 insertions(+), 27 deletions(-) diff --git a/python/build/autotools.py b/python/build/autotools.py index f8c4f42c4..600bb627c 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -1,11 +1,11 @@ import os.path, subprocess, sys -from typing import Collection, Iterable, Optional +from typing import Collection, Iterable, Optional, Sequence, Union from build.makeproject import MakeProject from .toolchain import AnyToolchain class AutotoolsProject(MakeProject): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, configure_args: Iterable[str]=[], autogen: bool=False, autoreconf: bool=False, diff --git a/python/build/cmake.py b/python/build/cmake.py index a26d7d6f8..1971a6cd8 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -1,7 +1,7 @@ import os import re import subprocess -from typing import Optional, TextIO +from typing import Optional, Sequence, TextIO, Union from collections.abc import Mapping from build.project import Project @@ -94,7 +94,7 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[], subprocess.check_call(configure, env=env, cwd=build) class CmakeProject(Project): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, configure_args: list[str]=[], windows_configure_args: list[str]=[], env: Optional[Mapping[str, str]]=None, diff --git a/python/build/download.py b/python/build/download.py index 3cc491e11..c170c841a 100644 --- a/python/build/download.py +++ b/python/build/download.py @@ -1,13 +1,50 @@ +from typing import Sequence, Union import os +import sys import urllib.request from .verify import verify_file_digest -def download_and_verify(url, md5, parent_path): +def __to_string_sequence(x: Union[str, Sequence[str]]) -> Sequence[str]: + if isinstance(x, str): + return (x,) + else: + return x + +def __get_any(x: Union[str, Sequence[str]]) -> str: + if isinstance(x, str): + return x + else: + return x[0] + +def __download_one(url: str, path: str) -> None: + print("download", url) + urllib.request.urlretrieve(url, path) + +def __download(urls: Sequence[str], path: str) -> None: + for url in urls[:-1]: + try: + __download_one(url, path) + return + except: + print("download error:", sys.exc_info()[0]) + __download_one(urls[-1], path) + +def __download_and_verify_to(urls: Sequence[str], md5: str, path: str) -> None: + __download(urls, path) + if not verify_file_digest(path, md5): + raise RuntimeError("Digest mismatch") + +def download_basename(urls: Union[str, Sequence[str]]) -> str: + return os.path.basename(__get_any(urls)) + +def download_and_verify(urls: Union[str, Sequence[str]], md5: str, parent_path: str) -> str: """Download a file, verify its MD5 checksum and return the local path.""" + base = download_basename(urls) + os.makedirs(parent_path, exist_ok=True) - path = os.path.join(parent_path, os.path.basename(url)) + path = os.path.join(parent_path, base) try: if verify_file_digest(path, md5): return path @@ -17,11 +54,6 @@ def download_and_verify(url, md5, parent_path): tmp_path = path + '.tmp' - print("download", url) - urllib.request.urlretrieve(url, tmp_path) - if not verify_file_digest(tmp_path, md5): - os.unlink(tmp_path) - raise RuntimeError("Digest mismatch") - + __download_and_verify_to(__to_string_sequence(urls), md5, tmp_path) os.rename(tmp_path, path) return path diff --git a/python/build/libs.py b/python/build/libs.py index 54f444d9f..15b7a0169 100644 --- a/python/build/libs.py +++ b/python/build/libs.py @@ -57,7 +57,8 @@ flac = AutotoolsProject( ) zlib = ZlibProject( - 'http://zlib.net/zlib-1.3.tar.xz', + ('http://zlib.net/zlib-1.3.tar.xz', + 'https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.xz'), '8a9ba2898e1d0d774eca6ba5b4627a11e5588ba85c8851336eb38de4683050a7', 'lib/libz.a', ) @@ -599,13 +600,15 @@ ffmpeg = FfmpegProject( ) openssl = OpenSSLProject( - 'https://www.openssl.org/source/openssl-3.1.3.tar.gz', + ('https://www.openssl.org/source/openssl-3.1.3.tar.gz', + 'https://artfiles.org/openssl.org/source/openssl-3.1.3.tar.gz'), 'f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6', 'include/openssl/ossl_typ.h', ) curl = CmakeProject( - 'https://curl.se/download/curl-8.2.1.tar.xz', + ('https://curl.se/download/curl-8.2.1.tar.xz', + 'https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.xz'), 'dd322f6bd0a20e6cebdfd388f69e98c3d183bed792cf4713c8a7ef498cba4894', 'lib/libcurl.a', [ diff --git a/python/build/makeproject.py b/python/build/makeproject.py index d8d0910ac..b0acf7807 100644 --- a/python/build/makeproject.py +++ b/python/build/makeproject.py @@ -1,11 +1,11 @@ import subprocess, multiprocessing -from typing import Optional +from typing import Optional, Sequence, Union from build.project import Project from .toolchain import AnyToolchain class MakeProject(Project): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, install_target: str='install', **kwargs): Project.__init__(self, url, md5, installed, **kwargs) diff --git a/python/build/meson.py b/python/build/meson.py index ffcb5d38a..6469e4d58 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -1,7 +1,7 @@ import os import subprocess import platform -from typing import Optional +from typing import Optional, Sequence, Union from build.project import Project from .toolchain import AnyToolchain @@ -105,7 +105,7 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[]) subprocess.check_call(configure, env=env) class MesonProject(Project): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, configure_args: list[str]=[], **kwargs): Project.__init__(self, url, md5, installed, **kwargs) diff --git a/python/build/openssl.py b/python/build/openssl.py index df63f2738..81396ce02 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -1,11 +1,11 @@ import subprocess -from typing import Optional +from typing import Optional, Sequence, Union from build.makeproject import MakeProject from .toolchain import AnyToolchain class OpenSSLProject(MakeProject): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, install_target='install_dev', **kwargs) diff --git a/python/build/project.py b/python/build/project.py index 12791a05a..f636b5118 100644 --- a/python/build/project.py +++ b/python/build/project.py @@ -1,21 +1,21 @@ import os, shutil import re -from typing import cast, BinaryIO, Optional +from typing import cast, BinaryIO, Optional, Sequence, Union -from build.download import download_and_verify +from build.download import download_basename, download_and_verify from build.tar import untar from build.quilt import push_all from .toolchain import AnyToolchain class Project: - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, name: Optional[str]=None, version: Optional[str]=None, base: Optional[str]=None, patches: Optional[str]=None, edits=None, use_cxx: bool=False): if base is None: - basename = os.path.basename(url) + basename = download_basename(url) m = re.match(r'^(.+)\.(tar(\.(gz|bz2|xz|lzma))?|zip)$', basename) if not m: raise RuntimeError('Could not identify tarball name: ' + basename) self.base = m.group(1) diff --git a/python/build/zlib.py b/python/build/zlib.py index 2007c6c6d..41138c825 100644 --- a/python/build/zlib.py +++ b/python/build/zlib.py @@ -1,11 +1,11 @@ import subprocess -from typing import Optional +from typing import Optional, Sequence, Union from build.makeproject import MakeProject from .toolchain import AnyToolchain class ZlibProject(MakeProject): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, **kwargs) From a9f1bed922bde81e83386017ef16ceeb6992476d Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:12:51 +0200 Subject: [PATCH 12/30] build/python/cmake: add cast to fix mypy warning --- python/build/cmake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/build/cmake.py b/python/build/cmake.py index 1971a6cd8..3b7e86405 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -1,7 +1,7 @@ import os import re import subprocess -from typing import Optional, Sequence, TextIO, Union +from typing import cast, Optional, Sequence, TextIO, Union from collections.abc import Mapping from build.project import Project @@ -59,7 +59,7 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[], cross_args = [] if toolchain.is_windows: - cross_args.append('-DCMAKE_RC_COMPILER=' + toolchain.windres) + cross_args.append('-DCMAKE_RC_COMPILER=' + cast(str, toolchain.windres)) # Several targets need a sysroot to prevent pkg-config from # looking for libraries on the build host (TODO: fix this From 3fedd978a2cec2f9b74468501de76829b5bd22d3 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:27:42 +0200 Subject: [PATCH 13/30] python/build/meson: disable ccache because Meson detects it automatically --- python/build/meson.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/build/meson.py b/python/build/meson.py index 6469e4d58..1a7311955 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -6,6 +6,11 @@ from typing import Optional, Sequence, Union from build.project import Project from .toolchain import AnyToolchain +def __no_ccache(cmd: str) -> str: + if cmd.startswith('ccache '): + cmd = cmd[7:] + return cmd + def make_cross_file(toolchain: AnyToolchain) -> str: if toolchain.is_windows: system = 'windows' @@ -40,8 +45,8 @@ def make_cross_file(toolchain: AnyToolchain) -> str: with open(path, 'w') as f: f.write(f""" [binaries] -c = '{toolchain.cc}' -cpp = '{toolchain.cxx}' +c = '{__no_ccache(toolchain.cc)}' +cpp = '{__no_ccache(toolchain.cxx)}' ar = '{toolchain.ar}' strip = '{toolchain.strip}' pkgconfig = '{toolchain.pkg_config}' From 23802f44897c968bc20756c4c351dca12c5dd440 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:28:22 +0200 Subject: [PATCH 14/30] python/build/tarball: Python type hints --- python/build/tar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/build/tar.py b/python/build/tar.py index 8247d2c22..208d0a606 100644 --- a/python/build/tar.py +++ b/python/build/tar.py @@ -1,6 +1,7 @@ import os, shutil, subprocess -def untar(tarball_path, parent_path, base, lazy=False): +def untar(tarball_path: str, parent_path: str, base: str, + lazy: bool=False) -> str: path = os.path.join(parent_path, base) if lazy and os.path.isdir(path): return path From 485c7805ebabe0985ea0f8fc83faa2610d7a3d87 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:30:38 +0200 Subject: [PATCH 15/30] python/build/autotools: use list.extend() to append configure_args --- python/build/autotools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/build/autotools.py b/python/build/autotools.py index 600bb627c..ae0f5f63e 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -54,7 +54,9 @@ class AutotoolsProject(MakeProject): '--host=' + toolchain.arch, '--prefix=' + toolchain.install_prefix, '--disable-silent-rules', - ] + self.configure_args + ] + + configure.extend(self.configure_args) try: print(configure) From a0892b852e5af2f6cdd029da0ff7c47ab2193e63 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 22 Sep 2023 20:11:57 +0200 Subject: [PATCH 16/30] build/python/autotools: add option `per_arch_cflags` --- python/build/autotools.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/build/autotools.py b/python/build/autotools.py index ae0f5f63e..a99aa5d4f 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -1,5 +1,6 @@ import os.path, subprocess, sys from typing import Collection, Iterable, Optional, Sequence, Union +from collections.abc import Mapping from build.makeproject import MakeProject from .toolchain import AnyToolchain @@ -9,6 +10,7 @@ class AutotoolsProject(MakeProject): configure_args: Iterable[str]=[], autogen: bool=False, autoreconf: bool=False, + per_arch_cflags: Optional[Mapping[str, str]]=None, cppflags: str='', ldflags: str='', libs: str='', @@ -18,6 +20,7 @@ class AutotoolsProject(MakeProject): self.configure_args = configure_args self.autogen = autogen self.autoreconf = autoreconf + self.per_arch_cflags = per_arch_cflags self.cppflags = cppflags self.ldflags = ldflags self.libs = libs @@ -38,12 +41,16 @@ class AutotoolsProject(MakeProject): build = self.make_build_path(toolchain) + arch_cflags = '' + if self.per_arch_cflags is not None and toolchain.host_triplet is not None: + arch_cflags = self.per_arch_cflags.get(toolchain.host_triplet, '') + configure = [ os.path.join(src, 'configure'), 'CC=' + toolchain.cc, 'CXX=' + toolchain.cxx, - 'CFLAGS=' + toolchain.cflags, - 'CXXFLAGS=' + toolchain.cxxflags, + 'CFLAGS=' + toolchain.cflags + ' ' + arch_cflags, + 'CXXFLAGS=' + toolchain.cxxflags + ' ' + arch_cflags, 'CPPFLAGS=' + toolchain.cppflags + ' ' + self.cppflags, 'LDFLAGS=' + toolchain.ldflags + ' ' + self.ldflags, 'LIBS=' + toolchain.libs + ' ' + self.libs, From 48cc76f114762deb677733f25259fd2f07dea3a0 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:46:59 +0200 Subject: [PATCH 17/30] python/build/toolchain: add is_android, is_darwin --- python/build/cmake.py | 4 ++-- python/build/meson.py | 2 +- python/build/toolchain.py | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/build/cmake.py b/python/build/cmake.py index 3b7e86405..53db47ddf 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -15,7 +15,7 @@ def __write_cmake_compiler(f: TextIO, language: str, compiler: str) -> None: print(f'set(CMAKE_{language}_COMPILER {compiler})', file=f) def __write_cmake_toolchain_file(f: TextIO, toolchain: AnyToolchain) -> None: - if '-darwin' in toolchain.actual_arch: + if toolchain.is_darwin: cmake_system_name = 'Darwin' elif toolchain.is_windows: cmake_system_name = 'Windows' @@ -65,7 +65,7 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[], # looking for libraries on the build host (TODO: fix this # properly); but we must not do that on Android because the NDK # has a sysroot already - if '-android' not in toolchain.actual_arch and '-darwin' not in toolchain.actual_arch: + if not toolchain.is_android and not toolchain.is_darwin: cross_args.append('-DCMAKE_SYSROOT=' + toolchain.install_prefix) os.makedirs(build, exist_ok=True) diff --git a/python/build/meson.py b/python/build/meson.py index 1a7311955..b6517d33b 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -63,7 +63,7 @@ pkgconfig = '{toolchain.pkg_config}' root = '{toolchain.install_prefix}' """) - if 'android' in toolchain.arch: + if toolchain.is_android: f.write(""" # Keep Meson from executing Android-x86 test binariees needs_exe_wrapper = true diff --git a/python/build/toolchain.py b/python/build/toolchain.py index 15a29ca20..3e615286f 100644 --- a/python/build/toolchain.py +++ b/python/build/toolchain.py @@ -91,6 +91,8 @@ class AndroidNdkToolchain: self.is_armv7 = self.is_arm and 'armv7' in self.cflags self.is_aarch64 = ndk_arch == 'arm64' self.is_windows = False + self.is_android = True + self.is_darwin = False libstdcxx_flags = '' libstdcxx_cxxflags = '' @@ -162,6 +164,8 @@ class MingwToolchain: self.is_armv7 = self.is_arm and 'armv7' in self.cflags self.is_aarch64 = arch == 'aarch64' self.is_windows = 'mingw32' in arch + self.is_android = False + self.is_darwin = False self.env = dict(os.environ) From 446f8f29d35d7192ef5b0b5e060e3bec60fcc7a6 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:02:01 +0200 Subject: [PATCH 18/30] python/build/openssl: add Darwin archs --- python/build/openssl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/build/openssl.py b/python/build/openssl.py index 81396ce02..790d2c00a 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -41,12 +41,16 @@ class OpenSSLProject(MakeProject): 'i686-linux-android': 'linux-x86-clang', 'x86_64-linux-android': 'linux-x86_64-clang', - # Kobo + # generic Linux 'arm-linux-gnueabihf': 'linux-generic32', # Windows 'i686-w64-mingw32': 'mingw', 'x86_64-w64-mingw32': 'mingw64', + + # Apple + 'x86_64-apple-darwin': 'darwin64-x86_64-cc', + 'aarch64-apple-darwin': 'darwin64-arm64-cc', } openssl_arch = openssl_archs[toolchain.arch] From c4da87a0cb2534e45bd169916a4de683689678e8 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:02:46 +0200 Subject: [PATCH 19/30] python/build/openssl: add `configure` variable --- python/build/openssl.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/python/build/openssl.py b/python/build/openssl.py index 790d2c00a..4af321996 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -55,13 +55,18 @@ class OpenSSLProject(MakeProject): openssl_arch = openssl_archs[toolchain.arch] - subprocess.check_call(['./Configure', - 'no-shared', - 'no-module', 'no-engine', 'no-static-engine', - 'no-async', - 'no-tests', - 'no-asm', # "asm" causes build failures on Windows - openssl_arch, - '--prefix=' + toolchain.install_prefix], - cwd=src, env=toolchain.env) + configure = [ + './Configure', + 'no-shared', + 'no-module', + 'no-engine', + 'no-static-engine', + 'no-async', + 'no-tests', + 'no-asm', # "asm" causes build failures on Windows + openssl_arch, + '--prefix=' + toolchain.install_prefix, + ] + + subprocess.check_call(configure, cwd=src, env=toolchain.env) self.build_make(toolchain, src) From 12dff8e382c7b63ee39a0513bfd1df1c3485a234 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:03:43 +0200 Subject: [PATCH 20/30] python/build/openssl: use `no-asm` only on Windows --- python/build/openssl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/build/openssl.py b/python/build/openssl.py index 4af321996..b0638e3e3 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -63,10 +63,13 @@ class OpenSSLProject(MakeProject): 'no-static-engine', 'no-async', 'no-tests', - 'no-asm', # "asm" causes build failures on Windows openssl_arch, '--prefix=' + toolchain.install_prefix, ] + if toolchain.is_windows: + # workaround for build failures + configure.append('no-asm') + subprocess.check_call(configure, cwd=src, env=toolchain.env) self.build_make(toolchain, src) From 6571b5d1181d18bef100bed060bfd0543c8a4241 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:05:04 +0200 Subject: [PATCH 21/30] python/build/openssl: add option "no-makedepend" We do not need "make" dependencies for one-time builds. --- python/build/openssl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/build/openssl.py b/python/build/openssl.py index b0638e3e3..2bf542c5b 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -63,6 +63,7 @@ class OpenSSLProject(MakeProject): 'no-static-engine', 'no-async', 'no-tests', + 'no-makedepend', openssl_arch, '--prefix=' + toolchain.install_prefix, ] From 9a5eac4ea931de5f9dbcfff4921e77fb77a86b9b Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:09:38 +0200 Subject: [PATCH 22/30] python/build/toolchain: set arch=llvm_triple --- python/build/openssl.py | 2 +- python/build/toolchain.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/python/build/openssl.py b/python/build/openssl.py index 2bf542c5b..953220ec9 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -36,7 +36,7 @@ class OpenSSLProject(MakeProject): # to know where the SDK is, but our own build scripts # prepared everything already to look like a regular Linux # build - 'arm-linux-androideabi': 'linux-generic32', + 'armv7a-linux-androideabi': 'linux-generic32', 'aarch64-linux-android': 'linux-aarch64', 'i686-linux-android': 'linux-x86-clang', 'x86_64-linux-android': 'linux-x86_64-clang', diff --git a/python/build/toolchain.py b/python/build/toolchain.py index 3e615286f..bfde287f5 100644 --- a/python/build/toolchain.py +++ b/python/build/toolchain.py @@ -4,30 +4,26 @@ from typing import Union android_abis = { 'armeabi-v7a': { - 'arch': 'arm-linux-androideabi', + 'arch': 'armv7a-linux-androideabi', 'ndk_arch': 'arm', - 'llvm_triple': 'armv7-linux-androideabi', 'cflags': '-fpic -mfpu=neon -mfloat-abi=softfp', }, 'arm64-v8a': { 'arch': 'aarch64-linux-android', 'ndk_arch': 'arm64', - 'llvm_triple': 'aarch64-linux-android', 'cflags': '-fpic', }, 'x86': { 'arch': 'i686-linux-android', 'ndk_arch': 'x86', - 'llvm_triple': 'i686-linux-android', 'cflags': '-fPIC -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32', }, 'x86_64': { 'arch': 'x86_64-linux-android', 'ndk_arch': 'x86_64', - 'llvm_triple': 'x86_64-linux-android', 'cflags': '-fPIC -m64', }, } @@ -60,7 +56,7 @@ class AndroidNdkToolchain: self.install_prefix = install_prefix llvm_path = os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', build_arch) - llvm_triple = abi_info['llvm_triple'] + android_api_level + llvm_triple = arch + android_api_level common_flags = '-Os -g' common_flags += ' ' + abi_info['cflags'] From da7f32bddb20e44291f5bea3fad31113e4be03cb Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:19:39 +0200 Subject: [PATCH 23/30] python/build/toolchain: rename `arch` to `host_triplet` --- python/build/autotools.py | 2 +- python/build/cmake.py | 6 +++--- python/build/meson.py | 2 +- python/build/openssl.py | 2 +- python/build/toolchain.py | 34 ++++++++++++++++------------------ 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/python/build/autotools.py b/python/build/autotools.py index a99aa5d4f..bf5d94a58 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -58,7 +58,7 @@ class AutotoolsProject(MakeProject): 'ARFLAGS=' + toolchain.arflags, 'RANLIB=' + toolchain.ranlib, 'STRIP=' + toolchain.strip, - '--host=' + toolchain.arch, + '--host=' + toolchain.host_triplet, '--prefix=' + toolchain.install_prefix, '--disable-silent-rules', ] diff --git a/python/build/cmake.py b/python/build/cmake.py index 53db47ddf..e48f4771e 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -24,10 +24,10 @@ def __write_cmake_toolchain_file(f: TextIO, toolchain: AnyToolchain) -> None: f.write(f""" set(CMAKE_SYSTEM_NAME {cmake_system_name}) -set(CMAKE_SYSTEM_PROCESSOR {toolchain.actual_arch.split('-', 1)[0]}) +set(CMAKE_SYSTEM_PROCESSOR {toolchain.host_triplet.split('-', 1)[0]}) -set(CMAKE_C_COMPILER_TARGET {toolchain.actual_arch}) -set(CMAKE_CXX_COMPILER_TARGET {toolchain.actual_arch}) +set(CMAKE_C_COMPILER_TARGET {toolchain.host_triplet}) +set(CMAKE_CXX_COMPILER_TARGET {toolchain.host_triplet}) set(CMAKE_C_FLAGS_INIT "{toolchain.cflags} {toolchain.cppflags}") set(CMAKE_CXX_FLAGS_INIT "{toolchain.cxxflags} {toolchain.cppflags}") diff --git a/python/build/meson.py b/python/build/meson.py index b6517d33b..43a52188e 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -30,7 +30,7 @@ def make_cross_file(toolchain: AnyToolchain) -> str: cpu = 'arm64-v8a' else: cpu_family = 'x86' - if 'x86_64' in toolchain.arch: + if 'x86_64' in toolchain.host_triplet: cpu = 'x86_64' else: cpu = 'i686' diff --git a/python/build/openssl.py b/python/build/openssl.py index 953220ec9..245a0d983 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -53,7 +53,7 @@ class OpenSSLProject(MakeProject): 'aarch64-apple-darwin': 'darwin64-arm64-cc', } - openssl_arch = openssl_archs[toolchain.arch] + openssl_arch = openssl_archs[toolchain.host_triplet] configure = [ './Configure', diff --git a/python/build/toolchain.py b/python/build/toolchain.py index bfde287f5..70ea5289b 100644 --- a/python/build/toolchain.py +++ b/python/build/toolchain.py @@ -38,9 +38,9 @@ class AndroidNdkToolchain: # select the NDK target abi_info = android_abis[android_abi] - arch = abi_info['arch'] + host_triplet = abi_info['arch'] - arch_path = os.path.join(lib_path, arch) + arch_path = os.path.join(lib_path, host_triplet) self.tarball_path = tarball_path self.src_path = src_path @@ -51,12 +51,11 @@ class AndroidNdkToolchain: install_prefix = os.path.join(arch_path, 'root') - self.arch = arch - self.actual_arch = arch + self.host_triplet = host_triplet self.install_prefix = install_prefix llvm_path = os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', build_arch) - llvm_triple = arch + android_api_level + llvm_triple = host_triplet + android_api_level common_flags = '-Os -g' common_flags += ' ' + abi_info['cflags'] @@ -118,24 +117,23 @@ class AndroidNdkToolchain: class MingwToolchain: def __init__(self, top_path: str, - toolchain_path, arch, x64: bool, + toolchain_path, host_triplet, x64: bool, tarball_path, src_path, build_path, install_prefix): - self.arch = arch - self.actual_arch = arch + self.host_triplet = host_triplet self.tarball_path = tarball_path self.src_path = src_path self.build_path = build_path self.install_prefix = install_prefix toolchain_bin = os.path.join(toolchain_path, 'bin') - self.cc = os.path.join(toolchain_bin, arch + '-gcc') - self.cxx = os.path.join(toolchain_bin, arch + '-g++') - self.ar = os.path.join(toolchain_bin, arch + '-ar') + self.cc = os.path.join(toolchain_bin, host_triplet + '-gcc') + self.cxx = os.path.join(toolchain_bin, host_triplet + '-g++') + self.ar = os.path.join(toolchain_bin, host_triplet + '-ar') self.arflags = 'rcs' - self.ranlib = os.path.join(toolchain_bin, arch + '-ranlib') - self.nm = os.path.join(toolchain_bin, arch + '-nm') - self.strip = os.path.join(toolchain_bin, arch + '-strip') - self.windres = os.path.join(toolchain_bin, arch + '-windres') + self.ranlib = os.path.join(toolchain_bin, host_triplet + '-ranlib') + self.nm = os.path.join(toolchain_bin, host_triplet + '-nm') + self.strip = os.path.join(toolchain_bin, host_triplet + '-strip') + self.windres = os.path.join(toolchain_bin, host_triplet + '-windres') common_flags = '-O2 -g' @@ -156,10 +154,10 @@ class MingwToolchain: # enable it. self.cppflags += ' -D_FORTIFY_SOURCE=0' - self.is_arm = arch.startswith('arm') + self.is_arm = host_triplet.startswith('arm') self.is_armv7 = self.is_arm and 'armv7' in self.cflags - self.is_aarch64 = arch == 'aarch64' - self.is_windows = 'mingw32' in arch + self.is_aarch64 = host_triplet == 'aarch64' + self.is_windows = 'mingw32' in host_triplet self.is_android = False self.is_darwin = False From 5b83c834ac012af8bf806188e1589b5619d8e547 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:26:16 +0200 Subject: [PATCH 24/30] python/build/meson: write cross-file only if cross-compiling --- python/build/meson.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/build/meson.py b/python/build/meson.py index 43a52188e..11e5471a3 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -88,7 +88,6 @@ endian = '{endian}' return path def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[]) -> None: - cross_file = make_cross_file(toolchain) configure = [ 'meson', 'setup', build, src, @@ -98,10 +97,13 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[]) '--buildtype', 'plain', '--default-library=static', - - '--cross-file', cross_file, ] + args + if toolchain.host_triplet is not None: + # cross-compiling: write a cross-file + cross_file = make_cross_file(toolchain) + configure.append(f'--cross-file={cross_file}') + env = toolchain.env.copy() # Meson 0.54 requires the BOOST_ROOT environment variable From 0ffbe5b5eacade0682259c9c408be3f11bc6a87f Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:26:41 +0200 Subject: [PATCH 25/30] python/build/autotools: add --host=... only if cross-compiling --- python/build/autotools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/build/autotools.py b/python/build/autotools.py index bf5d94a58..23c382c78 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -58,11 +58,13 @@ class AutotoolsProject(MakeProject): 'ARFLAGS=' + toolchain.arflags, 'RANLIB=' + toolchain.ranlib, 'STRIP=' + toolchain.strip, - '--host=' + toolchain.host_triplet, '--prefix=' + toolchain.install_prefix, '--disable-silent-rules', ] + if toolchain.host_triplet is not None: + configure.append('--host=' + toolchain.host_triplet) + configure.extend(self.configure_args) try: From 4d6f220a2fc1876d4b0fef6e5e614ebd40049d99 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:27:37 +0200 Subject: [PATCH 26/30] python/build/cmake: write toolchain file only if cross-compiling --- python/build/cmake.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/python/build/cmake.py b/python/build/cmake.py index e48f4771e..74d5f286a 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -56,35 +56,38 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) """) def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[], env: Optional[Mapping[str, str]]=None) -> None: - cross_args = [] + cross_args: list[str] = [] if toolchain.is_windows: cross_args.append('-DCMAKE_RC_COMPILER=' + cast(str, toolchain.windres)) - # Several targets need a sysroot to prevent pkg-config from - # looking for libraries on the build host (TODO: fix this - # properly); but we must not do that on Android because the NDK - # has a sysroot already - if not toolchain.is_android and not toolchain.is_darwin: - cross_args.append('-DCMAKE_SYSROOT=' + toolchain.install_prefix) - - os.makedirs(build, exist_ok=True) - cmake_toolchain_file = os.path.join(build, 'cmake_toolchain_file') - with open(cmake_toolchain_file, 'w') as f: - __write_cmake_toolchain_file(f, toolchain) - configure = [ 'cmake', src, - '-DCMAKE_TOOLCHAIN_FILE=' + cmake_toolchain_file, - '-DCMAKE_INSTALL_PREFIX=' + toolchain.install_prefix, '-DCMAKE_BUILD_TYPE=release', '-GNinja', ] + cross_args + args + if toolchain.host_triplet is not None: + # cross-compiling: write a toolchain file + os.makedirs(build, exist_ok=True) + + # Several targets need a sysroot to prevent pkg-config from + # looking for libraries on the build host (TODO: fix this + # properly); but we must not do that on Android because the NDK + # has a sysroot already + if not toolchain.is_android and not toolchain.is_darwin: + cross_args.append('-DCMAKE_SYSROOT=' + toolchain.install_prefix) + + cmake_toolchain_file = os.path.join(build, 'cmake_toolchain_file') + with open(cmake_toolchain_file, 'w') as f: + __write_cmake_toolchain_file(f, toolchain) + + configure.append('-DCMAKE_TOOLCHAIN_FILE=' + cmake_toolchain_file) + if env is None: env = toolchain.env else: From cf179ec294a49ccd6cd30935f25cfb40457710e6 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:30:22 +0200 Subject: [PATCH 27/30] python/build/openssl: pass --libdir=lib to Configure Without this, the AMD64 build installs to "lib64/" which then cannot be found by CURL. --- python/build/openssl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/build/openssl.py b/python/build/openssl.py index 245a0d983..88f221c9e 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -64,6 +64,7 @@ class OpenSSLProject(MakeProject): 'no-async', 'no-tests', 'no-makedepend', + '--libdir=lib', # no "lib64" on amd64, please openssl_arch, '--prefix=' + toolchain.install_prefix, ] From 2d25f6f57f78e6ebe7a16136fd75d3e3305848b5 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:31:00 +0200 Subject: [PATCH 28/30] python/build/openssl: pass arch only if cross-compiling --- python/build/openssl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/build/openssl.py b/python/build/openssl.py index 88f221c9e..a266db389 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -65,7 +65,6 @@ class OpenSSLProject(MakeProject): 'no-tests', 'no-makedepend', '--libdir=lib', # no "lib64" on amd64, please - openssl_arch, '--prefix=' + toolchain.install_prefix, ] @@ -73,5 +72,8 @@ class OpenSSLProject(MakeProject): # workaround for build failures configure.append('no-asm') + if toolchain.host_triplet is not None: + configure.append(openssl_archs[toolchain.host_triplet]) + subprocess.check_call(configure, cwd=src, env=toolchain.env) self.build_make(toolchain, src) From 94af199c49ee3475e7d8198bda5d7910359ab705 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:31:20 +0200 Subject: [PATCH 29/30] python/build/openssl: add --cross-compile-prefix=... --- python/build/openssl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/build/openssl.py b/python/build/openssl.py index a266db389..154658e33 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -74,6 +74,7 @@ class OpenSSLProject(MakeProject): if toolchain.host_triplet is not None: configure.append(openssl_archs[toolchain.host_triplet]) + configure.append(f'--cross-compile-prefix={toolchain.host_triplet}-') subprocess.check_call(configure, cwd=src, env=toolchain.env) self.build_make(toolchain, src) From 49130c201825cd277590ac53a2bf805fa9242001 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 15:32:40 +0200 Subject: [PATCH 30/30] python/build/openssl: remove obsolete variable --- python/build/openssl.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/build/openssl.py b/python/build/openssl.py index 154658e33..8b0224598 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -30,7 +30,7 @@ class OpenSSLProject(MakeProject): # OpenSSL has a weird target architecture scheme with lots of # hard-coded architectures; this table translates between our - # "toolchain_arch" (HOST_TRIPLET) and the OpenSSL target + # host triplet and the OpenSSL target openssl_archs = { # not using "android-*" because those OpenSSL targets want # to know where the SDK is, but our own build scripts @@ -53,8 +53,6 @@ class OpenSSLProject(MakeProject): 'aarch64-apple-darwin': 'darwin64-arm64-cc', } - openssl_arch = openssl_archs[toolchain.host_triplet] - configure = [ './Configure', 'no-shared',