From eb23788fec8e4724a138544a36e1dc331911c35c Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 26 Sep 2023 14:14:33 +0200 Subject: [PATCH] python/build: add support for fallback download URLs --- python/build/autotools.py | 4 ++-- python/build/cmake.py | 4 ++-- python/build/download.py | 48 ++++++++++++++++++++++++++++++------- python/build/libs.py | 9 ++++--- python/build/makeproject.py | 4 ++-- python/build/meson.py | 4 ++-- python/build/openssl.py | 4 ++-- python/build/project.py | 8 +++---- python/build/zlib.py | 4 ++-- 9 files changed, 62 insertions(+), 27 deletions(-) diff --git a/python/build/autotools.py b/python/build/autotools.py index f8c4f42c4..600bb627c 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -1,11 +1,11 @@ import os.path, subprocess, sys -from typing import Collection, Iterable, Optional +from typing import Collection, Iterable, Optional, Sequence, Union from build.makeproject import MakeProject from .toolchain import AnyToolchain class AutotoolsProject(MakeProject): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, configure_args: Iterable[str]=[], autogen: bool=False, autoreconf: bool=False, diff --git a/python/build/cmake.py b/python/build/cmake.py index a26d7d6f8..1971a6cd8 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -1,7 +1,7 @@ import os import re import subprocess -from typing import Optional, TextIO +from typing import Optional, Sequence, TextIO, Union from collections.abc import Mapping from build.project import Project @@ -94,7 +94,7 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[], subprocess.check_call(configure, env=env, cwd=build) class CmakeProject(Project): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, configure_args: list[str]=[], windows_configure_args: list[str]=[], env: Optional[Mapping[str, str]]=None, diff --git a/python/build/download.py b/python/build/download.py index 3cc491e11..c170c841a 100644 --- a/python/build/download.py +++ b/python/build/download.py @@ -1,13 +1,50 @@ +from typing import Sequence, Union import os +import sys import urllib.request from .verify import verify_file_digest -def download_and_verify(url, md5, parent_path): +def __to_string_sequence(x: Union[str, Sequence[str]]) -> Sequence[str]: + if isinstance(x, str): + return (x,) + else: + return x + +def __get_any(x: Union[str, Sequence[str]]) -> str: + if isinstance(x, str): + return x + else: + return x[0] + +def __download_one(url: str, path: str) -> None: + print("download", url) + urllib.request.urlretrieve(url, path) + +def __download(urls: Sequence[str], path: str) -> None: + for url in urls[:-1]: + try: + __download_one(url, path) + return + except: + print("download error:", sys.exc_info()[0]) + __download_one(urls[-1], path) + +def __download_and_verify_to(urls: Sequence[str], md5: str, path: str) -> None: + __download(urls, path) + if not verify_file_digest(path, md5): + raise RuntimeError("Digest mismatch") + +def download_basename(urls: Union[str, Sequence[str]]) -> str: + return os.path.basename(__get_any(urls)) + +def download_and_verify(urls: Union[str, Sequence[str]], md5: str, parent_path: str) -> str: """Download a file, verify its MD5 checksum and return the local path.""" + base = download_basename(urls) + os.makedirs(parent_path, exist_ok=True) - path = os.path.join(parent_path, os.path.basename(url)) + path = os.path.join(parent_path, base) try: if verify_file_digest(path, md5): return path @@ -17,11 +54,6 @@ def download_and_verify(url, md5, parent_path): tmp_path = path + '.tmp' - print("download", url) - urllib.request.urlretrieve(url, tmp_path) - if not verify_file_digest(tmp_path, md5): - os.unlink(tmp_path) - raise RuntimeError("Digest mismatch") - + __download_and_verify_to(__to_string_sequence(urls), md5, tmp_path) os.rename(tmp_path, path) return path diff --git a/python/build/libs.py b/python/build/libs.py index 54f444d9f..15b7a0169 100644 --- a/python/build/libs.py +++ b/python/build/libs.py @@ -57,7 +57,8 @@ flac = AutotoolsProject( ) zlib = ZlibProject( - 'http://zlib.net/zlib-1.3.tar.xz', + ('http://zlib.net/zlib-1.3.tar.xz', + 'https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.xz'), '8a9ba2898e1d0d774eca6ba5b4627a11e5588ba85c8851336eb38de4683050a7', 'lib/libz.a', ) @@ -599,13 +600,15 @@ ffmpeg = FfmpegProject( ) openssl = OpenSSLProject( - 'https://www.openssl.org/source/openssl-3.1.3.tar.gz', + ('https://www.openssl.org/source/openssl-3.1.3.tar.gz', + 'https://artfiles.org/openssl.org/source/openssl-3.1.3.tar.gz'), 'f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6', 'include/openssl/ossl_typ.h', ) curl = CmakeProject( - 'https://curl.se/download/curl-8.2.1.tar.xz', + ('https://curl.se/download/curl-8.2.1.tar.xz', + 'https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.xz'), 'dd322f6bd0a20e6cebdfd388f69e98c3d183bed792cf4713c8a7ef498cba4894', 'lib/libcurl.a', [ diff --git a/python/build/makeproject.py b/python/build/makeproject.py index d8d0910ac..b0acf7807 100644 --- a/python/build/makeproject.py +++ b/python/build/makeproject.py @@ -1,11 +1,11 @@ import subprocess, multiprocessing -from typing import Optional +from typing import Optional, Sequence, Union from build.project import Project from .toolchain import AnyToolchain class MakeProject(Project): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, install_target: str='install', **kwargs): Project.__init__(self, url, md5, installed, **kwargs) diff --git a/python/build/meson.py b/python/build/meson.py index ffcb5d38a..6469e4d58 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -1,7 +1,7 @@ import os import subprocess import platform -from typing import Optional +from typing import Optional, Sequence, Union from build.project import Project from .toolchain import AnyToolchain @@ -105,7 +105,7 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[]) subprocess.check_call(configure, env=env) class MesonProject(Project): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, configure_args: list[str]=[], **kwargs): Project.__init__(self, url, md5, installed, **kwargs) diff --git a/python/build/openssl.py b/python/build/openssl.py index df63f2738..81396ce02 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -1,11 +1,11 @@ import subprocess -from typing import Optional +from typing import Optional, Sequence, Union from build.makeproject import MakeProject from .toolchain import AnyToolchain class OpenSSLProject(MakeProject): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, install_target='install_dev', **kwargs) diff --git a/python/build/project.py b/python/build/project.py index 12791a05a..f636b5118 100644 --- a/python/build/project.py +++ b/python/build/project.py @@ -1,21 +1,21 @@ import os, shutil import re -from typing import cast, BinaryIO, Optional +from typing import cast, BinaryIO, Optional, Sequence, Union -from build.download import download_and_verify +from build.download import download_basename, download_and_verify from build.tar import untar from build.quilt import push_all from .toolchain import AnyToolchain class Project: - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, name: Optional[str]=None, version: Optional[str]=None, base: Optional[str]=None, patches: Optional[str]=None, edits=None, use_cxx: bool=False): if base is None: - basename = os.path.basename(url) + basename = download_basename(url) m = re.match(r'^(.+)\.(tar(\.(gz|bz2|xz|lzma))?|zip)$', basename) if not m: raise RuntimeError('Could not identify tarball name: ' + basename) self.base = m.group(1) diff --git a/python/build/zlib.py b/python/build/zlib.py index 2007c6c6d..41138c825 100644 --- a/python/build/zlib.py +++ b/python/build/zlib.py @@ -1,11 +1,11 @@ import subprocess -from typing import Optional +from typing import Optional, Sequence, Union from build.makeproject import MakeProject from .toolchain import AnyToolchain class ZlibProject(MakeProject): - def __init__(self, url: str, md5: str, installed: str, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, **kwargs)