python/build: add support for fallback download URLs

python/build/autotools.py

@@ -1,11 +1,11 @@
 import os.path, subprocess, sys
-from typing import Collection, Iterable, Optional
+from typing import Collection, Iterable, Optional, Sequence, Union
 
 from build.makeproject import MakeProject
 from .toolchain import AnyToolchain
 
 class AutotoolsProject(MakeProject):
-    def __init__(self, url: str, md5: str, installed: str,
+    def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str,
                  configure_args: Iterable[str]=[],
                  autogen: bool=False,
                  autoreconf: bool=False,

python/build/cmake.py

@@ -1,7 +1,7 @@
 import os
 import re
 import subprocess
-from typing import Optional, TextIO
+from typing import Optional, Sequence, TextIO, Union
 from collections.abc import Mapping
 
 from build.project import Project
@@ -94,7 +94,7 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[],
     subprocess.check_call(configure, env=env, cwd=build)
 
 class CmakeProject(Project):
-    def __init__(self, url: str, md5: str, installed: str,
+    def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str,
                  configure_args: list[str]=[],
                  windows_configure_args: list[str]=[],
                  env: Optional[Mapping[str, str]]=None,

python/build/download.py

@@ -1,13 +1,50 @@
+from typing import Sequence, Union
 import os
+import sys
 import urllib.request
 
 from .verify import verify_file_digest
 
-def download_and_verify(url, md5, parent_path):
+def __to_string_sequence(x: Union[str, Sequence[str]]) -> Sequence[str]:
+    if isinstance(x, str):
+        return (x,)
+    else:
+        return x
+
+def __get_any(x: Union[str, Sequence[str]]) -> str:
+    if isinstance(x, str):
+        return x
+    else:
+        return x[0]
+
+def __download_one(url: str, path: str) -> None:
+    print("download", url)
+    urllib.request.urlretrieve(url, path)
+
+def __download(urls: Sequence[str], path: str) -> None:
+    for url in urls[:-1]:
+        try:
+            __download_one(url, path)
+            return
+        except:
+            print("download error:", sys.exc_info()[0])
+    __download_one(urls[-1], path)
+
+def __download_and_verify_to(urls: Sequence[str], md5: str, path: str) -> None:
+    __download(urls, path)
+    if not verify_file_digest(path, md5):
+        raise RuntimeError("Digest mismatch")
+
+def download_basename(urls: Union[str, Sequence[str]]) -> str:
+    return os.path.basename(__get_any(urls))
+
+def download_and_verify(urls: Union[str, Sequence[str]], md5: str, parent_path: str) -> str:
     """Download a file, verify its MD5 checksum and return the local path."""
 
+    base = download_basename(urls)
+
     os.makedirs(parent_path, exist_ok=True)
-    path = os.path.join(parent_path, os.path.basename(url))
+    path = os.path.join(parent_path, base)
 
     try:
         if verify_file_digest(path, md5): return path
@@ -17,11 +54,6 @@ def download_and_verify(url, md5, parent_path):
 
     tmp_path = path + '.tmp'
 
-    print("download", url)
-    urllib.request.urlretrieve(url, tmp_path)
-    if not verify_file_digest(tmp_path, md5):
-        os.unlink(tmp_path)
-        raise RuntimeError("Digest mismatch")
-
+    __download_and_verify_to(__to_string_sequence(urls), md5, tmp_path)
     os.rename(tmp_path, path)
     return path

python/build/libs.py

@@ -57,7 +57,8 @@ flac = AutotoolsProject(
 )
 
 zlib = ZlibProject(
-    'http://zlib.net/zlib-1.3.tar.xz',
+    ('http://zlib.net/zlib-1.3.tar.xz',
+     'https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.xz'),
     '8a9ba2898e1d0d774eca6ba5b4627a11e5588ba85c8851336eb38de4683050a7',
     'lib/libz.a',
 )
@@ -599,13 +600,15 @@ ffmpeg = FfmpegProject(
 )
 
 openssl = OpenSSLProject(
-    'https://www.openssl.org/source/openssl-3.1.3.tar.gz',
+    ('https://www.openssl.org/source/openssl-3.1.3.tar.gz',
+     'https://artfiles.org/openssl.org/source/openssl-3.1.3.tar.gz'),
     'f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6',
     'include/openssl/ossl_typ.h',
 )
 
 curl = CmakeProject(
-    'https://curl.se/download/curl-8.2.1.tar.xz',
+    ('https://curl.se/download/curl-8.2.1.tar.xz',
+     'https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.xz'),
     'dd322f6bd0a20e6cebdfd388f69e98c3d183bed792cf4713c8a7ef498cba4894',
     'lib/libcurl.a',
     [

python/build/makeproject.py

@@ -1,11 +1,11 @@
 import subprocess, multiprocessing
-from typing import Optional
+from typing import Optional, Sequence, Union
 
 from build.project import Project
 from .toolchain import AnyToolchain
 
 class MakeProject(Project):
-    def __init__(self, url: str, md5: str, installed: str,
+    def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str,
                  install_target: str='install',
                  **kwargs):
         Project.__init__(self, url, md5, installed, **kwargs)

python/build/meson.py

@@ -1,7 +1,7 @@
 import os
 import subprocess
 import platform
-from typing import Optional
+from typing import Optional, Sequence, Union
 
 from build.project import Project
 from .toolchain import AnyToolchain
@@ -105,7 +105,7 @@ def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[])
     subprocess.check_call(configure, env=env)
 
 class MesonProject(Project):
-    def __init__(self, url: str, md5: str, installed: str,
+    def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str,
                  configure_args: list[str]=[],
                  **kwargs):
         Project.__init__(self, url, md5, installed, **kwargs)

python/build/openssl.py

@@ -1,11 +1,11 @@
 import subprocess
-from typing import Optional
+from typing import Optional, Sequence, Union
 
 from build.makeproject import MakeProject
 from .toolchain import AnyToolchain
 
 class OpenSSLProject(MakeProject):
-    def __init__(self, url: str, md5: str, installed: str,
+    def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str,
                  **kwargs):
         MakeProject.__init__(self, url, md5, installed, install_target='install_dev', **kwargs)
 

python/build/project.py

@@ -1,21 +1,21 @@
 import os, shutil
 import re
-from typing import cast, BinaryIO, Optional
+from typing import cast, BinaryIO, Optional, Sequence, Union
 
-from build.download import download_and_verify
+from build.download import download_basename, download_and_verify
 from build.tar import untar
 from build.quilt import push_all
 from .toolchain import AnyToolchain
 
 class Project:
-    def __init__(self, url: str, md5: str, installed: str,
+    def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str,
                  name: Optional[str]=None, version: Optional[str]=None,
                  base: Optional[str]=None,
                  patches: Optional[str]=None,
                  edits=None,
                  use_cxx: bool=False):
         if base is None:
-            basename = os.path.basename(url)
+            basename = download_basename(url)
             m = re.match(r'^(.+)\.(tar(\.(gz|bz2|xz|lzma))?|zip)$', basename)
             if not m: raise RuntimeError('Could not identify tarball name: ' + basename)
             self.base = m.group(1)

python/build/zlib.py

@@ -1,11 +1,11 @@
 import subprocess
-from typing import Optional
+from typing import Optional, Sequence, Union
 
 from build.makeproject import MakeProject
 from .toolchain import AnyToolchain
 
 class ZlibProject(MakeProject):
-    def __init__(self, url: str, md5: str, installed: str,
+    def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str,
                  **kwargs):
         MakeProject.__init__(self, url, md5, installed, **kwargs)