From b476745de6478b7fe08389dc9dd6dc11698ac5b3 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Sun, 14 Apr 2024 19:26:11 +0200 Subject: [PATCH] packages/mediawiki-extensions: use stable url --- flake.nix | 2 +- .../delete-batch/default.nix | 12 +- .../pluggable-auth/default.nix | 12 +- .../simple-saml-php/default.nix | 12 +- .../update-mediawiki-extensions.py | 111 +++++++++++++----- .../user-merge/default.nix | 12 +- .../visual-editor/default.nix | 12 +- 7 files changed, 127 insertions(+), 46 deletions(-) diff --git a/flake.nix b/flake.nix index 814801d..2310ce7 100644 --- a/flake.nix +++ b/flake.nix @@ -142,7 +142,7 @@ simplesamlphp = pkgs.callPackage ./packages/simplesamlphp { }; - # mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { }; + mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { }; } // nixlib.genAttrs allMachines (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel); }; diff --git a/packages/mediawiki-extensions/delete-batch/default.nix b/packages/mediawiki-extensions/delete-batch/default.nix index dc7de57..23b176f 100644 --- a/packages/mediawiki-extensions/delete-batch/default.nix +++ b/packages/mediawiki-extensions/delete-batch/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "a53af3b8269ed19ede3cf1fa811e7ec8cb00af92"; + project-name = "UserMerge"; + tracking-branch = "REL1_41"; +in fetchzip { name = "mediawiki-delete-batch"; - url = "https://extdist.wmflabs.org/dist/extensions/DeleteBatch-REL1_41-5774fdd.tar.gz"; - hash = "sha256-ROkn93lf0mNXBvij9X2pMhd8LXZ0azOz7ZRaqZvhh8k="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; + hash = "sha256-0ofCZhhv4aVTGq469Fdu7k0oVQu3kG3HFa8zaBbUr/M="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } diff --git a/packages/mediawiki-extensions/pluggable-auth/default.nix b/packages/mediawiki-extensions/pluggable-auth/default.nix index 58b793e..077c5ac 100644 --- a/packages/mediawiki-extensions/pluggable-auth/default.nix +++ b/packages/mediawiki-extensions/pluggable-auth/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "d5b3ad8f03b65d3746e025cdd7fe3254ad6e4026"; + project-name = "PluggableAuth"; + tracking-branch = "REL1_41"; +in fetchzip { name = "mediawiki-pluggable-auth-source"; - url = "https://extdist.wmflabs.org/dist/extensions/PluggableAuth-REL1_41-d5b3ad8.tar.gz"; - hash = "sha256-OLlkKeSlfNgWXWwDdINrYRZpYuSGRwzZHgU8EYW6rYU="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project_name}/+archive/${commit}.tar.gz"; + hash = "sha256-mLepavgeaNUGYxrrCKVpybGO2ecjc3B5IU8q+gZTx2U="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } diff --git a/packages/mediawiki-extensions/simple-saml-php/default.nix b/packages/mediawiki-extensions/simple-saml-php/default.nix index 59ef5fa..ce1800e 100644 --- a/packages/mediawiki-extensions/simple-saml-php/default.nix +++ b/packages/mediawiki-extensions/simple-saml-php/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "9ae0678d77a9175285a1cfadd5adf28379dbdb3d"; + project-name = "SimpleSAMLphp"; + tracking-branch = "REL1_41"; +in fetchzip { name = "mediawiki-simple-saml-php-source"; - url = "https://extdist.wmflabs.org/dist/extensions/SimpleSAMLphp-REL1_41-9ae0678.tar.gz"; - hash = "sha256-AmCaG5QXMJvi3N6zFyWylwYDt8GvyIk/0GFpM1Y0vkY="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; + hash = "sha256-s6Uw1fNzGBF0HEMl0LIRLhJkOHugrCE0aTnqawYi/pE="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } diff --git a/packages/mediawiki-extensions/update-mediawiki-extensions.py b/packages/mediawiki-extensions/update-mediawiki-extensions.py index 099b18d..3509c7e 100755 --- a/packages/mediawiki-extensions/update-mediawiki-extensions.py +++ b/packages/mediawiki-extensions/update-mediawiki-extensions.py @@ -7,36 +7,65 @@ import re import subprocess from collections import defaultdict from pprint import pprint +from dataclasses import dataclass import bs4 import requests -BASE_URL = "https://extdist.wmflabs.org/dist/extensions" +BASE_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions" -def fetch_plugin_list(skip_master=True) -> dict[str, list[str]]: - content = requests.get(BASE_URL).text +# def fetch_plugin_list(skip_master=True) -> dict[str, list[str]]: +# content = requests.get(BASE_URL).text +# soup = bs4.BeautifulSoup(content, features="html.parser") +# result = defaultdict(list) +# for a in soup.find_all('a'): +# if skip_master and 'master' in a.text: +# continue +# split = a.text.split('-') +# result[split[0]].append(a.text) +# return result + +@dataclass +class PluginMetadata: + project_name: str + tracking_branch: str + commit: str + + +def get_metadata(file_content: str) -> dict[str,str] | None: + commit_search = re.search(f'commit = "([^"]*?)";', file_content) + tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content) + project_name_search = re.search(f'project-name = "([^"]+?)";', file_content) + if commit_search is None: + print("Could not find commit in file:") + print(file_content) + return None + if tracking_branch_search is None: + print("Could not find tracking branch in file:") + print(file_content) + return None + if project_name_search is None: + print("Could not find project name in file:") + print(file_content) + return None + return PluginMetadata( + commit = commit_search.group(1), + tracking_branch = tracking_branch_search.group(1), + project_name = project_name_search.group(1), + ) + + +def get_newest_commit(project_name: str, tracking_branch: str) -> str: + content = requests.get(f"{BASE_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text soup = bs4.BeautifulSoup(content, features="html.parser") - result = defaultdict(list) - for a in soup.find_all('a'): - if skip_master and 'master' in a.text: - continue - split = a.text.split('-') - result[split[0]].append(a.text) - return result + a = soup.find('li').findChild('a') + commit_sha = a['href'].split('/')[-1] + return commit_sha -def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None: - assert package_file.is_file() - with open(package_file) as file: - content = file.read() - - tarball = re.search(f'url = "{BASE_URL}/(.+\.tar\.gz)";', content).group(1) - split = tarball.split('-') - updated_tarball = plugin_list[split[0]][-1] - - _hash = re.search(f'hash = "(.+?)";', content).group(1) +def get_nix_hash(tar_gz_url: str) -> str: out, err = subprocess.Popen( - ["nix-prefetch-url", "--unpack", "--type", "sha256", f"{BASE_URL}/{updated_tarball}"], + ["nix-prefetch-url", "--unpack", "--type", "sha256", tar_gz_url], stdout=subprocess.PIPE, stderr=subprocess.PIPE ).communicate() @@ -46,21 +75,43 @@ def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None: stderr=subprocess.PIPE ).communicate() - updated_hash = out.decode().strip() + return out.decode().strip() - if tarball == updated_tarball and _hash == updated_hash: + +def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str: + result = file_content + result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result) + result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result) + return result + +def update(package_file: Path) -> None: + with open(package_file) as file: + file_content = file.read() + + metadata = get_metadata(file_content) + if metadata is None: + return + if metadata.commit == "": + metadata.commit = "" + + new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch) + if new_commit == metadata.commit: return - print(f"Updating: {tarball} ({_hash[7:14]}) -> {updated_tarball} ({updated_hash[7:14]})") + new_url = f"{BASE_URL}/{metadata.project_name}/+archive/{new_commit}.tar.gz" + new_hash = get_nix_hash(new_url) + + print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}") + + new_file_content = set_commit_and_hash(file_content, new_commit, new_hash) - updated_text = re.sub(f'url = "{BASE_URL}/.+?\.tar\.gz";', f'url = "{BASE_URL}/{updated_tarball}";', content) - updated_text = re.sub('hash = ".+";', f'hash = "{updated_hash}";', updated_text) with open(package_file, 'w') as file: - file.write(updated_text) + file.write(new_file_content) + if __name__ == "__main__": - plugin_list = fetch_plugin_list() - for direntry in os.scandir(Path(__file__).parent): if direntry.is_dir(): - update(Path(direntry) / "default.nix", plugin_list) + package_file = Path(direntry) / "default.nix" + assert package_file.is_file() + update(package_file) diff --git a/packages/mediawiki-extensions/user-merge/default.nix b/packages/mediawiki-extensions/user-merge/default.nix index d918297..a5918a6 100644 --- a/packages/mediawiki-extensions/user-merge/default.nix +++ b/packages/mediawiki-extensions/user-merge/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "a53af3b8269ed19ede3cf1fa811e7ec8cb00af92"; + project-name = "UserMerge"; + tracking-branch = "REL1_41"; +in fetchzip { name = "mediawiki-user-merge-source"; - url = "https://extdist.wmflabs.org/dist/extensions/UserMerge-REL1_41-a53af3b.tar.gz"; - hash = "sha256-TxUkEqMW79thYl1la2r+w9laRnd3uSYYg1xDB+1he1g="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; + hash = "sha256-0ofCZhhv4aVTGq469Fdu7k0oVQu3kG3HFa8zaBbUr/M="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } diff --git a/packages/mediawiki-extensions/visual-editor/default.nix b/packages/mediawiki-extensions/visual-editor/default.nix index cfb9a0c..b365a32 100644 --- a/packages/mediawiki-extensions/visual-editor/default.nix +++ b/packages/mediawiki-extensions/visual-editor/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "bb92d4b0bb81cebd73a3dbabfb497213dac349f2"; + project-name = "VisualEditor"; + tracking-branch = "REL1_40"; +in fetchzip { name = "mediawiki-visual-editor-source"; - url = "https://extdist.wmflabs.org/dist/extensions/VisualEditor-REL1_40-5f8c97e.tar.gz"; - hash = "sha256-oBMmEDKsFxrD0tpN2dy264IXK164BrZWrNK3v3FNX6w="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; + hash = "sha256-lShpSoR+NLfdd5i7soM6J40pq+MzCMG0M1tSYsS+jAg="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; }