From 9d3f1381bf243741547d0097b3c14903222c6203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98ystein=20Tveit?= Date: Wed, 10 Jul 2024 18:59:16 +0200 Subject: [PATCH] packages/mediawiki-extensions: refactor --- packages/mediawiki-extensions/default.nix | 66 ++++++++- .../delete-batch/default.nix | 14 -- .../pluggable-auth/default.nix | 14 -- .../simple-saml-php/default.nix | 14 -- .../update-mediawiki-extensions.py | 140 ++++++++++++------ .../user-merge/default.nix | 14 -- .../visual-editor/default.nix | 14 -- 7 files changed, 155 insertions(+), 121 deletions(-) delete mode 100644 packages/mediawiki-extensions/delete-batch/default.nix delete mode 100644 packages/mediawiki-extensions/pluggable-auth/default.nix delete mode 100644 packages/mediawiki-extensions/simple-saml-php/default.nix delete mode 100644 packages/mediawiki-extensions/user-merge/default.nix delete mode 100644 packages/mediawiki-extensions/visual-editor/default.nix diff --git a/packages/mediawiki-extensions/default.nix b/packages/mediawiki-extensions/default.nix index 43c67e5c..086f0816 100644 --- a/packages/mediawiki-extensions/default.nix +++ b/packages/mediawiki-extensions/default.nix @@ -1,8 +1,60 @@ { pkgs, lib }: -{ - DeleteBatch = pkgs.callPackage ./delete-batch { }; - PluggableAuth = pkgs.callPackage ./pluggable-auth { }; - SimpleSAMLphp = pkgs.callPackage ./simple-saml-php { }; - UserMerge = pkgs.callPackage ./user-merge { }; - VisualEditor = pkgs.callPackage ./visual-editor { }; -} +let + kebab-case-name = project-name: lib.pipe project-name [ + (builtins.replaceStrings + lib.upperChars + (map (x: "-${x}") lib.lowerChars) + ) + (lib.removePrefix "-") + ]; + + mw-ext = { + name + , commit + , hash + , tracking-branch ? "REL1_41" + , kebab-name ? kebab-case-name name + , fetchgit ? pkgs.fetchgit + }: + { + ${name} = (fetchgit { + name = "mediawiki-${kebab-name}-source"; + url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${name}"; + rev = commit; + inherit hash; + }).overrideAttrs (_: { + passthru = { inherit name kebab-name tracking-branch; }; + }); + }; +in +# NOTE: to add another extension, you can add an mw-ext expression +# with an empty (or even wrong) commit and empty hash, and +# run the update script +lib.mergeAttrsList [ + (mw-ext { + name = "DeleteBatch"; + commit = "cad869fbd95637902673f744581b29e0f3e3f61a"; + hash = "sha256-M1ek1WdO1/uTjeYlrk3Tz+nlb/fFZH+O0Ok7b10iKak="; + }) + (mw-ext { + name = "PluggableAuth"; + commit = "4111a57c34e25bde579cce5d14ea094021e450c8"; + hash = "sha256-aPtN8A9gDxLlq2+EloRZBO0DfHtE0E5kbV/adk82jvM="; + }) + (mw-ext { + name = "SimpleSAMLphp"; + kebab-name = "simple-saml-php"; + commit = "ecb47191fecd1e0dc4c9d8b90a9118e393d82c23"; + hash = "sha256-gKu+O49XrAVt6hXdt36Ru7snjsKX6g2CYJ0kk/d+CI8="; + }) + (mw-ext { + name = "UserMerge"; + commit = "c17c919bdb9b67bb69f80df43e9ee9d33b1ecf1b"; + hash = "sha256-+mkzTCo8RVlGoFyfCrSb5YMh4J6Pbi1PZLFu5ps8bWY="; + }) + (mw-ext { + name = "VisualEditor"; + commit = "90bb3d455892e25317029ffd4bda93159e8faac8"; + hash = "sha256-SZAVELQUKZtwSM6NVlxvIHdFPodko8fhZ/uwB0LCFDA="; + }) +] diff --git a/packages/mediawiki-extensions/delete-batch/default.nix b/packages/mediawiki-extensions/delete-batch/default.nix deleted file mode 100644 index 4ed3b82f..00000000 --- a/packages/mediawiki-extensions/delete-batch/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ fetchgit }: -let - commit = "cad869fbd95637902673f744581b29e0f3e3f61a"; - project-name = "DeleteBatch"; - tracking-branch = "REL1_41"; -in -(fetchgit { - name = "mediawiki-delete-batch-source"; - url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}"; - rev = "refs/heads/${tracking-branch}"; - hash = "sha256-M1ek1WdO1/uTjeYlrk3Tz+nlb/fFZH+O0Ok7b10iKak="; -}).overrideAttrs (_: { - passthru = { inherit project-name tracking-branch; }; -}) diff --git a/packages/mediawiki-extensions/pluggable-auth/default.nix b/packages/mediawiki-extensions/pluggable-auth/default.nix deleted file mode 100644 index 924e5f88..00000000 --- a/packages/mediawiki-extensions/pluggable-auth/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ fetchgit }: -let - commit = "4111a57c34e25bde579cce5d14ea094021e450c8"; - project-name = "PluggableAuth"; - tracking-branch = "REL1_41"; -in -(fetchgit { - name = "mediawiki-pluggable-auth-source"; - url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}"; - rev = "refs/heads/${tracking-branch}"; - hash = "sha256-aPtN8A9gDxLlq2+EloRZBO0DfHtE0E5kbV/adk82jvM="; -}).overrideAttrs (_: { - passthru = { inherit project-name tracking-branch; }; -}) diff --git a/packages/mediawiki-extensions/simple-saml-php/default.nix b/packages/mediawiki-extensions/simple-saml-php/default.nix deleted file mode 100644 index 28025e74..00000000 --- a/packages/mediawiki-extensions/simple-saml-php/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ fetchgit }: -let - commit = "ecb47191fecd1e0dc4c9d8b90a9118e393d82c23"; - project-name = "SimpleSAMLphp"; - tracking-branch = "REL1_41"; -in -(fetchgit { - name = "mediawiki-simple-saml-php-source"; - url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}"; - rev = "refs/heads/${tracking-branch}"; - hash = "sha256-gKu+O49XrAVt6hXdt36Ru7snjsKX6g2CYJ0kk/d+CI8="; -}).overrideAttrs (_: { - passthru = { inherit project-name tracking-branch; }; -}) diff --git a/packages/mediawiki-extensions/update-mediawiki-extensions.py b/packages/mediawiki-extensions/update-mediawiki-extensions.py index b133ba13..c3ce47a4 100755 --- a/packages/mediawiki-extensions/update-mediawiki-extensions.py +++ b/packages/mediawiki-extensions/update-mediawiki-extensions.py @@ -8,49 +8,87 @@ import subprocess from collections import defaultdict from pprint import pprint from dataclasses import dataclass +from functools import cache import json import bs4 import requests + BASE_WEB_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions" BASE_GIT_URL = "https://gerrit.wikimedia.org/r/mediawiki/extensions/" + @dataclass class PluginMetadata: project_name: str - tracking_branch: str + tracking_branch: str | None commit: str + hash_: str -def get_metadata(file_content: str) -> dict[str,str] | None: - commit_search = re.search(f'commit = "([^"]*?)";', file_content) - tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content) - project_name_search = re.search(f'project-name = "([^"]+?)";', file_content) - if commit_search is None: - print("Could not find commit in file:") - print(file_content) - return None - if tracking_branch_search is None: - print("Could not find tracking branch in file:") - print(file_content) - return None +@cache +def get_package_listing_path(): + return Path(__file__).parent / "default.nix" + + +@cache +def get_global_tracking_branch() -> str: + with open(get_package_listing_path()) as file: + file_content = file.read() + return re.search(r'\btracking-branch\b \? "([^"]+?)"', file_content).group(1) + + +def get_metadata(package_expression: str) -> PluginMetadata | None: + project_name_search = re.search(r'\bname\b = "([^"]+?)";', package_expression) + tracking_branch_search = re.search(r'\btracking-branch\b = "([^"]+?)";', package_expression) + commit_search = re.search(r'\bcommit\b = "([^"]*?)";', package_expression) + hash_search = re.search(r'\bhash\b = "([^"]*?)";', package_expression) + if project_name_search is None: - print("Could not find project name in file:") - print(file_content) + print("Could not find project name in package:") + print(package_expression) return None + + tracking_branch = None; + if tracking_branch_search is not None: + tracking_branch = tracking_branch_search.group(1) + + if commit_search is None: + print("Could not find commit in package:") + print(package_expression) + return None + + if hash_search is None: + print("Could not find hash in package:") + print(package_expression) + return None + return PluginMetadata( commit = commit_search.group(1), - tracking_branch = tracking_branch_search.group(1), + tracking_branch = tracking_branch, project_name = project_name_search.group(1), + hash_ = hash_search.group(1), ) +def update_metadata(package_expression: str, metadata: PluginMetadata) -> str: + result = package_expression + result = re.sub(r'\bcommit\b = "[^"]*";', f'commit = "{metadata.commit}";', result) + result = re.sub(r'\bhash\b = "[^"]*";', f'hash = "{metadata.hash_}";', result) + return result + + def get_newest_commit(project_name: str, tracking_branch: str) -> str: content = requests.get(f"{BASE_WEB_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text soup = bs4.BeautifulSoup(content, features="html.parser") - a = soup.find('li').findChild('a') - commit_sha = a['href'].split('/')[-1] + try: + a = soup.find('li').findChild('a') + commit_sha = a['href'].split('/')[-1] + except AttributeError: + print(f"ERROR: Could not parse page for {project_name}:") + print(soup.prettify()) + exit(1) return commit_sha @@ -64,40 +102,54 @@ def get_nix_hash(url: str, commit: str) -> str: return json.loads(out.decode().strip())['hash'] -def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str: - result = file_content - result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result) - result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result) - return result - -def update(package_file: Path) -> None: - with open(package_file) as file: - file_content = file.read() - - metadata = get_metadata(file_content) - if metadata is None: - print(f"ERROR: could not find metadata for {package_file}") +def update_expression(package_expression: str) -> str: + old_metadata = get_metadata(package_expression) + if old_metadata is None: + print("ERROR: could not find metadata for expression:") + print(package_expression) return - if metadata.commit == "": - metadata.commit = "" - new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch) - new_hash = get_nix_hash(f"{BASE_GIT_URL}/{metadata.project_name}", new_commit) + if old_metadata.commit == "": + old_metadata.commit = "" + if old_metadata.hash_ == "": + old_metadata.hash_ = "" + + tracking_branch = old_metadata.tracking_branch + if tracking_branch is None: + tracking_branch = get_global_tracking_branch() + + new_commit = get_newest_commit(old_metadata.project_name, tracking_branch) + new_hash = get_nix_hash(f"{BASE_GIT_URL}/{old_metadata.project_name}", new_commit) if new_hash is None or new_hash == "": - print(f"ERROR: could not fetch hash for {metadata.project_name}") + print(f"ERROR: could not fetch hash for {old_metadata.project_name}") exit(1) - print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}") + print(f"Updating {old_metadata.project_name}[{tracking_branch}]: {old_metadata.commit} -> {new_commit}") - new_file_content = set_commit_and_hash(file_content, new_commit, new_hash) + new_metadata = PluginMetadata( + project_name = old_metadata.project_name, + tracking_branch = old_metadata.tracking_branch, + commit = new_commit, + hash_ = new_hash, + ) - with open(package_file, 'w') as file: + return update_metadata(package_expression, new_metadata) + + +def update_all_expressions_in_default_nix() -> None: + with open(get_package_listing_path()) as file: + file_content = file.read() + + new_file_content = re.sub( + r"\(mw-ext\s*\{(?:.|\n)+?\}\)", + lambda m: update_expression(m.group(0)), + file_content, + flags = re.MULTILINE, + ) + + with open(get_package_listing_path(), 'w') as file: file.write(new_file_content) if __name__ == "__main__": - for direntry in os.scandir(Path(__file__).parent): - if direntry.is_dir(): - package_file = Path(direntry) / "default.nix" - assert package_file.is_file() - update(package_file) + update_all_expressions_in_default_nix() diff --git a/packages/mediawiki-extensions/user-merge/default.nix b/packages/mediawiki-extensions/user-merge/default.nix deleted file mode 100644 index 7f7d3902..00000000 --- a/packages/mediawiki-extensions/user-merge/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ fetchgit }: -let - commit = "c17c919bdb9b67bb69f80df43e9ee9d33b1ecf1b"; - project-name = "UserMerge"; - tracking-branch = "REL1_41"; -in -(fetchgit { - name = "mediawiki-user-merge-source"; - url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}"; - rev = "refs/heads/${tracking-branch}"; - hash = "sha256-+mkzTCo8RVlGoFyfCrSb5YMh4J6Pbi1PZLFu5ps8bWY="; -}).overrideAttrs (_: { - passthru = { inherit project-name tracking-branch; }; -}) diff --git a/packages/mediawiki-extensions/visual-editor/default.nix b/packages/mediawiki-extensions/visual-editor/default.nix deleted file mode 100644 index 2210a717..00000000 --- a/packages/mediawiki-extensions/visual-editor/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ fetchgit }: -let - commit = "170d19aad1f28dc6bd3f98ee277680cabba9db0c"; - project-name = "VisualEditor"; - tracking-branch = "REL1_41"; -in -(fetchgit { - name = "mediawiki-visual-editor-source"; - url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}"; - rev = "refs/heads/${tracking-branch}"; - hash = "sha256-5WVlO/OEk4eln5j/w4Tu/MXSmlvjIn7l6H+OTPaV+t4="; -}).overrideAttrs (_: { - passthru = { inherit project-name tracking-branch; }; -})