packages/mediawiki-extensions: refactor

packages/mediawiki-extensions/default.nix

@@ -1,8 +1,60 @@
 { pkgs, lib }:
+let
+  kebab-case-name = project-name: lib.pipe project-name [
+    (builtins.replaceStrings
+      lib.upperChars
+      (map (x: "-${x}") lib.lowerChars)
+    )
+    (lib.removePrefix "-")
+  ];
+
+  mw-ext = {
+    name
+  , commit
+  , hash
+  , tracking-branch ? "REL1_41"
+  , kebab-name ? kebab-case-name name
+  , fetchgit ? pkgs.fetchgit
+  }:
   {
-  DeleteBatch = pkgs.callPackage ./delete-batch { };
+    ${name} = (fetchgit {
-  PluggableAuth = pkgs.callPackage ./pluggable-auth { };
+      name = "mediawiki-${kebab-name}-source";
-  SimpleSAMLphp = pkgs.callPackage ./simple-saml-php { };
+      url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${name}";
-  UserMerge = pkgs.callPackage ./user-merge { };
+      rev = commit;
-  VisualEditor = pkgs.callPackage ./visual-editor { };
+      inherit hash;
-}
+    }).overrideAttrs (_: {
+      passthru = { inherit name kebab-name tracking-branch; };
+    });
+  };
+in
+# NOTE: to add another extension, you can add an mw-ext expression
+#       with an empty (or even wrong) commit and empty hash, and
+#       run the update script
+lib.mergeAttrsList [
+  (mw-ext {
+    name = "DeleteBatch";
+    commit = "cad869fbd95637902673f744581b29e0f3e3f61a";
+    hash = "sha256-M1ek1WdO1/uTjeYlrk3Tz+nlb/fFZH+O0Ok7b10iKak=";
+  })
+  (mw-ext {
+    name = "PluggableAuth";
+    commit = "4111a57c34e25bde579cce5d14ea094021e450c8";
+    hash = "sha256-aPtN8A9gDxLlq2+EloRZBO0DfHtE0E5kbV/adk82jvM=";
+  })
+  (mw-ext {
+    name = "SimpleSAMLphp";
+    kebab-name = "simple-saml-php";
+    commit = "ecb47191fecd1e0dc4c9d8b90a9118e393d82c23";
+    hash = "sha256-gKu+O49XrAVt6hXdt36Ru7snjsKX6g2CYJ0kk/d+CI8=";
+  })
+  (mw-ext {
+    name = "UserMerge";
+    commit = "c17c919bdb9b67bb69f80df43e9ee9d33b1ecf1b";
+    hash = "sha256-+mkzTCo8RVlGoFyfCrSb5YMh4J6Pbi1PZLFu5ps8bWY=";
+  })
+  (mw-ext {
+    name = "VisualEditor";
+    commit = "90bb3d455892e25317029ffd4bda93159e8faac8";
+    hash = "sha256-SZAVELQUKZtwSM6NVlxvIHdFPodko8fhZ/uwB0LCFDA=";
+  })
+]

packages/mediawiki-extensions/delete-batch/default.nix

@@ -1,14 +0,0 @@
-{ fetchgit }:
-let
-  commit = "cad869fbd95637902673f744581b29e0f3e3f61a";
-  project-name = "DeleteBatch";
-  tracking-branch = "REL1_41";
-in
-(fetchgit {
-  name = "mediawiki-delete-batch-source";
-  url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
-  rev = "refs/heads/${tracking-branch}";
-  hash = "sha256-M1ek1WdO1/uTjeYlrk3Tz+nlb/fFZH+O0Ok7b10iKak=";
-}).overrideAttrs (_: {
-  passthru = { inherit project-name tracking-branch; };
-})

packages/mediawiki-extensions/pluggable-auth/default.nix

@@ -1,14 +0,0 @@
-{ fetchgit }:
-let
-  commit = "4111a57c34e25bde579cce5d14ea094021e450c8";
-  project-name = "PluggableAuth";
-  tracking-branch = "REL1_41";
-in
-(fetchgit {
-  name = "mediawiki-pluggable-auth-source";
-  url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
-  rev = "refs/heads/${tracking-branch}";
-  hash = "sha256-aPtN8A9gDxLlq2+EloRZBO0DfHtE0E5kbV/adk82jvM=";
-}).overrideAttrs (_: {
-  passthru = { inherit project-name tracking-branch; };
-})

packages/mediawiki-extensions/simple-saml-php/default.nix

@@ -1,14 +0,0 @@
-{ fetchgit }:
-let
-  commit = "ecb47191fecd1e0dc4c9d8b90a9118e393d82c23";
-  project-name = "SimpleSAMLphp";
-  tracking-branch = "REL1_41";
-in
-(fetchgit {
-  name = "mediawiki-simple-saml-php-source";
-  url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
-  rev = "refs/heads/${tracking-branch}";
-  hash = "sha256-gKu+O49XrAVt6hXdt36Ru7snjsKX6g2CYJ0kk/d+CI8=";
-}).overrideAttrs (_: {
-  passthru = { inherit project-name tracking-branch; };
-})

packages/mediawiki-extensions/update-mediawiki-extensions.py

@@ -8,49 +8,87 @@ import subprocess
 from collections import defaultdict
 from pprint import pprint
 from dataclasses import dataclass
+from functools import cache
 import json
 
 import bs4
 import requests
 
+
 BASE_WEB_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions"
 BASE_GIT_URL = "https://gerrit.wikimedia.org/r/mediawiki/extensions/"
 
+
 @dataclass
 class PluginMetadata:
     project_name: str
-    tracking_branch: str
+    tracking_branch: str | None
     commit: str
+    hash_: str
 
 
-def get_metadata(file_content: str) -> dict[str,str] | None:
+@cache
-    commit_search = re.search(f'commit = "([^"]*?)";', file_content)
+def get_package_listing_path():
-    tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content)
+    return Path(__file__).parent / "default.nix"
-    project_name_search = re.search(f'project-name = "([^"]+?)";', file_content)
+
-    if commit_search is None:
+
-        print("Could not find commit in file:")
+@cache
-        print(file_content)
+def get_global_tracking_branch() -> str:
-        return None
+    with open(get_package_listing_path()) as file:
-    if tracking_branch_search is None:
+        file_content = file.read()
-        print("Could not find tracking branch in file:")
+    return re.search(r'\btracking-branch\b \? "([^"]+?)"', file_content).group(1)
-        print(file_content)
+
-        return None
+
+def get_metadata(package_expression: str) -> PluginMetadata | None:
+    project_name_search = re.search(r'\bname\b = "([^"]+?)";', package_expression)
+    tracking_branch_search = re.search(r'\btracking-branch\b = "([^"]+?)";', package_expression)
+    commit_search = re.search(r'\bcommit\b = "([^"]*?)";', package_expression)
+    hash_search = re.search(r'\bhash\b = "([^"]*?)";', package_expression)
+
     if project_name_search is None:
-        print("Could not find project name in file:")
+        print("Could not find project name in package:")
-        print(file_content)
+        print(package_expression)
         return None
+
+    tracking_branch = None;
+    if tracking_branch_search is not None:
+        tracking_branch = tracking_branch_search.group(1)
+
+    if commit_search is None:
+        print("Could not find commit in package:")
+        print(package_expression)
+        return None
+
+    if hash_search is None:
+        print("Could not find hash in package:")
+        print(package_expression)
+        return None
+
     return PluginMetadata(
         commit = commit_search.group(1),
-        tracking_branch = tracking_branch_search.group(1),
+        tracking_branch = tracking_branch,
         project_name = project_name_search.group(1),
+        hash_ = hash_search.group(1),
     )
 
 
+def update_metadata(package_expression: str, metadata: PluginMetadata) -> str:
+    result = package_expression
+    result = re.sub(r'\bcommit\b = "[^"]*";', f'commit = "{metadata.commit}";', result)
+    result = re.sub(r'\bhash\b = "[^"]*";', f'hash = "{metadata.hash_}";', result)
+    return result
+
+
 def get_newest_commit(project_name: str, tracking_branch: str) -> str:
     content = requests.get(f"{BASE_WEB_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text
     soup = bs4.BeautifulSoup(content, features="html.parser")
+    try:
         a = soup.find('li').findChild('a')
         commit_sha = a['href'].split('/')[-1]
+    except AttributeError:
+        print(f"ERROR: Could not parse page for {project_name}:")
+        print(soup.prettify())
+        exit(1)
     return commit_sha
 
 
@@ -64,40 +102,54 @@ def get_nix_hash(url: str, commit: str) -> str:
     return json.loads(out.decode().strip())['hash']
 
 
-def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str:
+def update_expression(package_expression: str) -> str:
-    result = file_content
+    old_metadata = get_metadata(package_expression)
-    result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result)
+    if old_metadata is None:
-    result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result)
+        print("ERROR: could not find metadata for expression:")
-    return result
+        print(package_expression)
-
-def update(package_file: Path) -> None:
-    with open(package_file) as file:
-        file_content = file.read()
-
-    metadata = get_metadata(file_content)
-    if metadata is None:
-        print(f"ERROR: could not find metadata for {package_file}")
         return
-    if metadata.commit == "":
-        metadata.commit = "<none>"
 
-    new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch)
+    if old_metadata.commit == "":
-    new_hash = get_nix_hash(f"{BASE_GIT_URL}/{metadata.project_name}", new_commit)
+        old_metadata.commit = "<none>"
+    if old_metadata.hash_ == "":
+        old_metadata.hash_ = "<none>"
+
+    tracking_branch = old_metadata.tracking_branch
+    if tracking_branch is None:
+        tracking_branch = get_global_tracking_branch()
+
+    new_commit = get_newest_commit(old_metadata.project_name, tracking_branch)
+    new_hash = get_nix_hash(f"{BASE_GIT_URL}/{old_metadata.project_name}", new_commit)
     if new_hash is None or new_hash == "":
-        print(f"ERROR: could not fetch hash for {metadata.project_name}")
+        print(f"ERROR: could not fetch hash for {old_metadata.project_name}")
         exit(1)
 
-    print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}")
+    print(f"Updating {old_metadata.project_name}[{tracking_branch}]: {old_metadata.commit} -> {new_commit}")
 
-    new_file_content = set_commit_and_hash(file_content, new_commit, new_hash)
+    new_metadata = PluginMetadata(
+        project_name = old_metadata.project_name,
+        tracking_branch = old_metadata.tracking_branch,
+        commit = new_commit,
+        hash_ = new_hash,
+    )
 
-    with open(package_file, 'w') as file:
+    return update_metadata(package_expression, new_metadata)
+
+
+def update_all_expressions_in_default_nix() -> None:
+    with open(get_package_listing_path()) as file:
+        file_content = file.read()
+
+    new_file_content = re.sub(
+        r"\(mw-ext\s*\{(?:.|\n)+?\}\)",
+        lambda m: update_expression(m.group(0)),
+        file_content,
+        flags = re.MULTILINE,
+    )
+
+    with open(get_package_listing_path(), 'w') as file:
         file.write(new_file_content)
 
 
 if __name__ == "__main__":
-    for direntry in os.scandir(Path(__file__).parent):
+    update_all_expressions_in_default_nix()
-        if direntry.is_dir():
-            package_file = Path(direntry) / "default.nix"
-            assert package_file.is_file()
-            update(package_file)

packages/mediawiki-extensions/user-merge/default.nix

@@ -1,14 +0,0 @@
-{ fetchgit }:
-let
-  commit = "c17c919bdb9b67bb69f80df43e9ee9d33b1ecf1b";
-  project-name = "UserMerge";
-  tracking-branch = "REL1_41";
-in
-(fetchgit {
-  name = "mediawiki-user-merge-source";
-  url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
-  rev = "refs/heads/${tracking-branch}";
-  hash = "sha256-+mkzTCo8RVlGoFyfCrSb5YMh4J6Pbi1PZLFu5ps8bWY=";
-}).overrideAttrs (_: {
-  passthru = { inherit project-name tracking-branch; };
-})

packages/mediawiki-extensions/visual-editor/default.nix

@@ -1,14 +0,0 @@
-{ fetchgit }:
-let
-  commit = "170d19aad1f28dc6bd3f98ee277680cabba9db0c";
-  project-name = "VisualEditor";
-  tracking-branch = "REL1_41";
-in
-(fetchgit {
-  name = "mediawiki-visual-editor-source";
-  url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
-  rev = "refs/heads/${tracking-branch}";
-  hash = "sha256-5WVlO/OEk4eln5j/w4Tu/MXSmlvjIn7l6H+OTPaV+t4=";
-}).overrideAttrs (_: {
-  passthru = { inherit project-name tracking-branch; };
-})