Merge pull request 'packages/mediawiki-extensions: use stable url' (!35) from fix-mediawiki-extensions-url into main

Reviewed-on: #35

flake.nix

@@ -140,8 +140,13 @@
 
         simplesamlphp = pkgs.callPackage ./packages/simplesamlphp { };
 
-        # mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { };
-      } // nixlib.genAttrs allMachines
+      } //
+      (nixlib.pipe null [
+        (_: pkgs.callPackage ./packages/mediawiki-extensions { })
+        (nixlib.flip builtins.removeAttrs ["override" "overrideDerivation"])
+        (nixlib.mapAttrs' (name: nixlib.nameValuePair "mediawiki-${name}"))
+      ])
+      // nixlib.genAttrs allMachines
         (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel);
     };
   };

packages/mediawiki-extensions/default.nix

@@ -1,8 +1,8 @@
 { pkgs, lib }:
-lib.makeScope pkgs.newScope (self: {
-  DeleteBatch = self.callPackage ./delete-batch { };
-  PluggableAuth = self.callPackage ./pluggable-auth { };
-  SimpleSAMLphp = self.callPackage ./simple-saml-php { };
-  UserMerge = self.callPackage ./user-merge { };
-  VisualEditor = self.callPackage ./visual-editor { };
-})
+{
+  DeleteBatch = pkgs.callPackage ./delete-batch { };
+  PluggableAuth = pkgs.callPackage ./pluggable-auth { };
+  SimpleSAMLphp = pkgs.callPackage ./simple-saml-php { };
+  UserMerge = pkgs.callPackage ./user-merge { };
+  VisualEditor = pkgs.callPackage ./visual-editor { };
+}

packages/mediawiki-extensions/delete-batch/default.nix

@@ -1,7 +1,13 @@
 { fetchzip }:
-
+let
+  commit = "a53af3b8269ed19ede3cf1fa811e7ec8cb00af92";
+  project-name = "UserMerge";
+  tracking-branch = "REL1_41";
+in
 fetchzip {
   name = "mediawiki-delete-batch";
-  url = "https://extdist.wmflabs.org/dist/extensions/DeleteBatch-REL1_41-5774fdd.tar.gz";
-  hash = "sha256-ROkn93lf0mNXBvij9X2pMhd8LXZ0azOz7ZRaqZvhh8k=";
+  url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
+  hash = "sha256-0ofCZhhv4aVTGq469Fdu7k0oVQu3kG3HFa8zaBbUr/M=";
+  stripRoot = false;
+  passthru = { inherit project-name tracking-branch; };
 }

packages/mediawiki-extensions/pluggable-auth/default.nix

@@ -1,7 +1,13 @@
 { fetchzip }:
-
+let
+  commit = "d5b3ad8f03b65d3746e025cdd7fe3254ad6e4026";
+  project-name = "PluggableAuth";
+  tracking-branch = "REL1_41";
+in
 fetchzip {
   name = "mediawiki-pluggable-auth-source";
-  url = "https://extdist.wmflabs.org/dist/extensions/PluggableAuth-REL1_41-d5b3ad8.tar.gz";
-  hash = "sha256-OLlkKeSlfNgWXWwDdINrYRZpYuSGRwzZHgU8EYW6rYU=";
+  url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
+  hash = "sha256-mLepavgeaNUGYxrrCKVpybGO2ecjc3B5IU8q+gZTx2U=";
+  stripRoot = false;
+  passthru = { inherit project-name tracking-branch; };
 }

packages/mediawiki-extensions/simple-saml-php/default.nix

@@ -1,7 +1,13 @@
 { fetchzip }:
-
+let
+  commit = "9ae0678d77a9175285a1cfadd5adf28379dbdb3d";
+  project-name = "SimpleSAMLphp";
+  tracking-branch = "REL1_41";
+in
 fetchzip {
   name = "mediawiki-simple-saml-php-source";
-  url = "https://extdist.wmflabs.org/dist/extensions/SimpleSAMLphp-REL1_41-9ae0678.tar.gz";
-  hash = "sha256-AmCaG5QXMJvi3N6zFyWylwYDt8GvyIk/0GFpM1Y0vkY=";
+  url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
+  hash = "sha256-s6Uw1fNzGBF0HEMl0LIRLhJkOHugrCE0aTnqawYi/pE=";
+  stripRoot = false;
+  passthru = { inherit project-name tracking-branch; };
 }

packages/mediawiki-extensions/update-mediawiki-extensions.py

@@ -7,36 +7,54 @@ import re
 import subprocess
 from collections import defaultdict
 from pprint import pprint
+from dataclasses import dataclass
 
 import bs4
 import requests
 
-BASE_URL = "https://extdist.wmflabs.org/dist/extensions"
+BASE_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions"
 
-def fetch_plugin_list(skip_master=True) -> dict[str, list[str]]:
-    content = requests.get(BASE_URL).text
+@dataclass
+class PluginMetadata:
+    project_name: str
+    tracking_branch: str
+    commit: str
+
+
+def get_metadata(file_content: str) -> dict[str,str] | None:
+    commit_search = re.search(f'commit = "([^"]*?)";', file_content)
+    tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content)
+    project_name_search = re.search(f'project-name = "([^"]+?)";', file_content)
+    if commit_search is None:
+        print("Could not find commit in file:")
+        print(file_content)
+        return None
+    if tracking_branch_search is None:
+        print("Could not find tracking branch in file:")
+        print(file_content)
+        return None
+    if project_name_search is None:
+        print("Could not find project name in file:")
+        print(file_content)
+        return None
+    return PluginMetadata(
+        commit = commit_search.group(1),
+        tracking_branch = tracking_branch_search.group(1),
+        project_name = project_name_search.group(1),
+    )
+
+
+def get_newest_commit(project_name: str, tracking_branch: str) -> str:
+    content = requests.get(f"{BASE_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text
     soup = bs4.BeautifulSoup(content, features="html.parser")
-    result = defaultdict(list)
-    for a in soup.find_all('a'):
-        if skip_master and 'master' in a.text:
-            continue
-        split = a.text.split('-')
-        result[split[0]].append(a.text)
-    return result
+    a = soup.find('li').findChild('a')
+    commit_sha = a['href'].split('/')[-1]
+    return commit_sha
 
-def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None:
-    assert package_file.is_file()
-    with open(package_file) as file:
-        content = file.read()
-
-    tarball = re.search(f'url = "{BASE_URL}/(.+\.tar\.gz)";', content).group(1)
-    split = tarball.split('-')
-    updated_tarball = plugin_list[split[0]][-1]
-
-    _hash = re.search(f'hash = "(.+?)";', content).group(1)
 
+def get_nix_hash(tar_gz_url: str) -> str:
     out, err = subprocess.Popen(
-        ["nix-prefetch-url", "--unpack", "--type", "sha256", f"{BASE_URL}/{updated_tarball}"],
+        ["nix-prefetch-url", "--unpack", "--type", "sha256", tar_gz_url],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE
     ).communicate()
@@ -46,21 +64,43 @@ def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None:
         stderr=subprocess.PIPE
     ).communicate()
 
-    updated_hash = out.decode().strip()
+    return out.decode().strip()
 
-    if tarball == updated_tarball and _hash == updated_hash:
+
+def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str:
+    result = file_content
+    result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result)
+    result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result)
+    return result
+
+def update(package_file: Path) -> None:
+    with open(package_file) as file:
+        file_content = file.read()
+
+    metadata = get_metadata(file_content)
+    if metadata is None:
+        return
+    if metadata.commit == "":
+        metadata.commit = "<none>"
+
+    new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch)
+    if new_commit == metadata.commit:
         return
 
-    print(f"Updating: {tarball} ({_hash[7:14]}) -> {updated_tarball} ({updated_hash[7:14]})")
+    new_url = f"{BASE_URL}/{metadata.project_name}/+archive/{new_commit}.tar.gz"
+    new_hash = get_nix_hash(new_url)
+
+    print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}")
+
+    new_file_content = set_commit_and_hash(file_content, new_commit, new_hash)
 
-    updated_text = re.sub(f'url = "{BASE_URL}/.+?\.tar\.gz";', f'url = "{BASE_URL}/{updated_tarball}";', content)
-    updated_text = re.sub('hash = ".+";', f'hash = "{updated_hash}";', updated_text)
     with open(package_file, 'w') as file:
-        file.write(updated_text)
+        file.write(new_file_content)
+
 
 if __name__ == "__main__":
-    plugin_list = fetch_plugin_list()
-
     for direntry in os.scandir(Path(__file__).parent):
         if direntry.is_dir():
-            update(Path(direntry) / "default.nix", plugin_list)
+            package_file = Path(direntry) / "default.nix"
+            assert package_file.is_file()
+            update(package_file)

packages/mediawiki-extensions/user-merge/default.nix

@@ -1,7 +1,13 @@
 { fetchzip }:
-
+let
+  commit = "a53af3b8269ed19ede3cf1fa811e7ec8cb00af92";
+  project-name = "UserMerge";
+  tracking-branch = "REL1_41";
+in
 fetchzip {
   name = "mediawiki-user-merge-source";
-  url = "https://extdist.wmflabs.org/dist/extensions/UserMerge-REL1_41-a53af3b.tar.gz";
-  hash = "sha256-TxUkEqMW79thYl1la2r+w9laRnd3uSYYg1xDB+1he1g=";
+  url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
+  hash = "sha256-0ofCZhhv4aVTGq469Fdu7k0oVQu3kG3HFa8zaBbUr/M=";
+  stripRoot = false;
+  passthru = { inherit project-name tracking-branch; };
 }

packages/mediawiki-extensions/visual-editor/default.nix

@@ -1,7 +1,13 @@
 { fetchzip }:
-
+let
+  commit = "bb92d4b0bb81cebd73a3dbabfb497213dac349f2";
+  project-name = "VisualEditor";
+  tracking-branch = "REL1_40";
+in
 fetchzip {
   name = "mediawiki-visual-editor-source";
-  url = "https://extdist.wmflabs.org/dist/extensions/VisualEditor-REL1_40-5f8c97e.tar.gz";
-  hash = "sha256-oBMmEDKsFxrD0tpN2dy264IXK164BrZWrNK3v3FNX6w=";
+  url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
+  hash = "sha256-lShpSoR+NLfdd5i7soM6J40pq+MzCMG0M1tSYsS+jAg=";
+  stripRoot = false;
+  passthru = { inherit project-name tracking-branch; };
 }