packages/mediawiki-extensions: use stable url

packages/mediawiki-extensions/update-mediawiki-extensions.py

@@ -7,36 +7,54 @@ import re
 import subprocess
 from collections import defaultdict
 from pprint import pprint
+from dataclasses import dataclass
 
 import bs4
 import requests
 
-BASE_URL = "https://extdist.wmflabs.org/dist/extensions"
+BASE_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions"
 
-def fetch_plugin_list(skip_master=True) -> dict[str, list[str]]:
-    content = requests.get(BASE_URL).text
+@dataclass
+class PluginMetadata:
+    project_name: str
+    tracking_branch: str
+    commit: str
+
+
+def get_metadata(file_content: str) -> dict[str,str] | None:
+    commit_search = re.search(f'commit = "([^"]*?)";', file_content)
+    tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content)
+    project_name_search = re.search(f'project-name = "([^"]+?)";', file_content)
+    if commit_search is None:
+        print("Could not find commit in file:")
+        print(file_content)
+        return None
+    if tracking_branch_search is None:
+        print("Could not find tracking branch in file:")
+        print(file_content)
+        return None
+    if project_name_search is None:
+        print("Could not find project name in file:")
+        print(file_content)
+        return None
+    return PluginMetadata(
+        commit = commit_search.group(1),
+        tracking_branch = tracking_branch_search.group(1),
+        project_name = project_name_search.group(1),
+    )
+
+
+def get_newest_commit(project_name: str, tracking_branch: str) -> str:
+    content = requests.get(f"{BASE_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text
     soup = bs4.BeautifulSoup(content, features="html.parser")
-    result = defaultdict(list)
-    for a in soup.find_all('a'):
-        if skip_master and 'master' in a.text:
-            continue
-        split = a.text.split('-')
-        result[split[0]].append(a.text)
-    return result
+    a = soup.find('li').findChild('a')
+    commit_sha = a['href'].split('/')[-1]
+    return commit_sha
 
-def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None:
-    assert package_file.is_file()
-    with open(package_file) as file:
-        content = file.read()
-
-    tarball = re.search(f'url = "{BASE_URL}/(.+\.tar\.gz)";', content).group(1)
-    split = tarball.split('-')
-    updated_tarball = plugin_list[split[0]][-1]
-
-    _hash = re.search(f'hash = "(.+?)";', content).group(1)
 
+def get_nix_hash(tar_gz_url: str) -> str:
     out, err = subprocess.Popen(
-        ["nix-prefetch-url", "--unpack", "--type", "sha256", f"{BASE_URL}/{updated_tarball}"],
+        ["nix-prefetch-url", "--unpack", "--type", "sha256", tar_gz_url],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE
     ).communicate()
@@ -46,21 +64,43 @@ def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None:
         stderr=subprocess.PIPE
     ).communicate()
 
-    updated_hash = out.decode().strip()
+    return out.decode().strip()
 
-    if tarball == updated_tarball and _hash == updated_hash:
+
+def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str:
+    result = file_content
+    result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result)
+    result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result)
+    return result
+
+def update(package_file: Path) -> None:
+    with open(package_file) as file:
+        file_content = file.read()
+
+    metadata = get_metadata(file_content)
+    if metadata is None:
+        return
+    if metadata.commit == "":
+        metadata.commit = "<none>"
+
+    new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch)
+    if new_commit == metadata.commit:
         return
 
-    print(f"Updating: {tarball} ({_hash[7:14]}) -> {updated_tarball} ({updated_hash[7:14]})")
+    new_url = f"{BASE_URL}/{metadata.project_name}/+archive/{new_commit}.tar.gz"
+    new_hash = get_nix_hash(new_url)
+
+    print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}")
+
+    new_file_content = set_commit_and_hash(file_content, new_commit, new_hash)
 
-    updated_text = re.sub(f'url = "{BASE_URL}/.+?\.tar\.gz";', f'url = "{BASE_URL}/{updated_tarball}";', content)
-    updated_text = re.sub('hash = ".+";', f'hash = "{updated_hash}";', updated_text)
     with open(package_file, 'w') as file:
-        file.write(updated_text)
+        file.write(new_file_content)
+
 
 if __name__ == "__main__":
-    plugin_list = fetch_plugin_list()
-
     for direntry in os.scandir(Path(__file__).parent):
         if direntry.is_dir():
-            update(Path(direntry) / "default.nix", plugin_list)
+            package_file = Path(direntry) / "default.nix"
+            assert package_file.is_file()
+            update(package_file)