pvv-nixos-config/packages/mediawiki-extensions/update-mediawiki-extensions.py

156 lines
4.8 KiB
Python
Raw Normal View History

#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages(ps: with ps; [ beautifulsoup4 requests ])" nix-prefetch-git
import os
from pathlib import Path
import re
import subprocess
from collections import defaultdict
from pprint import pprint
from dataclasses import dataclass
from functools import cache
import json
import bs4
import requests
BASE_WEB_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions"
BASE_GIT_URL = "https://gerrit.wikimedia.org/r/mediawiki/extensions/"
@dataclass
class PluginMetadata:
project_name: str
tracking_branch: str | None
commit: str
hash_: str
@cache
def get_package_listing_path():
return Path(__file__).parent / "default.nix"
@cache
def get_global_tracking_branch() -> str:
with open(get_package_listing_path()) as file:
file_content = file.read()
return re.search(r'\btracking-branch\b \? "([^"]+?)"', file_content).group(1)
def get_metadata(package_expression: str) -> PluginMetadata | None:
project_name_search = re.search(r'\bname\b = "([^"]+?)";', package_expression)
tracking_branch_search = re.search(r'\btracking-branch\b = "([^"]+?)";', package_expression)
commit_search = re.search(r'\bcommit\b = "([^"]*?)";', package_expression)
hash_search = re.search(r'\bhash\b = "([^"]*?)";', package_expression)
if project_name_search is None:
print("Could not find project name in package:")
print(package_expression)
return None
tracking_branch = None;
if tracking_branch_search is not None:
tracking_branch = tracking_branch_search.group(1)
if commit_search is None:
print("Could not find commit in package:")
print(package_expression)
return None
if hash_search is None:
print("Could not find hash in package:")
print(package_expression)
return None
return PluginMetadata(
commit = commit_search.group(1),
tracking_branch = tracking_branch,
project_name = project_name_search.group(1),
hash_ = hash_search.group(1),
)
def update_metadata(package_expression: str, metadata: PluginMetadata) -> str:
result = package_expression
result = re.sub(r'\bcommit\b = "[^"]*";', f'commit = "{metadata.commit}";', result)
result = re.sub(r'\bhash\b = "[^"]*";', f'hash = "{metadata.hash_}";', result)
return result
def get_newest_commit(project_name: str, tracking_branch: str) -> str:
content = requests.get(f"{BASE_WEB_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text
soup = bs4.BeautifulSoup(content, features="html.parser")
try:
a = soup.find('li').findChild('a')
commit_sha = a['href'].split('/')[-1]
except AttributeError:
print(f"ERROR: Could not parse page for {project_name}:")
print(soup.prettify())
exit(1)
return commit_sha
def get_nix_hash(url: str, commit: str) -> str:
out, err = subprocess.Popen(
["nix-prefetch-git", "--url", url, "--rev", commit, "--fetch-submodules", "--quiet"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
return json.loads(out.decode().strip())['hash']
def update_expression(package_expression: str) -> str:
old_metadata = get_metadata(package_expression)
if old_metadata is None:
print("ERROR: could not find metadata for expression:")
print(package_expression)
return
if old_metadata.commit == "":
old_metadata.commit = "<none>"
if old_metadata.hash_ == "":
old_metadata.hash_ = "<none>"
tracking_branch = old_metadata.tracking_branch
if tracking_branch is None:
tracking_branch = get_global_tracking_branch()
new_commit = get_newest_commit(old_metadata.project_name, tracking_branch)
new_hash = get_nix_hash(f"{BASE_GIT_URL}/{old_metadata.project_name}", new_commit)
if new_hash is None or new_hash == "":
print(f"ERROR: could not fetch hash for {old_metadata.project_name}")
exit(1)
print(f"Updating {old_metadata.project_name}[{tracking_branch}]: {old_metadata.commit} -> {new_commit}")
new_metadata = PluginMetadata(
project_name = old_metadata.project_name,
tracking_branch = old_metadata.tracking_branch,
commit = new_commit,
hash_ = new_hash,
)
return update_metadata(package_expression, new_metadata)
def update_all_expressions_in_default_nix() -> None:
with open(get_package_listing_path()) as file:
file_content = file.read()
new_file_content = re.sub(
r"\(mw-ext\s*\{(?:.|\n)+?\}\)",
lambda m: update_expression(m.group(0)),
file_content,
flags = re.MULTILINE,
)
with open(get_package_listing_path(), 'w') as file:
file.write(new_file_content)
if __name__ == "__main__":
update_all_expressions_in_default_nix()