1
0
Fork 0

packages/mediawiki-extensions: refactor

This commit is contained in:
Oystein Kristoffer Tveit 2024-07-10 18:59:16 +02:00
parent f3a29429aa
commit 9d3f1381bf
7 changed files with 155 additions and 121 deletions

View File

@ -1,8 +1,60 @@
{ pkgs, lib }:
{
DeleteBatch = pkgs.callPackage ./delete-batch { };
PluggableAuth = pkgs.callPackage ./pluggable-auth { };
SimpleSAMLphp = pkgs.callPackage ./simple-saml-php { };
UserMerge = pkgs.callPackage ./user-merge { };
VisualEditor = pkgs.callPackage ./visual-editor { };
}
let
kebab-case-name = project-name: lib.pipe project-name [
(builtins.replaceStrings
lib.upperChars
(map (x: "-${x}") lib.lowerChars)
)
(lib.removePrefix "-")
];
mw-ext = {
name
, commit
, hash
, tracking-branch ? "REL1_41"
, kebab-name ? kebab-case-name name
, fetchgit ? pkgs.fetchgit
}:
{
${name} = (fetchgit {
name = "mediawiki-${kebab-name}-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${name}";
rev = commit;
inherit hash;
}).overrideAttrs (_: {
passthru = { inherit name kebab-name tracking-branch; };
});
};
in
# NOTE: to add another extension, you can add an mw-ext expression
# with an empty (or even wrong) commit and empty hash, and
# run the update script
lib.mergeAttrsList [
(mw-ext {
name = "DeleteBatch";
commit = "cad869fbd95637902673f744581b29e0f3e3f61a";
hash = "sha256-M1ek1WdO1/uTjeYlrk3Tz+nlb/fFZH+O0Ok7b10iKak=";
})
(mw-ext {
name = "PluggableAuth";
commit = "4111a57c34e25bde579cce5d14ea094021e450c8";
hash = "sha256-aPtN8A9gDxLlq2+EloRZBO0DfHtE0E5kbV/adk82jvM=";
})
(mw-ext {
name = "SimpleSAMLphp";
kebab-name = "simple-saml-php";
commit = "ecb47191fecd1e0dc4c9d8b90a9118e393d82c23";
hash = "sha256-gKu+O49XrAVt6hXdt36Ru7snjsKX6g2CYJ0kk/d+CI8=";
})
(mw-ext {
name = "UserMerge";
commit = "c17c919bdb9b67bb69f80df43e9ee9d33b1ecf1b";
hash = "sha256-+mkzTCo8RVlGoFyfCrSb5YMh4J6Pbi1PZLFu5ps8bWY=";
})
(mw-ext {
name = "VisualEditor";
commit = "90bb3d455892e25317029ffd4bda93159e8faac8";
hash = "sha256-SZAVELQUKZtwSM6NVlxvIHdFPodko8fhZ/uwB0LCFDA=";
})
]

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "cad869fbd95637902673f744581b29e0f3e3f61a";
project-name = "DeleteBatch";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-delete-batch-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-M1ek1WdO1/uTjeYlrk3Tz+nlb/fFZH+O0Ok7b10iKak=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "4111a57c34e25bde579cce5d14ea094021e450c8";
project-name = "PluggableAuth";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-pluggable-auth-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-aPtN8A9gDxLlq2+EloRZBO0DfHtE0E5kbV/adk82jvM=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "ecb47191fecd1e0dc4c9d8b90a9118e393d82c23";
project-name = "SimpleSAMLphp";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-simple-saml-php-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-gKu+O49XrAVt6hXdt36Ru7snjsKX6g2CYJ0kk/d+CI8=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})

View File

@ -8,49 +8,87 @@ import subprocess
from collections import defaultdict
from pprint import pprint
from dataclasses import dataclass
from functools import cache
import json
import bs4
import requests
BASE_WEB_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions"
BASE_GIT_URL = "https://gerrit.wikimedia.org/r/mediawiki/extensions/"
@dataclass
class PluginMetadata:
project_name: str
tracking_branch: str
tracking_branch: str | None
commit: str
hash_: str
def get_metadata(file_content: str) -> dict[str,str] | None:
commit_search = re.search(f'commit = "([^"]*?)";', file_content)
tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content)
project_name_search = re.search(f'project-name = "([^"]+?)";', file_content)
if commit_search is None:
print("Could not find commit in file:")
print(file_content)
return None
if tracking_branch_search is None:
print("Could not find tracking branch in file:")
print(file_content)
return None
@cache
def get_package_listing_path():
return Path(__file__).parent / "default.nix"
@cache
def get_global_tracking_branch() -> str:
with open(get_package_listing_path()) as file:
file_content = file.read()
return re.search(r'\btracking-branch\b \? "([^"]+?)"', file_content).group(1)
def get_metadata(package_expression: str) -> PluginMetadata | None:
project_name_search = re.search(r'\bname\b = "([^"]+?)";', package_expression)
tracking_branch_search = re.search(r'\btracking-branch\b = "([^"]+?)";', package_expression)
commit_search = re.search(r'\bcommit\b = "([^"]*?)";', package_expression)
hash_search = re.search(r'\bhash\b = "([^"]*?)";', package_expression)
if project_name_search is None:
print("Could not find project name in file:")
print(file_content)
print("Could not find project name in package:")
print(package_expression)
return None
tracking_branch = None;
if tracking_branch_search is not None:
tracking_branch = tracking_branch_search.group(1)
if commit_search is None:
print("Could not find commit in package:")
print(package_expression)
return None
if hash_search is None:
print("Could not find hash in package:")
print(package_expression)
return None
return PluginMetadata(
commit = commit_search.group(1),
tracking_branch = tracking_branch_search.group(1),
tracking_branch = tracking_branch,
project_name = project_name_search.group(1),
hash_ = hash_search.group(1),
)
def update_metadata(package_expression: str, metadata: PluginMetadata) -> str:
result = package_expression
result = re.sub(r'\bcommit\b = "[^"]*";', f'commit = "{metadata.commit}";', result)
result = re.sub(r'\bhash\b = "[^"]*";', f'hash = "{metadata.hash_}";', result)
return result
def get_newest_commit(project_name: str, tracking_branch: str) -> str:
content = requests.get(f"{BASE_WEB_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text
soup = bs4.BeautifulSoup(content, features="html.parser")
try:
a = soup.find('li').findChild('a')
commit_sha = a['href'].split('/')[-1]
except AttributeError:
print(f"ERROR: Could not parse page for {project_name}:")
print(soup.prettify())
exit(1)
return commit_sha
@ -64,40 +102,54 @@ def get_nix_hash(url: str, commit: str) -> str:
return json.loads(out.decode().strip())['hash']
def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str:
result = file_content
result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result)
result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result)
return result
def update(package_file: Path) -> None:
with open(package_file) as file:
file_content = file.read()
metadata = get_metadata(file_content)
if metadata is None:
print(f"ERROR: could not find metadata for {package_file}")
def update_expression(package_expression: str) -> str:
old_metadata = get_metadata(package_expression)
if old_metadata is None:
print("ERROR: could not find metadata for expression:")
print(package_expression)
return
if metadata.commit == "":
metadata.commit = "<none>"
new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch)
new_hash = get_nix_hash(f"{BASE_GIT_URL}/{metadata.project_name}", new_commit)
if old_metadata.commit == "":
old_metadata.commit = "<none>"
if old_metadata.hash_ == "":
old_metadata.hash_ = "<none>"
tracking_branch = old_metadata.tracking_branch
if tracking_branch is None:
tracking_branch = get_global_tracking_branch()
new_commit = get_newest_commit(old_metadata.project_name, tracking_branch)
new_hash = get_nix_hash(f"{BASE_GIT_URL}/{old_metadata.project_name}", new_commit)
if new_hash is None or new_hash == "":
print(f"ERROR: could not fetch hash for {metadata.project_name}")
print(f"ERROR: could not fetch hash for {old_metadata.project_name}")
exit(1)
print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}")
print(f"Updating {old_metadata.project_name}[{tracking_branch}]: {old_metadata.commit} -> {new_commit}")
new_file_content = set_commit_and_hash(file_content, new_commit, new_hash)
new_metadata = PluginMetadata(
project_name = old_metadata.project_name,
tracking_branch = old_metadata.tracking_branch,
commit = new_commit,
hash_ = new_hash,
)
with open(package_file, 'w') as file:
return update_metadata(package_expression, new_metadata)
def update_all_expressions_in_default_nix() -> None:
with open(get_package_listing_path()) as file:
file_content = file.read()
new_file_content = re.sub(
r"\(mw-ext\s*\{(?:.|\n)+?\}\)",
lambda m: update_expression(m.group(0)),
file_content,
flags = re.MULTILINE,
)
with open(get_package_listing_path(), 'w') as file:
file.write(new_file_content)
if __name__ == "__main__":
for direntry in os.scandir(Path(__file__).parent):
if direntry.is_dir():
package_file = Path(direntry) / "default.nix"
assert package_file.is_file()
update(package_file)
update_all_expressions_in_default_nix()

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "c17c919bdb9b67bb69f80df43e9ee9d33b1ecf1b";
project-name = "UserMerge";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-user-merge-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-+mkzTCo8RVlGoFyfCrSb5YMh4J6Pbi1PZLFu5ps8bWY=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "170d19aad1f28dc6bd3f98ee277680cabba9db0c";
project-name = "VisualEditor";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-visual-editor-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-5WVlO/OEk4eln5j/w4Tu/MXSmlvjIn7l6H+OTPaV+t4=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})