1
0
Fork 0

packages/mediawiki-extensions: refactor

This commit is contained in:
Oystein Kristoffer Tveit 2024-07-10 18:59:16 +02:00
parent f3a29429aa
commit 9d3f1381bf
7 changed files with 155 additions and 121 deletions

View File

@ -1,8 +1,60 @@
{ pkgs, lib }: { pkgs, lib }:
let
kebab-case-name = project-name: lib.pipe project-name [
(builtins.replaceStrings
lib.upperChars
(map (x: "-${x}") lib.lowerChars)
)
(lib.removePrefix "-")
];
mw-ext = {
name
, commit
, hash
, tracking-branch ? "REL1_41"
, kebab-name ? kebab-case-name name
, fetchgit ? pkgs.fetchgit
}:
{ {
DeleteBatch = pkgs.callPackage ./delete-batch { }; ${name} = (fetchgit {
PluggableAuth = pkgs.callPackage ./pluggable-auth { }; name = "mediawiki-${kebab-name}-source";
SimpleSAMLphp = pkgs.callPackage ./simple-saml-php { }; url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${name}";
UserMerge = pkgs.callPackage ./user-merge { }; rev = commit;
VisualEditor = pkgs.callPackage ./visual-editor { }; inherit hash;
} }).overrideAttrs (_: {
passthru = { inherit name kebab-name tracking-branch; };
});
};
in
# NOTE: to add another extension, you can add an mw-ext expression
# with an empty (or even wrong) commit and empty hash, and
# run the update script
lib.mergeAttrsList [
(mw-ext {
name = "DeleteBatch";
commit = "cad869fbd95637902673f744581b29e0f3e3f61a";
hash = "sha256-M1ek1WdO1/uTjeYlrk3Tz+nlb/fFZH+O0Ok7b10iKak=";
})
(mw-ext {
name = "PluggableAuth";
commit = "4111a57c34e25bde579cce5d14ea094021e450c8";
hash = "sha256-aPtN8A9gDxLlq2+EloRZBO0DfHtE0E5kbV/adk82jvM=";
})
(mw-ext {
name = "SimpleSAMLphp";
kebab-name = "simple-saml-php";
commit = "ecb47191fecd1e0dc4c9d8b90a9118e393d82c23";
hash = "sha256-gKu+O49XrAVt6hXdt36Ru7snjsKX6g2CYJ0kk/d+CI8=";
})
(mw-ext {
name = "UserMerge";
commit = "c17c919bdb9b67bb69f80df43e9ee9d33b1ecf1b";
hash = "sha256-+mkzTCo8RVlGoFyfCrSb5YMh4J6Pbi1PZLFu5ps8bWY=";
})
(mw-ext {
name = "VisualEditor";
commit = "90bb3d455892e25317029ffd4bda93159e8faac8";
hash = "sha256-SZAVELQUKZtwSM6NVlxvIHdFPodko8fhZ/uwB0LCFDA=";
})
]

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "cad869fbd95637902673f744581b29e0f3e3f61a";
project-name = "DeleteBatch";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-delete-batch-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-M1ek1WdO1/uTjeYlrk3Tz+nlb/fFZH+O0Ok7b10iKak=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "4111a57c34e25bde579cce5d14ea094021e450c8";
project-name = "PluggableAuth";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-pluggable-auth-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-aPtN8A9gDxLlq2+EloRZBO0DfHtE0E5kbV/adk82jvM=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "ecb47191fecd1e0dc4c9d8b90a9118e393d82c23";
project-name = "SimpleSAMLphp";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-simple-saml-php-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-gKu+O49XrAVt6hXdt36Ru7snjsKX6g2CYJ0kk/d+CI8=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})

View File

@ -8,49 +8,87 @@ import subprocess
from collections import defaultdict from collections import defaultdict
from pprint import pprint from pprint import pprint
from dataclasses import dataclass from dataclasses import dataclass
from functools import cache
import json import json
import bs4 import bs4
import requests import requests
BASE_WEB_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions" BASE_WEB_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions"
BASE_GIT_URL = "https://gerrit.wikimedia.org/r/mediawiki/extensions/" BASE_GIT_URL = "https://gerrit.wikimedia.org/r/mediawiki/extensions/"
@dataclass @dataclass
class PluginMetadata: class PluginMetadata:
project_name: str project_name: str
tracking_branch: str tracking_branch: str | None
commit: str commit: str
hash_: str
def get_metadata(file_content: str) -> dict[str,str] | None: @cache
commit_search = re.search(f'commit = "([^"]*?)";', file_content) def get_package_listing_path():
tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content) return Path(__file__).parent / "default.nix"
project_name_search = re.search(f'project-name = "([^"]+?)";', file_content)
if commit_search is None:
print("Could not find commit in file:") @cache
print(file_content) def get_global_tracking_branch() -> str:
return None with open(get_package_listing_path()) as file:
if tracking_branch_search is None: file_content = file.read()
print("Could not find tracking branch in file:") return re.search(r'\btracking-branch\b \? "([^"]+?)"', file_content).group(1)
print(file_content)
return None
def get_metadata(package_expression: str) -> PluginMetadata | None:
project_name_search = re.search(r'\bname\b = "([^"]+?)";', package_expression)
tracking_branch_search = re.search(r'\btracking-branch\b = "([^"]+?)";', package_expression)
commit_search = re.search(r'\bcommit\b = "([^"]*?)";', package_expression)
hash_search = re.search(r'\bhash\b = "([^"]*?)";', package_expression)
if project_name_search is None: if project_name_search is None:
print("Could not find project name in file:") print("Could not find project name in package:")
print(file_content) print(package_expression)
return None return None
tracking_branch = None;
if tracking_branch_search is not None:
tracking_branch = tracking_branch_search.group(1)
if commit_search is None:
print("Could not find commit in package:")
print(package_expression)
return None
if hash_search is None:
print("Could not find hash in package:")
print(package_expression)
return None
return PluginMetadata( return PluginMetadata(
commit = commit_search.group(1), commit = commit_search.group(1),
tracking_branch = tracking_branch_search.group(1), tracking_branch = tracking_branch,
project_name = project_name_search.group(1), project_name = project_name_search.group(1),
hash_ = hash_search.group(1),
) )
def update_metadata(package_expression: str, metadata: PluginMetadata) -> str:
result = package_expression
result = re.sub(r'\bcommit\b = "[^"]*";', f'commit = "{metadata.commit}";', result)
result = re.sub(r'\bhash\b = "[^"]*";', f'hash = "{metadata.hash_}";', result)
return result
def get_newest_commit(project_name: str, tracking_branch: str) -> str: def get_newest_commit(project_name: str, tracking_branch: str) -> str:
content = requests.get(f"{BASE_WEB_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text content = requests.get(f"{BASE_WEB_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text
soup = bs4.BeautifulSoup(content, features="html.parser") soup = bs4.BeautifulSoup(content, features="html.parser")
try:
a = soup.find('li').findChild('a') a = soup.find('li').findChild('a')
commit_sha = a['href'].split('/')[-1] commit_sha = a['href'].split('/')[-1]
except AttributeError:
print(f"ERROR: Could not parse page for {project_name}:")
print(soup.prettify())
exit(1)
return commit_sha return commit_sha
@ -64,40 +102,54 @@ def get_nix_hash(url: str, commit: str) -> str:
return json.loads(out.decode().strip())['hash'] return json.loads(out.decode().strip())['hash']
def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str: def update_expression(package_expression: str) -> str:
result = file_content old_metadata = get_metadata(package_expression)
result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result) if old_metadata is None:
result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result) print("ERROR: could not find metadata for expression:")
return result print(package_expression)
def update(package_file: Path) -> None:
with open(package_file) as file:
file_content = file.read()
metadata = get_metadata(file_content)
if metadata is None:
print(f"ERROR: could not find metadata for {package_file}")
return return
if metadata.commit == "":
metadata.commit = "<none>"
new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch) if old_metadata.commit == "":
new_hash = get_nix_hash(f"{BASE_GIT_URL}/{metadata.project_name}", new_commit) old_metadata.commit = "<none>"
if old_metadata.hash_ == "":
old_metadata.hash_ = "<none>"
tracking_branch = old_metadata.tracking_branch
if tracking_branch is None:
tracking_branch = get_global_tracking_branch()
new_commit = get_newest_commit(old_metadata.project_name, tracking_branch)
new_hash = get_nix_hash(f"{BASE_GIT_URL}/{old_metadata.project_name}", new_commit)
if new_hash is None or new_hash == "": if new_hash is None or new_hash == "":
print(f"ERROR: could not fetch hash for {metadata.project_name}") print(f"ERROR: could not fetch hash for {old_metadata.project_name}")
exit(1) exit(1)
print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}") print(f"Updating {old_metadata.project_name}[{tracking_branch}]: {old_metadata.commit} -> {new_commit}")
new_file_content = set_commit_and_hash(file_content, new_commit, new_hash) new_metadata = PluginMetadata(
project_name = old_metadata.project_name,
tracking_branch = old_metadata.tracking_branch,
commit = new_commit,
hash_ = new_hash,
)
with open(package_file, 'w') as file: return update_metadata(package_expression, new_metadata)
def update_all_expressions_in_default_nix() -> None:
with open(get_package_listing_path()) as file:
file_content = file.read()
new_file_content = re.sub(
r"\(mw-ext\s*\{(?:.|\n)+?\}\)",
lambda m: update_expression(m.group(0)),
file_content,
flags = re.MULTILINE,
)
with open(get_package_listing_path(), 'w') as file:
file.write(new_file_content) file.write(new_file_content)
if __name__ == "__main__": if __name__ == "__main__":
for direntry in os.scandir(Path(__file__).parent): update_all_expressions_in_default_nix()
if direntry.is_dir():
package_file = Path(direntry) / "default.nix"
assert package_file.is_file()
update(package_file)

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "c17c919bdb9b67bb69f80df43e9ee9d33b1ecf1b";
project-name = "UserMerge";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-user-merge-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-+mkzTCo8RVlGoFyfCrSb5YMh4J6Pbi1PZLFu5ps8bWY=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})

View File

@ -1,14 +0,0 @@
{ fetchgit }:
let
commit = "170d19aad1f28dc6bd3f98ee277680cabba9db0c";
project-name = "VisualEditor";
tracking-branch = "REL1_41";
in
(fetchgit {
name = "mediawiki-visual-editor-source";
url = "https://gerrit.wikimedia.org/r/mediawiki/extensions/${project-name}";
rev = "refs/heads/${tracking-branch}";
hash = "sha256-5WVlO/OEk4eln5j/w4Tu/MXSmlvjIn7l6H+OTPaV+t4=";
}).overrideAttrs (_: {
passthru = { inherit project-name tracking-branch; };
})