packages/mediawiki-extensions: use stable url
Eval nix flake / evals (push) Failing after 1m53s Details
Eval nix flake / evals (pull_request) Failing after 1m57s Details

This commit is contained in:
Oystein Kristoffer Tveit 2024-04-14 19:26:11 +02:00
parent 1e4195ea9d
commit b476745de6
Signed by: oysteikt
GPG Key ID: 9F2F7D8250F35146
7 changed files with 127 additions and 46 deletions

View File

@ -142,7 +142,7 @@
simplesamlphp = pkgs.callPackage ./packages/simplesamlphp { };
# mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { };
mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { };
} // nixlib.genAttrs allMachines
(machine: self.nixosConfigurations.${machine}.config.system.build.toplevel);
};

View File

@ -1,7 +1,13 @@
{ fetchzip }:
let
commit = "a53af3b8269ed19ede3cf1fa811e7ec8cb00af92";
project-name = "UserMerge";
tracking-branch = "REL1_41";
in
fetchzip {
name = "mediawiki-delete-batch";
url = "https://extdist.wmflabs.org/dist/extensions/DeleteBatch-REL1_41-5774fdd.tar.gz";
hash = "sha256-ROkn93lf0mNXBvij9X2pMhd8LXZ0azOz7ZRaqZvhh8k=";
url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
hash = "sha256-0ofCZhhv4aVTGq469Fdu7k0oVQu3kG3HFa8zaBbUr/M=";
stripRoot = false;
passthru = { inherit project-name tracking-branch; };
}

View File

@ -1,7 +1,13 @@
{ fetchzip }:
let
commit = "d5b3ad8f03b65d3746e025cdd7fe3254ad6e4026";
project-name = "PluggableAuth";
tracking-branch = "REL1_41";
in
fetchzip {
name = "mediawiki-pluggable-auth-source";
url = "https://extdist.wmflabs.org/dist/extensions/PluggableAuth-REL1_41-d5b3ad8.tar.gz";
hash = "sha256-OLlkKeSlfNgWXWwDdINrYRZpYuSGRwzZHgU8EYW6rYU=";
url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project_name}/+archive/${commit}.tar.gz";
hash = "sha256-mLepavgeaNUGYxrrCKVpybGO2ecjc3B5IU8q+gZTx2U=";
stripRoot = false;
passthru = { inherit project-name tracking-branch; };
}

View File

@ -1,7 +1,13 @@
{ fetchzip }:
let
commit = "9ae0678d77a9175285a1cfadd5adf28379dbdb3d";
project-name = "SimpleSAMLphp";
tracking-branch = "REL1_41";
in
fetchzip {
name = "mediawiki-simple-saml-php-source";
url = "https://extdist.wmflabs.org/dist/extensions/SimpleSAMLphp-REL1_41-9ae0678.tar.gz";
hash = "sha256-AmCaG5QXMJvi3N6zFyWylwYDt8GvyIk/0GFpM1Y0vkY=";
url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
hash = "sha256-s6Uw1fNzGBF0HEMl0LIRLhJkOHugrCE0aTnqawYi/pE=";
stripRoot = false;
passthru = { inherit project-name tracking-branch; };
}

View File

@ -7,36 +7,65 @@ import re
import subprocess
from collections import defaultdict
from pprint import pprint
from dataclasses import dataclass
import bs4
import requests
BASE_URL = "https://extdist.wmflabs.org/dist/extensions"
BASE_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions"
def fetch_plugin_list(skip_master=True) -> dict[str, list[str]]:
content = requests.get(BASE_URL).text
# def fetch_plugin_list(skip_master=True) -> dict[str, list[str]]:
# content = requests.get(BASE_URL).text
# soup = bs4.BeautifulSoup(content, features="html.parser")
# result = defaultdict(list)
# for a in soup.find_all('a'):
# if skip_master and 'master' in a.text:
# continue
# split = a.text.split('-')
# result[split[0]].append(a.text)
# return result
@dataclass
class PluginMetadata:
project_name: str
tracking_branch: str
commit: str
def get_metadata(file_content: str) -> dict[str,str] | None:
commit_search = re.search(f'commit = "([^"]*?)";', file_content)
tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content)
project_name_search = re.search(f'project-name = "([^"]+?)";', file_content)
if commit_search is None:
print("Could not find commit in file:")
print(file_content)
return None
if tracking_branch_search is None:
print("Could not find tracking branch in file:")
print(file_content)
return None
if project_name_search is None:
print("Could not find project name in file:")
print(file_content)
return None
return PluginMetadata(
commit = commit_search.group(1),
tracking_branch = tracking_branch_search.group(1),
project_name = project_name_search.group(1),
)
def get_newest_commit(project_name: str, tracking_branch: str) -> str:
content = requests.get(f"{BASE_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text
soup = bs4.BeautifulSoup(content, features="html.parser")
result = defaultdict(list)
for a in soup.find_all('a'):
if skip_master and 'master' in a.text:
continue
split = a.text.split('-')
result[split[0]].append(a.text)
return result
a = soup.find('li').findChild('a')
commit_sha = a['href'].split('/')[-1]
return commit_sha
def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None:
assert package_file.is_file()
with open(package_file) as file:
content = file.read()
tarball = re.search(f'url = "{BASE_URL}/(.+\.tar\.gz)";', content).group(1)
split = tarball.split('-')
updated_tarball = plugin_list[split[0]][-1]
_hash = re.search(f'hash = "(.+?)";', content).group(1)
def get_nix_hash(tar_gz_url: str) -> str:
out, err = subprocess.Popen(
["nix-prefetch-url", "--unpack", "--type", "sha256", f"{BASE_URL}/{updated_tarball}"],
["nix-prefetch-url", "--unpack", "--type", "sha256", tar_gz_url],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
@ -46,21 +75,43 @@ def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None:
stderr=subprocess.PIPE
).communicate()
updated_hash = out.decode().strip()
return out.decode().strip()
if tarball == updated_tarball and _hash == updated_hash:
def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str:
result = file_content
result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result)
result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result)
return result
def update(package_file: Path) -> None:
with open(package_file) as file:
file_content = file.read()
metadata = get_metadata(file_content)
if metadata is None:
return
if metadata.commit == "":
metadata.commit = "<none>"
new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch)
if new_commit == metadata.commit:
return
print(f"Updating: {tarball} ({_hash[7:14]}) -> {updated_tarball} ({updated_hash[7:14]})")
new_url = f"{BASE_URL}/{metadata.project_name}/+archive/{new_commit}.tar.gz"
new_hash = get_nix_hash(new_url)
print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}")
new_file_content = set_commit_and_hash(file_content, new_commit, new_hash)
updated_text = re.sub(f'url = "{BASE_URL}/.+?\.tar\.gz";', f'url = "{BASE_URL}/{updated_tarball}";', content)
updated_text = re.sub('hash = ".+";', f'hash = "{updated_hash}";', updated_text)
with open(package_file, 'w') as file:
file.write(updated_text)
file.write(new_file_content)
if __name__ == "__main__":
plugin_list = fetch_plugin_list()
for direntry in os.scandir(Path(__file__).parent):
if direntry.is_dir():
update(Path(direntry) / "default.nix", plugin_list)
package_file = Path(direntry) / "default.nix"
assert package_file.is_file()
update(package_file)

View File

@ -1,7 +1,13 @@
{ fetchzip }:
let
commit = "a53af3b8269ed19ede3cf1fa811e7ec8cb00af92";
project-name = "UserMerge";
tracking-branch = "REL1_41";
in
fetchzip {
name = "mediawiki-user-merge-source";
url = "https://extdist.wmflabs.org/dist/extensions/UserMerge-REL1_41-a53af3b.tar.gz";
hash = "sha256-TxUkEqMW79thYl1la2r+w9laRnd3uSYYg1xDB+1he1g=";
url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
hash = "sha256-0ofCZhhv4aVTGq469Fdu7k0oVQu3kG3HFa8zaBbUr/M=";
stripRoot = false;
passthru = { inherit project-name tracking-branch; };
}

View File

@ -1,7 +1,13 @@
{ fetchzip }:
let
commit = "bb92d4b0bb81cebd73a3dbabfb497213dac349f2";
project-name = "VisualEditor";
tracking-branch = "REL1_40";
in
fetchzip {
name = "mediawiki-visual-editor-source";
url = "https://extdist.wmflabs.org/dist/extensions/VisualEditor-REL1_40-5f8c97e.tar.gz";
hash = "sha256-oBMmEDKsFxrD0tpN2dy264IXK164BrZWrNK3v3FNX6w=";
url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz";
hash = "sha256-lShpSoR+NLfdd5i7soM6J40pq+MzCMG0M1tSYsS+jAg=";
stripRoot = false;
passthru = { inherit project-name tracking-branch; };
}