From 6f6721ce07735629bcb5f1358fd679c75b9dd013 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Sun, 14 Apr 2024 19:26:11 +0200 Subject: [PATCH 1/4] packages/mediawiki-extensions: use stable url --- flake.nix | 2 +- .../delete-batch/default.nix | 12 ++- .../pluggable-auth/default.nix | 12 ++- .../simple-saml-php/default.nix | 12 ++- .../update-mediawiki-extensions.py | 100 ++++++++++++------ .../user-merge/default.nix | 12 ++- .../visual-editor/default.nix | 12 ++- 7 files changed, 116 insertions(+), 46 deletions(-) diff --git a/flake.nix b/flake.nix index 0c65b2d..6147832 100644 --- a/flake.nix +++ b/flake.nix @@ -140,7 +140,7 @@ simplesamlphp = pkgs.callPackage ./packages/simplesamlphp { }; - # mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { }; + mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { }; } // nixlib.genAttrs allMachines (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel); }; diff --git a/packages/mediawiki-extensions/delete-batch/default.nix b/packages/mediawiki-extensions/delete-batch/default.nix index dc7de57..23b176f 100644 --- a/packages/mediawiki-extensions/delete-batch/default.nix +++ b/packages/mediawiki-extensions/delete-batch/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "a53af3b8269ed19ede3cf1fa811e7ec8cb00af92"; + project-name = "UserMerge"; + tracking-branch = "REL1_41"; +in fetchzip { name = "mediawiki-delete-batch"; - url = "https://extdist.wmflabs.org/dist/extensions/DeleteBatch-REL1_41-5774fdd.tar.gz"; - hash = "sha256-ROkn93lf0mNXBvij9X2pMhd8LXZ0azOz7ZRaqZvhh8k="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; + hash = "sha256-0ofCZhhv4aVTGq469Fdu7k0oVQu3kG3HFa8zaBbUr/M="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } diff --git a/packages/mediawiki-extensions/pluggable-auth/default.nix b/packages/mediawiki-extensions/pluggable-auth/default.nix index 58b793e..077c5ac 100644 --- a/packages/mediawiki-extensions/pluggable-auth/default.nix +++ b/packages/mediawiki-extensions/pluggable-auth/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "d5b3ad8f03b65d3746e025cdd7fe3254ad6e4026"; + project-name = "PluggableAuth"; + tracking-branch = "REL1_41"; +in fetchzip { name = "mediawiki-pluggable-auth-source"; - url = "https://extdist.wmflabs.org/dist/extensions/PluggableAuth-REL1_41-d5b3ad8.tar.gz"; - hash = "sha256-OLlkKeSlfNgWXWwDdINrYRZpYuSGRwzZHgU8EYW6rYU="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project_name}/+archive/${commit}.tar.gz"; + hash = "sha256-mLepavgeaNUGYxrrCKVpybGO2ecjc3B5IU8q+gZTx2U="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } diff --git a/packages/mediawiki-extensions/simple-saml-php/default.nix b/packages/mediawiki-extensions/simple-saml-php/default.nix index 59ef5fa..ce1800e 100644 --- a/packages/mediawiki-extensions/simple-saml-php/default.nix +++ b/packages/mediawiki-extensions/simple-saml-php/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "9ae0678d77a9175285a1cfadd5adf28379dbdb3d"; + project-name = "SimpleSAMLphp"; + tracking-branch = "REL1_41"; +in fetchzip { name = "mediawiki-simple-saml-php-source"; - url = "https://extdist.wmflabs.org/dist/extensions/SimpleSAMLphp-REL1_41-9ae0678.tar.gz"; - hash = "sha256-AmCaG5QXMJvi3N6zFyWylwYDt8GvyIk/0GFpM1Y0vkY="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; + hash = "sha256-s6Uw1fNzGBF0HEMl0LIRLhJkOHugrCE0aTnqawYi/pE="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } diff --git a/packages/mediawiki-extensions/update-mediawiki-extensions.py b/packages/mediawiki-extensions/update-mediawiki-extensions.py index 099b18d..c2eb748 100755 --- a/packages/mediawiki-extensions/update-mediawiki-extensions.py +++ b/packages/mediawiki-extensions/update-mediawiki-extensions.py @@ -7,36 +7,54 @@ import re import subprocess from collections import defaultdict from pprint import pprint +from dataclasses import dataclass import bs4 import requests -BASE_URL = "https://extdist.wmflabs.org/dist/extensions" +BASE_URL = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions" -def fetch_plugin_list(skip_master=True) -> dict[str, list[str]]: - content = requests.get(BASE_URL).text +@dataclass +class PluginMetadata: + project_name: str + tracking_branch: str + commit: str + + +def get_metadata(file_content: str) -> dict[str,str] | None: + commit_search = re.search(f'commit = "([^"]*?)";', file_content) + tracking_branch_search = re.search(f'tracking-branch = "([^"]+?)";', file_content) + project_name_search = re.search(f'project-name = "([^"]+?)";', file_content) + if commit_search is None: + print("Could not find commit in file:") + print(file_content) + return None + if tracking_branch_search is None: + print("Could not find tracking branch in file:") + print(file_content) + return None + if project_name_search is None: + print("Could not find project name in file:") + print(file_content) + return None + return PluginMetadata( + commit = commit_search.group(1), + tracking_branch = tracking_branch_search.group(1), + project_name = project_name_search.group(1), + ) + + +def get_newest_commit(project_name: str, tracking_branch: str) -> str: + content = requests.get(f"{BASE_URL}/{project_name}/+log/refs/heads/{tracking_branch}/").text soup = bs4.BeautifulSoup(content, features="html.parser") - result = defaultdict(list) - for a in soup.find_all('a'): - if skip_master and 'master' in a.text: - continue - split = a.text.split('-') - result[split[0]].append(a.text) - return result + a = soup.find('li').findChild('a') + commit_sha = a['href'].split('/')[-1] + return commit_sha -def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None: - assert package_file.is_file() - with open(package_file) as file: - content = file.read() - - tarball = re.search(f'url = "{BASE_URL}/(.+\.tar\.gz)";', content).group(1) - split = tarball.split('-') - updated_tarball = plugin_list[split[0]][-1] - - _hash = re.search(f'hash = "(.+?)";', content).group(1) +def get_nix_hash(tar_gz_url: str) -> str: out, err = subprocess.Popen( - ["nix-prefetch-url", "--unpack", "--type", "sha256", f"{BASE_URL}/{updated_tarball}"], + ["nix-prefetch-url", "--unpack", "--type", "sha256", tar_gz_url], stdout=subprocess.PIPE, stderr=subprocess.PIPE ).communicate() @@ -46,21 +64,43 @@ def update(package_file: Path, plugin_list: dict[str, list[str]]) -> None: stderr=subprocess.PIPE ).communicate() - updated_hash = out.decode().strip() + return out.decode().strip() - if tarball == updated_tarball and _hash == updated_hash: + +def set_commit_and_hash(file_content: str, commit: str, sha256: str) -> str: + result = file_content + result = re.sub('commit = "[^"]*";', f'commit = "{commit}";', result) + result = re.sub('hash = "[^"]*";', f'hash = "{sha256}";', result) + return result + +def update(package_file: Path) -> None: + with open(package_file) as file: + file_content = file.read() + + metadata = get_metadata(file_content) + if metadata is None: + return + if metadata.commit == "": + metadata.commit = "" + + new_commit = get_newest_commit(metadata.project_name, metadata.tracking_branch) + if new_commit == metadata.commit: return - print(f"Updating: {tarball} ({_hash[7:14]}) -> {updated_tarball} ({updated_hash[7:14]})") + new_url = f"{BASE_URL}/{metadata.project_name}/+archive/{new_commit}.tar.gz" + new_hash = get_nix_hash(new_url) + + print(f"Updating {metadata.project_name}: {metadata.commit} -> {new_commit}") + + new_file_content = set_commit_and_hash(file_content, new_commit, new_hash) - updated_text = re.sub(f'url = "{BASE_URL}/.+?\.tar\.gz";', f'url = "{BASE_URL}/{updated_tarball}";', content) - updated_text = re.sub('hash = ".+";', f'hash = "{updated_hash}";', updated_text) with open(package_file, 'w') as file: - file.write(updated_text) + file.write(new_file_content) + if __name__ == "__main__": - plugin_list = fetch_plugin_list() - for direntry in os.scandir(Path(__file__).parent): if direntry.is_dir(): - update(Path(direntry) / "default.nix", plugin_list) + package_file = Path(direntry) / "default.nix" + assert package_file.is_file() + update(package_file) diff --git a/packages/mediawiki-extensions/user-merge/default.nix b/packages/mediawiki-extensions/user-merge/default.nix index d918297..a5918a6 100644 --- a/packages/mediawiki-extensions/user-merge/default.nix +++ b/packages/mediawiki-extensions/user-merge/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "a53af3b8269ed19ede3cf1fa811e7ec8cb00af92"; + project-name = "UserMerge"; + tracking-branch = "REL1_41"; +in fetchzip { name = "mediawiki-user-merge-source"; - url = "https://extdist.wmflabs.org/dist/extensions/UserMerge-REL1_41-a53af3b.tar.gz"; - hash = "sha256-TxUkEqMW79thYl1la2r+w9laRnd3uSYYg1xDB+1he1g="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; + hash = "sha256-0ofCZhhv4aVTGq469Fdu7k0oVQu3kG3HFa8zaBbUr/M="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } diff --git a/packages/mediawiki-extensions/visual-editor/default.nix b/packages/mediawiki-extensions/visual-editor/default.nix index cfb9a0c..b365a32 100644 --- a/packages/mediawiki-extensions/visual-editor/default.nix +++ b/packages/mediawiki-extensions/visual-editor/default.nix @@ -1,7 +1,13 @@ { fetchzip }: - +let + commit = "bb92d4b0bb81cebd73a3dbabfb497213dac349f2"; + project-name = "VisualEditor"; + tracking-branch = "REL1_40"; +in fetchzip { name = "mediawiki-visual-editor-source"; - url = "https://extdist.wmflabs.org/dist/extensions/VisualEditor-REL1_40-5f8c97e.tar.gz"; - hash = "sha256-oBMmEDKsFxrD0tpN2dy264IXK164BrZWrNK3v3FNX6w="; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; + hash = "sha256-lShpSoR+NLfdd5i7soM6J40pq+MzCMG0M1tSYsS+jAg="; + stripRoot = false; + passthru = { inherit project-name tracking-branch; }; } -- 2.44.2 From af54cc2df45fd3db9acac2f2ab8892f2489a334b Mon Sep 17 00:00:00 2001 From: Daniel Olsen Date: Mon, 15 Apr 2024 01:17:56 +0200 Subject: [PATCH 2/4] packages/mediawiki/pluggable-auth: fix typo --- packages/mediawiki-extensions/pluggable-auth/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mediawiki-extensions/pluggable-auth/default.nix b/packages/mediawiki-extensions/pluggable-auth/default.nix index 077c5ac..3d0977e 100644 --- a/packages/mediawiki-extensions/pluggable-auth/default.nix +++ b/packages/mediawiki-extensions/pluggable-auth/default.nix @@ -6,7 +6,7 @@ let in fetchzip { name = "mediawiki-pluggable-auth-source"; - url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project_name}/+archive/${commit}.tar.gz"; + url = "https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/${project-name}/+archive/${commit}.tar.gz"; hash = "sha256-mLepavgeaNUGYxrrCKVpybGO2ecjc3B5IU8q+gZTx2U="; stripRoot = false; passthru = { inherit project-name tracking-branch; }; -- 2.44.2 From 96024efa287ac554e3f1b7a538e3eaca883740b0 Mon Sep 17 00:00:00 2001 From: Daniel Olsen Date: Mon, 15 Apr 2024 01:38:25 +0200 Subject: [PATCH 3/4] flake: move mediawiki-extensions to legacyPackages --- flake.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 6147832..4f1acda 100644 --- a/flake.nix +++ b/flake.nix @@ -139,10 +139,16 @@ (nixlib.getAttrs allMachines self.packages.x86_64-linux); simplesamlphp = pkgs.callPackage ./packages/simplesamlphp { }; - - mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { }; } // nixlib.genAttrs allMachines (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel); + } ; + + legacyPackages = { + "x86_64-linux" = let + pkgs = nixpkgs.legacyPackages."x86_64-linux"; + in rec { + mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { }; + }; }; }; } -- 2.44.2 From 3a972f03f73c97fe4b79749f13d7bcc9cc8a0d25 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Sun, 26 May 2024 02:17:48 +0200 Subject: [PATCH 4/4] flake: move mediawiki-extensions back to packages --- flake.nix | 17 ++++++++--------- packages/mediawiki-extensions/default.nix | 14 +++++++------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/flake.nix b/flake.nix index 4f1acda..929c79d 100644 --- a/flake.nix +++ b/flake.nix @@ -139,16 +139,15 @@ (nixlib.getAttrs allMachines self.packages.x86_64-linux); simplesamlphp = pkgs.callPackage ./packages/simplesamlphp { }; - } // nixlib.genAttrs allMachines - (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel); - } ; - legacyPackages = { - "x86_64-linux" = let - pkgs = nixpkgs.legacyPackages."x86_64-linux"; - in rec { - mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { }; - }; + } // + (nixlib.pipe null [ + (_: pkgs.callPackage ./packages/mediawiki-extensions { }) + (nixlib.flip builtins.removeAttrs ["override" "overrideDerivation"]) + (nixlib.mapAttrs' (name: nixlib.nameValuePair "mediawiki-${name}")) + ]) + // nixlib.genAttrs allMachines + (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel); }; }; } diff --git a/packages/mediawiki-extensions/default.nix b/packages/mediawiki-extensions/default.nix index b7195c8..43c67e5 100644 --- a/packages/mediawiki-extensions/default.nix +++ b/packages/mediawiki-extensions/default.nix @@ -1,8 +1,8 @@ { pkgs, lib }: -lib.makeScope pkgs.newScope (self: { - DeleteBatch = self.callPackage ./delete-batch { }; - PluggableAuth = self.callPackage ./pluggable-auth { }; - SimpleSAMLphp = self.callPackage ./simple-saml-php { }; - UserMerge = self.callPackage ./user-merge { }; - VisualEditor = self.callPackage ./visual-editor { }; -}) +{ + DeleteBatch = pkgs.callPackage ./delete-batch { }; + PluggableAuth = pkgs.callPackage ./pluggable-auth { }; + SimpleSAMLphp = pkgs.callPackage ./simple-saml-php { }; + UserMerge = pkgs.callPackage ./user-merge { }; + VisualEditor = pkgs.callPackage ./visual-editor { }; +} -- 2.44.2