From 94f4afd680e657f758bf8cc1756044237168b78d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 25 Mar 2024 02:49:30 +0100 Subject: [PATCH 01/18] feat(translate_apps): add a first script to push/updates translations from apps to weblate --- .../push_or_update_apps_on_repository.py | 181 ++++++++++++++++++ translate_apps/requirements.txt | 1 + 2 files changed, 182 insertions(+) create mode 100644 translate_apps/push_or_update_apps_on_repository.py create mode 100644 translate_apps/requirements.txt diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py new file mode 100644 index 00000000..98d6daae --- /dev/null +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -0,0 +1,181 @@ +import os +import time +import json +import tempfile +import subprocess + +from collections import defaultdict +from pathlib import Path +from typing import Union + +import wlc +import tomlkit + +github_webhook_secret = open("github_webhook_secret", "r").read().strip() + +login = open("login").read().strip() +token = open("token").read().strip() + +weblate_token = open("weblate_token").read().strip() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" +my_env["GIT_AUTHOR_NAME"] = "yunohost-bot" +my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" +my_env["GIT_COMMITTER_NAME"] = "yunohost-bot" +my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" + + +class Repository: + def __init__(self, url, branch): + self.url = url + self.branch = branch + + def __enter__(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.path = Path(self.temporary_directory.name) + self.run_command( + [ + "git", + "clone", + self.url, + "--single-branch", + "--branch", + self.branch, + self.path, + ] + ) + + return self + + def run_command( + self, command: Union[str, list], capture_output=False + ) -> Union[str, int]: + if isinstance(command, str): + kwargs = {"args": f"cd {self.path} && {command}", "shell": True} + + elif isinstance(command, list): + kwargs = {"args": command, "cwd": self.path} + + if capture_output: + return subprocess.check_output(**kwargs).decode() + else: + print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") + return subprocess.check_call(**kwargs) + + def file_exists(self, file_name: str) -> bool: + return (self.path / file_name).exists() + + def read_file(self, file_name: str) -> str: + return open((self.path / file_name).resolve(), "r").read() + + def write_file(self, file_name: str, content: str) -> None: + open((self.path / file_name).resolve(), "w").write(content) + + def remove_file(self, file_name: str) -> None: + os.remove(self.path / file_name) + + def append_to_file(self, file_name: str, content: str) -> None: + open((self.path / file_name).resolve(), "a").write(content) + + def __repr__(self): + return f'<__main__.Repository "{self.url.split("@")[1]}" path="{self.path}">' + + def __exit__(self, *args, **kwargs): + pass + + +def extract_strings_to_translate_from_apps(apps, translations_repository): + weblate = wlc.Weblate(key=weblate_token, url="https://translate.yunohost.org/api/") + + for app, infos in apps.items(): + repository_uri = infos["git"]["url"].replace("https://github.com/", "") + branch = infos["git"]["branch"] + + if "github.com" not in infos["git"]["url"]: + continue + + print(app) + print(f"{repository_uri} -> branch '{branch}'") + + with Repository( + f"https://{login}:{token}@github.com/{repository_uri}", branch + ) as repository: + if not repository.file_exists("manifest.toml"): + continue + + manifest = tomlkit.loads(repository.read_file("manifest.toml")) + + translations_path = Path(f"translations/apps/{app}/") + + newly_created_translation = False + if not translations_repository.file_exists(translations_path): + (translations_repository.path / translations_path).mkdir(parents=True) + newly_created_translation = True + + translations = defaultdict(dict) + for language, strings_to_translate in manifest.get( + "description", {} + ).items(): + translations[language]["description"] = strings_to_translate + + for question in manifest.get("install", {}): + for strings_to_translate in ["ask", "help"]: + for language, message in ( + manifest["install"][question] + .get(strings_to_translate, {}) + .items() + ): + translations[language][ + f"install_{question}_{strings_to_translate}" + ] = message + + if newly_created_translation: + for language, translated_strings in translations.items(): + translations_repository.write_file( + translations_path / f"{language}.json", + json.dumps(translated_strings, indent=4, sort_keys=True), + ) + else: + translations_repository.write_file( + translations_path / "en.json", + json.dumps(translations["en"], indent=4, sort_keys=True), + ) + + # if something has been modified + if translations_repository.run_command("git status -s", capture_output=True).strip(): + translations_repository.run_command("git status -s") + translations_repository.run_command(["git", "add", translations_path]) + translations_repository.run_command( + [ + "git", + "commit", + "-m", + f"feat(apps/i18n): extract strings to translate for application {app}", + ] + ) + translations_repository.run_command(["git", "push"]) + + if newly_created_translation: + weblate.create_component( + "yunohost-apps", + name=app, + slug=app, + file_format="json", + filemask=f"translations/apps/{app}/*.json", + repo="https://github.com/yunohost/apps_translations", + new_base=f"translations/apps/{app}/en.json", + template=f"translations/apps/{app}/en.json", + push="git@github.com:yunohost/apps_translations.git", + ) + + time.sleep(2) + + +if __name__ == "__main__": + apps = json.load(open("../../builds/default/v3/apps.json"))["apps"] + + with Repository( + f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" + ) as repository: + extract_strings_to_translate_from_apps(apps, repository) diff --git a/translate_apps/requirements.txt b/translate_apps/requirements.txt new file mode 100644 index 00000000..cf79a9fc --- /dev/null +++ b/translate_apps/requirements.txt @@ -0,0 +1 @@ +wlc # weblate api From 4c7b9bbb40a8e459e93ccee6ce97e1e74c62875e Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 25 Mar 2024 05:58:47 +0100 Subject: [PATCH 02/18] feat(translate_apps): recreate weblate component also if it's not present --- .../push_or_update_apps_on_repository.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 98d6daae..f516258a 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -26,6 +26,15 @@ my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" +def get_weblate_component(weblate, component_path): + try: + weblate.get_component(component_path) + except wlc.WeblateException: + return False + else: + return True + + class Repository: def __init__(self, url, branch): self.url = url @@ -156,24 +165,30 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): ) translations_repository.run_command(["git", "push"]) - if newly_created_translation: + if newly_created_translation or not get_weblate_component(weblate, f"yunohost-apps/{app}"): + print("Creating component on weblate...") weblate.create_component( "yunohost-apps", name=app, slug=app, - file_format="json", + if newly_created_translation or not get_weblate_component( + weblate, f"yunohost-apps/{app}" + ): filemask=f"translations/apps/{app}/*.json", repo="https://github.com/yunohost/apps_translations", new_base=f"translations/apps/{app}/en.json", template=f"translations/apps/{app}/en.json", push="git@github.com:yunohost/apps_translations.git", ) + print(f"Component created at https://translate.yunohost.org/projects/yunohost-apps/{app}/") time.sleep(2) if __name__ == "__main__": - apps = json.load(open("../../builds/default/v3/apps.json"))["apps"] + print( + f"Component created at https://translate.yunohost.org/projects/yunohost-apps/{app}/" + ) with Repository( f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" From d7facfb03834db2600f59480d59ee4f46b49024e Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 25 Mar 2024 06:24:46 +0100 Subject: [PATCH 03/18] feat(translate_apps): add a script that pushes new translation to apps via PR --- translate_apps/apps_translations_to_apps.py | 165 ++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 translate_apps/apps_translations_to_apps.py diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py new file mode 100644 index 00000000..0ec471fa --- /dev/null +++ b/translate_apps/apps_translations_to_apps.py @@ -0,0 +1,165 @@ +import os +import time +import json +import tempfile +import subprocess + +from typing import Union +from pathlib import Path + +import tomlkit + +github_webhook_secret = open("github_webhook_secret", "r").read().strip() + +login = open("login").read().strip() +token = open("token").read().strip() + +weblate_token = open("weblate_token").read().strip() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" +my_env["GIT_AUTHOR_NAME"] = "yunohost-bot" +my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" +my_env["GIT_COMMITTER_NAME"] = "yunohost-bot" +my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" +my_env["GITHUB_USER"] = login +my_env["GITHUB_TOKEN"] = token + + +class Repository: + def __init__(self, url, branch): + self.url = url + self.branch = branch + + def __enter__(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.path = Path(self.temporary_directory.name) + self.run_command( + [ + "git", + "clone", + self.url, + "--single-branch", + "--branch", + self.branch, + self.path, + ] + ) + + return self + + def run_command( + self, command: Union[str, list], capture_output=False + ) -> Union[str, int]: + if isinstance(command, str): + kwargs = {"args": f"cd {self.path} && {command}", "shell": True, "env": my_env} + + elif isinstance(command, list): + kwargs = {"args": command, "cwd": self.path, "env": my_env} + + if capture_output: + return subprocess.check_output(**kwargs).decode() + else: + print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") + return subprocess.check_call(**kwargs) + + def file_exists(self, file_name: str) -> bool: + return (self.path / file_name).exists() + + def read_file(self, file_name: str) -> str: + return open((self.path / file_name).resolve(), "r").read() + + def write_file(self, file_name: str, content: str) -> None: + open((self.path / file_name).resolve(), "w").write(content) + + def remove_file(self, file_name: str) -> None: + os.remove(self.path / file_name) + + def append_to_file(self, file_name: str, content: str) -> None: + open((self.path / file_name).resolve(), "a").write(content) + + def __repr__(self): + return f'<__main__.Repository "{self.url.split("@")[1]}" path="{self.path}">' + + def __exit__(self, *args, **kwargs): + pass + + +def extract_strings_to_translate_from_apps(apps, translations_repository): + for app, infos in apps.items(): + repository_uri = infos["git"]["url"].replace("https://github.com/", "") + branch = infos["git"]["branch"] + + if "github.com" not in infos["git"]["url"]: + continue + + # xxx + repository_uri = "YunoHost-Apps/test_app_for_translation_ynh" + branch = "master" + app = "test_app_for_translation" + + print(app) + print(f"{repository_uri} -> branch '{branch}'") + + translations_path = Path(f"translations/apps/{app}/") + + if not translations_repository.file_exists(translations_path): + print(f"App {app} doesn't have translations on github.com/yunohost/apps_translations, skip") + continue + + with Repository( + f"https://{login}:{token}@github.com/{repository_uri}", branch + ) as repository: + if not repository.file_exists("manifest.toml"): + continue + + manifest = tomlkit.loads(repository.read_file("manifest.toml")) + + for translation in (translations_repository.path / f"translations/apps/{app}/").glob("*.json"): + language = translation.name[:-len(".json")] + + # english version is the base, never modify it + if language == "en": + continue + + translation = json.load(open(translation)) + + if translation.get("description", "").strip(): + manifest["description"][language] = translation["description"] + + for question in manifest.get("install", {}): + for strings_to_translate in ["ask", "help"]: + translation_key = f"install_{question}_{strings_to_translate}" + if not translation.get(translation_key, "").strip(): + continue + + if strings_to_translate not in manifest["install"][question]: + continue + + manifest["install"][question][strings_to_translate][language] = translation[translation_key] + + repository.write_file("manifest.toml", tomlkit.dumps(manifest)) + + if not repository.run_command("git status -s", capture_output=True).strip(): + continue + + # create or update merge request + repository.run_command("git diff") + repository.run_command("git add manifest.toml") + repository.run_command(["git", "commit", "-m", "feat(i18n): update translations for manifest.toml"]) + repository.run_command(["git", "push", "-f", "origin", "master:manifest_toml_i18n"]) + + # if no PR exist, create one + if not repository.run_command("hub pr list -h manifest_toml_i18n", capture_output=True): + repository.run_command(["hub", "pull-request", "-m", "Update translations for manifest.toml", "-b", branch, "-h", "manifest_toml_i18n", "-p"]) + + time.sleep(2) + + +if __name__ == "__main__": + apps = json.load(open("../../builds/default/v3/apps.json"))["apps"] + + with Repository( + f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" + ) as repository: + extract_strings_to_translate_from_apps(apps, repository) From e26a9d174373e949a754053083e609472e551e20 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 28 Mar 2024 04:22:54 +0100 Subject: [PATCH 04/18] feat(translate_apps): add a list of apps for the beta testing phase --- translate_apps/apps_translations_to_apps.py | 13 +++++++++---- translate_apps/push_or_update_apps_on_repository.py | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index 0ec471fa..1f5ed1ab 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -93,10 +93,15 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if "github.com" not in infos["git"]["url"]: continue - # xxx - repository_uri = "YunoHost-Apps/test_app_for_translation_ynh" - branch = "master" - app = "test_app_for_translation" + if app not in ( + "gotosocial", + "fluffychat", + "cinny", + "fittrackee", + "funkwhale", + "photoprism", + ): + continue print(app) print(f"{repository_uri} -> branch '{branch}'") diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index f516258a..3bff1dc3 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -104,6 +104,9 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if "github.com" not in infos["git"]["url"]: continue + if app not in ("gotosocial", "fluffychat", "cinny", "fittrackee", "funkwhale", "photoprism"): + continue + print(app) print(f"{repository_uri} -> branch '{branch}'") From 762424e9c07759671a8d62632170d15a1d6b375c Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 28 Mar 2024 04:23:14 +0100 Subject: [PATCH 05/18] feat(translate_apps): on update from apps to weblate, app new strings from apps --- .../push_or_update_apps_on_repository.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 3bff1dc3..4f532351 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -154,6 +154,30 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): json.dumps(translations["en"], indent=4, sort_keys=True), ) + # add strings that aren't already present but don't overwrite existing ones + for language, translated_strings in translations.items(): + if language == "en": + continue + + # if the translation file doesn't exist yet, dump it + if not (translations_path / f"{language}.json").exists(): + translations_repository.write_file( + translations_path / f"{language}.json", + json.dumps(translated_strings, indent=4, sort_keys=True), + ) + else: # if it exists, only add keys that aren't already present + language_file = json.load((translations_path / f"{language}.json").open()) + for key, translated_string in translated_strings.items(): + if key not in language_file: + language_file[key] = translated_string + language_file = json.load( + (translations_path / f"{language}.json").open() + ) + translations_repository.write_file( + translations_path / f"{language}.json", + json.dumps(language_file, indent=4, sort_keys=True), + ) + # if something has been modified if translations_repository.run_command("git status -s", capture_output=True).strip(): translations_repository.run_command("git status -s") From 9da577c5d482542de6f3dd6032a08d9741215ec0 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 28 Mar 2024 04:23:28 +0100 Subject: [PATCH 06/18] feat(translate_apps): add a git diff in push_or_update_apps_on_repository.py for logging --- translate_apps/push_or_update_apps_on_repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 4f532351..5be1421d 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -181,6 +181,7 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): # if something has been modified if translations_repository.run_command("git status -s", capture_output=True).strip(): translations_repository.run_command("git status -s") + translations_repository.run_command("git diff") translations_repository.run_command(["git", "add", translations_path]) translations_repository.run_command( [ From 23ffc9fda26c0c957656819d2d5382d8e7c17a47 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 28 Mar 2024 05:35:19 +0100 Subject: [PATCH 07/18] feat(translate_apps): add message to the PR --- translate_apps/apps_translations_to_apps.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index 1f5ed1ab..60c0f489 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -156,7 +156,7 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): # if no PR exist, create one if not repository.run_command("hub pr list -h manifest_toml_i18n", capture_output=True): - repository.run_command(["hub", "pull-request", "-m", "Update translations for manifest.toml", "-b", branch, "-h", "manifest_toml_i18n", "-p"]) + repository.run_command(["hub", "pull-request", "-m", "Update translations for manifest.toml", "-b", branch, "-h", "manifest_toml_i18n", "-p", "-m", f"This pull request is automatically generated by scripts from the [YunoHost/apps](https://github.com/YunoHost/apps) repository.\n\nThe translation is pull from weblate and is located here: https://translate.yunohost.org/projects/yunohost-apps/{app}/\n\nIf you wish to modify the translation (other than in english), please do that directly on weblate since this is now the source of authority for it.\n\nDon't hesitate to reach the YunoHost team on [matrix](https://matrix.to/#/#yunohost:matrix.org) if there is any problem :heart:"]) time.sleep(2) @@ -168,3 +168,21 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" ) as repository: extract_strings_to_translate_from_apps(apps, repository) + if not repository.run_command( + "hub pr list -h manifest_toml_i18n", capture_output=True + ): + repository.run_command( + [ + "hub", + "pull-request", + "-m", + "Update translations for manifest.toml", + "-b", + branch, + "-h", + "manifest_toml_i18n", + "-p", + "-m", + f"This pull request is automatically generated by scripts from the [YunoHost/apps](https://github.com/YunoHost/apps) repository.\n\nThe translation is pull from weblate and is located here: https://translate.yunohost.org/projects/yunohost-apps/{app}/\n\nIf you wish to modify the translation (other than in english), please do that directly on weblate since this is now the source of authority for it.\n\nDon't hesitate to reach the YunoHost team on [matrix](https://matrix.to/#/#yunohost:matrix.org) if there is any problem :heart:", + ] + ) From 94ef20c02231b22dec476e89f44f0221f0957604 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 28 Mar 2024 05:35:24 +0100 Subject: [PATCH 08/18] refactor(translate_apps): move translations in a manifest subfolder --- translate_apps/apps_translations_to_apps.py | 6 ++++-- translate_apps/push_or_update_apps_on_repository.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index 60c0f489..0f83f4e4 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -106,12 +106,14 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): print(app) print(f"{repository_uri} -> branch '{branch}'") - translations_path = Path(f"translations/apps/{app}/") + translations_path = Path(f"translations/apps/{app}/manifest/") if not translations_repository.file_exists(translations_path): print(f"App {app} doesn't have translations on github.com/yunohost/apps_translations, skip") continue + translations_path = translations_repository.path / translations_path + with Repository( f"https://{login}:{token}@github.com/{repository_uri}", branch ) as repository: @@ -120,7 +122,7 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): manifest = tomlkit.loads(repository.read_file("manifest.toml")) - for translation in (translations_repository.path / f"translations/apps/{app}/").glob("*.json"): + for translation in translations_path.glob("*.json"): language = translation.name[:-len(".json")] # english version is the base, never modify it diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 5be1421d..83f0d4ad 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -118,7 +118,7 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): manifest = tomlkit.loads(repository.read_file("manifest.toml")) - translations_path = Path(f"translations/apps/{app}/") + translations_path = Path(f"translations/apps/{app}/manifest/") newly_created_translation = False if not translations_repository.file_exists(translations_path): From 948c67a2e31a6a935f8c28a48f3ae23475338c59 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 28 Mar 2024 05:48:08 +0100 Subject: [PATCH 09/18] refactor(translate_apps): create a base.py file to avoid duplicating code --- translate_apps/apps_translations_to_apps.py | 79 +---------------- translate_apps/base.py | 85 +++++++++++++++++++ .../push_or_update_apps_on_repository.py | 79 +---------------- 3 files changed, 88 insertions(+), 155 deletions(-) create mode 100644 translate_apps/base.py diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index 0f83f4e4..b4ec6948 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -1,88 +1,11 @@ -import os import time import json -import tempfile -import subprocess -from typing import Union from pathlib import Path import tomlkit -github_webhook_secret = open("github_webhook_secret", "r").read().strip() - -login = open("login").read().strip() -token = open("token").read().strip() - -weblate_token = open("weblate_token").read().strip() - -my_env = os.environ.copy() -my_env["GIT_TERMINAL_PROMPT"] = "0" -my_env["GIT_AUTHOR_NAME"] = "yunohost-bot" -my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" -my_env["GIT_COMMITTER_NAME"] = "yunohost-bot" -my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" -my_env["GITHUB_USER"] = login -my_env["GITHUB_TOKEN"] = token - - -class Repository: - def __init__(self, url, branch): - self.url = url - self.branch = branch - - def __enter__(self): - self.temporary_directory = tempfile.TemporaryDirectory() - self.path = Path(self.temporary_directory.name) - self.run_command( - [ - "git", - "clone", - self.url, - "--single-branch", - "--branch", - self.branch, - self.path, - ] - ) - - return self - - def run_command( - self, command: Union[str, list], capture_output=False - ) -> Union[str, int]: - if isinstance(command, str): - kwargs = {"args": f"cd {self.path} && {command}", "shell": True, "env": my_env} - - elif isinstance(command, list): - kwargs = {"args": command, "cwd": self.path, "env": my_env} - - if capture_output: - return subprocess.check_output(**kwargs).decode() - else: - print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") - return subprocess.check_call(**kwargs) - - def file_exists(self, file_name: str) -> bool: - return (self.path / file_name).exists() - - def read_file(self, file_name: str) -> str: - return open((self.path / file_name).resolve(), "r").read() - - def write_file(self, file_name: str, content: str) -> None: - open((self.path / file_name).resolve(), "w").write(content) - - def remove_file(self, file_name: str) -> None: - os.remove(self.path / file_name) - - def append_to_file(self, file_name: str, content: str) -> None: - open((self.path / file_name).resolve(), "a").write(content) - - def __repr__(self): - return f'<__main__.Repository "{self.url.split("@")[1]}" path="{self.path}">' - - def __exit__(self, *args, **kwargs): - pass +from base import Repository, login, token def extract_strings_to_translate_from_apps(apps, translations_repository): diff --git a/translate_apps/base.py b/translate_apps/base.py new file mode 100644 index 00000000..b0f5b957 --- /dev/null +++ b/translate_apps/base.py @@ -0,0 +1,85 @@ +import os +import tempfile +import subprocess + +from typing import Union +from pathlib import Path + +github_webhook_secret = open("github_webhook_secret", "r").read().strip() + +login = open("login").read().strip() +token = open("token").read().strip() + +weblate_token = open("weblate_token").read().strip() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" +my_env["GIT_AUTHOR_NAME"] = "yunohost-bot" +my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" +my_env["GIT_COMMITTER_NAME"] = "yunohost-bot" +my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" +my_env["GITHUB_USER"] = login +my_env["GITHUB_TOKEN"] = token + + +class Repository: + def __init__(self, url, branch): + self.url = url + self.branch = branch + + def __enter__(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.path = Path(self.temporary_directory.name) + self.run_command( + [ + "git", + "clone", + self.url, + "--single-branch", + "--branch", + self.branch, + self.path, + ] + ) + + return self + + def run_command( + self, command: Union[str, list], capture_output=False + ) -> Union[str, int, subprocess.CompletedProcess]: + if isinstance(command, str): + kwargs = { + "args": f"cd {self.path} && {command}", + "shell": True, + "env": my_env, + } + + elif isinstance(command, list): + kwargs = {"args": command, "cwd": self.path, "env": my_env} + + if capture_output: + return subprocess.check_output(**kwargs).decode() + else: + print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") + return subprocess.check_call(**kwargs) + + def file_exists(self, file_name: str) -> bool: + return (self.path / file_name).exists() + + def read_file(self, file_name: str) -> str: + return open((self.path / file_name).resolve(), "r").read() + + def write_file(self, file_name: str, content: str) -> None: + open((self.path / file_name).resolve(), "w").write(content) + + def remove_file(self, file_name: str) -> None: + os.remove(self.path / file_name) + + def append_to_file(self, file_name: str, content: str) -> None: + open((self.path / file_name).resolve(), "a").write(content) + + def __repr__(self): + return f'<__main__.Repository "{self.url.split("@")[1]}" path="{self.path}">' + + def __exit__(self, *args, **kwargs): + pass diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 83f0d4ad..e8119ae3 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -1,29 +1,13 @@ -import os import time import json -import tempfile -import subprocess -from collections import defaultdict from pathlib import Path -from typing import Union +from collections import defaultdict import wlc import tomlkit -github_webhook_secret = open("github_webhook_secret", "r").read().strip() - -login = open("login").read().strip() -token = open("token").read().strip() - -weblate_token = open("weblate_token").read().strip() - -my_env = os.environ.copy() -my_env["GIT_TERMINAL_PROMPT"] = "0" -my_env["GIT_AUTHOR_NAME"] = "yunohost-bot" -my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" -my_env["GIT_COMMITTER_NAME"] = "yunohost-bot" -my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" +from base import Repository, login, token, weblate_token def get_weblate_component(weblate, component_path): @@ -35,65 +19,6 @@ def get_weblate_component(weblate, component_path): return True -class Repository: - def __init__(self, url, branch): - self.url = url - self.branch = branch - - def __enter__(self): - self.temporary_directory = tempfile.TemporaryDirectory() - self.path = Path(self.temporary_directory.name) - self.run_command( - [ - "git", - "clone", - self.url, - "--single-branch", - "--branch", - self.branch, - self.path, - ] - ) - - return self - - def run_command( - self, command: Union[str, list], capture_output=False - ) -> Union[str, int]: - if isinstance(command, str): - kwargs = {"args": f"cd {self.path} && {command}", "shell": True} - - elif isinstance(command, list): - kwargs = {"args": command, "cwd": self.path} - - if capture_output: - return subprocess.check_output(**kwargs).decode() - else: - print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") - return subprocess.check_call(**kwargs) - - def file_exists(self, file_name: str) -> bool: - return (self.path / file_name).exists() - - def read_file(self, file_name: str) -> str: - return open((self.path / file_name).resolve(), "r").read() - - def write_file(self, file_name: str, content: str) -> None: - open((self.path / file_name).resolve(), "w").write(content) - - def remove_file(self, file_name: str) -> None: - os.remove(self.path / file_name) - - def append_to_file(self, file_name: str, content: str) -> None: - open((self.path / file_name).resolve(), "a").write(content) - - def __repr__(self): - return f'<__main__.Repository "{self.url.split("@")[1]}" path="{self.path}">' - - def __exit__(self, *args, **kwargs): - pass - - def extract_strings_to_translate_from_apps(apps, translations_repository): weblate = wlc.Weblate(key=weblate_token, url="https://translate.yunohost.org/api/") From c748938a6cec04e467d85c9ef2e028fba923291c Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 28 Mar 2024 07:09:30 +0100 Subject: [PATCH 10/18] feat(translate_apps): use testing branch if it exists --- translate_apps/apps_translations_to_apps.py | 21 +++++++++++++++++-- translate_apps/base.py | 16 ++++++++++++++ .../push_or_update_apps_on_repository.py | 8 +++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index b4ec6948..ade5cc64 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -5,7 +5,7 @@ import tomlkit -from base import Repository, login, token +from base import Repository, login, token, WORKING_BRANCH def extract_strings_to_translate_from_apps(apps, translations_repository): @@ -43,6 +43,23 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if not repository.file_exists("manifest.toml"): continue + if repository.run_command_as_if(["git", "rev-parse", "--verify", "origin/testing"]): + repository.run_command(["git", "checkout", "-b", WORKING_BRANCH, "--track", "origin/testing"]) + if repository.run_command_as_if( + ["git", "rev-parse", "--verify", "origin/testing"] + ): + repository.run_command( + [ + "git", + "checkout", + "-b", + WORKING_BRANCH, + "--track", + "origin/testing", + ] + ) + repository.run_command(["git", "checkout", "-b", WORKING_BRANCH]) + manifest = tomlkit.loads(repository.read_file("manifest.toml")) for translation in translations_path.glob("*.json"): @@ -77,7 +94,7 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): repository.run_command("git diff") repository.run_command("git add manifest.toml") repository.run_command(["git", "commit", "-m", "feat(i18n): update translations for manifest.toml"]) - repository.run_command(["git", "push", "-f", "origin", "master:manifest_toml_i18n"]) + repository.run_command(["git", "push", "-f", "origin", f"{WORKING_BRANCH}:manifest_toml_i18n"]) # if no PR exist, create one if not repository.run_command("hub pr list -h manifest_toml_i18n", capture_output=True): diff --git a/translate_apps/base.py b/translate_apps/base.py index b0f5b957..f98beb9e 100644 --- a/translate_apps/base.py +++ b/translate_apps/base.py @@ -21,6 +21,8 @@ my_env["GITHUB_USER"] = login my_env["GITHUB_TOKEN"] = token +WORKING_BRANCH = "manifest_toml_i18n" + class Repository: def __init__(self, url, branch): @@ -63,6 +65,20 @@ def run_command( print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") return subprocess.check_call(**kwargs) + def run_command_as_if(self, command: Union[str, list]) -> bool: + if isinstance(command, str): + kwargs = { + "args": f"cd {self.path} && {command}", + "shell": True, + "env": my_env, + } + + elif isinstance(command, list): + kwargs = {"args": command, "cwd": self.path, "env": my_env} + + print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") + return subprocess.run(**kwargs).returncode == 0 + def file_exists(self, file_name: str) -> bool: return (self.path / file_name).exists() diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index e8119ae3..236cbebb 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -41,6 +41,14 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if not repository.file_exists("manifest.toml"): continue + # base our work on the testing branch if it exists + if repository.run_command_as_if( + ["git", "rev-parse", "--verify", "origin/testing"] + ): + repository.run_command( + ["git", "checkout", "-b", "testing", "--track", "origin/testing"] + ) + manifest = tomlkit.loads(repository.read_file("manifest.toml")) translations_path = Path(f"translations/apps/{app}/manifest/") From 01e619b08427f05628a6466f6cf0c076bd65cabb Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 28 Mar 2024 07:51:31 +0100 Subject: [PATCH 11/18] fix(translate_apps): correctly indent a new translation --- translate_apps/apps_translations_to_apps.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index ade5cc64..20477922 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -83,7 +83,10 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if strings_to_translate not in manifest["install"][question]: continue + one_of_the_existing_languages = list(manifest["install"][question][strings_to_translate].keys())[0] + current_identation = len(manifest["install"][question][strings_to_translate][one_of_the_existing_languages].trivia.indent) manifest["install"][question][strings_to_translate][language] = translation[translation_key] + manifest["install"][question][strings_to_translate][language].indent(current_identation) repository.write_file("manifest.toml", tomlkit.dumps(manifest)) @@ -93,10 +96,20 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): # create or update merge request repository.run_command("git diff") repository.run_command("git add manifest.toml") - repository.run_command(["git", "commit", "-m", "feat(i18n): update translations for manifest.toml"]) - repository.run_command(["git", "push", "-f", "origin", f"{WORKING_BRANCH}:manifest_toml_i18n"]) - - # if no PR exist, create one + one_of_the_existing_languages = list( + manifest["install"][question][strings_to_translate].keys() + )[0] + current_identation = len( + manifest["install"][question][strings_to_translate][ + one_of_the_existing_languages + ].trivia.indent + ) + manifest["install"][question][strings_to_translate][ + language + ] = translation[translation_key] + manifest["install"][question][strings_to_translate][ + language + ].indent(current_identation) if not repository.run_command("hub pr list -h manifest_toml_i18n", capture_output=True): repository.run_command(["hub", "pull-request", "-m", "Update translations for manifest.toml", "-b", branch, "-h", "manifest_toml_i18n", "-p", "-m", f"This pull request is automatically generated by scripts from the [YunoHost/apps](https://github.com/YunoHost/apps) repository.\n\nThe translation is pull from weblate and is located here: https://translate.yunohost.org/projects/yunohost-apps/{app}/\n\nIf you wish to modify the translation (other than in english), please do that directly on weblate since this is now the source of authority for it.\n\nDon't hesitate to reach the YunoHost team on [matrix](https://matrix.to/#/#yunohost:matrix.org) if there is any problem :heart:"]) From c22005dee2678789d9cdb1422f561f970b204ace Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 29 Mar 2024 00:01:04 +0100 Subject: [PATCH 12/18] feat(translate_apps): make output more readable --- translate_apps/apps_translations_to_apps.py | 2 ++ translate_apps/push_or_update_apps_on_repository.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index 20477922..66a1abe0 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -26,7 +26,9 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): ): continue + print() print(app) + print("=" * len(app)) print(f"{repository_uri} -> branch '{branch}'") translations_path = Path(f"translations/apps/{app}/manifest/") diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 236cbebb..c0e92573 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -32,7 +32,9 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if app not in ("gotosocial", "fluffychat", "cinny", "fittrackee", "funkwhale", "photoprism"): continue + print() print(app) + print("=" * len(app)) print(f"{repository_uri} -> branch '{branch}'") with Repository( From d4e4eb96686189be6b34e0c8cb6ec389e7c94562 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 29 Mar 2024 05:59:35 +0100 Subject: [PATCH 13/18] chore: fix git absorb mess --- translate_apps/apps_translations_to_apps.py | 57 ++++++++++--------- .../push_or_update_apps_on_repository.py | 22 +++---- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index 66a1abe0..0f745e56 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -46,10 +46,6 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): continue if repository.run_command_as_if(["git", "rev-parse", "--verify", "origin/testing"]): - repository.run_command(["git", "checkout", "-b", WORKING_BRANCH, "--track", "origin/testing"]) - if repository.run_command_as_if( - ["git", "rev-parse", "--verify", "origin/testing"] - ): repository.run_command( [ "git", @@ -60,6 +56,8 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): "origin/testing", ] ) + branch = "testing" + else: repository.run_command(["git", "checkout", "-b", WORKING_BRANCH]) manifest = tomlkit.loads(repository.read_file("manifest.toml")) @@ -85,19 +83,6 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if strings_to_translate not in manifest["install"][question]: continue - one_of_the_existing_languages = list(manifest["install"][question][strings_to_translate].keys())[0] - current_identation = len(manifest["install"][question][strings_to_translate][one_of_the_existing_languages].trivia.indent) - manifest["install"][question][strings_to_translate][language] = translation[translation_key] - manifest["install"][question][strings_to_translate][language].indent(current_identation) - - repository.write_file("manifest.toml", tomlkit.dumps(manifest)) - - if not repository.run_command("git status -s", capture_output=True).strip(): - continue - - # create or update merge request - repository.run_command("git diff") - repository.run_command("git add manifest.toml") one_of_the_existing_languages = list( manifest["install"][question][strings_to_translate].keys() )[0] @@ -112,19 +97,18 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): manifest["install"][question][strings_to_translate][ language ].indent(current_identation) - if not repository.run_command("hub pr list -h manifest_toml_i18n", capture_output=True): - repository.run_command(["hub", "pull-request", "-m", "Update translations for manifest.toml", "-b", branch, "-h", "manifest_toml_i18n", "-p", "-m", f"This pull request is automatically generated by scripts from the [YunoHost/apps](https://github.com/YunoHost/apps) repository.\n\nThe translation is pull from weblate and is located here: https://translate.yunohost.org/projects/yunohost-apps/{app}/\n\nIf you wish to modify the translation (other than in english), please do that directly on weblate since this is now the source of authority for it.\n\nDon't hesitate to reach the YunoHost team on [matrix](https://matrix.to/#/#yunohost:matrix.org) if there is any problem :heart:"]) - time.sleep(2) + repository.write_file("manifest.toml", tomlkit.dumps(manifest)) + if not repository.run_command("git status -s", capture_output=True).strip(): + continue -if __name__ == "__main__": - apps = json.load(open("../../builds/default/v3/apps.json"))["apps"] + # create or update merge request + repository.run_command("git diff") + repository.run_command("git add manifest.toml") + repository.run_command(["git", "commit", "-m", "feat(i18n): update translations for manifest.toml"]) + repository.run_command(["git", "push", "-f", "origin", f"{WORKING_BRANCH}:manifest_toml_i18n"]) - with Repository( - f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" - ) as repository: - extract_strings_to_translate_from_apps(apps, repository) if not repository.run_command( "hub pr list -h manifest_toml_i18n", capture_output=True ): @@ -140,6 +124,25 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): "manifest_toml_i18n", "-p", "-m", - f"This pull request is automatically generated by scripts from the [YunoHost/apps](https://github.com/YunoHost/apps) repository.\n\nThe translation is pull from weblate and is located here: https://translate.yunohost.org/projects/yunohost-apps/{app}/\n\nIf you wish to modify the translation (other than in english), please do that directly on weblate since this is now the source of authority for it.\n\nDon't hesitate to reach the YunoHost team on [matrix](https://matrix.to/#/#yunohost:matrix.org) if there is any problem :heart:", + "This pull request is automatically generated by scripts from the " + "[YunoHost/apps](https://github.com/YunoHost/apps) repository.\n\n" + "The translation is pull from weblate and is located here: " + f"https://translate.yunohost.org/projects/yunohost-apps/{app}/\n\n" + "If you wish to modify the translation (other than in english), please do " + "that directly on weblate since this is now the source of authority for it." + "\n\nDon't hesitate to reach the YunoHost team on " + "[matrix](https://matrix.to/#/#yunohost:matrix.org) if there is any " + "problem :heart:", ] ) + + time.sleep(2) + + +if __name__ == "__main__": + apps = json.load(open("../../builds/default/v3/apps.json"))["apps"] + + with Repository( + f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" + ) as repository: + extract_strings_to_translate_from_apps(apps, repository) diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index c0e92573..75ad679e 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -105,9 +105,7 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): for key, translated_string in translated_strings.items(): if key not in language_file: language_file[key] = translated_string - language_file = json.load( - (translations_path / f"{language}.json").open() - ) + translations_repository.write_file( translations_path / f"{language}.json", json.dumps(language_file, indent=4, sort_keys=True), @@ -128,19 +126,19 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): ) translations_repository.run_command(["git", "push"]) - if newly_created_translation or not get_weblate_component(weblate, f"yunohost-apps/{app}"): + if newly_created_translation or not get_weblate_component( + weblate, f"yunohost-apps/{app}" + ): print("Creating component on weblate...") weblate.create_component( "yunohost-apps", name=app, slug=app, - if newly_created_translation or not get_weblate_component( - weblate, f"yunohost-apps/{app}" - ): - filemask=f"translations/apps/{app}/*.json", + file_format="json", + filemask=f"translations/apps/{app}/manifest/*.json", repo="https://github.com/yunohost/apps_translations", - new_base=f"translations/apps/{app}/en.json", - template=f"translations/apps/{app}/en.json", + new_base=f"translations/apps/{app}/manifest/en.json", + template=f"translations/apps/{app}/manifest/en.json", push="git@github.com:yunohost/apps_translations.git", ) print(f"Component created at https://translate.yunohost.org/projects/yunohost-apps/{app}/") @@ -149,9 +147,7 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if __name__ == "__main__": - print( - f"Component created at https://translate.yunohost.org/projects/yunohost-apps/{app}/" - ) + apps = json.load(open("../../builds/default/v3/apps.json"))["apps"] with Repository( f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" From 781a8d25c136e2723cbb39b507c78aa53c798017 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 29 Mar 2024 06:35:13 +0100 Subject: [PATCH 14/18] feat(translate_apps): for every app component add as available languages for translation all the ones available in yunohost/core --- translate_apps/push_or_update_apps_on_repository.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 75ad679e..b6b551bf 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -22,6 +22,9 @@ def get_weblate_component(weblate, component_path): def extract_strings_to_translate_from_apps(apps, translations_repository): weblate = wlc.Weblate(key=weblate_token, url="https://translate.yunohost.org/api/") + # put all languages used on core by default for each component + core_languages_list = {x["language_code"] for x in weblate.get("components/yunohost/core/translations/")["results"]} + for app, infos in apps.items(): repository_uri = infos["git"]["url"].replace("https://github.com/", "") branch = infos["git"]["branch"] @@ -143,6 +146,11 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): ) print(f"Component created at https://translate.yunohost.org/projects/yunohost-apps/{app}/") + component_existing_languages = {x["language_code"] for x in weblate.get(f"components/yunohost-apps/{app}/translations/")["results"]} + for language_code in sorted(core_languages_list - component_existing_languages): + print(f"Adding available language for translation: {language_code}") + weblate.post(f"components/yunohost-apps/{app}/translations/", **{"language_code": language_code}) + time.sleep(2) From c526d413bf7676900f0a31c93547139baf7632cb Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 31 Mar 2024 06:44:12 +0200 Subject: [PATCH 15/18] feat(translate_apps): use json.dumps the same way than weblate to avoid duplications --- translate_apps/push_or_update_apps_on_repository.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index b6b551bf..6a64ef3e 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -84,12 +84,12 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): for language, translated_strings in translations.items(): translations_repository.write_file( translations_path / f"{language}.json", - json.dumps(translated_strings, indent=4, sort_keys=True), + json.dumps(translated_strings, indent=4, sort_keys=True, ensure_ascii=False) + "\n", ) else: translations_repository.write_file( translations_path / "en.json", - json.dumps(translations["en"], indent=4, sort_keys=True), + json.dumps(translations["en"], indent=4, sort_keys=True, ensure_ascii=False) + "\n", ) # add strings that aren't already present but don't overwrite existing ones @@ -101,8 +101,9 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): if not (translations_path / f"{language}.json").exists(): translations_repository.write_file( translations_path / f"{language}.json", - json.dumps(translated_strings, indent=4, sort_keys=True), + json.dumps(translated_strings, indent=4, sort_keys=True, ensure_ascii=False) + "\n", ) + else: # if it exists, only add keys that aren't already present language_file = json.load((translations_path / f"{language}.json").open()) for key, translated_string in translated_strings.items(): @@ -111,7 +112,7 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): translations_repository.write_file( translations_path / f"{language}.json", - json.dumps(language_file, indent=4, sort_keys=True), + json.dumps(language_file, indent=4, sort_keys=True, ensure_ascii=False) + "\n", ) # if something has been modified From c7b3d908b38f1598f64b867f9b5de8fc44eb0ce3 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 31 Mar 2024 06:44:43 +0200 Subject: [PATCH 16/18] fix(translate_apps): the path to check if files existed was in the wrong PWD --- translate_apps/push_or_update_apps_on_repository.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 6a64ef3e..545702e1 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -98,14 +98,15 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): continue # if the translation file doesn't exist yet, dump it - if not (translations_path / f"{language}.json").exists(): + if not translations_repository.file_exists(translations_path / f"{language}.json"): translations_repository.write_file( translations_path / f"{language}.json", json.dumps(translated_strings, indent=4, sort_keys=True, ensure_ascii=False) + "\n", ) else: # if it exists, only add keys that aren't already present - language_file = json.load((translations_path / f"{language}.json").open()) + language_file = json.loads(translations_repository.read_file(translations_path / f"{language}.json")) + for key, translated_string in translated_strings.items(): if key not in language_file: language_file[key] = translated_string From e729dbdf6004a0cfc9da2123a1bfcce4dd8b5d19 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 31 Mar 2024 06:46:07 +0200 Subject: [PATCH 17/18] feat(translate_apps): handle also updating translation description on existing files --- translate_apps/push_or_update_apps_on_repository.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index 545702e1..afdbfefd 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -107,6 +107,9 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): else: # if it exists, only add keys that aren't already present language_file = json.loads(translations_repository.read_file(translations_path / f"{language}.json")) + if "description" in translated_strings and "description" not in language_file: + language_file["description"] = translated_strings["description"] + for key, translated_string in translated_strings.items(): if key not in language_file: language_file[key] = translated_string From 15340bc8eb9743fa0ef1e96956d5736d68714384 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 2 Apr 2024 07:10:39 +0200 Subject: [PATCH 18/18] fix(translate_apps): testing branch existance needed to be checked before cloning --- translate_apps/apps_translations_to_apps.py | 29 +++++++++---------- translate_apps/base.py | 15 ++++++++++ .../push_or_update_apps_on_repository.py | 13 +++------ 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/translate_apps/apps_translations_to_apps.py b/translate_apps/apps_translations_to_apps.py index 0f745e56..6d77c17d 100644 --- a/translate_apps/apps_translations_to_apps.py +++ b/translate_apps/apps_translations_to_apps.py @@ -5,7 +5,7 @@ import tomlkit -from base import Repository, login, token, WORKING_BRANCH +from base import Repository, login, token, WORKING_BRANCH, get_repository_branches def extract_strings_to_translate_from_apps(apps, translations_repository): @@ -39,26 +39,25 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): translations_path = translations_repository.path / translations_path + if "testing" in get_repository_branches(repository_uri, token): + branch = "testing" + with Repository( f"https://{login}:{token}@github.com/{repository_uri}", branch ) as repository: if not repository.file_exists("manifest.toml"): continue - if repository.run_command_as_if(["git", "rev-parse", "--verify", "origin/testing"]): - repository.run_command( - [ - "git", - "checkout", - "-b", - WORKING_BRANCH, - "--track", - "origin/testing", - ] - ) - branch = "testing" - else: - repository.run_command(["git", "checkout", "-b", WORKING_BRANCH]) + repository.run_command( + [ + "git", + "checkout", + "-b", + WORKING_BRANCH, + "--track", + "origin/{branch}", + ] + ) manifest = tomlkit.loads(repository.read_file("manifest.toml")) diff --git a/translate_apps/base.py b/translate_apps/base.py index f98beb9e..d739da8d 100644 --- a/translate_apps/base.py +++ b/translate_apps/base.py @@ -2,6 +2,8 @@ import tempfile import subprocess +import requests + from typing import Union from pathlib import Path @@ -24,6 +26,19 @@ WORKING_BRANCH = "manifest_toml_i18n" +def get_repository_branches(repository, token): + branches = requests.get( + f"https://api.github.com/repos/{repository}/branches", + headers={ + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "Accept": "application/vnd.github+json", + }, + ).json() + + return {x["name"] for x in branches} + + class Repository: def __init__(self, url, branch): self.url = url diff --git a/translate_apps/push_or_update_apps_on_repository.py b/translate_apps/push_or_update_apps_on_repository.py index afdbfefd..453d8939 100644 --- a/translate_apps/push_or_update_apps_on_repository.py +++ b/translate_apps/push_or_update_apps_on_repository.py @@ -7,7 +7,7 @@ import wlc import tomlkit -from base import Repository, login, token, weblate_token +from base import Repository, login, token, weblate_token, get_repository_branches def get_weblate_component(weblate, component_path): @@ -40,20 +40,15 @@ def extract_strings_to_translate_from_apps(apps, translations_repository): print("=" * len(app)) print(f"{repository_uri} -> branch '{branch}'") + if "testing" in get_repository_branches(repository_uri, token): + branch = "testing" + with Repository( f"https://{login}:{token}@github.com/{repository_uri}", branch ) as repository: if not repository.file_exists("manifest.toml"): continue - # base our work on the testing branch if it exists - if repository.run_command_as_if( - ["git", "rev-parse", "--verify", "origin/testing"] - ): - repository.run_command( - ["git", "checkout", "-b", "testing", "--track", "origin/testing"] - ) - manifest = tomlkit.loads(repository.read_file("manifest.toml")) translations_path = Path(f"translations/apps/{app}/manifest/")