Skip to content

Commit

Permalink
feat: Improved Crowdin command. Can choose reference languages
Browse files Browse the repository at this point in the history
  • Loading branch information
@kargnas
kargnas committed Aug 2, 2024
1 parent e3e3bf1 commit 1adec22
Showing 1 changed file with 139 additions and 93 deletions.
232 changes: 139 additions & 93 deletions src/Console/TranslateCrowdin.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ class TranslateCrowdin extends Command

protected Crowdin $crowdin;
protected array $selectedProject;
protected array $referenceLanguages = [];
protected string $targetLanguage;

public function handle() {
if (!env('CROWDIN_API_KEY')) {
Expand All @@ -72,6 +74,18 @@ public function handle() {
$this->chunkSize = $this->ask('How many strings to translate at once?', 30);

$this->choiceProjects();
$this->targetLanguage = $this->choiceLanguages("Choose a language to translate", false);


if ($this->ask('Do you want to choose reference languages? (y/n)', 'n') === 'y') {
$this->referenceLanguages = $this->choiceLanguages("Choose a language to reference when translating, preferably one that has already been vetted and translated to a high quality. You can select multiple languages via ',' (e.g. '1, 2')", true);
}

$this->info("Target language: {$this->targetLanguage}");
if ($this->referenceLanguages) {
$this->info("Reference languages: " . implode(", ", $this->referenceLanguages));
}

$this->translate();
}

Expand Down Expand Up @@ -256,116 +270,148 @@ public function choiceProjects() {
$this->info("Selected project: {$this->selectedProject['name']} ({$this->selectedProject['id']})");
}

public function choiceLanguages($question, $multiple, $default = null) {
$locales = collect($this->selectedProject['targetLanguages'])->sortBy('id')->pluck('id')->values()->toArray();

$selectedLocales = $this->choice(
$question,
$locales,
$default,
3,
$multiple);

return $selectedLocales;
}

public function translate() {
$languages = collect($this->selectedProject['targetLanguages'])->sortBy(function ($languagae) {
$firstOrder = [
'vi', 'en', 'ko', 'zh', 'ja',
];
if (in_array($languagae['twoLettersCode'], $firstOrder)) {
return array_search($languagae['twoLettersCode'], $firstOrder);
}
return 100;
})->values();
$targetLanguage = collect($this->selectedProject['targetLanguages'])->where('id', $this->targetLanguage)->first();

foreach ($languages as $targetLanguage) {
$locale = $targetLanguage['locale'];
$pluralRules = $targetLanguage['pluralRules'];
$pluralExamples = $targetLanguage['pluralExamples'];
$locale = $targetLanguage['locale'];
$pluralRules = $targetLanguage['pluralRules'];
$pluralExamples = $targetLanguage['pluralExamples'];

$this->info("Translating to {$targetLanguage['name']} ({$targetLanguage['id']})");
$this->info(" Locale: {$locale}");
$this->info(" Plural Rules: {$pluralRules}");
$this->info(" Plural Examples: " . implode(", ", array_keys($pluralExamples)));
$this->info("Source Language: {$this->selectedProject['sourceLanguage']['name']} ({$this->selectedProject['sourceLanguage']['id']})");
$this->info("Translating to {$targetLanguage['name']} ({$targetLanguage['id']})");
$this->info(" Locale: {$locale}");
$this->info(" Plural Rules: {$pluralRules}");
$this->info(" Plural Examples: " . implode(", ", array_keys($pluralExamples)));

$this->ask('Press any key to continue', 'continue', ['continue', 'c']);
foreach ($this->getAllDirectories($this->selectedProject['id']) as $directory) {
$directory = $directory->getData();
$files = collect($this->getAllFiles($this->selectedProject['id'], $directory['id']))->map(function ($file) {
return $file->getData();
});
if ($files->count() === 0) continue;

foreach ($this->getAllDirectories($this->selectedProject['id']) as $directory) {
$directory = $directory->getData();
$files = collect($this->getAllFiles($this->selectedProject['id'], $directory['id']))->map(function ($file) {
return $file->getData();
});
if ($files->count() === 0) continue;
$this->info(" Directory: {$directory['path']} ({$files->count()} files)");

$this->info(" Directory: {$directory['path']} ({$files->count()} files)");
foreach ($files as $file) {
$this->info(" File: {$file['name']} ({$file['id']})");

foreach ($files as $file) {
$this->info(" File: {$file['name']} ({$file['id']})");
$this->line(" Retrieving strings...");
$allStrings = $this->getAllSourceString($this->selectedProject['id'], $file['id']);

$allStrings = $this->getAllSourceString($this->selectedProject['id'], $file['id']);
$approvals = $this->getApprovals($this->selectedProject['id'], $file['id'], $targetLanguage['id']);
$this->line(" Retrieving approvals...");
$approvals = $this->getApprovals($this->selectedProject['id'], $file['id'], $targetLanguage['id']);

$untranslatedStrings = $allStrings
->filter(function (SourceString $sourceString) use ($approvals) {
if (!$sourceString->getIdentifier()) return false;
$referenceApprovals = collect($this->referenceLanguages)->reject($this->selectedProject['sourceLanguage']['id'])->mapWithKeys(function ($refLocale) use ($allStrings, $file) {
$this->line(" Retrieving approvals for reference language...: {$refLocale}");
$approvals = $this->getApprovals($this->selectedProject['id'], $file['id'], $refLocale);

if ($sourceString->isHidden()) {
$this->line(" Retrieving translations for reference language...: {$refLocale}");
$allTranslations = $this->getAllLanguageTranslations($this->selectedProject['id'], $file['id'], $refLocale);

return [
$refLocale => collect($allStrings)->mapWithKeys(function (SourceString $sourceString) use ($approvals, $allTranslations) {
$approved = $approvals->map(fn(StringTranslationApproval $ap) => $ap->getData())->where('stringId', $sourceString->getId())->first();
if (!$approved) return [];

$approvedTranslation = $allTranslations->map(fn(LanguageTranslation $t) => $t->getData())->where('translationId', $approved['translationId'])->first();
if (!$approvedTranslation) return [];

return [
$sourceString->getIdentifier() => $approvedTranslation['text'],
];
}),
];
});

$untranslatedStrings = $allStrings
->filter(function (SourceString $sourceString) use ($approvals) {
if (!$sourceString->getIdentifier()) return false;

if ($sourceString->isHidden()) {
// $this->line(" Skip: {$sourceString->getIdentifier()}: {$sourceString->getText()} (hidden)");
return false;
}
return false;
}

if (!$approvals->filter(fn(StringTranslationApproval $ap) => $ap->getStringId() == $sourceString->getId())->isEmpty()) {
if (!$approvals->filter(fn(StringTranslationApproval $ap) => $ap->getStringId() == $sourceString->getId())->isEmpty()) {
// $this->line(" Skip: {$sourceString->getIdentifier()}: {$sourceString->getText()} (approved)");
return false;
}

return true;
})
->map(function (SourceString $sourceString) use ($targetLanguage) {
return $sourceString->getData();
});

$this->info(" Total: {$allStrings->count()} strings");
$this->info(" Untranslated: {$untranslatedStrings->count()} strings");

$untranslatedStrings
->chunk($this->chunkSize)
->each(function ($chunk) use ($file, $targetLanguage, $untranslatedStrings) {
$translator = new AIProvider(
filename: $file['name'],
strings: $chunk->mapWithKeys(function ($string) {
$context = $string['context'] ?? null;
$context = preg_replace("/[\.\s\->]/", "", $context);
if (preg_replace("/[\.\s\->]/", "", $string['identifier']) === $context) {
$context = null;
}

return [
$string['identifier'] => [
'text' => $string['text'],
'context' => $context,
],
];
})->toArray(),
sourceLanguage: $this->selectedProject['sourceLanguage']['name'],
targetLanguage: $targetLanguage['name'],
additionalRules: static::getAdditionalRules($targetLanguage['locale']),
);

$translated = $translator->translate();

foreach ($translated as $item) {
$targetString = $untranslatedStrings->where('identifier', $item->key)->first();

$existsTranslations = $this->getAllTranslations($this->selectedProject['id'], $targetString['id'], $targetLanguage['id']);
$existsTranslations = $existsTranslations->sortByDesc(fn(StringTranslation $t) => Carbon::make($t->getDataProperty('created_at')))->values();

// 같은 번역이 있다면 패스
if ($existsTranslations->filter(fn(StringTranslation $t) => $t->getText() === $item->translated)->isNotEmpty()) {
$this->info("Skipping translation: {$item->key} [{$targetString['id']}]: {$item->translated} (Duplicated)");
continue;
return false;
}

return true;
})
->map(function (SourceString $sourceString) use ($targetLanguage) {
return $sourceString->getData();
});

$this->info(" Total: {$allStrings->count()} strings");
$this->info(" Untranslated: {$untranslatedStrings->count()} strings");

$untranslatedStrings
->chunk($this->chunkSize)
->each(function ($chunk) use ($file, $targetLanguage, $untranslatedStrings, $referenceApprovals) {
$translator = new AIProvider(
filename: $file['name'],
strings: $chunk->mapWithKeys(function ($string) use ($referenceApprovals) {
$context = $string['context'] ?? null;
$context = preg_replace("/[\.\s\->]/", "", $context);
if (preg_replace("/[\.\s\->]/", "", $string['identifier']) === $context) {
$context = null;
}

$this->info("Adding translation: {$item->key} [{$targetString['id']}]: {$item->translated}");
$myTransitions = $existsTranslations->filter(fn(StringTranslation $t) => $t->getUser()['id'] === 16501205);
if ($myTransitions->count() > 0) {
$this->delTranslation($this->selectedProject['id'], $myTransitions->first()->getId());
}
$this->addTranslation($this->selectedProject['id'], $targetString['id'], $targetLanguage['id'], $item->translated);
return [
$string['identifier'] => [
'text' => $string['text'],
'context' => $context,
'references' => $referenceApprovals->map(function ($items) use ($string) {
return $items[$string['identifier']] ?? "";
})->filter(function ($value) {
return strlen($value) > 0;
}),
],
];
})->toArray(),
sourceLanguage: $this->selectedProject['sourceLanguage']['name'],
targetLanguage: $targetLanguage['name'],
additionalRules: static::getAdditionalRules($targetLanguage['locale']),
);

$translated = $translator->translate();

foreach ($translated as $item) {
$targetString = $untranslatedStrings->where('identifier', $item->key)->first();

$existsTranslations = $this->getAllTranslations($this->selectedProject['id'], $targetString['id'], $targetLanguage['id']);
$existsTranslations = $existsTranslations->sortByDesc(fn(StringTranslation $t) => Carbon::make($t->getDataProperty('created_at')))->values();

// 같은 번역이 있다면 패스
if ($existsTranslations->filter(fn(StringTranslation $t) => $t->getText() === $item->translated)->isNotEmpty()) {
$this->info("Skipping translation: {$item->key} [{$targetString['id']}]: {$item->translated} (Duplicated)");
continue;
}

$this->info("Adding translation: {$item->key} [{$targetString['id']}]: {$item->translated}");
$myTransitions = $existsTranslations->filter(fn(StringTranslation $t) => $t->getUser()['id'] === 16501205);
if ($myTransitions->count() > 0) {
$this->delTranslation($this->selectedProject['id'], $myTransitions->first()->getId());
}
});
$this->addTranslation($this->selectedProject['id'], $targetString['id'], $targetLanguage['id'], $item->translated);
}
});

// dd($sourceStrings);
}
// dd($sourceStrings);
}
}
}
Expand Down

0 comments on commit 1adec22

Please sign in to comment.