From 58312843945dd3439dc9d5bdba72ed9eb79dfd08 Mon Sep 17 00:00:00 2001 From: icnocop Date: Sat, 17 Nov 2018 21:59:13 -0800 Subject: [PATCH 1/4] Added import test to fix import for parse_rc Fixes #92 --- .travis.yml | 1 + t/data/importer/common.serge | 28 +++ .../00/reference-output/database/files | 4 + .../00/reference-output/database/items | 8 + .../00/reference-output/database/properties | 7 + .../00/reference-output/database/strings | 8 + .../00/reference-output/database/translations | 3 + .../parse_rc/00/resources/application.rc | 47 +++++ t/data/importer/parse_rc/00/translate.serge | 13 ++ t/importer.t | 170 ++++++++++++++++++ 10 files changed, 289 insertions(+) create mode 100644 t/data/importer/common.serge create mode 100644 t/data/importer/parse_rc/00/reference-output/database/files create mode 100644 t/data/importer/parse_rc/00/reference-output/database/items create mode 100644 t/data/importer/parse_rc/00/reference-output/database/properties create mode 100644 t/data/importer/parse_rc/00/reference-output/database/strings create mode 100644 t/data/importer/parse_rc/00/reference-output/database/translations create mode 100644 t/data/importer/parse_rc/00/resources/application.rc create mode 100644 t/data/importer/parse_rc/00/translate.serge create mode 100644 t/importer.t diff --git a/.travis.yml b/.travis.yml index c8391e34..2bc8095d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ script: - cover -delete - perl -MDevel::Cover t/compile.t - perl -MDevel::Cover t/engine.t + - perl -MDevel::Cover t/importer.t after_success: - cover -report codecov perl: diff --git a/t/data/importer/common.serge b/t/data/importer/common.serge new file mode 100644 index 00000000..080e2d99 --- /dev/null +++ b/t/data/importer/common.serge @@ -0,0 +1,28 @@ +job_template +{ + name Test Job + id test_job + active YES + source_language en + destination_languages test + source_dir ./resources + db_source DBI:SQLite:dbname=:memory: + db_namespace test_namespace + ts_file_path ./test-output/po/%LOCALE%/%FILE%.po + output_bom NO + output_file_path ./test-output/localized-resources/%LOCALE%/%FILE% + + callback_plugins + { + :test_language + { + plugin test_language + phase get_translation_pre + + data + { + save_translations YES + } + } + } +} diff --git a/t/data/importer/parse_rc/00/reference-output/database/files b/t/data/importer/parse_rc/00/reference-output/database/files new file mode 100644 index 00000000..3ba7767a --- /dev/null +++ b/t/data/importer/parse_rc/00/reference-output/database/files @@ -0,0 +1,4 @@ +files +{ + 0 1 1 test_job test_namespace application.rc 0 +} diff --git a/t/data/importer/parse_rc/00/reference-output/database/items b/t/data/importer/parse_rc/00/reference-output/database/items new file mode 100644 index 00000000..0949511b --- /dev/null +++ b/t/data/importer/parse_rc/00/reference-output/database/items @@ -0,0 +1,8 @@ +items +{ + 0 1 3 1 1 IDS_TEST_1 NO 0 + 1 2 5 1 2 IDS_TEST_2 NO 0 + 2 3 7 1 3 IDS_ERROR NO 0 + 3 4 9 1 4 IDS_TEST_3 NO 0 + 4 5 11 1 5 IDS_TEST_4 NO 0 +} diff --git a/t/data/importer/parse_rc/00/reference-output/database/properties b/t/data/importer/parse_rc/00/reference-output/database/properties new file mode 100644 index 00000000..8897b851 --- /dev/null +++ b/t/data/importer/parse_rc/00/reference-output/database/properties @@ -0,0 +1,7 @@ +properties +{ + 0 1 source:1 b88e2e32e7cd852f7d4b76ba913a78a1 + 1 2 hash:1 b88e2e32e7cd852f7d4b76ba913a78a1 + 2 3 size:1 923 + 3 4 items:1 1,2,3,4,5 +} diff --git a/t/data/importer/parse_rc/00/reference-output/database/strings b/t/data/importer/parse_rc/00/reference-output/database/strings new file mode 100644 index 00000000..f99049b3 --- /dev/null +++ b/t/data/importer/parse_rc/00/reference-output/database/strings @@ -0,0 +1,8 @@ +strings +{ + 0 1 2 `Value 1` NO 0 + 1 2 4 `Value 2` NO 0 + 2 3 6 `Error %s` NO 0 + 3 4 8 `Value 3` NO 0 + 4 5 10 `Value 4` NO 0 +} diff --git a/t/data/importer/parse_rc/00/reference-output/database/translations b/t/data/importer/parse_rc/00/reference-output/database/translations new file mode 100644 index 00000000..244145a8 --- /dev/null +++ b/t/data/importer/parse_rc/00/reference-output/database/translations @@ -0,0 +1,3 @@ +translations +{ +} diff --git a/t/data/importer/parse_rc/00/resources/application.rc b/t/data/importer/parse_rc/00/resources/application.rc new file mode 100644 index 00000000..92b213df --- /dev/null +++ b/t/data/importer/parse_rc/00/resources/application.rc @@ -0,0 +1,47 @@ +// Microsoft Visual C++ generated resource script. +// +#include "resource.h" + +///////////////////////////////////////////////////////////////////////////// +// +// Test Version +// + +VS_VERSION_INFO VERSIONINFO + VERSION 1,0,0,1111 + PRODUCTVERSION 4,0,0,1111 + FILEFLAGSMASK 0x12L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "Sample Name", "Sample Name Value" + VALUE "Description", "Good description" + VALUE "Version", "1,0,0,1111" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Alternative", 0x409, 1200 + END +END + +///////////////////////////////////////////////////////////////////////////// +// +// Test Strings +// + +STRINGTABLE +BEGIN + IDS_TEST_1 "Value 1" + IDS_TEST_2 "Value 2" + IDS_ERROR "Error %s" +END + +STRINGTABLE +BEGIN + IDS_TEST_3 + "Value 3" + IDS_TEST_4 "Value 4" +END \ No newline at end of file diff --git a/t/data/importer/parse_rc/00/translate.serge b/t/data/importer/parse_rc/00/translate.serge new file mode 100644 index 00000000..f8b21306 --- /dev/null +++ b/t/data/importer/parse_rc/00/translate.serge @@ -0,0 +1,13 @@ +jobs +{ + { + @inherit ../../common.serge#job_template + + source_match \.rc$ + + parser + { + plugin parse_rc + } + } +} diff --git a/t/importer.t b/t/importer.t new file mode 100644 index 00000000..63c64f92 --- /dev/null +++ b/t/importer.t @@ -0,0 +1,170 @@ +#!/usr/bin/env perl + +use strict; + +# HOW TO USE THIS TEST +# +# By default, this test runs over all directories in t/data/importer/. To run +# the test only for specific directories, pass the directory names to this +# script or assign them to the environment variable SERGE_IMPORTER_TESTS as a +# comma-separated list. The following two examples are equivalent: +# +# perl t/importer.t parse_json parse_strings +# SERGE_IMPORTER_TESTS=parse_json,parse_strings prove t/importer.t + +BEGIN { + use Cwd qw(abs_path); + use File::Basename; + use File::Spec::Functions qw(catfile); + map { unshift(@INC, catfile(dirname(abs_path(__FILE__)), $_)) } qw(lib ../lib); +} + +use Data::Dumper; +use File::Copy::Recursive qw/dircopy/; +use File::Find qw(find); +use File::Path; +use File::Spec::Functions qw(catfile); +use Getopt::Long; +use Test::Config; +use Test::Diff; +use Test::More; +use Test::DB::Dumper; +use Serge::Importer; +use Serge::Engine::Job; + +$| = 1; # disable output buffering + +# to get the same results between tests, +# we override the `file_mtime` function to return a constant value; +# Serge::Importer automatically imports this function from Serge::Util +# (this is why it actually appears in Serge::Importer namespace) +sub Serge::Importer::file_mtime { + return 12345678; +} + +my $this_dir = dirname(abs_path(__FILE__)); +my $tests_dir = catfile($this_dir, 'data', 'importer'); + +my @importer_confs; + +my ($init_references); + +GetOptions("init" => \$init_references); + +my @importer_dirs = @ARGV; +if (my $env_dirs = $ENV{SERGE_IMPORTER_TESTS}) { + push @importer_dirs, split(/,/, $env_dirs); +} + +unless (@importer_dirs) { + find(sub { + push @importer_confs, $File::Find::name if(-f $_ && /\.serge$/ && $_ ne 'common.serge'); + }, $tests_dir); +} else { + for my $dir (@importer_dirs) { + find(sub { + push @importer_confs, $File::Find::name if(-f $_ && /\.serge$/ && $_ ne 'common.serge'); + }, catfile($tests_dir, $dir)); + } +} + +sub delete_directory { + my ($path, $ignore_errors) = @_; + + my $err; + + if (-e $path) { + rmtree($path, { error => \$err }); + if (@$err && !$ignore_errors) { + my $err_text = ''; + + map { + foreach my $key (keys %$_) { + $err_text .= $key.': '.$_->{$key}."\n"; + } + } @$err; + + BAIL_OUT("Directory '".$path."' couldn't be removed\n$err_text"); + } + } +} + +for my $config_file (@importer_confs) { + + subtest "Test config: $config_file" => sub { + my $cfg = Test::Config->new($config_file); + + SKIP: { + my $ok = ok(defined $cfg, 'Config file read'); + skip "<$config_file>", $init_references ? 2 : 4 if !$ok; + + my $err; + + delete_directory($cfg->output_path); + if ($init_references) { + delete_directory($cfg->reference_output_path); + } + + my $engine = Serge::Importer->new(); + $engine->{optimizations} = undef; # force generate all the files + $cfg->chdir; + + foreach my $job_data (@{$cfg->{data}->{jobs}}) { + my $job; + + eval { + $job = Serge::Engine::Job->new($job_data, $engine, $cfg->{base_dir}); + $engine->process_job($job); + }; + + if ($@) { + my $error = $@; + # cleanup error message to avoid having file paths that will differ across installations + $error =~ s/\s+$//sg; + $error =~ s/ at .*? line \d+\.$//s; + $error =~ s/ \(\@INC contains: .*\)$//s; + $error =~ s/\@INC.+$/\@INC/s; + + print "Job '$job_data->{id}' will be skipped: $error\n"; + + eval { mkpath($cfg->errors_path) }; + die "Couldn't create $cfg->errors_path: $@" if $@; + my $filename = catfile($cfg->errors_path, $job_data->{id}.'.txt'); + open(OUT, ">$filename"); + binmode(OUT, ':unix :utf8'); + print OUT $error; + close(OUT); + } + } + + #ok(!$@, 'Processing all jobs in config file') or BAIL_OUT('Engine failed to run some of the jobs'); + + if ($cfg->can_dump_db) { + my $dumper = Test::DB::Dumper->new($engine); + $dumper->dump_l10n_tables($Test::DB::Dumper::TYPE_NEAT, $cfg->db_path); + } else { + print "Skipped dumping the database, as this is not applicable for this test\n"; + } + + $engine->cleanup; + + if ($init_references) { + ok(dircopy($cfg->output_path, $cfg->reference_output_path), "Initialized ".$cfg->reference_output_path); + } else { + $ok &= dir_diff($cfg->errors_path, $cfg->reference_errors_path, { base_dir => $cfg->{base_dir} } ); + $ok &= dir_diff($cfg->db_path, $cfg->reference_db_path, { base_dir => $cfg->{base_dir} } ); + $ok &= dir_diff($cfg->ts_path, $cfg->reference_ts_path, { base_dir => $cfg->{base_dir} } ); + $ok &= dir_diff($cfg->data_path, $cfg->reference_data_path, { base_dir => $cfg->{base_dir} } ) if $cfg->output_lang_files; + } + + # Under Windows, deleting just created files may fail with 'Permission denied' + # for an unknown reason, and only closing the process will release the file handles. + # Since we will be removing test output at the beginning of each test anyway, + # don't bail out this time if some files failed to be removed + delete_directory($cfg->output_path, 1) if $ok; + } + } +} + + +done_testing(); From b260bdeab0ad1c6952027be892430933435b3c8b Mon Sep 17 00:00:00 2001 From: icnocop Date: Fri, 8 Mar 2019 03:28:16 -0800 Subject: [PATCH 2/4] Use source path when importing Fixed parsing rc file and import test --- lib/Serge/Engine/Plugin/parse_rc.pm | 3 ++- lib/Serge/Importer.pm | 2 +- .../parse_rc/00/reference-output/database/translations | 5 +++++ t/importer.t | 5 +++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/Serge/Engine/Plugin/parse_rc.pm b/lib/Serge/Engine/Plugin/parse_rc.pm index dad46292..589656ab 100644 --- a/lib/Serge/Engine/Plugin/parse_rc.pm +++ b/lib/Serge/Engine/Plugin/parse_rc.pm @@ -71,7 +71,8 @@ sub parse { # STRINGTABLE BEGIN...END block contents } elsif ($stringtable && $blocklevel) { if ($line =~ m/^[\t ]*(\w+)[\t ]+"((.*?("")*)*?)"/) { # test for one-line string definitions - $hint = $1; + $idstr = $1; + $hint = $idstr; $orig_str = $2; } elsif ($line =~ m/^[\t ]*(\w+)[\t ]*(\/\/.*)*$/) { # test for the first line (id) of the two-line string definitions $idstr = $1; diff --git a/lib/Serge/Importer.pm b/lib/Serge/Importer.pm index 8ffe7d18..b51bba92 100644 --- a/lib/Serge/Importer.pm +++ b/lib/Serge/Importer.pm @@ -104,7 +104,7 @@ sub parse_localized_files_for_file_lang { $self->{current_lang} = $lang; - my $fullpath = $self->get_full_output_path($file, $lang); + my $fullpath = $self->get_full_source_path($file, $lang); $self->{stats}->{$lang} = {} unless exists $self->{stats}->{$lang}; diff --git a/t/data/importer/parse_rc/00/reference-output/database/translations b/t/data/importer/parse_rc/00/reference-output/database/translations index 244145a8..5f6f3296 100644 --- a/t/data/importer/parse_rc/00/reference-output/database/translations +++ b/t/data/importer/parse_rc/00/reference-output/database/translations @@ -1,3 +1,8 @@ translations { + 0 1 12 1 test `Value 1` NO 0 0 + 1 2 13 2 test `Value 2` NO 0 0 + 2 3 14 3 test `Error %s` NO 0 0 + 3 4 15 4 test `Value 3` NO 0 0 + 4 5 16 5 test `Value 4` NO 0 0 } diff --git a/t/importer.t b/t/importer.t index 63c64f92..20c2405f 100644 --- a/t/importer.t +++ b/t/importer.t @@ -36,9 +36,9 @@ $| = 1; # disable output buffering # to get the same results between tests, # we override the `file_mtime` function to return a constant value; -# Serge::Importer automatically imports this function from Serge::Util +# Serge::Engine automatically imports this function from Serge::Util # (this is why it actually appears in Serge::Importer namespace) -sub Serge::Importer::file_mtime { +sub Serge::Engine::file_mtime { return 12345678; } @@ -112,6 +112,7 @@ for my $config_file (@importer_confs) { foreach my $job_data (@{$cfg->{data}->{jobs}}) { my $job; + $engine->{debug} = $job_data->{debug}; eval { $job = Serge::Engine::Job->new($job_data, $engine, $cfg->{base_dir}); $engine->process_job($job); From eb7a3798d65d1683baea4499cc92bf51488b6f61 Mon Sep 17 00:00:00 2001 From: icnocop Date: Fri, 8 Mar 2019 17:33:52 -0800 Subject: [PATCH 3/4] More fixes to parse_rc.pm --- lib/Serge/Engine/Plugin/parse_rc.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/Serge/Engine/Plugin/parse_rc.pm b/lib/Serge/Engine/Plugin/parse_rc.pm index 589656ab..d68f658c 100644 --- a/lib/Serge/Engine/Plugin/parse_rc.pm +++ b/lib/Serge/Engine/Plugin/parse_rc.pm @@ -57,6 +57,7 @@ sub parse { if ($dialog && !$blocklevel) { if ($line =~ m/^[\t ]*(CAPTION)[\t ]+"((.*?("")*)*?)"/) { + $idstr = $1; $hint = $1; $orig_str = $2; } @@ -64,6 +65,7 @@ sub parse { # MENU and DIALOGEX BEGIN...END block contents } elsif (($menu || $dialog) && $blocklevel) { if ($line =~ m/^[\t ]*(\w+)[\t ]+"((.*?("")*)*?)"(,[\t ]*(\w+)){0,1}/) { + $idstr = $6 ? $6 : $1; $hint = $6 ? "$1 $6" : $1; $orig_str = $2; } From 8346d54321b14b59c7932dbae86031b6c5b8255c Mon Sep 17 00:00:00 2001 From: icnocop Date: Thu, 14 Mar 2019 01:45:44 -0700 Subject: [PATCH 4/4] Revert to using output path when importing --- lib/Serge/Importer.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Serge/Importer.pm b/lib/Serge/Importer.pm index b51bba92..8ffe7d18 100644 --- a/lib/Serge/Importer.pm +++ b/lib/Serge/Importer.pm @@ -104,7 +104,7 @@ sub parse_localized_files_for_file_lang { $self->{current_lang} = $lang; - my $fullpath = $self->get_full_source_path($file, $lang); + my $fullpath = $self->get_full_output_path($file, $lang); $self->{stats}->{$lang} = {} unless exists $self->{stats}->{$lang};