Added import test to fix import for parse_rc

.travis.yml

@@ -8,6 +8,7 @@ script:
   - cover -delete
   - perl -MDevel::Cover t/compile.t
   - perl -MDevel::Cover t/engine.t
+  - perl -MDevel::Cover t/importer.t
 after_success:
   - cover -report codecov
 perl:

lib/Serge/Engine/Plugin/parse_rc.pm

@@ -57,21 +57,24 @@ sub parse {
         if ($dialog && !$blocklevel) {
 
             if ($line =~ m/^[\t ]*(CAPTION)[\t ]+"((.*?("")*)*?)"/) {
+                $idstr = $1;
                 $hint = $1;
                 $orig_str = $2;
             }
 
         # MENU and DIALOGEX BEGIN...END block contents
         } elsif (($menu || $dialog) && $blocklevel) {
             if ($line =~ m/^[\t ]*(\w+)[\t ]+"((.*?("")*)*?)"(,[\t ]*(\w+)){0,1}/) {
+                $idstr = $6 ? $6 : $1;
                 $hint = $6 ? "$1 $6" : $1;
                 $orig_str = $2;
             }
 
         # STRINGTABLE BEGIN...END block contents
         } elsif ($stringtable && $blocklevel) {
             if ($line =~ m/^[\t ]*(\w+)[\t ]+"((.*?("")*)*?)"/) { # test for one-line string definitions
-                $hint = $1;
+                $idstr = $1;
+                $hint = $idstr;
                 $orig_str = $2;
             } elsif ($line =~ m/^[\t ]*(\w+)[\t ]*(\/\/.*)*$/) { # test for the first line (id) of the two-line string definitions
                 $idstr = $1;

t/data/importer/common.serge

@@ -0,0 +1,28 @@
+job_template
+{
+        name                                     Test Job
+        id                                       test_job
+        active                                   YES
+        source_language                          en
+        destination_languages                    test
+        source_dir                               ./resources
+        db_source                                DBI:SQLite:dbname=:memory:
+        db_namespace                             test_namespace
+        ts_file_path                             ./test-output/po/%LOCALE%/%FILE%.po
+        output_bom                               NO
+        output_file_path                         ./test-output/localized-resources/%LOCALE%/%FILE%
+
+        callback_plugins
+        {
+            :test_language
+            {
+                plugin                           test_language
+                phase                            get_translation_pre
+
+                data
+                {
+                    save_translations            YES
+                }
+            }
+        }
+}

t/data/importer/parse_rc/00/reference-output/database/files

@@ -0,0 +1,4 @@
+files
+{
+    0    1 1 test_job test_namespace application.rc 0
+}

t/data/importer/parse_rc/00/reference-output/database/items

@@ -0,0 +1,8 @@
+items
+{
+    0    1 3 1 1 IDS_TEST_1 NO 0
+    1    2 5 1 2 IDS_TEST_2 NO 0
+    2    3 7 1 3 IDS_ERROR NO 0
+    3    4 9 1 4 IDS_TEST_3 NO 0
+    4    5 11 1 5 IDS_TEST_4 NO 0
+}

t/data/importer/parse_rc/00/reference-output/database/properties

@@ -0,0 +1,7 @@
+properties
+{
+    0    1 source:1 b88e2e32e7cd852f7d4b76ba913a78a1
+    1    2 hash:1 b88e2e32e7cd852f7d4b76ba913a78a1
+    2    3 size:1 923
+    3    4 items:1 1,2,3,4,5
+}

t/data/importer/parse_rc/00/reference-output/database/strings

@@ -0,0 +1,8 @@
+strings
+{
+    0    1 2 `Value 1` NO 0
+    1    2 4 `Value 2` NO 0
+    2    3 6 `Error %s` NO 0
+    3    4 8 `Value 3` NO 0
+    4    5 10 `Value 4` NO 0
+}

t/data/importer/parse_rc/00/reference-output/database/translations

@@ -0,0 +1,8 @@
+translations
+{
+    0    1 12 1 test `Value 1` NO 0 0
+    1    2 13 2 test `Value 2` NO 0 0
+    2    3 14 3 test `Error %s` NO 0 0
+    3    4 15 4 test `Value 3` NO 0 0
+    4    5 16 5 test `Value 4` NO 0 0
+}

t/data/importer/parse_rc/00/resources/application.rc

@@ -0,0 +1,47 @@
+// Microsoft Visual C++ generated resource script.
+//
+#include "resource.h"
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Test Version
+//
+
+VS_VERSION_INFO VERSIONINFO
+ VERSION 1,0,0,1111
+ PRODUCTVERSION 4,0,0,1111
+ FILEFLAGSMASK 0x12L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+            VALUE "Sample Name", "Sample Name Value"
+            VALUE "Description", "Good description"
+            VALUE "Version", "1,0,0,1111"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Alternative", 0x409, 1200
+    END
+END
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Test Strings
+//
+
+STRINGTABLE
+BEGIN
+    IDS_TEST_1 "Value 1"
+    IDS_TEST_2 "Value 2"
+    IDS_ERROR  "Error %s"
+END
+
+STRINGTABLE
+BEGIN
+    IDS_TEST_3
+                            "Value 3"
+    IDS_TEST_4              "Value 4"
+END

t/data/importer/parse_rc/00/translate.serge

@@ -0,0 +1,13 @@
+jobs
+{
+    {
+        @inherit                                 ../../common.serge#job_template
+
+        source_match                             \.rc$
+
+        parser
+        {
+            plugin                               parse_rc
+        }
+    }
+}

t/importer.t

@@ -0,0 +1,171 @@
+#!/usr/bin/env perl
+
+use strict;
+
+# HOW TO USE THIS TEST
+#
+# By default, this test runs over all directories in t/data/importer/.  To run
+# the test only for specific directories, pass the directory names to this
+# script or assign them to the environment variable SERGE_IMPORTER_TESTS as a
+# comma-separated list.  The following two examples are equivalent:
+#
+# perl t/importer.t parse_json parse_strings
+# SERGE_IMPORTER_TESTS=parse_json,parse_strings prove t/importer.t
+
+BEGIN {
+    use Cwd qw(abs_path);
+    use File::Basename;
+    use File::Spec::Functions qw(catfile);
+    map { unshift(@INC, catfile(dirname(abs_path(__FILE__)), $_)) } qw(lib ../lib);
+}
+
+use Data::Dumper;
+use File::Copy::Recursive qw/dircopy/;
+use File::Find qw(find);
+use File::Path;
+use File::Spec::Functions qw(catfile);
+use Getopt::Long;
+use Test::Config;
+use Test::Diff;
+use Test::More;
+use Test::DB::Dumper;
+use Serge::Importer;
+use Serge::Engine::Job;
+
+$| = 1; # disable output buffering
+
+# to get the same results between tests,
+# we override the `file_mtime` function to return a constant value;
+# Serge::Engine automatically imports this function from Serge::Util
+# (this is why it actually appears in Serge::Importer namespace)
+sub Serge::Engine::file_mtime {
+    return 12345678;
+}
+
+my $this_dir = dirname(abs_path(__FILE__));
+my $tests_dir = catfile($this_dir, 'data', 'importer');
+
+my @importer_confs;
+
+my ($init_references);
+
+GetOptions("init" => \$init_references);
+
+my @importer_dirs = @ARGV;
+if (my $env_dirs = $ENV{SERGE_IMPORTER_TESTS}) {
+    push @importer_dirs, split(/,/, $env_dirs);
+}
+
+unless (@importer_dirs) {
+    find(sub {
+        push @importer_confs, $File::Find::name if(-f $_ && /\.serge$/ && $_ ne 'common.serge');
+    }, $tests_dir);
+} else {
+    for my $dir (@importer_dirs) {
+        find(sub {
+            push @importer_confs, $File::Find::name if(-f $_ && /\.serge$/ && $_ ne 'common.serge');
+        }, catfile($tests_dir, $dir));
+    }
+}
+
+sub delete_directory {
+    my ($path, $ignore_errors) = @_;
+
+    my $err;
+
+    if (-e $path) {
+        rmtree($path, { error => \$err });
+        if (@$err && !$ignore_errors) {
+            my $err_text = '';
+
+            map {
+                foreach my $key (keys %$_) {
+                    $err_text .= $key.': '.$_->{$key}."\n";
+                }
+            } @$err;
+
+            BAIL_OUT("Directory '".$path."' couldn't be removed\n$err_text");
+        }
+    }
+}
+
+for my $config_file (@importer_confs) {
+
+    subtest "Test config: $config_file" => sub {
+        my $cfg = Test::Config->new($config_file);
+
+        SKIP: {
+            my $ok = ok(defined $cfg, 'Config file read');
+            skip "<$config_file>", $init_references ? 2 : 4 if !$ok;
+
+            my $err;
+
+            delete_directory($cfg->output_path);
+            if ($init_references) {
+                delete_directory($cfg->reference_output_path);
+            }
+
+            my $engine = Serge::Importer->new();
+            $engine->{optimizations} = undef; # force generate all the files
+            $cfg->chdir;
+
+            foreach my $job_data (@{$cfg->{data}->{jobs}}) {
+                my $job;
+
+                $engine->{debug} = $job_data->{debug};
+                eval {
+                    $job = Serge::Engine::Job->new($job_data, $engine, $cfg->{base_dir});
+                    $engine->process_job($job);
+                };
+
+                if ($@) {
+                    my $error = $@;
+                    # cleanup error message to avoid having file paths that will differ across installations
+                    $error =~ s/\s+$//sg;
+                    $error =~ s/ at .*? line \d+\.$//s;
+                    $error =~ s/ \(\@INC contains: .*\)$//s;
+                    $error =~ s/\@INC.+$/\@INC/s;
+
+                    print "Job '$job_data->{id}' will be skipped: $error\n";
+
+                    eval { mkpath($cfg->errors_path) };
+                    die "Couldn't create $cfg->errors_path: $@" if $@;
+                    my $filename = catfile($cfg->errors_path, $job_data->{id}.'.txt');
+                    open(OUT, ">$filename");
+                    binmode(OUT, ':unix :utf8');
+                    print OUT $error;
+                    close(OUT);
+                }
+            }
+
+            #ok(!$@, 'Processing all jobs in config file') or BAIL_OUT('Engine failed to run some of the jobs');
+
+            if ($cfg->can_dump_db) {
+                my $dumper = Test::DB::Dumper->new($engine);
+                $dumper->dump_l10n_tables($Test::DB::Dumper::TYPE_NEAT, $cfg->db_path);
+            } else {
+                print "Skipped dumping the database, as this is not applicable for this test\n";
+            }
+
+            $engine->cleanup;
+
+            if ($init_references) {
+                ok(dircopy($cfg->output_path, $cfg->reference_output_path), "Initialized ".$cfg->reference_output_path);
+            } else {
+                $ok &= dir_diff($cfg->errors_path, $cfg->reference_errors_path, { base_dir => $cfg->{base_dir} } );
+                $ok &= dir_diff($cfg->db_path, $cfg->reference_db_path, { base_dir => $cfg->{base_dir} } );
+                $ok &= dir_diff($cfg->ts_path, $cfg->reference_ts_path, { base_dir => $cfg->{base_dir} } );
+                $ok &= dir_diff($cfg->data_path, $cfg->reference_data_path, { base_dir => $cfg->{base_dir} } ) if $cfg->output_lang_files;
+            }
+
+            # Under Windows, deleting just created files may fail with 'Permission denied'
+            # for an unknown reason, and only closing the process will release the file handles.
+            # Since we will be removing test output at the beginning of each test anyway,
+            # don't bail out this time if some files failed to be removed
+            delete_directory($cfg->output_path, 1) if $ok;
+        }
+    }
+}
+
+
+done_testing();