Merge pull request #26 from TDu/v11ready

Migrate to Python 3

.gitignore

@@ -78,9 +78,11 @@ target/
 # celery beat schedule file
 celerybeat-schedule
 
-# virtualenv
+# dotenv
 .env
 .env3
+
+# virtualenv
 venv/
 ENV/
 
@@ -90,3 +92,5 @@ ENV/
 # Rope project settings
 .ropeproject
 
+# editors
+TAGS

.travis.yml

@@ -1,13 +1,12 @@
 language: python
 
-env:
-  - TOXENV=py27
-
 matrix:
   include:
   - python: 2.7
     env: TOXENV=py27
-  - python: 2.7
+  - python: 3.5
+    env: TOXENV=py35
+  - python: 3.5
     env: TOXENV=lint
 
 before_script:

HISTORY.rst

@@ -12,6 +12,8 @@ Unreleased
 
 **Improvements**
 
+* Support for python3
+
 **Documentation**
 
 **Build**

MANIFEST.in

@@ -1,3 +1,3 @@
 include LICENSE
 include HISTORY.rst
-include-recursive marabunta/html
+include-recursive marabunta/html *

README.rst

@@ -23,9 +23,8 @@ follows::
   $ git clone https://github.com/camptocamp/marabunta.git
   Cloning into 'marabunta'...
   $ cd marabunta
-  $ python2 -m virtualenv env
+  $ virtualenv -p YOUR_PYTHON env
   $ source env/bin/activate
   $ pip install -e .
   $ pip install pytest
   $ py.test tests
-

marabunta/config.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# © 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 import argparse

marabunta/core.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# © 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 """

marabunta/database.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# © 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 import json

marabunta/exception.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# © 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 

marabunta/helpers.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 import sys

marabunta/model.py

@@ -1,15 +1,13 @@
 # -*- coding: utf-8 -*-
-# © 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 import shlex
 import sys
 
+from builtins import object
 from distutils.version import StrictVersion
-try:  # Python 2.x
-    from cStringIO import StringIO
-except ImportError:  # Python 3.x
-    from io import StringIO
+from io import StringIO
 
 import pexpect
 
@@ -208,22 +206,26 @@ def __init__(self, command):
 
     @staticmethod
     def _shlex_split_unicode(command):
-        return [l.decode('utf8') for l in shlex.split(command.encode('utf8'))]
+        if sys.version_info < (3, 4):
+            return [l.decode('utf8') for l in shlex.split(
+                command.encode('utf-8'))]
+        else:
+            return shlex.split(command)
 
-    def __nonzero__(self):
+    def __bool__(self):
         return bool(self.command)
 
     def _execute(self, log, interactive=True):
-        child = pexpect.spawn(self.command[0].encode('utf8'),
-                              [l.encode('utf8') for l in self.command[1:]],
-                              timeout=None,
-                              )
+        assert self.command
+        executable = self.command[0]
+        params = self.command[1:]
+        child = pexpect.spawn(executable, params, timeout=None,
+                              encoding='utf8')
         # interact() will transfer the child's stdout to
         # stdout, but we also copy the output in a buffer
         # so we can save the logs in the database
         log_buffer = StringIO()
         if interactive:
-            child.logfile = log_buffer
             # use the interactive mode so we can use pdb in the
             # migration scripts
             child.interact()
@@ -234,6 +236,7 @@ def _execute(self, log, interactive=True):
             child.expect(pexpect.EOF)
             # child.before contains all the the output of the child program
             # before the EOF
+            # child.before is unicode
             log_buffer.write(child.before)
         child.close()
         if child.signalstatus is not None:
@@ -253,10 +256,8 @@ def _execute(self, log, interactive=True):
         log_buffer.seek(0)
         # the pseudo-tty used for the child process returns
         # lines with \r\n endings
-        log('\n'.join(log_buffer.read().splitlines())
-                .decode('utf-8', errors='replace'),
-            decorated=False,
-            stdout=False)
+        msg = '\n'.join(log_buffer.read().splitlines())
+        log(msg, decorated=False, stdout=False)
 
     def execute(self, log):
         log(u'{}'.format(u' '.join(self.command)))

marabunta/output.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# © 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 from __future__ import print_function
@@ -26,5 +26,8 @@ def print_decorated(message, *args, **kwargs):
 def safe_print(ustring, errors='replace', **kwargs):
     """ Safely print a unicode string """
     encoding = sys.stdout.encoding or 'utf-8'
-    bytestr = ustring.encode(encoding, errors=errors)
-    print(bytestr, **kwargs)
+    if sys.version_info[0] == 3:
+        print(ustring, **kwargs)
+    else:
+        bytestr = ustring.encode(encoding, errors=errors)
+        print(bytestr, **kwargs)

marabunta/parser.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# © 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 from __future__ import print_function

marabunta/runner.py

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
-# © 2016 Camptocamp SA
+# Copyright 2016-2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 import traceback
+import sys
 
 from datetime import datetime
 from distutils.version import StrictVersion
@@ -138,10 +139,11 @@ def perform(self):
         try:
             self.perform_version(version)
         except Exception:
-            error = u'\n'.join(
-                self.logs +
-                [u'\n', traceback.format_exc().decode('utf8', errors='ignore')]
-            )
+            if sys.version_info < (3, 4):
+                msg = traceback.format_exc().decode('utf8', errors='ignore')
+            else:
+                msg = traceback.format_exc()
+            error = u'\n'.join(self.logs + [u'\n', msg])
             self.table.record_log(version.number, error)
             raise
         self.finish()

setup.py

@@ -25,8 +25,14 @@
                       "PyYAML",
                       "pexpect",
                       "werkzeug",
+                      "future",
                       ],
+    tests_require=["pytest",
+                   "mock"],
     include_package_data=True,
+    package_data={
+        'marabunta': ['html/*.html'],
+    },
     classifiers=(
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',

tests/examples/migration.yml

@@ -0,0 +1,34 @@
+migration:
+  options:
+    # --workers=0 --stop-after-init are automatically added
+    install_command: odoo
+    install_args: --log-level=debug
+  versions:
+    - version: 0.0.1
+      operations:
+        pre:  # executed before 'addons'
+          - echo 'pre-operation'
+        post:  # executed after 'addons'
+          - echo 'post-operation'
+      modes:
+        prod:
+          operations:
+            pre:
+              - echo 'pre-operation executed only when the mode is prod'
+        demo:
+          operations:
+            post:
+              - echo 'post-operation executed only when the mode is demo'
+
+    - version: 0.0.2
+      # nothing to do
+
+    - version: 0.0.3
+      operations:
+        pre:
+          - echo 'foobar'
+          - echo 'foobarbaz'
+        post:
+          - echo 'post-op with unicode é â'
+
+    - version: 0.0.4

tests/test_migration_file.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016-2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import pytest
+import os
+
+import mock
+
+from marabunta.config import Config
+from marabunta.database import Database, MigrationTable
+from marabunta.parser import YamlParser
+from marabunta.runner import Runner
+
+
+@pytest.fixture
+def runner_gen(request):
+    def runner(filename, allow_serie=True, mode=None):
+        migration_file = os.path.join(request.fspath.dirname,
+                                      'examples', filename)
+        config = Config(migration_file,
+                        'test',
+                        allow_serie=allow_serie,
+                        mode=mode)
+        migration_parser = YamlParser.parse_from_file(config.migration_file)
+        migration = migration_parser.parse()
+        table = mock.MagicMock(spec=MigrationTable)
+        table.versions.return_value = []
+        database = mock.MagicMock(spec=Database)
+        return Runner(config, migration, database, table)
+    return runner
+
+
+def test_example_file_output(runner_gen, request, capfd):
+    runner = runner_gen('migration.yml')
+    runner.perform()
+    expected = (
+        u'|> migration: processing version 0.0.1\n'
+        u'|> version 0.0.1: start\n'
+        u'|> version 0.0.1: execute base pre-operations\n'
+        u'|> version 0.0.1: echo pre-operation\n'
+        u'pre-operation\r\n'
+        u'|> version 0.0.1: installation / upgrade of addons\n'
+        u'|> version 0.0.1: execute base post-operations\n'
+        u'|> version 0.0.1: echo post-operation\n'
+        u'post-operation\r\n'
+        u'|> version 0.0.1: done\n'
+        u'|> migration: processing version 0.0.2\n'
+        u'|> version 0.0.2: start\n'
+        u'|> version 0.0.2: version 0.0.2 is a noop\n'
+        u'|> version 0.0.2: done\n'
+        u'|> migration: processing version 0.0.3\n'
+        u'|> version 0.0.3: start\n'
+        u'|> version 0.0.3: execute base pre-operations\n'
+        u'|> version 0.0.3: echo foobar\n'
+        u'foobar\r\n'
+        u'|> version 0.0.3: echo foobarbaz\n'
+        u'foobarbaz\r\n'
+        u'|> version 0.0.3: installation / upgrade of addons\n'
+        u'|> version 0.0.3: execute base post-operations\n'
+        u'|> version 0.0.3: echo post-op with unicode é â\n'
+        u'post-op with unicode é â\r\n'
+        u'|> version 0.0.3: done\n'
+        u'|> migration: processing version 0.0.4\n'
+        u'|> version 0.0.4: start\n'
+        u'|> version 0.0.4: version 0.0.4 is a noop\n'
+        u'|> version 0.0.4: done\n',
+        u''
+    )
+    assert capfd.readouterr() == expected
+
+
+def test_example_file_output_mode(runner_gen, request, capfd):
+    runner = runner_gen('migration.yml', mode='prod')
+    runner.perform()
+    expected = (
+        u'|> migration: processing version 0.0.1\n'
+        u'|> version 0.0.1: start\n'
+        u'|> version 0.0.1: execute base pre-operations\n'
+        u'|> version 0.0.1: echo pre-operation\n'
+        u'pre-operation\r\n'
+        u'|> version 0.0.1: execute prod pre-operations\n'
+        u'|> version 0.0.1: echo pre-operation executed only'
+        u' when the mode is prod\n'
+        u'pre-operation executed only when the mode is prod\r\n'
+        u'|> version 0.0.1: installation / upgrade of addons\n'
+        u'|> version 0.0.1: execute base post-operations\n'
+        u'|> version 0.0.1: echo post-operation\n'
+        u'post-operation\r\n'
+        u'|> version 0.0.1: execute prod post-operations\n'
+        u'|> version 0.0.1: done\n'
+        u'|> migration: processing version 0.0.2\n'
+        u'|> version 0.0.2: start\n'
+        u'|> version 0.0.2: version 0.0.2 is a noop\n'
+        u'|> version 0.0.2: done\n'
+        u'|> migration: processing version 0.0.3\n'
+        u'|> version 0.0.3: start\n'
+        u'|> version 0.0.3: execute base pre-operations\n'
+        u'|> version 0.0.3: echo foobar\n'
+        u'foobar\r\n'
+        u'|> version 0.0.3: echo foobarbaz\n'
+        u'foobarbaz\r\n'
+        u'|> version 0.0.3: execute prod pre-operations\n'
+        u'|> version 0.0.3: installation / upgrade of addons\n'
+        u'|> version 0.0.3: execute base post-operations\n'
+        u'|> version 0.0.3: echo post-op with unicode é â\n'
+        u'post-op with unicode é â\r\n'
+        u'|> version 0.0.3: execute prod post-operations\n'
+        u'|> version 0.0.3: done\n'
+        u'|> migration: processing version 0.0.4\n'
+        u'|> version 0.0.4: start\n'
+        u'|> version 0.0.4: version 0.0.4 is a noop\n'
+        u'|> version 0.0.4: done\n',
+        u''
+    )
+    assert capfd.readouterr() == expected