diff --git a/.gitignore b/.gitignore index 57060fd..708034d 100644 --- a/.gitignore +++ b/.gitignore @@ -78,9 +78,11 @@ target/ # celery beat schedule file celerybeat-schedule -# virtualenv +# dotenv .env .env3 + +# virtualenv venv/ ENV/ @@ -90,3 +92,5 @@ ENV/ # Rope project settings .ropeproject +# editors +TAGS diff --git a/.travis.yml b/.travis.yml index d64a75d..e4dd083 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,12 @@ language: python -env: - - TOXENV=py27 - matrix: include: - python: 2.7 env: TOXENV=py27 - - python: 2.7 + - python: 3.5 + env: TOXENV=py35 + - python: 3.5 env: TOXENV=lint before_script: diff --git a/HISTORY.rst b/HISTORY.rst index dd3b3be..cc7e2c1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,8 @@ Unreleased **Improvements** +* Support for python3 + **Documentation** **Build** diff --git a/MANIFEST.in b/MANIFEST.in index 5dd899c..2504439 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include LICENSE include HISTORY.rst -include-recursive marabunta/html +include-recursive marabunta/html * diff --git a/README.rst b/README.rst index f1eec3f..3b6c084 100644 --- a/README.rst +++ b/README.rst @@ -23,9 +23,8 @@ follows:: $ git clone https://github.com/camptocamp/marabunta.git Cloning into 'marabunta'... $ cd marabunta - $ python2 -m virtualenv env + $ virtualenv -p YOUR_PYTHON env $ source env/bin/activate $ pip install -e . $ pip install pytest $ py.test tests - diff --git a/marabunta/config.py b/marabunta/config.py index b906488..1933ae3 100644 --- a/marabunta/config.py +++ b/marabunta/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import argparse diff --git a/marabunta/core.py b/marabunta/core.py index 3286bf3..3599f8d 100644 --- a/marabunta/core.py +++ b/marabunta/core.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) """ diff --git a/marabunta/database.py b/marabunta/database.py index 63e4408..569cdfe 100644 --- a/marabunta/database.py +++ b/marabunta/database.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import json diff --git a/marabunta/exception.py b/marabunta/exception.py index 105ca15..f08a8fe 100644 --- a/marabunta/exception.py +++ b/marabunta/exception.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) diff --git a/marabunta/helpers.py b/marabunta/helpers.py index 267cea9..28b6ba7 100644 --- a/marabunta/helpers.py +++ b/marabunta/helpers.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import sys diff --git a/marabunta/model.py b/marabunta/model.py index 615123a..c276bc3 100644 --- a/marabunta/model.py +++ b/marabunta/model.py @@ -1,15 +1,13 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import shlex import sys +from builtins import object from distutils.version import StrictVersion -try: # Python 2.x - from cStringIO import StringIO -except ImportError: # Python 3.x - from io import StringIO +from io import StringIO import pexpect @@ -208,22 +206,26 @@ def __init__(self, command): @staticmethod def _shlex_split_unicode(command): - return [l.decode('utf8') for l in shlex.split(command.encode('utf8'))] + if sys.version_info < (3, 4): + return [l.decode('utf8') for l in shlex.split( + command.encode('utf-8'))] + else: + return shlex.split(command) - def __nonzero__(self): + def __bool__(self): return bool(self.command) def _execute(self, log, interactive=True): - child = pexpect.spawn(self.command[0].encode('utf8'), - [l.encode('utf8') for l in self.command[1:]], - timeout=None, - ) + assert self.command + executable = self.command[0] + params = self.command[1:] + child = pexpect.spawn(executable, params, timeout=None, + encoding='utf8') # interact() will transfer the child's stdout to # stdout, but we also copy the output in a buffer # so we can save the logs in the database log_buffer = StringIO() if interactive: - child.logfile = log_buffer # use the interactive mode so we can use pdb in the # migration scripts child.interact() @@ -234,6 +236,7 @@ def _execute(self, log, interactive=True): child.expect(pexpect.EOF) # child.before contains all the the output of the child program # before the EOF + # child.before is unicode log_buffer.write(child.before) child.close() if child.signalstatus is not None: @@ -253,10 +256,8 @@ def _execute(self, log, interactive=True): log_buffer.seek(0) # the pseudo-tty used for the child process returns # lines with \r\n endings - log('\n'.join(log_buffer.read().splitlines()) - .decode('utf-8', errors='replace'), - decorated=False, - stdout=False) + msg = '\n'.join(log_buffer.read().splitlines()) + log(msg, decorated=False, stdout=False) def execute(self, log): log(u'{}'.format(u' '.join(self.command))) diff --git a/marabunta/output.py b/marabunta/output.py index b93e323..77bcd0b 100644 --- a/marabunta/output.py +++ b/marabunta/output.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) from __future__ import print_function @@ -26,5 +26,8 @@ def print_decorated(message, *args, **kwargs): def safe_print(ustring, errors='replace', **kwargs): """ Safely print a unicode string """ encoding = sys.stdout.encoding or 'utf-8' - bytestr = ustring.encode(encoding, errors=errors) - print(bytestr, **kwargs) + if sys.version_info[0] == 3: + print(ustring, **kwargs) + else: + bytestr = ustring.encode(encoding, errors=errors) + print(bytestr, **kwargs) diff --git a/marabunta/parser.py b/marabunta/parser.py index 03add97..c4eeb1b 100644 --- a/marabunta/parser.py +++ b/marabunta/parser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) from __future__ import print_function diff --git a/marabunta/runner.py b/marabunta/runner.py index 3349cf1..f49966d 100644 --- a/marabunta/runner.py +++ b/marabunta/runner.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import traceback +import sys from datetime import datetime from distutils.version import StrictVersion @@ -138,10 +139,11 @@ def perform(self): try: self.perform_version(version) except Exception: - error = u'\n'.join( - self.logs + - [u'\n', traceback.format_exc().decode('utf8', errors='ignore')] - ) + if sys.version_info < (3, 4): + msg = traceback.format_exc().decode('utf8', errors='ignore') + else: + msg = traceback.format_exc() + error = u'\n'.join(self.logs + [u'\n', msg]) self.table.record_log(version.number, error) raise self.finish() diff --git a/setup.py b/setup.py index 718b595..1be961e 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,14 @@ "PyYAML", "pexpect", "werkzeug", + "future", ], + tests_require=["pytest", + "mock"], include_package_data=True, + package_data={ + 'marabunta': ['html/*.html'], + }, classifiers=( 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/tests/examples/migration.yml b/tests/examples/migration.yml new file mode 100644 index 0000000..b74d90d --- /dev/null +++ b/tests/examples/migration.yml @@ -0,0 +1,34 @@ +migration: + options: + # --workers=0 --stop-after-init are automatically added + install_command: odoo + install_args: --log-level=debug + versions: + - version: 0.0.1 + operations: + pre: # executed before 'addons' + - echo 'pre-operation' + post: # executed after 'addons' + - echo 'post-operation' + modes: + prod: + operations: + pre: + - echo 'pre-operation executed only when the mode is prod' + demo: + operations: + post: + - echo 'post-operation executed only when the mode is demo' + + - version: 0.0.2 + # nothing to do + + - version: 0.0.3 + operations: + pre: + - echo 'foobar' + - echo 'foobarbaz' + post: + - echo 'post-op with unicode é â' + + - version: 0.0.4 diff --git a/tests/test_migration_file.py b/tests/test_migration_file.py new file mode 100644 index 0000000..00eee5b --- /dev/null +++ b/tests/test_migration_file.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# Copyright 2016-2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import pytest +import os + +import mock + +from marabunta.config import Config +from marabunta.database import Database, MigrationTable +from marabunta.parser import YamlParser +from marabunta.runner import Runner + + +@pytest.fixture +def runner_gen(request): + def runner(filename, allow_serie=True, mode=None): + migration_file = os.path.join(request.fspath.dirname, + 'examples', filename) + config = Config(migration_file, + 'test', + allow_serie=allow_serie, + mode=mode) + migration_parser = YamlParser.parse_from_file(config.migration_file) + migration = migration_parser.parse() + table = mock.MagicMock(spec=MigrationTable) + table.versions.return_value = [] + database = mock.MagicMock(spec=Database) + return Runner(config, migration, database, table) + return runner + + +def test_example_file_output(runner_gen, request, capfd): + runner = runner_gen('migration.yml') + runner.perform() + expected = ( + u'|> migration: processing version 0.0.1\n' + u'|> version 0.0.1: start\n' + u'|> version 0.0.1: execute base pre-operations\n' + u'|> version 0.0.1: echo pre-operation\n' + u'pre-operation\r\n' + u'|> version 0.0.1: installation / upgrade of addons\n' + u'|> version 0.0.1: execute base post-operations\n' + u'|> version 0.0.1: echo post-operation\n' + u'post-operation\r\n' + u'|> version 0.0.1: done\n' + u'|> migration: processing version 0.0.2\n' + u'|> version 0.0.2: start\n' + u'|> version 0.0.2: version 0.0.2 is a noop\n' + u'|> version 0.0.2: done\n' + u'|> migration: processing version 0.0.3\n' + u'|> version 0.0.3: start\n' + u'|> version 0.0.3: execute base pre-operations\n' + u'|> version 0.0.3: echo foobar\n' + u'foobar\r\n' + u'|> version 0.0.3: echo foobarbaz\n' + u'foobarbaz\r\n' + u'|> version 0.0.3: installation / upgrade of addons\n' + u'|> version 0.0.3: execute base post-operations\n' + u'|> version 0.0.3: echo post-op with unicode é â\n' + u'post-op with unicode é â\r\n' + u'|> version 0.0.3: done\n' + u'|> migration: processing version 0.0.4\n' + u'|> version 0.0.4: start\n' + u'|> version 0.0.4: version 0.0.4 is a noop\n' + u'|> version 0.0.4: done\n', + u'' + ) + assert capfd.readouterr() == expected + + +def test_example_file_output_mode(runner_gen, request, capfd): + runner = runner_gen('migration.yml', mode='prod') + runner.perform() + expected = ( + u'|> migration: processing version 0.0.1\n' + u'|> version 0.0.1: start\n' + u'|> version 0.0.1: execute base pre-operations\n' + u'|> version 0.0.1: echo pre-operation\n' + u'pre-operation\r\n' + u'|> version 0.0.1: execute prod pre-operations\n' + u'|> version 0.0.1: echo pre-operation executed only' + u' when the mode is prod\n' + u'pre-operation executed only when the mode is prod\r\n' + u'|> version 0.0.1: installation / upgrade of addons\n' + u'|> version 0.0.1: execute base post-operations\n' + u'|> version 0.0.1: echo post-operation\n' + u'post-operation\r\n' + u'|> version 0.0.1: execute prod post-operations\n' + u'|> version 0.0.1: done\n' + u'|> migration: processing version 0.0.2\n' + u'|> version 0.0.2: start\n' + u'|> version 0.0.2: version 0.0.2 is a noop\n' + u'|> version 0.0.2: done\n' + u'|> migration: processing version 0.0.3\n' + u'|> version 0.0.3: start\n' + u'|> version 0.0.3: execute base pre-operations\n' + u'|> version 0.0.3: echo foobar\n' + u'foobar\r\n' + u'|> version 0.0.3: echo foobarbaz\n' + u'foobarbaz\r\n' + u'|> version 0.0.3: execute prod pre-operations\n' + u'|> version 0.0.3: installation / upgrade of addons\n' + u'|> version 0.0.3: execute base post-operations\n' + u'|> version 0.0.3: echo post-op with unicode é â\n' + u'post-op with unicode é â\r\n' + u'|> version 0.0.3: execute prod post-operations\n' + u'|> version 0.0.3: done\n' + u'|> migration: processing version 0.0.4\n' + u'|> version 0.0.4: start\n' + u'|> version 0.0.4: version 0.0.4 is a noop\n' + u'|> version 0.0.4: done\n', + u'' + ) + assert capfd.readouterr() == expected diff --git a/tests/test_operation.py b/tests/test_operation.py index cc668fa..9d6c0cd 100644 --- a/tests/test_operation.py +++ b/tests/test_operation.py @@ -1,27 +1,38 @@ # -*- coding: utf-8 -*- -# Copyright 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import unittest - from marabunta.model import Operation -class TestOperation(unittest.TestCase): +def test_from_single_unicode(): + op = Operation(u'ls -l') + assert op.command == ['ls', '-l'] + + +def test_from_single_str(): + op = Operation('ls -l') + assert op.command == ['ls', '-l'] + + +def test_from_list_of_unicode(): + op = Operation([u'ls', u'-l']) + assert op.command == ['ls', '-l'] + + +def test_from_list_of_str(): + op = Operation(['ls', '-l']) + assert op.command == ['ls', '-l'] - def test_from_single_unicode(self): - op = Operation(u'ls -l') - self.assertEqual(op.command, ['ls', '-l']) - def test_from_single_str(self): - op = Operation('ls -l') - self.assertEqual(op.command, ['ls', '-l']) +def test_log_execute_output(capfd): + op = Operation([u'echo', u'hello world']) + logs = [] - def test_from_list_of_unicode(self): - op = Operation([u'ls', u'-l']) - self.assertEqual(op.command, ['ls', '-l']) + def log(msg, **kwargs): + logs.append(msg) - def test_from_list_of_str(self): - op = Operation(['ls', '-l']) - self.assertEqual(op.command, ['ls', '-l']) + op.execute(log) + assert logs == [u'echo hello world', u'hello world'] + assert capfd.readouterr() == (u'hello world\r\n', '') diff --git a/tests/test_parse.py b/tests/test_parse.py index f54131e..ff0baf3 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,23 +1,15 @@ # -*- coding: utf-8 -*- -# © 2016 Camptocamp SA +# Copyright 2016-2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import unittest - from io import StringIO from marabunta.parser import YamlParser, YAML_EXAMPLE -class ParseTestSuite(unittest.TestCase): - - def test_parse_yaml_example(self): - file_example = StringIO(YAML_EXAMPLE) - parser = YamlParser.parser_from_buffer(file_example) - migration = parser.parse() - self.assertEqual(len(migration.versions), 4) - - -if __name__ == '__main__': - unittest.main() +def test_parse_yaml_example(): + file_example = StringIO(YAML_EXAMPLE) + parser = YamlParser.parser_from_buffer(file_example) + migration = parser.parse() + assert len(migration.versions) == 4 diff --git a/tox.ini b/tox.ini index 62c568f..cca3a07 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,14 @@ [tox] -envlist = py27,lint +envlist = py27,py35,lint [testenv] deps = pytest + mock commands = py.test {posargs} [testenv:lint] -basepython = python2.7 +basepython = python3.5 deps = flake8 readme_renderer