From 017bebff01de5fe468de48b8d35b6d78cc77bb07 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Mon, 27 May 2019 16:41:39 -0700 Subject: [PATCH 01/15] travis: Use the default osx image. XCode 7.3 (based on macOS 10.11) is out of support, and cannot complete tests because the `brew upgrade` step has so many packages to build. Use the default image instead, which is currently macOS 10.13 and Xcode 9.4.1. This seems to resolve the recent travis test failures on macOS. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d3372f6..3c00132 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,6 @@ matrix: sudo: false python: 3.5 - os: osx - osx_image: xcode7.3 language: generic env: PIP=pip3 MYPYTHON=python3 env: From d531979355bd10647627ae31b21ab1b5e1aad091 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Tue, 28 May 2019 08:19:18 -0700 Subject: [PATCH 02/15] travis: Drop extra coverage report regeration. Assume we have a more recent version (3 or later) of pytest which can be invoked as such. Also remove the report generation switches. The codecov.io docs don't suggest this, so I don't think it's necessary for upload to the site. There's no point generating the reports in ci if they're not uploaded anywhere. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3c00132..56cbaf0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,7 +74,7 @@ script: if [ "$MYPYTHON" == "jython" ]; then py.test else - py.test --cov=pyoracc --cov-report xml --cov-report html --runslow + pytest --cov=pyoracc --runslow fi - pep8 --exclude=parsetab.py . From bb30753d31bb968051b81881ccd990c53021c313 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Tue, 28 May 2019 08:43:25 -0700 Subject: [PATCH 03/15] travis: Upgrade to the lastest pytest. The default pytest included in the travis 2.7 package (3.3.0) is too old for the pytest_cov package we install. Use pip to upgrade it before running tests. Resolves the build failure with python 2.7 on linux. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 56cbaf0..88f3eed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,7 +62,7 @@ install: $PIP install setuptools $PIP install ply pep8 mako if [ "$MYPYTHON" != "jython" ]; then - $PIP install pytest pytest-cov codecov + $PIP install --upgrade pytest pytest-cov codecov fi script: From e9d6c429abe0f9abfb135a41e13c199930b61be2 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Tue, 28 May 2019 10:34:27 -0700 Subject: [PATCH 04/15] Rewrite yield tests as parameterized tests. Yield tests were removed in pytest 4.0 because they didn't interact well with test fixtures. Use @pytest.mark.parametrize decorators instead to generate the test variants. This is the recommended replacement, according to the documentation. For details, see https://docs.pytest.org/en/latest/deprecations.html#yield-tests --- pyoracc/test/atf/test_atffile.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/pyoracc/test/atf/test_atffile.py b/pyoracc/test/atf/test_atffile.py index ac72046..5000b1f 100644 --- a/pyoracc/test/atf/test_atffile.py +++ b/pyoracc/test/atf/test_atffile.py @@ -20,6 +20,7 @@ from pyoracc.atf.common.atffile import AtfFile from ..fixtures import anzu, belsunu, sample_file +import pytest def test_create(): @@ -88,36 +89,22 @@ def test_composite(): ] -def consider_composite(name, code): +@pytest.mark.parametrize('name, code', [ + (text[0], text[1]) for text in composites]) +def test_composite_code(name, code): """ - Parses ATF and checks CDLI ID coincides + Parses ATF and checks CDLI ID coincides. """ afile = AtfFile(sample_file(name)) assert afile.text.texts[0].code == code -def consider_file(name, code, description): +@pytest.mark.parametrize('name, code, description', [ + (text[0], text[1], text[2]) for text in texts]) +def test_text_designation(name, code, description): """ - Parses ATF and checks CDLI ID and text description coincide + Parses ATF and checks CDLI ID and text description coincide. """ afile = AtfFile(sample_file(name)) assert afile.text.code == code assert afile.text.description == description - - -def test_texts(): - """" - Go through list of selected filenames and check parser deals non-composite - files. - """ - for text in texts: - yield consider_file, text[0], text[1], text[2] - - -def test_composites(): - """ - Go through list of selected composites and check parser deals with - composite files correctly - """ - for composite in composites: - yield consider_composite, composite[0], composite[1] From ff02252d6238d9e37110646eedc65b0dd7c922fa Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Tue, 28 May 2019 10:52:54 -0700 Subject: [PATCH 05/15] Mark parser patterns as raw strings. pycodestyle/pep8 warns about invalid escape sequences in most of the ply.lex re patterns. Avoid this by marking them as raw strings, which means the python interpreter won't process the backslash-escapes itself but pass them as-is to the regex engine. This was already the case in a number of places; this just makes the use of raw strings more uniquitous. I'm not sure how to do the interaction with \uXXXX escapes, which warn either way. I've left them as they were so there are still a handful of warnings. --- pyoracc/atf/common/atflex.py | 70 ++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/pyoracc/atf/common/atflex.py b/pyoracc/atf/common/atflex.py index e528a92..e5bf3c0 100644 --- a/pyoracc/atf/common/atflex.py +++ b/pyoracc/atf/common/atflex.py @@ -69,36 +69,36 @@ def resolve_keyword(self, value, source, fallback=None, extra=None): states = AtfLexicon.STATES - t_AMPERSAND = "\&" - t_HASH = "\#" - t_EXCLAIM = "\!" - t_QUERY = "\?" - t_STAR = "\*" - t_DOLLAR = "\$" - t_MINUS = "\-" - t_FROM = "\<\<" - t_TO = "\>\>" - t_COMMA = "\," - t_PARBAR = "\|\|" - - t_INITIAL_transctrl_PARENTHETICALID = "\([^\n\r]*\)" + t_AMPERSAND = r'\&' + t_HASH = r'\#' + t_EXCLAIM = r'\!' + t_QUERY = r'\?' + t_STAR = r'\*' + t_DOLLAR = r'\$' + t_MINUS = r'\-' + t_FROM = r'\<\<' + t_TO = r'\>\>' + t_COMMA = r'\,' + t_PARBAR = r'\|\|' + + t_INITIAL_transctrl_PARENTHETICALID = r'\([^\n\r]*\)' def t_INITIAL_transctrl_WHITESPACE(self, t): r'[\t ]+' # NO TOKEN def t_MULTILINGUAL(self, t): - "\=\=" + r'\=\=' t.lexer.push_state("text") return t def t_EQUALBRACE(self, t): - "^\=\{" + r'^\=\{' t.lexer.push_state('text') return t def t_EQUALS(self, t): - "\=" + r'\=' t.lexer.push_state('flagged') return t @@ -121,7 +121,7 @@ def t_NEWLINE(self, t): return t def t_INITIAL_parallel_labeled_ATID(self, t): - '^\@[a-zA-Z][a-zA-Z0-9\[\]]*\+?' + r'^\@[a-zA-Z][a-zA-Z0-9\[\]]*\+?' t.value = t.value[1:] t.lexpos += 1 t.type = self.resolve_keyword(t.value, @@ -171,13 +171,13 @@ def t_INITIAL_parallel_labeled_ATID(self, t): return t def t_labeled_OPENR(self, t): - "\@\(" + r'\@\(' t.lexer.push_state("para") t.lexer.push_state("transctrl") return t def t_INITIAL_parallel_labeled_HASHID(self, t): - '\#[a-zA-Z][a-zA-Z0-9\[\]]+\:' + r'\#[a-zA-Z][a-zA-Z0-9\[\]]+\:' # Note that \:? absorbs a trailing colon in protocol keywords t.value = t.value[1:-1] t.lexpos += 1 @@ -303,7 +303,7 @@ def t_transctrl_ID(self, t): t.type = "REFERENCE" return t - t_parallel_QUERY = "\?" + t_parallel_QUERY = r'\?' def t_parallel_LINELABEL(self, t): r'^([^\.\ \t]*)\.[\ \t]*' @@ -311,14 +311,14 @@ def t_parallel_LINELABEL(self, t): return t def t_parallel_labeled_DOLLAR(self, t): - "^\$" + r'^\$' t.lexer.push_state("absorb") return t - t_transctrl_MINUS = "\-\ " + t_transctrl_MINUS = r'\-\ ' def t_transctrl_CLOSER(self, t): - "\)" + r'\)' t.lexer.pop_state() return t @@ -352,8 +352,8 @@ def t_labeled_NEWLINE(self, t): # fact that the first character may not be a ? # We are looking for a string that does not start with ? it may include # newlines if they are followed by a whitespace. - translation_regex1 = '([^\?\^\n\r]|([\n\r](?=[ \t])))' - translation_regex2 = '([^\^\n\r]|([\n\r](?=[ \t])))*' + translation_regex1 = r'([^\?\^\n\r]|([\n\r](?=[ \t])))' + translation_regex2 = r'([^\^\n\r]|([\n\r](?=[ \t])))*' translation_regex = white + translation_regex1 + translation_regex2 + white @lex.TOKEN(translation_regex) @@ -399,12 +399,12 @@ def t_flagged_ID(self, t): t.value = t.value.strip() return t - t_flagged_HASH = "\#" - t_flagged_EXCLAIM = "\!" - t_flagged_QUERY = "\?" - t_flagged_STAR = "\*" - t_flagged_parallel_para_HAT = "[\ \t]*\^[\ \t]*" - t_flagged_EQUALS = "\=" + t_flagged_HASH = r'\#' + t_flagged_EXCLAIM = r'\!' + t_flagged_QUERY = r'\?' + t_flagged_STAR = r'\*' + t_flagged_parallel_para_HAT = r'[\ \t]*\^[\ \t]*' + t_flagged_EQUALS = r'\=' # --- Rules for paragaph state---------------------------------- # Free text, ended by double new line @@ -441,11 +441,11 @@ def t_para_MAGICNEWLINE(self, t): # --- RULES FOR THE nonequals STATE ----- # Absorb everything except an equals def t_nonequals_ID(self, t): - "[^\=\n\r]+" + r'[^\=\n\r]+' t.value = t.value.strip() return t - t_nonequals_EQUALS = "\=" + t_nonequals_EQUALS = r'\=' # --- RULES FOR THE absorb STATE ----- # Absorb everything @@ -455,14 +455,14 @@ def t_absorb_ID(self, t): return t # --- RULES FOR THE text STATE ---- - t_text_ID = "[^\ \t \n\r]+" + t_text_ID = r'[^\ \t \n\r]+' def t_text_SPACE(self, t): r'[\ \t]' # No token generated # --- RULES FOR THE lemmatize STATE - t_lemmatize_ID = "[^\;\n\r]+" + t_lemmatize_ID = r'[^\;\n\r]+' t_lemmatize_SEMICOLON = r'\;[\ \t]*' # Error handling rule From e16f98434315bf4e0b4a583af2abc721f4e71a47 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Thu, 30 May 2019 07:49:27 -0700 Subject: [PATCH 06/15] Rename file variable to filename. The local variable in the Corpus loader named `file` shadows the built-in `file` object in python2. The the more specific `filename` instead. Corrects a lint warning in my editor. --- pyoracc/model/corpus.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyoracc/model/corpus.py b/pyoracc/model/corpus.py index 48001c0..4969fc4 100644 --- a/pyoracc/model/corpus.py +++ b/pyoracc/model/corpus.py @@ -33,10 +33,10 @@ def __init__(self, **kwargs): self.source = kwargs['source'] if 'source' in kwargs: for dirpath, _, files in os.walk(self.source): - for file in files: - if file.endswith('.atf'): + for filename in files: + if filename.endswith('.atf'): try: - path = os.path.join(dirpath, file) + path = os.path.join(dirpath, filename) print("Parsing file", path, "... ", end="") content = codecs.open(path, encoding='utf-8-sig').read() From 6418ba787d70a908c3a3cd8212bc47e53e8170b5 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Thu, 30 May 2019 08:35:03 -0700 Subject: [PATCH 07/15] Fix comment typos. Adjust these to standard written English. --- pyoracc/test/model/test_corpus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyoracc/test/model/test_corpus.py b/pyoracc/test/model/test_corpus.py index 9bdc26f..3b44cbc 100644 --- a/pyoracc/test/model/test_corpus.py +++ b/pyoracc/test/model/test_corpus.py @@ -51,8 +51,8 @@ def test_sample(): @slow def test_whole(): corpus = Corpus(source=whole_corpus(), atftype='oracc') - # there is a total of 8229 files in the corpus + # There are a total of 8229 files in the corpus. # We have ommmited lacost/00atf/cdliatf_unblocked.atf - # which is 61 MB and this to large to fit in the git repository + # which is 61 MB and too large to fit in the git repository. assert corpus.successes == 6750 assert corpus.failures == 1479 From 48296b2194cb76bbe5a6f3d627177743c82ad043 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Thu, 30 May 2019 11:32:56 -0700 Subject: [PATCH 08/15] Always run slow tests. Remove the --runslow option from the pytest config and instead always run the slow tests. Unless a large corpus is passed through the `oracc_corpus_path` environment variable, this only adds the included sample corpus. That adds about 60% to the pytest runtime, but is still under 10 seconds. The motivation for this is that the way the option was accessed is deprecated in pytest 4.1. https://docs.pytest.org/en/latest/deprecations.html#pytest-config-global Having less configuration is less code to worry about, and less variance between different test invocations, so removing the option to skip the sample corpus is the most straightforward fix. --- .travis.yml | 2 +- README.md | 2 +- conftest.py | 26 -------------------------- pyoracc/test/model/test_corpus.py | 8 -------- 4 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 conftest.py diff --git a/.travis.yml b/.travis.yml index 88f3eed..a07ed5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,7 +74,7 @@ script: if [ "$MYPYTHON" == "jython" ]; then py.test else - pytest --cov=pyoracc --runslow + pytest --cov=pyoracc fi - pep8 --exclude=parsetab.py . diff --git a/README.md b/README.md index c1749a4..069bfcd 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ Options: Before running pytest and coverage, install [py.test](https://docs.pytest.org/en/latest/getting-started.html) and [pytest-cov](https://pypi.org/project/pytest-cov/). - $ py.test --cov=pyoracc --cov-report xml --cov-report html --cov-report annotate --runslow + $ py.test --cov=pyoracc --cov-report xml --cov-report html --cov-report annotate Before running pycodestyle, install [pycodestyle](https://pypi.org/project/pycodestyle/). diff --git a/conftest.py b/conftest.py deleted file mode 100644 index a74f104..0000000 --- a/conftest.py +++ /dev/null @@ -1,26 +0,0 @@ -''' -Copyright 2015, 2016 University College London. - -This file is part of PyORACC. - -PyORACC is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -PyORACC is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with PyORACC. If not, see . -''' - - -import pytest - - -def pytest_addoption(parser): - parser.addoption("--runslow", action="store_true", - help="run slow tests") diff --git a/pyoracc/test/model/test_corpus.py b/pyoracc/test/model/test_corpus.py index 3b44cbc..65d7cc3 100644 --- a/pyoracc/test/model/test_corpus.py +++ b/pyoracc/test/model/test_corpus.py @@ -25,19 +25,12 @@ from ..fixtures import tiny_corpus, sample_corpus, whole_corpus -slow = pytest.mark.skipif( - not pytest.config.getoption("--runslow"), - reason="need --runslow option to run" -) - - def test_tiny(): corpus = Corpus(source=tiny_corpus(), atftype='oracc') assert corpus.successes == 1 assert corpus.failures == 1 -@slow def test_sample(): corpus = Corpus(source=sample_corpus(), atftype='oracc') assert corpus.successes == 37 @@ -48,7 +41,6 @@ def test_sample(): reason="Need to set oracc_corpus_path to point " "to the whole corpus, which is not bundled with " "pyoracc") -@slow def test_whole(): corpus = Corpus(source=whole_corpus(), atftype='oracc') # There are a total of 8229 files in the corpus. From 39ca75d08a70b4c056f6389efef849d235b3c130 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Wed, 5 Jun 2019 18:15:19 -0700 Subject: [PATCH 09/15] travis: Test on release jython 2.7.1. The current release of jython is 2.7.1. Drop testing on 2.7.0 and 2.7.1b3 in favour of just testing on the release. --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 88f3eed..bd26185 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,7 @@ matrix: - os: linux sudo: false python: 2.7 - env: MYPYTHON=jython - JYTHON_URL="http://search.maven.org/remotecontent?filepath=org/python/jython-installer/2.7.0/jython-installer-2.7.0.jar" - - os: linux - sudo: false - python: 2.7 - env: MYPYTHON=jython - JYTHON_URL="http://search.maven.org/remotecontent?filepath=org/python/jython-installer/2.7.1b3/jython-installer-2.7.1b3.jar" + env: MYPYTHON=jython - JYTHON_URL="https://search.maven.org/remotecontent?filepath=org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" - os: linux sudo: false python: 3.5 From 570d8e0686ec28005af0d385006eee8388fda916 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Wed, 5 Jun 2019 18:16:21 -0700 Subject: [PATCH 10/15] travis: Use current pip and pytest with jython. These restriction was added about a year ago with a note about fixing the build, so this may break in other ways, but it would be nice to use modern pytest. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd26185..196007a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,8 +51,7 @@ install: # Install dependencies $PIP install pip if [ "$MYPYTHON" == "jython" ]; then - $PIP install pip==8.1.2 - $PIP install pytest==2.9.2 + $PIP install pytest fi $PIP install wheel $PIP install setuptools From 32d99e0405bc14d7ff7119299bb96d84e900fb6c Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Tue, 4 Jun 2019 17:29:00 -0700 Subject: [PATCH 11/15] Implement AtfFile.to_json(). Returns a string contain a JSON (Javascript Object Notation) serialization of the contents of the file. This is handy for exploring the syntax tree the parser produced, and for passing the data to other tools. --- pyoracc/atf/common/atffile.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pyoracc/atf/common/atffile.py b/pyoracc/atf/common/atffile.py index 48dde7d..9cfc6bb 100644 --- a/pyoracc/atf/common/atffile.py +++ b/pyoracc/atf/common/atffile.py @@ -20,6 +20,7 @@ import codecs import sys import logging +import json from pyoracc.atf.cdli.atflex import AtfCDLILexer from pyoracc.atf.cdli.atfyacc import AtfCDLIParser @@ -72,6 +73,33 @@ def __str__(self): def serialize(self): return AtfFile.template.render_unicode(**vars(self)) + def to_json(self, skip_empty=True, **kwargs): + '''Return a JSON representation of the parsed file. + + The optional skip_empty argument determines whether keys + with empty values are included in the output. Set it to + False to see all possible object members. + + Otherwise it accepts the same optional arguments as + json.dumps().''' + def _make_serializable(obj): + '''Construct a dict representation of an object. + + This is necessary to handle our custom objects + which json.JSONEncoder doesn't know how to + serialize.''' + + return {k: v + for k, v in vars(obj).items() + if not str(k).startswith('_') and not ( + skip_empty and not v and not isinstance(v, bool) + )} + + kwargs.setdefault('indent', 2) + kwargs.setdefault('sort_keys', True) + kwargs.setdefault('default', _make_serializable) + return json.dumps(self.text, **kwargs) + def check_atf(infile, atftype, verbose=False): content = codecs.open(infile, From 33255a81a10f643ae15f405692a0bc1f9b0fcc97 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Wed, 5 Jun 2019 15:44:38 -0700 Subject: [PATCH 12/15] Add a test for AtfFile.to_json(). Run the sample_corpus files through the AtfFile and then the json serializer. Check that they're valid json, but not much else. --- pyoracc/test/atf/test_atffile.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pyoracc/test/atf/test_atffile.py b/pyoracc/test/atf/test_atffile.py index 5000b1f..5a06c3a 100644 --- a/pyoracc/test/atf/test_atffile.py +++ b/pyoracc/test/atf/test_atffile.py @@ -21,6 +21,7 @@ from pyoracc.atf.common.atffile import AtfFile from ..fixtures import anzu, belsunu, sample_file import pytest +import json def test_create(): @@ -108,3 +109,18 @@ def test_text_designation(name, code, description): afile = AtfFile(sample_file(name)) assert afile.text.code == code assert afile.text.description == description + + +@pytest.mark.parametrize('name', [text[0] for text in texts]) +def test_json_serialization(name): + """ + Parses ATF and verifies the to_json() method output. + """ + afile = AtfFile(sample_file(name)) + js = afile.to_json() + result = json.loads(js) + assert result + noskipjs = afile.to_json(skip_empty=False) + result = json.loads(noskipjs) + assert result + assert len(noskipjs) >= len(js) From 478bc08957e028a07814c024adf7aadc00e3bb79 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Wed, 5 Jun 2019 17:13:08 -0700 Subject: [PATCH 13/15] Don't sort json keys by default. Now that Python retains insertion order for dicts, this gives more readable output in most cases, with properties like `objectype`, and `words` coming early in the JSON serialization. Also works around a serialization problem with `Multilingual` objects. --- pyoracc/atf/common/atffile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyoracc/atf/common/atffile.py b/pyoracc/atf/common/atffile.py index 9cfc6bb..bc78fdb 100644 --- a/pyoracc/atf/common/atffile.py +++ b/pyoracc/atf/common/atffile.py @@ -96,7 +96,6 @@ def _make_serializable(obj): )} kwargs.setdefault('indent', 2) - kwargs.setdefault('sort_keys', True) kwargs.setdefault('default', _make_serializable) return json.dumps(self.text, **kwargs) From af363ab7cfaef5092557ede1883cd38d4c0cf2fe Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Wed, 5 Jun 2019 17:20:22 -0700 Subject: [PATCH 14/15] Mark bb_2_6 as expected to fail the json serialzation test. Call to_json() explictly with sort_keys=True to make sure to trigger the issue with the unmarked language being under the None key. This will detect if that behaviour ever changes, which would mean we can remove this xfail mark. --- pyoracc/test/atf/test_atffile.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pyoracc/test/atf/test_atffile.py b/pyoracc/test/atf/test_atffile.py index 5a06c3a..fd7fdf2 100644 --- a/pyoracc/test/atf/test_atffile.py +++ b/pyoracc/test/atf/test_atffile.py @@ -111,7 +111,19 @@ def test_text_designation(name, code, description): assert afile.text.description == description -@pytest.mark.parametrize('name', [text[0] for text in texts]) +# ATF filenames which fail the serialization tests. +_xfail_texts = [ + # Multilingual objects store the unmarked language + # under the `None` key in their `lines` dictionary, + # which is incompatible with `sort_keys=True`. + 'bb_2_6', + ] + + +@pytest.mark.parametrize('name', [ + name if name not in _xfail_texts + else pytest.param(name, marks=[pytest.mark.xfail()]) + for name in [text[0] for text in texts]]) def test_json_serialization(name): """ Parses ATF and verifies the to_json() method output. @@ -120,7 +132,7 @@ def test_json_serialization(name): js = afile.to_json() result = json.loads(js) assert result - noskipjs = afile.to_json(skip_empty=False) + noskipjs = afile.to_json(skip_empty=False, sort_keys=True) result = json.loads(noskipjs) assert result assert len(noskipjs) >= len(js) From 17aa9c350d8215014e6becf157b164e1ad8d2511 Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Thu, 6 Jun 2019 11:53:13 -0700 Subject: [PATCH 15/15] Don't skip 0 values when serializing to json. By default we skip serializing properties which test as False, to avoid cluttering the output with a lot of empty arrays. Booleans were excluded from this, since a property which is literally False still carries useful information. Also exclude numbers from this check, so such properties don't disappear if they happen to be zero-valued, which is equally surprising. Since `bool` is a subclass of `int` checking against `numbers.Number` includes both cases. --- pyoracc/atf/common/atffile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyoracc/atf/common/atffile.py b/pyoracc/atf/common/atffile.py index bc78fdb..4160492 100644 --- a/pyoracc/atf/common/atffile.py +++ b/pyoracc/atf/common/atffile.py @@ -21,6 +21,7 @@ import sys import logging import json +from numbers import Number from pyoracc.atf.cdli.atflex import AtfCDLILexer from pyoracc.atf.cdli.atfyacc import AtfCDLIParser @@ -92,7 +93,7 @@ def _make_serializable(obj): return {k: v for k, v in vars(obj).items() if not str(k).startswith('_') and not ( - skip_empty and not v and not isinstance(v, bool) + skip_empty and not v and not isinstance(v, Number) )} kwargs.setdefault('indent', 2)