diff --git a/petl/io/csv_py2.py b/petl/io/csv_py2.py index a7057ad3..13ad48b8 100644 --- a/petl/io/csv_py2.py +++ b/petl/io/csv_py2.py @@ -112,7 +112,10 @@ def __iter__(self): it = iter(self.table) # deal with header row - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if self.write_header: writer.writerow(hdr) # N.B., always yield header, even if we don't write it diff --git a/petl/io/csv_py3.py b/petl/io/csv_py3.py index 9b800baa..cf228c1c 100644 --- a/petl/io/csv_py3.py +++ b/petl/io/csv_py3.py @@ -20,15 +20,15 @@ def fromcsv_impl(source, **kwargs): class CSVView(Table): def __init__(self, source, encoding, errors, header, **csvargs): - self.source = source - self.encoding = encoding - self.errors = errors - self.csvargs = csvargs - self.header = header + self.source = source + self.encoding = encoding + self.errors = errors + self.csvargs = csvargs + self.header = header def __iter__(self): if self.header is not None: - yield tuple(self.header) + yield tuple(self.header) with self.source.open('rb') as buf: csvfile = io.TextIOWrapper(buf, encoding=self.encoding, errors=self.errors, newline='') @@ -86,7 +86,10 @@ def __iter__(self): try: writer = csv.writer(csvfile, **self.csvargs) it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if self.write_header: writer.writerow(hdr) yield tuple(hdr) diff --git a/petl/io/html.py b/petl/io/html.py index fe6ef07c..029723e4 100644 --- a/petl/io/html.py +++ b/petl/io/html.py @@ -75,7 +75,10 @@ def tohtml(table, source=None, encoding=None, errors='strict', caption=None, it = iter(table) # write header - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] _write_begin(f, hdr, lineterminator, caption, index_header, truncate) @@ -166,10 +169,13 @@ def __iter__(self): it = iter(table) # write header - hdr = next(it) + try: + hdr = next(it) + yield hdr + except StopIteration: + hdr = [] _write_begin(f, hdr, lineterminator, caption, index_header, truncate) - yield hdr # write body if tr_style and callable(tr_style): @@ -193,16 +199,17 @@ def _write_begin(f, flds, lineterminator, caption, index_header, truncate): f.write("" + lineterminator) if caption is not None: f.write(('' % caption) + lineterminator) - f.write('' + lineterminator) - f.write('' + lineterminator) - for i, h in enumerate(flds): - if index_header: - h = '%s|%s' % (i, h) - if truncate: - h = h[:truncate] - f.write(('' % h) + lineterminator) - f.write('' + lineterminator) - f.write('' + lineterminator) + if flds: + f.write('' + lineterminator) + f.write('' + lineterminator) + for i, h in enumerate(flds): + if index_header: + h = '%s|%s' % (i, h) + if truncate: + h = h[:truncate] + f.write(('' % h) + lineterminator) + f.write('' + lineterminator) + f.write('' + lineterminator) f.write('' + lineterminator) diff --git a/petl/io/pandas.py b/petl/io/pandas.py index f3632f0e..68961670 100644 --- a/petl/io/pandas.py +++ b/petl/io/pandas.py @@ -29,7 +29,10 @@ def todataframe(table, index=None, exclude=None, columns=None, """ import pandas as pd it = iter(table) - header = next(it) + try: + header = next(it) + except StopIteration: + header = None # Will create an Empty DataFrame if columns is None: columns = header return pd.DataFrame.from_records(it, index=index, exclude=exclude, diff --git a/petl/io/pickle.py b/petl/io/pickle.py index 0bf77802..e7ba4da6 100644 --- a/petl/io/pickle.py +++ b/petl/io/pickle.py @@ -119,7 +119,10 @@ def _writepickle(table, source, mode, protocol, write_header): source = write_source_from_arg(source, mode) with source.open(mode) as f: it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if write_header: pickle.dump(hdr, f, protocol) for row in it: @@ -153,7 +156,10 @@ def __iter__(self): source = write_source_from_arg(self.source) with source.open('wb') as f: it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if self.write_header: pickle.dump(hdr, f, protocol) yield tuple(hdr) diff --git a/petl/io/text.py b/petl/io/text.py index 570d2efe..f9f79126 100644 --- a/petl/io/text.py +++ b/petl/io/text.py @@ -194,7 +194,10 @@ def _writetext(table, source, mode, encoding, errors, template, prologue, if prologue is not None: f.write(prologue) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) for row in it: rec = asdict(flds, row) @@ -266,7 +269,10 @@ def _iterteetext(table, source, encoding, errors, template, prologue, epilogue): if prologue is not None: f.write(prologue) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) flds = list(map(text_type, hdr)) for row in it: diff --git a/petl/io/xls.py b/petl/io/xls.py index b9a4599b..8b6559fa 100644 --- a/petl/io/xls.py +++ b/petl/io/xls.py @@ -85,12 +85,15 @@ def toxls(tbl, filename, sheet, encoding=None, style_compression=0, else: # handle styles it = iter(tbl) - hdr = next(it) - flds = list(map(str, hdr)) - for c, f in enumerate(flds): - ws.write(0, c, label=f) - if f not in styles or styles[f] is None: - styles[f] = xlwt.Style.default_style + try: + hdr = next(it) + flds = list(map(str, hdr)) + for c, f in enumerate(flds): + ws.write(0, c, label=f) + if f not in styles or styles[f] is None: + styles[f] = xlwt.Style.default_style + except StopIteration: + pass # no header written # convert to list for easy zipping styles = [styles[f] for f in flds] for r, row in enumerate(it): diff --git a/petl/io/xlsx.py b/petl/io/xlsx.py index c1f8bfac..612f7c36 100644 --- a/petl/io/xlsx.py +++ b/petl/io/xlsx.py @@ -113,9 +113,12 @@ def toxlsx(tbl, filename, sheet=None, write_header=True, mode="replace"): ws = _insert_sheet_on_workbook(mode, sheet, wb) if write_header: it = iter(tbl) - hdr = next(it) - flds = list(map(text_type, hdr)) - rows = itertools.chain([flds], it) + try: + hdr = next(it) + flds = list(map(text_type, hdr)) + rows = itertools.chain([flds], it) + except StopIteration: + rows = it else: rows = data(tbl) for row in rows: @@ -184,9 +187,12 @@ def appendxlsx(tbl, filename, sheet=None, write_header=False): ws = wb[str(sheet)] if write_header: it = iter(tbl) - hdr = next(it) - flds = list(map(text_type, hdr)) - rows = itertools.chain([flds], it) + try: + hdr = next(it) + flds = list(map(text_type, hdr)) + rows = itertools.chain([flds], it) + except StopIteration: + rows = it else: rows = data(tbl) for row in rows: diff --git a/petl/test/helpers.py b/petl/test/helpers.py index 60c780f9..5a799558 100644 --- a/petl/test/helpers.py +++ b/petl/test/helpers.py @@ -10,7 +10,7 @@ def eq_(expect, actual, msg=None): """Test when two values from a python variable are exactly equals (==)""" - assert expect == actual, msg + assert expect == actual, msg or ('%r != %s' % (expect, actual)) def assert_almost_equal(first, second, places=None, msg=None): diff --git a/petl/test/io/test_html.py b/petl/test/io/test_html.py index 127f5d09..8d6b565a 100644 --- a/petl/test/io/test_html.py +++ b/petl/test/io/test_html.py @@ -118,3 +118,21 @@ def test_tohtml_with_style(): u"
%s
%s
%s
\n" ) eq_(expect, actual) + + +def test_tohtml_headerless(): + table = [] + + f = NamedTemporaryFile(delete=False) + tohtml(table, f.name, encoding='ascii', lineterminator='\n') + + # check what it did + with io.open(f.name, mode='rt', encoding='ascii', newline='') as o: + actual = o.read() + expect = ( + u"\n" + u"\n" + u"\n" + u"
\n" + ) + eq_(expect, actual) diff --git a/petl/test/io/test_pandas.py b/petl/test/io/test_pandas.py index d592e350..0361aec3 100644 --- a/petl/test/io/test_pandas.py +++ b/petl/test/io/test_pandas.py @@ -26,6 +26,12 @@ def test_todataframe(): actual = todataframe(tbl) assert expect.equals(actual) + def test_headerless(): + tbl = [] + expect = pd.DataFrame() + actual = todataframe(tbl) + assert expect.equals(actual) + def test_fromdataframe(): tbl = [('foo', 'bar', 'baz'), ('apples', 1, 2.5), diff --git a/petl/test/io/test_pickle.py b/petl/test/io/test_pickle.py index 7f8f160f..3107b5d3 100644 --- a/petl/test/io/test_pickle.py +++ b/petl/test/io/test_pickle.py @@ -10,6 +10,14 @@ from petl.io.pickle import frompickle, topickle, appendpickle +def picklereader(fl): + try: + while True: + yield pickle.load(fl) + except EOFError: + pass + + def test_frompickle(): f = NamedTemporaryFile(delete=False) @@ -36,13 +44,6 @@ def test_topickle_appendpickle(): f = NamedTemporaryFile(delete=False) topickle(table, f.name) - def picklereader(fl): - try: - while True: - yield pickle.load(fl) - except EOFError: - pass - # check what it did with open(f.name, 'rb') as o: actual = picklereader(o) @@ -66,3 +67,12 @@ def picklereader(fl): ('e', 9), ('f', 1)) ieq(expect, actual) + + +def test_topickle_headerless(): + table = [] + f = NamedTemporaryFile(delete=False) + topickle(table, f.name) + expect = [] + with open(f.name, 'rb') as o: + ieq(expect, picklereader(o)) diff --git a/petl/test/io/test_text.py b/petl/test/io/test_text.py index 4ffdf19d..90a31ca8 100644 --- a/petl/test/io/test_text.py +++ b/petl/test/io/test_text.py @@ -174,3 +174,18 @@ def test_totext_gz(): eq_(expect, actual) finally: o.close() + + +def test_totext_headerless(): + table = [] + f = NamedTemporaryFile(delete=False) + prologue = "-- START\n" + template = "+ {f1}\n" + epilogue = "-- END\n" + totext(table, f.name, encoding='ascii', template=template, + prologue=prologue, epilogue=epilogue) + + with io.open(f.name, mode='rt', encoding='ascii', newline='') as o: + actual = o.read() + expect = prologue + epilogue + eq_(expect, actual) diff --git a/petl/test/io/test_xls.py b/petl/test/io/test_xls.py index c29d51c8..b748255d 100644 --- a/petl/test/io/test_xls.py +++ b/petl/test/io/test_xls.py @@ -85,6 +85,15 @@ def test_toxls(): ieq(expect, actual) ieq(expect, actual) + def test_toxls_headerless(): + expect = [] + f = NamedTemporaryFile(delete=False) + f.close() + toxls(expect, f.name, 'Sheet1') + actual = fromxls(f.name, 'Sheet1') + ieq(expect, actual) + ieq(expect, actual) + def test_toxls_date(): expect = (('foo', 'bar'), (u'é', datetime(2012, 1, 1)), @@ -128,4 +137,4 @@ def wrapper(self, *args, **kwargs): ('C', 2), (u'é', datetime(2012, 1, 1))) ieq(expect, tbl) - ieq(expect, tbl) \ No newline at end of file + ieq(expect, tbl) diff --git a/petl/test/io/test_xlsx.py b/petl/test/io/test_xlsx.py index 9076e7e4..469d2fda 100644 --- a/petl/test/io/test_xlsx.py +++ b/petl/test/io/test_xlsx.py @@ -226,3 +226,13 @@ def test_appendxlsx_with_non_str_header(xlsx_table_with_non_str_header, xlsx_tes appendxlsx(xlsx_table_with_non_str_header, f.name, 'Sheet1') expect = etl.cat(xlsx_test_table, xlsx_table_with_non_str_header) ieq(expect, actual) + + +def test_toxlsx_headerless(): + expect = [] + f = NamedTemporaryFile(delete=False) + f.close() + toxlsx(expect, f.name) + actual = fromxlsx(f.name) + ieq(expect, actual) + ieq(expect, actual) diff --git a/petl/test/transform/test_basics.py b/petl/test/transform/test_basics.py index 2424dab6..dfcdaf2d 100644 --- a/petl/test/transform/test_basics.py +++ b/petl/test/transform/test_basics.py @@ -2,6 +2,7 @@ import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import ieq from petl.util import expr, empty, coalesce from petl.transform.basics import cut, cat, addfield, rowslice, head, tail, \ @@ -71,6 +72,13 @@ def test_cut_empty(): ieq(expect, actual) +def test_cut_headerless(): + table = () + with pytest.raises(FieldSelectionError): + for i in cut(table, 'bar'): + pass + + def test_cutout(): table = (('foo', 'bar', 'baz'), @@ -108,6 +116,13 @@ def test_cutout(): ieq(expectation, cut3) +def test_cutout_headerless(): + table = () + with pytest.raises(FieldSelectionError): + for i in cutout(table, 'bar'): + pass + + def test_cat(): table1 = (('foo', 'bar'), @@ -198,6 +213,16 @@ def test_cat_empty(): ieq(expect, actual) +def test_cat_headerless(): + table1 = (('foo', 'bar'), + (1, 'A'), + (2, 'B')) + table2 = () + expect = table1 # basically does nothing + actual = cat(table1, table2) + ieq(expect, actual) + + def test_cat_dupfields(): table1 = (('foo', 'foo'), (1, 'A'), @@ -253,6 +278,16 @@ def test_stack_dupfields(): ieq(expect, actual) +def test_stack_headerless(): + table1 = (('foo', 'bar'), + (1, 'A'), + (2, 'B')) + table2 = () + expect = table1 # basically does nothing + actual = stack(table1, table2) + ieq(expect, actual) + + def test_addfield(): table = (('foo', 'bar'), ('M', 12), @@ -308,6 +343,15 @@ def test_addfield_empty(): ieq(expect, actual) +def test_addfield_headerless(): + """When adding a field to a headerless table, implicitly add a header.""" + table = () + expect = (('foo',),) + actual = addfield(table, 'foo', 1) + ieq(expect, actual) + ieq(expect, actual) + + def test_addfield_coalesce(): table = (('foo', 'bar', 'baz', 'quux'), ('M', 12, 23, 44), @@ -438,6 +482,13 @@ def test_rowslice_empty(): ieq(expect, actual) +def test_rowslice_headerless(): + table = () + expect = () + actual = rowslice(table, 1, 2) + ieq(expect, actual) + + def test_head(): table1 = (('foo', 'bar'), @@ -505,6 +556,13 @@ def test_tail_empty(): ieq(expect, actual) +def test_tail_headerless(): + table = () + expect = () + actual = tail(table) + ieq(expect, actual) + + def test_skipcomments(): table1 = (('##aaa', 'bbb', 'ccc'), @@ -576,6 +634,16 @@ def test_annex_uneven_rows(): ieq(expect, actual) +def test_annex_headerless(): + table1 = (('foo', 'bar'), + ('C', 2)) + table2 = () # does nothing + expect = table1 + actual = annex(table1, table2) + ieq(expect, actual) + ieq(expect, actual) + + def test_addrownumbers(): table1 = (('foo', 'bar'), @@ -606,6 +674,15 @@ def test_addrownumbers_field_name(): ieq(expect, actual) +def test_addrownumbers_headerless(): + """Adds a column row if there is none.""" + table = () + expect = (('id',),) + actual = addrownumbers(table, field='id') + ieq(expect, actual) + ieq(expect, actual) + + def test_addcolumn(): table1 = (('foo', 'bar'), @@ -655,6 +732,17 @@ def test_empty_addcolumn(): ieq(expect, table3) +def test_addcolumn_headerless(): + """Adds a header row if none exists.""" + table1 = () + expect = (('foo',), + ('A',), + ('B',)) + actual = addcolumn(table1, 'foo', ['A', 'B']) + ieq(expect, actual) + ieq(expect, actual) + + def test_addfieldusingcontext(): table1 = (('foo', 'bar'), @@ -720,6 +808,30 @@ def downstream(prv, cur, nxt): ieq(expect, table3) +def test_addfieldusingcontext_empty(): + table = empty() + expect = (('foo',),) + + def query(prv, cur, nxt): + return 0 + + actual = addfieldusingcontext(table, 'foo', query) + ieq(expect, actual) + ieq(expect, actual) + + +def test_addfieldusingcontext_headerless(): + table = () + expect = (('foo',),) + + def query(prv, cur, nxt): + return 0 + + actual = addfieldusingcontext(table, 'foo', query) + ieq(expect, actual) + ieq(expect, actual) + + def test_movefield(): table1 = (('foo', 'bar', 'baz'), diff --git a/petl/test/transform/test_conversions.py b/petl/test/transform/test_conversions.py index 0a7e7400..caa22a58 100644 --- a/petl/test/transform/test_conversions.py +++ b/petl/test/transform/test_conversions.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import FieldSelectionError from petl.test.failonerror import assert_failonerror from petl.test.helpers import ieq from petl.transform.conversions import convert, convertall, convertnumbers, \ @@ -72,6 +74,19 @@ def test_convert_empty(): ieq(expect, actual) +def test_convert_headerless(): + table = () + with pytest.raises(FieldSelectionError): + for i in convert(table, 'foo', int): + pass + + +def test_convert_headerless_no_conversions(): + table = expect = () + actual = convert(table) + ieq(expect, actual) + + def test_convert_indexes(): table1 = (('foo', 'bar', 'baz'), diff --git a/petl/test/transform/test_dedup.py b/petl/test/transform/test_dedup.py index 519e9211..8f783188 100644 --- a/petl/test/transform/test_dedup.py +++ b/petl/test/transform/test_dedup.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import ieq from petl.transform.dedup import duplicates, unique, conflicts, distinct, \ isunique @@ -34,6 +36,23 @@ def test_duplicates(): ieq(expectation, result) +def test_duplicates_headerless_no_keys(): + """Removing the duplicates from an empty table without specifying which + columns shouldn't be a problem. + """ + table = [] + actual = duplicates(table) + expect = [] + ieq(expect, actual) + + +def test_duplicates_headerless_explicit(): + table = [] + with pytest.raises(FieldSelectionError): + for i in duplicates(table, 'foo'): + pass + + def test_duplicates_empty(): table = (('foo', 'bar'),) expect = (('foo', 'bar'),) diff --git a/petl/test/transform/test_fills.py b/petl/test/transform/test_fills.py index c6c8625e..e5df2cca 100644 --- a/petl/test/transform/test_fills.py +++ b/petl/test/transform/test_fills.py @@ -53,6 +53,13 @@ def test_filldown(): ieq(expect, actual) +def test_filldown_headerless(): + table = [] + actual = filldown(table, 'foo') + expect = [] + ieq(expect, actual) + + def test_fillright(): table = (('foo', 'bar', 'baz'), @@ -77,6 +84,13 @@ def test_fillright(): ieq(expect, actual) +def test_fillright_headerless(): + table = [] + actual = fillright(table, 'foo') + expect = [] + ieq(expect, actual) + + def test_fillleft(): table = (('foo', 'bar', 'baz'), @@ -99,3 +113,10 @@ def test_fillleft(): ('c', 'c', .72)) ieq(expect, actual) ieq(expect, actual) + + +def test_fillleft_headerless(): + table = [] + actual = fillleft(table, 'foo') + expect = [] + ieq(expect, actual) diff --git a/petl/test/transform/test_headers.py b/petl/test/transform/test_headers.py index a40ef26f..50b98449 100644 --- a/petl/test/transform/test_headers.py +++ b/petl/test/transform/test_headers.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, print_function, division +import pytest from petl.test.helpers import ieq from petl.errors import FieldSelectionError @@ -29,6 +30,13 @@ def test_setheader_empty(): ieq(expect2, table2) +def test_setheader_headerless(): + table = [] + actual = setheader(table, ['foo', 'bar']) + expect = [('foo', 'bar')] + ieq(expect, actual) + + def test_extendheader(): table1 = (('foo',), @@ -50,6 +58,14 @@ def test_extendheader_empty(): ieq(expect2, table2) +def test_extendheader_headerless(): + table = [] + actual = extendheader(table, ['foo', 'bar']) + expect = [('foo', 'bar')] + ieq(expect, actual) + ieq(expect, actual) + + def test_pushheader(): table1 = (('a', 1), @@ -81,6 +97,14 @@ def test_pushheader_empty(): ieq(expect2, table2) +def test_pushheader_headerless(): + table = [] + actual = pushheader(table, ['foo', 'bar']) + expect = [('foo', 'bar')] + ieq(expect, actual) + ieq(expect, actual) + + def test_pushheader_positional(): table1 = (('a', 1), @@ -152,6 +176,13 @@ def test_skip_empty(): ieq(expect2, table2) +def test_skip_headerless(): + table = [] + actual = skip(table, 2) + expect = [] + ieq(expect, actual) + + def test_rename(): table = (('foo', 'bar'), @@ -212,6 +243,13 @@ def test_rename_empty(): ieq(expect, actual) +def test_rename_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + for i in rename(table, 'foo', 'foofoo'): + pass + + def test_prefixheader(): table1 = (('foo', 'bar'), @@ -227,6 +265,13 @@ def test_prefixheader(): ieq(expect, actual) +def test_prefixheader_headerless(): + table = [] + actual = prefixheader(table, 'pre_') + expect = [] + ieq(expect, actual) + + def test_suffixheader(): table1 = (('foo', 'bar'), @@ -242,6 +287,13 @@ def test_suffixheader(): ieq(expect, actual) +def test_suffixheader_headerless(): + table = [] + actual = suffixheader(table, '_suf') + expect = [] + ieq(expect, actual) + + def test_sortheaders(): table1 = ( ('id', 'foo', 'bar', 'baz'), @@ -274,3 +326,10 @@ def test_sortheaders_duplicate_headers(): actual = sortheader(table1) ieq(expect, actual) + + +def test_sortheader_headerless(): + table = [] + actual = sortheader(table) + expect = [] + ieq(expect, actual) diff --git a/petl/test/transform/test_maps.py b/petl/test/transform/test_maps.py index 155f6c2f..c23c7108 100644 --- a/petl/test/transform/test_maps.py +++ b/petl/test/transform/test_maps.py @@ -91,6 +91,16 @@ def test_fieldmap_empty(): ieq(expect, actual) +def test_fieldmap_headerless(): + table = [] + expect = [] + mappings = OrderedDict() + mappings['foo'] = 'foo' + mappings['baz'] = 'bar', lambda v: v * 2 + actual = fieldmap(table, mappings) + ieq(expect, actual) + + def test_fieldmap_failonerror(): input_ = (('foo',), ('A',), (1,)) mapper_ = {'bar': ('foo', lambda v: v.lower())} @@ -160,6 +170,17 @@ def rowmapper(row): ieq(expect, actual) +def test_rowmap_headerless(): + table = [] + + def rowmapper(row): + return row + + actual = rowmap(table, rowmapper, header=['subject_id', 'gender']) + expect = [] + ieq(expect, actual) + + def test_rowmap_failonerror(): input_ = (('foo',), ('A',), (1,), ('B',)) mapper = lambda r: [r[0].lower()] @@ -287,3 +308,15 @@ def rowgenerator(rec): ieq(expect, actual) ieq(expect, actual) # can iteratate twice? + +def test_recordmapmany_headerless(): + table = [] + + def duplicate(rec): + yield rec + yield rec + + actual = rowmapmany(table, duplicate, header=['subject_id', 'variable']) + expect = [] + ieq(expect, actual) + ieq(expect, actual) # can iteratate twice? diff --git a/petl/test/transform/test_regex.py b/petl/test/transform/test_regex.py index 95305f5f..fead5160 100644 --- a/petl/test/transform/test_regex.py +++ b/petl/test/transform/test_regex.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, division +import pytest from petl.compat import next - - +from petl.errors import ArgumentError from petl.test.helpers import ieq, eq_ from petl.transform.regex import capture, split, search, searchcomplement, splitdown from petl.transform.basics import TransformError @@ -57,6 +57,13 @@ def test_capture_empty(): ieq(expect, actual) +def test_capture_headerless(): + table = [] + with pytest.raises(ArgumentError): + for i in capture(table, 'bar', r'(\w)(\d)', ('baz', 'qux')): + pass + + def test_capture_nonmatching(): table = (('id', 'variable', 'value'), @@ -148,6 +155,13 @@ def test_split_empty(): ieq(expect, actual) +def test_split_headerless(): + table = [] + with pytest.raises(ArgumentError): + for i in split(table, 'bar', 'd', ('baz', 'qux')): + pass + + def test_search(): table1 = (('foo', 'bar', 'baz'), @@ -193,6 +207,14 @@ def test_search_2(): ieq(expect, actual) +def test_search_headerless(): + table = [] + actual = search(table, 'foo', '[ab]{2}') + expect = [] + ieq(expect, actual) + ieq(expect, actual) + + def test_searchcomplement(): table1 = (('foo', 'bar', 'baz'), diff --git a/petl/test/transform/test_reshape.py b/petl/test/transform/test_reshape.py index 35502237..e69048a5 100644 --- a/petl/test/transform/test_reshape.py +++ b/petl/test/transform/test_reshape.py @@ -3,7 +3,9 @@ from datetime import datetime +import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import ieq from petl.transform.reshape import melt, recast, transpose, pivot, flatten, \ unflatten @@ -69,6 +71,13 @@ def test_melt_empty(): ieq(expect, actual) +def test_melt_headerless(): + table = [] + expect = [] + actual = melt(table, key='foo') + ieq(expect, actual) + + def test_melt_1_shortrow(): table = (('id', 'gender', 'age'), @@ -254,6 +263,13 @@ def test_recast_empty(): ieq(expect, actual) +def test_recast_headerless(): + table = [] + expect = [] + actual = recast(table) + ieq(expect, actual) + + def test_recast_date(): dt = datetime.now().replace @@ -383,6 +399,13 @@ def test_pivot_empty(): ieq(expect2, table2) +def test_pivot_headerless(): + table1 = [] + with pytest.raises(FieldSelectionError): + for i in pivot(table1, 'region', 'gender', 'units', sum): + pass + + def test_flatten(): table1 = (('foo', 'bar', 'baz'), @@ -406,6 +429,13 @@ def test_flatten_empty(): ieq(expect1, actual1) +def test_flatten_headerless(): + table1 = [] + expect1 = [] + actual1 = flatten(table1) + ieq(expect1, actual1) + + def test_unflatten(): table1 = (('lines',), diff --git a/petl/test/transform/test_selects.py b/petl/test/transform/test_selects.py index ec9ac913..24d1cf63 100644 --- a/petl/test/transform/test_selects.py +++ b/petl/test/transform/test_selects.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import ieq, eq_ from petl.comparison import Comparable from petl.transform.selects import select, selectin, selectcontains, \ @@ -113,6 +115,20 @@ def test_select_empty(): ieq(expect, actual) +def test_rowselect_headerless(): + table = [] + expect = [] + actual = select(table, 'True') + ieq(expect, actual) + + +def test_fieldselect_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + for i in select(table, 'foo', lambda v: v == 'a'): + pass + + def test_select_falsey(): table = (('foo',), ([],), diff --git a/petl/test/transform/test_sorts.py b/petl/test/transform/test_sorts.py index 24aff0b1..b962e1ea 100644 --- a/petl/test/transform/test_sorts.py +++ b/petl/test/transform/test_sorts.py @@ -11,7 +11,7 @@ from petl.compat import next - +from petl.errors import FieldSelectionError from petl.test.helpers import ieq, eq_ from petl.util import nrows from petl.transform.basics import cat @@ -361,6 +361,25 @@ def test_sort_none(): ieq(expectation, result) +def test_sort_headerless_no_keys(): + """ + Sorting a headerless table without specifying cols should be a no-op. + """ + table = [] + result = sort(table) + expectation = [] + ieq(expectation, result) + + +def test_sort_headerless_explicit(): + """ + But if you specify keys, they must exist. + """ + table = [] + with pytest.raises(FieldSelectionError): + for i in sort(table, 'foo'): + pass + # TODO test sort with native comparison diff --git a/petl/test/transform/test_unpacks.py b/petl/test/transform/test_unpacks.py index bb83e910..e913d1ff 100644 --- a/petl/test/transform/test_unpacks.py +++ b/petl/test/transform/test_unpacks.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import ArgumentError from petl.test.helpers import ieq from petl.transform.unpacks import unpack, unpackdict @@ -76,6 +78,13 @@ def test_unpack_empty(): ieq(expect2, table2) +def test_unpack_headerless(): + table = [] + with pytest.raises(ArgumentError): + for i in unpack(table, 'bar', ['baz', 'quux']): + pass + + def test_unpackdict(): table1 = (('foo', 'bar'), diff --git a/petl/test/transform/test_validation.py b/petl/test/transform/test_validation.py index a8e6dc28..cfe88121 100644 --- a/petl/test/transform/test_validation.py +++ b/petl/test/transform/test_validation.py @@ -130,3 +130,14 @@ def test_header(): ieq(expect, actual) ieq(expect, actual) + + +def test_validation_headerless(): + header = ('foo', 'bar', 'baz') + table = [] + # Expect only a missing header - no exceptions please + expect = (('name', 'row', 'field', 'value', 'error'), + ('__header__', 0, None, None, 'AssertionError')) + actual = validate(table, header=header) + ieq(expect, actual) + ieq(expect, actual) diff --git a/petl/test/util/test_base.py b/petl/test/util/test_base.py index 65431e2d..8cd54d1d 100644 --- a/petl/test/util/test_base.py +++ b/petl/test/util/test_base.py @@ -1,7 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest -from petl.errors import ArgumentError +from petl.errors import FieldSelectionError from petl.test.helpers import ieq, eq_ from petl.compat import next from petl.util.base import header, fieldnames, data, dicts, records, \ @@ -52,6 +53,13 @@ def test_data(): ieq(expect, actual) +def test_data_headerless(): + table = [] + actual = data(table) + expect = [] + ieq(expect, actual) + + def test_dicts(): table = (('foo', 'bar'), ('a', 1), ('b', 2)) actual = dicts(table) @@ -59,6 +67,13 @@ def test_dicts(): ieq(expect, actual) +def test_dicts_headerless(): + table = [] + actual = dicts(table) + expect = [] + ieq(expect, actual) + + def test_dicts_shortrows(): table = (('foo', 'bar'), ('a', 1), ('b',)) actual = dicts(table) @@ -94,6 +109,13 @@ def test_records(): eq_('qux', o.get('baz', default='qux')) +def test_records_headerless(): + table = [] + actual = records(table) + expect = [] + ieq(expect, actual) + + def test_records_errors(): table = (('foo', 'bar'), ('a', 1), ('b', 2)) actual = records(table) @@ -147,6 +169,13 @@ def test_namedtuples(): eq_(2, o.bar) +def test_namedtuples_headerless(): + table = [] + actual = namedtuples(table) + expect = [] + ieq(expect, actual) + + def test_namedtuples_unevenrows(): table = (('foo', 'bar'), ('a', 1, True), ('b',)) actual = namedtuples(table) @@ -187,6 +216,14 @@ def test_itervalues(): ieq(expect, actual) +def test_itervalues_headerless(): + table = [] + actual = itervalues(table, 'foo') + with pytest.raises(FieldSelectionError): + for i in actual: + pass + + def test_values(): table = (('foo', 'bar', 'baz'), @@ -222,6 +259,14 @@ def test_values(): ieq(expect, actual) +def test_values_headerless(): + table = [] + actual = values(table, 'foo') + with pytest.raises(FieldSelectionError): + for i in actual: + pass + + def test_rowgroupby(): table = (('foo', 'bar', 'baz'), @@ -279,3 +324,9 @@ def test_rowgroupby(): eq_(2, len(vals)) eq_(True, vals[0]) eq_(None, vals[1]) # gets padded + + +def test_rowgroupby_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + rowgroupby(table, 'foo') diff --git a/petl/test/util/test_lookups.py b/petl/test/util/test_lookups.py index 2a8c54bf..06002d21 100644 --- a/petl/test/util/test_lookups.py +++ b/petl/test/util/test_lookups.py @@ -1,7 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest -from petl.errors import DuplicateKeyError +from petl.errors import DuplicateKeyError, FieldSelectionError from petl.test.helpers import eq_ from petl import cut, lookup, lookupone, dictlookup, dictlookupone, \ recordlookup, recordlookupone @@ -42,6 +43,12 @@ def test_lookup(): eq_(expect, actual) +def test_lookup_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + lookup(table, 'foo', 'bar') + + def test_lookupone(): t1 = (('foo', 'bar'), ('a', 1), ('b', 2), ('b', 3)) @@ -85,6 +92,12 @@ def test_lookupone(): eq_(expect, actual) +def test_lookupone_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + lookupone(table, 'foo', 'bar') + + def test_dictlookup(): t1 = (('foo', 'bar'), ('a', 1), ('b', 2), ('b', 3)) diff --git a/petl/test/util/test_materialise.py b/petl/test/util/test_materialise.py index ae723720..b09be7f0 100644 --- a/petl/test/util/test_materialise.py +++ b/petl/test/util/test_materialise.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import eq_ from petl.util.materialise import columns, facetcolumns @@ -13,6 +15,19 @@ def test_columns(): eq_([1, 2, 3], cols['bar']) +def test_columns_empty(): + table = [('foo', 'bar')] + cols = columns(table) + eq_([], cols['foo']) + eq_([], cols['bar']) + + +def test_columns_headerless(): + table = [] + cols = columns(table) + eq_({}, cols) + + def test_facetcolumns(): table = [['foo', 'bar', 'baz'], @@ -27,3 +42,9 @@ def test_facetcolumns(): eq_(['b', 'b'], fc['b']['foo']) eq_([2, 3], fc['b']['bar']) eq_([True, None], fc['b']['baz']) + + +def test_facetcolumns_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + facetcolumns(table, 'foo') diff --git a/petl/test/util/test_vis.py b/petl/test/util/test_vis.py index b34b089e..b6227acc 100644 --- a/petl/test/util/test_vis.py +++ b/petl/test/util/test_vis.py @@ -189,3 +189,10 @@ def test_lookstr(): +-----+-----+ """ eq_(expect, actual) + + +def test_look_headerless(): + table = [] + actual = repr(look(table)) + expect = "" + eq_(expect, actual) diff --git a/petl/transform/basics.py b/petl/transform/basics.py index 8627e81a..faa3e4c0 100644 --- a/petl/transform/basics.py +++ b/petl/transform/basics.py @@ -130,7 +130,10 @@ def itercut(source, spec, missing=None): spec = tuple(spec) # make sure no-one can change midstream # convert field selection into field indices - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] indices = asindices(hdr, spec) # define a function to transform each row in the source data @@ -202,7 +205,10 @@ def itercutout(source, spec, missing=None): spec = tuple(spec) # make sure no-one can change midstream # convert field selection into field indices - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] indicesout = asindices(hdr, spec) indices = [i for i in range(len(hdr)) if i not in indicesout] @@ -340,7 +346,12 @@ def __iter__(self): def itercat(sources, missing, header): its = [iter(t) for t in sources] - hdrs = [list(next(it)) for it in its] + hdrs = [] + for it in its: + try: + hdrs.append(list(next(it))) + except StopIteration: + hdrs.append([]) if header is None: # determine output fields by gathering all fields found in the sources @@ -451,7 +462,12 @@ def __iter__(self): def iterstack(sources, missing, trim, pad): its = [iter(t) for t in sources] - hdrs = [next(it) for it in its] + hdrs = [] + for it in its: + try: + hdrs.append(next(it)) + except StopIteration: + hdrs.append([]) hdr = hdrs[0] n = len(hdr) yield tuple(hdr) @@ -526,7 +542,10 @@ def __iter__(self): def iteraddfield(source, field, value, index): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) # determine index of new field @@ -615,7 +634,10 @@ def __iter__(self): def iteraddfields(source, field_defs): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) # initialize output fields and indices @@ -826,7 +848,10 @@ def __iter__(self): def itertail(source, n): it = iter(source) - yield tuple(next(it)) # fields + try: + yield tuple(next(it)) # fields + except StopIteration: + return # stop generating cache = deque() for row in it: cache.append(row) @@ -910,7 +935,10 @@ def __iter__(self): it = iter(self.table) # determine output fields - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] outhdr = [f for f in hdr if f != self.field] outhdr.insert(self.index, self.field) yield tuple(outhdr) @@ -977,7 +1005,12 @@ def __iter__(self): def iterannex(tables, missing): its = [iter(t) for t in tables] - hdrs = [next(it) for it in its] + hdrs = [] + for it in its: + try: + hdrs.append(next(it)) + except StopIteration: + hdrs.append([]) outhdr = tuple(chain(*hdrs)) yield outhdr for rows in izip_longest(*its): @@ -1042,7 +1075,10 @@ def __iter__(self): def iteraddrownumbers(table, start, step, field): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] outhdr = [field] outhdr.extend(hdr) yield tuple(outhdr) @@ -1097,7 +1133,10 @@ def __iter__(self): def iteraddcolumn(table, field, col, index, missing): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] # determine position of new column if index is None: @@ -1186,13 +1225,19 @@ def __iter__(self): def iteraddfieldusingcontext(table, field, query): it = iter(table) - hdr = tuple(next(it)) + try: + hdr = tuple(next(it)) + except StopIteration: + hdr = () flds = list(map(text_type, hdr)) yield hdr + (field,) flds.append(field) it = (Record(row, flds) for row in it) prv = None - cur = next(it) + try: + cur = next(it) + except StopIteration: + return # no more items for nxt in it: v = query(prv, cur, nxt) yield tuple(cur) + (v,) diff --git a/petl/transform/conversions.py b/petl/transform/conversions.py index c699a795..5b952f10 100644 --- a/petl/transform/conversions.py +++ b/petl/transform/conversions.py @@ -354,9 +354,12 @@ def iterfieldconvert(source, converters, failonerror, errorvalue, where, # grab the fields in the source table it = iter(source) - hdr = next(it) - flds = list(map(text_type, hdr)) - yield tuple(hdr) # these are not modified + try: + hdr = next(it) + flds = list(map(text_type, hdr)) + yield tuple(hdr) # these are not modified + except StopIteration: + hdr = flds = [] # converters will fail selecting a field # build converter functions converter_functions = dict() diff --git a/petl/transform/dedup.py b/petl/transform/dedup.py index 5c64e5ca..3d27e525 100644 --- a/petl/transform/dedup.py +++ b/petl/transform/dedup.py @@ -89,7 +89,12 @@ def iterduplicates(source, key): # first need to sort the data it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + if key is None: + return # nothing to do on a table without headers + hdr = [] yield tuple(hdr) # convert field selection into field indices @@ -189,7 +194,10 @@ def iterunique(source, key): # first need to sort the data it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) # convert field selection into field indices @@ -326,7 +334,10 @@ def iterconflicts(source, key, missing, exclude, include): include = None it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) yield tuple(hdr) @@ -407,7 +418,10 @@ def __init__(self, table, key=None, count=None, presorted=False, def __iter__(self): it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return # convert field selection into field indices if self.key is None: diff --git a/petl/transform/fills.py b/petl/transform/fills.py index a3c8f8e4..8dcf28c0 100644 --- a/petl/transform/fills.py +++ b/petl/transform/fills.py @@ -104,7 +104,10 @@ def __iter__(self): def iterfilldown(table, fillfields, missing): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) if not fillfields: # fill down all fields fillfields = hdr @@ -177,7 +180,10 @@ def __iter__(self): def iterfillright(table, missing): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) for row in it: outrow = list(row) @@ -243,7 +249,10 @@ def __iter__(self): def iterfillleft(table, missing): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) for row in it: outrow = list(reversed(row)) diff --git a/petl/transform/headers.py b/petl/transform/headers.py index 641acffe..d5b81cb6 100644 --- a/petl/transform/headers.py +++ b/petl/transform/headers.py @@ -79,7 +79,10 @@ def __setitem__(self, key, value): def iterrename(source, spec, strict): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) if strict: for x in spec: @@ -138,7 +141,10 @@ def __iter__(self): def itersetheader(source, header): it = iter(source) - next(it) # discard source header + try: + next(it) # discard source header + except StopIteration: + pass # no previous header yield tuple(header) for row in it: yield tuple(row) @@ -185,7 +191,10 @@ def __iter__(self): def iterextendheader(source, fields): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] outhdr = list(hdr) outhdr.extend(fields) yield tuple(outhdr) @@ -308,7 +317,10 @@ def __init__(self, table, prefix): def __iter__(self): it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return outhdr = tuple((text_type(self.prefix) + text_type(f)) for f in hdr) yield outhdr for row in it: @@ -332,7 +344,10 @@ def __init__(self, table, suffix): def __iter__(self): it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return outhdr = tuple((text_type(f) + text_type(self.suffix)) for f in hdr) yield outhdr for row in it: @@ -361,7 +376,10 @@ def __init__(self, table, reverse, missing): def __iter__(self): it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return shdr = sorted(hdr) indices = asindices(hdr, shdr) transform = rowgetter(*indices) diff --git a/petl/transform/maps.py b/petl/transform/maps.py index ff64afe0..c268de02 100644 --- a/petl/transform/maps.py +++ b/petl/transform/maps.py @@ -88,7 +88,10 @@ def __iter__(self): def iterfieldmap(source, mappings, failonerror, errorvalue): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) outhdr = mappings.keys() yield tuple(outhdr) @@ -214,7 +217,10 @@ def __iter__(self): def iterrowmap(source, rowmapper, header, failonerror): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) yield tuple(header) it = (Record(row, flds) for row in it) @@ -308,7 +314,10 @@ def __iter__(self): def iterrowmapmany(source, rowgenerator, header, failonerror): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) yield tuple(header) it = (Record(row, flds) for row in it) diff --git a/petl/transform/regex.py b/petl/transform/regex.py index 112cbfd4..2efe00de 100644 --- a/petl/transform/regex.py +++ b/petl/transform/regex.py @@ -100,7 +100,10 @@ def itercapture(source, field, pattern, newfields, include_original, flags, it = iter(source) prog = re.compile(pattern, flags) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) if isinstance(field, int) and field < len(hdr): field_index = field @@ -197,7 +200,10 @@ def itersplit(source, field, pattern, newfields, include_original, maxsplit, it = iter(source) prog = re.compile(pattern, flags) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) if isinstance(field, int) and field < len(hdr): field_index = field @@ -312,7 +318,10 @@ def __iter__(self): def itersearch(table, pattern, field, flags, complement): prog = re.compile(pattern, flags) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) yield tuple(hdr) @@ -439,7 +448,10 @@ def itersplitdown(table, field, pattern, maxsplit, flags): prog = re.compile(pattern, flags) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) if isinstance(field, int) and field < len(hdr): diff --git a/petl/transform/reshape.py b/petl/transform/reshape.py index 055b1abc..c3f1e821 100644 --- a/petl/transform/reshape.py +++ b/petl/transform/reshape.py @@ -110,7 +110,10 @@ def itermelt(source, key, variables, variablefield, valuefield): raise ValueError('either key or variables must be specified') it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return # determine key and variable field indices key_indices = variables_indices = None @@ -298,7 +301,10 @@ def iterrecast(source, key, variablefield, valuefield, # TODO only make one pass through the data it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) # normalise some stuff @@ -538,7 +544,10 @@ def iterpivot(source, f1, f2, f3, aggfun, missing): # second pass - generate output it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) f1i = flds.index(f1) f2i = flds.index(f2) diff --git a/petl/transform/selects.py b/petl/transform/selects.py index 87456aef..77e45d29 100644 --- a/petl/transform/selects.py +++ b/petl/transform/selects.py @@ -112,8 +112,11 @@ def __iter__(self): def iterfieldselect(source, field, where, complement, missing): it = iter(source) - hdr = next(it) - yield tuple(hdr) + try: + hdr = next(it) + yield tuple(hdr) + except StopIteration: + hdr = [] # will raise FieldSelectionError below indices = asindices(hdr, field) getv = operator.itemgetter(*indices) for row in it: @@ -127,7 +130,10 @@ def iterfieldselect(source, field, where, complement, missing): def iterrowselect(source, where, missing, complement): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return # will yield nothing flds = list(map(text_type, hdr)) yield tuple(hdr) it = (Record(row, flds, missing=missing) for row in it) @@ -421,7 +427,10 @@ def __iter__(self): def iterselectusingcontext(table, query): it = iter(table) - hdr = tuple(next(it)) + try: + hdr = tuple(next(it)) + except StopIteration: + return # will yield nothing flds = list(map(text_type, hdr)) yield hdr it = (Record(row, flds) for row in it) diff --git a/petl/transform/sorts.py b/petl/transform/sorts.py index d591132e..801866b8 100755 --- a/petl/transform/sorts.py +++ b/petl/transform/sorts.py @@ -286,7 +286,12 @@ def _iternocache(self, source, key, reverse): self.clearcache() it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + if key is None: + return # nothing to do on a table without headers + hdr = [] yield tuple(hdr) if key is not None: @@ -480,7 +485,12 @@ def itermergesort(sources, key, header, missing, reverse): # borrow this from itercat - TODO remove code smells its = [iter(t) for t in sources] - src_hdrs = [next(it) for it in its] + src_hdrs = [] + for it in its: + try: + src_hdrs.append(next(it)) + except StopIteration: + src_hdrs.append([]) if header is None: # determine output fields by gathering all fields found in the sources @@ -562,7 +572,10 @@ def issorted(table, key=None, reverse=False, strict=False): op = operator.ge it = iter(table) - flds = [text_type(f) for f in next(it)] + try: + flds = [text_type(f) for f in next(it)] + except StopIteration: + flds = [] if key is None: prev = next(it) for curr in it: diff --git a/petl/transform/unpacks.py b/petl/transform/unpacks.py index 00d39b2e..5de74c11 100644 --- a/petl/transform/unpacks.py +++ b/petl/transform/unpacks.py @@ -64,7 +64,10 @@ def __iter__(self): def iterunpack(source, field, newfields, include_original, missing): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) if field in flds: field_index = flds.index(field) @@ -164,7 +167,10 @@ def iterunpackdict(table, field, keys, includeoriginal, samplesize, missing): # set up it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) fidx = flds.index(field) outhdr = list(flds) diff --git a/petl/transform/validation.py b/petl/transform/validation.py index 76cde4d3..26821f0f 100644 --- a/petl/transform/validation.py +++ b/petl/transform/validation.py @@ -112,7 +112,10 @@ def iterproblems(table, constraints, expected_header): yield outhdr it = iter(table) - actual_header = next(it) + try: + actual_header = next(it) + except StopIteration: + actual_header = [] if expected_header is None: flds = list(map(text_type, actual_header)) diff --git a/petl/util/base.py b/petl/util/base.py index d53b7f9e..b950d288 100644 --- a/petl/util/base.py +++ b/petl/util/base.py @@ -244,7 +244,10 @@ def itervalues(table, field, **kwargs): missing = kwargs.get('missing', None) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] indices = asindices(hdr, field) assert len(indices) > 0, 'no field selected' @@ -445,7 +448,10 @@ def __repr__(self): def iterdicts(table, *sliceargs, **kwargs): missing = kwargs.get('missing', None) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if sliceargs: it = islice(it, *sliceargs) for row in it: @@ -517,7 +523,10 @@ def iternamedtuples(table, *sliceargs, **kwargs): missing = kwargs.get('missing', None) name = kwargs.get('name', 'row') it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) nt = namedtuple(name, tuple(flds)) if sliceargs: @@ -639,7 +648,10 @@ def __repr__(self): def iterrecords(table, *sliceargs, **kwargs): missing = kwargs.get('missing', None) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) if sliceargs: it = islice(it, *sliceargs) @@ -695,7 +707,10 @@ def rowgroupby(table, key, value=None): """ it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) # wrap rows as records it = (Record(row, flds) for row in it) diff --git a/petl/util/lookups.py b/petl/util/lookups.py index 921b45e4..54b1f086 100644 --- a/petl/util/lookups.py +++ b/petl/util/lookups.py @@ -13,7 +13,10 @@ def _setup_lookup(table, key, value): # obtain iterator and header row it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] # prepare key getter keyindices = asindices(hdr, key) @@ -225,7 +228,10 @@ def dictlookup(table, key, dictionary=None): dictionary = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) keyindices = asindices(hdr, key) assert len(keyindices) > 0, 'no key selected' @@ -303,7 +309,10 @@ def dictlookupone(table, key, dictionary=None, strict=False): dictionary = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) keyindices = asindices(hdr, key) assert len(keyindices) > 0, 'no key selected' @@ -331,7 +340,10 @@ def recordlookup(table, key, dictionary=None): dictionary = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) keyindices = asindices(hdr, key) assert len(keyindices) > 0, 'no key selected' @@ -363,7 +375,10 @@ def recordlookupone(table, key, dictionary=None, strict=False): dictionary = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) keyindices = asindices(hdr, key) assert len(keyindices) > 0, 'no key selected' diff --git a/petl/util/materialise.py b/petl/util/materialise.py index 0728d77c..074dfb37 100644 --- a/petl/util/materialise.py +++ b/petl/util/materialise.py @@ -60,7 +60,10 @@ def columns(table, missing=None): cols = OrderedDict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) for f in flds: cols[f] = list() @@ -94,7 +97,10 @@ def facetcolumns(table, key, missing=None): fct = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) indices = asindices(hdr, key) assert len(indices) > 0, 'no key field selected' diff --git a/petl/util/vis.py b/petl/util/vis.py index e1d830a2..a75baf70 100644 --- a/petl/util/vis.py +++ b/petl/util/vis.py @@ -194,7 +194,10 @@ def _look_grid(table, vrepr, index_header, truncate, width): it = iter(table) # fields representation - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return '' flds = list(map(text_type, hdr)) if index_header: fldsrepr = ['%s|%s' % (i, r) for (i, r) in enumerate(flds)] @@ -294,7 +297,10 @@ def _look_simple(table, vrepr, index_header, truncate, width): it = iter(table) # fields representation - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return '' flds = list(map(text_type, hdr)) if index_header: fldsrepr = ['%s|%s' % (i, r) for (i, r) in enumerate(flds)] @@ -377,7 +383,10 @@ def _look_minimal(table, vrepr, index_header, truncate, width): it = iter(table) # fields representation - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return '' flds = list(map(text_type, hdr)) if index_header: fldsrepr = ['%s|%s' % (i, r) for (i, r) in enumerate(flds)] @@ -495,7 +504,10 @@ def __repr__(self): # construct output output = '' it = iter(table) - flds = next(it) + try: + flds = next(it) + except StopIteration: + return '' cols = defaultdict(list) for row in it: for i, f in enumerate(flds):