vrt-fix-characters

#! /usr/bin/env python3
# -*- mode: Python; -*-

from collections import OrderedDict
import html
from itertools import groupby, chain
import os, re, sys, traceback
import unicodedata

from vrtargslib import trans_args, trans_main
from vrtargslib import BadData, BadCode

from vrtnamelib import xname, isnames, namelist, nameindices
from vrtdatalib import asrecord, escape

def parsearguments():
    description = '''

    Replace invalid characters or character entities in attribute
    values and named fields with something that is valid in VRT and
    hopefully closer to what was intended.

    Input must already be UTF-8, much like VRT, have field names.

    Any annotations that may have been based on invalid input may
    become mysterious when the input fields change. It is also too
    late to introduce line breaks or tabs through entities when the
    data is posing as VRT; something is substituted for attempts at
    such.

    '''

    parser = trans_args(description = description)

    parser.add_argument('--attribute', '-a',
                        metavar = 'attr-name',
                        dest = 'attributes', type = xname,
                        action = 'append', default = [],
                        help = 'an attribute to edit (default: no attributes)')

    parser.add_argument('--field', '-f',
                        metavar = 'field-name',
                        dest = 'fields', type = xname,
                        action = 'append', default = [],
                        help = 'a field to edit (default: no fields)')

    # should allow only one of --entities and --controls and such
    # (TODO Microsoft code page interpretation of C1 as an option)
    # no actually leave them be with a guarantee that entities are
    # handled before actual character replacements but TODO then
    # entities could, er, what

    parser.add_argument('--entities',
                        action = 'store_true',
                        help = '''repair character entity references:
                        leave the standard few as such, replacing the
                        rest with an actual character (or an identifying
                        string when the character is not allowed)''')

    parser.add_argument('--amp-entities',
                        action = 'store_true',
                        help = '''repair character entity references
                        with the initial & encoded as &, such as
                        """; generally use with --entities''')

    parser.add_argument('--control',
                        action = 'store_true',
                        help = '''replace control characters with
                        choice of {Cc:sub:code-name}, _, or nothing;
                        sub is one of C0, del, C1; name is
                        a short name of the character''')

    parser.add_argument('--private',
                        action = 'store_true',
                        help = '''replace private-use characters with
                        choice of {Co:priv:code}, _, or nothing''')

    parser.add_argument('--nonchar',
                        action = 'store_true',
                        help = '''replace non-characters with choice of
                        {Cn:non:code}, _, or nothing''')

    parser.add_argument('--reserved',
                        action = 'store_true', dest = 'reservd',
                        help = '''replace reserved codes with choice of
                        {Cn:res:code}, _, or nothing''')

    parser.add_argument('--surrogate',
                        action = 'store_true', dest = 'surrogt',
                        help = '''replace surrogates with choice of
                        {Cs:surr:code}, _, or nothing.''')

    parser.add_argument('--replace',
                        choices = [
                            'identify',
                            'lowline',
                            'vanish'
                        ],
                        default = 'identify',
                        help = '''what to replace an unwanted character with:
                        informative identifier, _, or nothing''')

    parser.add_argument('--identify',
                        choices = [
                            'C1=cp1252',
                        ],
                        help = '''replace characters in a subset
                        of Unicode with an incorrect but
                        possibly intended other character (this happens),
                        for example, C1 control code MW
                        with {C1:cp1252:\N{BULLET}}
                        ''')

    parser.add_argument('--abuse',
                        choices = [
                            'C1=cp1252',
                        ],
                        help = '''replace  characters in a subset
                        of Unicode with other characters that happen
                        to have the same code in some other encoding,
                        for example, C1 control code MW
                        with \N{BULLET}
                        ''')

    parser.add_argument('--unsorted',
                        action = 'store_true',
                        help = '''retain input attribute order
                        (default: sort alphabetically)''')

    parser.add_argument('--allow-empty',
                        action = 'store_true',
                        help = '''keep empty field values empty
                        (default: replace with _)''')

    parser.add_argument('--apos',
                        action = 'store_true',
                        help = '''encode ' as ' in attributes
                        (default: keep ' literal)''')

    args = parser.parse_args()
    args.prog = parser.prog
    return args

def main(args, inf, ouf):

    # *all* these exceptions should probably go to trans_main, q.v.

    status = 1
    try:
        implement_main(args, inf, ouf)
        status = 0
    except BadData as exn:
        print(args.prog + ':', exn, file = sys.stderr)
    except BadCode as exn:
        print(args.prog + ':', exn, file = sys.stderr)
    except BrokenPipeError:
        print(args.prog + ': broken pipe in main', file = sys.stderr)
    except KeyboardInterrupt:
        print(args.prog + ': keyboard interrupt', file = sys.stderr)
    except Exception as exn:
        print(traceback.format_exc(), file = sys.stderr)

    return status

def implement_main(args, ins, ous):

    def issome(line): return not line.isspace()
    def ismeta(line): return line.startswith('<')

    # Is assigned to if args.apos
    global QUOTES

    anames = set(args.attributes)
    fnames = set(args.fields)
    emptyfield = '' if args.allow_empty else '_'
    attrsort = (lambda s: s) if args.unsorted else sorted
    if args.apos:
        QUOTES = QUOTES_APOS

    if not (anames or fnames):
        print('{}: warning: no attributes or fields specified'
              .format(args.prog),
              file = sys.stderr)

    fix = None
    for groupismeta, group in groupby(filter(issome, ins), ismeta):

        if groupismeta:
            for line in group:
                if isnames(line):
                    # find field indices to fix
                    fix = nameindices(namelist(line), *fnames)
                    print(line, end = '', file = ous)
                    continue
                
                if line.startswith(('<!', '</')):
                    print(line, end = '', file = ous)
                    continue

                # line is a start tag and may contain attributes
                print(fixmeta(args, line, anames, attrsort),
                      end = '', file = ous)
                continue

        # groupisdata aka token lines

        if fix is None:
            raise BadData('no names before tokens')

        for line in group:
            record = asrecord(line)

            for ix in fix:
                record[ix] = fixdata(args, record[ix]) or emptyfield

            print(*record, sep = '\t', file = ous)

def fixmeta(args, start, anames, attrsort):
    '''Return fixed start-tag line'''
    if not anames: return start
    tag, attrs = parsemeta(start)
    for aname in anames:
        if aname not in attrs: continue
        attrs[aname] = fixattr(args, attrs[aname])
    return composemeta(tag, attrs, end = '\n', attrsort = attrsort)

def fixattr(args, value):
    '''Return fixed attribute value'''
    if args.amp_entities: value = amp_entity.sub(r'&\1', value)
    if args.entities: value = re.sub(entitylike, unbreakity_quote, value)
    if args.identify or args.abuse: value = miscode(args, value)
    if args.control: value = ''.join(fixchar(args, value, catControl))
    if args.private: value = ''.join(fixchar(args, value, catPrivate))
    if args.nonchar: value = ''.join(fixchar(args, value, catNonchar))
    if args.reservd: value = ''.join(fixchar(args, value, catReservd))
    if args.surrogt: value = ''.join(fixchar(args, value, catSurrogt))
    # return '[{}]'.format(value) # testing
    return value

def fixdata(args, value):
    '''Return fixed positional-field value'''
    if args.amp_entities: value = amp_entity.sub(r'&\1', value)
    if args.entities: value = re.sub(entitylike, unbreakity, value)
    if args.identify or args.abuse: value = miscode(args, value)
    if args.control: value = ''.join(fixchar(args, value, catControl))
    if args.private: value = ''.join(fixchar(args, value, catPrivate))
    if args.nonchar: value = ''.join(fixchar(args, value, catNonchar))
    if args.reservd: value = ''.join(fixchar(args, value, catReservd))
    if args.surrogt: value = ''.join(fixchar(args, value, catSurrogt))
    # return '[{}]'.format(value) # testing
    return value

def unbreakity(match, quote = False):
    '''Consider any &\w+; an attempted named entity (ASCII only), also
    consider any numerical-looking entity up to and including the
    semicolon, and actually consider any string of such up to and
    including a semicolon - type "&lt&lt&lt;" that complicators of
    legend in their legendary wisdom used to consider a good idea,
    syntactically - and any lone & < > " ' as such - and return a
    canonical, minimal interpretation.

    '''
    ents = match.group()
    if len(ents) == 1:
        # ents is one of & < > " '
        result = (
            ENTITIES.get(ents) or
            (quote and QUOTES.get(ents)) or
            (not quote and ents in ('"', "'") and ents) or
            # this cannot happen
            '[??{}]'.format(repr(ents))
        )
    else:
        # ents looks like an entity sequence: undo and redo
        #print('*** un-breaking {} ***'.format(ents))
        #for mo in re.finditer('&[^&]+', ents):
        #    part = html.unescape(mo.group())
        #    print(mo.group(), '=>', repr(part))
        #    ENTITIES.get(part) and print('*** ENTITIES ***')
        #    QUOTES.get(part) and print('*** QUOTES ***')
        #    TANKEROES.get(part) and print('*** TANKEROES ***')
        result = ''.join([
            (
                misabuse(ent) or
                ENTITIES.get(part) or
                (quote and QUOTES.get(part)) or
                TANKEROES.get(part) or
                # to handle mispeling here
                # though really why bother
                escape(part)
            )
            for mo in re.finditer('&[^&]+', ents)
            for ent, part in [ (mo.group(), html.unescape(mo.group())) ]
        ])
    # value = '[{} => {}]'.format(ents, result)
    return result

unbreakity_quote = lambda match : unbreakity(match, quote = True)

entitylike = re.compile(R'''

    # eventual semicolon is mandatory on an entity sequence
    # or the entity sequence will not be recognized as such

    (?: & [a-z] [a-z0-9] *
      | & \# [0-9] +
      | & \# x [0-9a-f] + ) + ;
    | [&<>"']

''', re.ASCII | re.IGNORECASE | re.VERBOSE)

# &amp; followed by something like an entity name (or numeric
# reference) and ; (grouped for re.sub)
amp_entity = re.compile(R'''

    &amp;
    (
      (?: [a-z] [a-z0-9] *
        | \# [0-9] +
        | \# x [0-9a-f] + )
      ;
    )

''', re.ASCII | re.IGNORECASE | re.VERBOSE)

ENTITIES = {
    # Three characters are entified in both meta and data
    '&amp;' : '&amp;',
    '&lt;' : '&lt;',
    '&gt;' : '&gt;',
    '&' : '&amp;',
    '<' : '&lt;',
    '>' : '&gt;'
}

QUOTES = {
    # One more character is entified in meta; ' need not, but
    # recognize it to support --apos
    '&quot;' : '&quot;',
    '&apos;' : "'",
    '"' : '&quot;',
    "'" : "'"
}

QUOTES_APOS = {
    # With --apos (previously, always), two more characters are
    # entified in meta
    '&quot;' : '&quot;',
    '&apos;' : "&apos;",
    '"' : '&quot;',
    "'" : "&apos;"
}

TANKEROES = {
    # The characters that absolutely must not be allowed
    # because they would formally break lines or fields,
    # and something is trying to smuggle them in through
    # an entity reference. Replace with identification.
    # These should simply not happen.

    # U+0009 CHARACTER TABULATION
    # aka HT, HORIZONTAL TAB
    '\t' : '{ref=Cc:C0:09-HT}',

    # U+000a LINE FEED
    # aka LF? or is that something else?
    '\n' : '{ref=Cc:C0:0a-LF}',

    # U+000b LINE TABULATION
    # aka VT, VERTICAL TAB
    # but &#xb; vanishes in html.unescape (python 3.5.2)
    # so caught with MISABUSE instead
    '\v' : '{ref=Cc:C0:0b-VT}',

    # U+000c FORM FEED
    # aka FF
    '\x0c' : '{ref=Cc:C0:0c-FF}',

    # U+000d CARRIAGE RETURN
    # aka CR
    '\r' : '{ref=Cc:C0:0d-CR}',

    # TODO &#xe; &#xf; vanish, ...

    # U+001c INFORMATION SEPARATOR FOUR
    # aka FS, FILE SEPARATOR
    # vanishes so caught with MISABUSE instead
    '\x1c' : '{ref=Cc:C0:1c-FS}',

    # U+001d INFORMATION SEPARATOR THREE
    # aka GS, GROUP SEPARATOR
    # vanishes so caught with MISABUSE instead
    '\x1d' : '{ref=Cc:C0:1d-GS}',

    # U+001e INFORMATION SEPARATOR TWO
    # aka RS, RECORD SEPARATOR
    # vanishes so caught with MISABUSE instead
    '\x1e' : '{ref=Cc:C0:1e-RS}',

    # 0x1f US vanishes see MISABUSE

    # U+0085 NEXT LINE
    # aka NEL
    # but html.unescape appears to abuse &#x85; for
    # a horizontal ellipsis (in python 3.5.2), so
    # caught with MISABUSE instead
    '\x85' : '{ref=Cc:C1:85-NEL}',

    # U+2028 LINE SEPARATOR
    '\u2028' : '{ref=Zl:2028-LS}',

    # U+2029 PARAGRAPH SEPARATOR
    '\u2029' : '{ref=Zp:2029-PS}',
}

MISABUSE = {
    # Some of the character entity reference codes that html.unescape
    # appears to abuse somehow (including vanishing them without a
    # trace).

    # U+0000
    # aka NUL
    # &#x0; becomes ... something?
    0x00 : '{ref=Cc:C0:00-NUL}',

    # U+0008
    # aka BS
    # but &#x8; vanishes
    0x08 : '{ref=Cc:C0:08-BS}',

    # U+000b LINE TABULATION
    # aka VT, VERTICAL TAB
    # but &#xb; vanishes in html.unescape (python 3.5.2)
    0x0b : '{ref=Cc:C0:0b-VT}',

    # U+001b (what should this be?)
    # aka ESC
    # but &#x1b; vanishes
    0x1b : '{ref=Cc:C0:1b-ESC}',

    # U+001c INFORMATION SEPARATOR FOUR
    # aka FS, FILE SEPARATOR
    # vanishes
    0x1c : '{ref=Cc:C0:1c-FS}',

    # U+001d INFORMATION SEPARATOR THREE
    # aka GS, GROUP SEPARATOR
    # vanishes
    0x1d : '{ref=Cc:C0:1d-GS}',

    # U+001e INFORMATION SEPARATOR TWO
    # aka RS, RECORD SEPARATOR
    # vanishes
    0x1e : '{ref=Cc:C0:1e-RS}',

    # U+001f (what should this even be?)
    # aka US, UNIT SEPARATOR
    # vanishes
    0x1f : '{ref=Cc:C0:1f-US}',

    # U+0085 NEXT LINE
    # aka NEL
    # but html.unescape appears to abuse &#x85; for
    # a horizontal ellipsis (in python 3.5.2)
    0x85 : '{ref=Cc:C1:85-NEL}',
}

def misabuse(entity):
    '''Return an abuse identifier for a character entity reference that
    html.unescape (python 3.5.2) appears to abuse somehow, and None
    for others. These are presumably all numeric, either in decimal or
    in hexadecimal, but there may be any number of leading zeroes and
    there may or may not be a trailing semicolon.

    '''

    entity = entity.rstrip(';')
    if entity.startswith('&#x'):
        code, base = entity[3:], 16
    elif entity.startswith('&#'):
        code, base = entity[2:], 10
    else:
        return None

    return MISABUSE.get(int(code, base))

TODO = str.maketrans({

    # TODO replace this with catControl
    # but then use this for Windows
    # interpretation what are really
    # control codes in Unicode but
    # some data contains them as such

    # C0 (ASCII) controls
    '\x00' : '(C0:NUL)',
    '\x01' : '(C0:SOH)',
    '\x02' : '(C0:STX)',
    '\x03' : '(C0:ETX)',
    '\x04' : '(C0:EOT)',
    '\x05' : '(C0:ENQ)',
    '\x06' : '(C0:ACK)',
    '\x07' : '(C0:BEL)',
    '\x08' : '(C0:BS) ',
    '\x09' : '(C0:HT) ', # cannot happen
    '\x0A' : '(C0:LF) ', # cannot happen
    '\x0B' : '(C0:VT) ', # cannot happen
    '\x0C' : '(C0:FF) ', # cannot happen
    '\x0D' : '(C0:CR) ', # cannot happen
    '\x0E' : '(C0:SO) ',
    '\x0F' : '(C0:SI) ',
    '\x10' : '(C0:DLE)',
    '\x11' : '(C0:DC1)',
    '\x12' : '(C0:DC2)',
    '\x13' : '(C0:DC3)',
    '\x14' : '(C0:DC4)',
    '\x15' : '(C0:NAK)',
    '\x16' : '(C0:SYN)',
    '\x17' : '(C0:ETB)',
    '\x18' : '(C0:CAN)',
    '\x19' : '(C0:EM) ',
    '\x1A' : '(C0:SUB)',
    '\x1B' : '(C0:ESC)',
    '\x1C' : '(C0:FS) ', # cannot happen
    '\x1D' : '(C0:GS) ', # cannot happen
    '\x1E' : '(C0:RS) ', # cannot happen
    '\x1F' : '(C0:US) ',

    # DEL
    '\x7F' : '(CC:DEL)',

    # C1 controls
    '\x80' : '(C1:PAD)',
    '\x81' : '(C1:HOP)',
    '\x82' : '(C1:BPH)',
    '\x83' : '(C1:NBH)',
    '\x84' : '(C1:IND)',
    '\x85' : '(C1:NEL)', # cannot happen ...
    '\x86' : '(C1:SSA)',
    '\x87' : '(C1:ESA)',
    '\x88' : '(C1:HTS)',
    '\x89' : '(C1:HTJ)',
    '\x8A' : '(C1:VTS)',
    '\x8B' : '(C1:PLD)',
    '\x8C' : '(C1:PLU)',
    '\x8D' : '(C1:RI)',
    '\x8E' : '(C1:SS2)',
    '\x8F' : '(C1:SS3)',
    '\x90' : '(C1:DCS)',
    '\x91' : '(C1:PU1)',
    '\x92' : '(C1:PU2)',
    '\x93' : '(C1:STS)',
    '\x94' : '(C1:CCH)',
    '\x95' : '(C1:MW)',
    '\x96' : '(C1:SPA)',
    '\x97' : '(C1:EPA)',
    '\x98' : '(C1:SOS)',
    '\x99' : '(C1:SGCI)',
    '\x9A' : '(C1:SCI)',
    '\x9B' : '(C1:CSI)',
    '\x9C' : '(C1:ST)',
    '\x9D' : '(C1:OSC)',
    '\x9E' : '(C1:PM)',
    '\x9F' : '(C1:APC)',
})

def miscode(args, value):
    '''Replace Unicode characters in value to identify or abuse a
    different interpretation. Identify goes before abuse. Either goes
    before other redactions and before control codes in particular.

    '''
    if args.identify == 'C1=cp1252':
        value = value.translate(IDENTIFY_C1_CP1252)
    elif args.abuse == 'C1=cp1252':
        value = value.translate(ABUSE_C1_CP1252)
    else:
        raise BadCode('this cannot happen')

    return value

IDENTIFY_C1_CP1252 = str.maketrans({
    # C1 controls

    # U+0080 C1 PAD => U+20ac €
    '\x80' : '{C1:cp1252:\N{EURO SIGN}}',

    # U+0081 C1 HOP, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+0082 C1 BPH => U+201a ‚
    '\x82' : '{C1:cp1252:\N{SINGLE LOW-9 QUOTATION MARK}}',

    # U+0083 C1 NBH => U+0192 ƒ (apparently for florin currency)
    '\x83' : '{C1:cp1252:\N{LATIN SMALL LETTER F WITH HOOK}}',

    # U+0084 C1 IND => U+201e „
    '\x84' : '{C1:cp1252:\N{DOUBLE LOW-9 QUOTATION MARK}}',

    # U+0085 C1 NEL => U+2026 … but this cannot happen in VRT
    '\x85' : '{C1:cp1252:\N{HORIZONTAL ELLIPSIS}}',

    # U+0086 C1 SSA => U+2020 †
    '\x86' : '{C1:cp1252:\N{DAGGER}}',

    # U+0087 C1 ESA => U+2021 ‡
    '\x87' : '{C1:cp1252:\{DOUBLE DAGGER}}',

    # U+0088 C1 HTS => U+02c6 ˆ
    '\x88' : '{C1:cp1252:\N{MODIFIER LETTER CIRCUMFLEX ACCENT}}',

    # U+0089 C1 HTJ => U+2030 ‰
    '\x89' : '{C1:cp1252:\N{PER MILLE SIGN}}',

    # U+008a C1 VTS => U+0160 Š
    '\x8A' : '{C1:cp1252:\N{LATIN CAPITAL LETTER S WITH CARON}}',

    # U+008b C1 PLD => U+2039 ‹
    '\x8B' : '{C1:cp1252:\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}}',

    # U+008c C1 PLU => U+0152 Œ
    '\x8C' : '{C1:cp1252:\N{LATIN CAPITAL LIGATURE OE}}',

    # U+008d C1 RI, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+008e C1 SS2 => U+017d Ž
    '\x8E' : '{C1:cp1252:\N{LATIN CAPITAL LETTER Z WITH CARON}}',

    # U+008f C1 SS3, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+0090 C1 DCS, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+0091 C1 PU1 => U+2018 ‘
    '\x91' : '{C1:cp1252:\N{LEFT SINGLE QUOTATION MARK}}',

    # U+0092 C1 PU2 => U+2019 ’
    '\x92' : '{C1:cp1252:\N{RIGHT SINGLE QUOTATION MARK}}',

    # U+0093 C1 STS => U+201c “
    '\x93' : '{C1:cp1252:\N{LEFT DOUBLE QUOTATION MARK}}',

    # U+0094 C1 CCH => U+201d ”
    '\x94' : '{C1:cp1252:\N{RIGHT DOUBLE QUOTATION MARK}}',

    # U+0095 C1 MW => U+2022 •
    '\x95' : '{C1:cp1252:\N{BULLET}}',

    # U+0096 C1 SPA => U+2013 –
    '\x96' : '{C1:cp1252:\N{EN DASH}}',

    # U+0097 C1 EPA => U+2014 —
    '\x97' : '{C1:cp1252:\N{EM DASH}})',

    # U+0098 C1 SOS => U+02dc ˜
    '\x98' : '{C1:cp1252\N{SMALL TILDE}}',

    # U+0099 C1 SGCI => U+2122 ™
    '\x99' : '{C1:cp1252:\N{TRADE MARK SIGN}}',

    # U+009a C1 SCI => U+0161 š
    '\x9A' : '{C1:cp1252:\N{LATIN SMALL LETTER S WITH CARON}}',

    # U+009b C1 CSI => U+203a ›
    '\x9B' : '{C1:cp1252:\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}}',

    # U+009c C1 ST => U+0153 œ
    '\x9C' : '{C1:cp1252:\N{LATIN SMALL LIGATURE OE}}',

    # U+009d C1 OSC, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+009e C1 PM => U+017e ž
    '\x9E' : '{C1:cp1252:\N{LATIN SMALL LETTER Z WITH CARON}}',

    # U+009f C1 APC => U+0178 Ÿ
    '\x9F' : '{C1:cp1252:\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}}',
})

ABUSE_C1_CP1252 = str.maketrans({
    # C1 controls

    # U+0080 C1 PAD => U+20ac €
    '\x80' : '\N{EURO SIGN}',

    # U+0081 C1 HOP, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+0082 C1 BPH => U+201a ‚
    '\x82' : '\N{SINGLE LOW-9 QUOTATION MARK}',

    # U+0083 C1 NBH => U+0192 ƒ (apparently for florin currency)
    '\x83' : '\N{LATIN SMALL LETTER F WITH HOOK}',

    # U+0084 C1 IND => U+201e „
    '\x84' : '\N{DOUBLE LOW-9 QUOTATION MARK}',

    # U+0085 C1 NEL => U+2026 … but this cannot happen in VRT
    '\x85' : '\N{HORIZONTAL ELLIPSIS}',

    # U+0086 C1 SSA => U+2020 †
    '\x86' : '\N{DAGGER}',

    # U+0087 C1 ESA => U+2021 ‡
    '\x87' : '\{DOUBLE DAGGER}',

    # U+0088 C1 HTS => U+02c6 ˆ
    '\x88' : '\N{MODIFIER LETTER CIRCUMFLEX ACCENT}',

    # U+0089 C1 HTJ => U+2030 ‰
    '\x89' : '\N{PER MILLE SIGN}',

    # U+008a C1 VTS => U+0160 Š
    '\x8A' : '\N{LATIN CAPITAL LETTER S WITH CARON}',

    # U+008b C1 PLD => U+2039 ‹
    '\x8B' : '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}',

    # U+008c C1 PLU => U+0152 Œ
    '\x8C' : '\N{LATIN CAPITAL LIGATURE OE}',

    # U+008d C1 RI, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+008e C1 SS2 => U+017d Ž
    '\x8E' : '\N{LATIN CAPITAL LETTER Z WITH CARON}',

    # U+008f C1 SS3, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+0090 C1 DCS, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+0091 C1 PU1 => U+2018 ‘
    '\x91' : '\N{LEFT SINGLE QUOTATION MARK}',

    # U+0092 C1 PU2 => U+2019 ’
    '\x92' : '\N{RIGHT SINGLE QUOTATION MARK}',

    # U+0093 C1 STS => U+201c “
    '\x93' : '\N{LEFT DOUBLE QUOTATION MARK}',

    # U+0094 C1 CCH => U+201d ”
    '\x94' : '\N{RIGHT DOUBLE QUOTATION MARK}',

    # U+0095 C1 MW => U+2022 •
    '\x95' : '\N{BULLET}',

    # U+0096 C1 SPA => U+2013 –
    '\x96' : '\N{EN DASH}',

    # U+0097 C1 EPA => U+2014 —
    '\x97' : '\N{EM DASH}',

    # U+0098 C1 SOS => U+02dc ˜
    '\x98' : '{C1:cp1252\N{SMALL TILDE}',

    # U+0099 C1 SGCI => U+2122 ™
    '\x99' : '\N{TRADE MARK SIGN}',

    # U+009a C1 SCI => U+0161 š
    '\x9A' : '\N{LATIN SMALL LETTER S WITH CARON}',

    # U+009b C1 CSI => U+203a ›
    '\x9B' : '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}',

    # U+009c C1 ST => U+0153 œ
    '\x9C' : '\N{LATIN SMALL LIGATURE OE}',

    # U+009d C1 OSC, not defined in cp1252
    # THIS LINE INTENTIONALLY LEFT BLANK

    # U+009e C1 PM => U+017e ž
    '\x9E' : '\N{LATIN SMALL LETTER Z WITH CARON}',

    # U+009f C1 APC => U+0178 Ÿ
    '\x9F' : '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}',
})

def fixchar(args, value, cat):
    '''Yield characters in value, with replacements for private-use
    characters (Unicode category Co) or certain other categories, with
    something (depending on args.replace). Caller joins and guards
    against the whole value vanishing altogether.

    '''

    if args.replace == 'identify':
        yield from (cat(c) or c for c in value)
    elif args.replace == 'lowline':
        yield from ((c if (unicodedata.category(c) == 'Cn' and
                           ord(c) in NEWCHARS)
                     else '_')
                    if cat(c) else c for c in value)
    elif args.replace == 'vanish':
        yield from ((c if (unicodedata.category(c) == 'Cn' and
                           ord(c) in NEWCHARS)
                     else '')
                    if cat(c) else c for c in value)
    else:
        raise BadCode('this cannot happen')

def catControl(c):
    '''Return an identifying non-empty string for control character,
    None for other character

    '''
    return CONTROLS.get(c)

CONTROLS = {

    # C0 (ASCII) controls
    '\x00' : '{Cc:C0:00-NUL}',
    '\x01' : '{Cc:C0:01-SOH}',
    '\x02' : '{Cc:C0:02-STX}',
    '\x03' : '{Cc:C0:03-ETX}',
    '\x04' : '{Cc:C0:04-EOT}',
    '\x05' : '{Cc:C0:05-ENQ}',
    '\x06' : '{Cc:C0:06-ACK}',
    '\x07' : '{Cc:C0:07-BEL}',
    '\x08' : '{Cc:C0:08-BS} ',
    '\x09' : '{Cc:C0:09-HT} ', # cannot happen
    '\x0A' : '{Cc:C0:0A-LF} ', # cannot happen
    '\x0B' : '{Cc:C0:0B-VT} ', # cannot happen
    '\x0C' : '{Cc:C0:0C-FF} ', # cannot happen
    '\x0D' : '{Cc:C0:0D-CR} ', # cannot happen
    '\x0E' : '{Cc:C0:0E-SO} ',
    '\x0F' : '{Cc:C0:0F-SI} ',
    '\x10' : '{Cc:C0:10-DLE}',
    '\x11' : '{Cc:C0:11-DC1}',
    '\x12' : '{Cc:C0:12-DC2}',
    '\x13' : '{Cc:C0:13-DC3}',
    '\x14' : '{Cc:C0:14-DC4}',
    '\x15' : '{Cc:C0:15-NAK}',
    '\x16' : '{Cc:C0:16-SYN}',
    '\x17' : '{Cc:C0:17-ETB}',
    '\x18' : '{Cc:C0:18-CAN}',
    '\x19' : '{Cc:C0:19-EM} ',
    '\x1A' : '{Cc:C0:1A-SUB}',
    '\x1B' : '{Cc:C0:1B-ESC}',
    '\x1C' : '{Cc:C0:1C-FS} ', # cannot happen
    '\x1D' : '{Cc:C0:1D-GS} ', # cannot happen
    '\x1E' : '{Cc:C0:1E-RS} ', # cannot happen
    '\x1F' : '{Cc:C0:1F-US} ',

    # so DEL is not C0
    '\x7F' : '{Cc:del:7F-DEL}',

    # C1 controls
    '\x80' : '{Cc:C1:80-PAD}',
    '\x81' : '{Cc:C1:81-HOP}',
    '\x82' : '{Cc:C1:82-BPH}',
    '\x83' : '{Cc:C1:83-NBH}',
    '\x84' : '{Cc:C1:84-IND}',
    '\x85' : '{Cc:C1:85-NEL}', # cannot happen ...
    '\x86' : '{Cc:C1:86-SSA}',
    '\x87' : '{Cc:C1:87-ESA}',
    '\x88' : '{Cc:C1:88-HTS}',
    '\x89' : '{Cc:C1:89-HTJ}',
    '\x8A' : '{Cc:C1:8A-VTS}',
    '\x8B' : '{Cc:C1:8B-PLD}',
    '\x8C' : '{Cc:C1:8C-PLU}',
    '\x8D' : '{Cc:C1:8D-RI}',
    '\x8E' : '{Cc:C1:8E-SS2}',
    '\x8F' : '{Cc:C1:8F-SS3}',
    '\x90' : '{Cc:C1:90-DCS}',
    '\x91' : '{Cc:C1:91-PU1}',
    '\x92' : '{Cc:C1:92-PU2}',
    '\x93' : '{Cc:C1:93-STS}',
    '\x94' : '{Cc:C1:94-CCH}',
    '\x95' : '{Cc:C1:95-MW}',
    '\x96' : '{Cc:C1:96-SPA}',
    '\x97' : '{Cc:C1:97-EPA}',
    '\x98' : '{Cc:C1:98-SOS}',
    '\x99' : '{Cc:C1:99-SGCI}',
    '\x9A' : '{Cc:C1:9A-SCI}',
    '\x9B' : '{Cc:C1:9B-CSI}',
    '\x9C' : '{Cc:C1:9C-ST}',
    '\x9D' : '{Cc:C1:9D-OSC}',
    '\x9E' : '{Cc:C1:9E-PM}',
    '\x9F' : '{Cc:C1:9F-APC}',

    # LINE SEPARATOR is the only character in its category
    # so technically not control so may should not be *here*
    # (in a 'general whitespace' block)
    '\u2028' : '{Zl:2028-LS}',

    # PARAGRAPH SEPARATOR is the only character in its category
    # so technically not control so may should not be *here*
    # (in a 'general whitespace' block)
    '\u2029' : '{Zp:2029-PS}',
}

def catPrivate(c):
    '''Return an identifying non-empty string for private-use character,
    None for other character

    '''
    return (
        '{{Co:priv:{:04x}}}'.format(ord(c))
        if unicodedata.category(c) == 'Co' else
        None
    )

def catNonchar(c):
    '''Return an identifying non-empty string for non-character,
    None for other character

    '''
    return (
        '{{Cn:non:{:04x}}}'.format(ord(c))
        if ord(c) in NONCHARS else
        None
    )

NONCHARS = set(chain(range(0xfdd0, 0xfdef + 1),
                     (0x00fffe, 0x00ffff,
                      0x01fffe, 0x01ffff,
                      0x02fffe, 0x02ffff,
                      0x03fffe, 0x03ffff,
                      0x04fffe, 0x04ffff,
                      0x05fffe, 0x05ffff,
                      0x06fffe, 0x06ffff,
                      0x07fffe, 0x07ffff,
                      0x08fffe, 0x08ffff,
                      0x09fffe, 0x09ffff,
                      0x0afffe, 0x0affff,
                      0x0bfffe, 0x0bffff,
                      0x0cfffe, 0x0cffff,
                      0x0dfffe, 0x0dffff,
                      0x0efffe, 0x0effff,
                      0x0ffffe, 0x0fffff,
                      0x10fffe, 0x10ffff)))

NEWCHARS = {
    # codes that were reserved but have been assigned a meaning - an
    # old version of unicodedata in an old version of Python would
    # misidentify
    0x052a, # U+052a Lu CYRILLIC CAPITAL LETTER DZZHE (Unicode 7.0)
    0x20bd, # U+20bd Cs RUBLE SIGN (Unicode 7.0, June 2014)
    0x20bf, # TESTING - reserved even in miilu
}

def catReservd(c):
    '''Return an identifying non-empty string for reserved character, None
    for other character, and guard against misidentifying known new
    characters like the ruble sign in case unicodedata is too old

    '''
    return (
        '{{Cn:new:{:04x}}}'.format(ord(c))
        if (unicodedata.category(c) == 'Cn' and
            ord(c) in NEWCHARS)
        else
        '{{Cn:res:{:04x}}}'.format(ord(c))
        if (unicodedata.category(c) == 'Cn' and
            ord(c) not in NONCHARS)
        else
        None
    )

def catSurrogt(c):
    '''Return an identifying non-empty string for surrogate code, None for
    other character.

    '''
    return (
        '{{Cs:surr:{:04x}}}'.format(ord(c))
        if unicodedata.category(c) == 'Cs'
        else
        None
    )

def parsemeta(start):
    '''Parse a start tag into the element name and a dictionary of
    attributes.

    '''
    # capture element name and make sure that the rest of the line
    # consists of something that could be key="value" pairs - is it an
    # overkill to enforce that element name consists of letters only?
    name = re.fullmatch(R'<(\w+)(?: [\w\d._\-]+/?="[^"]*")*>\r?\n?',
                        start)

    if name is None:
        raise BadData('no match: {!r}'.format(start))

    # now be safe to capture (key, value) pairs with a lax pattern

    name = name.group(1)
    rest = re.finditer(R'(\S+?)="([^"]*)"', start)
    return name, OrderedDict(mo.groups() for mo in rest)

def composemeta(name, attrs, end = '', attrsort = sorted):
    '''Given element name and a dictionary of attributes, properly
    entified and all, construct the start tag line, sorted, spaced,
    double-quoted.

    '''
    return ( '<{name}{attrs}>{end}'
             .format(name = name,
                     attrs = ''.join(( ' {name}="{value}"'
                                       .format(name = name,
                                               value = value) )
                                     for name, value
                                     in attrsort(attrs.items())),
                     end = end) )

if __name__ == '__main__':
    trans_main(parsearguments(), main)