Data Structures
class	Codepoint

Functions
	print_record (codepoint, letter)

	is_mark_to_remove (codepoint)

	is_plain_letter (codepoint)

	is_mark (codepoint)

	is_letter_with_marks (codepoint, table)

	is_letter (codepoint, table)

	get_plain_letter (codepoint, table)

	is_ligature (codepoint, table)

	get_plain_letters (codepoint, table)

	parse_cldr_latin_ascii_transliterator (latinAsciiFilePath)

	special_cases ()

	main (args)

Variables
	stdout

tuple	PLAIN_LETTER_RANGES

tuple	COMBINING_MARK_RANGES

	parser = argparse.ArgumentParser(description='This script builds unaccent.rules on standard output when given the contents of UnicodeData.txt and Latin-ASCII.xml given as arguments.')

	help

	type

	str

	required

	True

	dest

	action

	args = parser.parse_args()

Function Documentation

◆ get_plain_letter()

generate_unaccent_rules.get_plain_letter	(	codepoint,
		table
	)

Return the base codepoint without marks. If this codepoint has more
than one combining character, do a recursive lookup on the table to
find out its plain base letter.

Definition at line 131 of file generate_unaccent_rules.py.

def get_plain_letter(codepoint, table):
    """Return the base codepoint without marks. If this codepoint has more
    than one combining character, do a recursive lookup on the table to
    find out its plain base letter."""
    if is_letter_with_marks(codepoint, table):
        if len(table[codepoint.combining_ids[0]].combining_ids) > 1:
            return get_plain_letter(table[codepoint.combining_ids[0]], table)
        elif is_plain_letter(table[codepoint.combining_ids[0]]):
            return table[codepoint.combining_ids[0]]
 
        # Should not come here
        assert False, 'Codepoint U+%0.2X' % codepoint.id
    elif is_plain_letter(codepoint):
        return codepoint
 
    # Should not come here
    assert False, 'Codepoint U+%0.2X' % codepoint.id
 
 

References fb(), get_plain_letter(), is_letter_with_marks(), is_plain_letter(), and len.

Referenced by get_plain_letter(), get_plain_letters(), and main().

◆ get_plain_letters()

generate_unaccent_rules.get_plain_letters	(	codepoint,
		table
	)

Return a list of plain letters from a ligature.

Definition at line 154 of file generate_unaccent_rules.py.

def get_plain_letters(codepoint, table):
    """Return a list of plain letters from a ligature."""
    assert(is_ligature(codepoint, table))
    return [get_plain_letter(table[id], table) for id in codepoint.combining_ids]
 
 

References assert, fb(), get_plain_letter(), and is_ligature().

Referenced by main().

◆ is_letter()

generate_unaccent_rules.is_letter	(	codepoint,
		table
	)

Return true for letter with or without diacritical marks.

Definition at line 126 of file generate_unaccent_rules.py.

def is_letter(codepoint, table):
    """Return true for letter with or without diacritical marks."""
    return is_plain_letter(codepoint) or is_letter_with_marks(codepoint, table)
 
 

References is_letter_with_marks(), and is_plain_letter().

Referenced by is_ligature().

◆ is_letter_with_marks()

generate_unaccent_rules.is_letter_with_marks	(	codepoint,
		table
	)

Returns true for letters combined with one or more marks.

Definition at line 103 of file generate_unaccent_rules.py.

def is_letter_with_marks(codepoint, table):
    """Returns true for letters combined with one or more marks."""
    # See https://www.unicode.org/reports/tr44/tr44-14.html#General_Category_Values
 
    # Some codepoints redirect directly to another, instead of doing any
    # "combining"...  but sometimes they redirect to a codepoint that doesn't
    # exist, so ignore those.
    if len(codepoint.combining_ids) == 1 and codepoint.combining_ids[0] in table:
        return is_letter_with_marks(table[codepoint.combining_ids[0]], table)
 
    # A letter without diacritical marks has none of them.
    if any(is_mark(table[i]) for i in codepoint.combining_ids[1:]) is False:
        return False
 
    # Check if the base letter of this letter has marks.
    codepoint_base = codepoint.combining_ids[0]
    if is_plain_letter(table[codepoint_base]) is False and \
       is_letter_with_marks(table[codepoint_base], table) is False:
        return False
 
    return True
 
 

References fb(), is_letter_with_marks(), is_mark(), is_plain_letter(), and len.

Referenced by get_plain_letter(), is_letter(), is_letter_with_marks(), and main().

◆ is_ligature()

generate_unaccent_rules.is_ligature	(	codepoint,
		table
	)

Return true for letters combined with letters.

Definition at line 150 of file generate_unaccent_rules.py.

def is_ligature(codepoint, table):
    """Return true for letters combined with letters."""
    return all(i in table and is_letter(table[i], table) for i in codepoint.combining_ids)
 

References fb(), and is_letter().

Referenced by get_plain_letters(), and main().

◆ is_mark()

generate_unaccent_rules.is_mark ( codepoint )

Returns true for diacritical marks (combining codepoints).

Definition at line 98 of file generate_unaccent_rules.py.

def is_mark(codepoint):
    """Returns true for diacritical marks (combining codepoints)."""
    return codepoint.general_category in ("Mn", "Me", "Mc")
 
 

References fb().

Referenced by is_letter_with_marks(), and is_mark_to_remove().

◆ is_mark_to_remove()

generate_unaccent_rules.is_mark_to_remove ( codepoint )

Return true if this is a combining mark to remove.

Definition at line 79 of file generate_unaccent_rules.py.

def is_mark_to_remove(codepoint):
    """Return true if this is a combining mark to remove."""
    if not is_mark(codepoint):
        return False
 
    for begin, end in COMBINING_MARK_RANGES:
        if codepoint.id >= begin and codepoint.id <= end:
            return True
    return False
 
 

References fb(), and is_mark().

Referenced by main().

◆ is_plain_letter()

generate_unaccent_rules.is_plain_letter ( codepoint )

Return true if codepoint represents a "plain letter".

Definition at line 90 of file generate_unaccent_rules.py.

def is_plain_letter(codepoint):
    """Return true if codepoint represents a "plain letter"."""
    for begin, end in PLAIN_LETTER_RANGES:
        if codepoint.id >= begin and codepoint.id <= end:
            return True
    return False
 
 

References fb().

Referenced by get_plain_letter(), is_letter(), and is_letter_with_marks().

◆ main()

generate_unaccent_rules.main ( args )

Definition at line 228 of file generate_unaccent_rules.py.

def main(args):
    # https://www.unicode.org/reports/tr44/tr44-14.html#Character_Decomposition_Mappings
    decomposition_type_pattern = re.compile(" *<[^>]*> *")
 
    table = {}
    all = []
 
    # unordered set for ensure uniqueness
    charactersSet = set()
 
    # read file UnicodeData.txt
    with codecs.open(
      args.unicodeDataFilePath, mode='r', encoding='UTF-8',
      ) as unicodeDataFile:
        # read everything we need into memory
        for line in unicodeDataFile:
            fields = line.split(";")
            if len(fields) > 5:
                # https://www.unicode.org/reports/tr44/tr44-14.html#UnicodeData.txt
                general_category = fields[2]
                decomposition = fields[5]
                decomposition = re.sub(decomposition_type_pattern, ' ', decomposition)
                id = int(fields[0], 16)
                combining_ids = [int(s, 16) for s in decomposition.split(" ") if s != ""]
                codepoint = Codepoint(id, general_category, combining_ids)
                table[id] = codepoint
                all.append(codepoint)
 
    # walk through all the codepoints looking for interesting mappings
    for codepoint in all:
        if codepoint.general_category.startswith('L') and \
           len(codepoint.combining_ids) > 0:
            if is_letter_with_marks(codepoint, table):
                charactersSet.add((codepoint.id,
                                   chr(get_plain_letter(codepoint, table).id)))
            elif args.noLigaturesExpansion is False and is_ligature(codepoint, table):
                charactersSet.add((codepoint.id,
                                   "".join(chr(combining_codepoint.id)
                                           for combining_codepoint
                                           in get_plain_letters(codepoint, table))))
        elif is_mark_to_remove(codepoint):
            charactersSet.add((codepoint.id, None))
 
    # add CLDR Latin-ASCII characters
    if not args.noLigaturesExpansion:
        charactersSet |= parse_cldr_latin_ascii_transliterator(args.latinAsciiFilePath)
        charactersSet |= special_cases()
 
    # sort for more convenient display
    charactersList = sorted(charactersSet, key=lambda characterPair: characterPair[0])
 
    for characterPair in charactersList:
        print_record(characterPair[0], characterPair[1])
 
 

References fb(), get_plain_letter(), get_plain_letters(), is_letter_with_marks(), is_ligature(), is_mark_to_remove(), len, parse_cldr_latin_ascii_transliterator(), print_record(), and special_cases().

◆ parse_cldr_latin_ascii_transliterator()

generate_unaccent_rules.parse_cldr_latin_ascii_transliterator ( latinAsciiFilePath )

Parse the XML file and return a set of tuples (src, trg), where "src"
is the original character and "trg" the substitute.

Definition at line 160 of file generate_unaccent_rules.py.

def parse_cldr_latin_ascii_transliterator(latinAsciiFilePath):
    """Parse the XML file and return a set of tuples (src, trg), where "src"
    is the original character and "trg" the substitute."""
    charactersSet = set()
 
    # RegEx to parse rules
    rulePattern = re.compile(r'^(?:(.)|(\\u[0-9a-fA-F]{4})) \u2192 (?:\'(.+)\'|(.+)) ;')
 
    # construct tree from XML
    transliterationTree = ET.parse(latinAsciiFilePath)
    transliterationTreeRoot = transliterationTree.getroot()
 
    # Fetch all the transliteration rules.  Since release 29 of Latin-ASCII.xml
    # all the transliteration rules are located in a single tRule block with
    # all rules separated into separate lines.
    blockRules = transliterationTreeRoot.findall("./transforms/transform/tRule")
    assert(len(blockRules) == 1)
 
    # Split the block of rules into one element per line.
    rules = blockRules[0].text.splitlines()
 
    # And finish the processing of each individual rule.
    for rule in rules:
        matches = rulePattern.search(rule)
 
        # The regular expression capture four groups corresponding
        # to the characters.
        #
        # Group 1: plain "src" char. Empty if group 2 is not.
        # Group 2: unicode-escaped "src" char (e.g. "\u0110"). Empty if group 1 is not.
        #
        # Group 3: plain "trg" char. Empty if group 4 is not.
        # Group 4: plain "trg" char between quotes. Empty if group 3 is not.
        if matches is not None:
            src = matches.group(1) if matches.group(1) is not None else bytes(matches.group(2), 'UTF-8').decode('unicode-escape')
            trg = matches.group(3) if matches.group(3) is not None else matches.group(4)
 
            # "'" and """ are escaped
            trg = trg.replace("\\'", "'").replace('\\"', '"')
 
            # the parser of unaccent only accepts non-whitespace characters
            # for "src" and "trg" (see unaccent.c)
            if not src.isspace() and not trg.isspace():
                if src == "\u210c":
                    # This mapping seems to be in error, and causes a collision
                    # by disagreeing with the main Unicode database file:
                    # https://unicode-org.atlassian.net/browse/CLDR-17656
                    continue
                charactersSet.add((ord(src), trg))
 
    return charactersSet
 
 

References assert, fb(), and len.

Referenced by main().

◆ print_record()

generate_unaccent_rules.print_record	(	codepoint,
		letter
	)

Definition at line 59 of file generate_unaccent_rules.py.

def print_record(codepoint, letter):
    if letter:
        # If the letter has whitespace or double quotes, escape double
        # quotes and apply more quotes around it.
        if (' ' in letter) or ('"' in letter):
            letter = '"' + letter.replace('"', '""') + '"'
        output = chr(codepoint) + "\t" + letter
    else:
        output = chr(codepoint)
 
    print(output)
 
 

References fb(), and print().

Referenced by main().

◆ special_cases()

generate_unaccent_rules.special_cases ( )

Returns the special cases which are not handled by other methods

Definition at line 213 of file generate_unaccent_rules.py.

def special_cases():
    """Returns the special cases which are not handled by other methods"""
    charactersSet = set()
 
    # Cyrillic
    charactersSet.add((0x0401, "\u0415"))  # CYRILLIC CAPITAL LETTER IO
    charactersSet.add((0x0451, "\u0435"))  # CYRILLIC SMALL LETTER IO
 
    # Symbols of "Letterlike Symbols" Unicode Block (U+2100 to U+214F)
    charactersSet.add((0x2103, "\xb0C"))   # DEGREE CELSIUS
    charactersSet.add((0x2109, "\xb0F"))   # DEGREE FAHRENHEIT
 
    return charactersSet
 
 

References fb().

Referenced by main().

Variable Documentation

◆ action

generate_unaccent_rules.action

Definition at line 287 of file generate_unaccent_rules.py.

◆ args

generate_unaccent_rules.args = parser.parse_args()

Definition at line 288 of file generate_unaccent_rules.py.

◆ COMBINING_MARK_RANGES

tuple generate_unaccent_rules.COMBINING_MARK_RANGES

Initial value:

=  ((0x0300, 0x0362),   # Mn: Accents, IPA
                         (0x20dd, 0x20E0),   # Me: Symbols
                         (0x20e2, 0x20e4),)

Definition at line 54 of file generate_unaccent_rules.py.

◆ dest

generate_unaccent_rules.dest

Definition at line 285 of file generate_unaccent_rules.py.

◆ help

generate_unaccent_rules.help ( void )

Definition at line 285 of file generate_unaccent_rules.py.

◆ parser

generate_unaccent_rules.parser = argparse.ArgumentParser(description='This script builds unaccent.rules on standard output when given the contents of UnicodeData.txt and Latin-ASCII.xml given as arguments.')

Definition at line 284 of file generate_unaccent_rules.py.

◆ PLAIN_LETTER_RANGES

tuple generate_unaccent_rules.PLAIN_LETTER_RANGES

Initial value:

=  ((ord('a'), ord('z')),  # Latin lower case
                       (ord('A'), ord('Z')),  # Latin upper case
                       (0x03b1, 0x03c9),      # GREEK SMALL LETTER ALPHA, GREEK SMALL LETTER OMEGA
                       (0x0391, 0x03a9))

Definition at line 41 of file generate_unaccent_rules.py.

◆ required

generate_unaccent_rules.required

Definition at line 285 of file generate_unaccent_rules.py.

◆ stdout

generate_unaccent_rules.stdout

Definition at line 35 of file generate_unaccent_rules.py.

◆ str

generate_unaccent_rules.str

Definition at line 285 of file generate_unaccent_rules.py.

◆ True

generate_unaccent_rules.True

Definition at line 285 of file generate_unaccent_rules.py.

◆ type

generate_unaccent_rules.type

Definition at line 285 of file generate_unaccent_rules.py.

Data Structures

Functions

Variables

Function Documentation

◆ get_plain_letter()

◆ get_plain_letters()

◆ is_letter()

◆ is_letter_with_marks()

◆ is_ligature()

◆ is_mark()

◆ is_mark_to_remove()

◆ is_plain_letter()

◆ main()

◆ parse_cldr_latin_ascii_transliterator()

◆ print_record()

◆ special_cases()

Variable Documentation

◆ action

◆ args

◆ COMBINING_MARK_RANGES

◆ dest

◆ help

◆ parser

◆ PLAIN_LETTER_RANGES

◆ required

◆ stdout

◆ str

◆ True

◆ type