Data Structures
class	Codepoint

Functions
def	bytes (source, encoding='ascii', errors='strict')

def	print_record (codepoint, letter)

def	is_mark_to_remove (codepoint)

def	is_plain_letter (codepoint)

def	is_mark (codepoint)

def	is_letter_with_marks (codepoint, table)

def	is_letter (codepoint, table)

def	get_plain_letter (codepoint, table)

def	is_ligature (codepoint, table)

def	get_plain_letters (codepoint, table)

def	parse_cldr_latin_ascii_transliterator (latinAsciiFilePath)

def	special_cases ()

def	main (args)

Variables
	stdout

	chr = unichr

tuple	PLAIN_LETTER_RANGES

tuple	COMBINING_MARK_RANGES

	parser = argparse.ArgumentParser(description='This script builds unaccent.rules on standard output when given the contents of UnicodeData.txt and Latin-ASCII.xml given as arguments.')

	help

	type

	str

	required

	True

	dest

	action

	args = parser.parse_args()

Function Documentation

◆ bytes()

def generate_unaccent_rules.bytes	(	source,
		encoding = `'ascii'`,
		errors = `'strict'`
	)

Definition at line 52 of file generate_unaccent_rules.py.

Referenced by array_get_slice(), bn_to_mpi(), chr(), decoct(), dir_open_for_write(), fillTrgm(), ginCompressPostingList(), mp_int_binary_len(), mp_int_unsigned_len(), mp_px_rand(), parse_cldr_latin_ascii_transliterator(), pg_rightmost_one_pos64(), pg_signal_thread(), PLyUnicode_Bytes(), qunique(), qunique_arg(), SPI_sql_row_to_xmlelement(), and tar_write_padding_data().

     def bytes(source, encoding='ascii', errors='strict'):
         return source.encode(encoding=encoding, errors=errors)
 else:
 # END: Python 2/3 compatibility - remove when Python 2 compatibility dropped

◆ get_plain_letter()

def generate_unaccent_rules.get_plain_letter	(	codepoint,
		table
	)

Return the base codepoint without marks. If this codepoint has more
than one combining character, do a recursive lookup on the table to
find out its plain base letter.

Definition at line 139 of file generate_unaccent_rules.py.

References assert, is_letter_with_marks(), and is_plain_letter().

Referenced by get_plain_letters(), and main().

 def get_plain_letter(codepoint, table):
     """Return the base codepoint without marks. If this codepoint has more
     than one combining character, do a recursive lookup on the table to
     find out its plain base letter."""
     if is_letter_with_marks(codepoint, table):
         if len(table[codepoint.combining_ids[0]].combining_ids) > 1:
             return get_plain_letter(table[codepoint.combining_ids[0]], table)
         elif is_plain_letter(table[codepoint.combining_ids[0]]):
             return table[codepoint.combining_ids[0]]
 
         # Should not come here
         assert(False)
     elif is_plain_letter(codepoint):
         return codepoint
 
     # Should not come here
     assert(False)
 

◆ get_plain_letters()

def generate_unaccent_rules.get_plain_letters	(	codepoint,
		table
	)

Return a list of plain letters from a ligature.

Definition at line 161 of file generate_unaccent_rules.py.

References assert, get_plain_letter(), and is_ligature().

Referenced by main().

 def get_plain_letters(codepoint, table):
     """Return a list of plain letters from a ligature."""
     assert(is_ligature(codepoint, table))
     return [get_plain_letter(table[id], table) for id in codepoint.combining_ids]
 

◆ is_letter()

def generate_unaccent_rules.is_letter	(	codepoint,
		table
	)

Return true for letter with or without diacritical marks.

Definition at line 135 of file generate_unaccent_rules.py.

References is_letter_with_marks(), and is_plain_letter().

Referenced by is_ligature().

 def is_letter(codepoint, table):
     """Return true for letter with or without diacritical marks."""
     return is_plain_letter(codepoint) or is_letter_with_marks(codepoint, table)
 

◆ is_letter_with_marks()

def generate_unaccent_rules.is_letter_with_marks	(	codepoint,
		table
	)

Returns true for letters combined with one or more marks.

Definition at line 114 of file generate_unaccent_rules.py.

References is_mark(), and is_plain_letter().

Referenced by get_plain_letter(), is_letter(), and main().

 def is_letter_with_marks(codepoint, table):
     """Returns true for letters combined with one or more marks."""
     # See https://www.unicode.org/reports/tr44/tr44-14.html#General_Category_Values
 
     # Letter may have no combining characters, in which case it has
     # no marks.
     if len(codepoint.combining_ids) == 1:
         return False
 
     # A letter without diacritical marks has none of them.
     if any(is_mark(table[i]) for i in codepoint.combining_ids[1:]) is False:
         return False
 
     # Check if the base letter of this letter has marks.
     codepoint_base = codepoint.combining_ids[0]
     if (is_plain_letter(table[codepoint_base]) is False and \
         is_letter_with_marks(table[codepoint_base], table) is False):
         return False
 
     return True
 

◆ is_ligature()

def generate_unaccent_rules.is_ligature	(	codepoint,
		table
	)

Return true for letters combined with letters.

Definition at line 157 of file generate_unaccent_rules.py.

References is_letter().

Referenced by get_plain_letters(), and main().

 def is_ligature(codepoint, table):
     """Return true for letters combined with letters."""
     return all(is_letter(table[i], table) for i in codepoint.combining_ids)
 

◆ is_mark()

def generate_unaccent_rules.is_mark ( codepoint )

Returns true for diacritical marks (combining codepoints).

Definition at line 110 of file generate_unaccent_rules.py.

Referenced by is_letter_with_marks(), and is_mark_to_remove().

 def is_mark(codepoint):
     """Returns true for diacritical marks (combining codepoints)."""
     return codepoint.general_category in ("Mn", "Me", "Mc")
 

◆ is_mark_to_remove()

def generate_unaccent_rules.is_mark_to_remove ( codepoint )

Return true if this is a combining mark to remove.

Definition at line 93 of file generate_unaccent_rules.py.

References is_mark().

Referenced by main().

 def is_mark_to_remove(codepoint):
     """Return true if this is a combining mark to remove."""
     if not is_mark(codepoint):
         return False
 
     for begin, end in COMBINING_MARK_RANGES:
         if codepoint.id >= begin and codepoint.id <= end:
             return True
     return False
 

◆ is_plain_letter()

def generate_unaccent_rules.is_plain_letter ( codepoint )

Return true if codepoint represents a "plain letter".

Definition at line 103 of file generate_unaccent_rules.py.

Referenced by get_plain_letter(), is_letter(), and is_letter_with_marks().

 def is_plain_letter(codepoint):
     """Return true if codepoint represents a "plain letter"."""
     for begin, end in PLAIN_LETTER_RANGES:
       if codepoint.id >= begin and codepoint.id <= end:
         return True
     return False
 

◆ main()

def generate_unaccent_rules.main ( args )

Definition at line 228 of file generate_unaccent_rules.py.

References get_plain_letter(), get_plain_letters(), is_letter_with_marks(), is_ligature(), is_mark_to_remove(), parse_cldr_latin_ascii_transliterator(), print_record(), and special_cases().

 def main(args):
     # https://www.unicode.org/reports/tr44/tr44-14.html#Character_Decomposition_Mappings
     decomposition_type_pattern = re.compile(" *<[^>]*> *")
 
     table = {}
     all = []
 
     # unordered set for ensure uniqueness
     charactersSet = set()
 
     # read file UnicodeData.txt
     with codecs.open(
       args.unicodeDataFilePath, mode='r', encoding='UTF-8',
       ) as unicodeDataFile:
         # read everything we need into memory
         for line in unicodeDataFile:
             fields = line.split(";")
             if len(fields) > 5:
                 # https://www.unicode.org/reports/tr44/tr44-14.html#UnicodeData.txt
                 general_category = fields[2]
                 decomposition = fields[5]
                 decomposition = re.sub(decomposition_type_pattern, ' ', decomposition)
                 id = int(fields[0], 16)
                 combining_ids = [int(s, 16) for s in decomposition.split(" ") if s != ""]
                 codepoint = Codepoint(id, general_category, combining_ids)
                 table[id] = codepoint
                 all.append(codepoint)
 
     # walk through all the codepoints looking for interesting mappings
     for codepoint in all:
         if codepoint.general_category.startswith('L') and \
            len(codepoint.combining_ids) > 1:
             if is_letter_with_marks(codepoint, table):
                 charactersSet.add((codepoint.id,
                              chr(get_plain_letter(codepoint, table).id)))
             elif args.noLigaturesExpansion is False and is_ligature(codepoint, table):
                 charactersSet.add((codepoint.id,
                              "".join(chr(combining_codepoint.id)
                                      for combining_codepoint \
                                      in get_plain_letters(codepoint, table))))
         elif is_mark_to_remove(codepoint):
             charactersSet.add((codepoint.id, None))
 
     # add CLDR Latin-ASCII characters
     if not args.noLigaturesExpansion:
         charactersSet |= parse_cldr_latin_ascii_transliterator(args.latinAsciiFilePath)
         charactersSet |= special_cases()
 
     # sort for more convenient display
     charactersList = sorted(charactersSet, key=lambda characterPair: characterPair[0])
 
     for characterPair in charactersList:
         print_record(characterPair[0], characterPair[1])
 

◆ parse_cldr_latin_ascii_transliterator()

def generate_unaccent_rules.parse_cldr_latin_ascii_transliterator ( latinAsciiFilePath )

Parse the XML file and return a set of tuples (src, trg), where "src"
is the original character and "trg" the substitute.

Definition at line 166 of file generate_unaccent_rules.py.

References assert, and bytes().

Referenced by main().

 def parse_cldr_latin_ascii_transliterator(latinAsciiFilePath):
     """Parse the XML file and return a set of tuples (src, trg), where "src"
     is the original character and "trg" the substitute."""
     charactersSet = set()
 
     # RegEx to parse rules
     rulePattern = re.compile(r'^(?:(.)|(\\u[0-9a-fA-F]{4})) \u2192 (?:\'(.+)\'|(.+)) ;')
 
     # construct tree from XML
     transliterationTree = ET.parse(latinAsciiFilePath)
     transliterationTreeRoot = transliterationTree.getroot()
 
     # Fetch all the transliteration rules.  Since release 29 of Latin-ASCII.xml
     # all the transliteration rules are located in a single tRule block with
     # all rules separated into separate lines.
     blockRules = transliterationTreeRoot.findall("./transforms/transform/tRule")
     assert(len(blockRules) == 1)
 
     # Split the block of rules into one element per line.
     rules = blockRules[0].text.splitlines()
 
     # And finish the processing of each individual rule.
     for rule in rules:
         matches = rulePattern.search(rule)
 
         # The regular expression capture four groups corresponding
         # to the characters.
         #
         # Group 1: plain "src" char. Empty if group 2 is not.
         # Group 2: unicode-escaped "src" char (e.g. "\u0110"). Empty if group 1 is not.
         #
         # Group 3: plain "trg" char. Empty if group 4 is not.
         # Group 4: plain "trg" char between quotes. Empty if group 3 is not.
         if matches is not None:
             src = matches.group(1) if matches.group(1) is not None else bytes(matches.group(2), 'UTF-8').decode('unicode-escape')
             trg = matches.group(3) if matches.group(3) is not None else matches.group(4)
 
             # "'" and """ are escaped
             trg = trg.replace("\\'", "'").replace('\\"', '"')
 
             # the parser of unaccent only accepts non-whitespace characters
             # for "src" and "trg" (see unaccent.c)
             if not src.isspace() and not trg.isspace():
                 charactersSet.add((ord(src), trg))
 
     return charactersSet
 

◆ print_record()

def generate_unaccent_rules.print_record	(	codepoint,
		letter
	)

Definition at line 79 of file generate_unaccent_rules.py.

References print().

Referenced by main().

 def print_record(codepoint, letter):
     if letter:
         output = chr(codepoint) + "\t" + letter
     else:
         output = chr(codepoint)
 
     print(output)
 

◆ special_cases()

def generate_unaccent_rules.special_cases ( )

Returns the special cases which are not handled by other methods

Definition at line 213 of file generate_unaccent_rules.py.

Referenced by main().

 def special_cases():
     """Returns the special cases which are not handled by other methods"""
     charactersSet = set()
 
     # Cyrillic
     charactersSet.add((0x0401, u"\u0415")) # CYRILLIC CAPITAL LETTER IO
     charactersSet.add((0x0451, u"\u0435")) # CYRILLIC SMALL LETTER IO
 
     # Symbols of "Letterlike Symbols" Unicode Block (U+2100 to U+214F)
     charactersSet.add((0x2103, u"\xb0C")) # DEGREE CELSIUS
     charactersSet.add((0x2109, u"\xb0F")) # DEGREE FAHRENHEIT
     charactersSet.add((0x2117, "(P)")) # SOUND RECORDING COPYRIGHT
 
     return charactersSet
 

Variable Documentation

◆ action

generate_unaccent_rules.action

Definition at line 286 of file generate_unaccent_rules.py.

Referenced by _copyAlterDefaultPrivilegesStmt(), _copyAlterExtensionContentsStmt(), _copyAlterRoleStmt(), _copyOnConflictClause(), _copyOnConflictExpr(), _equalAlterDefaultPrivilegesStmt(), _equalAlterExtensionContentsStmt(), _equalAlterRoleStmt(), _equalOnConflictClause(), _equalOnConflictExpr(), _outOnConflictExpr(), _readOnConflictExpr(), apply_dispatch(), brin_xlog_desummarize_page(), brin_xlog_insert_update(), brin_xlog_revmap_extend(), brin_xlog_samepage_update(), brin_xlog_update(), check_foreign_key(), computeLeafRecompressWALData(), ExecAlterDefaultPrivilegesStmt(), ExecSetVariableStmt(), generic_redo(), gistRedoClearFollowRight(), hash_search(), hash_xlog_delete(), hash_xlog_move_page_contents(), hash_xlog_split_allocate_page(), hash_xlog_split_complete(), hash_xlog_squeeze_page(), hash_xlog_vacuum_one_page(), heap_xlog_clean(), heap_xlog_insert(), heap_xlog_multi_insert(), heap_xlog_visible(), iterate_json_values(), iterate_jsonb_values(), logicalrep_read_delete(), logicalrep_read_insert(), logicalrep_read_update(), make_ruledef(), ModifyWaitEvent(), pgstat_get_db_entry(), pgstat_get_tab_entry(), process_source_file(), queue_listen(), register_on_commit_action(), spgRedoAddLeaf(), spgRedoAddNode(), spgRedoMoveLeafs(), spgRedoPickSplit(), spgRedoSplitTuple(), transformRuleStmt(), ts_headline_json_byid_opt(), ts_headline_jsonb_byid_opt(), and walkdir().

◆ args

generate_unaccent_rules.args = parser.parse_args()

Definition at line 287 of file generate_unaccent_rules.py.

Referenced by _copyAggref(), _copyBoolExpr(), _copyCaseExpr(), _copyCoalesceExpr(), _copyCreateTrigStmt(), _copyDefineStmt(), _copyDistinctExpr(), _copyDoStmt(), _copyFuncCall(), _copyFuncExpr(), _copyGroupingFunc(), _copyMinMaxExpr(), _copyNullIfExpr(), _copyOpExpr(), _copyRangeTableSample(), _copyRowExpr(), _copyScalarArrayOpExpr(), _copySubPlan(), _copyTableSampleClause(), _copyVariableSetStmt(), _copyWindowFunc(), _copyXmlExpr(), _equalAggref(), _equalBoolExpr(), _equalCaseExpr(), _equalCoalesceExpr(), _equalCreateTrigStmt(), _equalDefineStmt(), _equalDistinctExpr(), _equalDoStmt(), _equalFuncCall(), _equalFuncExpr(), _equalGroupingFunc(), _equalMinMaxExpr(), _equalNullIfExpr(), _equalOpExpr(), _equalRangeTableSample(), _equalRowExpr(), _equalScalarArrayOpExpr(), _equalSubPlan(), _equalTableSampleClause(), _equalVariableSetStmt(), _equalWindowFunc(), _equalXmlExpr(), _int_matchsel(), _outAggref(), _outBoolExpr(), _outCaseExpr(), _outCoalesceExpr(), _outDistinctExpr(), _outFuncCall(), _outFuncExpr(), _outGroupingFunc(), _outMinMaxExpr(), _outNullIfExpr(), _outOpExpr(), _outRangeTableSample(), _outRowExpr(), _outScalarArrayOpExpr(), _outSubPlan(), _outTableSampleClause(), _outWindowFunc(), _outXmlExpr(), _readAggref(), _readBoolExpr(), _readCaseExpr(), _readCoalesceExpr(), _readDistinctExpr(), _readFuncExpr(), _readGroupingFunc(), _readMinMaxExpr(), _readNullIfExpr(), _readOpExpr(), _readRowExpr(), _readScalarArrayOpExpr(), _readSubPlan(), _readTableSampleClause(), _readWindowFunc(), _readXmlExpr(), ahprintf(), appendPQExpBuffer(), appendStringInfo(), archprintf(), array_unnest_support(), arraycontsel(), autoinc(), build_aggregate_combinefn_expr(), build_aggregate_deserialfn_expr(), build_aggregate_finalfn_expr(), build_aggregate_serialfn_expr(), build_aggregate_transfn_expr(), build_coercion_expression(), check_agg_arguments(), check_agglevels_and_constraints(), check_foreign_key(), check_hashjoinable(), check_mergejoinable(), check_primary_key(), clause_selectivity(), clauselist_selectivity_simple(), coerce_record_to_complex(), count_rowexpr_columns(), CreateTrigger(), does_not_exist_skipping(), ECPGdescribe(), ECPGdo(), ECPGget_desc(), ECPGset_desc(), eqjoinsel(), eqsel_internal(), error(), eval_const_expressions_mutator(), evalStandardFunc(), evaluate_function(), ExecEvalFuncExprStrictFusage(), ExecInterpExpr(), ExecJustApplyFuncToCase(), executeQueryOrDie(), expand_function_arguments(), expression_tree_walker(), find_duplicate_ors(), flushbuffer(), FuncnameGetCandidates(), function_selectivity(), gen_partprune_steps_internal(), generate_series_int4_support(), generate_series_int8_support(), get_call_expr_arg_stable(), get_call_expr_argtype(), get_from_clause_item(), get_notclausearg(), get_oper_expr(), get_qual_for_hash(), get_rule_expr(), get_simple_binary_op_name(), inline_function(), insert_username(), isSimpleNode(), json_build_array(), json_build_object(), jsonb_build_array(), jsonb_build_object(), JumbleExpr(), llvm_compile_expr(), lo_manage(), ltreeparentsel(), make_jsp_entry_node(), make_jsp_expr_node(), make_op(), make_restrictinfo_internal(), make_scalar_array_op(), make_sub_restrictinfos(), makeBoolExpr(), makeFuncCall(), makeFuncExpr(), match_clause_to_partition_key(), MemoryContextSwitchTo(), moddatetime(), neqjoinsel(), networkjoinsel(), networksel(), parallel_exec_prog(), parallel_transfer_all_new_dbs(), patternsel(), pg_fatal(), pg_fprintf(), pg_get_object_address(), pg_get_ruledef_worker(), pg_get_viewdef_worker(), pg_identify_object_as_address(), pg_log(), pg_printf(), pg_snprintf(), pg_sprintf(), PLy_exception_set_with_details(), PLy_function_build_args(), PLy_generate_spi_exceptions(), PLy_spi_exception_set(), pqInternalNotice(), prep_status(), printfPQExpBuffer(), process_sublinks_mutator(), psprintf(), psql_command(), pull_ands(), pull_ors(), pull_up_sublinks_qual_recurse(), rangesel(), raw_expression_tree_walker(), reorder_function_arguments(), report_invalid_record(), report_status(), scalararraysel(), scalarineqsel_wrapper(), SetWALFileNameForCleanup(), simplify_function(), substitute_actual_parameters(), substitute_actual_srf_parameters(), tarPrintf(), test_support_func(), testexpr_is_hashable(), transformAExprBetween(), transformAExprIn(), transformAExprOp(), transformBoolExpr(), transformCoalesceExpr(), transformFuncCall(), transformGroupingFunc(), transformMinMaxExpr(), tsmatchsel(), ttdummy(), and warning().

◆ chr

generate_unaccent_rules.chr = unichr

Definition at line 49 of file generate_unaccent_rules.py.

◆ COMBINING_MARK_RANGES

tuple generate_unaccent_rules.COMBINING_MARK_RANGES

Initial value:

 =  ((0x0300, 0x0362),  # Mn: Accents, IPA
                          (0x20dd, 0x20E0),  # Me: Symbols
                          (0x20e2, 0x20e4),)

Definition at line 75 of file generate_unaccent_rules.py.

◆ dest

generate_unaccent_rules.dest

Definition at line 284 of file generate_unaccent_rules.py.

Referenced by _SPI_execute_plan(), begin_tup_output_tupdesc(), BeginCopy(), big5_to_euc_tw(), big5_to_mic(), big5_to_utf8(), brin_copy_tuple(), compute_scalar_stats(), CreateQueryDesc(), CreateReplicationSlot(), euc_cn_to_mic(), euc_cn_to_utf8(), euc_jis_2004_to_shift_jis_2004(), euc_jis_2004_to_utf8(), euc_jp_to_mic(), euc_jp_to_sjis(), euc_jp_to_utf8(), euc_kr_to_mic(), euc_kr_to_utf8(), euc_tw_to_big5(), euc_tw_to_mic(), euc_tw_to_utf8(), exec_execute_message(), exec_replication_command(), exec_simple_query(), ExecCreateTableAs(), ExecRefreshMatView(), execute_sql_string(), ExplainOnePlan(), gb18030_to_utf8(), gbk_to_utf8(), hashline_number(), IdentifySystem(), iso8859_1_to_utf8(), iso8859_to_utf8(), iso_to_koi8r(), iso_to_mic(), iso_to_win1251(), iso_to_win866(), johab_to_utf8(), koi8r_to_iso(), koi8r_to_mic(), koi8r_to_utf8(), koi8r_to_win1251(), koi8r_to_win866(), koi8u_to_utf8(), latin1_to_mic(), latin2_to_mic(), latin2_to_win1250(), latin3_to_mic(), latin4_to_mic(), mic_to_big5(), mic_to_euc_cn(), mic_to_euc_jp(), mic_to_euc_kr(), mic_to_euc_tw(), mic_to_iso(), mic_to_koi8r(), mic_to_latin1(), mic_to_latin2(), mic_to_latin3(), mic_to_latin4(), mic_to_sjis(), mic_to_win1250(), mic_to_win1251(), mic_to_win866(), partition_bounds_copy(), pglz_decompress(), PortalRunSelect(), PostmasterMarkPIDForWorkerNotify(), postquel_start(), PQcopyResult(), printtup_create_DR(), process_pipe_input(), ProcessUtility(), sepgsql_utility_command(), shift_jis_2004_to_euc_jis_2004(), shift_jis_2004_to_utf8(), sjis_to_euc_jp(), sjis_to_mic(), sjis_to_utf8(), standard_ExecutorRun(), StartReplication(), store_coded_char(), str_numth(), uhc_to_utf8(), utf8_to_big5(), utf8_to_euc_cn(), utf8_to_euc_jis_2004(), utf8_to_euc_jp(), utf8_to_euc_kr(), utf8_to_euc_tw(), utf8_to_gb18030(), utf8_to_gbk(), utf8_to_iso8859(), utf8_to_iso8859_1(), utf8_to_johab(), utf8_to_koi8r(), utf8_to_koi8u(), utf8_to_shift_jis_2004(), utf8_to_sjis(), utf8_to_uhc(), utf8_to_win(), win1250_to_latin2(), win1250_to_mic(), win1251_to_iso(), win1251_to_koi8r(), win1251_to_mic(), win1251_to_win866(), win866_to_iso(), win866_to_koi8r(), win866_to_mic(), win866_to_win1251(), and win_to_utf8().

◆ help

generate_unaccent_rules.help

Definition at line 284 of file generate_unaccent_rules.py.

Referenced by helpSQL().

◆ parser

generate_unaccent_rules.parser = argparse.ArgumentParser(description='This script builds unaccent.rules on standard output when given the contents of UnicodeData.txt and Latin-ASCII.xml given as arguments.')

Definition at line 283 of file generate_unaccent_rules.py.

◆ PLAIN_LETTER_RANGES

tuple generate_unaccent_rules.PLAIN_LETTER_RANGES

Initial value:

 =  ((ord('a'), ord('z')), # Latin lower case
                        (ord('A'), ord('Z')), # Latin upper case
                        (0x03b1, 0x03c9),     # GREEK SMALL LETTER ALPHA, GREEK SMALL LETTER OMEGA
                        (0x0391, 0x03a9))