PostgreSQL Source Code  git master
generate_unaccent_rules Namespace Reference

Data Structures

class  Codepoint
 

Functions

def print_record (codepoint, letter)
 
def is_mark_to_remove (codepoint)
 
def is_plain_letter (codepoint)
 
def is_mark (codepoint)
 
def is_letter_with_marks (codepoint, table)
 
def is_letter (codepoint, table)
 
def get_plain_letter (codepoint, table)
 
def is_ligature (codepoint, table)
 
def get_plain_letters (codepoint, table)
 
def parse_cldr_latin_ascii_transliterator (latinAsciiFilePath)
 
def special_cases ()
 
def main (args)
 

Variables

 stdout
 
tuple PLAIN_LETTER_RANGES
 
tuple COMBINING_MARK_RANGES
 
 parser = argparse.ArgumentParser(description='This script builds unaccent.rules on standard output when given the contents of UnicodeData.txt and Latin-ASCII.xml given as arguments.')
 
 help
 
 type
 
 str
 
 required
 
 True
 
 dest
 
 action
 
 args = parser.parse_args()
 

Function Documentation

◆ get_plain_letter()

def generate_unaccent_rules.get_plain_letter (   codepoint,
  table 
)
Return the base codepoint without marks. If this codepoint has more
than one combining character, do a recursive lookup on the table to
find out its plain base letter.

Definition at line 126 of file generate_unaccent_rules.py.

126 def get_plain_letter(codepoint, table):
127  """Return the base codepoint without marks. If this codepoint has more
128  than one combining character, do a recursive lookup on the table to
129  find out its plain base letter."""
130  if is_letter_with_marks(codepoint, table):
131  if len(table[codepoint.combining_ids[0]].combining_ids) > 1:
132  return get_plain_letter(table[codepoint.combining_ids[0]], table)
133  elif is_plain_letter(table[codepoint.combining_ids[0]]):
134  return table[codepoint.combining_ids[0]]
135 
136  # Should not come here
137  assert False, 'Codepoint U+%0.2X' % codepoint.id
138  elif is_plain_letter(codepoint):
139  return codepoint
140 
141  # Should not come here
142  assert False, 'Codepoint U+%0.2X' % codepoint.id
143 
144 
def get_plain_letter(codepoint, table)
def is_letter_with_marks(codepoint, table)
const void size_t len

References is_letter_with_marks(), is_plain_letter(), and len.

Referenced by get_plain_letters(), and main().

◆ get_plain_letters()

def generate_unaccent_rules.get_plain_letters (   codepoint,
  table 
)
Return a list of plain letters from a ligature.

Definition at line 150 of file generate_unaccent_rules.py.

150 def get_plain_letters(codepoint, table):
151  """Return a list of plain letters from a ligature."""
152  assert(is_ligature(codepoint, table))
153  return [get_plain_letter(table[id], table) for id in codepoint.combining_ids]
154 
155 
def get_plain_letters(codepoint, table)
def is_ligature(codepoint, table)
#define assert(x)
Definition: regcustom.h:55

References assert, get_plain_letter(), and is_ligature().

Referenced by main().

◆ is_letter()

def generate_unaccent_rules.is_letter (   codepoint,
  table 
)
Return true for letter with or without diacritical marks.

Definition at line 121 of file generate_unaccent_rules.py.

121 def is_letter(codepoint, table):
122  """Return true for letter with or without diacritical marks."""
123  return is_plain_letter(codepoint) or is_letter_with_marks(codepoint, table)
124 
125 
def is_letter(codepoint, table)

References is_letter_with_marks(), and is_plain_letter().

Referenced by is_ligature().

◆ is_letter_with_marks()

def generate_unaccent_rules.is_letter_with_marks (   codepoint,
  table 
)
Returns true for letters combined with one or more marks.

Definition at line 99 of file generate_unaccent_rules.py.

99 def is_letter_with_marks(codepoint, table):
100  """Returns true for letters combined with one or more marks."""
101  # See https://www.unicode.org/reports/tr44/tr44-14.html#General_Category_Values
102 
103  # Letter may have no combining characters, in which case it has
104  # no marks.
105  if len(codepoint.combining_ids) == 1:
106  return False
107 
108  # A letter without diacritical marks has none of them.
109  if any(is_mark(table[i]) for i in codepoint.combining_ids[1:]) is False:
110  return False
111 
112  # Check if the base letter of this letter has marks.
113  codepoint_base = codepoint.combining_ids[0]
114  if is_plain_letter(table[codepoint_base]) is False and \
115  is_letter_with_marks(table[codepoint_base], table) is False:
116  return False
117 
118  return True
119 
120 

References is_mark(), is_plain_letter(), and len.

Referenced by get_plain_letter(), is_letter(), and main().

◆ is_ligature()

def generate_unaccent_rules.is_ligature (   codepoint,
  table 
)
Return true for letters combined with letters.

Definition at line 145 of file generate_unaccent_rules.py.

145 def is_ligature(codepoint, table):
146  """Return true for letters combined with letters."""
147  return all(is_letter(table[i], table) for i in codepoint.combining_ids)
148 
149 

References is_letter().

Referenced by get_plain_letters(), and main().

◆ is_mark()

def generate_unaccent_rules.is_mark (   codepoint)
Returns true for diacritical marks (combining codepoints).

Definition at line 94 of file generate_unaccent_rules.py.

94 def is_mark(codepoint):
95  """Returns true for diacritical marks (combining codepoints)."""
96  return codepoint.general_category in ("Mn", "Me", "Mc")
97 
98 

Referenced by is_letter_with_marks(), and is_mark_to_remove().

◆ is_mark_to_remove()

def generate_unaccent_rules.is_mark_to_remove (   codepoint)
Return true if this is a combining mark to remove.

Definition at line 75 of file generate_unaccent_rules.py.

75 def is_mark_to_remove(codepoint):
76  """Return true if this is a combining mark to remove."""
77  if not is_mark(codepoint):
78  return False
79 
80  for begin, end in COMBINING_MARK_RANGES:
81  if codepoint.id >= begin and codepoint.id <= end:
82  return True
83  return False
84 
85 

References is_mark().

Referenced by main().

◆ is_plain_letter()

def generate_unaccent_rules.is_plain_letter (   codepoint)
Return true if codepoint represents a "plain letter".

Definition at line 86 of file generate_unaccent_rules.py.

86 def is_plain_letter(codepoint):
87  """Return true if codepoint represents a "plain letter"."""
88  for begin, end in PLAIN_LETTER_RANGES:
89  if codepoint.id >= begin and codepoint.id <= end:
90  return True
91  return False
92 
93 

Referenced by get_plain_letter(), is_letter(), and is_letter_with_marks().

◆ main()

def generate_unaccent_rules.main (   args)

Definition at line 219 of file generate_unaccent_rules.py.

219 def main(args):
220  # https://www.unicode.org/reports/tr44/tr44-14.html#Character_Decomposition_Mappings
221  decomposition_type_pattern = re.compile(" *<[^>]*> *")
222 
223  table = {}
224  all = []
225 
226  # unordered set for ensure uniqueness
227  charactersSet = set()
228 
229  # read file UnicodeData.txt
230  with codecs.open(
231  args.unicodeDataFilePath, mode='r', encoding='UTF-8',
232  ) as unicodeDataFile:
233  # read everything we need into memory
234  for line in unicodeDataFile:
235  fields = line.split(";")
236  if len(fields) > 5:
237  # https://www.unicode.org/reports/tr44/tr44-14.html#UnicodeData.txt
238  general_category = fields[2]
239  decomposition = fields[5]
240  decomposition = re.sub(decomposition_type_pattern, ' ', decomposition)
241  id = int(fields[0], 16)
242  combining_ids = [int(s, 16) for s in decomposition.split(" ") if s != ""]
243  codepoint = Codepoint(id, general_category, combining_ids)
244  table[id] = codepoint
245  all.append(codepoint)
246 
247  # walk through all the codepoints looking for interesting mappings
248  for codepoint in all:
249  if codepoint.general_category.startswith('L') and \
250  len(codepoint.combining_ids) > 1:
251  if is_letter_with_marks(codepoint, table):
252  charactersSet.add((codepoint.id,
253  chr(get_plain_letter(codepoint, table).id)))
254  elif args.noLigaturesExpansion is False and is_ligature(codepoint, table):
255  charactersSet.add((codepoint.id,
256  "".join(chr(combining_codepoint.id)
257  for combining_codepoint
258  in get_plain_letters(codepoint, table))))
259  elif is_mark_to_remove(codepoint):
260  charactersSet.add((codepoint.id, None))
261 
262  # add CLDR Latin-ASCII characters
263  if not args.noLigaturesExpansion:
264  charactersSet |= parse_cldr_latin_ascii_transliterator(args.latinAsciiFilePath)
265  charactersSet |= special_cases()
266 
267  # sort for more convenient display
268  charactersList = sorted(charactersSet, key=lambda characterPair: characterPair[0])
269 
270  for characterPair in charactersList:
271  print_record(characterPair[0], characterPair[1])
272 
273 
def print_record(codepoint, letter)
def parse_cldr_latin_ascii_transliterator(latinAsciiFilePath)

References get_plain_letter(), get_plain_letters(), is_letter_with_marks(), is_ligature(), is_mark_to_remove(), len, parse_cldr_latin_ascii_transliterator(), print_record(), and special_cases().

◆ parse_cldr_latin_ascii_transliterator()

def generate_unaccent_rules.parse_cldr_latin_ascii_transliterator (   latinAsciiFilePath)
Parse the XML file and return a set of tuples (src, trg), where "src"
is the original character and "trg" the substitute.

Definition at line 156 of file generate_unaccent_rules.py.

156 def parse_cldr_latin_ascii_transliterator(latinAsciiFilePath):
157  """Parse the XML file and return a set of tuples (src, trg), where "src"
158  is the original character and "trg" the substitute."""
159  charactersSet = set()
160 
161  # RegEx to parse rules
162  rulePattern = re.compile(r'^(?:(.)|(\\u[0-9a-fA-F]{4})) \u2192 (?:\'(.+)\'|(.+)) ;')
163 
164  # construct tree from XML
165  transliterationTree = ET.parse(latinAsciiFilePath)
166  transliterationTreeRoot = transliterationTree.getroot()
167 
168  # Fetch all the transliteration rules. Since release 29 of Latin-ASCII.xml
169  # all the transliteration rules are located in a single tRule block with
170  # all rules separated into separate lines.
171  blockRules = transliterationTreeRoot.findall("./transforms/transform/tRule")
172  assert(len(blockRules) == 1)
173 
174  # Split the block of rules into one element per line.
175  rules = blockRules[0].text.splitlines()
176 
177  # And finish the processing of each individual rule.
178  for rule in rules:
179  matches = rulePattern.search(rule)
180 
181  # The regular expression capture four groups corresponding
182  # to the characters.
183  #
184  # Group 1: plain "src" char. Empty if group 2 is not.
185  # Group 2: unicode-escaped "src" char (e.g. "\u0110"). Empty if group 1 is not.
186  #
187  # Group 3: plain "trg" char. Empty if group 4 is not.
188  # Group 4: plain "trg" char between quotes. Empty if group 3 is not.
189  if matches is not None:
190  src = matches.group(1) if matches.group(1) is not None else bytes(matches.group(2), 'UTF-8').decode('unicode-escape')
191  trg = matches.group(3) if matches.group(3) is not None else matches.group(4)
192 
193  # "'" and """ are escaped
194  trg = trg.replace("\\'", "'").replace('\\"', '"')
195 
196  # the parser of unaccent only accepts non-whitespace characters
197  # for "src" and "trg" (see unaccent.c)
198  if not src.isspace() and not trg.isspace():
199  charactersSet.add((ord(src), trg))
200 
201  return charactersSet
202 
203 

References assert, and len.

Referenced by main().

◆ print_record()

def generate_unaccent_rules.print_record (   codepoint,
  letter 
)

Definition at line 59 of file generate_unaccent_rules.py.

59 def print_record(codepoint, letter):
60  if letter:
61  output = chr(codepoint) + "\t" + letter
62  else:
63  output = chr(codepoint)
64 
65  print(output)
66 
67 
void print(const void *obj)
Definition: print.c:36

References print().

Referenced by main().

◆ special_cases()

def generate_unaccent_rules.special_cases ( )
Returns the special cases which are not handled by other methods

Definition at line 204 of file generate_unaccent_rules.py.

204 def special_cases():
205  """Returns the special cases which are not handled by other methods"""
206  charactersSet = set()
207 
208  # Cyrillic
209  charactersSet.add((0x0401, "\u0415")) # CYRILLIC CAPITAL LETTER IO
210  charactersSet.add((0x0451, "\u0435")) # CYRILLIC SMALL LETTER IO
211 
212  # Symbols of "Letterlike Symbols" Unicode Block (U+2100 to U+214F)
213  charactersSet.add((0x2103, "\xb0C")) # DEGREE CELSIUS
214  charactersSet.add((0x2109, "\xb0F")) # DEGREE FAHRENHEIT
215 
216  return charactersSet
217 
218 

Referenced by main().

Variable Documentation

◆ action

generate_unaccent_rules.action

Definition at line 278 of file generate_unaccent_rules.py.

Referenced by action_to_str(), apply_dispatch(), audit_attempt(), audit_failure(), audit_success(), brin_xlog_desummarize_page(), brin_xlog_insert_update(), brin_xlog_revmap_extend(), brin_xlog_samepage_update(), brin_xlog_update(), check_foreign_key(), computeLeafRecompressWALData(), DefineQueryRewrite(), emit_audit_message(), ExecAlterDefaultPrivilegesStmt(), ExecInitMerge(), ExecInitPartitionInfo(), ExecMergeNotMatched(), ExecSetVariableStmt(), expression_tree_mutator_impl(), expression_tree_walker_impl(), generic_redo(), gistRedoClearFollowRight(), grouping_planner(), handle_streamed_transaction(), hash_search(), hash_search_with_hash_value(), hash_xlog_delete(), hash_xlog_move_page_contents(), hash_xlog_split_allocate_page(), hash_xlog_split_complete(), hash_xlog_squeeze_page(), hash_xlog_vacuum_one_page(), heap_xlog_insert(), heap_xlog_multi_insert(), heap_xlog_prune(), heap_xlog_vacuum(), heap_xlog_visible(), index_set_state_flags(), InsertRule(), iterate_json_values(), iterate_jsonb_values(), logicalrep_message_type(), logicalrep_read_delete(), logicalrep_read_insert(), logicalrep_read_update(), make_ruledef(), parseCreateReplSlotOptions(), perform_pullup_replace_vars(), pgoutput_change(), pgoutput_row_filter(), preprocess_targetlist(), ProcessGUCArray(), push_old_value(), queue_listen(), register_on_commit_action(), REGRESS_utility_command(), RewriteQuery(), set_config_option(), set_config_option_ext(), set_plan_refs(), simple_action_list_append(), spgRedoAddLeaf(), spgRedoAddNode(), spgRedoMoveLeafs(), spgRedoPickSplit(), spgRedoSplitTuple(), stream_open_and_write_change(), stream_write_change(), subquery_planner(), transformMergeStmt(), transformRuleStmt(), ts_headline_json_byid_opt(), ts_headline_jsonb_byid_opt(), and walkdir().

◆ args

generate_unaccent_rules.args = parser.parse_args()

Definition at line 279 of file generate_unaccent_rules.py.

Referenced by _int_matchsel(), _outBoolExpr(), _readBoolExpr(), add_function_defaults(), ahprintf(), appendJSONKeyValueFmt(), appendPQExpBuffer(), appendPQExpBufferVA(), appendStringInfo(), appendStringInfoVA(), archprintf(), array_unnest_support(), arraycontsel(), autoinc(), build_aggregate_deserialfn_expr(), build_aggregate_finalfn_expr(), build_aggregate_serialfn_expr(), build_aggregate_transfn_expr(), build_coercion_expression(), check_agg_arguments(), check_agglevels_and_constraints(), check_foreign_key(), check_hashjoinable(), check_memoizable(), check_mergejoinable(), check_primary_key(), clause_selectivity_ext(), clauselist_selectivity_ext(), coerce_record_to_complex(), count_rowexpr_columns(), CreateTriggerFiringOn(), decrypt_internal(), DefineAggregate(), do_copy(), does_not_exist_skipping(), dopr(), ecpg_do(), ecpg_do_prologue(), ECPGdescribe(), ECPGdo(), ECPGget_desc(), ECPGset_desc(), encrypt_internal(), eqjoinsel(), eqsel_internal(), error(), eval_const_expressions_mutator(), evalFunc(), evalLazyFunc(), evalStandardFunc(), evaluate_function(), examine_opclause_args(), ExecEvalFuncExprStrictFusage(), ExecInitFunc(), ExecInterpExpr(), ExecJustApplyFuncToCase(), executeQueryOrDie(), expand_function_arguments(), expression_tree_walker_impl(), extract_variadic_args(), find_arguments(), find_duplicate_ors(), find_single_rel_for_clauses(), flatten_set_variable_args(), FuncnameGetCandidates(), function_selectivity(), gen_partprune_steps_internal(), generate_series_int4_support(), generate_series_int8_support(), generic_restriction_selectivity(), get_call_expr_arg_stable(), get_call_expr_argtype(), get_from_clause_item(), get_func_sql_syntax_time(), get_join_variables(), get_notclausearg(), get_oper_expr(), get_qual_for_hash(), get_restriction_variable(), get_rule_expr(), get_simple_binary_op_name(), info_cb(), init_work(), inline_function(), insert_username(), isSimpleNode(), join_selectivity(), json_build_array(), json_build_object(), jsonb_build_array(), jsonb_build_object(), libpq_append_conn_error(), libpq_append_error(), lo_manage(), ltreeparentsel(), make_jsp_entry_node(), make_jsp_expr_node(), make_jsp_expr_node_args(), make_op(), make_restrictinfo_internal(), make_scalar_array_op(), make_sub_restrictinfos(), makeBoolExpr(), makeFuncCall(), makeFuncExpr(), match_clause_to_partition_key(), matchingsel(), moddatetime(), multirangesel(), neqjoinsel(), networkjoinsel(), networksel(), parallel_exec_prog(), parse_args(), parse_slash_copy(), patternsel(), patternsel_common(), pg_fatal(), pg_fprintf(), pg_get_object_address(), pg_get_ruledef_worker(), pg_get_viewdef_worker(), pg_identify_object_as_address(), pg_log(), pg_printf(), pg_snprintf(), pg_sprintf(), pg_vfprintf(), pg_vprintf(), pg_vsnprintf(), pg_vsprintf(), PLy_cursor(), PLy_cursor_fetch(), PLy_cursor_plan(), PLy_debug(), PLy_error(), PLy_exception_set_with_details(), PLy_fatal(), PLy_function_build_args(), PLy_info(), PLy_log(), PLy_notice(), PLy_output(), PLy_plan_cursor(), PLy_plan_execute(), PLy_plan_status(), PLy_quote_ident(), PLy_quote_literal(), PLy_quote_nullable(), PLy_spi_exception_set(), PLy_spi_execute(), PLy_spi_prepare(), PLy_subtransaction_exit(), PLy_warning(), PQfn(), pqFunctionCall3(), pqInternalNotice(), prep_status(), prep_status_progress(), printfPQExpBuffer(), process_sublinks_mutator(), processTypesSpec(), psprintf(), psql_add_command(), pull_ands(), pull_ors(), pull_up_sublinks_qual_recurse(), pvsnprintf(), rangesel(), raw_expression_tree_walker_impl(), recheck_cast_function_args(), reorder_function_arguments(), report_invalid_record(), report_status(), restriction_selectivity(), scalararraysel(), scalarineqsel_wrapper(), SetPGVariable(), SetWALFileNameForCleanup(), simplify_and_arguments(), simplify_boolean_equality(), simplify_function(), simplify_or_arguments(), substitute_actual_parameters(), substitute_actual_srf_parameters(), tarPrintf(), test_support_func(), testexpr_is_hashable(), transformAExprBetween(), transformAExprIn(), transformAExprOp(), transformAggregateCall(), transformBoolExpr(), transformCoalesceExpr(), transformFuncCall(), transformGroupingFunc(), transformMinMaxExpr(), tsmatchsel(), ttdummy(), verror(), warning(), and xmlconcat().

◆ COMBINING_MARK_RANGES

tuple generate_unaccent_rules.COMBINING_MARK_RANGES
Initial value:
1 = ((0x0300, 0x0362), # Mn: Accents, IPA
2  (0x20dd, 0x20E0), # Me: Symbols
3  (0x20e2, 0x20e4),)

Definition at line 54 of file generate_unaccent_rules.py.

◆ dest

generate_unaccent_rules.dest

Definition at line 276 of file generate_unaccent_rules.py.

Referenced by _SPI_cursor_operation(), _SPI_execute_plan(), add_pos(), AppendInvalidationMessages(), AppendInvalidationMessageSubGroup(), ArrayCastAndSet(), ascii_safe_strlcpy(), begin_tup_output_tupdesc(), BeginCopyTo(), big5_to_euc_tw(), big5_to_mic(), big5_to_utf8(), brin_copy_tuple(), compute_scalar_stats(), copy_generic_path_info(), copy_plan_costsize(), CopyReadBinaryData(), CreateDestReceiver(), CreateQueryDesc(), CreateReplicationSlot(), DoPortalRunFetch(), EndCommand(), euc_cn_to_mic(), euc_cn_to_utf8(), euc_jis_2004_to_shift_jis_2004(), euc_jis_2004_to_utf8(), euc_jp_to_mic(), euc_jp_to_sjis(), euc_jp_to_utf8(), euc_kr_to_mic(), euc_kr_to_utf8(), euc_tw_to_big5(), euc_tw_to_mic(), euc_tw_to_utf8(), exec_execute_message(), exec_replication_command(), exec_simple_query(), ExecCreateTableAs(), ExecRefreshMatView(), execute_sql_string(), ExecuteCallStmt(), ExecutePlan(), ExecuteQuery(), ExplainOnePlan(), ExplainQuery(), from_char_parse_int(), from_char_parse_int_len(), from_char_seq_search(), from_char_set_int(), FullTransactionIdAdvance(), FullTransactionIdRetreat(), gb18030_to_utf8(), gbk_to_utf8(), get_publications_str(), GetPGVariable(), hashline_number(), heap_copytuple_with_tuple(), IdentifySystem(), init_var_from_num(), iso8859_1_to_utf8(), iso8859_to_utf8(), iso_to_koi8r(), iso_to_mic(), iso_to_win1251(), iso_to_win866(), johab_to_utf8(), koi8r_to_iso(), koi8r_to_mic(), koi8r_to_utf8(), koi8r_to_win1251(), koi8r_to_win866(), koi8u_to_utf8(), latin1_to_mic(), latin2_to_mic(), latin2_to_win1250(), latin3_to_mic(), latin4_to_mic(), ldchar(), LookupWSErrorMessage(), mic_to_big5(), mic_to_euc_cn(), mic_to_euc_jp(), mic_to_euc_kr(), mic_to_euc_tw(), mic_to_iso(), mic_to_koi8r(), mic_to_latin1(), mic_to_latin2(), mic_to_latin3(), mic_to_latin4(), mic_to_sjis(), mic_to_win1250(), mic_to_win1251(), mic_to_win866(), NullCommand(), partition_bounds_copy(), PerformPortalFetch(), pg_cryptohash_final(), pg_do_encoding_conversion_buf(), pg_hmac_final(), pg_md5_final(), pg_sha1_final(), pg_strnxfrm(), pg_strnxfrm_libc(), pg_strnxfrm_prefix(), pg_strxfrm(), pg_strxfrm_libc(), pg_strxfrm_prefix(), pg_to_ascii(), pglz_compress(), pglz_decompress(), pgss_ProcessUtility(), PortalRun(), PortalRunFetch(), PortalRunMulti(), PortalRunSelect(), PortalRunUtility(), postquel_start(), PQcopyResult(), printtup_create_DR(), process_pipe_input(), ProcessQuery(), ProcessUtility(), rbt_copy_data(), read_gucstate_binary(), ReadReplicationSlot(), ReadyForQuery(), refresh_matview_datafill(), REGRESS_utility_command(), RunFromStore(), SendTablespaceList(), SendTimeLineHistory(), SendXlogRecPtrResult(), sepgsql_utility_command(), SerializePendingSyncs(), set_input(), set_var_from_non_decimal_integer_str(), set_var_from_num(), set_var_from_str(), set_var_from_var(), shift_jis_2004_to_euc_jis_2004(), shift_jis_2004_to_utf8(), ShowAllGUCConfig(), ShowGUCConfigOption(), sjis_to_euc_jp(), sjis_to_mic(), sjis_to_utf8(), SnapBuildRestoreContents(), standard_ExecutorRun(), standard_ProcessUtility(), StartReplication(), store_coded_char(), str_numth(), uhc_to_utf8(), utf8_to_big5(), utf8_to_euc_cn(), utf8_to_euc_jis_2004(), utf8_to_euc_jp(), utf8_to_euc_kr(), utf8_to_euc_tw(), utf8_to_gb18030(), utf8_to_gbk(), utf8_to_iso8859(), utf8_to_iso8859_1(), utf8_to_johab(), utf8_to_koi8r(), utf8_to_koi8u(), utf8_to_shift_jis_2004(), utf8_to_sjis(), utf8_to_uhc(), utf8_to_win(), win1250_to_latin2(), win1250_to_mic(), win1251_to_iso(), win1251_to_koi8r(), win1251_to_mic(), win1251_to_win866(), win866_to_iso(), win866_to_koi8r(), win866_to_mic(), win866_to_win1251(), win_to_utf8(), write_pipe_chunks(), and XLogCompressBackupBlock().

◆ help

generate_unaccent_rules.help ( void  )

Definition at line 276 of file generate_unaccent_rules.py.

Referenced by helpSQL().

◆ parser

generate_unaccent_rules.parser = argparse.ArgumentParser(description='This script builds unaccent.rules on standard output when given the contents of UnicodeData.txt and Latin-ASCII.xml given as arguments.')

Definition at line 275 of file generate_unaccent_rules.py.

◆ PLAIN_LETTER_RANGES

tuple generate_unaccent_rules.PLAIN_LETTER_RANGES
Initial value:
1 = ((ord('a'), ord('z')), # Latin lower case
2  (ord('A'), ord('Z')), # Latin upper case
3  (0x03b1, 0x03c9), # GREEK SMALL LETTER ALPHA, GREEK SMALL LETTER OMEGA
4  (0x0391, 0x03a9))

Definition at line 41 of file generate_unaccent_rules.py.

◆ required

◆ stdout

◆ str

generate_unaccent_rules.str

Definition at line 276 of file generate_unaccent_rules.py.

Referenced by _add(), _outA_Const(), _outA_Expr(), _outBitString(), _outBoolean(), _outBoolExpr(), _outConst(), _outConstraint(), _outExtensibleNode(), _outFloat(), _outForeignKeyOptInfo(), _outInteger(), _outList(), _outString(), add_role_attribute(), add_stringlist_item(), add_typedefs_from_file(), AddToDataDirLockFile(), append_with_tabs(), appendArrayEscapedString(), appendBinaryPQExpBuffer(), appendBinaryStringInfo(), appendBinaryStringInfoNT(), appendByteaLiteral(), appendConnStrVal(), appendContextKeyword(), appendPQExpBuffer(), appendPQExpBufferChar(), appendPQExpBufferStr(), appendPQExpBufferVA(), appendShellString(), appendShellStringNoError(), appendStringInfo(), appendStringInfoChar(), appendStringInfoRegexpSubstr(), appendStringInfoSpaces(), appendStringInfoString(), appendStringInfoStringQuoted(), appendStringInfoText(), appendStringInfoVA(), appendStringLiteral(), appendStringLiteralConn(), appendStringLiteralDQ(), ArrayCount(), ASN1_STRING_to_text(), assign_text_var(), be_lo_from_bytea(), be_lo_put(), bitposition(), bits_to_text(), bmsToString(), boolin(), booltext(), box_in(), bpcharrecv(), brin_bloom_summary_out(), brin_minmax_multi_summary_out(), byleng(), bytea_substring(), bytealike(), byteanlike(), byteaoctetlen(), cash_in(), char2wchar(), check_recovery_target_time(), cidin(), circle_in(), circle_out(), citext_hash(), citext_hash_extended(), cliplen(), compact_trigram(), complex_in(), concat_internal(), conninfo_uri_decode(), CopySendString(), cpstrdup(), croak_cstr(), cstr2sv(), cstring_in(), cstring_out(), cstring_recv(), cstring_send(), csv_escaped_print(), csv_print_field(), cube_in(), date_in(), DateTimeParseError(), db_encoding_convert(), DCH_cache_fetch(), DCH_cache_getnew(), DCH_cache_search(), DCH_to_char(), deccvasc(), DecodeDate(), DecodeISO8601Interval(), DecodeNumber(), DecodeNumberField(), DecodePosixTimezone(), DecodeTextArrayToBitmapset(), DecodeTime(), DecodeTimeCommon(), DecodeTimeForInterval(), DecodeTimezone(), dectoasc(), defGetStringList(), destroyPQExpBuffer(), DirectInputFunctionCallSafe(), dostr(), DoubleMetaphone(), dtcvasc(), ean13_in(), ecpg_build_params(), ecpg_get_data(), ecpg_raise(), ecpg_store_input(), ECPGconnect(), ECPGdump_a_type(), ecpyalloc(), EncodeDateOnly(), EncodeDateTime(), EncodeInterval(), EncodeSpecialDate(), EncodeSpecialTimestamp(), EncodeTimeOnly(), EncodeTimezone(), enlargePQExpBuffer(), enlargeStringInfo(), err_generic_string(), err_gettext(), err_sendstring(), errdetail_params(), escape_json(), escape_param_str(), escape_xml(), escape_yaml(), exec_assign_c_string(), ExecInterpExpr(), executeLikeRegex(), ExecuteSqlCommandBuf(), ExplainPrintSettings(), ExplainProperty(), ExplainPropertyList(), ExplainQueryParameters(), fetch_function_defaults(), fill_str(), fillTrgm(), find_end_token(), find_str(), find_struct_member(), find_word(), findchar(), findchar2(), flush_pipe_input(), forkname_chars(), format_node_dump(), func_get_detail(), generate_trgm(), generate_trgm_only(), generate_wildcard_trgm(), Generic_Text_IC_like(), get_nextfield(), get_str_from_var(), get_str_from_var_sci(), get_string_attr(), get_wildcard_part(), getRightMostDot(), hstore_in(), indent_lines(), index_seq_search(), init_libpq_conn(), initPQExpBuffer(), initStringInfo(), InputFunctionCall(), InputFunctionCallSafe(), interpret_func_parallel(), interpret_func_volatility(), interval_in(), intoasc(), is_an_int(), is_separator_char(), isbn_in(), ismn_in(), issn_in(), IsValidJsonNumber(), json_recv(), jsonb_object(), jsonb_object_two_arg(), jsonb_pretty(), jsonb_recv(), JsonbValue_to_SV(), jsonpath_recv(), limit_printout_length(), line_decode(), line_in(), lowerstr(), lowerstr_with_len(), lquery_recv(), lseg_in(), ltree_recv(), ltxtq_recv(), macaddr8_in(), macaddr_in(), main(), make_text_key(), make_trigrams(), make_tsvector(), makeBitString(), makeitem(), makeString(), map_sql_value_to_xml_value(), markPQExpBufferBroken(), mb_strchr(), mxid_to_string(), my_pv_pretty(), nameiclike(), nameicnlike(), namelike(), namenlike(), namerecv(), namestrcmp(), namestrcpy(), nodeToString(), NUM_cache(), NUM_cache_fetch(), NUM_cache_getnew(), NUM_cache_search(), numeric_in(), numeric_normalize(), numeric_out(), numeric_out_sci(), object_to_string(), OidInputFunctionCall(), outBitmapset(), outChar(), outDatum(), outDouble(), outNode(), output_escaped_str(), outToken(), pair_decode(), pair_encode(), parse_affentry(), parse_args(), parse_format(), parse_hba_line(), parse_ooaffentry(), parse_snapshot(), ParseISO8601Number(), parseOidArray(), parseTypeString(), path_decode(), path_encode(), path_in(), pg_clean_ascii(), pg_dependencies_out(), pg_get_expr_worker(), pg_get_function_arg_default(), pg_get_indexdef_worker(), pg_get_partkeydef_worker(), pg_get_statisticsobj_worker(), pg_get_statisticsobjdef_expressions(), pg_GSS_error_int(), pg_input_is_valid_common(), pg_is_ascii(), pg_lsn_in(), pg_lsn_in_internal(), pg_ndistinct_out(), pg_parse_query(), pg_plan_query(), pg_rewrite_query(), pg_size_bytes(), pg_snapshot_in(), pg_snapshot_out(), pg_snprintf(), pg_sprintf(), pg_str_endswith(), pg_strerror_r(), pg_strfromd(), pg_strip_crlf(), pg_ultostr(), pg_ultostr_zeropad(), pg_vsnprintf(), pg_vsprintf(), pg_wchar_strlen(), pgtypes_strdup(), PGTYPESdate_defmt_asc(), PGTYPESdate_from_asc(), PGTYPESinterval_from_asc(), PGTYPESnumeric_from_asc(), PGTYPEStimestamp_defmt_asc(), PGTYPEStimestamp_defmt_scan(), PGTYPEStimestamp_from_asc(), placeChar(), plperl_sv_to_datum(), plpgsql_scanner_init(), PLy_quote_ident(), PLy_quote_literal(), PLy_quote_nullable(), PLyBytes_FromBytea(), PLyDecimal_FromNumeric(), PLyNumber_ToJsonbValue(), PLyObject_FromJsonbValue(), PLyObject_ToScalar(), PLyUnicode_ToComposite(), point_in(), poly_in(), populate_scalar(), pq_getmsgrawstring(), pq_getmsgstring(), pq_getmsgtext(), pq_puttextmessage(), pq_send_ascii_string(), pq_sendcountedtext(), pq_sendstring(), pq_sendtext(), pq_writestring(), PQenv2encoding(), PQescapeIdentifier(), PQescapeInternal(), PQescapeLiteral(), pqResultStrdup(), pretty_format_node_dump(), print_function_arguments(), printfPQExpBuffer(), printsimple(), process_pipe_input(), pset_quoted_string(), pts_error_callback(), quote_ident(), raw_parser(), rdatestr(), rdefmtdate(), read_tablespace_map(), regexp_count(), regexp_fixed_prefix(), regexp_instr(), regexp_like(), regexp_substr(), removeStringInfoSpaces(), replace_text(), resetPQExpBuffer(), resetStringInfo(), rfmtdate(), RS_compile(), RS_execute(), RS_isRegis(), rstrdate(), rupshift(), sanitize_line(), ScanKeywordLookup(), seg_in(), sendMessageToLeader(), sendMessageToWorker(), set_errdata_field(), set_string_attr(), set_var_from_non_decimal_integer_str(), set_var_from_str(), ShowTransactionState(), ShowTransactionStateRec(), ShowUsage(), single_encode(), str2uint(), str_udeescape(), string2ean(), string_matches_pattern(), string_to_bytea_const(), string_to_const(), string_to_datum(), string_to_text(), stringToNode(), stringToNodeInternal(), strlen_max_width(), strnlen(), strspace_len(), strtodouble(), strtoint(), strtoint64(), suff_search(), termPQExpBuffer(), text_format(), text_format_append_string(), text_format_string_conversion(), text_left(), text_length(), text_reverse(), text_right(), text_substring(), text_to_bits(), texticlike(), texticnlike(), textlen(), textlike(), textnlike(), textoctetlen(), textpos(), textrecv(), tidin(), time_in(), timestamp_in(), timestamptz_in(), timetz_in(), TParserInit(), TrimTrailingZeros(), typeStringToTypeName(), unaccent_dict(), unistr(), unknownin(), unknownout(), unknownrecv(), unknownsend(), upc_in(), utf_e2u(), uuid_generate_internal(), varcharrecv(), verify_cb(), wait_result_to_str(), widget_in(), widget_out(), writeNodeArray(), xid8in(), xidin(), xml_out_internal(), xml_recv(), xmlconcat(), xmlelement(), xmlroot(), XmlTableGetValue(), and XmlTableSetDocument().

◆ True

generate_unaccent_rules.True

Definition at line 276 of file generate_unaccent_rules.py.

◆ type

generate_unaccent_rules.type

Definition at line 276 of file generate_unaccent_rules.py.

Referenced by _equalList(), _getObjectDescription(), _jumbleNode(), _readBlockHeader(), accesstype_to_string(), addtt(), agg_args_support_sendreceive(), allocate_reloption(), AutoVacuumRequestWork(), bqarr_in(), brin_page_init(), brin_page_type(), build_mss(), buildACLCommands(), buildDefaultACLCommands(), cannotCastJsonbValue(), check_foreign_key(), cloneouts(), colorcomplement(), CreateStatistics(), cryptohash_internal(), datum_to_jsonb(), DecodeDate(), DecodeDateTime(), DecodeInterval(), DecodePosixTimezone(), DecodeSpecial(), DecodeTimeOnly(), DecodeTimezoneName(), DecodeUnits(), DefineAttr(), dependency_degree(), dumpACL(), dumpComment(), dumpCommentExtended(), dumpDefaultACL(), dumpSecLabel(), ean2isn(), ean2string(), ecpg_do_prologue(), ecpg_dynamic_type(), ecpg_dynamic_type_DDT(), ecpg_get_data(), ECPGdescribe(), ECPGdump_a_simple(), ECPGdump_a_struct(), ECPGdump_a_type(), ECPGfree_type(), ECPGget_desc(), ECPGis_noind_null(), ECPGmake_array_type(), ECPGmake_simple_type(), ECPGmake_struct_member(), ECPGmake_struct_type(), ECPGset_noind_null(), ECPGstruct_member_dup(), emit_audit_message(), EmitProcSignalBarrier(), enable_timeouts(), evalStandardFunc(), exprType(), extract_date(), extract_jsp_bool_expr(), fillTypeDesc(), FindAffixes(), findarc(), findoprnd(), findoprnd_recurse(), fmtfloat(), fmtint(), ForwardSyncRequest(), get_command_type(), get_docrep(), get_dtype(), get_parallel_object_list(), get_th(), get_typdefault(), get_type(), getScalar(), gettype(), getTypeBinaryInputInfo(), getTypeBinaryOutputInfo(), getTypeInputInfo(), getTypeOutputInfo(), GetVirtualXIDsDelayingChkpt(), gin_bool_consistent(), gin_extract_tsquery(), ginint4_queryextract(), hash_page_type(), HaveVirtualXIDsDelayingChkpt(), hladdword(), hlparsetext(), info_cb(), init_compress(), init_custom_variable(), init_litdata_packet(), interval_part_common(), interval_trunc(), isDataGram(), IsPreferredType(), isSimpleNode(), iterate_jsonb_values(), json_typeof(), jsonb_agg_transfn(), jsonb_object_agg_transfn(), jsonb_strip_nulls(), JsonbToCStringWorker(), JsonbType(), jspOperationName(), LexizeAddLemm(), libpq_traverse_files(), logicalrep_write_prepare_common(), main(), make_jsp_expr_node(), make_jsp_expr_node_args(), make_jsp_expr_node_binary(), makeAlterConfigCommand(), makepol(), map_sql_value_to_xml_value(), mkANode(), ndistinct_for_combination(), new_list(), new_variable(), NIAddAffix(), NIImportOOAffixes(), nodeRead(), NonFiniteTimestampTzPart(), outzone(), parse(), parse_compressed_data(), parse_jsonb_index_flags(), parse_literal_data(), parse_ooaffentry(), parse_sane_timezone(), parseAclItem(), parsebranch(), parseqatom(), parsetext(), pg_checksum_init(), pg_checksum_parse_type(), pg_checksum_type_name(), pg_cryptohash_create(), pg_decrypt(), pg_decrypt_iv(), pg_encrypt(), pg_encrypt_iv(), pg_event_trigger_ddl_commands(), pg_get_object_address(), pg_GSS_error_int(), pg_hmac_create(), pg_log(), pg_log_v(), pg_prewarm(), PGTYPEStimestamp_defmt_scan(), pgwin32_socket(), PLy_subtransaction_exit(), pqTraceOutputNR(), prepare_column_cache(), prepareCommandsInPipeline(), process_source_file(), process_target_file(), ProcessProcSignalBarrier(), prs_setup_firstcall(), pushquery(), pushval_asis(), radius_add_attribute(), rainbow(), RegisterSyncRequest(), reindex_one_database(), RememberSyncRequest(), RemovePgTempFilesInDir(), report_status(), roles_is_member_of(), rtypalign(), rtypmsize(), run_reindex_command(), sqlda_common_total_size(), sqlda_dynamic_type(), StartChildProcess(), statext_is_kind_built(), storeObjectDescription(), str_numth(), string2ean(), suff_search(), test_null(), testprs_getlexeme(), time_part_common(), timestamp_part_common(), timestamp_trunc(), timestamp_zone(), timestamptz_part_common(), timestamptz_trunc_internal(), timestamptz_zone(), timetz_part_common(), timetz_zone(), transform_jsonb_string_values(), TypeCategory(), typeidTypeRelid(), typeOrDomainTypeRelid(), verify_brin_page(), writezone(), and XLogWalRcvProcessMsg().