PostgreSQL Source Code  git master
relation.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  * relation.c
3  * PostgreSQL logical replication
4  *
5  * Copyright (c) 2016-2020, PostgreSQL Global Development Group
6  *
7  * IDENTIFICATION
8  * src/backend/replication/logical/relation.c
9  *
10  * NOTES
11  * This file contains helper functions for logical replication relation
12  * mapping cache.
13  *
14  *-------------------------------------------------------------------------
15  */
16 
17 #include "postgres.h"
18 
19 #include "access/sysattr.h"
20 #include "access/table.h"
21 #include "catalog/namespace.h"
23 #include "executor/executor.h"
24 #include "nodes/makefuncs.h"
27 #include "utils/builtins.h"
28 #include "utils/inval.h"
29 #include "utils/lsyscache.h"
30 #include "utils/memutils.h"
31 #include "utils/syscache.h"
32 
34 
35 static HTAB *LogicalRepRelMap = NULL;
36 static HTAB *LogicalRepTypMap = NULL;
37 
38 
39 /*
40  * Relcache invalidation callback for our relation map cache.
41  */
42 static void
44 {
45  LogicalRepRelMapEntry *entry;
46 
47  /* Just to be sure. */
48  if (LogicalRepRelMap == NULL)
49  return;
50 
51  if (reloid != InvalidOid)
52  {
54 
55  hash_seq_init(&status, LogicalRepRelMap);
56 
57  /* TODO, use inverse lookup hashtable? */
58  while ((entry = (LogicalRepRelMapEntry *) hash_seq_search(&status)) != NULL)
59  {
60  if (entry->localreloid == reloid)
61  {
62  entry->localreloid = InvalidOid;
63  hash_seq_term(&status);
64  break;
65  }
66  }
67  }
68  else
69  {
70  /* invalidate all cache entries */
72 
73  hash_seq_init(&status, LogicalRepRelMap);
74 
75  while ((entry = (LogicalRepRelMapEntry *) hash_seq_search(&status)) != NULL)
76  entry->localreloid = InvalidOid;
77  }
78 }
79 
80 /*
81  * Initialize the relation map cache.
82  */
83 static void
85 {
86  HASHCTL ctl;
87 
88  if (!LogicalRepRelMapContext)
89  LogicalRepRelMapContext =
91  "LogicalRepRelMapContext",
93 
94  /* Initialize the relation hash table. */
95  MemSet(&ctl, 0, sizeof(ctl));
96  ctl.keysize = sizeof(LogicalRepRelId);
97  ctl.entrysize = sizeof(LogicalRepRelMapEntry);
99 
100  LogicalRepRelMap = hash_create("logicalrep relation map cache", 128, &ctl,
102 
103  /* Initialize the type hash table. */
104  MemSet(&ctl, 0, sizeof(ctl));
105  ctl.keysize = sizeof(Oid);
106  ctl.entrysize = sizeof(LogicalRepTyp);
108 
109  /* This will usually be small. */
110  LogicalRepTypMap = hash_create("logicalrep type map cache", 2, &ctl,
112 
113  /* Watch for invalidation events. */
115  (Datum) 0);
116 }
117 
118 /*
119  * Free the entry of a relation map cache.
120  */
121 static void
123 {
124  LogicalRepRelation *remoterel;
125 
126  remoterel = &entry->remoterel;
127 
128  pfree(remoterel->nspname);
129  pfree(remoterel->relname);
130 
131  if (remoterel->natts > 0)
132  {
133  int i;
134 
135  for (i = 0; i < remoterel->natts; i++)
136  pfree(remoterel->attnames[i]);
137 
138  pfree(remoterel->attnames);
139  pfree(remoterel->atttyps);
140  }
141  bms_free(remoterel->attkeys);
142 
143  if (entry->attrmap)
144  pfree(entry->attrmap);
145 }
146 
147 /*
148  * Add new entry or update existing entry in the relation map cache.
149  *
150  * Called when new relation mapping is sent by the publisher to update
151  * our expected view of incoming data from said publisher.
152  */
153 void
155 {
156  MemoryContext oldctx;
157  LogicalRepRelMapEntry *entry;
158  bool found;
159  int i;
160 
161  if (LogicalRepRelMap == NULL)
163 
164  /*
165  * HASH_ENTER returns the existing entry if present or creates a new one.
166  */
167  entry = hash_search(LogicalRepRelMap, (void *) &remoterel->remoteid,
168  HASH_ENTER, &found);
169 
170  if (found)
172 
173  memset(entry, 0, sizeof(LogicalRepRelMapEntry));
174 
175  /* Make cached copy of the data */
176  oldctx = MemoryContextSwitchTo(LogicalRepRelMapContext);
177  entry->remoterel.remoteid = remoterel->remoteid;
178  entry->remoterel.nspname = pstrdup(remoterel->nspname);
179  entry->remoterel.relname = pstrdup(remoterel->relname);
180  entry->remoterel.natts = remoterel->natts;
181  entry->remoterel.attnames = palloc(remoterel->natts * sizeof(char *));
182  entry->remoterel.atttyps = palloc(remoterel->natts * sizeof(Oid));
183  for (i = 0; i < remoterel->natts; i++)
184  {
185  entry->remoterel.attnames[i] = pstrdup(remoterel->attnames[i]);
186  entry->remoterel.atttyps[i] = remoterel->atttyps[i];
187  }
188  entry->remoterel.replident = remoterel->replident;
189  entry->remoterel.attkeys = bms_copy(remoterel->attkeys);
190  MemoryContextSwitchTo(oldctx);
191 }
192 
193 /*
194  * Find attribute index in TupleDesc struct by attribute name.
195  *
196  * Returns -1 if not found.
197  */
198 static int
200 {
201  int i;
202 
203  for (i = 0; i < remoterel->natts; i++)
204  {
205  if (strcmp(remoterel->attnames[i], attname) == 0)
206  return i;
207  }
208 
209  return -1;
210 }
211 
212 /*
213  * Open the local relation associated with the remote one.
214  *
215  * Optionally rebuilds the Relcache mapping if it was invalidated
216  * by local DDL.
217  */
220 {
221  LogicalRepRelMapEntry *entry;
222  bool found;
223  Oid relid = InvalidOid;
224  LogicalRepRelation *remoterel;
225 
226  if (LogicalRepRelMap == NULL)
228 
229  /* Search for existing entry. */
230  entry = hash_search(LogicalRepRelMap, (void *) &remoteid,
231  HASH_FIND, &found);
232 
233  if (!found)
234  elog(ERROR, "no relation map entry for remote relation ID %u",
235  remoteid);
236 
237  remoterel = &entry->remoterel;
238 
239  /*
240  * When opening and locking a relation, pending invalidation messages are
241  * processed which can invalidate the relation. We need to update the
242  * local cache both when we are first time accessing the relation and when
243  * the relation is invalidated (aka entry->localreloid is set InvalidOid).
244  */
245  if (!OidIsValid(entry->localreloid))
246  {
247  /* Try to find and lock the relation by name. */
248  relid = RangeVarGetRelid(makeRangeVar(remoterel->nspname,
249  remoterel->relname, -1),
250  lockmode, true);
251  if (!OidIsValid(relid))
252  ereport(ERROR,
253  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
254  errmsg("logical replication target relation \"%s.%s\" does not exist",
255  remoterel->nspname, remoterel->relname)));
256  entry->localrel = table_open(relid, NoLock);
257 
258  }
259  else
260  {
261  relid = entry->localreloid;
262  entry->localrel = table_open(entry->localreloid, lockmode);
263  }
264 
265  if (!OidIsValid(entry->localreloid))
266  {
267  int found;
268  Bitmapset *idkey;
269  TupleDesc desc;
270  MemoryContext oldctx;
271  int i;
272 
273  /* Check for supported relkind. */
274  CheckSubscriptionRelkind(entry->localrel->rd_rel->relkind,
275  remoterel->nspname, remoterel->relname);
276 
277  /*
278  * Build the mapping of local attribute numbers to remote attribute
279  * numbers and validate that we don't miss any replicated columns as
280  * that would result in potentially unwanted data loss.
281  */
282  desc = RelationGetDescr(entry->localrel);
283  oldctx = MemoryContextSwitchTo(LogicalRepRelMapContext);
284  entry->attrmap = make_attrmap(desc->natts);
285  MemoryContextSwitchTo(oldctx);
286 
287  found = 0;
288  for (i = 0; i < desc->natts; i++)
289  {
290  int attnum;
291  Form_pg_attribute attr = TupleDescAttr(desc, i);
292 
293  if (attr->attisdropped || attr->attgenerated)
294  {
295  entry->attrmap->attnums[i] = -1;
296  continue;
297  }
298 
299  attnum = logicalrep_rel_att_by_name(remoterel,
300  NameStr(attr->attname));
301 
302  entry->attrmap->attnums[i] = attnum;
303  if (attnum >= 0)
304  found++;
305  }
306 
307  /* TODO, detail message with names of missing columns */
308  if (found < remoterel->natts)
309  ereport(ERROR,
310  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
311  errmsg("logical replication target relation \"%s.%s\" is missing "
312  "some replicated columns",
313  remoterel->nspname, remoterel->relname)));
314 
315  /*
316  * Check that replica identity matches. We allow for stricter replica
317  * identity (fewer columns) on subscriber as that will not stop us
318  * from finding unique tuple. IE, if publisher has identity
319  * (id,timestamp) and subscriber just (id) this will not be a problem,
320  * but in the opposite scenario it will.
321  *
322  * Don't throw any error here just mark the relation entry as not
323  * updatable, as replica identity is only for updates and deletes but
324  * inserts can be replicated even without it.
325  */
326  entry->updatable = true;
327  idkey = RelationGetIndexAttrBitmap(entry->localrel,
329  /* fallback to PK if no replica identity */
330  if (idkey == NULL)
331  {
332  idkey = RelationGetIndexAttrBitmap(entry->localrel,
334 
335  /*
336  * If no replica identity index and no PK, the published table
337  * must have replica identity FULL.
338  */
339  if (idkey == NULL && remoterel->replident != REPLICA_IDENTITY_FULL)
340  entry->updatable = false;
341  }
342 
343  i = -1;
344  while ((i = bms_next_member(idkey, i)) >= 0)
345  {
347 
348  if (!AttrNumberIsForUserDefinedAttr(attnum))
349  ereport(ERROR,
350  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
351  errmsg("logical replication target relation \"%s.%s\" uses "
352  "system columns in REPLICA IDENTITY index",
353  remoterel->nspname, remoterel->relname)));
354 
355  attnum = AttrNumberGetAttrOffset(attnum);
356 
357  if (entry->attrmap->attnums[attnum] < 0 ||
358  !bms_is_member(entry->attrmap->attnums[attnum], remoterel->attkeys))
359  {
360  entry->updatable = false;
361  break;
362  }
363  }
364 
365  entry->localreloid = relid;
366  }
367 
368  if (entry->state != SUBREL_STATE_READY)
370  entry->localreloid,
371  &entry->statelsn,
372  true);
373 
374  return entry;
375 }
376 
377 /*
378  * Close the previously opened logical relation.
379  */
380 void
382 {
383  table_close(rel->localrel, lockmode);
384  rel->localrel = NULL;
385 }
386 
387 /*
388  * Free the type map cache entry data.
389  */
390 static void
392 {
393  pfree(entry->nspname);
394  pfree(entry->typname);
395 }
396 
397 /*
398  * Add new entry or update existing entry in the type map cache.
399  */
400 void
402 {
403  MemoryContext oldctx;
404  LogicalRepTyp *entry;
405  bool found;
406 
407  if (LogicalRepTypMap == NULL)
409 
410  /*
411  * HASH_ENTER returns the existing entry if present or creates a new one.
412  */
413  entry = hash_search(LogicalRepTypMap, (void *) &remotetyp->remoteid,
414  HASH_ENTER, &found);
415 
416  if (found)
418 
419  /* Make cached copy of the data */
420  entry->remoteid = remotetyp->remoteid;
421  oldctx = MemoryContextSwitchTo(LogicalRepRelMapContext);
422  entry->nspname = pstrdup(remotetyp->nspname);
423  entry->typname = pstrdup(remotetyp->typname);
424  MemoryContextSwitchTo(oldctx);
425 }
426 
427 /*
428  * Fetch type name from the cache by remote type OID.
429  *
430  * Return a substitute value if we cannot find the data type; no message is
431  * sent to the log in that case, because this is used by error callback
432  * already.
433  */
434 char *
436 {
437  LogicalRepTyp *entry;
438  bool found;
439 
440  /* Internal types are mapped directly. */
441  if (remoteid < FirstGenbkiObjectId)
442  {
443  if (!get_typisdefined(remoteid))
444  {
445  /*
446  * This can be caused by having a publisher with a higher
447  * PostgreSQL major version than the subscriber.
448  */
449  return psprintf("unrecognized %u", remoteid);
450  }
451 
452  return format_type_be(remoteid);
453  }
454 
455  if (LogicalRepTypMap == NULL)
456  {
457  /*
458  * If the typemap is not initialized yet, we cannot possibly attempt
459  * to search the hash table; but there's no way we know the type
460  * locally yet, since we haven't received a message about this type,
461  * so this is the best we can do.
462  */
463  return psprintf("unrecognized %u", remoteid);
464  }
465 
466  /* search the mapping */
467  entry = hash_search(LogicalRepTypMap, (void *) &remoteid,
468  HASH_FIND, &found);
469  if (!found)
470  return psprintf("unrecognized %u", remoteid);
471 
472  Assert(OidIsValid(entry->remoteid));
473  return psprintf("%s.%s", entry->nspname, entry->typname);
474 }
Subscription * MySubscription
Definition: worker.c:103
static void logicalrep_relmap_free_entry(LogicalRepRelMapEntry *entry)
Definition: relation.c:122
#define AllocSetContextCreate
Definition: memutils.h:170
void table_close(Relation relation, LOCKMODE lockmode)
Definition: table.c:133
Bitmapset * bms_copy(const Bitmapset *a)
Definition: bitmapset.c:74
#define HASH_CONTEXT
Definition: hsearch.h:93
char * typname
Definition: logicalproto.h:60
#define HASH_ELEM
Definition: hsearch.h:87
MemoryContext hcxt
Definition: hsearch.h:78
#define RelationGetDescr(relation)
Definition: rel.h:454
int LOCKMODE
Definition: lockdefs.h:26
#define RangeVarGetRelid(relation, lockmode, missing_ok)
Definition: namespace.h:63
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:92
char * pstrdup(const char *in)
Definition: mcxt.c:1186
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
int bms_next_member(const Bitmapset *a, int prevbit)
Definition: bitmapset.c:1043
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Size entrysize
Definition: hsearch.h:73
#define AttrNumberIsForUserDefinedAttr(attributeNumber)
Definition: attnum.h:41
bool get_typisdefined(Oid typid)
Definition: lsyscache.c:1951
int errcode(int sqlerrcode)
Definition: elog.c:608
#define MemSet(start, val, len)
Definition: c.h:962
char * format_type_be(Oid type_oid)
Definition: format_type.c:326
void logicalrep_rel_close(LogicalRepRelMapEntry *rel, LOCKMODE lockmode)
Definition: relation.c:381
#define FirstLowInvalidHeapAttributeNumber
Definition: sysattr.h:27
void * hash_search(HTAB *hashp, const void *keyPtr, HASHACTION action, bool *foundPtr)
Definition: dynahash.c:906
Form_pg_class rd_rel
Definition: rel.h:84
unsigned int Oid
Definition: postgres_ext.h:31
static HTAB * LogicalRepTypMap
Definition: relation.c:36
#define OidIsValid(objectId)
Definition: c.h:645
Bitmapset * attkeys
Definition: logicalproto.h:52
LogicalRepRelId remoteid
Definition: logicalproto.h:45
void CacheRegisterRelcacheCallback(RelcacheCallbackFunction func, Datum arg)
Definition: inval.c:1468
void logicalrep_relmap_update(LogicalRepRelation *remoterel)
Definition: relation.c:154
Definition: dynahash.c:208
void pfree(void *pointer)
Definition: mcxt.c:1056
#define ERROR
Definition: elog.h:43
NameData attname
Definition: pg_attribute.h:40
static int logicalrep_rel_att_by_name(LogicalRepRelation *remoterel, const char *attname)
Definition: relation.c:199
LogicalRepRelation remoterel
static void logicalrep_relmap_invalidate_cb(Datum arg, Oid reloid)
Definition: relation.c:43
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:192
#define AttrNumberGetAttrOffset(attNum)
Definition: attnum.h:51
#define NoLock
Definition: lockdefs.h:34
static MemoryContext LogicalRepRelMapContext
Definition: relation.c:33
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:200
void CheckSubscriptionRelkind(char relkind, const char *nspname, const char *relname)
LogicalRepRelMapEntry * logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode)
Definition: relation.c:219
#define ereport(elevel, rest)
Definition: elog.h:141
struct LogicalRepTyp LogicalRepTyp
#define HASH_BLOBS
Definition: hsearch.h:88
struct LogicalRepRelMapEntry LogicalRepRelMapEntry
HTAB * hash_create(const char *tabname, long nelem, HASHCTL *info, int flags)
Definition: dynahash.c:316
uintptr_t Datum
Definition: postgres.h:367
Size keysize
Definition: hsearch.h:72
#define InvalidOid
Definition: postgres_ext.h:36
int16 attnum
Definition: pg_attribute.h:79
void bms_free(Bitmapset *a)
Definition: bitmapset.c:208
#define Assert(condition)
Definition: c.h:739
AttrNumber * attnums
Definition: attmap.h:36
void * hash_seq_search(HASH_SEQ_STATUS *status)
Definition: dynahash.c:1389
char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn, bool missing_ok)
void hash_seq_init(HASH_SEQ_STATUS *status, HTAB *hashp)
Definition: dynahash.c:1379
AttrMap * make_attrmap(int maplen)
Definition: attmap.c:41
char * logicalrep_typmap_gettypname(Oid remoteid)
Definition: relation.c:435
#define FirstGenbkiObjectId
Definition: transam.h:139
void * palloc(Size size)
Definition: mcxt.c:949
int errmsg(const char *fmt,...)
Definition: elog.c:822
#define elog(elevel,...)
Definition: elog.h:228
int i
Bitmapset * RelationGetIndexAttrBitmap(Relation relation, IndexAttrBitmapKind attrKind)
Definition: relcache.c:4760
#define NameStr(name)
Definition: c.h:616
void * arg
static void logicalrep_relmap_init(void)
Definition: relation.c:84
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:226
Relation table_open(Oid relationId, LOCKMODE lockmode)
Definition: table.c:39
bool bms_is_member(int x, const Bitmapset *a)
Definition: bitmapset.c:427
RangeVar * makeRangeVar(char *schemaname, char *relname, int location)
Definition: makefuncs.c:420
static void logicalrep_typmap_free_entry(LogicalRepTyp *entry)
Definition: relation.c:391
void hash_seq_term(HASH_SEQ_STATUS *status)
Definition: dynahash.c:1465
char * nspname
Definition: logicalproto.h:59
void logicalrep_typmap_update(LogicalRepTyp *remotetyp)
Definition: relation.c:401
static HTAB * LogicalRepRelMap
Definition: relation.c:35
uint32 LogicalRepRelId
Definition: logicalproto.h:39
MemoryContext CacheMemoryContext
Definition: mcxt.c:47