PostgreSQL Source Code  git master
logicalfuncs.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * logicalfuncs.c
4  *
5  * Support functions for using logical decoding and management of
6  * logical replication slots via SQL.
7  *
8  *
9  * Copyright (c) 2012-2022, PostgreSQL Global Development Group
10  *
11  * IDENTIFICATION
12  * src/backend/replication/logical/logicalfuncs.c
13  *-------------------------------------------------------------------------
14  */
15 
16 #include "postgres.h"
17 
18 #include <unistd.h>
19 
20 #include "access/xact.h"
21 #include "access/xlog_internal.h"
22 #include "access/xlogrecovery.h"
23 #include "access/xlogutils.h"
24 #include "catalog/pg_type.h"
25 #include "fmgr.h"
26 #include "funcapi.h"
27 #include "mb/pg_wchar.h"
28 #include "miscadmin.h"
29 #include "nodes/makefuncs.h"
30 #include "replication/decode.h"
31 #include "replication/logical.h"
32 #include "replication/message.h"
33 #include "storage/fd.h"
34 #include "utils/array.h"
35 #include "utils/builtins.h"
36 #include "utils/inval.h"
37 #include "utils/lsyscache.h"
38 #include "utils/memutils.h"
39 #include "utils/pg_lsn.h"
40 #include "utils/regproc.h"
41 #include "utils/resowner.h"
42 
43 /* Private data for writing out data */
44 typedef struct DecodingOutputState
45 {
51 
52 /*
53  * Prepare for an output plugin write.
54  */
55 static void
57  bool last_write)
58 {
59  resetStringInfo(ctx->out);
60 }
61 
62 /*
63  * Perform output plugin write into tuplestore.
64  */
65 static void
67  bool last_write)
68 {
69  Datum values[3];
70  bool nulls[3];
72 
73  /* SQL Datums can only be of a limited length... */
74  if (ctx->out->len > MaxAllocSize - VARHDRSZ)
75  elog(ERROR, "too much output for sql interface");
76 
78 
79  memset(nulls, 0, sizeof(nulls));
80  values[0] = LSNGetDatum(lsn);
81  values[1] = TransactionIdGetDatum(xid);
82 
83  /*
84  * Assert ctx->out is in database encoding when we're writing textual
85  * output.
86  */
87  if (!p->binary_output)
89  ctx->out->data, ctx->out->len,
90  false));
91 
92  /* ick, but cstring_to_text_with_len works for bytea perfectly fine */
94 
96  p->returned_rows++;
97 }
98 
99 /*
100  * Helper function for the various SQL callable logical decoding functions.
101  */
102 static Datum
103 pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool binary)
104 {
105  Name name;
106  XLogRecPtr upto_lsn;
107  int32 upto_nchanges;
108  ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
109  MemoryContext per_query_ctx;
110  MemoryContext oldcontext;
111  XLogRecPtr end_of_wal;
113  ResourceOwner old_resowner = CurrentResourceOwner;
114  ArrayType *arr;
115  Size ndim;
116  List *options = NIL;
118 
120 
122 
123  if (PG_ARGISNULL(0))
124  ereport(ERROR,
125  (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
126  errmsg("slot name must not be null")));
127  name = PG_GETARG_NAME(0);
128 
129  if (PG_ARGISNULL(1))
130  upto_lsn = InvalidXLogRecPtr;
131  else
132  upto_lsn = PG_GETARG_LSN(1);
133 
134  if (PG_ARGISNULL(2))
135  upto_nchanges = InvalidXLogRecPtr;
136  else
137  upto_nchanges = PG_GETARG_INT32(2);
138 
139  if (PG_ARGISNULL(3))
140  ereport(ERROR,
141  (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
142  errmsg("options array must not be null")));
143  arr = PG_GETARG_ARRAYTYPE_P(3);
144 
145  /* state to write output to */
146  p = palloc0(sizeof(DecodingOutputState));
147 
148  p->binary_output = binary;
149 
150  per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
151  oldcontext = MemoryContextSwitchTo(per_query_ctx);
152 
153  /* Deconstruct options array */
154  ndim = ARR_NDIM(arr);
155  if (ndim > 1)
156  {
157  ereport(ERROR,
158  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
159  errmsg("array must be one-dimensional")));
160  }
161  else if (array_contains_nulls(arr))
162  {
163  ereport(ERROR,
164  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
165  errmsg("array must not contain nulls")));
166  }
167  else if (ndim == 1)
168  {
169  int nelems;
170  Datum *datum_opts;
171  int i;
172 
173  Assert(ARR_ELEMTYPE(arr) == TEXTOID);
174 
175  deconstruct_array_builtin(arr, TEXTOID, &datum_opts, NULL, &nelems);
176 
177  if (nelems % 2 != 0)
178  ereport(ERROR,
179  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
180  errmsg("array must have even number of elements")));
181 
182  for (i = 0; i < nelems; i += 2)
183  {
184  char *name = TextDatumGetCString(datum_opts[i]);
185  char *opt = TextDatumGetCString(datum_opts[i + 1]);
186 
188  }
189  }
190 
191  InitMaterializedSRF(fcinfo, 0);
192  p->tupstore = rsinfo->setResult;
193  p->tupdesc = rsinfo->setDesc;
194 
195  /*
196  * Compute the current end-of-wal.
197  */
198  if (!RecoveryInProgress())
199  end_of_wal = GetFlushRecPtr(NULL);
200  else
201  end_of_wal = GetXLogReplayRecPtr(NULL);
202 
204 
205  PG_TRY();
206  {
207  /* restart at slot's confirmed_flush */
209  options,
210  false,
211  XL_ROUTINE(.page_read = read_local_xlog_page,
212  .segment_open = wal_segment_open,
213  .segment_close = wal_segment_close),
215  LogicalOutputWrite, NULL);
216 
217  /*
218  * After the sanity checks in CreateDecodingContext, make sure the
219  * restart_lsn is valid. Avoid "cannot get changes" wording in this
220  * errmsg because that'd be confusingly ambiguous about no changes
221  * being available.
222  */
224  ereport(ERROR,
225  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
226  errmsg("can no longer get changes from replication slot \"%s\"",
227  NameStr(*name)),
228  errdetail("This slot has never previously reserved WAL, or it has been invalidated.")));
229 
230  MemoryContextSwitchTo(oldcontext);
231 
232  /*
233  * Check whether the output plugin writes textual output if that's
234  * what we need.
235  */
236  if (!binary &&
237  ctx->options.output_type !=OUTPUT_PLUGIN_TEXTUAL_OUTPUT)
238  ereport(ERROR,
239  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
240  errmsg("logical decoding output plugin \"%s\" produces binary output, but function \"%s\" expects textual data",
242  format_procedure(fcinfo->flinfo->fn_oid))));
243 
244  ctx->output_writer_private = p;
245 
246  /*
247  * Decoding of WAL must start at restart_lsn so that the entirety of
248  * xacts that committed after the slot's confirmed_flush can be
249  * accumulated into reorder buffers.
250  */
252 
253  /* invalidate non-timetravel entries */
255 
256  /* Decode until we run out of records */
257  while (ctx->reader->EndRecPtr < end_of_wal)
258  {
259  XLogRecord *record;
260  char *errm = NULL;
261 
262  record = XLogReadRecord(ctx->reader, &errm);
263  if (errm)
264  elog(ERROR, "could not find record for logical decoding: %s", errm);
265 
266  /*
267  * The {begin_txn,change,commit_txn}_wrapper callbacks above will
268  * store the description into our tuplestore.
269  */
270  if (record != NULL)
271  LogicalDecodingProcessRecord(ctx, ctx->reader);
272 
273  /* check limits */
274  if (upto_lsn != InvalidXLogRecPtr &&
275  upto_lsn <= ctx->reader->EndRecPtr)
276  break;
277  if (upto_nchanges != 0 &&
278  upto_nchanges <= p->returned_rows)
279  break;
281  }
282 
283  /*
284  * Logical decoding could have clobbered CurrentResourceOwner during
285  * transaction management, so restore the executor's value. (This is
286  * a kluge, but it's not worth cleaning up right now.)
287  */
288  CurrentResourceOwner = old_resowner;
289 
290  /*
291  * Next time, start where we left off. (Hunting things, the family
292  * business..)
293  */
294  if (ctx->reader->EndRecPtr != InvalidXLogRecPtr && confirm)
295  {
296  LogicalConfirmReceivedLocation(ctx->reader->EndRecPtr);
297 
298  /*
299  * If only the confirmed_flush_lsn has changed the slot won't get
300  * marked as dirty by the above. Callers on the walsender
301  * interface are expected to keep track of their own progress and
302  * don't need it written out. But SQL-interface users cannot
303  * specify their own start positions and it's harder for them to
304  * keep track of their progress, so we should make more of an
305  * effort to save it for them.
306  *
307  * Dirty the slot so it's written out at the next checkpoint.
308  * We'll still lose its position on crash, as documented, but it's
309  * better than always losing the position even on clean restart.
310  */
312  }
313 
314  /* free context, call shutdown callback */
315  FreeDecodingContext(ctx);
316 
319  }
320  PG_CATCH();
321  {
322  /* clear all timetravel entries */
324 
325  PG_RE_THROW();
326  }
327  PG_END_TRY();
328 
329  return (Datum) 0;
330 }
331 
332 /*
333  * SQL function returning the changestream as text, consuming the data.
334  */
335 Datum
337 {
338  return pg_logical_slot_get_changes_guts(fcinfo, true, false);
339 }
340 
341 /*
342  * SQL function returning the changestream as text, only peeking ahead.
343  */
344 Datum
346 {
347  return pg_logical_slot_get_changes_guts(fcinfo, false, false);
348 }
349 
350 /*
351  * SQL function returning the changestream in binary, consuming the data.
352  */
353 Datum
355 {
356  return pg_logical_slot_get_changes_guts(fcinfo, true, true);
357 }
358 
359 /*
360  * SQL function returning the changestream in binary, only peeking ahead.
361  */
362 Datum
364 {
365  return pg_logical_slot_get_changes_guts(fcinfo, false, true);
366 }
367 
368 
369 /*
370  * SQL function for writing logical decoding message into WAL.
371  */
372 Datum
374 {
375  bool transactional = PG_GETARG_BOOL(0);
376  char *prefix = text_to_cstring(PG_GETARG_TEXT_PP(1));
378  XLogRecPtr lsn;
379 
381  transactional);
382  PG_RETURN_LSN(lsn);
383 }
384 
385 Datum
387 {
388  /* bytea and text are compatible */
389  return pg_logical_emit_message_bytea(fcinfo);
390 }
#define ARR_NDIM(a)
Definition: array.h:283
#define PG_GETARG_ARRAYTYPE_P(n)
Definition: array.h:256
#define ARR_ELEMTYPE(a)
Definition: array.h:285
bool array_contains_nulls(ArrayType *array)
Definition: arrayfuncs.c:3712
void deconstruct_array_builtin(ArrayType *array, Oid elmtype, Datum **elemsp, bool **nullsp, int *nelemsp)
Definition: arrayfuncs.c:3642
static Datum values[MAXATTR]
Definition: bootstrap.c:156
#define TextDatumGetCString(d)
Definition: builtins.h:86
#define NameStr(name)
Definition: c.h:682
signed int int32
Definition: c.h:430
#define VARHDRSZ
Definition: c.h:628
uint32 TransactionId
Definition: c.h:588
size_t Size
Definition: c.h:541
void LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *record)
Definition: decode.c:91
int errdetail(const char *fmt,...)
Definition: elog.c:1039
int errcode(int sqlerrcode)
Definition: elog.c:695
int errmsg(const char *fmt,...)
Definition: elog.c:906
#define PG_RE_THROW()
Definition: elog.h:350
#define PG_TRY(...)
Definition: elog.h:309
#define PG_END_TRY(...)
Definition: elog.h:334
#define ERROR
Definition: elog.h:35
#define PG_CATCH(...)
Definition: elog.h:319
#define ereport(elevel,...)
Definition: elog.h:145
const char * name
Definition: encode.c:561
#define PG_GETARG_BYTEA_PP(n)
Definition: fmgr.h:308
#define PG_GETARG_TEXT_PP(n)
Definition: fmgr.h:309
#define PG_ARGISNULL(n)
Definition: fmgr.h:209
#define PG_GETARG_NAME(n)
Definition: fmgr.h:278
#define PG_GETARG_INT32(n)
Definition: fmgr.h:269
#define PG_GETARG_BOOL(n)
Definition: fmgr.h:274
#define PG_FUNCTION_ARGS
Definition: fmgr.h:193
void InitMaterializedSRF(FunctionCallInfo fcinfo, bits32 flags)
Definition: funcapi.c:76
void InvalidateSystemCaches(void)
Definition: inval.c:702
int i
Definition: isn.c:73
Assert(fmt[strlen(fmt) - 1] !='\n')
List * lappend(List *list, void *datum)
Definition: list.c:338
void LogicalConfirmReceivedLocation(XLogRecPtr lsn)
Definition: logical.c:1736
void FreeDecodingContext(LogicalDecodingContext *ctx)
Definition: logical.c:634
void CheckLogicalDecodingRequirements(void)
Definition: logical.c:103
LogicalDecodingContext * CreateDecodingContext(XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, XLogReaderRoutine *xl_routine, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress)
Definition: logical.c:479
static void LogicalOutputPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write)
Definition: logicalfuncs.c:56
Datum pg_logical_slot_get_changes(PG_FUNCTION_ARGS)
Definition: logicalfuncs.c:336
static void LogicalOutputWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write)
Definition: logicalfuncs.c:66
Datum pg_logical_emit_message_bytea(PG_FUNCTION_ARGS)
Definition: logicalfuncs.c:373
Datum pg_logical_slot_get_binary_changes(PG_FUNCTION_ARGS)
Definition: logicalfuncs.c:354
Datum pg_logical_emit_message_text(PG_FUNCTION_ARGS)
Definition: logicalfuncs.c:386
Datum pg_logical_slot_peek_binary_changes(PG_FUNCTION_ARGS)
Definition: logicalfuncs.c:363
Datum pg_logical_slot_peek_changes(PG_FUNCTION_ARGS)
Definition: logicalfuncs.c:345
struct DecodingOutputState DecodingOutputState
static Datum pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool binary)
Definition: logicalfuncs.c:103
DefElem * makeDefElem(char *name, Node *arg, int location)
Definition: makefuncs.c:546
int GetDatabaseEncoding(void)
Definition: mbutils.c:1210
bool pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError)
Definition: mbutils.c:1515
void * palloc0(Size size)
Definition: mcxt.c:1230
#define MaxAllocSize
Definition: memutils.h:40
XLogRecPtr LogLogicalMessage(const char *prefix, const char *message, size_t size, bool transactional)
Definition: message.c:46
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:121
@ OUTPUT_PLUGIN_TEXTUAL_OUTPUT
Definition: output_plugin.h:20
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:135
const void * data
#define NIL
Definition: pg_list.h:66
#define PG_GETARG_LSN(n)
Definition: pg_lsn.h:33
static Datum LSNGetDatum(XLogRecPtr X)
Definition: pg_lsn.h:28
#define PG_RETURN_LSN(x)
Definition: pg_lsn.h:34
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:670
static Datum TransactionIdGetDatum(TransactionId X)
Definition: postgres.h:620
uintptr_t Datum
Definition: postgres.h:412
#define VARDATA_ANY(PTR)
Definition: postgres.h:362
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:355
char * format_procedure(Oid procedure_oid)
Definition: regproc.c:323
ResourceOwner CurrentResourceOwner
Definition: resowner.c:146
void ReplicationSlotMarkDirty(void)
Definition: slot.c:796
void ReplicationSlotAcquire(const char *name, bool nowait)
Definition: slot.c:450
ReplicationSlot * MyReplicationSlot
Definition: slot.c:98
void CheckSlotPermissions(void)
Definition: slot.c:1141
void ReplicationSlotRelease(void)
Definition: slot.c:547
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:75
Tuplestorestate * tupstore
Definition: logicalfuncs.c:46
MemoryContext ecxt_per_query_memory
Definition: execnodes.h:254
Oid fn_oid
Definition: fmgr.h:59
fmNodePtr resultinfo
Definition: fmgr.h:89
FmgrInfo * flinfo
Definition: fmgr.h:87
Definition: pg_list.h:52
void * output_writer_private
Definition: logical.h:81
StringInfo out
Definition: logical.h:71
Definition: nodes.h:112
XLogRecPtr restart_lsn
Definition: slot.h:73
ReplicationSlotPersistentData data
Definition: slot.h:147
ExprContext * econtext
Definition: execnodes.h:324
TupleDesc setDesc
Definition: execnodes.h:332
Tuplestorestate * setResult
Definition: execnodes.h:331
Definition: c.h:677
Definition: c.h:623
void tuplestore_putvalues(Tuplestorestate *state, TupleDesc tdesc, Datum *values, bool *isnull)
Definition: tuplestore.c:750
String * makeString(char *str)
Definition: value.c:63
char * text_to_cstring(const text *t)
Definition: varlena.c:222
text * cstring_to_text_with_len(const char *s, int len)
Definition: varlena.c:201
bool RecoveryInProgress(void)
Definition: xlog.c:5912
XLogRecPtr GetFlushRecPtr(TimeLineID *insertTLI)
Definition: xlog.c:6077
#define XLogRecPtrIsInvalid(r)
Definition: xlogdefs.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
XLogRecord * XLogReadRecord(XLogReaderState *state, char **errormsg)
Definition: xlogreader.c:422
void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
Definition: xlogreader.c:264
#define XL_ROUTINE(...)
Definition: xlogreader.h:117
XLogRecPtr GetXLogReplayRecPtr(TimeLineID *replayTLI)
void wal_segment_close(XLogReaderState *state)
Definition: xlogutils.c:859
void wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p)
Definition: xlogutils.c:834
int read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
Definition: xlogutils.c:878