PostgreSQL Source Code  git master
syncrep.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * syncrep.c
4  *
5  * Synchronous replication is new as of PostgreSQL 9.1.
6  *
7  * If requested, transaction commits wait until their commit LSN are
8  * acknowledged by the synchronous standbys.
9  *
10  * This module contains the code for waiting and release of backends.
11  * All code in this module executes on the primary. The core streaming
12  * replication transport remains within WALreceiver/WALsender modules.
13  *
14  * The essence of this design is that it isolates all logic about
15  * waiting/releasing onto the primary. The primary defines which standbys
16  * it wishes to wait for. The standbys are completely unaware of the
17  * durability requirements of transactions on the primary, reducing the
18  * complexity of the code and streamlining both standby operations and
19  * network bandwidth because there is no requirement to ship
20  * per-transaction state information.
21  *
22  * Replication is either synchronous or not synchronous (async). If it is
23  * async, we just fastpath out of here. If it is sync, then we wait for
24  * the write, flush or apply location on the standby before releasing
25  * the waiting backend. Further complexity in that interaction is
26  * expected in later releases.
27  *
28  * The best performing way to manage the waiting backends is to have a
29  * single ordered queue of waiting backends, so that we can avoid
30  * searching the through all waiters each time we receive a reply.
31  *
32  * In 9.5 or before only a single standby could be considered as
33  * synchronous. In 9.6 we support a priority-based multiple synchronous
34  * standbys. In 10.0 a quorum-based multiple synchronous standbys is also
35  * supported. The number of synchronous standbys that transactions
36  * must wait for replies from is specified in synchronous_standby_names.
37  * This parameter also specifies a list of standby names and the method
38  * (FIRST and ANY) to choose synchronous standbys from the listed ones.
39  *
40  * The method FIRST specifies a priority-based synchronous replication
41  * and makes transaction commits wait until their WAL records are
42  * replicated to the requested number of synchronous standbys chosen based
43  * on their priorities. The standbys whose names appear earlier in the list
44  * are given higher priority and will be considered as synchronous.
45  * Other standby servers appearing later in this list represent potential
46  * synchronous standbys. If any of the current synchronous standbys
47  * disconnects for whatever reason, it will be replaced immediately with
48  * the next-highest-priority standby.
49  *
50  * The method ANY specifies a quorum-based synchronous replication
51  * and makes transaction commits wait until their WAL records are
52  * replicated to at least the requested number of synchronous standbys
53  * in the list. All the standbys appearing in the list are considered as
54  * candidates for quorum synchronous standbys.
55  *
56  * If neither FIRST nor ANY is specified, FIRST is used as the method.
57  * This is for backward compatibility with 9.6 or before where only a
58  * priority-based sync replication was supported.
59  *
60  * Before the standbys chosen from synchronous_standby_names can
61  * become the synchronous standbys they must have caught up with
62  * the primary; that may take some time. Once caught up,
63  * the standbys which are considered as synchronous at that moment
64  * will release waiters from the queue.
65  *
66  * Portions Copyright (c) 2010-2023, PostgreSQL Global Development Group
67  *
68  * IDENTIFICATION
69  * src/backend/replication/syncrep.c
70  *
71  *-------------------------------------------------------------------------
72  */
73 #include "postgres.h"
74 
75 #include <unistd.h>
76 
77 #include "access/xact.h"
78 #include "miscadmin.h"
79 #include "pgstat.h"
80 #include "replication/syncrep.h"
81 #include "replication/walsender.h"
83 #include "storage/pmsignal.h"
84 #include "storage/proc.h"
85 #include "tcop/tcopprot.h"
86 #include "utils/builtins.h"
87 #include "utils/guc_hooks.h"
88 #include "utils/ps_status.h"
89 
90 /* User-settable parameters for sync rep */
92 
93 #define SyncStandbysDefined() \
94  (SyncRepStandbyNames != NULL && SyncRepStandbyNames[0] != '\0')
95 
96 static bool announce_next_takeover = true;
97 
100 
101 static void SyncRepQueueInsert(int mode);
102 static void SyncRepCancelWait(void);
103 static int SyncRepWakeQueue(bool all, int mode);
104 
105 static bool SyncRepGetSyncRecPtr(XLogRecPtr *writePtr,
106  XLogRecPtr *flushPtr,
107  XLogRecPtr *applyPtr,
108  bool *am_sync);
109 static void SyncRepGetOldestSyncRecPtr(XLogRecPtr *writePtr,
110  XLogRecPtr *flushPtr,
111  XLogRecPtr *applyPtr,
112  SyncRepStandbyData *sync_standbys,
113  int num_standbys);
114 static void SyncRepGetNthLatestSyncRecPtr(XLogRecPtr *writePtr,
115  XLogRecPtr *flushPtr,
116  XLogRecPtr *applyPtr,
117  SyncRepStandbyData *sync_standbys,
118  int num_standbys,
119  uint8 nth);
120 static int SyncRepGetStandbyPriority(void);
121 static int standby_priority_comparator(const void *a, const void *b);
122 static int cmp_lsn(const void *a, const void *b);
123 
124 #ifdef USE_ASSERT_CHECKING
125 static bool SyncRepQueueIsOrderedByLSN(int mode);
126 #endif
127 
128 /*
129  * ===========================================================
130  * Synchronous Replication functions for normal user backends
131  * ===========================================================
132  */
133 
134 /*
135  * Wait for synchronous replication, if requested by user.
136  *
137  * Initially backends start in state SYNC_REP_NOT_WAITING and then
138  * change that state to SYNC_REP_WAITING before adding ourselves
139  * to the wait queue. During SyncRepWakeQueue() a WALSender changes
140  * the state to SYNC_REP_WAIT_COMPLETE once replication is confirmed.
141  * This backend then resets its state to SYNC_REP_NOT_WAITING.
142  *
143  * 'lsn' represents the LSN to wait for. 'commit' indicates whether this LSN
144  * represents a commit record. If it doesn't, then we wait only for the WAL
145  * to be flushed if synchronous_commit is set to the higher level of
146  * remote_apply, because only commit records provide apply feedback.
147  */
148 void
149 SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
150 {
151  char *new_status = NULL;
152  const char *old_status;
153  int mode;
154 
155  /*
156  * This should be called while holding interrupts during a transaction
157  * commit to prevent the follow-up shared memory queue cleanups to be
158  * influenced by external interruptions.
159  */
161 
162  /*
163  * Fast exit if user has not requested sync replication, or there are no
164  * sync replication standby names defined.
165  *
166  * Since this routine gets called every commit time, it's important to
167  * exit quickly if sync replication is not requested. So we check
168  * WalSndCtl->sync_standbys_defined flag without the lock and exit
169  * immediately if it's false. If it's true, we need to check it again
170  * later while holding the lock, to check the flag and operate the sync
171  * rep queue atomically. This is necessary to avoid the race condition
172  * described in SyncRepUpdateSyncStandbysDefined(). On the other hand, if
173  * it's false, the lock is not necessary because we don't touch the queue.
174  */
175  if (!SyncRepRequested() ||
176  !((volatile WalSndCtlData *) WalSndCtl)->sync_standbys_defined)
177  return;
178 
179  /* Cap the level for anything other than commit to remote flush only. */
180  if (commit)
182  else
184 
186  Assert(WalSndCtl != NULL);
187 
188  LWLockAcquire(SyncRepLock, LW_EXCLUSIVE);
190 
191  /*
192  * We don't wait for sync rep if WalSndCtl->sync_standbys_defined is not
193  * set. See SyncRepUpdateSyncStandbysDefined.
194  *
195  * Also check that the standby hasn't already replied. Unlikely race
196  * condition but we'll be fetching that cache line anyway so it's likely
197  * to be a low cost check.
198  */
200  lsn <= WalSndCtl->lsn[mode])
201  {
202  LWLockRelease(SyncRepLock);
203  return;
204  }
205 
206  /*
207  * Set our waitLSN so WALSender will know when to wake us, and add
208  * ourselves to the queue.
209  */
210  MyProc->waitLSN = lsn;
213  Assert(SyncRepQueueIsOrderedByLSN(mode));
214  LWLockRelease(SyncRepLock);
215 
216  /* Alter ps display to show waiting for sync rep. */
218  {
219  int len;
220 
221  old_status = get_ps_display(&len);
222  new_status = (char *) palloc(len + 32 + 1);
223  memcpy(new_status, old_status, len);
224  sprintf(new_status + len, " waiting for %X/%X",
225  LSN_FORMAT_ARGS(lsn));
226  set_ps_display(new_status);
227  new_status[len] = '\0'; /* truncate off " waiting ..." */
228  }
229 
230  /*
231  * Wait for specified LSN to be confirmed.
232  *
233  * Each proc has its own wait latch, so we perform a normal latch
234  * check/wait loop here.
235  */
236  for (;;)
237  {
238  int rc;
239 
240  /* Must reset the latch before testing state. */
242 
243  /*
244  * Acquiring the lock is not needed, the latch ensures proper
245  * barriers. If it looks like we're done, we must really be done,
246  * because once walsender changes the state to SYNC_REP_WAIT_COMPLETE,
247  * it will never update it again, so we can't be seeing a stale value
248  * in that case.
249  */
251  break;
252 
253  /*
254  * If a wait for synchronous replication is pending, we can neither
255  * acknowledge the commit nor raise ERROR or FATAL. The latter would
256  * lead the client to believe that the transaction aborted, which is
257  * not true: it's already committed locally. The former is no good
258  * either: the client has requested synchronous replication, and is
259  * entitled to assume that an acknowledged commit is also replicated,
260  * which might not be true. So in this case we issue a WARNING (which
261  * some clients may be able to interpret) and shut off further output.
262  * We do NOT reset ProcDiePending, so that the process will die after
263  * the commit is cleaned up.
264  */
265  if (ProcDiePending)
266  {
268  (errcode(ERRCODE_ADMIN_SHUTDOWN),
269  errmsg("canceling the wait for synchronous replication and terminating connection due to administrator command"),
270  errdetail("The transaction has already committed locally, but might not have been replicated to the standby.")));
273  break;
274  }
275 
276  /*
277  * It's unclear what to do if a query cancel interrupt arrives. We
278  * can't actually abort at this point, but ignoring the interrupt
279  * altogether is not helpful, so we just terminate the wait with a
280  * suitable warning.
281  */
282  if (QueryCancelPending)
283  {
284  QueryCancelPending = false;
286  (errmsg("canceling wait for synchronous replication due to user request"),
287  errdetail("The transaction has already committed locally, but might not have been replicated to the standby.")));
289  break;
290  }
291 
292  /*
293  * Wait on latch. Any condition that should wake us up will set the
294  * latch, so no need for timeout.
295  */
298 
299  /*
300  * If the postmaster dies, we'll probably never get an acknowledgment,
301  * because all the wal sender processes will exit. So just bail out.
302  */
303  if (rc & WL_POSTMASTER_DEATH)
304  {
305  ProcDiePending = true;
308  break;
309  }
310  }
311 
312  /*
313  * WalSender has checked our LSN and has removed us from queue. Clean up
314  * state and leave. It's OK to reset these shared memory fields without
315  * holding SyncRepLock, because any walsenders will ignore us anyway when
316  * we're not on the queue. We need a read barrier to make sure we see the
317  * changes to the queue link (this might be unnecessary without
318  * assertions, but better safe than sorry).
319  */
320  pg_read_barrier();
323  MyProc->waitLSN = 0;
324 
325  if (new_status)
326  {
327  /* Reset ps display */
328  set_ps_display(new_status);
329  pfree(new_status);
330  }
331 }
332 
333 /*
334  * Insert MyProc into the specified SyncRepQueue, maintaining sorted invariant.
335  *
336  * Usually we will go at tail of queue, though it's possible that we arrive
337  * here out of order, so start at tail and work back to insertion point.
338  */
339 static void
341 {
342  dlist_head *queue;
343  dlist_iter iter;
344 
346  queue = &WalSndCtl->SyncRepQueue[mode];
347 
348  dlist_reverse_foreach(iter, queue)
349  {
350  PGPROC *proc = dlist_container(PGPROC, syncRepLinks, iter.cur);
351 
352  /*
353  * Stop at the queue element that we should insert after to ensure the
354  * queue is ordered by LSN.
355  */
356  if (proc->waitLSN < MyProc->waitLSN)
357  {
359  return;
360  }
361  }
362 
363  /*
364  * If we get here, the list was either empty, or this process needs to be
365  * at the head.
366  */
368 }
369 
370 /*
371  * Acquire SyncRepLock and cancel any wait currently in progress.
372  */
373 static void
375 {
376  LWLockAcquire(SyncRepLock, LW_EXCLUSIVE);
380  LWLockRelease(SyncRepLock);
381 }
382 
383 void
385 {
386  /*
387  * First check if we are removed from the queue without the lock to not
388  * slow down backend exit.
389  */
391  {
392  LWLockAcquire(SyncRepLock, LW_EXCLUSIVE);
393 
394  /* maybe we have just been removed, so recheck */
397 
398  LWLockRelease(SyncRepLock);
399  }
400 }
401 
402 /*
403  * ===========================================================
404  * Synchronous Replication functions for wal sender processes
405  * ===========================================================
406  */
407 
408 /*
409  * Take any action required to initialise sync rep state from config
410  * data. Called at WALSender startup and after each SIGHUP.
411  */
412 void
414 {
415  int priority;
416 
417  /*
418  * Determine if we are a potential sync standby and remember the result
419  * for handling replies from standby.
420  */
421  priority = SyncRepGetStandbyPriority();
422  if (MyWalSnd->sync_standby_priority != priority)
423  {
425  MyWalSnd->sync_standby_priority = priority;
427 
428  ereport(DEBUG1,
429  (errmsg_internal("standby \"%s\" now has synchronous standby priority %u",
430  application_name, priority)));
431  }
432 }
433 
434 /*
435  * Update the LSNs on each queue based upon our latest state. This
436  * implements a simple policy of first-valid-sync-standby-releases-waiter.
437  *
438  * Other policies are possible, which would change what we do here and
439  * perhaps also which information we store as well.
440  */
441 void
443 {
444  volatile WalSndCtlData *walsndctl = WalSndCtl;
445  XLogRecPtr writePtr;
446  XLogRecPtr flushPtr;
447  XLogRecPtr applyPtr;
448  bool got_recptr;
449  bool am_sync;
450  int numwrite = 0;
451  int numflush = 0;
452  int numapply = 0;
453 
454  /*
455  * If this WALSender is serving a standby that is not on the list of
456  * potential sync standbys then we have nothing to do. If we are still
457  * starting up, still running base backup or the current flush position is
458  * still invalid, then leave quickly also. Streaming or stopping WAL
459  * senders are allowed to release waiters.
460  */
461  if (MyWalSnd->sync_standby_priority == 0 ||
465  {
466  announce_next_takeover = true;
467  return;
468  }
469 
470  /*
471  * We're a potential sync standby. Release waiters if there are enough
472  * sync standbys and we are considered as sync.
473  */
474  LWLockAcquire(SyncRepLock, LW_EXCLUSIVE);
475 
476  /*
477  * Check whether we are a sync standby or not, and calculate the synced
478  * positions among all sync standbys. (Note: although this step does not
479  * of itself require holding SyncRepLock, it seems like a good idea to do
480  * it after acquiring the lock. This ensures that the WAL pointers we use
481  * to release waiters are newer than any previous execution of this
482  * routine used.)
483  */
484  got_recptr = SyncRepGetSyncRecPtr(&writePtr, &flushPtr, &applyPtr, &am_sync);
485 
486  /*
487  * If we are managing a sync standby, though we weren't prior to this,
488  * then announce we are now a sync standby.
489  */
490  if (announce_next_takeover && am_sync)
491  {
492  announce_next_takeover = false;
493 
495  ereport(LOG,
496  (errmsg("standby \"%s\" is now a synchronous standby with priority %u",
498  else
499  ereport(LOG,
500  (errmsg("standby \"%s\" is now a candidate for quorum synchronous standby",
501  application_name)));
502  }
503 
504  /*
505  * If the number of sync standbys is less than requested or we aren't
506  * managing a sync standby then just leave.
507  */
508  if (!got_recptr || !am_sync)
509  {
510  LWLockRelease(SyncRepLock);
511  announce_next_takeover = !am_sync;
512  return;
513  }
514 
515  /*
516  * Set the lsn first so that when we wake backends they will release up to
517  * this location.
518  */
519  if (walsndctl->lsn[SYNC_REP_WAIT_WRITE] < writePtr)
520  {
521  walsndctl->lsn[SYNC_REP_WAIT_WRITE] = writePtr;
522  numwrite = SyncRepWakeQueue(false, SYNC_REP_WAIT_WRITE);
523  }
524  if (walsndctl->lsn[SYNC_REP_WAIT_FLUSH] < flushPtr)
525  {
526  walsndctl->lsn[SYNC_REP_WAIT_FLUSH] = flushPtr;
527  numflush = SyncRepWakeQueue(false, SYNC_REP_WAIT_FLUSH);
528  }
529  if (walsndctl->lsn[SYNC_REP_WAIT_APPLY] < applyPtr)
530  {
531  walsndctl->lsn[SYNC_REP_WAIT_APPLY] = applyPtr;
532  numapply = SyncRepWakeQueue(false, SYNC_REP_WAIT_APPLY);
533  }
534 
535  LWLockRelease(SyncRepLock);
536 
537  elog(DEBUG3, "released %d procs up to write %X/%X, %d procs up to flush %X/%X, %d procs up to apply %X/%X",
538  numwrite, LSN_FORMAT_ARGS(writePtr),
539  numflush, LSN_FORMAT_ARGS(flushPtr),
540  numapply, LSN_FORMAT_ARGS(applyPtr));
541 }
542 
543 /*
544  * Calculate the synced Write, Flush and Apply positions among sync standbys.
545  *
546  * Return false if the number of sync standbys is less than
547  * synchronous_standby_names specifies. Otherwise return true and
548  * store the positions into *writePtr, *flushPtr and *applyPtr.
549  *
550  * On return, *am_sync is set to true if this walsender is connecting to
551  * sync standby. Otherwise it's set to false.
552  */
553 static bool
555  XLogRecPtr *applyPtr, bool *am_sync)
556 {
557  SyncRepStandbyData *sync_standbys;
558  int num_standbys;
559  int i;
560 
561  /* Initialize default results */
562  *writePtr = InvalidXLogRecPtr;
563  *flushPtr = InvalidXLogRecPtr;
564  *applyPtr = InvalidXLogRecPtr;
565  *am_sync = false;
566 
567  /* Quick out if not even configured to be synchronous */
568  if (SyncRepConfig == NULL)
569  return false;
570 
571  /* Get standbys that are considered as synchronous at this moment */
572  num_standbys = SyncRepGetCandidateStandbys(&sync_standbys);
573 
574  /* Am I among the candidate sync standbys? */
575  for (i = 0; i < num_standbys; i++)
576  {
577  if (sync_standbys[i].is_me)
578  {
579  *am_sync = true;
580  break;
581  }
582  }
583 
584  /*
585  * Nothing more to do if we are not managing a sync standby or there are
586  * not enough synchronous standbys.
587  */
588  if (!(*am_sync) ||
589  num_standbys < SyncRepConfig->num_sync)
590  {
591  pfree(sync_standbys);
592  return false;
593  }
594 
595  /*
596  * In a priority-based sync replication, the synced positions are the
597  * oldest ones among sync standbys. In a quorum-based, they are the Nth
598  * latest ones.
599  *
600  * SyncRepGetNthLatestSyncRecPtr() also can calculate the oldest
601  * positions. But we use SyncRepGetOldestSyncRecPtr() for that calculation
602  * because it's a bit more efficient.
603  *
604  * XXX If the numbers of current and requested sync standbys are the same,
605  * we can use SyncRepGetOldestSyncRecPtr() to calculate the synced
606  * positions even in a quorum-based sync replication.
607  */
609  {
610  SyncRepGetOldestSyncRecPtr(writePtr, flushPtr, applyPtr,
611  sync_standbys, num_standbys);
612  }
613  else
614  {
615  SyncRepGetNthLatestSyncRecPtr(writePtr, flushPtr, applyPtr,
616  sync_standbys, num_standbys,
618  }
619 
620  pfree(sync_standbys);
621  return true;
622 }
623 
624 /*
625  * Calculate the oldest Write, Flush and Apply positions among sync standbys.
626  */
627 static void
629  XLogRecPtr *flushPtr,
630  XLogRecPtr *applyPtr,
631  SyncRepStandbyData *sync_standbys,
632  int num_standbys)
633 {
634  int i;
635 
636  /*
637  * Scan through all sync standbys and calculate the oldest Write, Flush
638  * and Apply positions. We assume *writePtr et al were initialized to
639  * InvalidXLogRecPtr.
640  */
641  for (i = 0; i < num_standbys; i++)
642  {
643  XLogRecPtr write = sync_standbys[i].write;
644  XLogRecPtr flush = sync_standbys[i].flush;
645  XLogRecPtr apply = sync_standbys[i].apply;
646 
647  if (XLogRecPtrIsInvalid(*writePtr) || *writePtr > write)
648  *writePtr = write;
649  if (XLogRecPtrIsInvalid(*flushPtr) || *flushPtr > flush)
650  *flushPtr = flush;
651  if (XLogRecPtrIsInvalid(*applyPtr) || *applyPtr > apply)
652  *applyPtr = apply;
653  }
654 }
655 
656 /*
657  * Calculate the Nth latest Write, Flush and Apply positions among sync
658  * standbys.
659  */
660 static void
662  XLogRecPtr *flushPtr,
663  XLogRecPtr *applyPtr,
664  SyncRepStandbyData *sync_standbys,
665  int num_standbys,
666  uint8 nth)
667 {
668  XLogRecPtr *write_array;
669  XLogRecPtr *flush_array;
670  XLogRecPtr *apply_array;
671  int i;
672 
673  /* Should have enough candidates, or somebody messed up */
674  Assert(nth > 0 && nth <= num_standbys);
675 
676  write_array = (XLogRecPtr *) palloc(sizeof(XLogRecPtr) * num_standbys);
677  flush_array = (XLogRecPtr *) palloc(sizeof(XLogRecPtr) * num_standbys);
678  apply_array = (XLogRecPtr *) palloc(sizeof(XLogRecPtr) * num_standbys);
679 
680  for (i = 0; i < num_standbys; i++)
681  {
682  write_array[i] = sync_standbys[i].write;
683  flush_array[i] = sync_standbys[i].flush;
684  apply_array[i] = sync_standbys[i].apply;
685  }
686 
687  /* Sort each array in descending order */
688  qsort(write_array, num_standbys, sizeof(XLogRecPtr), cmp_lsn);
689  qsort(flush_array, num_standbys, sizeof(XLogRecPtr), cmp_lsn);
690  qsort(apply_array, num_standbys, sizeof(XLogRecPtr), cmp_lsn);
691 
692  /* Get Nth latest Write, Flush, Apply positions */
693  *writePtr = write_array[nth - 1];
694  *flushPtr = flush_array[nth - 1];
695  *applyPtr = apply_array[nth - 1];
696 
697  pfree(write_array);
698  pfree(flush_array);
699  pfree(apply_array);
700 }
701 
702 /*
703  * Compare lsn in order to sort array in descending order.
704  */
705 static int
706 cmp_lsn(const void *a, const void *b)
707 {
708  XLogRecPtr lsn1 = *((const XLogRecPtr *) a);
709  XLogRecPtr lsn2 = *((const XLogRecPtr *) b);
710 
711  if (lsn1 > lsn2)
712  return -1;
713  else if (lsn1 == lsn2)
714  return 0;
715  else
716  return 1;
717 }
718 
719 /*
720  * Return data about walsenders that are candidates to be sync standbys.
721  *
722  * *standbys is set to a palloc'd array of structs of per-walsender data,
723  * and the number of valid entries (candidate sync senders) is returned.
724  * (This might be more or fewer than num_sync; caller must check.)
725  */
726 int
728 {
729  int i;
730  int n;
731 
732  /* Create result array */
733  *standbys = (SyncRepStandbyData *)
735 
736  /* Quick exit if sync replication is not requested */
737  if (SyncRepConfig == NULL)
738  return 0;
739 
740  /* Collect raw data from shared memory */
741  n = 0;
742  for (i = 0; i < max_wal_senders; i++)
743  {
744  volatile WalSnd *walsnd; /* Use volatile pointer to prevent code
745  * rearrangement */
746  SyncRepStandbyData *stby;
747  WalSndState state; /* not included in SyncRepStandbyData */
748 
749  walsnd = &WalSndCtl->walsnds[i];
750  stby = *standbys + n;
751 
752  SpinLockAcquire(&walsnd->mutex);
753  stby->pid = walsnd->pid;
754  state = walsnd->state;
755  stby->write = walsnd->write;
756  stby->flush = walsnd->flush;
757  stby->apply = walsnd->apply;
759  SpinLockRelease(&walsnd->mutex);
760 
761  /* Must be active */
762  if (stby->pid == 0)
763  continue;
764 
765  /* Must be streaming or stopping */
766  if (state != WALSNDSTATE_STREAMING &&
768  continue;
769 
770  /* Must be synchronous */
771  if (stby->sync_standby_priority == 0)
772  continue;
773 
774  /* Must have a valid flush position */
775  if (XLogRecPtrIsInvalid(stby->flush))
776  continue;
777 
778  /* OK, it's a candidate */
779  stby->walsnd_index = i;
780  stby->is_me = (walsnd == MyWalSnd);
781  n++;
782  }
783 
784  /*
785  * In quorum mode, we return all the candidates. In priority mode, if we
786  * have too many candidates then return only the num_sync ones of highest
787  * priority.
788  */
790  n > SyncRepConfig->num_sync)
791  {
792  /* Sort by priority ... */
793  qsort(*standbys, n, sizeof(SyncRepStandbyData),
795  /* ... then report just the first num_sync ones */
796  n = SyncRepConfig->num_sync;
797  }
798 
799  return n;
800 }
801 
802 /*
803  * qsort comparator to sort SyncRepStandbyData entries by priority
804  */
805 static int
806 standby_priority_comparator(const void *a, const void *b)
807 {
808  const SyncRepStandbyData *sa = (const SyncRepStandbyData *) a;
809  const SyncRepStandbyData *sb = (const SyncRepStandbyData *) b;
810 
811  /* First, sort by increasing priority value */
812  if (sa->sync_standby_priority != sb->sync_standby_priority)
813  return sa->sync_standby_priority - sb->sync_standby_priority;
814 
815  /*
816  * We might have equal priority values; arbitrarily break ties by position
817  * in the WALSnd array. (This is utterly bogus, since that is arrival
818  * order dependent, but there are regression tests that rely on it.)
819  */
820  return sa->walsnd_index - sb->walsnd_index;
821 }
822 
823 
824 /*
825  * Check if we are in the list of sync standbys, and if so, determine
826  * priority sequence. Return priority if set, or zero to indicate that
827  * we are not a potential sync standby.
828  *
829  * Compare the parameter SyncRepStandbyNames against the application_name
830  * for this WALSender, or allow any name if we find a wildcard "*".
831  */
832 static int
834 {
835  const char *standby_name;
836  int priority;
837  bool found = false;
838 
839  /*
840  * Since synchronous cascade replication is not allowed, we always set the
841  * priority of cascading walsender to zero.
842  */
844  return 0;
845 
846  if (!SyncStandbysDefined() || SyncRepConfig == NULL)
847  return 0;
848 
849  standby_name = SyncRepConfig->member_names;
850  for (priority = 1; priority <= SyncRepConfig->nmembers; priority++)
851  {
852  if (pg_strcasecmp(standby_name, application_name) == 0 ||
853  strcmp(standby_name, "*") == 0)
854  {
855  found = true;
856  break;
857  }
858  standby_name += strlen(standby_name) + 1;
859  }
860 
861  if (!found)
862  return 0;
863 
864  /*
865  * In quorum-based sync replication, all the standbys in the list have the
866  * same priority, one.
867  */
868  return (SyncRepConfig->syncrep_method == SYNC_REP_PRIORITY) ? priority : 1;
869 }
870 
871 /*
872  * Walk the specified queue from head. Set the state of any backends that
873  * need to be woken, remove them from the queue, and then wake them.
874  * Pass all = true to wake whole queue; otherwise, just wake up to
875  * the walsender's LSN.
876  *
877  * The caller must hold SyncRepLock in exclusive mode.
878  */
879 static int
880 SyncRepWakeQueue(bool all, int mode)
881 {
882  volatile WalSndCtlData *walsndctl = WalSndCtl;
883  int numprocs = 0;
884  dlist_mutable_iter iter;
885 
888  Assert(SyncRepQueueIsOrderedByLSN(mode));
889 
891  {
892  PGPROC *proc = dlist_container(PGPROC, syncRepLinks, iter.cur);
893 
894  /*
895  * Assume the queue is ordered by LSN
896  */
897  if (!all && walsndctl->lsn[mode] < proc->waitLSN)
898  return numprocs;
899 
900  /*
901  * Remove from queue.
902  */
904 
905  /*
906  * SyncRepWaitForLSN() reads syncRepState without holding the lock, so
907  * make sure that it sees the queue link being removed before the
908  * syncRepState change.
909  */
911 
912  /*
913  * Set state to complete; see SyncRepWaitForLSN() for discussion of
914  * the various states.
915  */
917 
918  /*
919  * Wake only when we have set state and removed from queue.
920  */
921  SetLatch(&(proc->procLatch));
922 
923  numprocs++;
924  }
925 
926  return numprocs;
927 }
928 
929 /*
930  * The checkpointer calls this as needed to update the shared
931  * sync_standbys_defined flag, so that backends don't remain permanently wedged
932  * if synchronous_standby_names is unset. It's safe to check the current value
933  * without the lock, because it's only ever updated by one process. But we
934  * must take the lock to change it.
935  */
936 void
938 {
939  bool sync_standbys_defined = SyncStandbysDefined();
940 
941  if (sync_standbys_defined != WalSndCtl->sync_standbys_defined)
942  {
943  LWLockAcquire(SyncRepLock, LW_EXCLUSIVE);
944 
945  /*
946  * If synchronous_standby_names has been reset to empty, it's futile
947  * for backends to continue waiting. Since the user no longer wants
948  * synchronous replication, we'd better wake them up.
949  */
950  if (!sync_standbys_defined)
951  {
952  int i;
953 
954  for (i = 0; i < NUM_SYNC_REP_WAIT_MODE; i++)
955  SyncRepWakeQueue(true, i);
956  }
957 
958  /*
959  * Only allow people to join the queue when there are synchronous
960  * standbys defined. Without this interlock, there's a race
961  * condition: we might wake up all the current waiters; then, some
962  * backend that hasn't yet reloaded its config might go to sleep on
963  * the queue (and never wake up). This prevents that.
964  */
965  WalSndCtl->sync_standbys_defined = sync_standbys_defined;
966 
967  LWLockRelease(SyncRepLock);
968  }
969 }
970 
971 #ifdef USE_ASSERT_CHECKING
972 static bool
973 SyncRepQueueIsOrderedByLSN(int mode)
974 {
975  XLogRecPtr lastLSN;
976  dlist_iter iter;
977 
979 
980  lastLSN = 0;
981 
983  {
984  PGPROC *proc = dlist_container(PGPROC, syncRepLinks, iter.cur);
985 
986  /*
987  * Check the queue is ordered by LSN and that multiple procs don't
988  * have matching LSNs
989  */
990  if (proc->waitLSN <= lastLSN)
991  return false;
992 
993  lastLSN = proc->waitLSN;
994  }
995 
996  return true;
997 }
998 #endif
999 
1000 /*
1001  * ===========================================================
1002  * Synchronous Replication functions executed by any process
1003  * ===========================================================
1004  */
1005 
1006 bool
1008 {
1009  if (*newval != NULL && (*newval)[0] != '\0')
1010  {
1011  int parse_rc;
1012  SyncRepConfigData *pconf;
1013 
1014  /* Reset communication variables to ensure a fresh start */
1015  syncrep_parse_result = NULL;
1016  syncrep_parse_error_msg = NULL;
1017 
1018  /* Parse the synchronous_standby_names string */
1020  parse_rc = syncrep_yyparse();
1022 
1023  if (parse_rc != 0 || syncrep_parse_result == NULL)
1024  {
1025  GUC_check_errcode(ERRCODE_SYNTAX_ERROR);
1028  else
1029  GUC_check_errdetail("synchronous_standby_names parser failed");
1030  return false;
1031  }
1032 
1033  if (syncrep_parse_result->num_sync <= 0)
1034  {
1035  GUC_check_errmsg("number of synchronous standbys (%d) must be greater than zero",
1037  return false;
1038  }
1039 
1040  /* GUC extra value must be guc_malloc'd, not palloc'd */
1041  pconf = (SyncRepConfigData *)
1043  if (pconf == NULL)
1044  return false;
1046 
1047  *extra = (void *) pconf;
1048 
1049  /*
1050  * We need not explicitly clean up syncrep_parse_result. It, and any
1051  * other cruft generated during parsing, will be freed when the
1052  * current memory context is deleted. (This code is generally run in
1053  * a short-lived context used for config file processing, so that will
1054  * not be very long.)
1055  */
1056  }
1057  else
1058  *extra = NULL;
1059 
1060  return true;
1061 }
1062 
1063 void
1064 assign_synchronous_standby_names(const char *newval, void *extra)
1065 {
1066  SyncRepConfig = (SyncRepConfigData *) extra;
1067 }
1068 
1069 void
1071 {
1072  switch (newval)
1073  {
1076  break;
1079  break;
1082  break;
1083  default:
1085  break;
1086  }
1087 }
#define pg_read_barrier()
Definition: atomics.h:153
#define pg_write_barrier()
Definition: atomics.h:154
#define Min(x, y)
Definition: c.h:988
unsigned char uint8
Definition: c.h:488
@ DestNone
Definition: dest.h:87
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1156
int errdetail(const char *fmt,...)
Definition: elog.c:1202
int errcode(int sqlerrcode)
Definition: elog.c:858
int errmsg(const char *fmt,...)
Definition: elog.c:1069
#define LOG
Definition: elog.h:31
#define DEBUG3
Definition: elog.h:28
#define WARNING
Definition: elog.h:36
#define DEBUG1
Definition: elog.h:30
#define ereport(elevel,...)
Definition: elog.h:149
volatile uint32 InterruptHoldoffCount
Definition: globals.c:40
volatile sig_atomic_t QueryCancelPending
Definition: globals.c:31
struct Latch * MyLatch
Definition: globals.c:58
volatile sig_atomic_t ProcDiePending
Definition: globals.c:32
void GUC_check_errcode(int sqlerrcode)
Definition: guc.c:6680
void * guc_malloc(int elevel, size_t size)
Definition: guc.c:630
#define newval
#define GUC_check_errmsg
Definition: guc.h:433
#define GUC_check_errdetail
Definition: guc.h:437
GucSource
Definition: guc.h:108
char * application_name
Definition: guc_tables.c:516
static void dlist_insert_after(dlist_node *after, dlist_node *node)
Definition: ilist.h:381
#define dlist_foreach(iter, lhead)
Definition: ilist.h:623
static void dlist_delete_thoroughly(dlist_node *node)
Definition: ilist.h:416
static bool dlist_node_is_detached(const dlist_node *node)
Definition: ilist.h:525
#define dlist_reverse_foreach(iter, lhead)
Definition: ilist.h:654
static void dlist_push_head(dlist_head *head, dlist_node *node)
Definition: ilist.h:347
#define dlist_foreach_modify(iter, lhead)
Definition: ilist.h:640
#define dlist_container(type, membername, ptr)
Definition: ilist.h:593
#define write(a, b, c)
Definition: win32.h:14
int b
Definition: isn.c:70
int a
Definition: isn.c:69
int i
Definition: isn.c:73
void SetLatch(Latch *latch)
Definition: latch.c:607
void ResetLatch(Latch *latch)
Definition: latch.c:699
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:492
#define WL_LATCH_SET
Definition: latch.h:125
#define WL_POSTMASTER_DEATH
Definition: latch.h:129
Assert(fmt[strlen(fmt) - 1] !='\n')
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1195
bool LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1963
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1803
@ LW_EXCLUSIVE
Definition: lwlock.h:115
void pfree(void *pointer)
Definition: mcxt.c:1436
void * palloc(Size size)
Definition: mcxt.c:1210
static PgChecksumMode mode
Definition: pg_checksums.c:65
const void size_t len
static rewind_source * source
Definition: pg_rewind.c:81
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
#define sprintf
Definition: port.h:240
#define qsort(a, b, c, d)
Definition: port.h:445
CommandDest whereToSendOutput
Definition: postgres.c:84
const char * get_ps_display(int *displen)
Definition: ps_status.c:414
bool update_process_title
Definition: ps_status.c:35
void set_ps_display(const char *activity)
Definition: ps_status.c:342
#define SpinLockRelease(lock)
Definition: spin.h:64
#define SpinLockAcquire(lock)
Definition: spin.h:62
PGPROC * MyProc
Definition: proc.c:66
Definition: proc.h:162
XLogRecPtr waitLSN
Definition: proc.h:243
dlist_node syncRepLinks
Definition: proc.h:245
int syncRepState
Definition: proc.h:244
Latch procLatch
Definition: proc.h:170
uint8 syncrep_method
Definition: syncrep.h:68
char member_names[FLEXIBLE_ARRAY_MEMBER]
Definition: syncrep.h:71
int sync_standby_priority
Definition: syncrep.h:49
XLogRecPtr apply
Definition: syncrep.h:48
XLogRecPtr write
Definition: syncrep.h:46
XLogRecPtr flush
Definition: syncrep.h:47
XLogRecPtr lsn[NUM_SYNC_REP_WAIT_MODE]
WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]
dlist_head SyncRepQueue[NUM_SYNC_REP_WAIT_MODE]
slock_t mutex
XLogRecPtr flush
WalSndState state
XLogRecPtr write
int sync_standby_priority
XLogRecPtr apply
dlist_node * cur
Definition: ilist.h:179
dlist_node * cur
Definition: ilist.h:200
Definition: regguts.h:318
static int SyncRepWaitMode
Definition: syncrep.c:99
void SyncRepInitConfig(void)
Definition: syncrep.c:413
void SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
Definition: syncrep.c:149
static bool SyncRepGetSyncRecPtr(XLogRecPtr *writePtr, XLogRecPtr *flushPtr, XLogRecPtr *applyPtr, bool *am_sync)
Definition: syncrep.c:554
static void SyncRepGetNthLatestSyncRecPtr(XLogRecPtr *writePtr, XLogRecPtr *flushPtr, XLogRecPtr *applyPtr, SyncRepStandbyData *sync_standbys, int num_standbys, uint8 nth)
Definition: syncrep.c:661
void assign_synchronous_commit(int newval, void *extra)
Definition: syncrep.c:1070
void assign_synchronous_standby_names(const char *newval, void *extra)
Definition: syncrep.c:1064
static int standby_priority_comparator(const void *a, const void *b)
Definition: syncrep.c:806
static int SyncRepWakeQueue(bool all, int mode)
Definition: syncrep.c:880
SyncRepConfigData * SyncRepConfig
Definition: syncrep.c:98
int SyncRepGetCandidateStandbys(SyncRepStandbyData **standbys)
Definition: syncrep.c:727
void SyncRepReleaseWaiters(void)
Definition: syncrep.c:442
void SyncRepUpdateSyncStandbysDefined(void)
Definition: syncrep.c:937
static bool announce_next_takeover
Definition: syncrep.c:96
static int SyncRepGetStandbyPriority(void)
Definition: syncrep.c:833
char * SyncRepStandbyNames
Definition: syncrep.c:91
static void SyncRepQueueInsert(int mode)
Definition: syncrep.c:340
static void SyncRepCancelWait(void)
Definition: syncrep.c:374
bool check_synchronous_standby_names(char **newval, void **extra, GucSource source)
Definition: syncrep.c:1007
static void SyncRepGetOldestSyncRecPtr(XLogRecPtr *writePtr, XLogRecPtr *flushPtr, XLogRecPtr *applyPtr, SyncRepStandbyData *sync_standbys, int num_standbys)
Definition: syncrep.c:628
void SyncRepCleanupAtProcExit(void)
Definition: syncrep.c:384
static int cmp_lsn(const void *a, const void *b)
Definition: syncrep.c:706
#define SyncStandbysDefined()
Definition: syncrep.c:93
#define SYNC_REP_PRIORITY
Definition: syncrep.h:35
#define NUM_SYNC_REP_WAIT_MODE
Definition: syncrep.h:27
void syncrep_scanner_init(const char *str)
#define SyncRepRequested()
Definition: syncrep.h:18
#define SYNC_REP_NO_WAIT
Definition: syncrep.h:22
void syncrep_scanner_finish(void)
#define SYNC_REP_WAIT_WRITE
Definition: syncrep.h:23
#define SYNC_REP_WAITING
Definition: syncrep.h:31
int syncrep_yyparse(void)
PGDLLIMPORT SyncRepConfigData * syncrep_parse_result
#define SYNC_REP_WAIT_COMPLETE
Definition: syncrep.h:32
#define SYNC_REP_WAIT_FLUSH
Definition: syncrep.h:24
PGDLLIMPORT char * syncrep_parse_error_msg
#define SYNC_REP_NOT_WAITING
Definition: syncrep.h:30
#define SYNC_REP_WAIT_APPLY
Definition: syncrep.h:25
@ WAIT_EVENT_SYNC_REP
Definition: wait_event.h:130
WalSnd * MyWalSnd
Definition: walsender.c:113
int max_wal_senders
Definition: walsender.c:122
bool am_cascading_walsender
Definition: walsender.c:117
WalSndCtlData * WalSndCtl
Definition: walsender.c:110
WalSndState
@ WALSNDSTATE_STREAMING
@ WALSNDSTATE_STOPPING
@ SYNCHRONOUS_COMMIT_REMOTE_WRITE
Definition: xact.h:72
@ SYNCHRONOUS_COMMIT_REMOTE_APPLY
Definition: xact.h:75
@ SYNCHRONOUS_COMMIT_REMOTE_FLUSH
Definition: xact.h:74
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43
#define XLogRecPtrIsInvalid(r)
Definition: xlogdefs.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28