PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
varsup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * varsup.c
4  * postgres OID & XID variables support routines
5  *
6  * Copyright (c) 2000-2017, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/access/transam/varsup.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres.h"
15 
16 #include "access/clog.h"
17 #include "access/commit_ts.h"
18 #include "access/subtrans.h"
19 #include "access/transam.h"
20 #include "access/xact.h"
21 #include "access/xlog.h"
22 #include "commands/dbcommands.h"
23 #include "miscadmin.h"
24 #include "postmaster/autovacuum.h"
25 #include "storage/pmsignal.h"
26 #include "storage/proc.h"
27 #include "utils/syscache.h"
28 
29 
30 /* Number of OIDs to prefetch (preallocate) per XLOG write */
31 #define VAR_OID_PREFETCH 8192
32 
33 /* pointer to "variable cache" in shared memory (set up by shmem.c) */
35 
36 
37 /*
38  * Allocate the next XID for a new transaction or subtransaction.
39  *
40  * The new XID is also stored into MyPgXact before returning.
41  *
42  * Note: when this is called, we are actually already inside a valid
43  * transaction, since XIDs are now not allocated until the transaction
44  * does something. So it is safe to do a database lookup if we want to
45  * issue a warning about XID wrap.
46  */
48 GetNewTransactionId(bool isSubXact)
49 {
50  TransactionId xid;
51 
52  /*
53  * Workers synchronize transaction state at the beginning of each parallel
54  * operation, so we can't account for new XIDs after that point.
55  */
56  if (IsInParallelMode())
57  elog(ERROR, "cannot assign TransactionIds during a parallel operation");
58 
59  /*
60  * During bootstrap initialization, we return the special bootstrap
61  * transaction id.
62  */
64  {
65  Assert(!isSubXact);
68  }
69 
70  /* safety check, we should never get this far in a HS slave */
71  if (RecoveryInProgress())
72  elog(ERROR, "cannot assign TransactionIds during recovery");
73 
74  LWLockAcquire(XidGenLock, LW_EXCLUSIVE);
75 
76  xid = ShmemVariableCache->nextXid;
77 
78  /*----------
79  * Check to see if it's safe to assign another XID. This protects against
80  * catastrophic data loss due to XID wraparound. The basic rules are:
81  *
82  * If we're past xidVacLimit, start trying to force autovacuum cycles.
83  * If we're past xidWarnLimit, start issuing warnings.
84  * If we're past xidStopLimit, refuse to execute transactions, unless
85  * we are running in single-user mode (which gives an escape hatch
86  * to the DBA who somehow got past the earlier defenses).
87  *
88  * Note that this coding also appears in GetNewMultiXactId.
89  *----------
90  */
91  if (TransactionIdFollowsOrEquals(xid, ShmemVariableCache->xidVacLimit))
92  {
93  /*
94  * For safety's sake, we release XidGenLock while sending signals,
95  * warnings, etc. This is not so much because we care about
96  * preserving concurrency in this situation, as to avoid any
97  * possibility of deadlock while doing get_database_name(). First,
98  * copy all the shared values we'll need in this path.
99  */
100  TransactionId xidWarnLimit = ShmemVariableCache->xidWarnLimit;
101  TransactionId xidStopLimit = ShmemVariableCache->xidStopLimit;
102  TransactionId xidWrapLimit = ShmemVariableCache->xidWrapLimit;
103  Oid oldest_datoid = ShmemVariableCache->oldestXidDB;
104 
105  LWLockRelease(XidGenLock);
106 
107  /*
108  * To avoid swamping the postmaster with signals, we issue the autovac
109  * request only once per 64K transaction starts. This still gives
110  * plenty of chances before we get into real trouble.
111  */
112  if (IsUnderPostmaster && (xid % 65536) == 0)
114 
115  if (IsUnderPostmaster &&
116  TransactionIdFollowsOrEquals(xid, xidStopLimit))
117  {
118  char *oldest_datname = get_database_name(oldest_datoid);
119 
120  /* complain even if that DB has disappeared */
121  if (oldest_datname)
122  ereport(ERROR,
123  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
124  errmsg("database is not accepting commands to avoid wraparound data loss in database \"%s\"",
125  oldest_datname),
126  errhint("Stop the postmaster and vacuum that database in single-user mode.\n"
127  "You might also need to commit or roll back old prepared transactions.")));
128  else
129  ereport(ERROR,
130  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
131  errmsg("database is not accepting commands to avoid wraparound data loss in database with OID %u",
132  oldest_datoid),
133  errhint("Stop the postmaster and vacuum that database in single-user mode.\n"
134  "You might also need to commit or roll back old prepared transactions.")));
135  }
136  else if (TransactionIdFollowsOrEquals(xid, xidWarnLimit))
137  {
138  char *oldest_datname = get_database_name(oldest_datoid);
139 
140  /* complain even if that DB has disappeared */
141  if (oldest_datname)
143  (errmsg("database \"%s\" must be vacuumed within %u transactions",
144  oldest_datname,
145  xidWrapLimit - xid),
146  errhint("To avoid a database shutdown, execute a database-wide VACUUM in that database.\n"
147  "You might also need to commit or roll back old prepared transactions.")));
148  else
150  (errmsg("database with OID %u must be vacuumed within %u transactions",
151  oldest_datoid,
152  xidWrapLimit - xid),
153  errhint("To avoid a database shutdown, execute a database-wide VACUUM in that database.\n"
154  "You might also need to commit or roll back old prepared transactions.")));
155  }
156 
157  /* Re-acquire lock and start over */
158  LWLockAcquire(XidGenLock, LW_EXCLUSIVE);
159  xid = ShmemVariableCache->nextXid;
160  }
161 
162  /*
163  * If we are allocating the first XID of a new page of the commit log,
164  * zero out that commit-log page before returning. We must do this while
165  * holding XidGenLock, else another xact could acquire and commit a later
166  * XID before we zero the page. Fortunately, a page of the commit log
167  * holds 32K or more transactions, so we don't have to do this very often.
168  *
169  * Extend pg_subtrans and pg_commit_ts too.
170  */
171  ExtendCLOG(xid);
172  ExtendCommitTs(xid);
173  ExtendSUBTRANS(xid);
174 
175  /*
176  * Now advance the nextXid counter. This must not happen until after we
177  * have successfully completed ExtendCLOG() --- if that routine fails, we
178  * want the next incoming transaction to try it again. We cannot assign
179  * more XIDs until there is CLOG space for them.
180  */
181  TransactionIdAdvance(ShmemVariableCache->nextXid);
182 
183  /*
184  * We must store the new XID into the shared ProcArray before releasing
185  * XidGenLock. This ensures that every active XID older than
186  * latestCompletedXid is present in the ProcArray, which is essential for
187  * correct OldestXmin tracking; see src/backend/access/transam/README.
188  *
189  * XXX by storing xid into MyPgXact without acquiring ProcArrayLock, we
190  * are relying on fetch/store of an xid to be atomic, else other backends
191  * might see a partially-set xid here. But holding both locks at once
192  * would be a nasty concurrency hit. So for now, assume atomicity.
193  *
194  * Note that readers of PGXACT xid fields should be careful to fetch the
195  * value only once, rather than assume they can read a value multiple
196  * times and get the same answer each time.
197  *
198  * The same comments apply to the subxact xid count and overflow fields.
199  *
200  * A solution to the atomic-store problem would be to give each PGXACT its
201  * own spinlock used only for fetching/storing that PGXACT's xid and
202  * related fields.
203  *
204  * If there's no room to fit a subtransaction XID into PGPROC, set the
205  * cache-overflowed flag instead. This forces readers to look in
206  * pg_subtrans to map subtransaction XIDs up to top-level XIDs. There is a
207  * race-condition window, in that the new XID will not appear as running
208  * until its parent link has been placed into pg_subtrans. However, that
209  * will happen before anyone could possibly have a reason to inquire about
210  * the status of the XID, so it seems OK. (Snapshots taken during this
211  * window *will* include the parent XID, so they will deliver the correct
212  * answer later on when someone does have a reason to inquire.)
213  */
214  {
215  /*
216  * Use volatile pointer to prevent code rearrangement; other backends
217  * could be examining my subxids info concurrently, and we don't want
218  * them to see an invalid intermediate state, such as incrementing
219  * nxids before filling the array entry. Note we are assuming that
220  * TransactionId and int fetch/store are atomic.
221  */
222  volatile PGPROC *myproc = MyProc;
223  volatile PGXACT *mypgxact = MyPgXact;
224 
225  if (!isSubXact)
226  mypgxact->xid = xid;
227  else
228  {
229  int nxids = mypgxact->nxids;
230 
231  if (nxids < PGPROC_MAX_CACHED_SUBXIDS)
232  {
233  myproc->subxids.xids[nxids] = xid;
234  mypgxact->nxids = nxids + 1;
235  }
236  else
237  mypgxact->overflowed = true;
238  }
239  }
240 
241  LWLockRelease(XidGenLock);
242 
243  return xid;
244 }
245 
246 /*
247  * Read nextXid but don't allocate it.
248  */
251 {
252  TransactionId xid;
253 
254  LWLockAcquire(XidGenLock, LW_SHARED);
255  xid = ShmemVariableCache->nextXid;
256  LWLockRelease(XidGenLock);
257 
258  return xid;
259 }
260 
261 /*
262  * Determine the last safe XID to allocate given the currently oldest
263  * datfrozenxid (ie, the oldest XID that might exist in any database
264  * of our cluster), and the OID of the (or a) database with that value.
265  */
266 void
267 SetTransactionIdLimit(TransactionId oldest_datfrozenxid, Oid oldest_datoid)
268 {
269  TransactionId xidVacLimit;
270  TransactionId xidWarnLimit;
271  TransactionId xidStopLimit;
272  TransactionId xidWrapLimit;
273  TransactionId curXid;
274 
275  Assert(TransactionIdIsNormal(oldest_datfrozenxid));
276 
277  /*
278  * The place where we actually get into deep trouble is halfway around
279  * from the oldest potentially-existing XID. (This calculation is
280  * probably off by one or two counts, because the special XIDs reduce the
281  * size of the loop a little bit. But we throw in plenty of slop below,
282  * so it doesn't matter.)
283  */
284  xidWrapLimit = oldest_datfrozenxid + (MaxTransactionId >> 1);
285  if (xidWrapLimit < FirstNormalTransactionId)
286  xidWrapLimit += FirstNormalTransactionId;
287 
288  /*
289  * We'll refuse to continue assigning XIDs in interactive mode once we get
290  * within 1M transactions of data loss. This leaves lots of room for the
291  * DBA to fool around fixing things in a standalone backend, while not
292  * being significant compared to total XID space. (Note that since
293  * vacuuming requires one transaction per table cleaned, we had better be
294  * sure there's lots of XIDs left...)
295  */
296  xidStopLimit = xidWrapLimit - 1000000;
297  if (xidStopLimit < FirstNormalTransactionId)
298  xidStopLimit -= FirstNormalTransactionId;
299 
300  /*
301  * We'll start complaining loudly when we get within 10M transactions of
302  * the stop point. This is kind of arbitrary, but if you let your gas
303  * gauge get down to 1% of full, would you be looking for the next gas
304  * station? We need to be fairly liberal about this number because there
305  * are lots of scenarios where most transactions are done by automatic
306  * clients that won't pay attention to warnings. (No, we're not gonna make
307  * this configurable. If you know enough to configure it, you know enough
308  * to not get in this kind of trouble in the first place.)
309  */
310  xidWarnLimit = xidStopLimit - 10000000;
311  if (xidWarnLimit < FirstNormalTransactionId)
312  xidWarnLimit -= FirstNormalTransactionId;
313 
314  /*
315  * We'll start trying to force autovacuums when oldest_datfrozenxid gets
316  * to be more than autovacuum_freeze_max_age transactions old.
317  *
318  * Note: guc.c ensures that autovacuum_freeze_max_age is in a sane range,
319  * so that xidVacLimit will be well before xidWarnLimit.
320  *
321  * Note: autovacuum_freeze_max_age is a PGC_POSTMASTER parameter so that
322  * we don't have to worry about dealing with on-the-fly changes in its
323  * value. It doesn't look practical to update shared state from a GUC
324  * assign hook (too many processes would try to execute the hook,
325  * resulting in race conditions as well as crashes of those not connected
326  * to shared memory). Perhaps this can be improved someday. See also
327  * SetMultiXactIdLimit.
328  */
329  xidVacLimit = oldest_datfrozenxid + autovacuum_freeze_max_age;
330  if (xidVacLimit < FirstNormalTransactionId)
331  xidVacLimit += FirstNormalTransactionId;
332 
333  /* Grab lock for just long enough to set the new limit values */
334  LWLockAcquire(XidGenLock, LW_EXCLUSIVE);
335  ShmemVariableCache->oldestXid = oldest_datfrozenxid;
336  ShmemVariableCache->xidVacLimit = xidVacLimit;
337  ShmemVariableCache->xidWarnLimit = xidWarnLimit;
338  ShmemVariableCache->xidStopLimit = xidStopLimit;
339  ShmemVariableCache->xidWrapLimit = xidWrapLimit;
340  ShmemVariableCache->oldestXidDB = oldest_datoid;
341  curXid = ShmemVariableCache->nextXid;
342  LWLockRelease(XidGenLock);
343 
344  /* Log the info */
345  ereport(DEBUG1,
346  (errmsg("transaction ID wrap limit is %u, limited by database with OID %u",
347  xidWrapLimit, oldest_datoid)));
348 
349  /*
350  * If past the autovacuum force point, immediately signal an autovac
351  * request. The reason for this is that autovac only processes one
352  * database per invocation. Once it's finished cleaning up the oldest
353  * database, it'll call here, and we'll signal the postmaster to start
354  * another iteration immediately if there are still any old databases.
355  */
356  if (TransactionIdFollowsOrEquals(curXid, xidVacLimit) &&
359 
360  /* Give an immediate warning if past the wrap warn point */
361  if (TransactionIdFollowsOrEquals(curXid, xidWarnLimit) && !InRecovery)
362  {
363  char *oldest_datname;
364 
365  /*
366  * We can be called when not inside a transaction, for example during
367  * StartupXLOG(). In such a case we cannot do database access, so we
368  * must just report the oldest DB's OID.
369  *
370  * Note: it's also possible that get_database_name fails and returns
371  * NULL, for example because the database just got dropped. We'll
372  * still warn, even though the warning might now be unnecessary.
373  */
374  if (IsTransactionState())
375  oldest_datname = get_database_name(oldest_datoid);
376  else
377  oldest_datname = NULL;
378 
379  if (oldest_datname)
381  (errmsg("database \"%s\" must be vacuumed within %u transactions",
382  oldest_datname,
383  xidWrapLimit - curXid),
384  errhint("To avoid a database shutdown, execute a database-wide VACUUM in that database.\n"
385  "You might also need to commit or roll back old prepared transactions.")));
386  else
388  (errmsg("database with OID %u must be vacuumed within %u transactions",
389  oldest_datoid,
390  xidWrapLimit - curXid),
391  errhint("To avoid a database shutdown, execute a database-wide VACUUM in that database.\n"
392  "You might also need to commit or roll back old prepared transactions.")));
393  }
394 }
395 
396 
397 /*
398  * ForceTransactionIdLimitUpdate -- does the XID wrap-limit data need updating?
399  *
400  * We primarily check whether oldestXidDB is valid. The cases we have in
401  * mind are that that database was dropped, or the field was reset to zero
402  * by pg_resetwal. In either case we should force recalculation of the
403  * wrap limit. Also do it if oldestXid is old enough to be forcing
404  * autovacuums or other actions; this ensures we update our state as soon
405  * as possible once extra overhead is being incurred.
406  */
407 bool
409 {
410  TransactionId nextXid;
411  TransactionId xidVacLimit;
412  TransactionId oldestXid;
413  Oid oldestXidDB;
414 
415  /* Locking is probably not really necessary, but let's be careful */
416  LWLockAcquire(XidGenLock, LW_SHARED);
417  nextXid = ShmemVariableCache->nextXid;
418  xidVacLimit = ShmemVariableCache->xidVacLimit;
419  oldestXid = ShmemVariableCache->oldestXid;
420  oldestXidDB = ShmemVariableCache->oldestXidDB;
421  LWLockRelease(XidGenLock);
422 
423  if (!TransactionIdIsNormal(oldestXid))
424  return true; /* shouldn't happen, but just in case */
425  if (!TransactionIdIsValid(xidVacLimit))
426  return true; /* this shouldn't happen anymore either */
427  if (TransactionIdFollowsOrEquals(nextXid, xidVacLimit))
428  return true; /* past VacLimit, don't delay updating */
430  return true; /* could happen, per comments above */
431  return false;
432 }
433 
434 
435 /*
436  * GetNewObjectId -- allocate a new OID
437  *
438  * OIDs are generated by a cluster-wide counter. Since they are only 32 bits
439  * wide, counter wraparound will occur eventually, and therefore it is unwise
440  * to assume they are unique unless precautions are taken to make them so.
441  * Hence, this routine should generally not be used directly. The only
442  * direct callers should be GetNewOid() and GetNewRelFileNode() in
443  * catalog/catalog.c.
444  */
445 Oid
447 {
448  Oid result;
449 
450  /* safety check, we should never get this far in a HS slave */
451  if (RecoveryInProgress())
452  elog(ERROR, "cannot assign OIDs during recovery");
453 
454  LWLockAcquire(OidGenLock, LW_EXCLUSIVE);
455 
456  /*
457  * Check for wraparound of the OID counter. We *must* not return 0
458  * (InvalidOid); and as long as we have to check that, it seems a good
459  * idea to skip over everything below FirstNormalObjectId too. (This
460  * basically just avoids lots of collisions with bootstrap-assigned OIDs
461  * right after a wrap occurs, so as to avoid a possibly large number of
462  * iterations in GetNewOid.) Note we are relying on unsigned comparison.
463  *
464  * During initdb, we start the OID generator at FirstBootstrapObjectId, so
465  * we only wrap if before that point when in bootstrap or standalone mode.
466  * The first time through this routine after normal postmaster start, the
467  * counter will be forced up to FirstNormalObjectId. This mechanism
468  * leaves the OIDs between FirstBootstrapObjectId and FirstNormalObjectId
469  * available for automatic assignment during initdb, while ensuring they
470  * will never conflict with user-assigned OIDs.
471  */
472  if (ShmemVariableCache->nextOid < ((Oid) FirstNormalObjectId))
473  {
475  {
476  /* wraparound, or first post-initdb assignment, in normal mode */
477  ShmemVariableCache->nextOid = FirstNormalObjectId;
478  ShmemVariableCache->oidCount = 0;
479  }
480  else
481  {
482  /* we may be bootstrapping, so don't enforce the full range */
483  if (ShmemVariableCache->nextOid < ((Oid) FirstBootstrapObjectId))
484  {
485  /* wraparound in standalone mode (unlikely but possible) */
486  ShmemVariableCache->nextOid = FirstNormalObjectId;
487  ShmemVariableCache->oidCount = 0;
488  }
489  }
490  }
491 
492  /* If we run out of logged for use oids then we must log more */
493  if (ShmemVariableCache->oidCount == 0)
494  {
495  XLogPutNextOid(ShmemVariableCache->nextOid + VAR_OID_PREFETCH);
496  ShmemVariableCache->oidCount = VAR_OID_PREFETCH;
497  }
498 
499  result = ShmemVariableCache->nextOid;
500 
501  (ShmemVariableCache->nextOid)++;
502  (ShmemVariableCache->oidCount)--;
503 
504  LWLockRelease(OidGenLock);
505 
506  return result;
507 }
#define TransactionIdAdvance(dest)
Definition: transam.h:48
bool IsPostmasterEnvironment
Definition: globals.c:99
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:987
uint32 TransactionId
Definition: c.h:394
Definition: proc.h:197
uint32 oidCount
Definition: transam.h:112
PGPROC * MyProc
Definition: proc.c:67
#define VAR_OID_PREFETCH
Definition: varsup.c:31
TransactionId xid
Definition: proc.h:199
void ExtendCLOG(TransactionId newestXact)
Definition: clog.c:604
bool InRecovery
Definition: xlog.c:191
bool TransactionIdFollowsOrEquals(TransactionId id1, TransactionId id2)
Definition: transam.c:349
TransactionId GetNewTransactionId(bool isSubXact)
Definition: varsup.c:48
TransactionId oldestXid
Definition: transam.h:119
int errcode(int sqlerrcode)
Definition: elog.c:575
unsigned int Oid
Definition: postgres_ext.h:31
bool RecoveryInProgress(void)
Definition: xlog.c:7805
TransactionId xidVacLimit
Definition: transam.h:120
#define FirstNormalObjectId
Definition: transam.h:94
PGXACT * MyPgXact
Definition: proc.c:68
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1714
bool ForceTransactionIdLimitUpdate(void)
Definition: varsup.c:408
Oid GetNewObjectId(void)
Definition: varsup.c:446
#define SearchSysCacheExists1(cacheId, key1)
Definition: syscache.h:167
bool IsInParallelMode(void)
Definition: xact.c:912
#define FirstNormalTransactionId
Definition: transam.h:34
#define ObjectIdGetDatum(X)
Definition: postgres.h:515
#define ERROR
Definition: elog.h:43
int autovacuum_freeze_max_age
Definition: autovacuum.c:118
void ExtendSUBTRANS(TransactionId newestXact)
Definition: subtrans.c:312
TransactionId nextXid
Definition: transam.h:117
void XLogPutNextOid(Oid nextOid)
Definition: xlog.c:9285
char * get_database_name(Oid dbid)
Definition: dbcommands.c:2048
#define BootstrapTransactionId
Definition: transam.h:32
TransactionId xidStopLimit
Definition: transam.h:122
bool IsUnderPostmaster
Definition: globals.c:100
uint8 nxids
Definition: proc.h:213
VariableCache ShmemVariableCache
Definition: varsup.c:34
TransactionId ReadNewTransactionId(void)
Definition: varsup.c:250
#define FirstBootstrapObjectId
Definition: transam.h:93
struct XidCache subxids
Definition: proc.h:148
#define ereport(elevel, rest)
Definition: elog.h:122
#define WARNING
Definition: elog.h:40
#define MaxTransactionId
Definition: transam.h:35
bool overflowed
Definition: proc.h:209
#define PGPROC_MAX_CACHED_SUBXIDS
Definition: proc.h:35
TransactionId xids[PGPROC_MAX_CACHED_SUBXIDS]
Definition: proc.h:39
#define NULL
Definition: c.h:226
#define Assert(condition)
Definition: c.h:671
void ExtendCommitTs(TransactionId newestXact)
Definition: commit_ts.c:773
void SetTransactionIdLimit(TransactionId oldest_datfrozenxid, Oid oldest_datoid)
Definition: varsup.c:267
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1110
bool IsTransactionState(void)
Definition: xact.c:349
#define IsBootstrapProcessingMode()
Definition: miscadmin.h:365
TransactionId xidWarnLimit
Definition: transam.h:121
int errmsg(const char *fmt,...)
Definition: elog.c:797
TransactionId xidWrapLimit
Definition: transam.h:123
void SendPostmasterSignal(PMSignalReason reason)
Definition: pmsignal.c:113
#define elog
Definition: elog.h:219
#define TransactionIdIsValid(xid)
Definition: transam.h:41
#define TransactionIdIsNormal(xid)
Definition: transam.h:42
Definition: proc.h:84