PostgreSQL Source Code git master
copyfromparse.c File Reference
#include "postgres.h"
#include <ctype.h>
#include <unistd.h>
#include <sys/stat.h>
#include "commands/copyapi.h"
#include "commands/copyfrom_internal.h"
#include "commands/progress.h"
#include "executor/executor.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "port/pg_bswap.h"
#include "utils/builtins.h"
#include "utils/rel.h"
Include dependency graph for copyfromparse.c:

Go to the source code of this file.

Macros

#define ISOCTAL(c)   (((c) >= '0') && ((c) <= '7'))
 
#define OCTVALUE(c)   ((c) - '0')
 
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
 
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
 
#define REFILL_LINEBUF
 

Functions

static bool CopyReadLine (CopyFromState cstate, bool is_csv)
 
static bool CopyReadLineText (CopyFromState cstate, bool is_csv)
 
static int CopyReadAttributesText (CopyFromState cstate)
 
static int CopyReadAttributesCSV (CopyFromState cstate)
 
static Datum CopyReadBinaryAttribute (CopyFromState cstate, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
 
static pg_attribute_always_inline bool CopyFromTextLikeOneRow (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls, bool is_csv)
 
static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal (CopyFromState cstate, char ***fields, int *nfields, bool is_csv)
 
static int CopyGetData (CopyFromState cstate, void *databuf, int minread, int maxread)
 
static bool CopyGetInt32 (CopyFromState cstate, int32 *val)
 
static bool CopyGetInt16 (CopyFromState cstate, int16 *val)
 
static void CopyLoadInputBuf (CopyFromState cstate)
 
static int CopyReadBinaryData (CopyFromState cstate, char *dest, int nbytes)
 
void ReceiveCopyBegin (CopyFromState cstate)
 
void ReceiveCopyBinaryHeader (CopyFromState cstate)
 
static void CopyConvertBuf (CopyFromState cstate)
 
static void CopyConversionError (CopyFromState cstate)
 
static void CopyLoadRawBuf (CopyFromState cstate)
 
bool NextCopyFromRawFields (CopyFromState cstate, char ***fields, int *nfields)
 
bool NextCopyFrom (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
 
bool CopyFromTextOneRow (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
 
bool CopyFromCSVOneRow (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
 
bool CopyFromBinaryOneRow (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
 
static int GetDecimalFromHex (char hex)
 

Variables

static const char BinarySignature [11] = "PGCOPY\n\377\r\n\0"
 

Macro Definition Documentation

◆ IF_NEED_REFILL_AND_EOF_BREAK

#define IF_NEED_REFILL_AND_EOF_BREAK (   extralen)
Value:
if (1) \
{ \
if (input_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
{ \
if (extralen) \
input_buf_ptr = copy_buf_len; /* consume the partial character */ \
/* backslash just before EOF, treat as data char */ \
result = true; \
break; \
} \
} else ((void) 0)

Definition at line 109 of file copyfromparse.c.

◆ IF_NEED_REFILL_AND_NOT_EOF_CONTINUE

#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE (   extralen)
Value:
if (1) \
{ \
if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
{ \
input_buf_ptr = prev_raw_ptr; /* undo fetch */ \
need_data = true; \
continue; \
} \
} else ((void) 0)

Definition at line 97 of file copyfromparse.c.

◆ ISOCTAL

#define ISOCTAL (   c)    (((c) >= '0') && ((c) <= '7'))

Definition at line 78 of file copyfromparse.c.

◆ OCTVALUE

#define OCTVALUE (   c)    ((c) - '0')

Definition at line 79 of file copyfromparse.c.

◆ REFILL_LINEBUF

#define REFILL_LINEBUF
Value:
if (1) \
{ \
if (input_buf_ptr > cstate->input_buf_index) \
{ \
appendBinaryStringInfo(&cstate->line_buf, \
cstate->input_buf + cstate->input_buf_index, \
input_buf_ptr - cstate->input_buf_index); \
cstate->input_buf_index = input_buf_ptr; \
} \
} else ((void) 0)

Definition at line 126 of file copyfromparse.c.

Function Documentation

◆ CopyConversionError()

static void CopyConversionError ( CopyFromState  cstate)
static

Definition at line 533 of file copyfromparse.c.

534{
535 Assert(cstate->raw_buf_len > 0);
537
538 if (!cstate->need_transcoding)
539 {
540 /*
541 * Everything up to input_buf_len was successfully verified, and
542 * input_buf_len points to the invalid or incomplete character.
543 */
545 cstate->raw_buf + cstate->input_buf_len,
546 cstate->raw_buf_len - cstate->input_buf_len);
547 }
548 else
549 {
550 /*
551 * raw_buf_index points to the invalid or untranslatable character. We
552 * let the conversion routine report the error, because it can provide
553 * a more specific error message than we could here. An earlier call
554 * to the conversion routine in CopyConvertBuf() detected that there
555 * is an error, now we call the conversion routine again with
556 * noError=false, to have it throw the error.
557 */
558 unsigned char *src;
559 int srclen;
560 unsigned char *dst;
561 int dstlen;
562
563 src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
564 srclen = cstate->raw_buf_len - cstate->raw_buf_index;
565 dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
566 dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
567
569 cstate->file_encoding,
571 src, srclen,
572 dst, dstlen,
573 false);
574
575 /*
576 * The conversion routine should have reported an error, so this
577 * should not be reached.
578 */
579 elog(ERROR, "encoding conversion failed without error");
580 }
581}
#define INPUT_BUF_SIZE
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:225
Assert(PointerIsAligned(start, uint64))
int GetDatabaseEncoding(void)
Definition: mbutils.c:1261
int pg_do_encoding_conversion_buf(Oid proc, int src_encoding, int dest_encoding, unsigned char *src, int srclen, unsigned char *dest, int destlen, bool noError)
Definition: mbutils.c:469
void report_invalid_encoding(int encoding, const char *mbstr, int len)
Definition: mbutils.c:1698

References Assert(), CopyFromStateData::conversion_proc, elog, ERROR, CopyFromStateData::file_encoding, GetDatabaseEncoding(), CopyFromStateData::input_buf, CopyFromStateData::input_buf_len, INPUT_BUF_SIZE, CopyFromStateData::input_reached_error, CopyFromStateData::need_transcoding, pg_do_encoding_conversion_buf(), CopyFromStateData::raw_buf, CopyFromStateData::raw_buf_index, CopyFromStateData::raw_buf_len, and report_invalid_encoding().

Referenced by CopyLoadInputBuf().

◆ CopyConvertBuf()

static void CopyConvertBuf ( CopyFromState  cstate)
static

Definition at line 400 of file copyfromparse.c.

401{
402 /*
403 * If the file and server encoding are the same, no encoding conversion is
404 * required. However, we still need to verify that the input is valid for
405 * the encoding.
406 */
407 if (!cstate->need_transcoding)
408 {
409 /*
410 * When conversion is not required, input_buf and raw_buf are the
411 * same. raw_buf_len is the total number of bytes in the buffer, and
412 * input_buf_len tracks how many of those bytes have already been
413 * verified.
414 */
415 int preverifiedlen = cstate->input_buf_len;
416 int unverifiedlen = cstate->raw_buf_len - cstate->input_buf_len;
417 int nverified;
418
419 if (unverifiedlen == 0)
420 {
421 /*
422 * If no more raw data is coming, report the EOF to the caller.
423 */
424 if (cstate->raw_reached_eof)
425 cstate->input_reached_eof = true;
426 return;
427 }
428
429 /*
430 * Verify the new data, including any residual unverified bytes from
431 * previous round.
432 */
433 nverified = pg_encoding_verifymbstr(cstate->file_encoding,
434 cstate->raw_buf + preverifiedlen,
435 unverifiedlen);
436 if (nverified == 0)
437 {
438 /*
439 * Could not verify anything.
440 *
441 * If there is no more raw input data coming, it means that there
442 * was an incomplete multi-byte sequence at the end. Also, if
443 * there's "enough" input left, we should be able to verify at
444 * least one character, and a failure to do so means that we've
445 * hit an invalid byte sequence.
446 */
447 if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
448 cstate->input_reached_error = true;
449 return;
450 }
451 cstate->input_buf_len += nverified;
452 }
453 else
454 {
455 /*
456 * Encoding conversion is needed.
457 */
458 int nbytes;
459 unsigned char *src;
460 int srclen;
461 unsigned char *dst;
462 int dstlen;
463 int convertedlen;
464
465 if (RAW_BUF_BYTES(cstate) == 0)
466 {
467 /*
468 * If no more raw data is coming, report the EOF to the caller.
469 */
470 if (cstate->raw_reached_eof)
471 cstate->input_reached_eof = true;
472 return;
473 }
474
475 /*
476 * First, copy down any unprocessed data.
477 */
478 nbytes = INPUT_BUF_BYTES(cstate);
479 if (nbytes > 0 && cstate->input_buf_index > 0)
480 memmove(cstate->input_buf, cstate->input_buf + cstate->input_buf_index,
481 nbytes);
482 cstate->input_buf_index = 0;
483 cstate->input_buf_len = nbytes;
484 cstate->input_buf[nbytes] = '\0';
485
486 src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
487 srclen = cstate->raw_buf_len - cstate->raw_buf_index;
488 dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
489 dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
490
491 /*
492 * Do the conversion. This might stop short, if there is an invalid
493 * byte sequence in the input. We'll convert as much as we can in
494 * that case.
495 *
496 * Note: Even if we hit an invalid byte sequence, we don't report the
497 * error until all the valid bytes have been consumed. The input
498 * might contain an end-of-input marker (\.), and we don't want to
499 * report an error if the invalid byte sequence is after the
500 * end-of-input marker. We might unnecessarily convert some data
501 * after the end-of-input marker as long as it's valid for the
502 * encoding, but that's harmless.
503 */
504 convertedlen = pg_do_encoding_conversion_buf(cstate->conversion_proc,
505 cstate->file_encoding,
507 src, srclen,
508 dst, dstlen,
509 true);
510 if (convertedlen == 0)
511 {
512 /*
513 * Could not convert anything. If there is no more raw input data
514 * coming, it means that there was an incomplete multi-byte
515 * sequence at the end. Also, if there is plenty of input left,
516 * we should be able to convert at least one character, so a
517 * failure to do so must mean that we've hit a byte sequence
518 * that's invalid.
519 */
520 if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
521 cstate->input_reached_error = true;
522 return;
523 }
524 cstate->raw_buf_index += convertedlen;
525 cstate->input_buf_len += strlen((char *) dst);
526 }
527}
#define RAW_BUF_BYTES(cstate)
#define INPUT_BUF_BYTES(cstate)
#define MAX_CONVERSION_INPUT_LENGTH
Definition: pg_wchar.h:320
int pg_encoding_verifymbstr(int encoding, const char *mbstr, int len)
Definition: wchar.c:2163
int pg_encoding_max_length(int encoding)
Definition: wchar.c:2174

References CopyFromStateData::conversion_proc, CopyFromStateData::file_encoding, GetDatabaseEncoding(), CopyFromStateData::input_buf, INPUT_BUF_BYTES, CopyFromStateData::input_buf_index, CopyFromStateData::input_buf_len, INPUT_BUF_SIZE, CopyFromStateData::input_reached_eof, CopyFromStateData::input_reached_error, MAX_CONVERSION_INPUT_LENGTH, CopyFromStateData::need_transcoding, pg_do_encoding_conversion_buf(), pg_encoding_max_length(), pg_encoding_verifymbstr(), CopyFromStateData::raw_buf, RAW_BUF_BYTES, CopyFromStateData::raw_buf_index, CopyFromStateData::raw_buf_len, and CopyFromStateData::raw_reached_eof.

Referenced by CopyLoadInputBuf().

◆ CopyFromBinaryOneRow()

bool CopyFromBinaryOneRow ( CopyFromState  cstate,
ExprContext econtext,
Datum values,
bool *  nulls 
)

Definition at line 1086 of file copyfromparse.c.

1088{
1089 TupleDesc tupDesc;
1090 AttrNumber attr_count;
1091 FmgrInfo *in_functions = cstate->in_functions;
1092 Oid *typioparams = cstate->typioparams;
1093 int16 fld_count;
1094 ListCell *cur;
1095
1096 tupDesc = RelationGetDescr(cstate->rel);
1097 attr_count = list_length(cstate->attnumlist);
1098
1099 cstate->cur_lineno++;
1100
1101 if (!CopyGetInt16(cstate, &fld_count))
1102 {
1103 /* EOF detected (end of file, or protocol-level EOF) */
1104 return false;
1105 }
1106
1107 if (fld_count == -1)
1108 {
1109 /*
1110 * Received EOF marker. Wait for the protocol-level EOF, and complain
1111 * if it doesn't come immediately. In COPY FROM STDIN, this ensures
1112 * that we correctly handle CopyFail, if client chooses to send that
1113 * now. When copying from file, we could ignore the rest of the file
1114 * like in text mode, but we choose to be consistent with the COPY
1115 * FROM STDIN case.
1116 */
1117 char dummy;
1118
1119 if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
1120 ereport(ERROR,
1121 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1122 errmsg("received copy data after EOF marker")));
1123 return false;
1124 }
1125
1126 if (fld_count != attr_count)
1127 ereport(ERROR,
1128 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1129 errmsg("row field count is %d, expected %d",
1130 (int) fld_count, attr_count)));
1131
1132 foreach(cur, cstate->attnumlist)
1133 {
1134 int attnum = lfirst_int(cur);
1135 int m = attnum - 1;
1136 Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1137
1138 cstate->cur_attname = NameStr(att->attname);
1139 values[m] = CopyReadBinaryAttribute(cstate,
1140 &in_functions[m],
1141 typioparams[m],
1142 att->atttypmod,
1143 &nulls[m]);
1144 cstate->cur_attname = NULL;
1145 }
1146
1147 return true;
1148}
int16 AttrNumber
Definition: attnum.h:21
static Datum values[MAXATTR]
Definition: bootstrap.c:151
#define NameStr(name)
Definition: c.h:717
int16_t int16
Definition: c.h:497
static bool CopyGetInt16(CopyFromState cstate, int16 *val)
static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
struct cursor * cur
Definition: ecpg.c:29
int errcode(int sqlerrcode)
Definition: elog.c:853
int errmsg(const char *fmt,...)
Definition: elog.c:1070
#define ereport(elevel,...)
Definition: elog.h:149
int16 attnum
Definition: pg_attribute.h:74
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:200
static int list_length(const List *l)
Definition: pg_list.h:152
#define lfirst_int(lc)
Definition: pg_list.h:173
unsigned int Oid
Definition: postgres_ext.h:32
#define RelationGetDescr(relation)
Definition: rel.h:539
const char * cur_attname
Definition: fmgr.h:57
static FormData_pg_attribute * TupleDescAttr(TupleDesc tupdesc, int i)
Definition: tupdesc.h:154

References attnum, CopyFromStateData::attnumlist, CopyGetInt16(), CopyReadBinaryAttribute(), CopyReadBinaryData(), cur, CopyFromStateData::cur_attname, CopyFromStateData::cur_lineno, ereport, errcode(), errmsg(), ERROR, CopyFromStateData::in_functions, lfirst_int, list_length(), NameStr, CopyFromStateData::rel, RelationGetDescr, TupleDescAttr(), CopyFromStateData::typioparams, and values.

◆ CopyFromCSVOneRow()

bool CopyFromCSVOneRow ( CopyFromState  cstate,
ExprContext econtext,
Datum values,
bool *  nulls 
)

Definition at line 924 of file copyfromparse.c.

926{
927 return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
928}
static pg_attribute_always_inline bool CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls, bool is_csv)

References CopyFromTextLikeOneRow(), and values.

◆ CopyFromTextLikeOneRow()

static pg_attribute_always_inline bool CopyFromTextLikeOneRow ( CopyFromState  cstate,
ExprContext econtext,
Datum values,
bool *  nulls,
bool  is_csv 
)
static

Definition at line 937 of file copyfromparse.c.

939{
940 TupleDesc tupDesc;
941 AttrNumber attr_count;
942 FmgrInfo *in_functions = cstate->in_functions;
943 Oid *typioparams = cstate->typioparams;
944 ExprState **defexprs = cstate->defexprs;
945 char **field_strings;
946 ListCell *cur;
947 int fldct;
948 int fieldno;
949 char *string;
950
951 tupDesc = RelationGetDescr(cstate->rel);
952 attr_count = list_length(cstate->attnumlist);
953
954 /* read raw fields in the next line */
955 if (!NextCopyFromRawFieldsInternal(cstate, &field_strings, &fldct, is_csv))
956 return false;
957
958 /* check for overflowing fields */
959 if (attr_count > 0 && fldct > attr_count)
961 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
962 errmsg("extra data after last expected column")));
963
964 fieldno = 0;
965
966 /* Loop to read the user attributes on the line. */
967 foreach(cur, cstate->attnumlist)
968 {
969 int attnum = lfirst_int(cur);
970 int m = attnum - 1;
971 Form_pg_attribute att = TupleDescAttr(tupDesc, m);
972
973 if (fieldno >= fldct)
975 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
976 errmsg("missing data for column \"%s\"",
977 NameStr(att->attname))));
978 string = field_strings[fieldno++];
979
980 if (cstate->convert_select_flags &&
981 !cstate->convert_select_flags[m])
982 {
983 /* ignore input field, leaving column as NULL */
984 continue;
985 }
986
987 if (is_csv)
988 {
989 if (string == NULL &&
990 cstate->opts.force_notnull_flags[m])
991 {
992 /*
993 * FORCE_NOT_NULL option is set and column is NULL - convert
994 * it to the NULL string.
995 */
996 string = cstate->opts.null_print;
997 }
998 else if (string != NULL && cstate->opts.force_null_flags[m]
999 && strcmp(string, cstate->opts.null_print) == 0)
1000 {
1001 /*
1002 * FORCE_NULL option is set and column matches the NULL
1003 * string. It must have been quoted, or otherwise the string
1004 * would already have been set to NULL. Convert it to NULL as
1005 * specified.
1006 */
1007 string = NULL;
1008 }
1009 }
1010
1011 cstate->cur_attname = NameStr(att->attname);
1012 cstate->cur_attval = string;
1013
1014 if (string != NULL)
1015 nulls[m] = false;
1016
1017 if (cstate->defaults[m])
1018 {
1019 /* We must have switched into the per-tuple memory context */
1020 Assert(econtext != NULL);
1022
1023 values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
1024 }
1025
1026 /*
1027 * If ON_ERROR is specified with IGNORE, skip rows with soft errors
1028 */
1029 else if (!InputFunctionCallSafe(&in_functions[m],
1030 string,
1031 typioparams[m],
1032 att->atttypmod,
1033 (Node *) cstate->escontext,
1034 &values[m]))
1035 {
1037
1038 cstate->num_errors++;
1039
1041 {
1042 /*
1043 * Since we emit line number and column info in the below
1044 * notice message, we suppress error context information other
1045 * than the relation name.
1046 */
1047 Assert(!cstate->relname_only);
1048 cstate->relname_only = true;
1049
1050 if (cstate->cur_attval)
1051 {
1052 char *attval;
1053
1054 attval = CopyLimitPrintoutLength(cstate->cur_attval);
1056 errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"",
1057 (unsigned long long) cstate->cur_lineno,
1058 cstate->cur_attname,
1059 attval));
1060 pfree(attval);
1061 }
1062 else
1064 errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input",
1065 (unsigned long long) cstate->cur_lineno,
1066 cstate->cur_attname));
1067
1068 /* reset relname_only */
1069 cstate->relname_only = false;
1070 }
1071
1072 return true;
1073 }
1074
1075 cstate->cur_attname = NULL;
1076 cstate->cur_attval = NULL;
1077 }
1078
1079 Assert(fieldno == attr_count);
1080
1081 return true;
1082}
char * CopyLimitPrintoutLength(const char *str)
Definition: copyfrom.c:333
static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields, bool is_csv)
#define NOTICE
Definition: elog.h:35
static Datum ExecEvalExpr(ExprState *state, ExprContext *econtext, bool *isNull)
Definition: executor.h:374
bool InputFunctionCallSafe(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod, fmNodePtr escontext, Datum *result)
Definition: fmgr.c:1585
@ COPY_ON_ERROR_STOP
Definition: copy.h:39
@ COPY_LOG_VERBOSITY_VERBOSE
Definition: copy.h:51
void pfree(void *pointer)
Definition: mcxt.c:1524
MemoryContext CurrentMemoryContext
Definition: mcxt.c:143
char string[11]
Definition: preproc-type.c:52
CopyLogVerbosityChoice log_verbosity
Definition: copy.h:87
CopyOnErrorChoice on_error
Definition: copy.h:86
char * null_print
Definition: copy.h:68
bool * force_notnull_flags
Definition: copy.h:81
bool * force_null_flags
Definition: copy.h:84
CopyFormatOptions opts
const char * cur_attval
ErrorSaveContext * escontext
MemoryContext ecxt_per_tuple_memory
Definition: execnodes.h:276
Definition: nodes.h:131

References Assert(), attnum, CopyFromStateData::attnumlist, CopyFromStateData::convert_select_flags, COPY_LOG_VERBOSITY_VERBOSE, COPY_ON_ERROR_STOP, CopyLimitPrintoutLength(), cur, CopyFromStateData::cur_attname, CopyFromStateData::cur_attval, CopyFromStateData::cur_lineno, CurrentMemoryContext, CopyFromStateData::defaults, CopyFromStateData::defexprs, ExprContext::ecxt_per_tuple_memory, ereport, errcode(), errmsg(), ERROR, CopyFromStateData::escontext, ExecEvalExpr(), CopyFormatOptions::force_notnull_flags, CopyFormatOptions::force_null_flags, CopyFromStateData::in_functions, InputFunctionCallSafe(), lfirst_int, list_length(), CopyFormatOptions::log_verbosity, NameStr, NextCopyFromRawFieldsInternal(), NOTICE, CopyFormatOptions::null_print, CopyFromStateData::num_errors, CopyFormatOptions::on_error, CopyFromStateData::opts, pfree(), CopyFromStateData::rel, RelationGetDescr, CopyFromStateData::relname_only, TupleDescAttr(), CopyFromStateData::typioparams, and values.

Referenced by CopyFromCSVOneRow(), and CopyFromTextOneRow().

◆ CopyFromTextOneRow()

bool CopyFromTextOneRow ( CopyFromState  cstate,
ExprContext econtext,
Datum values,
bool *  nulls 
)

Definition at line 916 of file copyfromparse.c.

918{
919 return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, false);
920}

References CopyFromTextLikeOneRow(), and values.

◆ CopyGetData()

static int CopyGetData ( CopyFromState  cstate,
void *  databuf,
int  minread,
int  maxread 
)
static

Definition at line 245 of file copyfromparse.c.

246{
247 int bytesread = 0;
248
249 switch (cstate->copy_src)
250 {
251 case COPY_FILE:
252 bytesread = fread(databuf, 1, maxread, cstate->copy_file);
253 if (ferror(cstate->copy_file))
256 errmsg("could not read from COPY file: %m")));
257 if (bytesread == 0)
258 cstate->raw_reached_eof = true;
259 break;
260 case COPY_FRONTEND:
261 while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
262 {
263 int avail;
264
265 while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
266 {
267 /* Try to receive another message */
268 int mtype;
269 int maxmsglen;
270
271 readmessage:
274 mtype = pq_getbyte();
275 if (mtype == EOF)
277 (errcode(ERRCODE_CONNECTION_FAILURE),
278 errmsg("unexpected EOF on client connection with an open transaction")));
279 /* Validate message type and set packet size limit */
280 switch (mtype)
281 {
282 case PqMsg_CopyData:
283 maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
284 break;
285 case PqMsg_CopyDone:
286 case PqMsg_CopyFail:
287 case PqMsg_Flush:
288 case PqMsg_Sync:
289 maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
290 break;
291 default:
293 (errcode(ERRCODE_PROTOCOL_VIOLATION),
294 errmsg("unexpected message type 0x%02X during COPY from stdin",
295 mtype)));
296 maxmsglen = 0; /* keep compiler quiet */
297 break;
298 }
299 /* Now collect the message body */
300 if (pq_getmessage(cstate->fe_msgbuf, maxmsglen))
302 (errcode(ERRCODE_CONNECTION_FAILURE),
303 errmsg("unexpected EOF on client connection with an open transaction")));
305 /* ... and process it */
306 switch (mtype)
307 {
308 case PqMsg_CopyData:
309 break;
310 case PqMsg_CopyDone:
311 /* COPY IN correctly terminated by frontend */
312 cstate->raw_reached_eof = true;
313 return bytesread;
314 case PqMsg_CopyFail:
316 (errcode(ERRCODE_QUERY_CANCELED),
317 errmsg("COPY from stdin failed: %s",
318 pq_getmsgstring(cstate->fe_msgbuf))));
319 break;
320 case PqMsg_Flush:
321 case PqMsg_Sync:
322
323 /*
324 * Ignore Flush/Sync for the convenience of client
325 * libraries (such as libpq) that may send those
326 * without noticing that the command they just
327 * sent was COPY.
328 */
329 goto readmessage;
330 default:
331 Assert(false); /* NOT REACHED */
332 }
333 }
334 avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
335 if (avail > maxread)
336 avail = maxread;
337 pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
338 databuf = (void *) ((char *) databuf + avail);
339 maxread -= avail;
340 bytesread += avail;
341 }
342 break;
343 case COPY_CALLBACK:
344 bytesread = cstate->data_source_cb(databuf, minread, maxread);
345 break;
346 }
347
348 return bytesread;
349}
@ COPY_FILE
Definition: copyto.c:45
@ COPY_CALLBACK
Definition: copyto.c:47
@ COPY_FRONTEND
Definition: copyto.c:46
int errcode_for_file_access(void)
Definition: elog.c:876
#define PQ_SMALL_MESSAGE_LIMIT
Definition: libpq.h:30
#define PQ_LARGE_MESSAGE_LIMIT
Definition: libpq.h:31
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:141
#define RESUME_CANCEL_INTERRUPTS()
Definition: miscadmin.h:143
int pq_getmessage(StringInfo s, int maxlen)
Definition: pqcomm.c:1204
int pq_getbyte(void)
Definition: pqcomm.c:964
void pq_startmsgread(void)
Definition: pqcomm.c:1142
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:579
void pq_copymsgbytes(StringInfo msg, void *buf, int datalen)
Definition: pqformat.c:528
#define PqMsg_CopyDone
Definition: protocol.h:64
#define PqMsg_CopyData
Definition: protocol.h:65
#define PqMsg_Sync
Definition: protocol.h:27
#define PqMsg_CopyFail
Definition: protocol.h:29
#define PqMsg_Flush
Definition: protocol.h:24
copy_data_source_cb data_source_cb

References Assert(), COPY_CALLBACK, COPY_FILE, CopyFromStateData::copy_file, COPY_FRONTEND, CopyFromStateData::copy_src, StringInfoData::cursor, CopyFromStateData::data_source_cb, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, CopyFromStateData::fe_msgbuf, HOLD_CANCEL_INTERRUPTS, StringInfoData::len, pq_copymsgbytes(), pq_getbyte(), pq_getmessage(), pq_getmsgstring(), PQ_LARGE_MESSAGE_LIMIT, PQ_SMALL_MESSAGE_LIMIT, pq_startmsgread(), PqMsg_CopyData, PqMsg_CopyDone, PqMsg_CopyFail, PqMsg_Flush, PqMsg_Sync, CopyFromStateData::raw_reached_eof, and RESUME_CANCEL_INTERRUPTS.

Referenced by CopyLoadRawBuf(), and CopyReadLine().

◆ CopyGetInt16()

static bool CopyGetInt16 ( CopyFromState  cstate,
int16 val 
)
inlinestatic

Definition at line 379 of file copyfromparse.c.

380{
381 uint16 buf;
382
383 if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
384 {
385 *val = 0; /* suppress compiler warning */
386 return false;
387 }
388 *val = (int16) pg_ntoh16(buf);
389 return true;
390}
uint16_t uint16
Definition: c.h:501
long val
Definition: informix.c:689
#define pg_ntoh16(x)
Definition: pg_bswap.h:124
static char * buf
Definition: pg_test_fsync.c:72

References buf, CopyReadBinaryData(), pg_ntoh16, and val.

Referenced by CopyFromBinaryOneRow().

◆ CopyGetInt32()

static bool CopyGetInt32 ( CopyFromState  cstate,
int32 val 
)
inlinestatic

Definition at line 362 of file copyfromparse.c.

363{
364 uint32 buf;
365
366 if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
367 {
368 *val = 0; /* suppress compiler warning */
369 return false;
370 }
371 *val = (int32) pg_ntoh32(buf);
372 return true;
373}
int32_t int32
Definition: c.h:498
uint32_t uint32
Definition: c.h:502
#define pg_ntoh32(x)
Definition: pg_bswap.h:125

References buf, CopyReadBinaryData(), pg_ntoh32, and val.

Referenced by CopyReadBinaryAttribute(), and ReceiveCopyBinaryHeader().

◆ CopyLoadInputBuf()

static void CopyLoadInputBuf ( CopyFromState  cstate)
static

Definition at line 650 of file copyfromparse.c.

651{
652 int nbytes = INPUT_BUF_BYTES(cstate);
653
654 /*
655 * The caller has updated input_buf_index to indicate how much of the
656 * input has been consumed and isn't needed anymore. If input_buf is the
657 * same physical area as raw_buf, update raw_buf_index accordingly.
658 */
659 if (cstate->raw_buf == cstate->input_buf)
660 {
661 Assert(!cstate->need_transcoding);
662 Assert(cstate->input_buf_index >= cstate->raw_buf_index);
663 cstate->raw_buf_index = cstate->input_buf_index;
664 }
665
666 for (;;)
667 {
668 /* If we now have some unconverted data, try to convert it */
669 CopyConvertBuf(cstate);
670
671 /* If we now have some more input bytes ready, return them */
672 if (INPUT_BUF_BYTES(cstate) > nbytes)
673 return;
674
675 /*
676 * If we reached an invalid byte sequence, or we're at an incomplete
677 * multi-byte character but there is no more raw input data, report
678 * conversion error.
679 */
680 if (cstate->input_reached_error)
681 CopyConversionError(cstate);
682
683 /* no more input, and everything has been converted */
684 if (cstate->input_reached_eof)
685 break;
686
687 /* Try to load more raw data */
688 Assert(!cstate->raw_reached_eof);
689 CopyLoadRawBuf(cstate);
690 }
691}
static void CopyConversionError(CopyFromState cstate)
static void CopyLoadRawBuf(CopyFromState cstate)
static void CopyConvertBuf(CopyFromState cstate)

References Assert(), CopyConversionError(), CopyConvertBuf(), CopyLoadRawBuf(), CopyFromStateData::input_buf, INPUT_BUF_BYTES, CopyFromStateData::input_buf_index, CopyFromStateData::input_reached_eof, CopyFromStateData::input_reached_error, CopyFromStateData::need_transcoding, CopyFromStateData::raw_buf, CopyFromStateData::raw_buf_index, and CopyFromStateData::raw_reached_eof.

Referenced by CopyReadLineText().

◆ CopyLoadRawBuf()

static void CopyLoadRawBuf ( CopyFromState  cstate)
static

Definition at line 590 of file copyfromparse.c.

591{
592 int nbytes;
593 int inbytes;
594
595 /*
596 * In text mode, if encoding conversion is not required, raw_buf and
597 * input_buf point to the same buffer. Their len/index better agree, too.
598 */
599 if (cstate->raw_buf == cstate->input_buf)
600 {
601 Assert(!cstate->need_transcoding);
602 Assert(cstate->raw_buf_index == cstate->input_buf_index);
603 Assert(cstate->input_buf_len <= cstate->raw_buf_len);
604 }
605
606 /*
607 * Copy down the unprocessed data if any.
608 */
609 nbytes = RAW_BUF_BYTES(cstate);
610 if (nbytes > 0 && cstate->raw_buf_index > 0)
611 memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
612 nbytes);
613 cstate->raw_buf_len -= cstate->raw_buf_index;
614 cstate->raw_buf_index = 0;
615
616 /*
617 * If raw_buf and input_buf are in fact the same buffer, adjust the
618 * input_buf variables, too.
619 */
620 if (cstate->raw_buf == cstate->input_buf)
621 {
622 cstate->input_buf_len -= cstate->input_buf_index;
623 cstate->input_buf_index = 0;
624 }
625
626 /* Load more data */
627 inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len,
628 1, RAW_BUF_SIZE - cstate->raw_buf_len);
629 nbytes += inbytes;
630 cstate->raw_buf[nbytes] = '\0';
631 cstate->raw_buf_len = nbytes;
632
633 cstate->bytes_processed += inbytes;
635
636 if (inbytes == 0)
637 cstate->raw_reached_eof = true;
638}
void pgstat_progress_update_param(int index, int64 val)
#define RAW_BUF_SIZE
static int CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
#define PROGRESS_COPY_BYTES_PROCESSED
Definition: progress.h:142

References Assert(), CopyFromStateData::bytes_processed, CopyGetData(), CopyFromStateData::input_buf, CopyFromStateData::input_buf_index, CopyFromStateData::input_buf_len, CopyFromStateData::need_transcoding, pgstat_progress_update_param(), PROGRESS_COPY_BYTES_PROCESSED, CopyFromStateData::raw_buf, RAW_BUF_BYTES, CopyFromStateData::raw_buf_index, CopyFromStateData::raw_buf_len, RAW_BUF_SIZE, and CopyFromStateData::raw_reached_eof.

Referenced by CopyLoadInputBuf(), and CopyReadBinaryData().

◆ CopyReadAttributesCSV()

static int CopyReadAttributesCSV ( CopyFromState  cstate)
static

Definition at line 1818 of file copyfromparse.c.

1819{
1820 char delimc = cstate->opts.delim[0];
1821 char quotec = cstate->opts.quote[0];
1822 char escapec = cstate->opts.escape[0];
1823 int fieldno;
1824 char *output_ptr;
1825 char *cur_ptr;
1826 char *line_end_ptr;
1827
1828 /*
1829 * We need a special case for zero-column tables: check that the input
1830 * line is empty, and return.
1831 */
1832 if (cstate->max_fields <= 0)
1833 {
1834 if (cstate->line_buf.len != 0)
1835 ereport(ERROR,
1836 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1837 errmsg("extra data after last expected column")));
1838 return 0;
1839 }
1840
1842
1843 /*
1844 * The de-escaped attributes will certainly not be longer than the input
1845 * data line, so we can just force attribute_buf to be large enough and
1846 * then transfer data without any checks for enough space. We need to do
1847 * it this way because enlarging attribute_buf mid-stream would invalidate
1848 * pointers already stored into cstate->raw_fields[].
1849 */
1850 if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
1851 enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
1852 output_ptr = cstate->attribute_buf.data;
1853
1854 /* set pointer variables for loop */
1855 cur_ptr = cstate->line_buf.data;
1856 line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
1857
1858 /* Outer loop iterates over fields */
1859 fieldno = 0;
1860 for (;;)
1861 {
1862 bool found_delim = false;
1863 bool saw_quote = false;
1864 char *start_ptr;
1865 char *end_ptr;
1866 int input_len;
1867
1868 /* Make sure there is enough space for the next value */
1869 if (fieldno >= cstate->max_fields)
1870 {
1871 cstate->max_fields *= 2;
1872 cstate->raw_fields =
1873 repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
1874 }
1875
1876 /* Remember start of field on both input and output sides */
1877 start_ptr = cur_ptr;
1878 cstate->raw_fields[fieldno] = output_ptr;
1879
1880 /*
1881 * Scan data for field,
1882 *
1883 * The loop starts in "not quote" mode and then toggles between that
1884 * and "in quote" mode. The loop exits normally if it is in "not
1885 * quote" mode and a delimiter or line end is seen.
1886 */
1887 for (;;)
1888 {
1889 char c;
1890
1891 /* Not in quote */
1892 for (;;)
1893 {
1894 end_ptr = cur_ptr;
1895 if (cur_ptr >= line_end_ptr)
1896 goto endfield;
1897 c = *cur_ptr++;
1898 /* unquoted field delimiter */
1899 if (c == delimc)
1900 {
1901 found_delim = true;
1902 goto endfield;
1903 }
1904 /* start of quoted field (or part of field) */
1905 if (c == quotec)
1906 {
1907 saw_quote = true;
1908 break;
1909 }
1910 /* Add c to output string */
1911 *output_ptr++ = c;
1912 }
1913
1914 /* In quote */
1915 for (;;)
1916 {
1917 end_ptr = cur_ptr;
1918 if (cur_ptr >= line_end_ptr)
1919 ereport(ERROR,
1920 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1921 errmsg("unterminated CSV quoted field")));
1922
1923 c = *cur_ptr++;
1924
1925 /* escape within a quoted field */
1926 if (c == escapec)
1927 {
1928 /*
1929 * peek at the next char if available, and escape it if it
1930 * is an escape char or a quote char
1931 */
1932 if (cur_ptr < line_end_ptr)
1933 {
1934 char nextc = *cur_ptr;
1935
1936 if (nextc == escapec || nextc == quotec)
1937 {
1938 *output_ptr++ = nextc;
1939 cur_ptr++;
1940 continue;
1941 }
1942 }
1943 }
1944
1945 /*
1946 * end of quoted field. Must do this test after testing for
1947 * escape in case quote char and escape char are the same
1948 * (which is the common case).
1949 */
1950 if (c == quotec)
1951 break;
1952
1953 /* Add c to output string */
1954 *output_ptr++ = c;
1955 }
1956 }
1957endfield:
1958
1959 /* Terminate attribute value in output area */
1960 *output_ptr++ = '\0';
1961
1962 /* Check whether raw input matched null marker */
1963 input_len = end_ptr - start_ptr;
1964 if (!saw_quote && input_len == cstate->opts.null_print_len &&
1965 strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
1966 cstate->raw_fields[fieldno] = NULL;
1967 /* Check whether raw input matched default marker */
1968 else if (fieldno < list_length(cstate->attnumlist) &&
1969 cstate->opts.default_print &&
1970 input_len == cstate->opts.default_print_len &&
1971 strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
1972 {
1973 /* fieldno is 0-index and attnum is 1-index */
1974 int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
1975
1976 if (cstate->defexprs[m] != NULL)
1977 {
1978 /* defaults contain entries for all physical attributes */
1979 cstate->defaults[m] = true;
1980 }
1981 else
1982 {
1983 TupleDesc tupDesc = RelationGetDescr(cstate->rel);
1984 Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1985
1986 ereport(ERROR,
1987 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1988 errmsg("unexpected default marker in COPY data"),
1989 errdetail("Column \"%s\" has no default value.",
1990 NameStr(att->attname))));
1991 }
1992 }
1993
1994 fieldno++;
1995 /* Done if we hit EOL instead of a delim */
1996 if (!found_delim)
1997 break;
1998 }
1999
2000 /* Clean up state of attribute_buf */
2001 output_ptr--;
2002 Assert(*output_ptr == '\0');
2003 cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
2004
2005 return fieldno;
2006}
int errdetail(const char *fmt,...)
Definition: elog.c:1203
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1544
static int list_nth_int(const List *list, int n)
Definition: pg_list.h:310
char * c
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:126
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:337
int default_print_len
Definition: copy.h:72
int null_print_len
Definition: copy.h:69
char * quote
Definition: copy.h:74
char * escape
Definition: copy.h:75
char * delim
Definition: copy.h:73
char * default_print
Definition: copy.h:71
StringInfoData line_buf
StringInfoData attribute_buf

References Assert(), CopyFromStateData::attnumlist, CopyFromStateData::attribute_buf, StringInfoData::data, CopyFormatOptions::default_print, CopyFormatOptions::default_print_len, CopyFromStateData::defaults, CopyFromStateData::defexprs, CopyFormatOptions::delim, enlargeStringInfo(), ereport, errcode(), errdetail(), errmsg(), ERROR, CopyFormatOptions::escape, StringInfoData::len, CopyFromStateData::line_buf, list_length(), list_nth_int(), CopyFromStateData::max_fields, StringInfoData::maxlen, NameStr, CopyFormatOptions::null_print, CopyFormatOptions::null_print_len, CopyFromStateData::opts, CopyFormatOptions::quote, CopyFromStateData::raw_fields, CopyFromStateData::rel, RelationGetDescr, repalloc(), resetStringInfo(), and TupleDescAttr().

Referenced by NextCopyFromRawFieldsInternal().

◆ CopyReadAttributesText()

static int CopyReadAttributesText ( CopyFromState  cstate)
static

Definition at line 1564 of file copyfromparse.c.

1565{
1566 char delimc = cstate->opts.delim[0];
1567 int fieldno;
1568 char *output_ptr;
1569 char *cur_ptr;
1570 char *line_end_ptr;
1571
1572 /*
1573 * We need a special case for zero-column tables: check that the input
1574 * line is empty, and return.
1575 */
1576 if (cstate->max_fields <= 0)
1577 {
1578 if (cstate->line_buf.len != 0)
1579 ereport(ERROR,
1580 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1581 errmsg("extra data after last expected column")));
1582 return 0;
1583 }
1584
1586
1587 /*
1588 * The de-escaped attributes will certainly not be longer than the input
1589 * data line, so we can just force attribute_buf to be large enough and
1590 * then transfer data without any checks for enough space. We need to do
1591 * it this way because enlarging attribute_buf mid-stream would invalidate
1592 * pointers already stored into cstate->raw_fields[].
1593 */
1594 if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
1595 enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
1596 output_ptr = cstate->attribute_buf.data;
1597
1598 /* set pointer variables for loop */
1599 cur_ptr = cstate->line_buf.data;
1600 line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
1601
1602 /* Outer loop iterates over fields */
1603 fieldno = 0;
1604 for (;;)
1605 {
1606 bool found_delim = false;
1607 char *start_ptr;
1608 char *end_ptr;
1609 int input_len;
1610 bool saw_non_ascii = false;
1611
1612 /* Make sure there is enough space for the next value */
1613 if (fieldno >= cstate->max_fields)
1614 {
1615 cstate->max_fields *= 2;
1616 cstate->raw_fields =
1617 repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
1618 }
1619
1620 /* Remember start of field on both input and output sides */
1621 start_ptr = cur_ptr;
1622 cstate->raw_fields[fieldno] = output_ptr;
1623
1624 /*
1625 * Scan data for field.
1626 *
1627 * Note that in this loop, we are scanning to locate the end of field
1628 * and also speculatively performing de-escaping. Once we find the
1629 * end-of-field, we can match the raw field contents against the null
1630 * marker string. Only after that comparison fails do we know that
1631 * de-escaping is actually the right thing to do; therefore we *must
1632 * not* throw any syntax errors before we've done the null-marker
1633 * check.
1634 */
1635 for (;;)
1636 {
1637 char c;
1638
1639 end_ptr = cur_ptr;
1640 if (cur_ptr >= line_end_ptr)
1641 break;
1642 c = *cur_ptr++;
1643 if (c == delimc)
1644 {
1645 found_delim = true;
1646 break;
1647 }
1648 if (c == '\\')
1649 {
1650 if (cur_ptr >= line_end_ptr)
1651 break;
1652 c = *cur_ptr++;
1653 switch (c)
1654 {
1655 case '0':
1656 case '1':
1657 case '2':
1658 case '3':
1659 case '4':
1660 case '5':
1661 case '6':
1662 case '7':
1663 {
1664 /* handle \013 */
1665 int val;
1666
1667 val = OCTVALUE(c);
1668 if (cur_ptr < line_end_ptr)
1669 {
1670 c = *cur_ptr;
1671 if (ISOCTAL(c))
1672 {
1673 cur_ptr++;
1674 val = (val << 3) + OCTVALUE(c);
1675 if (cur_ptr < line_end_ptr)
1676 {
1677 c = *cur_ptr;
1678 if (ISOCTAL(c))
1679 {
1680 cur_ptr++;
1681 val = (val << 3) + OCTVALUE(c);
1682 }
1683 }
1684 }
1685 }
1686 c = val & 0377;
1687 if (c == '\0' || IS_HIGHBIT_SET(c))
1688 saw_non_ascii = true;
1689 }
1690 break;
1691 case 'x':
1692 /* Handle \x3F */
1693 if (cur_ptr < line_end_ptr)
1694 {
1695 char hexchar = *cur_ptr;
1696
1697 if (isxdigit((unsigned char) hexchar))
1698 {
1699 int val = GetDecimalFromHex(hexchar);
1700
1701 cur_ptr++;
1702 if (cur_ptr < line_end_ptr)
1703 {
1704 hexchar = *cur_ptr;
1705 if (isxdigit((unsigned char) hexchar))
1706 {
1707 cur_ptr++;
1708 val = (val << 4) + GetDecimalFromHex(hexchar);
1709 }
1710 }
1711 c = val & 0xff;
1712 if (c == '\0' || IS_HIGHBIT_SET(c))
1713 saw_non_ascii = true;
1714 }
1715 }
1716 break;
1717 case 'b':
1718 c = '\b';
1719 break;
1720 case 'f':
1721 c = '\f';
1722 break;
1723 case 'n':
1724 c = '\n';
1725 break;
1726 case 'r':
1727 c = '\r';
1728 break;
1729 case 't':
1730 c = '\t';
1731 break;
1732 case 'v':
1733 c = '\v';
1734 break;
1735
1736 /*
1737 * in all other cases, take the char after '\'
1738 * literally
1739 */
1740 }
1741 }
1742
1743 /* Add c to output string */
1744 *output_ptr++ = c;
1745 }
1746
1747 /* Check whether raw input matched null marker */
1748 input_len = end_ptr - start_ptr;
1749 if (input_len == cstate->opts.null_print_len &&
1750 strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
1751 cstate->raw_fields[fieldno] = NULL;
1752 /* Check whether raw input matched default marker */
1753 else if (fieldno < list_length(cstate->attnumlist) &&
1754 cstate->opts.default_print &&
1755 input_len == cstate->opts.default_print_len &&
1756 strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
1757 {
1758 /* fieldno is 0-indexed and attnum is 1-indexed */
1759 int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
1760
1761 if (cstate->defexprs[m] != NULL)
1762 {
1763 /* defaults contain entries for all physical attributes */
1764 cstate->defaults[m] = true;
1765 }
1766 else
1767 {
1768 TupleDesc tupDesc = RelationGetDescr(cstate->rel);
1769 Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1770
1771 ereport(ERROR,
1772 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1773 errmsg("unexpected default marker in COPY data"),
1774 errdetail("Column \"%s\" has no default value.",
1775 NameStr(att->attname))));
1776 }
1777 }
1778 else
1779 {
1780 /*
1781 * At this point we know the field is supposed to contain data.
1782 *
1783 * If we de-escaped any non-7-bit-ASCII chars, make sure the
1784 * resulting string is valid data for the db encoding.
1785 */
1786 if (saw_non_ascii)
1787 {
1788 char *fld = cstate->raw_fields[fieldno];
1789
1790 pg_verifymbstr(fld, output_ptr - fld, false);
1791 }
1792 }
1793
1794 /* Terminate attribute value in output area */
1795 *output_ptr++ = '\0';
1796
1797 fieldno++;
1798 /* Done if we hit EOL instead of a delim */
1799 if (!found_delim)
1800 break;
1801 }
1802
1803 /* Clean up state of attribute_buf */
1804 output_ptr--;
1805 Assert(*output_ptr == '\0');
1806 cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
1807
1808 return fieldno;
1809}
#define IS_HIGHBIT_SET(ch)
Definition: c.h:1126
#define OCTVALUE(c)
Definition: copyfromparse.c:79
#define ISOCTAL(c)
Definition: copyfromparse.c:78
static int GetDecimalFromHex(char hex)
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:78
bool pg_verifymbstr(const char *mbstr, int len, bool noError)
Definition: mbutils.c:1556

References Assert(), CopyFromStateData::attnumlist, CopyFromStateData::attribute_buf, StringInfoData::data, CopyFormatOptions::default_print, CopyFormatOptions::default_print_len, CopyFromStateData::defaults, CopyFromStateData::defexprs, CopyFormatOptions::delim, enlargeStringInfo(), ereport, errcode(), errdetail(), errmsg(), ERROR, GetDecimalFromHex(), if(), IS_HIGHBIT_SET, ISOCTAL, StringInfoData::len, CopyFromStateData::line_buf, list_length(), list_nth_int(), CopyFromStateData::max_fields, StringInfoData::maxlen, NameStr, CopyFormatOptions::null_print, CopyFormatOptions::null_print_len, OCTVALUE, CopyFromStateData::opts, pg_verifymbstr(), CopyFromStateData::raw_fields, CopyFromStateData::rel, RelationGetDescr, repalloc(), resetStringInfo(), TupleDescAttr(), and val.

Referenced by NextCopyFromRawFieldsInternal().

◆ CopyReadBinaryAttribute()

static Datum CopyReadBinaryAttribute ( CopyFromState  cstate,
FmgrInfo flinfo,
Oid  typioparam,
int32  typmod,
bool *  isnull 
)
static

Definition at line 2013 of file copyfromparse.c.

2016{
2017 int32 fld_size;
2018 Datum result;
2019
2020 if (!CopyGetInt32(cstate, &fld_size))
2021 ereport(ERROR,
2022 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2023 errmsg("unexpected EOF in COPY data")));
2024 if (fld_size == -1)
2025 {
2026 *isnull = true;
2027 return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
2028 }
2029 if (fld_size < 0)
2030 ereport(ERROR,
2031 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2032 errmsg("invalid field size")));
2033
2034 /* reset attribute_buf to empty, and load raw data in it */
2036
2037 enlargeStringInfo(&cstate->attribute_buf, fld_size);
2038 if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
2039 fld_size) != fld_size)
2040 ereport(ERROR,
2041 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2042 errmsg("unexpected EOF in COPY data")));
2043
2044 cstate->attribute_buf.len = fld_size;
2045 cstate->attribute_buf.data[fld_size] = '\0';
2046
2047 /* Call the column type's binary input converter */
2048 result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
2049 typioparam, typmod);
2050
2051 /* Trouble if it didn't eat the whole buffer */
2052 if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
2053 ereport(ERROR,
2054 (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
2055 errmsg("incorrect binary data format")));
2056
2057 *isnull = false;
2058 return result;
2059}
static bool CopyGetInt32(CopyFromState cstate, int32 *val)
Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, Oid typioparam, int32 typmod)
Definition: fmgr.c:1697
uintptr_t Datum
Definition: postgres.h:69

References CopyFromStateData::attribute_buf, CopyGetInt32(), CopyReadBinaryData(), StringInfoData::cursor, StringInfoData::data, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, StringInfoData::len, ReceiveFunctionCall(), and resetStringInfo().

Referenced by CopyFromBinaryOneRow().

◆ CopyReadBinaryData()

static int CopyReadBinaryData ( CopyFromState  cstate,
char *  dest,
int  nbytes 
)
static

Definition at line 701 of file copyfromparse.c.

702{
703 int copied_bytes = 0;
704
705 if (RAW_BUF_BYTES(cstate) >= nbytes)
706 {
707 /* Enough bytes are present in the buffer. */
708 memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes);
709 cstate->raw_buf_index += nbytes;
710 copied_bytes = nbytes;
711 }
712 else
713 {
714 /*
715 * Not enough bytes in the buffer, so must read from the file. Need
716 * to loop since 'nbytes' could be larger than the buffer size.
717 */
718 do
719 {
720 int copy_bytes;
721
722 /* Load more data if buffer is empty. */
723 if (RAW_BUF_BYTES(cstate) == 0)
724 {
725 CopyLoadRawBuf(cstate);
726 if (cstate->raw_reached_eof)
727 break; /* EOF */
728 }
729
730 /* Transfer some bytes. */
731 copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate));
732 memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes);
733 cstate->raw_buf_index += copy_bytes;
734 dest += copy_bytes;
735 copied_bytes += copy_bytes;
736 } while (copied_bytes < nbytes);
737 }
738
739 return copied_bytes;
740}
#define Min(x, y)
Definition: c.h:975

References CopyLoadRawBuf(), generate_unaccent_rules::dest, Min, CopyFromStateData::raw_buf, RAW_BUF_BYTES, CopyFromStateData::raw_buf_index, and CopyFromStateData::raw_reached_eof.

Referenced by CopyFromBinaryOneRow(), CopyGetInt16(), CopyGetInt32(), CopyReadBinaryAttribute(), and ReceiveCopyBinaryHeader().

◆ CopyReadLine()

static bool CopyReadLine ( CopyFromState  cstate,
bool  is_csv 
)
static

Definition at line 1158 of file copyfromparse.c.

1159{
1160 bool result;
1161
1162 resetStringInfo(&cstate->line_buf);
1163 cstate->line_buf_valid = false;
1164
1165 /* Parse data and transfer into line_buf */
1166 result = CopyReadLineText(cstate, is_csv);
1167
1168 if (result)
1169 {
1170 /*
1171 * Reached EOF. In protocol version 3, we should ignore anything
1172 * after \. up to the protocol end of copy data. (XXX maybe better
1173 * not to treat \. as special?)
1174 */
1175 if (cstate->copy_src == COPY_FRONTEND)
1176 {
1177 int inbytes;
1178
1179 do
1180 {
1181 inbytes = CopyGetData(cstate, cstate->input_buf,
1182 1, INPUT_BUF_SIZE);
1183 } while (inbytes > 0);
1184 cstate->input_buf_index = 0;
1185 cstate->input_buf_len = 0;
1186 cstate->raw_buf_index = 0;
1187 cstate->raw_buf_len = 0;
1188 }
1189 }
1190 else
1191 {
1192 /*
1193 * If we didn't hit EOF, then we must have transferred the EOL marker
1194 * to line_buf along with the data. Get rid of it.
1195 */
1196 switch (cstate->eol_type)
1197 {
1198 case EOL_NL:
1199 Assert(cstate->line_buf.len >= 1);
1200 Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1201 cstate->line_buf.len--;
1202 cstate->line_buf.data[cstate->line_buf.len] = '\0';
1203 break;
1204 case EOL_CR:
1205 Assert(cstate->line_buf.len >= 1);
1206 Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
1207 cstate->line_buf.len--;
1208 cstate->line_buf.data[cstate->line_buf.len] = '\0';
1209 break;
1210 case EOL_CRNL:
1211 Assert(cstate->line_buf.len >= 2);
1212 Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
1213 Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1214 cstate->line_buf.len -= 2;
1215 cstate->line_buf.data[cstate->line_buf.len] = '\0';
1216 break;
1217 case EOL_UNKNOWN:
1218 /* shouldn't get here */
1219 Assert(false);
1220 break;
1221 }
1222 }
1223
1224 /* Now it's safe to use the buffer in error messages */
1225 cstate->line_buf_valid = true;
1226
1227 return result;
1228}
@ EOL_CR
@ EOL_CRNL
@ EOL_UNKNOWN
@ EOL_NL
static bool CopyReadLineText(CopyFromState cstate, bool is_csv)

References Assert(), COPY_FRONTEND, CopyFromStateData::copy_src, CopyGetData(), CopyReadLineText(), StringInfoData::data, EOL_CR, EOL_CRNL, EOL_NL, CopyFromStateData::eol_type, EOL_UNKNOWN, CopyFromStateData::input_buf, CopyFromStateData::input_buf_index, CopyFromStateData::input_buf_len, INPUT_BUF_SIZE, StringInfoData::len, CopyFromStateData::line_buf, CopyFromStateData::line_buf_valid, CopyFromStateData::raw_buf_index, CopyFromStateData::raw_buf_len, and resetStringInfo().

Referenced by NextCopyFromRawFieldsInternal().

◆ CopyReadLineText()

static bool CopyReadLineText ( CopyFromState  cstate,
bool  is_csv 
)
static

Definition at line 1234 of file copyfromparse.c.

1235{
1236 char *copy_input_buf;
1237 int input_buf_ptr;
1238 int copy_buf_len;
1239 bool need_data = false;
1240 bool hit_eof = false;
1241 bool result = false;
1242
1243 /* CSV variables */
1244 bool in_quote = false,
1245 last_was_esc = false;
1246 char quotec = '\0';
1247 char escapec = '\0';
1248
1249 if (is_csv)
1250 {
1251 quotec = cstate->opts.quote[0];
1252 escapec = cstate->opts.escape[0];
1253 /* ignore special escape processing if it's the same as quotec */
1254 if (quotec == escapec)
1255 escapec = '\0';
1256 }
1257
1258 /*
1259 * The objective of this loop is to transfer the entire next input line
1260 * into line_buf. Hence, we only care for detecting newlines (\r and/or
1261 * \n) and the end-of-copy marker (\.).
1262 *
1263 * In CSV mode, \r and \n inside a quoted field are just part of the data
1264 * value and are put in line_buf. We keep just enough state to know if we
1265 * are currently in a quoted field or not.
1266 *
1267 * The input has already been converted to the database encoding. All
1268 * supported server encodings have the property that all bytes in a
1269 * multi-byte sequence have the high bit set, so a multibyte character
1270 * cannot contain any newline or escape characters embedded in the
1271 * multibyte sequence. Therefore, we can process the input byte-by-byte,
1272 * regardless of the encoding.
1273 *
1274 * For speed, we try to move data from input_buf to line_buf in chunks
1275 * rather than one character at a time. input_buf_ptr points to the next
1276 * character to examine; any characters from input_buf_index to
1277 * input_buf_ptr have been determined to be part of the line, but not yet
1278 * transferred to line_buf.
1279 *
1280 * For a little extra speed within the loop, we copy input_buf and
1281 * input_buf_len into local variables.
1282 */
1283 copy_input_buf = cstate->input_buf;
1284 input_buf_ptr = cstate->input_buf_index;
1285 copy_buf_len = cstate->input_buf_len;
1286
1287 for (;;)
1288 {
1289 int prev_raw_ptr;
1290 char c;
1291
1292 /*
1293 * Load more data if needed.
1294 *
1295 * TODO: We could just force four bytes of read-ahead and avoid the
1296 * many calls to IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(). That was
1297 * unsafe with the old v2 COPY protocol, but we don't support that
1298 * anymore.
1299 */
1300 if (input_buf_ptr >= copy_buf_len || need_data)
1301 {
1303
1304 CopyLoadInputBuf(cstate);
1305 /* update our local variables */
1306 hit_eof = cstate->input_reached_eof;
1307 input_buf_ptr = cstate->input_buf_index;
1308 copy_buf_len = cstate->input_buf_len;
1309
1310 /*
1311 * If we are completely out of data, break out of the loop,
1312 * reporting EOF.
1313 */
1314 if (INPUT_BUF_BYTES(cstate) <= 0)
1315 {
1316 result = true;
1317 break;
1318 }
1319 need_data = false;
1320 }
1321
1322 /* OK to fetch a character */
1323 prev_raw_ptr = input_buf_ptr;
1324 c = copy_input_buf[input_buf_ptr++];
1325
1326 if (is_csv)
1327 {
1328 /*
1329 * If character is '\r', we may need to look ahead below. Force
1330 * fetch of the next character if we don't already have it. We
1331 * need to do this before changing CSV state, in case '\r' is also
1332 * the quote or escape character.
1333 */
1334 if (c == '\r')
1335 {
1337 }
1338
1339 /*
1340 * Dealing with quotes and escapes here is mildly tricky. If the
1341 * quote char is also the escape char, there's no problem - we
1342 * just use the char as a toggle. If they are different, we need
1343 * to ensure that we only take account of an escape inside a
1344 * quoted field and immediately preceding a quote char, and not
1345 * the second in an escape-escape sequence.
1346 */
1347 if (in_quote && c == escapec)
1348 last_was_esc = !last_was_esc;
1349 if (c == quotec && !last_was_esc)
1350 in_quote = !in_quote;
1351 if (c != escapec)
1352 last_was_esc = false;
1353
1354 /*
1355 * Updating the line count for embedded CR and/or LF chars is
1356 * necessarily a little fragile - this test is probably about the
1357 * best we can do. (XXX it's arguable whether we should do this
1358 * at all --- is cur_lineno a physical or logical count?)
1359 */
1360 if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
1361 cstate->cur_lineno++;
1362 }
1363
1364 /* Process \r */
1365 if (c == '\r' && (!is_csv || !in_quote))
1366 {
1367 /* Check for \r\n on first line, _and_ handle \r\n. */
1368 if (cstate->eol_type == EOL_UNKNOWN ||
1369 cstate->eol_type == EOL_CRNL)
1370 {
1371 /*
1372 * If need more data, go back to loop top to load it.
1373 *
1374 * Note that if we are at EOF, c will wind up as '\0' because
1375 * of the guaranteed pad of input_buf.
1376 */
1378
1379 /* get next char */
1380 c = copy_input_buf[input_buf_ptr];
1381
1382 if (c == '\n')
1383 {
1384 input_buf_ptr++; /* eat newline */
1385 cstate->eol_type = EOL_CRNL; /* in case not set yet */
1386 }
1387 else
1388 {
1389 /* found \r, but no \n */
1390 if (cstate->eol_type == EOL_CRNL)
1391 ereport(ERROR,
1392 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1393 !is_csv ?
1394 errmsg("literal carriage return found in data") :
1395 errmsg("unquoted carriage return found in data"),
1396 !is_csv ?
1397 errhint("Use \"\\r\" to represent carriage return.") :
1398 errhint("Use quoted CSV field to represent carriage return.")));
1399
1400 /*
1401 * if we got here, it is the first line and we didn't find
1402 * \n, so don't consume the peeked character
1403 */
1404 cstate->eol_type = EOL_CR;
1405 }
1406 }
1407 else if (cstate->eol_type == EOL_NL)
1408 ereport(ERROR,
1409 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1410 !is_csv ?
1411 errmsg("literal carriage return found in data") :
1412 errmsg("unquoted carriage return found in data"),
1413 !is_csv ?
1414 errhint("Use \"\\r\" to represent carriage return.") :
1415 errhint("Use quoted CSV field to represent carriage return.")));
1416 /* If reach here, we have found the line terminator */
1417 break;
1418 }
1419
1420 /* Process \n */
1421 if (c == '\n' && (!is_csv || !in_quote))
1422 {
1423 if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
1424 ereport(ERROR,
1425 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1426 !is_csv ?
1427 errmsg("literal newline found in data") :
1428 errmsg("unquoted newline found in data"),
1429 !is_csv ?
1430 errhint("Use \"\\n\" to represent newline.") :
1431 errhint("Use quoted CSV field to represent newline.")));
1432 cstate->eol_type = EOL_NL; /* in case not set yet */
1433 /* If reach here, we have found the line terminator */
1434 break;
1435 }
1436
1437 /*
1438 * Process backslash, except in CSV mode where backslash is a normal
1439 * character.
1440 */
1441 if (c == '\\' && !is_csv)
1442 {
1443 char c2;
1444
1447
1448 /* -----
1449 * get next character
1450 * Note: we do not change c so if it isn't \., we can fall
1451 * through and continue processing.
1452 * -----
1453 */
1454 c2 = copy_input_buf[input_buf_ptr];
1455
1456 if (c2 == '.')
1457 {
1458 input_buf_ptr++; /* consume the '.' */
1459 if (cstate->eol_type == EOL_CRNL)
1460 {
1461 /* Get the next character */
1463 /* if hit_eof, c2 will become '\0' */
1464 c2 = copy_input_buf[input_buf_ptr++];
1465
1466 if (c2 == '\n')
1467 ereport(ERROR,
1468 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1469 errmsg("end-of-copy marker does not match previous newline style")));
1470 else if (c2 != '\r')
1471 ereport(ERROR,
1472 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1473 errmsg("end-of-copy marker is not alone on its line")));
1474 }
1475
1476 /* Get the next character */
1478 /* if hit_eof, c2 will become '\0' */
1479 c2 = copy_input_buf[input_buf_ptr++];
1480
1481 if (c2 != '\r' && c2 != '\n')
1482 ereport(ERROR,
1483 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1484 errmsg("end-of-copy marker is not alone on its line")));
1485
1486 if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
1487 (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
1488 (cstate->eol_type == EOL_CR && c2 != '\r'))
1489 ereport(ERROR,
1490 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1491 errmsg("end-of-copy marker does not match previous newline style")));
1492
1493 /*
1494 * If there is any data on this line before the \., complain.
1495 */
1496 if (cstate->line_buf.len > 0 ||
1497 prev_raw_ptr > cstate->input_buf_index)
1498 ereport(ERROR,
1499 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1500 errmsg("end-of-copy marker is not alone on its line")));
1501
1502 /*
1503 * Discard the \. and newline, then report EOF.
1504 */
1505 cstate->input_buf_index = input_buf_ptr;
1506 result = true; /* report EOF */
1507 break;
1508 }
1509 else
1510 {
1511 /*
1512 * If we are here, it means we found a backslash followed by
1513 * something other than a period. In non-CSV mode, anything
1514 * after a backslash is special, so we skip over that second
1515 * character too. If we didn't do that \\. would be
1516 * considered an eof-of copy, while in non-CSV mode it is a
1517 * literal backslash followed by a period.
1518 */
1519 input_buf_ptr++;
1520 }
1521 }
1522 } /* end of outer loop */
1523
1524 /*
1525 * Transfer any still-uncopied data to line_buf.
1526 */
1528
1529 return result;
1530}
#define REFILL_LINEBUF
static void CopyLoadInputBuf(CopyFromState cstate)
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
Definition: copyfromparse.c:97
int errhint(const char *fmt,...)
Definition: elog.c:1317

References CopyLoadInputBuf(), CopyFromStateData::cur_lineno, EOL_CR, EOL_CRNL, EOL_NL, CopyFromStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errhint(), errmsg(), ERROR, CopyFormatOptions::escape, IF_NEED_REFILL_AND_EOF_BREAK, IF_NEED_REFILL_AND_NOT_EOF_CONTINUE, CopyFromStateData::input_buf, INPUT_BUF_BYTES, CopyFromStateData::input_buf_index, CopyFromStateData::input_buf_len, CopyFromStateData::input_reached_eof, StringInfoData::len, CopyFromStateData::line_buf, CopyFromStateData::opts, CopyFormatOptions::quote, and REFILL_LINEBUF.

Referenced by CopyReadLine().

◆ GetDecimalFromHex()

static int GetDecimalFromHex ( char  hex)
static

Definition at line 1536 of file copyfromparse.c.

1537{
1538 if (isdigit((unsigned char) hex))
1539 return hex - '0';
1540 else
1541 return tolower((unsigned char) hex) - 'a' + 10;
1542}

Referenced by CopyReadAttributesText().

◆ NextCopyFrom()

bool NextCopyFrom ( CopyFromState  cstate,
ExprContext econtext,
Datum values,
bool *  nulls 
)

Definition at line 871 of file copyfromparse.c.

873{
874 TupleDesc tupDesc;
875 AttrNumber num_phys_attrs,
876 num_defaults = cstate->num_defaults;
877 int i;
878 int *defmap = cstate->defmap;
879 ExprState **defexprs = cstate->defexprs;
880
881 tupDesc = RelationGetDescr(cstate->rel);
882 num_phys_attrs = tupDesc->natts;
883
884 /* Initialize all values for row to NULL */
885 MemSet(values, 0, num_phys_attrs * sizeof(Datum));
886 MemSet(nulls, true, num_phys_attrs * sizeof(bool));
887 MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
888
889 /* Get one row from source */
890 if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls))
891 return false;
892
893 /*
894 * Now compute and insert any defaults available for the columns not
895 * provided by the input data. Anything not processed here or above will
896 * remain NULL.
897 */
898 for (i = 0; i < num_defaults; i++)
899 {
900 /*
901 * The caller must supply econtext and have switched into the
902 * per-tuple memory context in it.
903 */
904 Assert(econtext != NULL);
906
907 values[defmap[i]] = ExecEvalExpr(defexprs[defmap[i]], econtext,
908 &nulls[defmap[i]]);
909 }
910
911 return true;
912}
#define MemSet(start, val, len)
Definition: c.h:991
int i
Definition: isn.c:74
bool(* CopyFromOneRow)(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
Definition: copyapi.h:96
const struct CopyFromRoutine * routine

References Assert(), CopyFromRoutine::CopyFromOneRow, CurrentMemoryContext, CopyFromStateData::defaults, CopyFromStateData::defexprs, CopyFromStateData::defmap, ExprContext::ecxt_per_tuple_memory, ExecEvalExpr(), i, MemSet, TupleDescData::natts, CopyFromStateData::num_defaults, CopyFromStateData::rel, RelationGetDescr, CopyFromStateData::routine, and values.

Referenced by CopyFrom(), file_acquire_sample_rows(), and fileIterateForeignScan().

◆ NextCopyFromRawFields()

bool NextCopyFromRawFields ( CopyFromState  cstate,
char ***  fields,
int *  nfields 
)

Definition at line 747 of file copyfromparse.c.

748{
749 return NextCopyFromRawFieldsInternal(cstate, fields, nfields,
750 cstate->opts.csv_mode);
751}
bool csv_mode
Definition: copy.h:66

References CopyFormatOptions::csv_mode, NextCopyFromRawFieldsInternal(), and CopyFromStateData::opts.

◆ NextCopyFromRawFieldsInternal()

static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal ( CopyFromState  cstate,
char ***  fields,
int *  nfields,
bool  is_csv 
)
static

Definition at line 771 of file copyfromparse.c.

772{
773 int fldct;
774 bool done;
775
776 /* only available for text or csv input */
777 Assert(!cstate->opts.binary);
778
779 /* on input check that the header line is correct if needed */
780 if (cstate->cur_lineno == 0 && cstate->opts.header_line)
781 {
782 ListCell *cur;
783 TupleDesc tupDesc;
784
785 tupDesc = RelationGetDescr(cstate->rel);
786
787 cstate->cur_lineno++;
788 done = CopyReadLine(cstate, is_csv);
789
790 if (cstate->opts.header_line == COPY_HEADER_MATCH)
791 {
792 int fldnum;
793
794 if (is_csv)
795 fldct = CopyReadAttributesCSV(cstate);
796 else
797 fldct = CopyReadAttributesText(cstate);
798
799 if (fldct != list_length(cstate->attnumlist))
801 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
802 errmsg("wrong number of fields in header line: got %d, expected %d",
803 fldct, list_length(cstate->attnumlist))));
804
805 fldnum = 0;
806 foreach(cur, cstate->attnumlist)
807 {
808 int attnum = lfirst_int(cur);
809 char *colName;
810 Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
811
812 Assert(fldnum < cstate->max_fields);
813
814 colName = cstate->raw_fields[fldnum++];
815 if (colName == NULL)
817 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
818 errmsg("column name mismatch in header line field %d: got null value (\"%s\"), expected \"%s\"",
819 fldnum, cstate->opts.null_print, NameStr(attr->attname))));
820
821 if (namestrcmp(&attr->attname, colName) != 0)
822 {
824 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
825 errmsg("column name mismatch in header line field %d: got \"%s\", expected \"%s\"",
826 fldnum, colName, NameStr(attr->attname))));
827 }
828 }
829 }
830
831 if (done)
832 return false;
833 }
834
835 cstate->cur_lineno++;
836
837 /* Actually read the line into memory here */
838 done = CopyReadLine(cstate, is_csv);
839
840 /*
841 * EOF at start of line means we're done. If we see EOF after some
842 * characters, we act as though it was newline followed by EOF, ie,
843 * process the line and then exit loop on next iteration.
844 */
845 if (done && cstate->line_buf.len == 0)
846 return false;
847
848 /* Parse the line into de-escaped field values */
849 if (is_csv)
850 fldct = CopyReadAttributesCSV(cstate);
851 else
852 fldct = CopyReadAttributesText(cstate);
853
854 *fields = cstate->raw_fields;
855 *nfields = fldct;
856 return true;
857}
static int CopyReadAttributesCSV(CopyFromState cstate)
static int CopyReadAttributesText(CopyFromState cstate)
static bool CopyReadLine(CopyFromState cstate, bool is_csv)
@ COPY_HEADER_MATCH
Definition: copy.h:30
int namestrcmp(Name name, const char *str)
Definition: name.c:247
bool binary
Definition: copy.h:64
CopyHeaderChoice header_line
Definition: copy.h:67

References Assert(), attnum, CopyFromStateData::attnumlist, CopyFormatOptions::binary, COPY_HEADER_MATCH, CopyReadAttributesCSV(), CopyReadAttributesText(), CopyReadLine(), cur, CopyFromStateData::cur_lineno, ereport, errcode(), errmsg(), ERROR, CopyFormatOptions::header_line, StringInfoData::len, lfirst_int, CopyFromStateData::line_buf, list_length(), NameStr, namestrcmp(), CopyFormatOptions::null_print, CopyFromStateData::opts, CopyFromStateData::raw_fields, CopyFromStateData::rel, RelationGetDescr, and TupleDescAttr().

Referenced by CopyFromTextLikeOneRow(), and NextCopyFromRawFields().

◆ ReceiveCopyBegin()

void ReceiveCopyBegin ( CopyFromState  cstate)

Definition at line 170 of file copyfromparse.c.

171{
173 int natts = list_length(cstate->attnumlist);
174 int16 format = (cstate->opts.binary ? 1 : 0);
175 int i;
176
178 pq_sendbyte(&buf, format); /* overall format */
179 pq_sendint16(&buf, natts);
180 for (i = 0; i < natts; i++)
181 pq_sendint16(&buf, format); /* per-column formats */
183 cstate->copy_src = COPY_FRONTEND;
184 cstate->fe_msgbuf = makeStringInfo();
185 /* We *must* flush here to ensure FE knows it can send. */
186 pq_flush();
187}
#define pq_flush()
Definition: libpq.h:46
static char format
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:296
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:160
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:136
#define PqMsg_CopyInResponse
Definition: protocol.h:45
StringInfo makeStringInfo(void)
Definition: stringinfo.c:72

References CopyFromStateData::attnumlist, CopyFormatOptions::binary, buf, COPY_FRONTEND, CopyFromStateData::copy_src, CopyFromStateData::fe_msgbuf, format, i, list_length(), makeStringInfo(), CopyFromStateData::opts, pq_beginmessage(), pq_endmessage(), pq_flush, pq_sendbyte(), pq_sendint16(), and PqMsg_CopyInResponse.

Referenced by BeginCopyFrom().

◆ ReceiveCopyBinaryHeader()

void ReceiveCopyBinaryHeader ( CopyFromState  cstate)

Definition at line 190 of file copyfromparse.c.

191{
192 char readSig[11];
193 int32 tmp;
194
195 /* Signature */
196 if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
197 memcmp(readSig, BinarySignature, 11) != 0)
199 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
200 errmsg("COPY file signature not recognized")));
201 /* Flags field */
202 if (!CopyGetInt32(cstate, &tmp))
204 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
205 errmsg("invalid COPY file header (missing flags)")));
206 if ((tmp & (1 << 16)) != 0)
208 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
209 errmsg("invalid COPY file header (WITH OIDS)")));
210 tmp &= ~(1 << 16);
211 if ((tmp >> 16) != 0)
213 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
214 errmsg("unrecognized critical flags in COPY file header")));
215 /* Header extension length */
216 if (!CopyGetInt32(cstate, &tmp) ||
217 tmp < 0)
219 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
220 errmsg("invalid COPY file header (missing length)")));
221 /* Skip extension header, if present */
222 while (tmp-- > 0)
223 {
224 if (CopyReadBinaryData(cstate, readSig, 1) != 1)
226 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
227 errmsg("invalid COPY file header (wrong length)")));
228 }
229}
static const char BinarySignature[11]

References BinarySignature, CopyGetInt32(), CopyReadBinaryData(), ereport, errcode(), errmsg(), and ERROR.

Referenced by CopyFromBinaryStart().

Variable Documentation

◆ BinarySignature

const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"
static

Definition at line 139 of file copyfromparse.c.

Referenced by ReceiveCopyBinaryHeader().