PostgreSQL Source Code  git master
copyfrom_internal.h
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copyfrom_internal.h
4  * Internal definitions for COPY FROM command.
5  *
6  *
7  * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
8  * Portions Copyright (c) 1994, Regents of the University of California
9  *
10  * src/include/commands/copyfrom_internal.h
11  *
12  *-------------------------------------------------------------------------
13  */
14 #ifndef COPYFROM_INTERNAL_H
15 #define COPYFROM_INTERNAL_H
16 
17 #include "commands/copy.h"
18 #include "commands/trigger.h"
19 
20 /*
21  * Represents the different source cases we need to worry about at
22  * the bottom level
23  */
24 typedef enum CopySource
25 {
26  COPY_FILE, /* from file (or a piped program) */
27  COPY_FRONTEND, /* from frontend */
28  COPY_CALLBACK /* from callback function */
30 
31 /*
32  * Represents the end-of-line terminator type of the input
33  */
34 typedef enum EolType
35 {
39  EOL_CRNL
41 
42 /*
43  * Represents the insert method to be used during COPY FROM.
44  */
45 typedef enum CopyInsertMethod
46 {
47  CIM_SINGLE, /* use table_tuple_insert or ExecForeignInsert */
48  CIM_MULTI, /* always use table_multi_insert or
49  * ExecForeignBatchInsert */
50  CIM_MULTI_CONDITIONAL /* use table_multi_insert or
51  * ExecForeignBatchInsert only if valid */
53 
54 /*
55  * This struct contains all the state variables used throughout a COPY FROM
56  * operation.
57  */
58 typedef struct CopyFromStateData
59 {
60  /* low-level state data */
61  CopySource copy_src; /* type of copy source */
62  FILE *copy_file; /* used if copy_src == COPY_FILE */
63  StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
64 
65  EolType eol_type; /* EOL type of input */
66  int file_encoding; /* file or remote side's character encoding */
67  bool need_transcoding; /* file encoding diff from server? */
68  Oid conversion_proc; /* encoding conversion function */
69 
70  /* parameters from the COPY command */
71  Relation rel; /* relation to copy from */
72  List *attnumlist; /* integer list of attnums to copy */
73  char *filename; /* filename, or NULL for STDIN */
74  bool is_program; /* is 'filename' a program to popen? */
75  copy_data_source_cb data_source_cb; /* function for reading data */
76 
78  bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
79  Node *whereClause; /* WHERE condition (or NULL) */
80 
81  /* these are just for error messages, see CopyFromErrorCallback */
82  const char *cur_relname; /* table name for error messages */
83  uint64 cur_lineno; /* line number for error messages */
84  const char *cur_attname; /* current att for error messages */
85  const char *cur_attval; /* current att value for error messages */
86  bool relname_only; /* don't output line number, att, etc. */
87 
88  /*
89  * Working state
90  */
91  MemoryContext copycontext; /* per-copy execution context */
92 
93  AttrNumber num_defaults; /* count of att that are missing and have
94  * default value */
95  FmgrInfo *in_functions; /* array of input functions for each attrs */
96  Oid *typioparams; /* array of element types for in_functions */
97  int *defmap; /* array of default att numbers related to
98  * missing att */
99  ExprState **defexprs; /* array of default att expressions for all
100  * att */
101  bool *defaults; /* if DEFAULT marker was found for
102  * corresponding att */
103  bool volatile_defexprs; /* is any of defexprs volatile? */
104  List *range_table; /* single element list of RangeTblEntry */
105  List *rteperminfos; /* single element list of RTEPermissionInfo */
107 
109 
110  /*
111  * These variables are used to reduce overhead in COPY FROM.
112  *
113  * attribute_buf holds the separated, de-escaped text for each field of
114  * the current line. The CopyReadAttributes functions return arrays of
115  * pointers into this buffer. We avoid palloc/pfree overhead by re-using
116  * the buffer on each cycle.
117  *
118  * In binary COPY FROM, attribute_buf holds the binary data for the
119  * current field, but the usage is otherwise similar.
120  */
122 
123  /* field raw data pointers found by COPY FROM */
124 
126  char **raw_fields;
127 
128  /*
129  * Similarly, line_buf holds the whole input line being processed. The
130  * input cycle is first to read the whole line into line_buf, and then
131  * extract the individual attribute fields into attribute_buf. line_buf
132  * is preserved unmodified so that we can display it in error messages if
133  * appropriate. (In binary mode, line_buf is not used.)
134  */
136  bool line_buf_valid; /* contains the row being processed? */
137 
138  /*
139  * input_buf holds input data, already converted to database encoding.
140  *
141  * In text mode, CopyReadLine parses this data sufficiently to locate line
142  * boundaries, then transfers the data to line_buf. We guarantee that
143  * there is a \0 at input_buf[input_buf_len] at all times. (In binary
144  * mode, input_buf is not used.)
145  *
146  * If encoding conversion is not required, input_buf is not a separate
147  * buffer but points directly to raw_buf. In that case, input_buf_len
148  * tracks the number of bytes that have been verified as valid in the
149  * database encoding, and raw_buf_len is the total number of bytes stored
150  * in the buffer.
151  */
152 #define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
153  char *input_buf;
154  int input_buf_index; /* next byte to process */
155  int input_buf_len; /* total # of bytes stored */
156  bool input_reached_eof; /* true if we reached EOF */
157  bool input_reached_error; /* true if a conversion error happened */
158  /* Shorthand for number of unconsumed bytes available in input_buf */
159 #define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
160 
161  /*
162  * raw_buf holds raw input data read from the data source (file or client
163  * connection), not yet converted to the database encoding. Like with
164  * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
165  */
166 #define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
167  char *raw_buf;
168  int raw_buf_index; /* next byte to process */
169  int raw_buf_len; /* total # of bytes stored */
170  bool raw_reached_eof; /* true if we reached EOF */
171 
172  /* Shorthand for number of unconsumed bytes available in raw_buf */
173 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
174 
175  uint64 bytes_processed; /* number of bytes processed so far */
177 
178 extern void ReceiveCopyBegin(CopyFromState cstate);
179 extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
180 
181 #endif /* COPYFROM_INTERNAL_H */
int16 AttrNumber
Definition: attnum.h:21
struct CopyFromStateData CopyFromStateData
CopySource
@ COPY_FILE
@ COPY_CALLBACK
@ COPY_FRONTEND
CopyInsertMethod
@ CIM_SINGLE
@ CIM_MULTI_CONDITIONAL
@ CIM_MULTI
void ReceiveCopyBinaryHeader(CopyFromState cstate)
void ReceiveCopyBegin(CopyFromState cstate)
@ EOL_CR
@ EOL_CRNL
@ EOL_UNKNOWN
@ EOL_NL
int(* copy_data_source_cb)(void *outbuf, int minread, int maxread)
Definition: copy.h:70
unsigned int Oid
Definition: postgres_ext.h:31
ExprState ** defexprs
copy_data_source_cb data_source_cb
StringInfoData line_buf
CopyFormatOptions opts
StringInfoData attribute_buf
TransitionCaptureState * transition_capture
MemoryContext copycontext
const char * cur_attval
const char * cur_attname
const char * cur_relname
Definition: fmgr.h:57
Definition: pg_list.h:54
Definition: nodes.h:129