Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_backup_tar.c
4 : *
5 : * This file is copied from the 'files' format file, but dumps data into
6 : * one temp file then sends it to the output TAR archive.
7 : *
8 : * The tar format also includes a 'restore.sql' script which is there for
9 : * the benefit of humans. This script is never used by pg_restore.
10 : *
11 : * NOTE: If you untar the created 'tar' file, the resulting files are
12 : * compatible with the 'directory' format. Please keep the two formats in
13 : * sync.
14 : *
15 : * See the headers to pg_backup_directory & pg_restore for more details.
16 : *
17 : * Copyright (c) 2000, Philip Warner
18 : * Rights are granted to use this software in any way so long
19 : * as this notice is not removed.
20 : *
21 : * The author is not responsible for loss or damages that may
22 : * result from its use.
23 : *
24 : *
25 : * IDENTIFICATION
26 : * src/bin/pg_dump/pg_backup_tar.c
27 : *
28 : *-------------------------------------------------------------------------
29 : */
30 : #include "postgres_fe.h"
31 :
32 : #include <sys/stat.h>
33 : #include <ctype.h>
34 : #include <limits.h>
35 : #include <unistd.h>
36 :
37 : #include "common/file_utils.h"
38 : #include "fe_utils/string_utils.h"
39 : #include "pg_backup_archiver.h"
40 : #include "pg_backup_tar.h"
41 : #include "pg_backup_utils.h"
42 : #include "pgtar.h"
43 :
44 : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45 : static void _StartData(ArchiveHandle *AH, TocEntry *te);
46 : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47 : static void _EndData(ArchiveHandle *AH, TocEntry *te);
48 : static int _WriteByte(ArchiveHandle *AH, const int i);
49 : static int _ReadByte(ArchiveHandle *AH);
50 : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51 : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52 : static void _CloseArchive(ArchiveHandle *AH);
53 : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54 : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55 : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56 : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57 :
58 : static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
59 : static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
60 : static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
61 : static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
62 :
63 : #define K_STD_BUF_SIZE 1024
64 :
65 :
66 : typedef struct
67 : {
68 : FILE *nFH;
69 : FILE *tarFH;
70 : FILE *tmpFH;
71 : char *targetFile;
72 : char mode;
73 : pgoff_t pos;
74 : pgoff_t fileLen;
75 : ArchiveHandle *AH;
76 : } TAR_MEMBER;
77 :
78 : typedef struct
79 : {
80 : int hasSeek;
81 : pgoff_t filePos;
82 : TAR_MEMBER *loToc;
83 : FILE *tarFH;
84 : pgoff_t tarFHpos;
85 : pgoff_t tarNextMember;
86 : TAR_MEMBER *FH;
87 : int isSpecialScript;
88 : TAR_MEMBER *scriptTH;
89 : } lclContext;
90 :
91 : typedef struct
92 : {
93 : TAR_MEMBER *TH;
94 : char *filename;
95 : } lclTocEntry;
96 :
97 : static void _LoadLOs(ArchiveHandle *AH, TocEntry *te);
98 :
99 : static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
100 : static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th);
101 :
102 : #ifdef __NOT_USED__
103 : static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
104 : #endif
105 : static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2, 3);
106 :
107 : static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
108 : static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
109 : static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
110 : static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
111 : static void _tarWriteHeader(TAR_MEMBER *th);
112 : static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
113 : static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
114 :
115 : static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
116 :
117 : /*
118 : * Initializer
119 : */
120 : void
121 0 : InitArchiveFmt_Tar(ArchiveHandle *AH)
122 : {
123 0 : lclContext *ctx;
124 :
125 : /* Assuming static functions, this can be copied for each format. */
126 0 : AH->ArchiveEntryPtr = _ArchiveEntry;
127 0 : AH->StartDataPtr = _StartData;
128 0 : AH->WriteDataPtr = _WriteData;
129 0 : AH->EndDataPtr = _EndData;
130 0 : AH->WriteBytePtr = _WriteByte;
131 0 : AH->ReadBytePtr = _ReadByte;
132 0 : AH->WriteBufPtr = _WriteBuf;
133 0 : AH->ReadBufPtr = _ReadBuf;
134 0 : AH->ClosePtr = _CloseArchive;
135 0 : AH->ReopenPtr = NULL;
136 0 : AH->PrintTocDataPtr = _PrintTocData;
137 0 : AH->ReadExtraTocPtr = _ReadExtraToc;
138 0 : AH->WriteExtraTocPtr = _WriteExtraToc;
139 0 : AH->PrintExtraTocPtr = _PrintExtraToc;
140 :
141 0 : AH->StartLOsPtr = _StartLOs;
142 0 : AH->StartLOPtr = _StartLO;
143 0 : AH->EndLOPtr = _EndLO;
144 0 : AH->EndLOsPtr = _EndLOs;
145 0 : AH->ClonePtr = NULL;
146 0 : AH->DeClonePtr = NULL;
147 :
148 0 : AH->WorkerJobDumpPtr = NULL;
149 0 : AH->WorkerJobRestorePtr = NULL;
150 :
151 : /*
152 : * Set up some special context used in compressing data.
153 : */
154 0 : ctx = pg_malloc0_object(lclContext);
155 0 : AH->formatData = ctx;
156 0 : ctx->filePos = 0;
157 0 : ctx->isSpecialScript = 0;
158 :
159 : /*
160 : * Now open the tar file, and load the TOC if we're in read mode.
161 : */
162 0 : if (AH->mode == archModeWrite)
163 : {
164 0 : if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
165 : {
166 0 : ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
167 0 : if (ctx->tarFH == NULL)
168 0 : pg_fatal("could not open TOC file \"%s\" for output: %m",
169 : AH->fSpec);
170 0 : }
171 : else
172 : {
173 0 : ctx->tarFH = stdout;
174 0 : if (ctx->tarFH == NULL)
175 0 : pg_fatal("could not open TOC file for output: %m");
176 : }
177 :
178 0 : ctx->tarFHpos = 0;
179 :
180 : /*
181 : * Make unbuffered since we will dup() it, and the buffers screw each
182 : * other
183 : */
184 : /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
185 :
186 0 : ctx->hasSeek = checkSeek(ctx->tarFH);
187 :
188 : /*
189 : * We don't support compression because reading the files back is not
190 : * possible since gzdopen uses buffered IO which totally screws file
191 : * positioning.
192 : */
193 0 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
194 0 : pg_fatal("compression is not supported by tar archive format");
195 0 : }
196 : else
197 : { /* Read Mode */
198 0 : if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
199 : {
200 0 : ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
201 0 : if (ctx->tarFH == NULL)
202 0 : pg_fatal("could not open TOC file \"%s\" for input: %m",
203 : AH->fSpec);
204 0 : }
205 : else
206 : {
207 0 : ctx->tarFH = stdin;
208 0 : if (ctx->tarFH == NULL)
209 0 : pg_fatal("could not open TOC file for input: %m");
210 : }
211 :
212 : /*
213 : * Make unbuffered since we will dup() it, and the buffers screw each
214 : * other
215 : */
216 : /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
217 :
218 0 : ctx->tarFHpos = 0;
219 :
220 0 : ctx->hasSeek = checkSeek(ctx->tarFH);
221 :
222 0 : ctx->FH = tarOpen(AH, "toc.dat", 'r');
223 0 : ReadHead(AH);
224 0 : ReadToc(AH);
225 0 : tarClose(AH, ctx->FH); /* Nothing else in the file... */
226 : }
227 0 : }
228 :
229 : /*
230 : * - Start a new TOC entry
231 : * Setup the output file name.
232 : */
233 : static void
234 0 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
235 : {
236 0 : lclTocEntry *ctx;
237 0 : char fn[K_STD_BUF_SIZE];
238 :
239 0 : ctx = pg_malloc0_object(lclTocEntry);
240 0 : if (te->dataDumper != NULL)
241 : {
242 0 : snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
243 0 : ctx->filename = pg_strdup(fn);
244 0 : }
245 : else
246 : {
247 0 : ctx->filename = NULL;
248 0 : ctx->TH = NULL;
249 : }
250 0 : te->formatData = ctx;
251 0 : }
252 :
253 : static void
254 0 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
255 : {
256 0 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
257 :
258 0 : if (ctx->filename)
259 0 : WriteStr(AH, ctx->filename);
260 : else
261 0 : WriteStr(AH, "");
262 0 : }
263 :
264 : static void
265 0 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
266 : {
267 0 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
268 :
269 0 : if (ctx == NULL)
270 : {
271 0 : ctx = pg_malloc0_object(lclTocEntry);
272 0 : te->formatData = ctx;
273 0 : }
274 :
275 0 : ctx->filename = ReadStr(AH);
276 0 : if (strlen(ctx->filename) == 0)
277 : {
278 0 : free(ctx->filename);
279 0 : ctx->filename = NULL;
280 0 : }
281 0 : ctx->TH = NULL;
282 0 : }
283 :
284 : static void
285 0 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
286 : {
287 0 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
288 :
289 0 : if (AH->public.verbose && ctx->filename != NULL)
290 0 : ahprintf(AH, "-- File: %s\n", ctx->filename);
291 0 : }
292 :
293 : static void
294 0 : _StartData(ArchiveHandle *AH, TocEntry *te)
295 : {
296 0 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
297 :
298 0 : tctx->TH = tarOpen(AH, tctx->filename, 'w');
299 0 : }
300 :
301 : static TAR_MEMBER *
302 0 : tarOpen(ArchiveHandle *AH, const char *filename, char mode)
303 : {
304 0 : lclContext *ctx = (lclContext *) AH->formatData;
305 0 : TAR_MEMBER *tm;
306 :
307 0 : if (mode == 'r')
308 : {
309 0 : tm = _tarPositionTo(AH, filename);
310 0 : if (!tm) /* Not found */
311 : {
312 0 : if (filename)
313 : {
314 : /*
315 : * Couldn't find the requested file. Future: do SEEK(0) and
316 : * retry.
317 : */
318 0 : pg_fatal("could not find file \"%s\" in archive", filename);
319 0 : }
320 : else
321 : {
322 : /* Any file OK, none left, so return NULL */
323 0 : return NULL;
324 : }
325 0 : }
326 :
327 0 : if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
328 0 : tm->nFH = ctx->tarFH;
329 : else
330 0 : pg_fatal("compression is not supported by tar archive format");
331 0 : }
332 : else
333 : {
334 0 : int old_umask;
335 :
336 0 : tm = pg_malloc0_object(TAR_MEMBER);
337 :
338 : /*
339 : * POSIX does not require, but permits, tmpfile() to restrict file
340 : * permissions. Given an OS crash after we write data, the filesystem
341 : * might retain the data but forget tmpfile()'s unlink(). If so, the
342 : * file mode protects confidentiality of the data written.
343 : */
344 0 : old_umask = umask(S_IRWXG | S_IRWXO);
345 :
346 : #ifndef WIN32
347 0 : tm->tmpFH = tmpfile();
348 : #else
349 :
350 : /*
351 : * On WIN32, tmpfile() generates a filename in the root directory,
352 : * which requires administrative permissions on certain systems. Loop
353 : * until we find a unique file name we can create.
354 : */
355 : while (1)
356 : {
357 : char *name;
358 : int fd;
359 :
360 : name = _tempnam(NULL, "pg_temp_");
361 : if (name == NULL)
362 : break;
363 : fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
364 : O_TEMPORARY, S_IRUSR | S_IWUSR);
365 : free(name);
366 :
367 : if (fd != -1) /* created a file */
368 : {
369 : tm->tmpFH = fdopen(fd, "w+b");
370 : break;
371 : }
372 : else if (errno != EEXIST) /* failure other than file exists */
373 : break;
374 : }
375 : #endif
376 :
377 0 : if (tm->tmpFH == NULL)
378 0 : pg_fatal("could not generate temporary file name: %m");
379 :
380 0 : umask(old_umask);
381 :
382 0 : if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
383 0 : tm->nFH = tm->tmpFH;
384 : else
385 0 : pg_fatal("compression is not supported by tar archive format");
386 :
387 0 : tm->AH = AH;
388 0 : tm->targetFile = pg_strdup(filename);
389 0 : }
390 :
391 0 : tm->mode = mode;
392 0 : tm->tarFH = ctx->tarFH;
393 :
394 0 : return tm;
395 0 : }
396 :
397 : static void
398 0 : tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
399 : {
400 0 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
401 0 : pg_fatal("compression is not supported by tar archive format");
402 :
403 0 : if (th->mode == 'w')
404 0 : _tarAddFile(AH, th); /* This will close the temp file */
405 :
406 : /*
407 : * else Nothing to do for normal read since we don't dup() normal file
408 : * handle, and we don't use temp files.
409 : */
410 :
411 0 : free(th->targetFile);
412 :
413 0 : th->nFH = NULL;
414 0 : }
415 :
416 : #ifdef __NOT_USED__
417 : static char *
418 : tarGets(char *buf, size_t len, TAR_MEMBER *th)
419 : {
420 : char *s;
421 : size_t cnt = 0;
422 : char c = ' ';
423 : int eof = 0;
424 :
425 : /* Can't read past logical EOF */
426 : if (len > (th->fileLen - th->pos))
427 : len = th->fileLen - th->pos;
428 :
429 : while (cnt < len && c != '\n')
430 : {
431 : if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
432 : {
433 : eof = 1;
434 : break;
435 : }
436 : buf[cnt++] = c;
437 : }
438 :
439 : if (eof && cnt == 0)
440 : s = NULL;
441 : else
442 : {
443 : buf[cnt++] = '\0';
444 : s = buf;
445 : }
446 :
447 : if (s)
448 : {
449 : len = strlen(s);
450 : th->pos += len;
451 : }
452 :
453 : return s;
454 : }
455 : #endif
456 :
457 : /*
458 : * Just read bytes from the archive. This is the low level read routine
459 : * that is used for ALL reads on a tar file.
460 : */
461 : static size_t
462 0 : _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
463 : {
464 0 : lclContext *ctx = (lclContext *) AH->formatData;
465 0 : size_t avail;
466 0 : size_t used = 0;
467 0 : size_t res = 0;
468 :
469 0 : Assert(th || fh);
470 :
471 0 : avail = AH->lookaheadLen - AH->lookaheadPos;
472 0 : if (avail > 0)
473 : {
474 : /* We have some lookahead bytes to use */
475 0 : if (avail >= len) /* Just use the lookahead buffer */
476 0 : used = len;
477 : else
478 0 : used = avail;
479 :
480 : /* Copy, and adjust buffer pos */
481 0 : memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
482 0 : AH->lookaheadPos += used;
483 :
484 : /* Adjust required length */
485 0 : len -= used;
486 0 : }
487 :
488 : /* Read the file if len > 0 */
489 0 : if (len > 0)
490 : {
491 0 : if (fh)
492 : {
493 0 : res = fread(&((char *) buf)[used], 1, len, fh);
494 0 : if (res != len && !feof(fh))
495 0 : READ_ERROR_EXIT(fh);
496 0 : }
497 0 : else if (th)
498 : {
499 0 : res = fread(&((char *) buf)[used], 1, len, th->nFH);
500 0 : if (res != len && !feof(th->nFH))
501 0 : READ_ERROR_EXIT(th->nFH);
502 0 : }
503 0 : }
504 :
505 0 : ctx->tarFHpos += res + used;
506 :
507 0 : return (res + used);
508 0 : }
509 :
510 : static size_t
511 0 : tarRead(void *buf, size_t len, TAR_MEMBER *th)
512 : {
513 0 : size_t res;
514 :
515 0 : if (th->pos + len > th->fileLen)
516 0 : len = th->fileLen - th->pos;
517 :
518 0 : if (len <= 0)
519 0 : return 0;
520 :
521 0 : res = _tarReadRaw(th->AH, buf, len, th, NULL);
522 :
523 0 : th->pos += res;
524 :
525 0 : return res;
526 0 : }
527 :
528 : static size_t
529 0 : tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
530 : {
531 0 : size_t res;
532 :
533 0 : res = fwrite(buf, 1, len, th->nFH);
534 :
535 0 : th->pos += res;
536 0 : return res;
537 0 : }
538 :
539 : static void
540 0 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
541 : {
542 0 : lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
543 :
544 0 : if (tarWrite(data, dLen, tctx->TH) != dLen)
545 0 : WRITE_ERROR_EXIT;
546 0 : }
547 :
548 : static void
549 0 : _EndData(ArchiveHandle *AH, TocEntry *te)
550 : {
551 0 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
552 :
553 : /* Close the file */
554 0 : tarClose(AH, tctx->TH);
555 0 : tctx->TH = NULL;
556 0 : }
557 :
558 : /*
559 : * Print data for a given file
560 : */
561 : static void
562 0 : _PrintFileData(ArchiveHandle *AH, char *filename)
563 : {
564 0 : lclContext *ctx = (lclContext *) AH->formatData;
565 0 : char buf[4096];
566 0 : size_t cnt;
567 0 : TAR_MEMBER *th;
568 :
569 0 : if (!filename)
570 0 : return;
571 :
572 0 : th = tarOpen(AH, filename, 'r');
573 0 : ctx->FH = th;
574 :
575 0 : while ((cnt = tarRead(buf, 4095, th)) > 0)
576 : {
577 0 : buf[cnt] = '\0';
578 0 : ahwrite(buf, 1, cnt, AH);
579 : }
580 :
581 0 : tarClose(AH, th);
582 0 : }
583 :
584 :
585 : /*
586 : * Print data for a given TOC entry
587 : */
588 : static void
589 0 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
590 : {
591 0 : lclContext *ctx = (lclContext *) AH->formatData;
592 0 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
593 0 : int pos1;
594 :
595 0 : if (!tctx->filename)
596 0 : return;
597 :
598 : /*
599 : * If we're writing the special restore.sql script, emit a suitable
600 : * command to include each table's data from the corresponding file.
601 : *
602 : * In the COPY case this is a bit klugy because the regular COPY command
603 : * was already printed before we get control.
604 : */
605 0 : if (ctx->isSpecialScript)
606 : {
607 0 : if (te->copyStmt)
608 : {
609 : /* Abort the COPY FROM stdin */
610 0 : ahprintf(AH, "\\.\n");
611 :
612 : /*
613 : * The COPY statement should look like "COPY ... FROM stdin;\n",
614 : * see dumpTableData().
615 : */
616 0 : pos1 = (int) strlen(te->copyStmt) - 13;
617 0 : if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
618 0 : strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
619 0 : pg_fatal("unexpected COPY statement syntax: \"%s\"",
620 : te->copyStmt);
621 :
622 : /* Emit all but the FROM part ... */
623 0 : ahwrite(te->copyStmt, 1, pos1, AH);
624 : /* ... and insert modified FROM */
625 0 : ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
626 0 : }
627 : else
628 : {
629 : /* --inserts mode, no worries, just include the data file */
630 0 : ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
631 : }
632 :
633 0 : return;
634 : }
635 :
636 0 : if (strcmp(te->desc, "BLOBS") == 0)
637 0 : _LoadLOs(AH, te);
638 : else
639 0 : _PrintFileData(AH, tctx->filename);
640 0 : }
641 :
642 : static void
643 0 : _LoadLOs(ArchiveHandle *AH, TocEntry *te)
644 : {
645 0 : Oid oid;
646 0 : lclContext *ctx = (lclContext *) AH->formatData;
647 0 : TAR_MEMBER *th;
648 0 : size_t cnt;
649 0 : bool foundLO = false;
650 0 : char buf[4096];
651 :
652 0 : StartRestoreLOs(AH);
653 :
654 : /*
655 : * The blobs_NNN.toc or blobs.toc file is fairly useless to us because it
656 : * will appear only after the associated blob_NNN.dat files. For archive
657 : * versions >= 16 we can look at the BLOBS entry's te->tag to discover the
658 : * OID of the first blob we want to restore, and then search forward to
659 : * find the appropriate blob_<oid>.dat file. For older versions we rely
660 : * on the knowledge that there was only one BLOBS entry and just search
661 : * for the first blob_<oid>.dat file. Once we find the first blob file to
662 : * restore, restore all blobs until we reach the blobs[_NNN].toc file.
663 : */
664 0 : if (AH->version >= K_VERS_1_16)
665 : {
666 : /* We rely on atooid to not complain about nnnn..nnnn tags */
667 0 : oid = atooid(te->tag);
668 0 : snprintf(buf, sizeof(buf), "blob_%u.dat", oid);
669 0 : th = tarOpen(AH, buf, 'r'); /* Advance to first desired file */
670 0 : }
671 : else
672 0 : th = tarOpen(AH, NULL, 'r'); /* Open next file */
673 :
674 0 : while (th != NULL)
675 : {
676 0 : ctx->FH = th;
677 :
678 0 : if (strncmp(th->targetFile, "blob_", 5) == 0)
679 : {
680 0 : oid = atooid(&th->targetFile[5]);
681 0 : if (oid != 0)
682 : {
683 0 : pg_log_info("restoring large object with OID %u", oid);
684 :
685 0 : StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
686 :
687 0 : while ((cnt = tarRead(buf, 4095, th)) > 0)
688 : {
689 0 : buf[cnt] = '\0';
690 0 : ahwrite(buf, 1, cnt, AH);
691 : }
692 0 : EndRestoreLO(AH, oid);
693 0 : foundLO = true;
694 0 : }
695 0 : tarClose(AH, th);
696 0 : }
697 : else
698 : {
699 0 : tarClose(AH, th);
700 :
701 : /*
702 : * Once we have found the first LO, stop at the first non-LO entry
703 : * (which will be 'blobs[_NNN].toc'). This coding would eat all
704 : * the rest of the archive if there are no LOs ... but this
705 : * function shouldn't be called at all in that case.
706 : */
707 0 : if (foundLO)
708 0 : break;
709 : }
710 :
711 0 : th = tarOpen(AH, NULL, 'r');
712 : }
713 0 : EndRestoreLOs(AH);
714 0 : }
715 :
716 :
717 : static int
718 0 : _WriteByte(ArchiveHandle *AH, const int i)
719 : {
720 0 : lclContext *ctx = (lclContext *) AH->formatData;
721 0 : char b = i; /* Avoid endian problems */
722 :
723 0 : if (tarWrite(&b, 1, ctx->FH) != 1)
724 0 : WRITE_ERROR_EXIT;
725 :
726 0 : ctx->filePos += 1;
727 0 : return 1;
728 0 : }
729 :
730 : static int
731 0 : _ReadByte(ArchiveHandle *AH)
732 : {
733 0 : lclContext *ctx = (lclContext *) AH->formatData;
734 0 : size_t res;
735 0 : unsigned char c;
736 :
737 0 : res = tarRead(&c, 1, ctx->FH);
738 0 : if (res != 1)
739 : /* We already would have exited for errors on reads, must be EOF */
740 0 : pg_fatal("could not read from input file: end of file");
741 0 : ctx->filePos += 1;
742 0 : return c;
743 0 : }
744 :
745 : static void
746 0 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
747 : {
748 0 : lclContext *ctx = (lclContext *) AH->formatData;
749 :
750 0 : if (tarWrite(buf, len, ctx->FH) != len)
751 0 : WRITE_ERROR_EXIT;
752 :
753 0 : ctx->filePos += len;
754 0 : }
755 :
756 : static void
757 0 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
758 : {
759 0 : lclContext *ctx = (lclContext *) AH->formatData;
760 :
761 0 : if (tarRead(buf, len, ctx->FH) != len)
762 : /* We already would have exited for errors on reads, must be EOF */
763 0 : pg_fatal("could not read from input file: end of file");
764 :
765 0 : ctx->filePos += len;
766 0 : }
767 :
768 : static void
769 0 : _CloseArchive(ArchiveHandle *AH)
770 : {
771 0 : lclContext *ctx = (lclContext *) AH->formatData;
772 0 : TAR_MEMBER *th;
773 0 : RestoreOptions *ropt;
774 0 : RestoreOptions *savRopt;
775 0 : DumpOptions *savDopt;
776 0 : int savVerbose,
777 : i;
778 :
779 0 : if (AH->mode == archModeWrite)
780 : {
781 : /*
782 : * Write the Header & TOC to the archive FIRST
783 : */
784 0 : th = tarOpen(AH, "toc.dat", 'w');
785 0 : ctx->FH = th;
786 0 : WriteHead(AH);
787 0 : WriteToc(AH);
788 0 : tarClose(AH, th); /* Not needed any more */
789 :
790 : /*
791 : * Now send the data (tables & LOs)
792 : */
793 0 : WriteDataChunks(AH, NULL);
794 :
795 : /*
796 : * Now this format wants to append a script which does a full restore
797 : * if the files have been extracted.
798 : */
799 0 : th = tarOpen(AH, "restore.sql", 'w');
800 :
801 0 : tarPrintf(th, "--\n"
802 : "-- NOTE:\n"
803 : "--\n"
804 : "-- File paths need to be edited. Search for $$PATH$$ and\n"
805 : "-- replace it with the path to the directory containing\n"
806 : "-- the extracted data files.\n"
807 : "--\n");
808 :
809 0 : AH->CustomOutPtr = _scriptOut;
810 :
811 0 : ctx->isSpecialScript = 1;
812 0 : ctx->scriptTH = th;
813 :
814 0 : ropt = NewRestoreOptions();
815 0 : memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
816 0 : ropt->filename = NULL;
817 0 : ropt->dropSchema = 1;
818 0 : ropt->superuser = NULL;
819 0 : ropt->suppressDumpWarnings = true;
820 :
821 0 : savDopt = AH->public.dopt;
822 0 : savRopt = AH->public.ropt;
823 :
824 0 : SetArchiveOptions((Archive *) AH, NULL, ropt);
825 :
826 0 : savVerbose = AH->public.verbose;
827 0 : AH->public.verbose = 0;
828 :
829 0 : RestoreArchive((Archive *) AH);
830 :
831 0 : SetArchiveOptions((Archive *) AH, savDopt, savRopt);
832 :
833 0 : AH->public.verbose = savVerbose;
834 :
835 0 : tarClose(AH, th);
836 :
837 0 : ctx->isSpecialScript = 0;
838 :
839 : /*
840 : * EOF marker for tar files is two blocks of NULLs.
841 : */
842 0 : for (i = 0; i < TAR_BLOCK_SIZE * 2; i++)
843 : {
844 0 : if (fputc(0, ctx->tarFH) == EOF)
845 0 : WRITE_ERROR_EXIT;
846 0 : }
847 :
848 : /* Sync the output file if one is defined */
849 0 : if (AH->dosync && AH->fSpec)
850 0 : (void) fsync_fname(AH->fSpec, false);
851 0 : }
852 :
853 0 : AH->FH = NULL;
854 0 : }
855 :
856 : static size_t
857 0 : _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
858 : {
859 0 : lclContext *ctx = (lclContext *) AH->formatData;
860 :
861 0 : return tarWrite(buf, len, ctx->scriptTH);
862 0 : }
863 :
864 : /*
865 : * Large Object support
866 : */
867 :
868 : /*
869 : * Called by the archiver when starting to save BLOB DATA (not schema).
870 : * This routine should save whatever format-specific information is needed
871 : * to read the LOs back into memory.
872 : *
873 : * It is called just prior to the dumper's DataDumper routine.
874 : *
875 : * Optional, but strongly recommended.
876 : *
877 : */
878 : static void
879 0 : _StartLOs(ArchiveHandle *AH, TocEntry *te)
880 : {
881 0 : lclContext *ctx = (lclContext *) AH->formatData;
882 0 : char fname[K_STD_BUF_SIZE];
883 :
884 0 : sprintf(fname, "blobs_%d.toc", te->dumpId);
885 0 : ctx->loToc = tarOpen(AH, fname, 'w');
886 0 : }
887 :
888 : /*
889 : * Called by the archiver when the dumper calls StartLO.
890 : *
891 : * Mandatory.
892 : *
893 : * Must save the passed OID for retrieval at restore-time.
894 : */
895 : static void
896 0 : _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
897 : {
898 0 : lclContext *ctx = (lclContext *) AH->formatData;
899 0 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
900 0 : char fname[255];
901 :
902 0 : if (oid == 0)
903 0 : pg_fatal("invalid OID for large object (%u)", oid);
904 :
905 0 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
906 0 : pg_fatal("compression is not supported by tar archive format");
907 :
908 0 : sprintf(fname, "blob_%u.dat", oid);
909 :
910 0 : tarPrintf(ctx->loToc, "%u %s\n", oid, fname);
911 :
912 0 : tctx->TH = tarOpen(AH, fname, 'w');
913 0 : }
914 :
915 : /*
916 : * Called by the archiver when the dumper calls EndLO.
917 : *
918 : * Optional.
919 : *
920 : */
921 : static void
922 0 : _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
923 : {
924 0 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
925 :
926 0 : tarClose(AH, tctx->TH);
927 0 : }
928 :
929 : /*
930 : * Called by the archiver when finishing saving BLOB DATA.
931 : *
932 : * Optional.
933 : *
934 : */
935 : static void
936 0 : _EndLOs(ArchiveHandle *AH, TocEntry *te)
937 : {
938 0 : lclContext *ctx = (lclContext *) AH->formatData;
939 :
940 : /* Write out a fake zero OID to mark end-of-LOs. */
941 : /* WriteInt(AH, 0); */
942 :
943 0 : tarClose(AH, ctx->loToc);
944 0 : }
945 :
946 :
947 :
948 : /*------------
949 : * TAR Support
950 : *------------
951 : */
952 :
953 : static int
954 0 : tarPrintf(TAR_MEMBER *th, const char *fmt,...)
955 : {
956 0 : int save_errno = errno;
957 0 : char *p;
958 0 : size_t len = 128; /* initial assumption about buffer size */
959 0 : size_t cnt;
960 :
961 0 : for (;;)
962 : {
963 0 : va_list args;
964 :
965 : /* Allocate work buffer. */
966 0 : p = (char *) pg_malloc(len);
967 :
968 : /* Try to format the data. */
969 0 : errno = save_errno;
970 0 : va_start(args, fmt);
971 0 : cnt = pvsnprintf(p, len, fmt, args);
972 0 : va_end(args);
973 :
974 0 : if (cnt < len)
975 0 : break; /* success */
976 :
977 : /* Release buffer and loop around to try again with larger len. */
978 0 : free(p);
979 0 : len = cnt;
980 0 : }
981 :
982 0 : cnt = tarWrite(p, cnt, th);
983 0 : free(p);
984 0 : return (int) cnt;
985 0 : }
986 :
987 : bool
988 0 : isValidTarHeader(char *header)
989 : {
990 0 : int sum;
991 0 : int chk = tarChecksum(header);
992 :
993 0 : sum = read_tar_number(&header[TAR_OFFSET_CHECKSUM], 8);
994 :
995 0 : if (sum != chk)
996 0 : return false;
997 :
998 : /* POSIX tar format */
999 0 : if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar\0", 6) == 0 &&
1000 0 : memcmp(&header[TAR_OFFSET_VERSION], "00", 2) == 0)
1001 0 : return true;
1002 : /* GNU tar format */
1003 0 : if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar \0", 8) == 0)
1004 0 : return true;
1005 : /* not-quite-POSIX format written by pre-9.3 pg_dump */
1006 0 : if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar00\0", 8) == 0)
1007 0 : return true;
1008 :
1009 0 : return false;
1010 0 : }
1011 :
1012 : /* Given the member, write the TAR header & copy the file */
1013 : static void
1014 0 : _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
1015 : {
1016 0 : lclContext *ctx = (lclContext *) AH->formatData;
1017 0 : FILE *tmp = th->tmpFH; /* Grab it for convenience */
1018 0 : char buf[32768];
1019 0 : size_t cnt;
1020 0 : pgoff_t len = 0;
1021 0 : size_t res;
1022 0 : size_t i,
1023 : pad;
1024 :
1025 : /*
1026 : * Find file len & go back to start.
1027 : */
1028 0 : if (fseeko(tmp, 0, SEEK_END) != 0)
1029 0 : pg_fatal("error during file seek: %m");
1030 0 : th->fileLen = ftello(tmp);
1031 0 : if (th->fileLen < 0)
1032 0 : pg_fatal("could not determine seek position in archive file: %m");
1033 0 : if (fseeko(tmp, 0, SEEK_SET) != 0)
1034 0 : pg_fatal("error during file seek: %m");
1035 :
1036 0 : _tarWriteHeader(th);
1037 :
1038 0 : while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1039 : {
1040 0 : if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1041 0 : WRITE_ERROR_EXIT;
1042 0 : len += res;
1043 : }
1044 0 : if (!feof(tmp))
1045 0 : READ_ERROR_EXIT(tmp);
1046 :
1047 0 : if (fclose(tmp) != 0) /* This *should* delete it... */
1048 0 : pg_fatal("could not close temporary file: %m");
1049 :
1050 0 : if (len != th->fileLen)
1051 0 : pg_fatal("actual file length (%lld) does not match expected (%lld)",
1052 : (long long) len, (long long) th->fileLen);
1053 :
1054 0 : pad = tarPaddingBytesRequired(len);
1055 0 : for (i = 0; i < pad; i++)
1056 : {
1057 0 : if (fputc('\0', th->tarFH) == EOF)
1058 0 : WRITE_ERROR_EXIT;
1059 0 : }
1060 :
1061 0 : ctx->tarFHpos += len + pad;
1062 0 : }
1063 :
1064 : /* Locate the file in the archive, read header and position to data */
1065 : static TAR_MEMBER *
1066 0 : _tarPositionTo(ArchiveHandle *AH, const char *filename)
1067 : {
1068 0 : lclContext *ctx = (lclContext *) AH->formatData;
1069 0 : TAR_MEMBER *th = pg_malloc0_object(TAR_MEMBER);
1070 0 : char c;
1071 0 : char header[TAR_BLOCK_SIZE];
1072 0 : size_t i,
1073 : len,
1074 : blks;
1075 0 : int id;
1076 :
1077 0 : th->AH = AH;
1078 :
1079 : /* Go to end of current file, if any */
1080 0 : if (ctx->tarFHpos != 0)
1081 : {
1082 0 : pg_log_debug("moving from position %lld to next member at file position %lld",
1083 : (long long) ctx->tarFHpos, (long long) ctx->tarNextMember);
1084 :
1085 0 : while (ctx->tarFHpos < ctx->tarNextMember)
1086 0 : _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1087 0 : }
1088 :
1089 0 : pg_log_debug("now at file position %lld", (long long) ctx->tarFHpos);
1090 :
1091 : /* We are at the start of the file, or at the next member */
1092 :
1093 : /* Get the header */
1094 0 : if (!_tarGetHeader(AH, th))
1095 : {
1096 0 : if (filename)
1097 0 : pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1098 : else
1099 : {
1100 : /*
1101 : * We're just scanning the archive for the next file, so return
1102 : * null
1103 : */
1104 0 : free(th);
1105 0 : return NULL;
1106 : }
1107 0 : }
1108 :
1109 0 : while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1110 : {
1111 0 : pg_log_debug("skipping tar member %s", th->targetFile);
1112 :
1113 0 : id = atoi(th->targetFile);
1114 0 : if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1115 0 : pg_fatal("restoring data out of order is not supported in this archive format: "
1116 : "\"%s\" is required, but comes before \"%s\" in the archive file.",
1117 : th->targetFile, filename);
1118 :
1119 : /* Header doesn't match, so read to next header */
1120 0 : len = th->fileLen;
1121 0 : len += tarPaddingBytesRequired(th->fileLen);
1122 0 : blks = len / TAR_BLOCK_SIZE; /* # of tar blocks */
1123 :
1124 0 : for (i = 0; i < blks; i++)
1125 0 : _tarReadRaw(AH, &header[0], TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1126 :
1127 0 : if (!_tarGetHeader(AH, th))
1128 0 : pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1129 : }
1130 :
1131 0 : ctx->tarNextMember = ctx->tarFHpos + th->fileLen
1132 0 : + tarPaddingBytesRequired(th->fileLen);
1133 0 : th->pos = 0;
1134 :
1135 0 : return th;
1136 0 : }
1137 :
1138 : /* Read & verify a header */
1139 : static int
1140 0 : _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
1141 : {
1142 0 : lclContext *ctx = (lclContext *) AH->formatData;
1143 0 : char h[TAR_BLOCK_SIZE];
1144 0 : char tag[100 + 1];
1145 0 : int sum,
1146 : chk;
1147 0 : pgoff_t len;
1148 0 : pgoff_t hPos;
1149 0 : bool gotBlock = false;
1150 :
1151 0 : while (!gotBlock)
1152 : {
1153 : /* Save the pos for reporting purposes */
1154 0 : hPos = ctx->tarFHpos;
1155 :
1156 : /* Read the next tar block, return EOF, exit if short */
1157 0 : len = _tarReadRaw(AH, h, TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1158 0 : if (len == 0) /* EOF */
1159 0 : return 0;
1160 :
1161 0 : if (len != TAR_BLOCK_SIZE)
1162 0 : pg_fatal(ngettext("incomplete tar header found (%lu byte)",
1163 : "incomplete tar header found (%lu bytes)",
1164 : len),
1165 : (unsigned long) len);
1166 :
1167 : /* Calc checksum */
1168 0 : chk = tarChecksum(h);
1169 0 : sum = read_tar_number(&h[TAR_OFFSET_CHECKSUM], 8);
1170 :
1171 : /*
1172 : * If the checksum failed, see if it is a null block. If so, silently
1173 : * continue to the next block.
1174 : */
1175 0 : if (chk == sum)
1176 0 : gotBlock = true;
1177 : else
1178 : {
1179 0 : int i;
1180 :
1181 0 : for (i = 0; i < TAR_BLOCK_SIZE; i++)
1182 : {
1183 0 : if (h[i] != 0)
1184 : {
1185 0 : gotBlock = true;
1186 0 : break;
1187 : }
1188 0 : }
1189 0 : }
1190 : }
1191 :
1192 : /* Name field is 100 bytes, might not be null-terminated */
1193 0 : strlcpy(tag, &h[TAR_OFFSET_NAME], 100 + 1);
1194 :
1195 0 : len = read_tar_number(&h[TAR_OFFSET_SIZE], 12);
1196 :
1197 0 : pg_log_debug("TOC Entry %s at %llu (length %llu, checksum %d)",
1198 : tag, (unsigned long long) hPos, (unsigned long long) len, sum);
1199 :
1200 0 : if (chk != sum)
1201 0 : pg_fatal("corrupt tar header found in %s (expected %d, computed %d) file position %llu",
1202 : tag, sum, chk, (unsigned long long) ftello(ctx->tarFH));
1203 :
1204 0 : th->targetFile = pg_strdup(tag);
1205 0 : th->fileLen = len;
1206 :
1207 0 : return 1;
1208 0 : }
1209 :
1210 :
1211 : static void
1212 0 : _tarWriteHeader(TAR_MEMBER *th)
1213 : {
1214 0 : char h[TAR_BLOCK_SIZE];
1215 :
1216 0 : tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1217 0 : 0600, 04000, 02000, time(NULL));
1218 :
1219 : /* Now write the completed header. */
1220 0 : if (fwrite(h, 1, TAR_BLOCK_SIZE, th->tarFH) != TAR_BLOCK_SIZE)
1221 0 : WRITE_ERROR_EXIT;
1222 0 : }
|
