diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f99d0f27..7393e2650 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,9 +1,9 @@ name: Build Probackup on: - push: - branches: - - "**" + #push: + # branches: + # - "**" # Runs triggered by pull requests are disabled to prevent executing potentially unsafe code from public pull requests # pull_request: # branches: diff --git a/.gitignore b/.gitignore index 97d323ceb..1cdc7809a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Object files *.o +*.bc # Libraries *.lib @@ -36,15 +37,8 @@ /tests/helpers/*pyc # Extra files -/src/pg_crc.c -/src/receivelog.c -/src/receivelog.h -/src/streamutil.c -/src/streamutil.h -/src/xlogreader.c -/src/walmethods.c -/src/walmethods.h -/src/instr_time.h +/src/borrowed/ +/borrowed.mk # Doc files /doc/*html @@ -57,7 +51,7 @@ /backup_restore.sh # Packaging -/build +/pkg-build /packaging/pkg/tarballs/pgpro.tar.bz2 /packaging/repo/pg_probackup /packaging/repo/pg_probackup-forks diff --git a/.travis.yml b/.travis.yml index 074ae3d02..6ebddbfb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,8 +70,6 @@ env: - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup_test.BackupTest.test_full_backup - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup_test.BackupTest.test_full_backup_stream # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup @@ -92,7 +90,6 @@ env: jobs: allow_failures: - if: env(PG_BRANCH) = master - - if: env(PG_BRANCH) = REL9_5_STABLE # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) # Only run CI for master branch commits to limit our travis usage diff --git a/Makefile b/Makefile index f93cc37a4..c588c44c9 100644 --- a/Makefile +++ b/Makefile @@ -1,43 +1,70 @@ -PROGRAM = pg_probackup -WORKDIR ?= $(CURDIR) -BUILDDIR = $(WORKDIR)/build/ -PBK_GIT_REPO = https://github.com/postgrespro/pg_probackup +# pg_probackup build system +# +# You can build pg_probackup in different ways: +# +# 1. in source tree using PGXS (with already installed PG and existing PG sources) +# git clone https://github.com/postgrespro/pg_probackup pg_probackup +# cd pg_probackup +# make USE_PGXS=1 PG_CONFIG= top_srcdir= +# +# 2. out of source using PGXS +# git clone https://github.com/postgrespro/pg_probackup pg_probackup-src +# mkdir pg_probackup-build && cd pg_probackup-build +# make USE_PGXS=1 PG_CONFIG= top_srcdir= -f ../pg_probackup-src/Makefile +# +# 3. in PG source (without PGXS -- using only PG sources) +# git clone https://git.postgresql.org/git/postgresql.git postgresql +# git clone https://github.com/postgrespro/pg_probackup postgresql/contrib/pg_probackup +# cd postgresql +# ./configure ... && make +# make -C contrib/pg_probackup +# +# 4. out of PG source and without PGXS +# git clone https://git.postgresql.org/git/postgresql.git postgresql-src +# git clone https://github.com/postgrespro/pg_probackup postgresql-src/contrib/pg_probackup +# mkdir postgresql-build && cd postgresql-build +# ../postgresql-src/configure ... && make +# make -C contrib/pg_probackup +# +top_pbk_srcdir := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) -# utils -OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ - src/utils/parray.o src/utils/pgut.o src/utils/thread.o src/utils/remote.o src/utils/file.o +PROGRAM := pg_probackup -OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ - src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ +# pg_probackup sources +PGPOBJS := src/utils/configuration.o src/utils/json.o src/utils/logger.o \ + src/utils/parray.o src/utils/pgut.o src/utils/thread.o src/utils/remote.o src/utils/file.o +PGPOBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ + src/delete.o src/dir.o src/help.o src/init.o src/merge.o \ src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/stream.o \ - src/util.o src/validate.o src/datapagemap.o src/catchup.o + src/util.o src/validate.o src/datapagemap.o src/catchup.o \ + src/compatibility/pg-11.o src/utils/simple_prompt.o +PGPOBJS += src/compatibility/file_compat.o src/compatibility/receivelog.o \ + src/compatibility/streamutil.o \ + src/compatibility/walmethods.o src/compatibility/file_compat10.o -# borrowed files -OBJS += src/pg_crc.o src/receivelog.o src/streamutil.o \ - src/xlogreader.o +# sources borrowed from postgresql (paths are relative to pg top dir) +BORROWED_H_SRC := +BORROWED_C_SRC := \ + src/backend/access/transam/xlogreader.c -EXTRA_CLEAN = src/pg_crc.c \ - src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \ - src/xlogreader.c src/instr_time.h +BORROW_DIR := src/borrowed +BORROWED_H := $(addprefix $(BORROW_DIR)/, $(notdir $(BORROWED_H_SRC))) +BORROWED_C := $(addprefix $(BORROW_DIR)/, $(notdir $(BORROWED_C_SRC))) +PGPOBJS += $(patsubst %.c, %.o, $(BORROWED_C)) +EXTRA_CLEAN := $(BORROWED_H) $(BORROWED_C) $(BORROW_DIR) borrowed.mk -ifdef top_srcdir -srchome := $(abspath $(top_srcdir)) -else -top_srcdir=../.. -ifneq (,$(wildcard ../../../contrib/pg_probackup)) -# separate build directory support -srchome := $(abspath $(top_srcdir)/..) -else -srchome := $(abspath $(top_srcdir)) -endif -endif +PGPOBJS += src/fu_util/impl/ft_impl.o src/fu_util/impl/fo_impl.o -# OBJS variable must be finally defined before invoking the include directive -ifneq (,$(wildcard $(srchome)/src/bin/pg_basebackup/walmethods.c)) -OBJS += src/walmethods.o -EXTRA_CLEAN += src/walmethods.c src/walmethods.h +# off-source build support +ifneq ($(abspath $(CURDIR))/, $(top_pbk_srcdir)) +VPATH := $(top_pbk_srcdir) endif +# artificial file for `main` function +OBJS += $(PGPOBJS) src/main.o + +# standard PGXS stuff +# all PGPOBJS must be defined above this ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -49,41 +76,43 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(srchome)/$(subdir)/src +# +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -I$(top_pbk_srcdir)src -I$(BORROW_DIR) -Isrc/compatibility -Isrc/utils +PG_CPPFLAGS += -I$(top_pbk_srcdir)src/fu_util -Wno-declaration-after-statement +ifdef VPATH +PG_CPPFLAGS += -Isrc +endif override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} +CFLAGS += -fexceptions -src/utils/configuration.o: src/datapagemap.h -src/archive.o: src/instr_time.h -src/backup.o: src/receivelog.h src/streamutil.h +# additional dependencies on borrowed files +src/backup.o src/catchup.o src/pg_probackup.o: src/compatibility/streamutil.h +src/stream.o src/compatibility/receivelog.o src/compatibility/streamutil.o src/compatibility/walmethods.o: src/compatibility/receivelog.h +src/compatibility/receivelog.h: src/compatibility/walmethods.h -src/instr_time.h: $(srchome)/src/include/portability/instr_time.h - rm -f $@ && $(LN_S) $(srchome)/src/include/portability/instr_time.h $@ -src/pg_crc.c: $(srchome)/src/backend/utils/hash/pg_crc.c - rm -f $@ && $(LN_S) $(srchome)/src/backend/utils/hash/pg_crc.c $@ -src/receivelog.c: $(srchome)/src/bin/pg_basebackup/receivelog.c - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@ -ifneq (,$(wildcard $(srchome)/src/bin/pg_basebackup/walmethods.c)) -src/receivelog.h: src/walmethods.h $(srchome)/src/bin/pg_basebackup/receivelog.h -else -src/receivelog.h: $(srchome)/src/bin/pg_basebackup/receivelog.h -endif - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.h $@ -src/streamutil.c: $(srchome)/src/bin/pg_basebackup/streamutil.c - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ -src/streamutil.h: $(srchome)/src/bin/pg_basebackup/streamutil.h - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ -src/xlogreader.c: $(srchome)/src/backend/access/transam/xlogreader.c - rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ -src/walmethods.c: $(srchome)/src/bin/pg_basebackup/walmethods.c - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ -src/walmethods.h: $(srchome)/src/bin/pg_basebackup/walmethods.h - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ +# generate separate makefile to handle borrowed files +borrowed.mk: $(firstword $(MAKEFILE_LIST)) + $(file >$@,# This file is autogenerated. Do not edit!) + $(foreach borrowed_file, $(BORROWED_H_SRC) $(BORROWED_C_SRC), \ + $(file >>$@,$(addprefix $(BORROW_DIR)/, $(notdir $(borrowed_file))): | $(CURDIR)/$(BORROW_DIR)/ $(realpath $(top_srcdir)/$(borrowed_file))) \ + $(file >>$@,$(shell echo " "'$$(LN_S) -f $(realpath $(top_srcdir)/$(borrowed_file)) $$@')) \ + ) +include borrowed.mk -ifeq ($(PORTNAME), aix) - CC=xlc_r -endif +# create needed directories for borrowed files and off-source build +OBJDIRS = $(addprefix $(CURDIR)/, $(sort $(dir $(OBJS)))) +$(OBJS): | $(OBJDIRS) +$(OBJDIRS): + mkdir -p $@ + +# packaging infrastructure +WORKDIR ?= $(CURDIR) +PBK_PKG_BUILDDIR = $(WORKDIR)/pkg-build/ +PBK_GIT_REPO = https://github.com/postgrespro/pg_probackup + +include $(top_pbk_srcdir)/packaging/Makefile.pkg +include $(top_pbk_srcdir)/packaging/Makefile.repo +include $(top_pbk_srcdir)/packaging/Makefile.test -include packaging/Makefile.pkg -include packaging/Makefile.repo -include packaging/Makefile.test +include $(top_pbk_srcdir)/unit/Makefile \ No newline at end of file diff --git a/README.md b/README.md index 7486a6ca6..f47e22c3e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.6, 10, 11, 12, 13, 14, 15; +* PostgreSQL 10, 11, 12, 13, 14, 15; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -137,46 +137,46 @@ sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10} +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10}-dbg #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} +sudo apt-get install pg-probackup-{std,ent}-{12,11,10}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10} +yum install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10} +yum install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10} +yum install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 2cb10e379..45634bf42 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -164,7 +164,7 @@ doc/src/sgml/pgprobackup.sgml recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. - pg_probackup supports PostgreSQL 9.5 or higher. + pg_probackup supports PostgreSQL 10 or higher. @@ -401,11 +401,6 @@ doc/src/sgml/pgprobackup.sgml pg_probackup currently has the following limitations: - - - pg_probackup only supports PostgreSQL 9.5 and higher. - - The remote mode is not supported on Windows systems. @@ -413,7 +408,7 @@ doc/src/sgml/pgprobackup.sgml - On Unix systems, for PostgreSQL 10 or lower, + On Unix systems, for PostgreSQL 10, a backup can be made only by the same OS user that has started the PostgreSQL server. For example, if PostgreSQL server is started by user postgres, the backup command must also be run @@ -422,17 +417,6 @@ doc/src/sgml/pgprobackup.sgml option to postgres. - - - For PostgreSQL 9.5, functions - pg_create_restore_point(text) and - pg_switch_xlog() can be executed only if - the backup role is a superuser, so backup of a - cluster with low amount of WAL traffic by a non-superuser - role can take longer than the backup of the same cluster by - a superuser role. - - The PostgreSQL server from which the backup was taken and @@ -612,46 +596,6 @@ pg_probackup add-instance -B backup_dir -D used for connection to the PostgreSQL server: - - For PostgreSQL 9.5: - - -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -COMMIT; - - - For PostgreSQL 9.6: - - -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; -COMMIT; - For PostgreSQL versions 10 — 14: @@ -901,7 +845,7 @@ archive_command = '"install_dir/pg_probackup" archive Setting up Backup from Standby - For PostgreSQL 9.6 or higher, pg_probackup can take backups from + pg_probackup can take backups from a standby server. This requires the following additional setup: @@ -4797,8 +4741,7 @@ pg_probackup catchup -b catchup_mode Specifies the LSN of the write-ahead log location up to which - recovery will proceed. Can be used only when restoring - a database cluster of major version 10 or higher. + recovery will proceed. @@ -5760,96 +5703,6 @@ pg_probackup catchup -b catchup_mode - - Replica Options - - This section describes the options related to taking a backup - from standby. - - - - Starting from pg_probackup 2.0.24, backups can be - taken from standby without connecting to the master server, - so these options are no longer required. In lower versions, - pg_probackup had to connect to the master to determine - recovery time — the earliest moment for which you can - restore a consistent state of the database cluster. - - - - - - - - - Deprecated. Specifies the name of the database on the master - server to connect to. The connection is used only for managing - the backup process, so you can connect to any existing - database. Can be set in the pg_probackup.conf using the - command. - - - Default: postgres, the default PostgreSQL database - - - - - - - - - Deprecated. Specifies the host name of the system on which the - master server is running. - - - - - - - - - Deprecated. Specifies the TCP port or the local Unix domain - socket file extension on which the master server is listening - for connections. - - - Default: 5432, the PostgreSQL default port - - - - - - - - - Deprecated. User name to connect as. - - - Default: postgres, - the PostgreSQL default user name - - - - - - - - - - Deprecated. Wait time for WAL segment streaming via - replication, in seconds. By default, pg_probackup waits 300 - seconds. You can also define this parameter in the - pg_probackup.conf configuration file using the - command. - - - Default: 300 sec - - - - - - @@ -6061,8 +5914,6 @@ xlog-seg-size = 16777216 pgdatabase = backupdb pghost = postgres_host pguser = backup -# Replica parameters -replica-timeout = 5min # Archive parameters archive-timeout = 5min # Logging parameters diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 8143b7d0d..c912329fa 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -214,6 +214,7 @@ sub build_pgprobackup $probackup->AddIncludeDir("$currpath"); $probackup->AddIncludeDir("$currpath/src"); $probackup->AddIncludeDir("$currpath/src/utils"); + $probackup->AddIncludeDir("$currpath/src/fu_util"); if ($libpgfeutils) { diff --git a/packaging/Makefile.pkg b/packaging/Makefile.pkg index e17243614..7ebfe315c 100644 --- a/packaging/Makefile.pkg +++ b/packaging/Makefile.pkg @@ -36,7 +36,7 @@ build/prepare: mkdir -p build build/clean: build/prepare - find $(BUILDDIR) -maxdepth 1 -type f -exec rm -f {} \; + find $(PBK_PKG_BUILDDIR) -maxdepth 1 -type f -exec rm -f {} \; build/all: build/debian build/ubuntu build/centos build/oraclelinux build/alt build/suse build/rhel @echo Packaging is done @@ -76,8 +76,8 @@ define build_deb --rm pgpro/$1:$2 /app/in/scripts/deb.sh endef -include packaging/pkg/Makefile.debian -include packaging/pkg/Makefile.ubuntu +include $(top_pbk_srcdir)/packaging/pkg/Makefile.debian +include $(top_pbk_srcdir)/packaging/pkg/Makefile.ubuntu # CENTOS build/centos: build/centos_7 build/centos_8 #build/rpm_repo_package_centos @@ -127,9 +127,9 @@ define build_rpm --rm pgpro/$1:$2 /app/in/scripts/rpm.sh endef -include packaging/pkg/Makefile.centos -include packaging/pkg/Makefile.rhel -include packaging/pkg/Makefile.oraclelinux +include $(top_pbk_srcdir)/packaging/pkg/Makefile.centos +include $(top_pbk_srcdir)/packaging/pkg/Makefile.rhel +include $(top_pbk_srcdir)/packaging/pkg/Makefile.oraclelinux # Alt Linux @@ -157,7 +157,7 @@ define build_alt --rm pgpro/$1:$2 /app/in/scripts/alt.sh endef -include packaging/pkg/Makefile.alt +include $(top_pbk_srcdir)/packaging/pkg/Makefile.alt # SUSE Linux build/suse: build/suse_15.1 build/suse_15.2 @@ -182,4 +182,4 @@ define build_suse --rm pgpro/$1:$2 /app/in/scripts/suse.sh endef -include packaging/pkg/Makefile.suse +include $(top_pbk_srcdir)/packaging/pkg/Makefile.suse diff --git a/packaging/Makefile.repo b/packaging/Makefile.repo index 10fb27137..fdfb1d427 100644 --- a/packaging/Makefile.repo +++ b/packaging/Makefile.repo @@ -100,9 +100,9 @@ build/repo_suse_15.2: repo_finish: # cd build/data/www/$(PBK_PKG_REPO)/ - cd $(BUILDDIR)/data/www/$(PBK_PKG_REPO)/rpm && sudo ln -nsf $(PBK_VERSION) latest + cd $(PBK_PKG_BUILDDIR)/data/www/$(PBK_PKG_REPO)/rpm && sudo ln -nsf $(PBK_VERSION) latest # following line only for vanilla - cd $(BUILDDIR)/data/www/$(PBK_PKG_REPO)/srpm && sudo ln -nsf $(PBK_VERSION) latest + cd $(PBK_PKG_BUILDDIR)/data/www/$(PBK_PKG_REPO)/srpm && sudo ln -nsf $(PBK_VERSION) latest # sudo ln -rfs build/data/www/$(PBK_PKG_REPO)/rpm/${PBK_VERSION} build/data/www/$(PBK_PKG_REPO)/rpm/latest # sudo ln -rfs build/data/www/$(PBK_PKG_REPO)/srpm/${PBK_VERSION} build/data/www/$(PBK_PKG_REPO)/srpm/latest diff --git a/packaging/Makefile.test b/packaging/Makefile.test index 11c63619a..525af8583 100644 --- a/packaging/Makefile.test +++ b/packaging/Makefile.test @@ -39,7 +39,7 @@ define test_deb docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ docker run \ -v $(WORKDIR)/packaging/test:/app/in \ - -v $(BUILDDIR)/data/www:/app/www \ + -v $(PBK_PKG_BUILDDIR)/data/www:/app/www \ -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg-probackup-$(PKG_NAME_SUFFIX)$4" \ -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ @@ -47,8 +47,8 @@ define test_deb --rm pgpro/$1:$2 /app/in/scripts/deb$(SCRIPT_SUFFIX).sh endef -include packaging/test/Makefile.debian -include packaging/test/Makefile.ubuntu +include $(top_pbk_srcdir)/packaging/test/Makefile.debian +include $(top_pbk_srcdir)/packaging/test/Makefile.ubuntu # CENTOS build/test_centos: build/test_centos_7 build/test_centos_8 @@ -86,7 +86,7 @@ define test_rpm docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ docker run \ -v $(WORKDIR)/packaging/test:/app/in \ - -v $(BUILDDIR)/data/www:/app/www \ + -v $(PBK_PKG_BUILDDIR)/data/www:/app/www \ -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ @@ -94,9 +94,9 @@ define test_rpm --rm pgpro/$1:$2 /app/in/scripts/rpm$(SCRIPT_SUFFIX).sh endef -include packaging/test/Makefile.centos -include packaging/test/Makefile.rhel -include packaging/test/Makefile.oraclelinux +include $(top_pbk_srcdir)/packaging/test/Makefile.centos +include $(top_pbk_srcdir)/packaging/test/Makefile.rhel +include $(top_pbk_srcdir)/packaging/test/Makefile.oraclelinux # Alt Linux build/test_alt: build/test_alt_8 build/test_alt_9 @@ -115,7 +115,7 @@ define test_alt docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ docker run \ -v $(WORKDIR)/packaging/test:/app/in \ - -v $(BUILDDIR)/data/www:/app/www \ + -v $(PBK_PKG_BUILDDIR)/data/www:/app/www \ -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ @@ -123,7 +123,7 @@ define test_alt --rm pgpro/$1:$2 /app/in/scripts/alt$(SCRIPT_SUFFIX).sh endef -include packaging/test/Makefile.alt +include $(top_pbk_srcdir)/packaging/test/Makefile.alt # SUSE Linux build/test_suse: build/test_suse_15.1 build/test_suse_15.2 @@ -139,7 +139,7 @@ define test_suse docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ docker run \ -v $(WORKDIR)/packaging/test:/app/in \ - -v $(BUILDDIR)/data/www:/app/www \ + -v $(PBK_PKG_BUILDDIR)/data/www:/app/www \ -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ @@ -147,4 +147,4 @@ define test_suse --rm pgpro/$1:$2 /app/in/scripts/suse$(SCRIPT_SUFFIX).sh endef -include packaging/test/Makefile.suse +include $(top_pbk_srcdir)/packaging/test/Makefile.suse diff --git a/src/archive.c b/src/archive.c index 734602cac..560308a8f 100644 --- a/src/archive.c +++ b/src/archive.c @@ -11,25 +11,18 @@ #include #include "pg_probackup.h" #include "utils/thread.h" -#include "instr_time.h" - -static int push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - uint32 archive_timeout); -#ifdef HAVE_LIBZ -static int push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - int compress_level, uint32 archive_timeout); -#endif +#include "portability/instr_time.h" + +static err_i push_file_internal(const char *wal_file_name, + const char *pg_xlog_dir, + const char *archive_dir, + bool overwrite, bool no_sync, + bool is_compress, int compress_level, + uint32 archive_timeout, bool *skipped); static void *push_files(void *arg); static void *get_files(void *arg); static bool get_wal_file(const char *filename, const char *from_path, const char *to_path, bool prefetch_mode); -static int get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, - bool is_decompress); -#ifdef HAVE_LIBZ -static const char *get_gz_error(gzFile gzf, int errnum); -#endif //static void copy_file_attributes(const char *from_path, // fio_location from_location, // const char *to_path, fio_location to_location, @@ -99,7 +92,7 @@ static int push_file(WALSegno *xlogfile, const char *archive_status_dir, bool no_ready_rename, bool is_compress, int compress_level); -static parray *setup_push_filelist(const char *archive_status_dir, +static parray *setup_push_filelist(pioDrive_i drive, const char *archive_status_dir, const char *first_file, int batch_size); /* @@ -148,14 +141,15 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg #endif /* Setup filelist and locks */ - batch_files = setup_push_filelist(archive_status_dir, wal_file_name, batch_size); + batch_files = setup_push_filelist(instanceState->database_location, + archive_status_dir, wal_file_name, batch_size); n_threads = num_threads; if (num_threads > parray_num(batch_files)) n_threads = parray_num(batch_files); elog(INFO, "pg_probackup archive-push WAL file: %s, " - "threads: %i/%i, batch: %lu/%i, compression: %s", + "threads: %i/%i, batch: %zu/%i, compression: %s", wal_file_name, n_threads, num_threads, parray_num(batch_files), batch_size, is_compress ? "zlib" : "none"); @@ -286,7 +280,7 @@ push_files(void *arg) int rc; archive_push_arg *args = (archive_push_arg *) arg; - my_thread_num = args->thread_num; + set_my_thread_num(args->thread_num); for (i = 0; i < parray_num(args->files); i++) { @@ -331,21 +325,21 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, bool no_ready_rename, bool is_compress, int compress_level) { - int rc; + FOBJ_FUNC_ARP(); + bool skipped = false; + err_i err; + pioDBDrive_i drive = pioDBDriveForLocation(FIO_DB_HOST); elog(LOG, "pushing file \"%s\"", xlogfile->name); - /* If compression is not required, then just copy it as is */ - if (!is_compress) - rc = push_file_internal_uncompressed(xlogfile->name, pg_xlog_dir, - archive_dir, overwrite, no_sync, - archive_timeout); -#ifdef HAVE_LIBZ - else - rc = push_file_internal_gz(xlogfile->name, pg_xlog_dir, archive_dir, - overwrite, no_sync, compress_level, - archive_timeout); -#endif + err = push_file_internal(xlogfile->name, pg_xlog_dir, + archive_dir, overwrite, no_sync, + is_compress, compress_level, + archive_timeout, &skipped); + if ($haserr(err)) + { + ft_logerr(FT_ERROR, $errmsg(err), "Archiving %s", xlogfile->name); + } /* take '--no-ready-rename' flag into account */ if (!no_ready_rename && archive_status_dir != NULL) @@ -364,521 +358,147 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, elog(LOG, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done); /* do not error out, if rename failed */ - if (fio_rename(wal_file_ready, wal_file_done, FIO_DB_HOST) < 0) - elog(WARNING, "Cannot rename ready file \"%s\" to \"%s\": %s", - wal_file_ready, wal_file_done, strerror(errno)); + err = $i(pioRename, drive, wal_file_ready, wal_file_done); + if ($haserr(err)) + ft_logerr(FT_WARNING, $errmsg(err), "Renaming ready file"); } - return rc; + return skipped; } /* * Copy non WAL file, such as .backup or .history file, into WAL archive. - * Such files are not compressed. + * Optionally apply streaming compression to it. * Returns: * 0 - file was successfully pushed * 1 - push was skipped because file already exists in the archive and * has the same checksum */ -int -push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - uint32 archive_timeout) +err_i +push_file_internal(const char *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + bool is_compress, int compress_level, + uint32 archive_timeout, bool *skipped) { - FILE *in = NULL; - int out = -1; - char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; - /* partial handling */ - struct stat st; - char to_fullpath_part[MAXPGPATH]; - int partial_try_count = 0; - int partial_file_size = 0; - bool partial_is_stale = true; - /* remote agent error message */ - char *errmsg = NULL; - - /* from path */ - join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); - canonicalize_path(from_fullpath); - /* to path */ - join_path_components(to_fullpath, archive_dir, wal_file_name); - canonicalize_path(to_fullpath); + FOBJ_FUNC_ARP(); + pioReadStream_i in; + pioWriteCloser_i out; + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; +/* partial handling */ + size_t len; + err_i err = $noerr(); + + pioDrive_i db_drive = pioDriveForLocation(FIO_DB_HOST); + pioDrive_i backup_drive = pioDriveForLocation(FIO_BACKUP_HOST); + + /* from path */ + join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); + canonicalize_path(from_fullpath); + /* to path */ + join_path_components(to_fullpath, archive_dir, wal_file_name); + canonicalize_path(to_fullpath); + if (is_compress) + { + /* destination file with .gz suffix */ + len = ft_strlcat(to_fullpath, ".gz", sizeof(to_fullpath)); + if (len >= sizeof(to_fullpath)) + return $iresult($err(RT, "File path too long: {path:q}", + path(to_fullpath))); + } + /* open destination partial file for write */ + + if ($i(pioExists, backup_drive, .path = to_fullpath, .err = &err)) + { + pg_crc32 crc32_src; + pg_crc32 crc32_dst; + + crc32_src = $i(pioGetCRC32, db_drive, from_fullpath, + .compressed = false, .err = &err); + if ($haserr(err)) + return $iresult(err); + + crc32_dst = $i(pioGetCRC32, backup_drive, to_fullpath, + .compressed = is_compress, .err = &err); + if ($err_has_kind(GZ, err) && overwrite) + { + elog(LOG, "WAL file already exists and looks like it is damaged, overwriting: %s", + $errmsg(err)); + } + else if ($err_has_kind(GZ, err)) + { + return $iresult($err(RT, "WAL file already exists and looks like it is damaged: {cause}", + cause(err.self))); + } + else if ($haserr(err)) + { + return $iresult(err); + } + else if (crc32_src == crc32_dst) + { + elog(LOG, "WAL file already exists in archive with the same " + "checksum, skip pushing: \"%s\"", from_fullpath); + *skipped = true; + return $noerr(); + } + else if (overwrite) + { + elog(LOG, "WAL file already exists in archive with " + "different checksum, overwriting: \"%s\"", + to_fullpath); + } + else + { + return $iresult($err(RT, "WAL file already exists in archive with " + "different checksum: {path:q}", + path(to_fullpath))); + } + } + else if ($haserr(err)) + { + return $iresult(err); + } /* Open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - elog(ERROR, "Cannot open source file \"%s\": %s", from_fullpath, strerror(errno)); - - /* disable stdio buffering for input file */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - - /* open destination partial file for write */ - snprintf(to_fullpath_part, sizeof(to_fullpath_part), "%s.part", to_fullpath); - - /* Grab lock by creating temp file in exclusive mode */ - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - { - if (errno != EEXIST) - elog(ERROR, "Failed to open temp WAL file \"%s\": %s", - to_fullpath_part, strerror(errno)); - /* Already existing destination temp file is not an error condition */ - } - else - goto part_opened; - - /* - * Partial file already exists, it could have happened due to: - * 1. failed archive-push - * 2. concurrent archiving - * - * For ARCHIVE_TIMEOUT period we will try to create partial file - * and look for the size of already existing partial file, to - * determine if it is changing or not. - * If after ARCHIVE_TIMEOUT we still failed to create partial - * file, we will make a decision about discarding - * already existing partial file. - */ - - while (partial_try_count < archive_timeout) - { - if (fio_stat(to_fullpath_part, &st, false, FIO_BACKUP_HOST) < 0) - { - if (errno == ENOENT) - { - //part file is gone, lets try to grab it - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - { - if (errno != EEXIST) - elog(ERROR, "Failed to open temp WAL file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - else - /* Successfully created partial file */ - break; - } - else - elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); - } - - /* first round */ - if (!partial_try_count) - { - elog(LOG, "Temp WAL file already exists, waiting on it %u seconds: \"%s\"", - archive_timeout, to_fullpath_part); - partial_file_size = st.st_size; - } - - /* file size is changing */ - if (st.st_size > partial_file_size) - partial_is_stale = false; - - sleep(1); - partial_try_count++; - } - /* The possible exit conditions: - * 1. File is grabbed - * 2. File is not grabbed, and it is not stale - * 2. File is not grabbed, and it is stale. - */ - - /* - * If temp file was not grabbed for ARCHIVE_TIMEOUT and temp file is not stale, - * then exit with error. - */ - if (out < 0) - { - if (!partial_is_stale) - elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", - to_fullpath_part, archive_timeout); - - /* Partial segment is considered stale, so reuse it */ - elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_part); - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); - } - -part_opened: - elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_part); - /* Check if possible to skip copying */ - if (fileExists(to_fullpath, FIO_BACKUP_HOST)) - { - pg_crc32 crc32_src; - pg_crc32 crc32_dst; - - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); - crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false, false); - - if (crc32_src == crc32_dst) - { - elog(LOG, "WAL file already exists in archive with the same " - "checksum, skip pushing: \"%s\"", from_fullpath); - /* cleanup */ - fclose(in); - fio_close(out); - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - return 1; - } - else - { - if (overwrite) - elog(LOG, "WAL file already exists in archive with " - "different checksum, overwriting: \"%s\"", to_fullpath); - else - { - /* Overwriting is forbidden, - * so we must unlink partial file and exit with error. - */ - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "WAL file already exists in archive with " - "different checksum: \"%s\"", to_fullpath); - } - } - } - - /* copy content */ - errno = 0; - for (;;) - { - size_t read_len = 0; - - read_len = fread(buf, 1, OUT_BUF_SIZE, in); - - if (ferror(in)) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot read source file \"%s\": %s", - from_fullpath, strerror(errno)); - } - - if (read_len > 0 && fio_write_async(out, buf, read_len) != read_len) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write to destination temp file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - - if (feof(in)) - break; - } - - /* close source file */ - fclose(in); - - /* Writing is asynchronous in case of push in remote mode, so check agent status */ - if (fio_check_error_fd(out, &errmsg)) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write to the remote file \"%s\": %s", - to_fullpath_part, errmsg); - } - - /* close temp file */ - if (fio_close(out) != 0) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot close temp WAL file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - - /* sync temp file to disk */ - if (!no_sync) - { - if (fio_sync(to_fullpath_part, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Failed to sync file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - - elog(LOG, "Rename \"%s\" to \"%s\"", to_fullpath_part, to_fullpath); - - //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); - - /* Rename temp file to destination file */ - if (fio_rename(to_fullpath_part, to_fullpath, FIO_BACKUP_HOST) < 0) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - to_fullpath_part, to_fullpath, strerror(errno)); - } - - pg_free(buf); - return 0; -} - + in = $i(pioOpenReadStream, db_drive, from_fullpath, .err = &err); + if ($haserr(err)) + return $iresult(err); + + out = $i(pioOpenRewrite, backup_drive, .path = to_fullpath, + .sync = !no_sync, .err = &err); + if ($haserr(err)) + return $iresult(err); + + /* enable streaming compression */ + if (is_compress) + { #ifdef HAVE_LIBZ -/* - * Push WAL segment into archive and apply streaming compression to it. - * Returns: - * 0 - file was successfully pushed - * 1 - push was skipped because file already exists in the archive and - * has the same checksum - */ -int -push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - int compress_level, uint32 archive_timeout) -{ - FILE *in = NULL; - gzFile out = NULL; - char *buf = pgut_malloc(OUT_BUF_SIZE); - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; - char to_fullpath_gz[MAXPGPATH]; - - /* partial handling */ - struct stat st; - - char to_fullpath_gz_part[MAXPGPATH]; - int partial_try_count = 0; - int partial_file_size = 0; - bool partial_is_stale = true; - /* remote agent errormsg */ - char *errmsg = NULL; - - /* from path */ - join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); - canonicalize_path(from_fullpath); - /* to path */ - join_path_components(to_fullpath, archive_dir, wal_file_name); - canonicalize_path(to_fullpath); - - /* destination file with .gz suffix */ - snprintf(to_fullpath_gz, sizeof(to_fullpath_gz), "%s.gz", to_fullpath); - /* destination temp file */ - snprintf(to_fullpath_gz_part, sizeof(to_fullpath_gz_part), "%s.part", to_fullpath_gz); - - /* Open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - elog(ERROR, "Cannot open source WAL file \"%s\": %s", - from_fullpath, strerror(errno)); - - /* disable stdio buffering for input file */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - - /* Grab lock by creating temp file in exclusive mode */ - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); - if (out == NULL) - { - if (errno != EEXIST) - elog(ERROR, "Cannot open temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - /* Already existing destination temp file is not an error condition */ - } - else - goto part_opened; - - /* - * Partial file already exists, it could have happened due to: - * 1. failed archive-push - * 2. concurrent archiving - * - * For ARCHIVE_TIMEOUT period we will try to create partial file - * and look for the size of already existing partial file, to - * determine if it is changing or not. - * If after ARCHIVE_TIMEOUT we still failed to create partial - * file, we will make a decision about discarding - * already existing partial file. - */ - - while (partial_try_count < archive_timeout) - { - if (fio_stat(to_fullpath_gz_part, &st, false, FIO_BACKUP_HOST) < 0) - { - if (errno == ENOENT) - { - //part file is gone, lets try to grab it - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); - if (out == NULL) - { - if (errno != EEXIST) - elog(ERROR, "Failed to open temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - else - /* Successfully created partial file */ - break; - } - else - elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - - /* first round */ - if (!partial_try_count) - { - elog(LOG, "Temp WAL file already exists, waiting on it %u seconds: \"%s\"", - archive_timeout, to_fullpath_gz_part); - partial_file_size = st.st_size; - } - - /* file size is changing */ - if (st.st_size > partial_file_size) - partial_is_stale = false; - - sleep(1); - partial_try_count++; - } - /* The possible exit conditions: - * 1. File is grabbed - * 2. File is not grabbed, and it is not stale - * 2. File is not grabbed, and it is stale. - */ - - /* - * If temp file was not grabbed for ARCHIVE_TIMEOUT and temp file is not stale, - * then exit with error. - */ - if (out == NULL) - { - if (!partial_is_stale) - elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", - to_fullpath_gz_part, archive_timeout); - - /* Partial segment is considered stale, so reuse it */ - elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_gz_part); - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); - if (out == NULL) - elog(ERROR, "Cannot open temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - -part_opened: - elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_gz_part); - /* Check if possible to skip copying, - */ - if (fileExists(to_fullpath_gz, FIO_BACKUP_HOST)) - { - pg_crc32 crc32_src; - pg_crc32 crc32_dst; - - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); - crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true, false); - - if (crc32_src == crc32_dst) - { - elog(LOG, "WAL file already exists in archive with the same " - "checksum, skip pushing: \"%s\"", from_fullpath); - /* cleanup */ - fclose(in); - fio_gzclose(out); - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - return 1; - } - else - { - if (overwrite) - elog(LOG, "WAL file already exists in archive with " - "different checksum, overwriting: \"%s\"", to_fullpath_gz); - else - { - /* Overwriting is forbidden, - * so we must unlink partial file and exit with error. - */ - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "WAL file already exists in archive with " - "different checksum: \"%s\"", to_fullpath_gz); - } - } - } - - /* copy content */ - /* TODO: move to separate function */ - for (;;) - { - size_t read_len = 0; - - read_len = fread(buf, 1, OUT_BUF_SIZE, in); - - if (ferror(in)) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot read from source file \"%s\": %s", - from_fullpath, strerror(errno)); - } - - if (read_len > 0 && fio_gzwrite(out, buf, read_len) != read_len) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write to compressed temp WAL file \"%s\": %s", - to_fullpath_gz_part, get_gz_error(out, errno)); - } - - if (feof(in)) - break; - } - - /* close source file */ - fclose(in); - - /* Writing is asynchronous in case of push in remote mode, so check agent status */ - if (fio_check_error_fd_gz(out, &errmsg)) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write to the remote compressed file \"%s\": %s", - to_fullpath_gz_part, errmsg); - } - - /* close temp file, TODO: make it synchronous */ - if (fio_gzclose(out) != 0) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot close compressed temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - - /* sync temp file to disk */ - if (!no_sync) - { - if (fio_sync(to_fullpath_gz_part, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Failed to sync file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - - elog(LOG, "Rename \"%s\" to \"%s\"", - to_fullpath_gz_part, to_fullpath_gz); + pioFilter_i flt = pioGZCompressFilter(compress_level); + err = pioCopy($reduce(pioWriteFlush, out), + $reduce(pioRead, in), + flt); +#else + elog(ERROR, "Compression is requested, but not compiled it"); +#endif + } + else + { + err = pioCopy($reduce(pioWriteFlush, out), + $reduce(pioRead, in)); + } - //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); + /* close source file */ + $i(pioClose, in); /* ignore error */ - /* Rename temp file to destination file */ - if (fio_rename(to_fullpath_gz_part, to_fullpath_gz, FIO_BACKUP_HOST) < 0) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); - } + if ($haserr(err)) + return $iresult(err); - pg_free(buf); - - return 0; -} -#endif - -#ifdef HAVE_LIBZ -/* - * Show error during work with compressed file - */ -static const char * -get_gz_error(gzFile gzf, int errnum) -{ - int gz_errnum; - const char *errmsg; + err = $i(pioClose, out); + if ($haserr(err)) + return $iresult(err); - errmsg = fio_gzerror(gzf, &gz_errnum); - if (gz_errnum == Z_ERRNO) - return strerror(errnum); - else - return errmsg; + return $noerr(); } -#endif /* Copy file attributes */ //static void @@ -888,7 +508,7 @@ get_gz_error(gzFile gzf, int errnum) //{ // struct stat st; // -// if (fio_stat(from_path, &st, true, from_location) == -1) +// if (fio_stat(from_location, from_path, &st, true) == -1) // { // if (unlink_on_error) // fio_unlink(to_path, to_location); @@ -896,7 +516,7 @@ get_gz_error(gzFile gzf, int errnum) // from_path, strerror(errno)); // } // -// if (fio_chmod(to_path, st.st_mode, to_location) == -1) +// if (fio_chmod(to_location, to_path, st.st_mode) == -1) // { // if (unlink_on_error) // fio_unlink(to_path, to_location); @@ -909,13 +529,15 @@ get_gz_error(gzFile gzf, int errnum) * and pack such files into batch sized array. */ parray * -setup_push_filelist(const char *archive_status_dir, const char *first_file, - int batch_size) +setup_push_filelist(pioDrive_i drive, const char *archive_status_dir, + const char *first_file, int batch_size) { - int i; + FOBJ_FUNC_ARP(); WALSegno *xlogfile = NULL; - parray *status_files = NULL; parray *batch_files = parray_new(); + pioDirIter_i iter; + pio_dirent_t entry; + err_i err; /* guarantee that first filename is in batch list */ xlogfile = palloc(sizeof(WALSegno)); @@ -927,18 +549,17 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, return batch_files; /* get list of files from archive_status */ - status_files = parray_new(); - dir_list_file(status_files, archive_status_dir, false, false, false, false, true, 0, FIO_DB_HOST); - parray_qsort(status_files, pgFileCompareName); + iter = $i(pioOpenDir, drive, archive_status_dir, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Reading push filelist"); - for (i = 0; i < parray_num(status_files); i++) + while ((entry=$i(pioDirNext, iter, .err=&err)).stat.pst_kind) { int result = 0; char filename[MAXFNAMELEN]; char suffix[MAXFNAMELEN]; - pgFile *file = (pgFile *) parray_get(status_files, i); - result = sscanf(file->name, "%[^.]%s", (char *) &filename, (char *) &suffix); + result = sscanf(entry.name.ptr, "%[^.]%s", (char *) &filename, (char *) &suffix); if (result != 2) continue; @@ -960,9 +581,9 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, break; } - /* cleanup */ - parray_walk(status_files, pgFileFree); - parray_free(status_files); + $i(pioClose, iter); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Reading push filelist"); return batch_files; } @@ -1033,7 +654,7 @@ do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const cha num_threads = n_actual_threads; elog(VERBOSE, "Obtaining XLOG_SEG_SIZE from pg_control file"); - instance->xlog_seg_size = get_xlog_seg_size(current_dir); + instance->xlog_seg_size = get_xlog_seg_size(instanceState->database_location, current_dir); /* Prefetch optimization kicks in only if simple XLOG segments is requested * and batching is enabled. @@ -1251,7 +872,7 @@ uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, /* init thread args */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (archive_get_arg *) palloc(sizeof(archive_get_arg) * num_threads); + threads_args = (archive_get_arg *) palloc0(sizeof(archive_get_arg) * num_threads); for (i = 0; i < num_threads; i++) { @@ -1294,7 +915,7 @@ get_files(void *arg) char from_fullpath[MAXPGPATH]; archive_get_arg *args = (archive_get_arg *) arg; - my_thread_num = args->thread_num; + set_my_thread_num(args->thread_num); for (i = 0; i < parray_num(args->files); i++) { @@ -1336,286 +957,139 @@ bool get_wal_file(const char *filename, const char *from_fullpath, const char *to_fullpath, bool prefetch_mode) { - int rc = FILE_MISSING; - FILE *out; - char from_fullpath_gz[MAXPGPATH]; - bool src_partial = false; + FOBJ_FUNC_ARP(); + pioDBWriter_i out = {NULL}; + pioReadStream_i in = {NULL}; + char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ + err_i err = $noerr(); + char from_fullpath_gz[MAXPGPATH]; + bool compressed = false; + bool src_partial = false; + + pioDBDrive_i db_drive = pioDBDriveForLocation(FIO_DB_HOST); + pioDrive_i backup_drive = pioDriveForLocation(FIO_BACKUP_HOST); + + snprintf(from_fullpath_gz, sizeof(from_fullpath_gz), "%s.gz", + from_fullpath); + + /* open destination file */ + out = $i(pioOpenWrite, db_drive, .path = to_fullpath, .err = &err, + .exclusive = true); + if ($haserr(err)) + { + elog(WARNING, "%s", $errmsg(err)); + return false; + } - snprintf(from_fullpath_gz, sizeof(from_fullpath_gz), "%s.gz", from_fullpath); - /* open destination file */ - out = fopen(to_fullpath, PG_BINARY_W); - if (!out) - { - elog(WARNING, "Failed to open file '%s': %s", - to_fullpath, strerror(errno)); - return false; - } - - if (chmod(to_fullpath, FILE_PERMISSION) == -1) - { - elog(WARNING, "Cannot change mode of file '%s': %s", - to_fullpath, strerror(errno)); - fclose(out); - unlink(to_fullpath); - return false; - } - - /* disable buffering for output file */ - setvbuf(out, NULL, _IONBF, BUFSIZ); - - /* In prefetch mode, we do look only for full WAL segments - * In non-prefetch mode, do look up '.partial' and '.gz.partial' - * segments. - */ - if (fio_is_remote(FIO_BACKUP_HOST)) - { - char *errmsg = NULL; - /* get file via ssh */ #ifdef HAVE_LIBZ - /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ - if (IsXLogFileName(filename)) - rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg); - if (rc == FILE_MISSING) + /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ + if (IsXLogFileName(filename)) + { + in = $i(pioOpenReadStream, backup_drive, from_fullpath_gz, .err = &err); + compressed = in.self != NULL; + if ($haserr(err) && getErrno(err) != ENOENT) + elog(ERROR, "Source file: %s", $errmsg(err)); + } #endif - /* ... failing that, use uncompressed */ - rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg); - - /* When not in prefetch mode, try to use partial file */ - if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) - { - char from_partial[MAXPGPATH]; - + if (in.self == NULL) + { + in = $i(pioOpenReadStream, backup_drive, from_fullpath, .err = &err); + if ($haserr(err) && getErrno(err) != ENOENT) + elog(ERROR, "Source file: %s", $errmsg(err)); + } + /* try partial file */ + if (in.self == NULL && !prefetch_mode && IsXLogFileName(filename)) + { + char from_partial[MAXPGPATH]; #ifdef HAVE_LIBZ - /* '.gz.partial' goes first ... */ - snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = fio_send_file_gz(from_partial, out, &errmsg); - if (rc == FILE_MISSING) -#endif - { - /* ... failing that, use '.partial' */ - snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); - rc = fio_send_file(from_partial, out, false, NULL, &errmsg); - } - - if (rc == SEND_OK) - src_partial = true; - } + snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", + from_fullpath); - if (rc == WRITE_FAILED) - elog(WARNING, "Cannot write to file '%s': %s", - to_fullpath, strerror(errno)); - - if (errmsg) - elog(WARNING, "%s", errmsg); - - pg_free(errmsg); - } - else - { - /* get file locally */ -#ifdef HAVE_LIBZ - /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ - if (IsXLogFileName(filename)) - rc = get_wal_file_internal(from_fullpath_gz, to_fullpath, out, true); - if (rc == FILE_MISSING) + in = $i(pioOpenReadStream, backup_drive, from_partial, .err = &err); + compressed = in.self != NULL; + if ($haserr(err) && getErrno(err) != ENOENT) + elog(ERROR, "Source partial file: %s", $errmsg(err)); #endif - /* ... failing that, use uncompressed */ - rc = get_wal_file_internal(from_fullpath, to_fullpath, out, false); - /* When not in prefetch mode, try to use partial file */ - if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) - { - char from_partial[MAXPGPATH]; + if (in.self == NULL) + { + snprintf(from_partial, sizeof(from_partial), "%s.partial", + from_fullpath); + in = $i(pioOpenReadStream, backup_drive, .path = from_partial, + .err = &err); + if ($haserr(err) && getErrno(err) != ENOENT) + elog(ERROR, "Source partial file: %s", $errmsg(err)); + } + + src_partial = true; + } + + if (in.self == NULL) + { + $i(pioClose, out); + $i(pioRemove, db_drive, to_fullpath, true); + free(buf); + if (!prefetch_mode) + elog(LOG, "Target WAL file is missing: %s", filename); + return false; + } #ifdef HAVE_LIBZ - /* '.gz.partial' goes first ... */ - snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = get_wal_file_internal(from_partial, to_fullpath, out, true); - if (rc == FILE_MISSING) + if (compressed) + { + pioFilter_i flt = pioGZDecompressFilter(src_partial); + err = pioCopy($reduce(pioWriteFlush, out), + $reduce(pioRead, in), + flt); + } + else #endif - { - /* ... failing that, use '.partial' */ - snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); - rc = get_wal_file_internal(from_partial, to_fullpath, out, false); - } - - if (rc == SEND_OK) - src_partial = true; - } - } - - if (!prefetch_mode && (rc == FILE_MISSING)) - elog(LOG, "Target WAL file is missing: %s", filename); - - if (rc < 0) - { - fclose(out); - unlink(to_fullpath); - return false; - } - - /* If partial file was used as source, then it is very likely that destination - * file is not equal to XLOG_SEG_SIZE - that is the way pg_receivexlog works. - * We must manually extent it up to XLOG_SEG_SIZE. - */ - if (src_partial) - { - - if (fflush(out) != 0) - { - elog(WARNING, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno)); - fclose(out); - unlink(to_fullpath); - return false; - } - - if (ftruncate(fileno(out), xlog_seg_size) != 0) - { - elog(WARNING, "Cannot extend file \"%s\": %s", to_fullpath, strerror(errno)); - fclose(out); - unlink(to_fullpath); - return false; - } - } - - if (fclose(out) != 0) - { - elog(WARNING, "Cannot close file '%s': %s", to_fullpath, strerror(errno)); - unlink(to_fullpath); - return false; - } - - elog(LOG, "WAL file successfully %s: %s", - prefetch_mode ? "prefetched" : "copied", filename); - return true; -} - -/* - * Copy WAL segment with possible decompression from local archive. - * Return codes: - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - * WRITE_FAILED (-4) - * ZLIB_ERROR (-5) - */ -int -get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, - bool is_decompress) -{ -#ifdef HAVE_LIBZ - gzFile gz_in = NULL; -#endif - FILE *in = NULL; - char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ - int exit_code = 0; - - elog(LOG, "Attempting to %s WAL file '%s'", - is_decompress ? "open compressed" : "open", from_path); - - /* open source file for read */ - if (!is_decompress) - { - in = fopen(from_path, PG_BINARY_R); - if (in == NULL) - { - if (errno == ENOENT) - exit_code = FILE_MISSING; - else - { - elog(WARNING, "Cannot open source WAL file \"%s\": %s", - from_path, strerror(errno)); - exit_code = OPEN_FAILED; - } - goto cleanup; - } - - /* disable stdio buffering */ - setvbuf(out, NULL, _IONBF, BUFSIZ); - } -#ifdef HAVE_LIBZ - else - { - gz_in = gzopen(from_path, PG_BINARY_R); - if (gz_in == NULL) - { - if (errno == ENOENT) - exit_code = FILE_MISSING; - else - { - elog(WARNING, "Cannot open compressed WAL file \"%s\": %s", - from_path, strerror(errno)); - exit_code = OPEN_FAILED; - } - - goto cleanup; - } - } -#endif - - /* copy content */ - for (;;) - { - int read_len = 0; - -#ifdef HAVE_LIBZ - if (is_decompress) - { - read_len = gzread(gz_in, buf, OUT_BUF_SIZE); - - if (read_len <= 0) - { - if (gzeof(gz_in)) - break; - else - { - elog(WARNING, "Cannot read compressed WAL file \"%s\": %s", - from_path, get_gz_error(gz_in, errno)); - exit_code = READ_FAILED; - break; - } - } - } - else -#endif - { - read_len = fread(buf, 1, OUT_BUF_SIZE, in); - - if (ferror(in)) - { - elog(WARNING, "Cannot read source WAL file \"%s\": %s", - from_path, strerror(errno)); - exit_code = READ_FAILED; - break; - } - - if (read_len == 0 && feof(in)) - break; - } - - if (read_len > 0) - { - if (fwrite(buf, 1, read_len, out) != read_len) - { - elog(WARNING, "Cannot write to WAL file '%s': %s", - to_path, strerror(errno)); - exit_code = WRITE_FAILED; - break; - } - } - } - -cleanup: -#ifdef HAVE_LIBZ - if (gz_in) - gzclose(gz_in); -#endif - if (in) - fclose(in); - - pg_free(buf); - return exit_code; + { + err = pioCopy($reduce(pioWriteFlush, out), + $reduce(pioRead, in)); + } + + /* close source file */ + $i(pioClose, in); /* ignore error */ + + if ($haserr(err)) + { + $i(pioClose, out); + $i(pioRemove, db_drive, to_fullpath, true); + elog(ERROR, "%s", $errmsg(err)); + } + + /* If partial file was used as source, then it is very likely that destination + * file is not equal to XLOG_SEG_SIZE - that is the way pg_receivexlog works. + * We must manually extent it up to XLOG_SEG_SIZE. + */ + if (src_partial) + { + err = $i(pioTruncate, out, xlog_seg_size); + if ($haserr(err)) + { + elog(WARNING, "Extend file: %s", $errmsg(err)); + $i(pioClose, out); + $i(pioRemove, db_drive, to_fullpath, true); + free(buf); + return false; + } + } + + err = $i(pioClose, out); + if ($haserr(err)) + { + elog(WARNING, "%s", $errmsg(err)); + $i(pioRemove, db_drive, to_fullpath, true); + free(buf); + return false; + } + + elog(LOG, "WAL file successfully %s: %s", + prefetch_mode ? "prefetched" : "copied", filename); + free(buf); + return true; } bool next_wal_segment_exists(TimeLineID tli, XLogSegNo segno, const char *prefetch_dir, uint32 wal_seg_size) diff --git a/src/backup.c b/src/backup.c index 35fc98092..66b1a2201 100644 --- a/src/backup.c +++ b/src/backup.c @@ -17,7 +17,6 @@ #include "pgtar.h" #include "streamutil.h" -#include #include #include @@ -28,21 +27,17 @@ /* list of files contained in backup */ parray *backup_files_list = NULL; +parray *backup_files_hash = NULL; /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; -// TODO: move to PGnodeInfo -bool exclusive_backup = false; - /* Is pg_start_backup() was executed */ bool backup_in_progress = false; /* * Backup routines */ -static void backup_cleanup(bool fatal, void *userdata); - static void *backup_files(void *arg); static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, @@ -50,10 +45,7 @@ static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, static void pg_switch_wal(PGconn *conn); -static void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); -static void check_external_for_tablespaces(parray *external_list, - PGconn *backup_conn); static parray *get_database_map(PGconn *pg_startbackup_conn); /* pgpro specific functions */ @@ -80,7 +72,7 @@ backup_stopbackup_callback(bool fatal, void *userdata) { elog(WARNING, "backup in progress, stop backup"); /* don't care about stop_lsn in case of error */ - pg_stop_backup_send(st->conn, st->server_version, current.from_replica, exclusive_backup, NULL); + pg_stop_backup_send(st->conn, st->server_version, current.from_replica, NULL); } } @@ -104,6 +96,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; + parray *prev_backup_hashtable = NULL; parray *backup_list = NULL; parray *external_dirs = NULL; parray *database_map = NULL; @@ -116,6 +109,10 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, char pretty_time[20]; char pretty_bytes[20]; + pioSyncTree_i syncer; + err_i err = $noerr(); + + elog(INFO, "Database backup start"); if(current.external_dir_str) { @@ -133,11 +130,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn); /* Obtain current timeline */ -#if PG_VERSION_NUM >= 90600 current.tli = get_current_timeline(backup_conn); -#else - current.tli = get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); -#endif /* * In incremental backup mode ensure that already-validated @@ -242,16 +235,20 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !current.stream) { /* Check that archive_dir can be reached */ - if (fio_access(instanceState->instance_wal_subdir_path, F_OK, FIO_BACKUP_HOST) != 0) + err_i err = $noerr(); + + if (!$i(pioExists, current.backup_location, .path = instanceState->instance_wal_subdir_path, + .expected_kind = PIO_KIND_DIRECTORY, .err = &err)) elog(ERROR, "WAL archive directory is not accessible \"%s\": %s", - instanceState->instance_wal_subdir_path, strerror(errno)); + instanceState->instance_wal_subdir_path, + $haserr(err) ? $errmsg(err) : "no such file or directory"); /* * Do not wait start_lsn for stream backup. * Because WAL streaming will start after pg_start_backup() in stream * mode. */ - wait_wal_lsn(instanceState->instance_wal_subdir_path, current.start_lsn, true, current.tli, false, true, ERROR, false); + wait_wal_lsn(instanceState->instance_wal_subdir_path, current.start_lsn, true, current.tli, true, ERROR); } /* start stream replication */ @@ -260,7 +257,12 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, char stream_xlog_path[MAXPGPATH]; join_path_components(stream_xlog_path, current.database_dir, PG_XLOG_DIR); - fio_mkdir(stream_xlog_path, DIR_PERMISSION, FIO_BACKUP_HOST); + err = $i(pioMakeDir, current.backup_location, .path = stream_xlog_path, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create WAL directory: %s", $errmsg(err)); + } start_WAL_streaming(backup_conn, stream_xlog_path, &instance_config.conn_opt, current.start_lsn, current.tli, true); @@ -269,7 +271,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, * PAGE backup in stream mode is waited twice, first for * segment in WAL archive and then for streamed segment */ - wait_wal_lsn(stream_xlog_path, current.start_lsn, true, current.tli, false, true, ERROR, true); + wait_WAL_streaming_starts(); } /* initialize backup's file list */ @@ -277,8 +279,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, join_path_components(external_prefix, current.root_dir, EXTERNAL_DIR); /* list files with the logical path. omit $PGDATA */ - fio_list_dir(backup_files_list, instance_config.pgdata, - true, true, false, backup_logs, true, 0); + db_list_dir(backup_files_list, instance_config.pgdata, true, backup_logs, 0, FIO_DB_HOST); /* * Get database_map (name to oid) for use in partial restore feature. @@ -294,14 +295,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, { for (i = 0; i < parray_num(external_dirs); i++) { - /* External dirs numeration starts with 1. - * 0 value is not external dir */ - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(backup_files_list, parray_get(external_dirs, i), - false, true, false, false, true, i+1); - else - dir_list_file(backup_files_list, parray_get(external_dirs, i), - false, true, false, false, true, i+1, FIO_LOCAL_HOST); + db_list_dir(backup_files_list, parray_get(external_dirs, i), + false, false, i + 1, FIO_DB_HOST); } } @@ -336,6 +331,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, /* Extract information about files in backup_list parsing their names:*/ parse_filelist_filenames(backup_files_list, instance_config.pgdata); + backup_files_hash = make_filelist_hashtable(backup_files_list); + elog(INFO, "Current Start LSN: %X/%X, TLI: %X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), current.tli); @@ -398,7 +395,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, pgFile *file = (pgFile *) parray_get(backup_files_list, i); /* if the entry was a directory, create it in the backup */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) { char dirpath[MAXPGPATH]; @@ -413,7 +410,13 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, join_path_components(dirpath, current.database_dir, file->rel_path); elog(LOG, "Create directory '%s'", dirpath); - fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); + err = $i(pioMakeDir, current.backup_location, .path = dirpath, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create instance backup directory: %s", + $errmsg(err)); + } } } @@ -422,10 +425,10 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, pfilearray_clear_locks(backup_files_list); /* Sort by size for load balancing */ - parray_qsort(backup_files_list, pgFileCompareSize); + parray_qsort(backup_files_list, pgFileCompareSizeDesc); /* Sort the array for binary search */ if (prev_backup_filelist) - parray_qsort(prev_backup_filelist, pgFileCompareRelPathWithExternal); + prev_backup_hashtable = make_filelist_hashtable(prev_backup_filelist); /* write initial backup_content.control file and update backup.control */ write_backup_filelist(¤t, backup_files_list, @@ -444,12 +447,13 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, backup_files_arg *arg = &(threads_args[i]); arg->nodeInfo = nodeInfo; + arg->instanceState = instanceState; arg->from_root = instance_config.pgdata; arg->to_root = current.database_dir; arg->external_prefix = external_prefix; arg->external_dirs = external_dirs; arg->files_list = backup_files_list; - arg->prev_filelist = prev_backup_filelist; + arg->prev_filehash = prev_backup_hashtable; arg->prev_start_lsn = prev_backup_start_lsn; arg->hdr_map = &(current.hdr_map); arg->thread_num = i+1; @@ -492,15 +496,16 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, { parray_walk(prev_backup_filelist, pgFileFree); parray_free(prev_backup_filelist); + parray_free(prev_backup_hashtable); } /* Notify end of backup */ pg_stop_backup(instanceState, ¤t, backup_conn, nodeInfo); - /* In case of backup from replica >= 9.6 we must fix minRecPoint, + /* In case of backup from replica we must fix minRecPoint, * First we must find pg_control in backup_files_list. */ - if (current.from_replica && !exclusive_backup) + if (current.from_replica) { pgFile *pg_control = NULL; @@ -520,16 +525,14 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, elog(ERROR, "Failed to find file \"%s\" in backup filelist.", XLOG_CONTROL_FILE); - set_min_recovery_point(pg_control, current.database_dir, current.stop_lsn); + set_min_recovery_point(instanceState->database_location, instanceState->backup_location, + pg_control, current.database_dir, current.stop_lsn); } /* close and sync page header map */ - if (current.hdr_map.fp) + if ($notNULL(current.hdr_map.fp)) { cleanup_header_map(&(current.hdr_map)); - - if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); } /* close ssh session in main thread */ @@ -558,37 +561,23 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, /* Sync all copied files unless '--no-sync' flag is used */ if (no_sync) elog(WARNING, "Backup files are not synced to disk"); - else + else if ($implements(pioSyncTree, current.backup_location.self, &syncer)) { + char external_dst[MAXPGPATH]; elog(INFO, "Syncing backup files to disk"); time(&start_time); - for (i = 0; i < parray_num(backup_files_list); i++) - { - char to_fullpath[MAXPGPATH]; - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - - /* TODO: sync directory ? */ - if (S_ISDIR(file->mode)) - continue; - - if (file->write_size <= 0) - continue; + err = $i(pioSyncTree, syncer, current.database_dir); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Syncing backup's database_dir"); - /* construct fullpath */ - if (file->external_dir_num == 0) - join_path_components(to_fullpath, current.database_dir, file->rel_path); - else - { - char external_dst[MAXPGPATH]; - - makeExternalDirPathByNum(external_dst, external_prefix, - file->external_dir_num); - join_path_components(to_fullpath, external_dst, file->rel_path); - } - - if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", to_fullpath, strerror(errno)); + for (i = 1; i <= parray_num(external_dirs); i++) + { + makeExternalDirPathByNum(external_dst, external_prefix, i); + err = $i(pioSyncTree, syncer, external_dst); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), + "Syncing backup's external dir %d", i); } time(&end_time); @@ -625,7 +614,9 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, parray_walk(backup_files_list, pgFileFree); parray_free(backup_files_list); + parray_free(backup_files_hash); backup_files_list = NULL; + backup_files_hash = NULL; } /* @@ -810,13 +801,11 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, * instance we opened connection to. And that target backup database PGDATA * belogns to the same instance. */ - check_system_identifiers(backup_conn, instance_config.pgdata); + check_system_identifiers(instanceState->database_location, backup_conn, instance_config.pgdata); /* below perform checks specific for backup command */ -#if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(backup_conn)) elog(ERROR, "Failed to retrieve wal_segment_size"); -#endif get_ptrack_version(backup_conn, &nodeInfo); // elog(WARNING, "ptrack_version_num %d", ptrack_version_num); @@ -836,11 +825,6 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, } } - if (current.from_replica && exclusive_backup) - /* Check master connection options */ - if (instance_config.master_conn_opt.pghost == NULL) - elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); - /* add note to backup if requested */ if (set_backup_params && set_backup_params->note) add_note(¤t, set_backup_params->note); @@ -921,22 +905,12 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) elog(ERROR, "Unknown server version %d", nodeInfo->server_version); if (nodeInfo->server_version < 100000) - sprintf(nodeInfo->server_version_str, "%d.%d", - nodeInfo->server_version / 10000, - (nodeInfo->server_version / 100) % 100); - else - sprintf(nodeInfo->server_version_str, "%d", - nodeInfo->server_version / 10000); - - if (nodeInfo->server_version < 90500) elog(ERROR, "server version is %s, must be %s or higher", - nodeInfo->server_version_str, "9.5"); + nodeInfo->server_version_str, "10"); - if (current.from_replica && nodeInfo->server_version < 90600) - elog(ERROR, - "server version is %s, must be %s or higher for backup from replica", - nodeInfo->server_version_str, "9.6"); + sprintf(nodeInfo->server_version_str, "%d", + nodeInfo->server_version / 10000); if (nodeInfo->pgpro_support) res = pgut_execute(conn, "SELECT pg_catalog.pgpro_edition()", 0, NULL); @@ -988,9 +962,6 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) if (res) PQclear(res); - - /* Do exclusive backup only for PostgreSQL 9.5 */ - exclusive_backup = nodeInfo->server_version < 90600; } /* @@ -1000,12 +971,12 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) * All system identifiers must be equal. */ void -check_system_identifiers(PGconn *conn, const char *pgdata) +check_system_identifiers(pioDrive_i drive, PGconn *conn, const char *pgdata) { uint64 system_id_conn; uint64 system_id_pgdata; - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); + system_id_pgdata = get_system_identifier(drive, pgdata, false); system_id_conn = get_remote_system_identifier(conn); /* for checkdb check only system_id_pgdata and system_id_conn */ @@ -1078,12 +1049,12 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, params[1] = smooth ? "false" : "true"; res = pgut_execute(conn, #if PG_VERSION_NUM >= 150000 - "SELECT pg_catalog.pg_backup_start($1, $2)", + "SELECT pg_catalog.pg_backup_start($1, $2)", #else - "SELECT pg_catalog.pg_start_backup($1, $2, false)", + "SELECT pg_catalog.pg_start_backup($1, $2, false)", #endif - 2, - params); + 2, + params); /* * Set flag that pg_start_backup() was called. If an error will happen it @@ -1102,21 +1073,16 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, PQclear(res); if ((!backup->stream || backup->backup_mode == BACKUP_MODE_DIFF_PAGE) && - !backup->from_replica && - !(nodeInfo->server_version < 90600 && - !nodeInfo->is_superuser)) + !backup->from_replica) /* * Switch to a new WAL segment. It is necessary to get archived WAL * segment, which includes start LSN of current backup. - * Don`t do this for replica backups and for PG 9.5 if pguser is not superuser - * (because in 9.5 only superuser can switch WAL) */ pg_switch_wal(conn); } /* * Switch to a new WAL segment. It should be called only for master. - * For PG 9.5 it should be called only if pguser is superuser. */ void pg_switch_wal(PGconn *conn) @@ -1125,11 +1091,7 @@ pg_switch_wal(PGconn *conn) pg_silent_client_messages(conn); -#if PG_VERSION_NUM >= 100000 res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_wal()", 0, NULL); -#else - res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_xlog()", 0, NULL); -#endif PQclear(res); } @@ -1215,6 +1177,8 @@ get_database_map(PGconn *conn) parray_append(database_map, db_entry); } + PQclear(res); + return database_map; } @@ -1277,17 +1241,8 @@ pg_is_superuser(PGconn *conn) * streamed in 'archive_dir' or 'pg_wal' directory. * * If flag 'is_start_lsn' is set then issue warning for first-time users. - * If flag 'in_prev_segment' is set, look for LSN in previous segment, - * with EndRecPtr >= Target LSN. It should be used only for solving - * invalid XRecOff problem. * If flag 'segment_only' is set, then, instead of waiting for LSN, wait for segment, * containing that LSN. - * If flags 'in_prev_segment' and 'segment_only' are both set, then wait for - * previous segment. - * - * Flag 'in_stream_dir' determine whether we looking for WAL in 'pg_wal' directory or - * in archive. Do note, that we cannot rely sorely on global variable 'stream_wal' (current.stream) because, - * for example, PAGE backup must(!) look for start_lsn in archive regardless of wal_mode. * * 'timeout_elevel' determine the elevel for timeout elog message. If elevel lighter than * ERROR is used, then return InvalidXLogRecPtr. TODO: return something more concrete, for example 1. @@ -1297,16 +1252,15 @@ pg_is_superuser(PGconn *conn) */ XLogRecPtr wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, - bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir) + bool segment_only, int timeout_elevel) { XLogSegNo targetSegNo; char wal_segment_path[MAXPGPATH], wal_segment[MAXFNAMELEN]; + char* try_segment_path = NULL; bool file_exists = false; uint32 try_count = 0, timeout; - char *wal_delivery_str = in_stream_dir ? "streamed":"archived"; #ifdef HAVE_LIBZ char gz_wal_segment_path[MAXPGPATH]; @@ -1314,7 +1268,7 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l /* Compute the name of the WAL file containing requested LSN */ GetXLogSegNo(target_lsn, targetSegNo, instance_config.xlog_seg_size); - if (in_prev_segment) + if (target_lsn % instance_config.xlog_seg_size == 0) targetSegNo--; GetXLogFileName(wal_segment, tli, targetSegNo, instance_config.xlog_seg_size); @@ -1358,11 +1312,17 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l #ifdef HAVE_LIBZ file_exists = fileExists(gz_wal_segment_path, FIO_BACKUP_HOST); if (file_exists) - elog(LOG, "Found compressed WAL segment: %s", wal_segment_path); + { + elog(LOG, "Found compressed WAL segment: %s", gz_wal_segment_path); + try_segment_path = gz_wal_segment_path; + } #endif } else + { elog(LOG, "Found WAL segment: %s", wal_segment_path); + try_segment_path = wal_segment_path; + } } if (file_exists) @@ -1371,18 +1331,6 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l if (segment_only) return InvalidXLogRecPtr; - /* - * A WAL segment found. Look for target LSN in it. - */ - if (!XRecOffIsNull(target_lsn) && - wal_contains_lsn(wal_segment_dir, target_lsn, tli, - instance_config.xlog_seg_size)) - /* Target LSN was found */ - { - elog(LOG, "Found LSN: %X/%X", (uint32) (target_lsn >> 32), (uint32) target_lsn); - return target_lsn; - } - /* * If we failed to get target LSN in a reasonable time, try * to get LSN of last valid record prior to the target LSN. But only @@ -1397,39 +1345,42 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l * 2. Replica returened endpoint LSN with NullXRecOff. We want to look * for previous record which endpoint points greater or equal LSN in previous WAL segment. */ - if (current.from_replica && - (XRecOffIsNull(target_lsn) || try_count > timeout / 2)) - { - XLogRecPtr res; + XLogRecPtr prev; - res = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, - in_prev_segment, instance_config.xlog_seg_size); + prev = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, + instance_config.xlog_seg_size); - if (!XLogRecPtrIsInvalid(res)) - { - /* LSN of the prior record was found */ - elog(LOG, "Found prior LSN: %X/%X", - (uint32) (res >> 32), (uint32) res); - return res; - } + if (!XLogRecPtrIsInvalid(prev)) + { + /* LSN of the prior record was found */ + elog(LOG, "Found prior LSN: %X/%X", + (uint32) (prev >> 32), (uint32) prev); + return target_lsn; } + elog(ERROR, "Attempt %d: prior lsn is not found in %s", try_count+1, + try_segment_path); + } + else + { + elog(LOG, "Attempt %d: file %s is not found", try_count+1, + wal_segment_path); } sleep(1); if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during waiting for WAL %s", in_stream_dir ? "streaming" : "archiving"); + elog(ERROR, "Interrupted during waiting for WAL archiving"); try_count++; /* Inform user if WAL segment is absent in first attempt */ if (try_count == 1) { if (segment_only) - elog(INFO, "Wait for WAL segment %s to be %s", - wal_segment_path, wal_delivery_str); + elog(INFO, "Wait for WAL segment %s to be archived", + wal_segment_path); else - elog(INFO, "Wait for LSN %X/%X in %s WAL segment %s", + elog(INFO, "Wait for LSN %X/%X in archived WAL segment %s", (uint32) (target_lsn >> 32), (uint32) target_lsn, - wal_delivery_str, wal_segment_path); + wal_segment_path); } if (!current.stream && is_start_lsn && try_count == 30) @@ -1440,158 +1391,22 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l if (timeout > 0 && try_count > timeout) { if (file_exists) - elog(timeout_elevel, "WAL segment %s was %s, " - "but target LSN %X/%X could not be %s in %d seconds", - wal_segment, wal_delivery_str, + elog(timeout_elevel, "WAL segment %s was archived, " + "but target LSN %X/%X could not be archived in %d seconds", + wal_segment, (uint32) (target_lsn >> 32), (uint32) target_lsn, - wal_delivery_str, timeout); + timeout); /* If WAL segment doesn't exist or we wait for previous segment */ else elog(timeout_elevel, - "WAL segment %s could not be %s in %d seconds", - wal_segment, wal_delivery_str, timeout); + "WAL segment %s could not be archived in %d seconds", + wal_segment, timeout); return InvalidXLogRecPtr; } } } -/* - * Check stop_lsn (returned from pg_stop_backup()) and update backup->stop_lsn - */ -void -wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup) -{ - bool stop_lsn_exists = false; - - /* It is ok for replica to return invalid STOP LSN - * UPD: Apparently it is ok even for a master. - */ - if (!XRecOffIsValid(stop_lsn)) - { - XLogSegNo segno = 0; - XLogRecPtr lsn_tmp = InvalidXLogRecPtr; - - /* - * Even though the value is invalid, it's expected postgres behaviour - * and we're trying to fix it below. - */ - elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", - (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - /* - * Note: even with gdb it is very hard to produce automated tests for - * contrecord + invalid LSN, so emulate it for manual testing. - */ - //lsn = lsn - XLOG_SEG_SIZE; - //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", - // (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - GetXLogSegNo(stop_lsn, segno, instance_config.xlog_seg_size); - - /* - * Note, that there is no guarantee that corresponding WAL file even exists. - * Replica may return LSN from future and keep staying in present. - * Or it can return invalid LSN. - * - * That's bad, since we want to get real LSN to save it in backup label file - * and to use it in WAL validation. - * - * So we try to do the following: - * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and - * look for the first valid record in it. - * It solves the problem of occasional invalid LSN on write-busy system. - * 2. Failing that, look for record in previous segment with endpoint - * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN - * on write-idle system. If that fails too, error out. - */ - - /* stop_lsn is pointing to a 0 byte of xlog segment */ - if (stop_lsn % instance_config.xlog_seg_size == 0) - { - /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ - wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, - false, true, WARNING, backup->stream); - - /* Get the first record in segment with current stop_lsn */ - lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout); - - /* Check that returned LSN is valid and greater than stop_lsn */ - if (XLogRecPtrIsInvalid(lsn_tmp) || - !XRecOffIsValid(lsn_tmp) || - lsn_tmp < stop_lsn) - { - /* Backup from master should error out here */ - if (!backup->from_replica) - elog(ERROR, "Failed to get next WAL record after %X/%X", - (uint32) (stop_lsn >> 32), - (uint32) (stop_lsn)); - - /* No luck, falling back to looking up for previous record */ - elog(WARNING, "Failed to get next WAL record after %X/%X, " - "looking for previous WAL record", - (uint32) (stop_lsn >> 32), - (uint32) (stop_lsn)); - - /* Despite looking for previous record there is not guarantee of success - * because previous record can be the contrecord. - */ - lsn_tmp = wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, - true, false, ERROR, backup->stream); - - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record prior to %X/%X", - (uint32) (stop_lsn >> 32), - (uint32) (stop_lsn)); - } - } - /* stop lsn is aligned to xlog block size, just find next lsn */ - else if (stop_lsn % XLOG_BLCKSZ == 0) - { - /* Wait for segment with current stop_lsn */ - wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, - false, true, ERROR, backup->stream); - - /* Get the next closest record in segment with current stop_lsn */ - lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout, - stop_lsn); - - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record next to %X/%X", - (uint32) (stop_lsn >> 32), - (uint32) (stop_lsn)); - } - /* PostgreSQL returned something very illegal as STOP_LSN, error out */ - else - elog(ERROR, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - /* Setting stop_backup_lsn will set stop point for streaming */ - stop_backup_lsn = lsn_tmp; - stop_lsn_exists = true; - } - - elog(INFO, "stop_lsn: %X/%X", - (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - /* - * Wait for stop_lsn to be archived or streamed. - * If replica returned valid STOP_LSN of not actually existing record, - * look for previous record with endpoint >= STOP_LSN. - */ - if (!stop_lsn_exists) - stop_backup_lsn = wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, - false, false, ERROR, backup->stream); - - backup->stop_lsn = stop_backup_lsn; -} - /* Remove annoying NOTICE messages generated by backend */ void pg_silent_client_messages(PGconn *conn) @@ -1619,20 +1434,9 @@ pg_create_restore_point(PGconn *conn, time_t backup_start_time) } void -pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text) +pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, char **query_text) { static const char - stop_exlusive_backup_query[] = - /* - * Stop the non-exclusive backup. Besides stop_lsn it returns from - * pg_stop_backup(false) copy of the backup label and tablespace map - * so they can be written to disk by the caller. - * TODO, question: add NULLs as backup_label and tablespace_map? - */ - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " pg_catalog.pg_stop_backup() as lsn", stop_backup_on_master_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1641,14 +1445,6 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " labelfile," " spcmapfile" " FROM pg_catalog.pg_stop_backup(false, false)", - stop_backup_on_master_before10_query[] = - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " lsn," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)", stop_backup_on_master_after15_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1658,7 +1454,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " spcmapfile" " FROM pg_catalog.pg_backup_stop(false)", /* - * In case of backup from replica >= 9.6 we do not trust minRecPoint + * In case of backup from replica we do not trust minRecPoint * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. */ stop_backup_on_replica_query[] = @@ -1669,14 +1465,6 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " labelfile," " spcmapfile" " FROM pg_catalog.pg_stop_backup(false, false)", - stop_backup_on_replica_before10_query[] = - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " pg_catalog.pg_last_xlog_replay_location()," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)", stop_backup_on_replica_after15_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1687,22 +1475,14 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " FROM pg_catalog.pg_backup_stop(false)"; const char * const stop_backup_query = - is_exclusive ? - stop_exlusive_backup_query : server_version >= 150000 ? (is_started_on_replica ? stop_backup_on_replica_after15_query : stop_backup_on_master_after15_query ) : - (server_version >= 100000 ? - (is_started_on_replica ? - stop_backup_on_replica_query : - stop_backup_on_master_query - ) : - (is_started_on_replica ? - stop_backup_on_replica_before10_query : - stop_backup_on_master_before10_query - ) + (is_started_on_replica ? + stop_backup_on_replica_query : + stop_backup_on_master_query ); bool sent = false; @@ -1740,7 +1520,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica */ void pg_stop_backup_consume(PGconn *conn, int server_version, - bool is_exclusive, uint32 timeout, const char *query_text, + uint32 timeout, const char *query_text, PGStopBackupResult *result) { PGresult *query_result; @@ -1852,54 +1632,31 @@ pg_stop_backup_consume(PGconn *conn, int server_version, } /* get backup_label_content */ - result->backup_label_content = NULL; // if (!PQgetisnull(query_result, 0, backup_label_colno)) - if (!is_exclusive) - { - result->backup_label_content_len = PQgetlength(query_result, 0, backup_label_colno); - if (result->backup_label_content_len > 0) - result->backup_label_content = pgut_strndup(PQgetvalue(query_result, 0, backup_label_colno), - result->backup_label_content_len); - } else { - result->backup_label_content_len = 0; - } + result->backup_label_content = ft_strdupc(PQgetvalue(query_result, 0, backup_label_colno)); /* get tablespace_map_content */ - result->tablespace_map_content = NULL; // if (!PQgetisnull(query_result, 0, tablespace_map_colno)) - if (!is_exclusive) - { - result->tablespace_map_content_len = PQgetlength(query_result, 0, tablespace_map_colno); - if (result->tablespace_map_content_len > 0) - result->tablespace_map_content = pgut_strndup(PQgetvalue(query_result, 0, tablespace_map_colno), - result->tablespace_map_content_len); - } else { - result->tablespace_map_content_len = 0; - } + result->tablespace_map_content = ft_strdupc(PQgetvalue(query_result, 0, tablespace_map_colno)); } /* * helper routine used to write backup_label and tablespace_map in pg_stop_backup() */ void -pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, - const void *data, size_t len, parray *file_list) +pg_stop_backup_write_file_helper(pioDrive_i drive, const char *path, const char *filename, const char *error_msg_filename, + ft_str_t data, parray *file_list) { - FILE *fp; pgFile *file; char full_filename[MAXPGPATH]; + err_i err = $noerr(); join_path_components(full_filename, path, filename); - fp = fio_fopen(full_filename, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "can't open %s file \"%s\": %s", - error_msg_filename, full_filename, strerror(errno)); - if (fio_fwrite(fp, data, len) != len || - fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "can't write %s file \"%s\": %s", - error_msg_filename, full_filename, strerror(errno)); + err = $i(pioWriteFile, drive, .path = full_filename, + .content = ft_str2bytes(data)); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "writting stop backup file"); /* * It's vital to check if files_list is initialized, @@ -1907,16 +1664,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c */ if (file_list) { - file = pgFileNew(full_filename, filename, true, 0, - FIO_BACKUP_HOST); - - if (S_ISREG(file->mode)) - { - file->crc = pgFileGetCRC(full_filename, true, false); - - file->write_size = file->size; - file->uncompressed_size = file->size; - } + file = pgFileNew(full_filename, filename, true, true, drive); parray_append(file_list, file); } } @@ -1924,7 +1672,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c /* * Notify end of backup to PostgreSQL server. */ -static void +void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo) { @@ -1943,64 +1691,61 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb /* Create restore point * Only if backup is from master. - * For PG 9.5 create restore point only if pguser is superuser. */ - if (!backup->from_replica && - !(nodeInfo->server_version < 90600 && - !nodeInfo->is_superuser)) //TODO: check correctness + if (!backup->from_replica) pg_create_restore_point(pg_startbackup_conn, backup->start_time); /* Execute pg_stop_backup using PostgreSQL connection */ - pg_stop_backup_send(pg_startbackup_conn, nodeInfo->server_version, backup->from_replica, exclusive_backup, &query_text); + pg_stop_backup_send(pg_startbackup_conn, nodeInfo->server_version, backup->from_replica, &query_text); /* * Wait for the result of pg_stop_backup(), but no longer than * archive_timeout seconds */ - pg_stop_backup_consume(pg_startbackup_conn, nodeInfo->server_version, exclusive_backup, timeout, query_text, &stop_backup_result); + pg_stop_backup_consume(pg_startbackup_conn, nodeInfo->server_version, timeout, query_text, &stop_backup_result); if (backup->stream) { join_path_components(stream_xlog_path, backup->database_dir, PG_XLOG_DIR); xlog_path = stream_xlog_path; + /* This function will also add list of xlog files + * to the passed filelist */ + if(wait_WAL_streaming_end(backup_files_list, xlog_path, + stop_backup_result.lsn, backup)) + elog(ERROR, "WAL streaming failed"); + elog(INFO, "backup->stop_lsn %X/%X", + (uint32_t)(backup->stop_lsn>>32), (uint32_t)backup->stop_lsn); + } else + { xlog_path = instanceState->instance_wal_subdir_path; - wait_wal_and_calculate_stop_lsn(xlog_path, stop_backup_result.lsn, backup); + backup->stop_lsn = wait_wal_lsn(xlog_path, stop_backup_result.lsn, false, backup->tli, + false, ERROR); + if (XLogRecPtrIsInvalid(backup->stop_lsn)) + elog(ERROR, "We couldn't wait for %llX", + (long long)stop_backup_result.lsn); + ft_assert(backup->stop_lsn == stop_backup_result.lsn); + } /* Write backup_label and tablespace_map */ - if (!exclusive_backup) - { - Assert(stop_backup_result.backup_label_content != NULL); - - /* Write backup_label */ - pg_stop_backup_write_file_helper(backup->database_dir, PG_BACKUP_LABEL_FILE, "backup label", - stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, - backup_files_list); - free(stop_backup_result.backup_label_content); - stop_backup_result.backup_label_content = NULL; - stop_backup_result.backup_label_content_len = 0; - - /* Write tablespace_map */ - if (stop_backup_result.tablespace_map_content != NULL) - { - pg_stop_backup_write_file_helper(backup->database_dir, PG_TABLESPACE_MAP_FILE, "tablespace map", - stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, - backup_files_list); - free(stop_backup_result.tablespace_map_content); - stop_backup_result.tablespace_map_content = NULL; - stop_backup_result.tablespace_map_content_len = 0; - } - } + Assert(stop_backup_result.backup_label_content.len != 0); - if (backup->stream) + /* Write backup_label */ + pg_stop_backup_write_file_helper(backup->backup_location, + backup->database_dir, PG_BACKUP_LABEL_FILE, "backup label", + stop_backup_result.backup_label_content, backup_files_list); + ft_str_free(&stop_backup_result.backup_label_content); + + /* Write tablespace_map */ + if (stop_backup_result.tablespace_map_content.len != 0) { - /* This function will also add list of xlog files - * to the passed filelist */ - if(wait_WAL_streaming_end(backup_files_list)) - elog(ERROR, "WAL streaming failed"); + pg_stop_backup_write_file_helper(backup->backup_location, + backup->database_dir, PG_TABLESPACE_MAP_FILE, "tablespace map", + stop_backup_result.tablespace_map_content, backup_files_list); } + ft_str_free(&stop_backup_result.tablespace_map_content); backup->recovery_xid = stop_backup_result.snapshot_xid; @@ -2025,7 +1770,7 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb * of the DB cluster. * Also update backup status to ERROR when the backup is not finished. */ -static void +void backup_cleanup(bool fatal, void *userdata) { /* @@ -2053,6 +1798,7 @@ backup_cleanup(bool fatal, void *userdata) static void * backup_files(void *arg) { + FOBJ_FUNC_ARP(); int i; char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; @@ -2060,6 +1806,8 @@ backup_files(void *arg) backup_files_arg *arguments = (backup_files_arg *) arg; int n_backup_files_list = parray_num(arguments->files_list); + pioDrive_i db_drive = arguments->instanceState->database_location; + pioDrive_i backup_drive = arguments->instanceState->backup_location; prev_time = current.start_time; @@ -2070,7 +1818,7 @@ backup_files(void *arg) pgFile *prev_file = NULL; /* We have already copied all directories */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) continue; if (arguments->thread_num == 1) @@ -2125,21 +1873,20 @@ backup_files(void *arg) } /* Encountered some strange beast */ - if (!S_ISREG(file->mode)) - elog(WARNING, "Unexpected type %d of file \"%s\", skipping", - file->mode, from_fullpath); + if (file->kind != PIO_KIND_REGULAR) + elog(WARNING, "Unexpected type %s of file \"%s\", skipping", + pio_file_kind2str(file->kind, from_fullpath), from_fullpath); /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) { - pgFile **prev_file_tmp = NULL; - prev_file_tmp = (pgFile **) parray_bsearch(arguments->prev_filelist, - file, pgFileCompareRelPathWithExternal); + pgFile *prev_file_tmp = NULL; + prev_file_tmp = search_file_in_hashtable(arguments->prev_filehash, file); if (prev_file_tmp) { /* File exists in previous backup */ file->exists_in_prev = true; - prev_file = *prev_file_tmp; + prev_file = prev_file_tmp; } } @@ -2152,12 +1899,14 @@ backup_files(void *arg) instance_config.compress_alg, instance_config.compress_level, arguments->nodeInfo->checksum_version, - arguments->hdr_map, false); + arguments->hdr_map, false, false); } else { - backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, - current.backup_mode, current.parent_backup, true); + backup_non_data_file(db_drive, backup_drive, + file, prev_file, from_fullpath, to_fullpath, + current.backup_mode, current.parent_backup, + true, false); } if (file->write_size == FILE_NOT_FOUND) @@ -2200,7 +1949,7 @@ parse_filelist_filenames(parray *files, const char *root) pgFile *file = (pgFile *) parray_get(files, i); int sscanf_result; - if (S_ISREG(file->mode) && + if (file->kind == PIO_KIND_REGULAR && path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) { /* @@ -2227,7 +1976,7 @@ parse_filelist_filenames(parray *files, const char *root) } } - if (S_ISREG(file->mode) && file->tblspcOid != 0 && + if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 && file->name && file->name[0]) { if (file->forkName == init) @@ -2297,7 +2046,7 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL) { - if (S_ISREG(prev_file->mode) && prev_file->is_datafile) + if (prev_file->kind == PIO_KIND_REGULAR && prev_file->is_datafile) { elog(LOG, "Setting 'is_cfs' on file %s, name %s", prev_file->rel_path, prev_file->name); @@ -2324,8 +2073,8 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) char *rel_path; BlockNumber blkno_inseg; int segno; - pgFile **file_item; - pgFile f; + pgFile *file_item; + pgFile f = {0}; segno = blkno / RELSEG_SIZE; blkno_inseg = blkno % RELSEG_SIZE; @@ -2339,8 +2088,7 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) f.external_dir_num = 0; /* backup_files_list should be sorted before */ - file_item = (pgFile **) parray_bsearch(backup_files_list, &f, - pgFileCompareRelPathWithExternal); + file_item = search_file_in_hashtable(backup_files_hash, &f); /* * If we don't have any record of this file in the file map, it means @@ -2354,7 +2102,7 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) if (num_threads > 1) pthread_lock(&backup_pagemap_mutex); - datapagemap_add(&(*file_item)->pagemap, blkno_inseg); + datapagemap_add(&file_item->pagemap, blkno_inseg); if (num_threads > 1) pthread_mutex_unlock(&backup_pagemap_mutex); @@ -2372,6 +2120,7 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) PGresult *res; int i = 0; int j = 0; + int ntups; char *tablespace_path = NULL; char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " "FROM pg_catalog.pg_tablespace " @@ -2383,7 +2132,8 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) if (!res) elog(ERROR, "Failed to get list of tablespaces"); - for (i = 0; i < res->ntups; i++) + ntups = PQntuples(res); + for (i = 0; i < ntups; i++) { tablespace_path = PQgetvalue(res, i, 0); Assert (strlen(tablespace_path) > 0); @@ -2452,7 +2202,7 @@ calculate_datasize_of_filelist(parray *filelist) if (file->external_dir_num != 0 || file->excluded) continue; - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) { // TODO is a dir always 4K? bytes += 4096; diff --git a/src/catalog.c b/src/catalog.c index 92a2d84b7..2f873c5ed 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -3,7 +3,7 @@ * catalog.c: backup catalog operation * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -13,23 +13,21 @@ #include #include -#include #include #include "utils/file.h" #include "utils/configuration.h" -static pgBackup* get_closest_backup(timelineInfo *tlinfo); -static pgBackup* get_oldest_backup(timelineInfo *tlinfo); +static pgBackup* get_closest_backup(timelineInfo *tlinfo, uint32_t xlog_seg_size); +static pgBackup* get_oldest_backup(timelineInfo *tlinfo, uint32_t xlog_seg_size); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; -static pgBackup *readBackupControlFile(const char *path); -static int create_backup_dir(pgBackup *backup, const char *backup_instance_path); +static err_i create_backup_dir(pgBackup *backup, const char *backup_instance_path); static bool backup_lock_exit_hook_registered = false; static parray *locks = NULL; static int grab_excl_lock_file(const char *backup_dir, const char *backup_id, bool strict); -static int grab_shared_lock_file(pgBackup *backup); +static int grab_shared_lock_file(const char *backup_dir); static int wait_shared_owners(pgBackup *backup); @@ -65,13 +63,20 @@ timelineInfoNew(TimeLineID tli) return tlinfo; } +static void +xlogFile_free(xlogFile* fl) +{ + ft_str_free(&fl->name); + ft_free(fl); +} + /* free timelineInfo object */ void timelineInfoFree(void *tliInfo) { timelineInfo *tli = (timelineInfo *) tliInfo; - parray_walk(tli->xlog_filelist, pgFileFree); + parray_walk(tli->xlog_filelist, (void(*)(void*))xlogFile_free); parray_free(tli->xlog_filelist); if (tli->backups) @@ -109,13 +114,13 @@ unlink_lock_atexit(bool unused_fatal, void *unused_userdata) * If no backup matches, return NULL. */ pgBackup * -read_backup(const char *root_dir) +read_backup(pioDrive_i drive, const char *root_dir) { char conf_path[MAXPGPATH]; join_path_components(conf_path, root_dir, BACKUP_CONTROL_FILE); - return readBackupControlFile(conf_path); + return readBackupControlFile(drive, conf_path); } /* @@ -130,7 +135,7 @@ write_backup_status(pgBackup *backup, BackupStatus status, { pgBackup *tmp; - tmp = read_backup(backup->root_dir); + tmp = read_backup(backup->backup_location, backup->root_dir); if (!tmp) { /* @@ -232,7 +237,7 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) if (exclusive) rc = wait_shared_owners(backup); else - rc = grab_shared_lock_file(backup); + rc = grab_shared_lock_file(backup->root_dir); if (rc != 0) { @@ -298,13 +303,26 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) int grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) { + FOBJ_FUNC_ARP(); + pioDrive_i drive = pioDriveForLocation(FIO_BACKUP_HOST); char lock_file[MAXPGPATH]; - int fd = 0; + FILE *fp = NULL; char buffer[256]; int ntries = LOCK_TIMEOUT; int empty_tries = LOCK_STALE_TIMEOUT; - int len; - int encoded_pid; + size_t len; + pid_t encoded_pid; + int save_errno = 0; + enum { + GELF_FAILED_WRITE = 1, + GELF_FAILED_CLOSE = 2, + } failed_action = 0; + + if ($i(pioIsRemote, drive)) + { + elog(INFO, "Skipping exclusive lock on remote drive"); + return LOCK_OK; + } join_path_components(lock_file, root_dir, BACKUP_LOCK_FILE); @@ -315,19 +333,17 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) */ do { - FILE *fp_out = NULL; - if (interrupted) elog(ERROR, "Interrupted while locking backup %s", backup_id); /* - * Try to create the lock file --- O_EXCL makes this atomic. + * Try to create the lock file --- "wx" makes this atomic. * * Think not to make the file protection weaker than 0600. See * comments below. */ - fd = fio_open(lock_file, O_RDWR | O_CREAT | O_EXCL, FIO_BACKUP_HOST); - if (fd >= 0) + fp = fopen(lock_file, "wx"); + if (fp != NULL) break; /* Success; exit the retry loop */ /* read-only fs is a special case */ @@ -343,7 +359,6 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) * If file already exists or we have some permission problem (???), * then retry; */ -// if ((errno != EEXIST && errno != EACCES)) if (errno != EEXIST) elog(ERROR, "Could not create lock file \"%s\": %s", lock_file, strerror(errno)); @@ -353,18 +368,19 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) * here: file might have been deleted since we tried to create it. */ - fp_out = fopen(lock_file, "r"); - if (fp_out == NULL) + fp = fopen(lock_file, "r"); + if (fp == NULL) { if (errno == ENOENT) continue; /* race condition; try again */ elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); } - len = fread(buffer, 1, sizeof(buffer) - 1, fp_out); - if (ferror(fp_out)) + len = fread(buffer, 1, sizeof(buffer) - 1, fp); + if (ferror(fp)) elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); - fclose(fp_out); + fclose(fp); + fp = NULL; /* * There are several possible reasons for lock file @@ -401,7 +417,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) continue; } - encoded_pid = atoi(buffer); + encoded_pid = (pid_t)atoll(buffer); if (encoded_pid <= 0) { @@ -423,8 +439,8 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) /* complain every fifth interval */ if ((ntries % LOG_FREQ) == 0) { - elog(WARNING, "Process %d is using backup %s, and is still running", - encoded_pid, backup_id); + elog(WARNING, "Process %lld is using backup %s, and is still running", + (long long)encoded_pid, backup_id); elog(WARNING, "Waiting %u seconds on exclusive lock for backup %s", ntries, backup_id); @@ -438,8 +454,8 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) else { if (errno == ESRCH) - elog(WARNING, "Process %d which used backup %s no longer exists", - encoded_pid, backup_id); + elog(WARNING, "Process %lld which used backup %s no longer exists", + (long long)encoded_pid, backup_id); else elog(ERROR, "Failed to send signal 0 to a process %d: %s", encoded_pid, strerror(errno)); @@ -451,7 +467,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) * it. Need a loop because of possible race condition against other * would-be creators. */ - if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) + if (remove(lock_file) < 0) { if (errno == ENOENT) continue; /* race condition, again */ @@ -462,39 +478,33 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) } while (ntries--); /* Failed to acquire exclusive lock in time */ - if (fd <= 0) + if (fp == NULL) return LOCK_FAIL_TIMEOUT; /* * Successfully created the file, now fill it. */ - snprintf(buffer, sizeof(buffer), "%d\n", my_pid); - errno = 0; - if (fio_write(fd, buffer, strlen(buffer)) != strlen(buffer)) - { - int save_errno = errno; + fprintf(fp, "%lld\n", (long long)my_pid); + fflush(fp); - fio_close(fd); - fio_unlink(lock_file, FIO_BACKUP_HOST); - - /* In lax mode if we failed to grab lock because of 'out of space error', - * then treat backup as locked. - * Only delete command should be run in lax mode. - */ - if (!strict && save_errno == ENOSPC) - return LOCK_FAIL_ENOSPC; - else - elog(ERROR, "Could not write lock file \"%s\": %s", - lock_file, strerror(save_errno)); + if (ferror(fp)) + { + failed_action = GELF_FAILED_WRITE; + save_errno = errno; + clearerr(fp); } - if (fio_flush(fd) != 0) + if (fclose(fp) && save_errno == 0) { - int save_errno = errno; + failed_action = GELF_FAILED_CLOSE; + save_errno = errno; + } - fio_close(fd); - fio_unlink(lock_file, FIO_BACKUP_HOST); + if (save_errno) + { + if (remove(lock_file) != 0) + elog(WARNING, "Cannot remove lock file \"%s\": %s", lock_file, strerror(errno)); /* In lax mode if we failed to grab lock because of 'out of space error', * then treat backup as locked. @@ -502,20 +512,10 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) */ if (!strict && save_errno == ENOSPC) return LOCK_FAIL_ENOSPC; - else - elog(ERROR, "Could not flush lock file \"%s\": %s", - lock_file, strerror(save_errno)); - } - - if (fio_close(fd) != 0) - { - int save_errno = errno; - - fio_unlink(lock_file, FIO_BACKUP_HOST); - - if (!strict && errno == ENOSPC) - return LOCK_FAIL_ENOSPC; - else + else if (failed_action == GELF_FAILED_WRITE) + elog(ERROR, "Could not write lock file \"%s\": %s", + lock_file, strerror(save_errno)); + else if (failed_action == GELF_FAILED_CLOSE) elog(ERROR, "Could not close lock file \"%s\": %s", lock_file, strerror(save_errno)); } @@ -572,8 +572,8 @@ wait_shared_owners(pgBackup *backup) /* complain from time to time */ if ((ntries % LOG_FREQ) == 0) { - elog(WARNING, "Process %d is using backup %s in shared mode, and is still running", - encoded_pid, backup_id_of(backup)); + elog(WARNING, "Process %lld is using backup %s in shared mode, and is still running", + (long long)encoded_pid, backup_id_of(backup)); elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, backup_id_of(backup)); @@ -585,8 +585,8 @@ wait_shared_owners(pgBackup *backup) continue; } else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); + elog(ERROR, "Failed to send signal 0 to a process %lld: %s", + (long long)encoded_pid, strerror(errno)); /* locker is dead */ break; @@ -603,36 +603,32 @@ wait_shared_owners(pgBackup *backup) /* some shared owners are still alive */ if (ntries <= 0) { - elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %u owns shared lock", - backup_id_of(backup), encoded_pid); + elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %llu owns shared lock", + backup_id_of(backup), (long long)encoded_pid); return 1; } - /* unlink shared lock file */ - fio_unlink(lock_file, FIO_BACKUP_HOST); + /* remove shared lock file */ + if (fio_remove(FIO_BACKUP_HOST, lock_file, true) != 0) + elog(ERROR, "Cannot remove shared lock file \"%s\": %s", lock_file, strerror(errno)); return 0; } +#define FT_SLICE pid +#define FT_SLICE_TYPE pid_t +#include + /* - * Lock backup in shared mode - * 0 - successs - * 1 - fail + * returns array of pids stored in shared lock file and still alive. + * It excludes our own pid, so no need to exclude it explicitely. */ -int -grab_shared_lock_file(pgBackup *backup) +static ft_arr_pid_t +read_shared_lock_file(const char *lock_file) { FILE *fp_in = NULL; - FILE *fp_out = NULL; char buf_in[256]; pid_t encoded_pid; - char lock_file[MAXPGPATH]; - - char buffer[8192]; /*TODO: should be enough, but maybe malloc+realloc is better ? */ - char lock_file_tmp[MAXPGPATH]; - int buffer_len = 0; - - join_path_components(lock_file, backup->root_dir, BACKUP_RO_LOCK_FILE); - snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); + ft_arr_pid_t pids = ft_arr_init(); /* open already existing lock files */ fp_in = fopen(lock_file, "r"); @@ -642,7 +638,7 @@ grab_shared_lock_file(pgBackup *backup) /* read PIDs of owners */ while (fp_in && fgets(buf_in, sizeof(buf_in), fp_in)) { - encoded_pid = atoi(buf_in); + encoded_pid = (pid_t)atoll(buf_in); if (encoded_pid <= 0) { elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buf_in); @@ -658,12 +654,12 @@ grab_shared_lock_file(pgBackup *backup) * Somebody is still using this backup in shared mode, * copy this pid into a new file. */ - buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); + ft_arr_pid_push(&pids, encoded_pid); } else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); - } + elog(ERROR, "Failed to send signal 0 to a process %lld: %s", + (long long)encoded_pid, strerror(errno)); + } if (fp_in) { @@ -672,31 +668,77 @@ grab_shared_lock_file(pgBackup *backup) fclose(fp_in); } + return pids; +} + +static void +write_shared_lock_file(const char *lock_file, ft_arr_pid_t pids) +{ + FOBJ_FUNC_ARP(); + pioDrive_i drive = pioDriveForLocation(FIO_BACKUP_HOST); + FILE *fp_out = NULL; + char lock_file_tmp[MAXPGPATH]; + ssize_t i; + + if ($i(pioIsRemote, drive)) + { + elog(INFO, "Skipping write lock on remote drive"); + return; + } + + snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); + fp_out = fopen(lock_file_tmp, "w"); if (fp_out == NULL) { if (errno == EROFS) - return 0; + return; elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); } - /* add my own pid */ - buffer_len += snprintf(buffer+buffer_len, sizeof(buffer), "%u\n", my_pid); - /* write out the collected PIDs to temp lock file */ - fwrite(buffer, 1, buffer_len, fp_out); + for (i = 0; i < pids.len; i++) + fprintf(fp_out, "%lld\n", (long long)ft_arr_pid_at(&pids, i)); + fflush(fp_out); if (ferror(fp_out)) + { + fclose(fp_out); + remove(lock_file_tmp); elog(ERROR, "Cannot write to lock file: \"%s\"", lock_file_tmp); + } if (fclose(fp_out) != 0) + { + remove(lock_file_tmp); elog(ERROR, "Cannot close temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + } if (rename(lock_file_tmp, lock_file) < 0) elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - lock_file_tmp, lock_file, strerror(errno)); + lock_file_tmp, lock_file, strerror(errno)); +} + +/* + * Lock backup in shared mode + * 0 - successs + * 1 - fail + */ +int +grab_shared_lock_file(const char *backup_dir) +{ + char lock_file[MAXPGPATH]; + ft_arr_pid_t pids; + + join_path_components(lock_file, backup_dir, BACKUP_RO_LOCK_FILE); + pids = read_shared_lock_file(lock_file); + /* add my own pid */ + ft_arr_pid_push(&pids, my_pid); + + write_shared_lock_file(lock_file, pids); + ft_arr_pid_free(&pids); return 0; } @@ -727,93 +769,38 @@ release_excl_lock_file(const char *backup_dir) /* TODO Sanity check: maybe we should check, that pid in lock file is my_pid */ - /* unlink pid file */ - fio_unlink(lock_file, FIO_BACKUP_HOST); + /* remove pid file */ + /* exclusive locks releasing multiple times -> missing_ok = true */ + if (fio_remove(FIO_BACKUP_HOST, lock_file, true) != 0) + elog(ERROR, "Cannot remove exclusive lock file \"%s\": %s", lock_file, strerror(errno)); } void release_shared_lock_file(const char *backup_dir) { - FILE *fp_in = NULL; - FILE *fp_out = NULL; - char buf_in[256]; - pid_t encoded_pid; char lock_file[MAXPGPATH]; - - char buffer[8192]; /*TODO: should be enough, but maybe malloc+realloc is better ? */ - char lock_file_tmp[MAXPGPATH]; - int buffer_len = 0; + ft_arr_pid_t pids; join_path_components(lock_file, backup_dir, BACKUP_RO_LOCK_FILE); - snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); - /* open lock file */ - fp_in = fopen(lock_file, "r"); - if (fp_in == NULL) - { - if (errno == ENOENT) - return; - else - elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); - } - - /* read PIDs of owners */ - while (fgets(buf_in, sizeof(buf_in), fp_in)) + pids = read_shared_lock_file(lock_file); + /* read_shared_lock_file already had deleted my own pid */ + if (pids.len == 0) { - encoded_pid = atoi(buf_in); - - if (encoded_pid <= 0) - { - elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buf_in); - continue; - } - - /* remove my pid */ - if (encoded_pid == my_pid) - continue; - - if (kill(encoded_pid, 0) == 0) - { - /* - * Somebody is still using this backup in shared mode, - * copy this pid into a new file. - */ - buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); - } - else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); - } - - if (ferror(fp_in)) - elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); - fclose(fp_in); - - /* if there is no active pid left, then there is nothing to do */ - if (buffer_len == 0) - { - fio_unlink(lock_file, FIO_BACKUP_HOST); + ft_arr_pid_free(&pids); + /* + * TODO: we should not call 'release_shared_lock_file' if we don't hold it. + * Therefore we should not ignore ENOENT. + * We should find why ENOENT happens, but until then lets ignore it as + * it were ignored for a while. + */ + if (remove(lock_file) != 0 && errno != ENOENT) + elog(ERROR, "Cannot remove shared lock file \"%s\": %s", lock_file, strerror(errno)); return; } - fp_out = fopen(lock_file_tmp, "w"); - if (fp_out == NULL) - elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); - - /* write out the collected PIDs to temp lock file */ - fwrite(buffer, 1, buffer_len, fp_out); - - if (ferror(fp_out)) - elog(ERROR, "Cannot write to lock file: \"%s\"", lock_file_tmp); - - if (fclose(fp_out) != 0) - elog(ERROR, "Cannot close temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); - - if (rename(lock_file_tmp, lock_file) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - lock_file_tmp, lock_file, strerror(errno)); - - return; + write_shared_lock_file(lock_file, pids); + ft_arr_pid_free(&pids); } /* @@ -838,15 +825,19 @@ pgBackupGetBackupMode(pgBackup *backup, bool show_color) return backupModes[backup->backup_mode]; } -static bool -IsDir(const char *dirpath, const char *entry, fio_location location) +/* Build `CatalogState' from `backup_path' */ +CatalogState * +catalog_new(const char *backup_path) { - char path[MAXPGPATH]; - struct stat st; - - join_path_components(path, dirpath, entry); - - return fio_stat(path, &st, false, location) == 0 && S_ISDIR(st.st_mode); + CatalogState *catalogState = pgut_new0(CatalogState); + strncpy(catalogState->catalog_path, backup_path, MAXPGPATH); + join_path_components(catalogState->backup_subdir_path, + catalogState->catalog_path, BACKUPS_DIR); + join_path_components(catalogState->wal_subdir_path, + catalogState->catalog_path, WAL_SUBDIR); + catalogState->backup_location = pioDriveForLocation(FIO_BACKUP_HOST); + + return catalogState; } /* @@ -857,64 +848,42 @@ IsDir(const char *dirpath, const char *entry, fio_location location) parray * catalog_get_instance_list(CatalogState *catalogState) { - DIR *dir; - struct dirent *dent; + FOBJ_FUNC_ARP(); + pioDirIter_i data_dir; + pio_dirent_t ent; + err_i err = $noerr(); parray *instances; instances = parray_new(); /* open directory and list contents */ - dir = opendir(catalogState->backup_subdir_path); - if (dir == NULL) - elog(ERROR, "Cannot open directory \"%s\": %s", - catalogState->backup_subdir_path, strerror(errno)); + data_dir = $i(pioOpenDir, catalogState->backup_location, + catalogState->backup_subdir_path, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Failed to get backup list"); - while (errno = 0, (dent = readdir(dir)) != NULL) + while ((ent = $i(pioDirNext, data_dir, .err=&err)).stat.pst_kind) { - char child[MAXPGPATH]; - struct stat st; InstanceState *instanceState = NULL; - /* skip entries point current dir or parent dir */ - if (strcmp(dent->d_name, ".") == 0 || - strcmp(dent->d_name, "..") == 0) + if (ent.stat.pst_kind != PIO_KIND_DIRECTORY) continue; - join_path_components(child, catalogState->backup_subdir_path, dent->d_name); - - if (lstat(child, &st) == -1) - elog(ERROR, "Cannot stat file \"%s\": %s", - child, strerror(errno)); - - if (!S_ISDIR(st.st_mode)) - continue; - - instanceState = pgut_new(InstanceState); - - strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); - join_path_components(instanceState->instance_backup_subdir_path, - catalogState->backup_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_wal_subdir_path, - catalogState->wal_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_config_path, - instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); + instanceState = makeInstanceState(catalogState, ent.name.ptr); instanceState->config = readInstanceConfigFile(instanceState); parray_append(instances, instanceState); } + $i(pioClose, data_dir); // ignore error + + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Read backup root directory"); + /* TODO 3.0: switch to ERROR */ if (parray_num(instances) == 0) elog(WARNING, "This backup catalog contains no backup instances. Backup instance can be added via 'add-instance' command."); - if (errno) - elog(ERROR, "Cannot read directory \"%s\": %s", - catalogState->backup_subdir_path, strerror(errno)); - - if (closedir(dir)) - elog(ERROR, "Cannot close directory \"%s\": %s", - catalogState->backup_subdir_path, strerror(errno)); - return instances; } @@ -927,50 +896,58 @@ catalog_get_instance_list(CatalogState *catalogState) parray * catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id) { - DIR *data_dir = NULL; - struct dirent *data_ent = NULL; + FOBJ_FUNC_ARP(); + pioDirIter_i data_dir; + pio_dirent_t data_ent; + err_i err = $noerr(); parray *backups = NULL; int i; /* open backup instance backups directory */ - data_dir = fio_opendir(instanceState->instance_backup_subdir_path, FIO_BACKUP_HOST); - if (data_dir == NULL) + data_dir = $i(pioOpenDir, instanceState->backup_location, + instanceState->instance_backup_subdir_path, .err = &err); + if ($haserr(err) && getErrno(err) != ENOENT) { - elog(WARNING, "cannot open directory \"%s\": %s", instanceState->instance_backup_subdir_path, - strerror(errno)); - goto err_proc; + ft_logerr(FT_FATAL, $errmsg(err), "Failed to get backup list"); } /* scan the directory and list backups */ backups = parray_new(); - for (; (data_ent = fio_readdir(data_dir)) != NULL; errno = 0) + if ($isNULL(data_dir)) + { + elog(WARNING, "Cannot find any backups in \"%s\"", + instanceState->instance_backup_subdir_path); + return backups; + } + + while ((data_ent = $i(pioDirNext, data_dir, .err=&err)).stat.pst_kind) { char backup_conf_path[MAXPGPATH]; char data_path[MAXPGPATH]; pgBackup *backup = NULL; - /* skip not-directory entries and hidden entries */ - if (!IsDir(instanceState->instance_backup_subdir_path, data_ent->d_name, FIO_BACKUP_HOST) - || data_ent->d_name[0] == '.') + /* skip not-directory entries (hidden are skipped already) */ + if (data_ent.stat.pst_kind != PIO_KIND_DIRECTORY) continue; /* open subdirectory of specific backup */ - join_path_components(data_path, instanceState->instance_backup_subdir_path, data_ent->d_name); + join_path_components(data_path, instanceState->instance_backup_subdir_path, + data_ent.name.ptr); /* read backup information from BACKUP_CONTROL_FILE */ join_path_components(backup_conf_path, data_path, BACKUP_CONTROL_FILE); - backup = readBackupControlFile(backup_conf_path); + backup = readBackupControlFile(instanceState->backup_location, backup_conf_path); if (!backup) { backup = pgut_new0(pgBackup); - pgBackupInit(backup); - backup->start_time = base36dec(data_ent->d_name); + pgBackupInit(backup, instanceState->backup_location); + backup->start_time = base36dec(data_ent.name.ptr); /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ Assert(backup->backup_id == 0 || backup->backup_id == backup->start_time); backup->backup_id = backup->start_time; } - else if (strcmp(backup_id_of(backup), data_ent->d_name) != 0) + else if (strcmp(backup_id_of(backup), data_ent.name.ptr) != 0) { /* TODO there is no such guarantees */ elog(WARNING, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", @@ -995,16 +972,14 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id parray_append(backups, backup); } - if (errno) + $i(pioClose, data_dir); // ignore error + + if ($haserr(err)) { - elog(WARNING, "Cannot read backup root directory \"%s\": %s", - instanceState->instance_backup_subdir_path, strerror(errno)); + ft_logerr(FT_WARNING, $errmsg(err), "Read backup root directory"); goto err_proc; } - fio_closedir(data_dir); - data_dir = NULL; - parray_qsort(backups, pgBackupCompareIdDesc); /* Link incremental backups with their ancestors.*/ @@ -1027,8 +1002,6 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id return backups; err_proc: - if (data_dir) - fio_closedir(data_dir); if (backups) parray_walk(backups, pgBackupFree); parray_free(backups); @@ -1044,32 +1017,38 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id parray * get_backup_filelist(pgBackup *backup, bool strict) { + FOBJ_FUNC_ARP(); parray *files = NULL; char backup_filelist_path[MAXPGPATH]; - FILE *fp; - char buf[BLCKSZ]; - char stdio_buf[STDIO_BUFSIZE]; pg_crc32 content_crc = 0; + pb_control_line ft_cleanup(deinit_pb_control_line) + pb_line = {0}; + pio_line_reader ft_cleanup(deinit_pio_line_reader) + line_reader = {0}; + ft_bytes_t line; + err_i err = $noerr(); + pioReadStream_i fl; join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST); - fp = fio_open_stream(backup_filelist_path, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "cannot open \"%s\": %s", backup_filelist_path, strerror(errno)); + fl = $i(pioOpenReadStream, backup->backup_location, .path = backup_filelist_path, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Opening backup filelist"); - /* enable stdio buffering for local file */ - if (!fio_is_remote(FIO_BACKUP_HOST)) - setvbuf(fp, stdio_buf, _IOFBF, STDIO_BUFSIZE); + init_pio_line_reader(&line_reader, $reduce(pioRead, fl), IN_BUF_SIZE); files = parray_new(); - INIT_FILE_CRC32(true, content_crc); + INIT_CRC32C(content_crc); + + init_pb_control_line(&pb_line); - while (fgets(buf, lengthof(buf), fp)) + for(;;) { - char path[MAXPGPATH]; - char linked[MAXPGPATH]; - char compress_alg_string[MAXPGPATH]; + ft_str_t path; + ft_str_t linked; + ft_str_t compress_alg; + ft_str_t kind; int64 write_size, uncompressed_size, mode, /* bit length of mode_t depends on platforms */ @@ -1086,63 +1065,81 @@ get_backup_filelist(pgBackup *backup, bool strict) hdr_size; pgFile *file; - COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); + line = pio_line_reader_getline(&line_reader, &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Reading backup filelist"); + if (line.len == 0) + break; + + COMP_CRC32C(content_crc, line.ptr, line.len); + + parse_pb_control_line(&pb_line, line); + + path = pb_control_line_get_str(&pb_line, "path"); + write_size = pb_control_line_get_int64(&pb_line, "size"); + mode = pb_control_line_get_int64(&pb_line, "mode"); + is_datafile = pb_control_line_get_int64(&pb_line, "is_datafile"); + crc = pb_control_line_get_int64(&pb_line, "crc"); + + pb_control_line_try_int64(&pb_line, "is_cfs", &is_cfs); + pb_control_line_try_int64(&pb_line, "dbOid", &dbOid); + pb_control_line_try_str(&pb_line, "compress_alg", &compress_alg); + pb_control_line_try_int64(&pb_line, "external_dir_num", &external_dir_num); - get_control_value_str(buf, "path", path, sizeof(path),true); - get_control_value_int64(buf, "size", &write_size, true); - get_control_value_int64(buf, "mode", &mode, true); - get_control_value_int64(buf, "is_datafile", &is_datafile, true); - get_control_value_int64(buf, "is_cfs", &is_cfs, false); - get_control_value_int64(buf, "crc", &crc, true); - get_control_value_str(buf, "compress_alg", compress_alg_string, sizeof(compress_alg_string), false); - get_control_value_int64(buf, "external_dir_num", &external_dir_num, false); - get_control_value_int64(buf, "dbOid", &dbOid, false); + if (path.len > MAXPGPATH) + elog(ERROR, "File path in "DATABASE_FILE_LIST" is too long: '%.*s'", + (int)line.len, line.ptr); - file = pgFileInit(path); + file = pgFileInit(path.ptr); file->write_size = (int64) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->is_cfs = is_cfs ? true : false; file->crc = (pg_crc32) crc; - file->compress_alg = parse_compress_alg(compress_alg_string); - file->external_dir_num = external_dir_num; + file->compress_alg = parse_compress_alg(compress_alg.ptr); + file->external_dir_num = (int)external_dir_num; file->dbOid = dbOid ? dbOid : 0; /* * Optional fields */ - if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0]) + if (pb_control_line_try_str(&pb_line, "kind", &kind)) + file->kind = pio_str2file_kind(kind.ptr, path.ptr); + else /* fallback to mode for old backups */ + file->kind = pio_statmode2file_kind(file->mode, path.ptr); + + if (pb_control_line_try_str(&pb_line, "linked", &linked) && linked.len > 0) { - file->linked = pgut_strdup(linked); + file->linked = ft_strdup(linked).ptr; canonicalize_path(file->linked); } - if (get_control_value_int64(buf, "segno", &segno, false)) + if (pb_control_line_try_int64(&pb_line, "segno", &segno)) file->segno = (int) segno; - if (get_control_value_int64(buf, "n_blocks", &n_blocks, false)) + if (pb_control_line_try_int64(&pb_line, "n_blocks", &n_blocks)) file->n_blocks = (int) n_blocks; - if (get_control_value_int64(buf, "n_headers", &n_headers, false)) + if (pb_control_line_try_int64(&pb_line, "n_headers", &n_headers)) file->n_headers = (int) n_headers; - if (get_control_value_int64(buf, "hdr_crc", &hdr_crc, false)) + if (pb_control_line_try_int64(&pb_line, "hdr_crc", &hdr_crc)) file->hdr_crc = (pg_crc32) hdr_crc; - if (get_control_value_int64(buf, "hdr_off", &hdr_off, false)) + if (pb_control_line_try_int64(&pb_line, "hdr_off", &hdr_off)) file->hdr_off = hdr_off; - if (get_control_value_int64(buf, "hdr_size", &hdr_size, false)) + if (pb_control_line_try_int64(&pb_line, "hdr_size", &hdr_size)) file->hdr_size = (int) hdr_size; - if (get_control_value_int64(buf, "full_size", &uncompressed_size, false)) + if (pb_control_line_try_int64(&pb_line, "full_size", &uncompressed_size)) file->uncompressed_size = uncompressed_size; else file->uncompressed_size = write_size; if (!file->is_datafile || file->is_cfs) file->size = file->uncompressed_size; - if (file->external_dir_num == 0 && S_ISREG(file->mode)) + if (file->external_dir_num == 0 && file->kind == PIO_KIND_REGULAR) { bool is_datafile = file->is_datafile; set_forkname(file); @@ -1163,12 +1160,9 @@ get_backup_filelist(pgBackup *backup, bool strict) parray_append(files, file); } - FIN_FILE_CRC32(true, content_crc); - - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", backup_filelist_path); + FIN_CRC32C(content_crc); - fio_close_stream(fp); + err = $i(pioClose, fl); if (backup->content_crc != 0 && backup->content_crc != content_crc) @@ -1187,6 +1181,64 @@ get_backup_filelist(pgBackup *backup, bool strict) return files; } +static uint32_t +pgFileHashRelPathWithExternal(pgFile* file) +{ + uint32_t hash = ft_small_cstr_hash(file->rel_path); + hash = ft_mix32(hash ^ file->external_dir_num); + return hash ? hash : 1; +} + +parray * +make_filelist_hashtable(parray* files) +{ + parray* buckets; + size_t nbuckets; + pgFile* file; + size_t i; + size_t pos; + + if (parray_num(files) == 0) + return NULL; + + buckets = parray_new(); + nbuckets = ft_nextpow2(parray_num(files)) / 2; + nbuckets = ft_max(ft_min(nbuckets, UINT32_MAX/2+1), 1); + parray_set(buckets, nbuckets-1, NULL); /* ensure size will be == nbuckets */ + + for (i = 0; i < parray_num(files); i++) + { + file = (pgFile*)parray_get(files, i); + file->hash = pgFileHashRelPathWithExternal(file); + pos = file->hash & (nbuckets - 1); + file->next = parray_get(buckets, pos); + parray_set(buckets, pos, file); + } + + return buckets; +} + +pgFile* +search_file_in_hashtable(parray* buckets, pgFile* file) +{ + pgFile* ent; + size_t pos; + + if (!file->hash) + file->hash = pgFileHashRelPathWithExternal(file); + + pos = file->hash & (parray_num(buckets)-1); + ent = (pgFile*) parray_get(buckets, pos); + while (ent != NULL) + { + if (ent->hash == file->hash && + pgFileCompareRelPathWithExternal(&file, &ent) == 0) + return ent; + ent = ent->next; + } + return NULL; +} + /* * Lock list of backups. Function goes in backward direction. */ @@ -1444,16 +1496,17 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, * It may be ok or maybe not, so it's up to the caller * to fix it or let it be. */ - void pgBackupInitDir(pgBackup *backup, const char *backup_instance_path) { int i; char temp[MAXPGPATH]; parray *subdirs; + err_i err = $noerr(); /* Try to create backup directory at first */ - if (create_backup_dir(backup, backup_instance_path) != 0) + err = create_backup_dir(backup, backup_instance_path); + if ($haserr(err)) { /* Clear backup_id as indication of error */ reset_backup_id(backup); @@ -1489,7 +1542,12 @@ pgBackupInitDir(pgBackup *backup, const char *backup_instance_path) for (i = 0; i < parray_num(subdirs); i++) { join_path_components(temp, backup->root_dir, parray_get(subdirs, i)); - fio_mkdir(temp, DIR_PERMISSION, FIO_BACKUP_HOST); + err = $i(pioMakeDir, backup->backup_location, .path = temp, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create backup subdirectory: %s", $errmsg(err)); + } } free_dir_list(subdirs); @@ -1502,22 +1560,25 @@ pgBackupInitDir(pgBackup *backup, const char *backup_instance_path) * 0 - ok * -1 - error (warning message already emitted) */ -int +static err_i create_backup_dir(pgBackup *backup, const char *backup_instance_path) { - int rc; char path[MAXPGPATH]; + err_i err; join_path_components(path, backup_instance_path, backup_id_of(backup)); /* TODO: add wrapper for remote mode */ - rc = dir_create_dir(path, DIR_PERMISSION, true); - - if (rc == 0) + err = $i(pioMakeDir, backup->backup_location, .path = path, + .mode = DIR_PERMISSION, .strict = true); + if (!$haserr(err)) + { backup->root_dir = pgut_strdup(path); - else - elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno)); - return rc; + } else if (getErrno(err) != EEXIST) { + elog(ERROR, "Can not create backup directory: %s", $errmsg(err)); + } + + return err; } /* @@ -1527,50 +1588,72 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path) parray * catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) { + FOBJ_FUNC_ARP(); int i,j,k; - parray *xlog_files_list = parray_new(); parray *timelineinfos; parray *backups; timelineInfo *tlinfo; + ft_arr_dirent_t xlog_files_list = ft_arr_init(); + pioDirIter_i dir; + pio_dirent_t file; + err_i err; /* for fancy reporting */ char begin_segno_str[MAXFNAMELEN]; char end_segno_str[MAXFNAMELEN]; + + dir = $i(pioOpenDir, instanceState->backup_location, + .path = instanceState->instance_wal_subdir_path, + .err = &err); + /* read all xlog files that belong to this archive */ - dir_list_file(xlog_files_list, instanceState->instance_wal_subdir_path, - false, true, false, false, true, 0, FIO_BACKUP_HOST); - parray_qsort(xlog_files_list, pgFileCompareName); + if (!$isNULL(dir)) + { + while ((file = $i(pioDirNext, dir, .err = &err)).stat.pst_kind) + { + if (file.stat.pst_kind != PIO_KIND_REGULAR) + continue; + file.name = ft_strdup(file.name); + ft_arr_dirent_push(&xlog_files_list, file); + } + + ft_qsort_dirent(xlog_files_list.ptr, xlog_files_list.len, compare_dirent_by_name); + + $i(pioClose, dir); + } + if ($haserr(err) && getErrno(err) != ENOENT) + ft_logerr(FT_FATAL, $errmsg(err), "Reading wal dir"); timelineinfos = parray_new(); tlinfo = NULL; /* walk through files and collect info about timelines */ - for (i = 0; i < parray_num(xlog_files_list); i++) + for (i = 0; i < xlog_files_list.len; ft_str_free(&file.name), i++) { - pgFile *file = (pgFile *) parray_get(xlog_files_list, i); TimeLineID tli; parray *timelines; xlogFile *wal_file = NULL; + file = xlog_files_list.ptr[i]; /* * Regular WAL file. * IsXLogFileName() cannot be used here */ - if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) + if (strspn(file.name.ptr, "0123456789ABCDEF") == XLOG_FNAME_LEN) { int result = 0; uint32 log, seg; XLogSegNo segno = 0; char suffix[MAXFNAMELEN]; - result = sscanf(file->name, "%08X%08X%08X.%s", + result = sscanf(file.name.ptr, "%08X%08X%08X.%s", &tli, &log, &seg, (char *) &suffix); /* sanity */ if (result < 3) { - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + elog(WARNING, "unexpected WAL file name \"%s\"", file.name.ptr); continue; } @@ -1581,9 +1664,9 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) if (result == 4) { /* backup history file. Currently we don't use them */ - if (IsBackupHistoryFileName(file->name)) + if (IsBackupHistoryFileName(file.name.ptr)) { - elog(VERBOSE, "backup history file \"%s\"", file->name); + elog(VERBOSE, "backup history file \"%s\"", file.name.ptr); if (!tlinfo || tlinfo->tli != tli) { @@ -1592,8 +1675,9 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) } /* append file to xlog file list */ - wal_file = palloc(sizeof(xlogFile)); - wal_file->file = *file; + wal_file = ft_calloc(sizeof(xlogFile)); + wal_file->name = ft_str_steal(&file.name); + wal_file->size = file.stat.pst_size; wal_file->segno = segno; wal_file->type = BACKUP_HISTORY_FILE; wal_file->keep = false; @@ -1601,10 +1685,10 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) continue; } /* partial WAL segment */ - else if (IsPartialXLogFileName(file->name) || - IsPartialCompressXLogFileName(file->name)) + else if (IsPartialXLogFileName(file.name.ptr) || + IsPartialCompressXLogFileName(file.name)) { - elog(VERBOSE, "partial WAL file \"%s\"", file->name); + elog(VERBOSE, "partial WAL file \"%s\"", file.name.ptr); if (!tlinfo || tlinfo->tli != tli) { @@ -1613,8 +1697,9 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) } /* append file to xlog file list */ - wal_file = palloc(sizeof(xlogFile)); - wal_file->file = *file; + wal_file = ft_calloc(sizeof(xlogFile)); + wal_file->name = ft_str_steal(&file.name); + wal_file->size = file.stat.pst_size; wal_file->segno = segno; wal_file->type = PARTIAL_SEGMENT; wal_file->keep = false; @@ -1622,10 +1707,10 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) continue; } /* temp WAL segment */ - else if (IsTempXLogFileName(file->name) || - IsTempCompressXLogFileName(file->name)) + else if (IsTempXLogFileName(file.name) || + IsTempCompressXLogFileName(file.name)) { - elog(VERBOSE, "temp WAL file \"%s\"", file->name); + elog(VERBOSE, "temp WAL file \"%s\"", file.name.ptr); if (!tlinfo || tlinfo->tli != tli) { @@ -1634,8 +1719,9 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) } /* append file to xlog file list */ - wal_file = palloc(sizeof(xlogFile)); - wal_file->file = *file; + wal_file = ft_calloc(sizeof(xlogFile)); + wal_file->name = ft_str_steal(&file.name); + wal_file->size = file.stat.pst_size; wal_file->segno = segno; wal_file->type = TEMP_SEGMENT; wal_file->keep = false; @@ -1645,7 +1731,7 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) /* we only expect compressed wal files with .gz suffix */ else if (strcmp(suffix, "gz") != 0) { - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + elog(WARNING, "unexpected WAL file name \"%s\"", file.name.ptr); continue; } } @@ -1693,22 +1779,23 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) tlinfo->end_segno = segno; /* update counters */ tlinfo->n_xlog_files++; - tlinfo->size += file->size; + tlinfo->size += file.stat.pst_size; /* append file to xlog file list */ - wal_file = palloc(sizeof(xlogFile)); - wal_file->file = *file; + wal_file = ft_calloc(sizeof(xlogFile)); + wal_file->name = ft_str_steal(&file.name); + wal_file->size = file.stat.pst_size; wal_file->segno = segno; wal_file->type = SEGMENT; wal_file->keep = false; parray_append(tlinfo->xlog_filelist, wal_file); } /* timeline history file */ - else if (IsTLHistoryFileName(file->name)) + else if (IsTLHistoryFileName(file.name.ptr)) { TimeLineHistoryEntry *tln; - sscanf(file->name, "%08X.history", &tli); + sscanf(file.name.ptr, "%08X.history", &tli); timelines = read_timeline_history(instanceState->instance_wal_subdir_path, tli, true); /* History file is empty or corrupted, disregard it */ @@ -1743,7 +1830,7 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) parray_free(timelines); } else - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + elog(WARNING, "unexpected WAL file name \"%s\"", file.name.ptr); } /* save information about backups belonging to each timeline */ @@ -1770,8 +1857,8 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) { timelineInfo *tlinfo = parray_get(timelineinfos, i); - tlinfo->oldest_backup = get_oldest_backup(tlinfo); - tlinfo->closest_backup = get_closest_backup(tlinfo); + tlinfo->oldest_backup = get_oldest_backup(tlinfo, instance->xlog_seg_size); + tlinfo->closest_backup = get_closest_backup(tlinfo, instance->xlog_seg_size); } /* determine which WAL segments must be kept because of wal retention */ @@ -2135,7 +2222,7 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) * timeline is unreachable. Return NULL. */ pgBackup* -get_closest_backup(timelineInfo *tlinfo) +get_closest_backup(timelineInfo *tlinfo, uint32 xlog_seg_size) { pgBackup *closest_backup = NULL; int i; @@ -2158,7 +2245,7 @@ get_closest_backup(timelineInfo *tlinfo) * should be considered. */ if (!XLogRecPtrIsInvalid(backup->stop_lsn) && - XRecOffIsValid(backup->stop_lsn) && + XRecEndLooksGood(backup->stop_lsn, xlog_seg_size) && backup->stop_lsn <= tlinfo->switchpoint && (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE)) @@ -2186,7 +2273,7 @@ get_closest_backup(timelineInfo *tlinfo) * there is no backups on this timeline. Return NULL. */ pgBackup* -get_oldest_backup(timelineInfo *tlinfo) +get_oldest_backup(timelineInfo *tlinfo, uint32_t xlog_seg_size) { pgBackup *oldest_backup = NULL; int i; @@ -2200,7 +2287,7 @@ get_oldest_backup(timelineInfo *tlinfo) /* Backups with invalid START LSN can be safely skipped */ if (XLogRecPtrIsInvalid(backup->start_lsn) || - !XRecOffIsValid(backup->start_lsn)) + !XRecPtrLooksGood(backup->start_lsn, xlog_seg_size)) continue; /* @@ -2335,104 +2422,106 @@ add_note(pgBackup *target_backup, char *note) } /* - * Write information about backup.in to stream "out". + * Write information about backup.in to ft_strbuf_t". */ -void -pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc) +ft_str_t +pgBackupWriteControl(pgBackup *backup, bool utc) { char timestamp[100]; + ft_strbuf_t buf = ft_strbuf_zero(); - fio_fprintf(out, "#Configuration\n"); - fio_fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup, false)); - fio_fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); - fio_fprintf(out, "compress-alg = %s\n", + ft_strbuf_catf(&buf, "#Configuration\n"); + ft_strbuf_catf(&buf, "backup-mode = %s\n", pgBackupGetBackupMode(backup, false)); + ft_strbuf_catf(&buf, "stream = %s\n", backup->stream ? "true" : "false"); + ft_strbuf_catf(&buf, "compress-alg = %s\n", deparse_compress_alg(backup->compress_alg)); - fio_fprintf(out, "compress-level = %d\n", backup->compress_level); - fio_fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); + ft_strbuf_catf(&buf, "compress-level = %d\n", backup->compress_level); + ft_strbuf_catf(&buf, "from-replica = %s\n", backup->from_replica ? "true" : "false"); - fio_fprintf(out, "\n#Compatibility\n"); - fio_fprintf(out, "block-size = %u\n", backup->block_size); - fio_fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); - fio_fprintf(out, "checksum-version = %u\n", backup->checksum_version); + ft_strbuf_catf(&buf, "\n#Compatibility\n"); + ft_strbuf_catf(&buf, "block-size = %u\n", backup->block_size); + ft_strbuf_catf(&buf, "xlog-block-size = %u\n", backup->wal_block_size); + ft_strbuf_catf(&buf, "checksum-version = %u\n", backup->checksum_version); if (backup->program_version[0] != '\0') - fio_fprintf(out, "program-version = %s\n", backup->program_version); + ft_strbuf_catf(&buf, "program-version = %s\n", backup->program_version); if (backup->server_version[0] != '\0') - fio_fprintf(out, "server-version = %s\n", backup->server_version); + ft_strbuf_catf(&buf, "server-version = %s\n", backup->server_version); - fio_fprintf(out, "\n#Result backup info\n"); - fio_fprintf(out, "timelineid = %d\n", backup->tli); + ft_strbuf_catf(&buf, "\n#Result backup info\n"); + ft_strbuf_catf(&buf, "timelineid = %d\n", backup->tli); /* LSN returned by pg_start_backup */ - fio_fprintf(out, "start-lsn = %X/%X\n", + ft_strbuf_catf(&buf, "start-lsn = %X/%X\n", (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); /* LSN returned by pg_stop_backup */ - fio_fprintf(out, "stop-lsn = %X/%X\n", + ft_strbuf_catf(&buf, "stop-lsn = %X/%X\n", (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); time2iso(timestamp, lengthof(timestamp), backup->start_time, utc); - fio_fprintf(out, "start-time = '%s'\n", timestamp); + ft_strbuf_catf(&buf, "start-time = '%s'\n", timestamp); if (backup->merge_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->merge_time, utc); - fio_fprintf(out, "merge-time = '%s'\n", timestamp); + ft_strbuf_catf(&buf, "merge-time = '%s'\n", timestamp); } if (backup->end_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->end_time, utc); - fio_fprintf(out, "end-time = '%s'\n", timestamp); + ft_strbuf_catf(&buf, "end-time = '%s'\n", timestamp); } - fio_fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); + ft_strbuf_catf(&buf, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->recovery_time, utc); - fio_fprintf(out, "recovery-time = '%s'\n", timestamp); + ft_strbuf_catf(&buf, "recovery-time = '%s'\n", timestamp); } if (backup->expire_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->expire_time, utc); - fio_fprintf(out, "expire-time = '%s'\n", timestamp); + ft_strbuf_catf(&buf, "expire-time = '%s'\n", timestamp); } if (backup->merge_dest_backup != 0) - fio_fprintf(out, "merge-dest-id = '%s'\n", base36enc(backup->merge_dest_backup)); + ft_strbuf_catf(&buf, "merge-dest-id = '%s'\n", base36enc(backup->merge_dest_backup)); /* * Size of PGDATA directory. The size does not include size of related * WAL segments in archive 'wal' directory. */ if (backup->data_bytes != BYTES_INVALID) - fio_fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + ft_strbuf_catf(&buf, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); if (backup->wal_bytes != BYTES_INVALID) - fio_fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + ft_strbuf_catf(&buf, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); if (backup->uncompressed_bytes >= 0) - fio_fprintf(out, "uncompressed-bytes = " INT64_FORMAT "\n", backup->uncompressed_bytes); + ft_strbuf_catf(&buf, "uncompressed-bytes = " INT64_FORMAT "\n", backup->uncompressed_bytes); if (backup->pgdata_bytes >= 0) - fio_fprintf(out, "pgdata-bytes = " INT64_FORMAT "\n", backup->pgdata_bytes); + ft_strbuf_catf(&buf, "pgdata-bytes = " INT64_FORMAT "\n", backup->pgdata_bytes); - fio_fprintf(out, "status = %s\n", status2str(backup->status)); + ft_strbuf_catf(&buf, "status = %s\n", status2str(backup->status)); /* 'parent_backup' is set if it is incremental backup */ if (backup->parent_backup != 0) - fio_fprintf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); + ft_strbuf_catf(&buf, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); /* print connection info except password */ if (backup->primary_conninfo) - fio_fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); + ft_strbuf_catf(&buf, "primary_conninfo = '%s'\n", backup->primary_conninfo); /* print external directories list */ if (backup->external_dir_str) - fio_fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str); + ft_strbuf_catf(&buf, "external-dirs = '%s'\n", backup->external_dir_str); if (backup->note) - fio_fprintf(out, "note = '%s'\n", backup->note); + ft_strbuf_catf(&buf, "note = '%s'\n", backup->note); if (backup->content_crc != 0) - fio_fprintf(out, "content-crc = %u\n", backup->content_crc); + ft_strbuf_catf(&buf, "content-crc = %u\n", backup->content_crc); + return ft_strbuf_steal(&buf); } /* @@ -2444,60 +2533,23 @@ pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc) void write_backup(pgBackup *backup, bool strict) { - FILE *fp = NULL; + ft_str_t buf; char path[MAXPGPATH]; - char path_temp[MAXPGPATH]; - char buf[8192]; + err_i err = $noerr(); join_path_components(path, backup->root_dir, BACKUP_CONTROL_FILE); - snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); - - fp = fopen(path_temp, PG_BINARY_W); - if (fp == NULL) - elog(ERROR, "Cannot open control file \"%s\": %s", - path_temp, strerror(errno)); - if (chmod(path_temp, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp, - strerror(errno)); - - setvbuf(fp, buf, _IOFBF, sizeof(buf)); + buf = pgBackupWriteControl(backup, true); + err = $i(pioWriteFile, backup->backup_location, .path = path, + .content = ft_bytes(buf.ptr, buf.len), .binary = false); - pgBackupWriteControl(fp, backup, true); - - /* Ignore 'out of space' error in lax mode */ - if (fflush(fp) != 0) - { - int elevel = ERROR; - int save_errno = errno; - - if (!strict && (errno == ENOSPC)) - elevel = WARNING; - - elog(elevel, "Cannot flush control file \"%s\": %s", - path_temp, strerror(save_errno)); - - if (!strict && (save_errno == ENOSPC)) - { - fclose(fp); - fio_unlink(path_temp, FIO_BACKUP_HOST); - return; - } - } + ft_str_free(&buf); - if (fclose(fp) != 0) - elog(ERROR, "Cannot close control file \"%s\": %s", - path_temp, strerror(errno)); - - if (fio_sync(path_temp, FIO_BACKUP_HOST) < 0) - elog(ERROR, "Cannot sync control file \"%s\": %s", - path_temp, strerror(errno)); - - if (rename(path_temp, path) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - path_temp, path, strerror(errno)); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Writting " BACKUP_CONTROL_FILE ".tmp"); } + /* * Output the list of files to backup catalog DATABASE_FILE_LIST */ @@ -2505,53 +2557,50 @@ void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync) { - FILE *out; + FOBJ_FUNC_ARP(); char control_path[MAXPGPATH]; - char control_path_temp[MAXPGPATH]; size_t i = 0; - #define BUFFERSZ (1024*1024) - char *buf; int64 backup_size_on_disk = 0; int64 uncompressed_size_on_disk = 0; int64 wal_size_on_disk = 0; - join_path_components(control_path, backup->root_dir, DATABASE_FILE_LIST); - snprintf(control_path_temp, sizeof(control_path_temp), "%s.tmp", control_path); + pioWriteCloser_i out; + pioCRC32Counter* crc; + pioWriteFlush_i wrapped; + pioDrive_i backup_drive = backup->backup_location; + err_i err; - out = fopen(control_path_temp, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open file list \"%s\": %s", control_path_temp, - strerror(errno)); + ft_strbuf_t line = ft_strbuf_zero(); - if (chmod(control_path_temp, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", control_path_temp, - strerror(errno)); + join_path_components(control_path, backup->root_dir, DATABASE_FILE_LIST); - buf = pgut_malloc(BUFFERSZ); - setvbuf(out, buf, _IOFBF, BUFFERSZ); + out = $i(pioOpenRewrite, backup_drive, control_path, .sync = sync, .err = &err); + if ($haserr(err)) + elog(ERROR, "Cannot open file list \"%s\": %s", control_path, + strerror(errno)); - if (sync) - INIT_FILE_CRC32(true, backup->content_crc); + crc = pioCRC32Counter_alloc(); + wrapped = pioWrapWriteFilter($reduce(pioWriteFlush, out), + $bind(pioFilter, crc), + OUT_BUF_SIZE); /* print each file in the list */ for (i = 0; i < parray_num(files); i++) { - int len = 0; - char line[BLCKSZ]; - pgFile *file = (pgFile *) parray_get(files, i); + pgFile *file = (pgFile *) parray_get(files, i); /* Ignore disappeared file */ if (file->write_size == FILE_NOT_FOUND) continue; - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) { backup_size_on_disk += 4096; uncompressed_size_on_disk += 4096; } /* Count the amount of the data actually copied */ - if (S_ISREG(file->mode) && file->write_size > 0) + if (file->kind == PIO_KIND_REGULAR && file->write_size > 0) { /* * Size of WAL files in 'pg_wal' is counted separately @@ -2566,12 +2615,14 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, } } - len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " - "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + ft_strbuf_catf(&line,"{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"kind\":\"%s\", \"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " "\"dbOid\":\"%u\"", - file->rel_path, file->write_size, file->mode, + file->rel_path, file->write_size, + pio_file_kind2str(file->kind, file->rel_path), + file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, @@ -2581,52 +2632,48 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, if (file->uncompressed_size != 0 && file->uncompressed_size != file->write_size) - len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"", + ft_strbuf_catf(&line, ",\"full_size\":\"" INT64_FORMAT "\"", file->uncompressed_size); if (file->is_datafile) - len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno); + ft_strbuf_catf(&line, ",\"segno\":\"%d\"", file->segno); if (file->linked) - len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked); + ft_strbuf_catf(&line, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks > 0) - len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); + ft_strbuf_catf(&line, ",\"n_blocks\":\"%i\"", file->n_blocks); if (file->n_headers > 0) { - len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers); - len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc); - len += sprintf(line+len, ",\"hdr_off\":\"%llu\"", file->hdr_off); - len += sprintf(line+len, ",\"hdr_size\":\"%i\"", file->hdr_size); + ft_strbuf_catf(&line, ",\"n_headers\":\"%i\"", file->n_headers); + ft_strbuf_catf(&line, ",\"hdr_crc\":\"%u\"", file->hdr_crc); + ft_strbuf_catf(&line, ",\"hdr_off\":\"%llu\"", file->hdr_off); + ft_strbuf_catf(&line, ",\"hdr_size\":\"%i\"", file->hdr_size); } - sprintf(line+len, "}\n"); + ft_strbuf_catf(&line, "}\n"); - if (sync) - COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line)); + err = $i(pioWrite, wrapped, ft_bytes(line.ptr, line.len)); - fprintf(out, "%s", line); - } + ft_strbuf_reset_for_reuse(&line); - if (sync) - FIN_FILE_CRC32(true, backup->content_crc); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Writing into " DATABASE_FILE_LIST ".tmp"); + } - if (fflush(out) != 0) - elog(ERROR, "Cannot flush file list \"%s\": %s", - control_path_temp, strerror(errno)); + ft_strbuf_free(&line); - if (sync && fsync(fileno(out)) < 0) - elog(ERROR, "Cannot sync file list \"%s\": %s", - control_path_temp, strerror(errno)); + err = $i(pioWriteFinish, wrapped); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Flushing " DATABASE_FILE_LIST ".tmp"); - if (fclose(out) != 0) - elog(ERROR, "Cannot close file list \"%s\": %s", - control_path_temp, strerror(errno)); + if (sync) + backup->content_crc = pioCRC32Counter_getCRC32(crc); - if (rename(control_path_temp, control_path) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - control_path_temp, control_path, strerror(errno)); + err = $i(pioClose, out); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Closing " DATABASE_FILE_LIST ".tmp"); /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; @@ -2634,8 +2681,6 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, if (backup->stream) backup->wal_bytes = wal_size_on_disk; - - free(buf); } /* @@ -2643,8 +2688,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, * - Comment starts with ';'. * - Do not care section. */ -static pgBackup * -readBackupControlFile(const char *path) +pgBackup * +readBackupControlFile(pioDrive_i drive, const char *path) { pgBackup *backup = pgut_new0(pgBackup); char *backup_mode = NULL; @@ -2657,6 +2702,7 @@ readBackupControlFile(const char *path) char *server_version = NULL; char *compress_alg = NULL; int parsed_options; + err_i err; ConfigOption options[] = { @@ -2693,15 +2739,18 @@ readBackupControlFile(const char *path) {0} }; - pgBackupInit(backup); - if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) + pgBackupInit(backup, drive); + + parsed_options = config_read_opt(drive, path, options, WARNING, true, &err); + + if (getErrno(err) == ENOENT) { elog(WARNING, "Control file \"%s\" doesn't exist", path); pgBackupFree(backup); return NULL; } - - parsed_options = config_read_opt(path, options, WARNING, true, true); + else if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Control file"); if (parsed_options == 0) { @@ -2921,7 +2970,7 @@ pgNodeInit(PGNodeInfo *node) * Fill pgBackup struct with default values. */ void -pgBackupInit(pgBackup *backup) +pgBackupInit(pgBackup *backup, pioDrive_i drive) { backup->backup_id = INVALID_BACKUP_ID; backup->backup_mode = BACKUP_MODE_INVALID; @@ -2962,6 +3011,8 @@ pgBackupInit(pgBackup *backup) backup->files = NULL; backup->note = NULL; backup->content_crc = 0; + + backup->backup_location = drive; } /* free pgBackup object */ @@ -2970,6 +3021,9 @@ pgBackupFree(void *backup) { pgBackup *b = (pgBackup *) backup; + /* Both point to global static vars */ + b->backup_location.self = NULL; + pg_free(b->primary_conninfo); pg_free(b->external_dir_str); pg_free(b->root_dir); @@ -3168,3 +3222,24 @@ append_children(parray *backup_list, pgBackup *target_backup, parray *append_lis } } } + +InstanceState* makeInstanceState(CatalogState* catalogState, const char* name) +{ + InstanceState* instanceState; + + instanceState = pgut_new0(InstanceState); + + instanceState->catalog_state = catalogState; + + strncpy(instanceState->instance_name, name, MAXPGPATH); + join_path_components(instanceState->instance_backup_subdir_path, + catalogState->backup_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_wal_subdir_path, + catalogState->wal_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_config_path, + instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); + + instanceState->backup_location = catalogState->backup_location; + + return instanceState; +} diff --git a/src/catchup.c b/src/catchup.c index 79e3361a8..9f690dbb2 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -17,7 +17,6 @@ #include "pgtar.h" #include "streamutil.h" -#include #include #include @@ -27,8 +26,9 @@ /* * Catchup routines */ -static PGconn *catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata); -static void catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, const char *source_pgdata, +static PGconn *catchup_init_state(pioDrive_i src_drive, PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata); +static void catchup_preflight_checks(pioDrive_i source_drive, pioDrive_i dest_drive, + PGNodeInfo *source_node_info, PGconn *source_conn, const char *source_pgdata, const char *dest_pgdata); static void catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn); static parray* catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli); @@ -39,7 +39,7 @@ static parray* catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID t * Prepare for work: fill some globals, open connection to source database */ static PGconn * -catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata) +catchup_init_state(pioDrive_i src_drive, PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata) { PGconn *source_conn; @@ -47,8 +47,8 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons pgNodeInit(source_node_info); /* Get WAL segments size and system ID of source PG instance */ - instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata); - instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false); + instance_config.xlog_seg_size = get_xlog_seg_size(src_drive, source_pgdata); + instance_config.system_identifier = get_system_identifier(src_drive, source_pgdata, false); current.start_time = time(NULL); strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); @@ -56,22 +56,15 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons /* Do some compatibility checks and fill basic info about PG instance */ source_conn = pgdata_basic_setup(instance_config.conn_opt, source_node_info); -#if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(source_conn)) elog(ERROR, "Failed to retrieve wal_segment_size"); -#endif get_ptrack_version(source_conn, source_node_info); if (source_node_info->ptrack_version_num > 0) source_node_info->is_ptrack_enabled = pg_is_ptrack_enabled(source_conn, source_node_info->ptrack_version_num); /* Obtain current timeline */ -#if PG_VERSION_NUM >= 90600 current.tli = get_current_timeline(source_conn); -#else - instance_config.pgdata = source_pgdata; - current.tli = get_current_timeline_from_control(source_pgdata, FIO_DB_HOST, false); -#endif elog(INFO, "Catchup start, pg_probackup version: %s, " "PostgreSQL version: %s, " @@ -91,7 +84,8 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons * this function is for checks, that can be performed without modification of data on disk */ static void -catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, +catchup_preflight_checks(pioDrive_i source_drive, pioDrive_i dest_drive, + PGNodeInfo *source_node_info, PGconn *source_conn, const char *source_pgdata, const char *dest_pgdata) { /* TODO @@ -133,7 +127,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, if (current.backup_mode != BACKUP_MODE_FULL) { pid_t pid; - pid = fio_check_postmaster(dest_pgdata, FIO_LOCAL_HOST); + pid = fio_check_postmaster(FIO_LOCAL_HOST, dest_pgdata); if (pid == 1) /* postmaster.pid is mangled */ { char pid_filename[MAXPGPATH]; @@ -143,8 +137,8 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, } else if (pid > 1) /* postmaster is up */ { - elog(ERROR, "Postmaster with pid %u is running in destination directory \"%s\"", - pid, dest_pgdata); + elog(ERROR, "Postmaster with pid %lld is running in destination directory \"%s\"", + (long long)pid, dest_pgdata); } } @@ -152,9 +146,10 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, if (current.backup_mode != BACKUP_MODE_FULL) { char backup_label_filename[MAXPGPATH]; + err_i err = $noerr(); join_path_components(backup_label_filename, dest_pgdata, PG_BACKUP_LABEL_FILE); - if (fio_access(backup_label_filename, F_OK, FIO_LOCAL_HOST) == 0) + if($i(pioExists, current.backup_location, .path = backup_label_filename, .err = &err)) elog(ERROR, "Destination directory contains \"" PG_BACKUP_LABEL_FILE "\" file"); } @@ -163,18 +158,18 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, uint64 source_conn_id, source_id, dest_id; source_conn_id = get_remote_system_identifier(source_conn); - source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */ + source_id = get_system_identifier(source_drive, source_pgdata, false); /* same as instance_config.system_identifier */ if (source_conn_id != source_id) - elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", - source_conn_id, source_pgdata, source_id); + elog(ERROR, "Database identifiers mismatch: we connected to DB id %llu, but in \"%s\" we found id %llu", + (long long)source_conn_id, source_pgdata, (long long)source_id); if (current.backup_mode != BACKUP_MODE_FULL) { - dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false); + dest_id = get_system_identifier(dest_drive, dest_pgdata, false); if (source_conn_id != dest_id) - elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", - source_conn_id, dest_pgdata, dest_id); + elog(ERROR, "Database identifiers mismatch: we connected to DB id %llu, but in \"%s\" we found id %llu", + (long long)source_conn_id, dest_pgdata, (long long)dest_id); } } @@ -190,9 +185,6 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, elog(ERROR, "Ptrack is disabled"); } - if (current.from_replica && exclusive_backup) - elog(ERROR, "Catchup from standby is only available for PostgreSQL >= 9.6"); - /* check that we don't overwrite tablespace in source pgdata */ catchup_check_tablespaces_existance_in_tbsmapping(source_conn); @@ -202,7 +194,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; /* fill dest_redo.lsn and dest_redo.tli */ - get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); + get_redo(dest_drive, dest_pgdata, &dest_redo); elog(LOG, "source.tli = %X, dest_redo.lsn = %X/%X, dest_redo.tli = %X", current.tli, (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn, dest_redo.tli); @@ -238,6 +230,7 @@ catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn) { PGresult *res; int i; + int ntups; char *tablespace_path = NULL; const char *linked_path = NULL; char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " @@ -249,7 +242,8 @@ catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn) if (!res) elog(ERROR, "Failed to get list of tablespaces"); - for (i = 0; i < res->ntups; i++) + ntups = PQntuples(res); + for (i = 0; i < ntups; i++) { tablespace_path = PQgetvalue(res, i, 0); Assert (strlen(tablespace_path) > 0); @@ -359,7 +353,7 @@ typedef struct const char *from_root; const char *to_root; parray *source_filelist; - parray *dest_filelist; + parray *dest_filehash; XLogRecPtr sync_lsn; BackupMode backup_mode; int thread_num; @@ -371,6 +365,8 @@ typedef struct static void * catchup_thread_runner(void *arg) { + pioDrive_i drive_from = pioDriveForLocation(FIO_DB_HOST); + pioDrive_i drive_to = pioDriveForLocation(FIO_BACKUP_HOST); int i; char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; @@ -385,7 +381,7 @@ catchup_thread_runner(void *arg) pgFile *dest_file = NULL; /* We have already copied all directories */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) continue; if (file->excluded) @@ -407,21 +403,20 @@ catchup_thread_runner(void *arg) join_path_components(to_fullpath, arguments->to_root, file->rel_path); /* Encountered some strange beast */ - if (!S_ISREG(file->mode)) - elog(WARNING, "Unexpected type %d of file \"%s\", skipping", - file->mode, from_fullpath); + if (file->kind != PIO_KIND_REGULAR) + elog(WARNING, "Unexpected kind %s of file \"%s\", skipping", + pio_file_kind2str(file->kind, from_fullpath), from_fullpath); /* Check that file exist in dest pgdata */ if (arguments->backup_mode != BACKUP_MODE_FULL) { - pgFile **dest_file_tmp = NULL; - dest_file_tmp = (pgFile **) parray_bsearch(arguments->dest_filelist, - file, pgFileCompareRelPathWithExternal); + pgFile *dest_file_tmp = NULL; + dest_file_tmp = search_file_in_hashtable(arguments->dest_filehash, file); if (dest_file_tmp) { /* File exists in destination PGDATA */ file->exists_in_prev = true; - dest_file = *dest_file_tmp; + dest_file = dest_file_tmp; } } @@ -436,8 +431,10 @@ catchup_thread_runner(void *arg) } else { - backup_non_data_file(file, dest_file, from_fullpath, to_fullpath, - arguments->backup_mode, current.parent_backup, true); + backup_non_data_file(drive_from, drive_to, + file, dest_file, from_fullpath, to_fullpath, + arguments->backup_mode, current.parent_backup, + true, false); } /* file went missing during catchup */ @@ -446,7 +443,7 @@ catchup_thread_runner(void *arg) if (file->write_size == BYTES_INVALID) { - elog(LOG, "Skipping the unchanged file: \"%s\", read %li bytes", from_fullpath, file->read_size); + elog(LOG, "Skipping the unchanged file: \"%s\", read %lld bytes", from_fullpath, (long long)file->read_size); continue; } @@ -475,7 +472,7 @@ catchup_multithreaded_copy(int num_threads, const char *source_pgdata_path, const char *dest_pgdata_path, parray *source_filelist, - parray *dest_filelist, + parray *dest_filehash, XLogRecPtr sync_lsn, BackupMode backup_mode) { @@ -495,7 +492,7 @@ catchup_multithreaded_copy(int num_threads, .from_root = source_pgdata_path, .to_root = dest_pgdata_path, .source_filelist = source_filelist, - .dest_filelist = dest_filelist, + .dest_filehash = dest_filehash, .sync_lsn = sync_lsn, .backup_mode = backup_mode, .thread_num = i + 1, @@ -533,41 +530,19 @@ catchup_multithreaded_copy(int num_threads, * Sync every file in destination directory to disk */ static void -catchup_sync_destination_files(const char* pgdata_path, fio_location location, parray *filelist, pgFile *pg_control_file) +catchup_sync_destination_files(const char* pgdata_path, fio_location location) { - char fullpath[MAXPGPATH]; time_t start_time, end_time; char pretty_time[20]; - int i; + pioDBDrive_i drive = pioDBDriveForLocation(location); + err_i err; elog(INFO, "Syncing copied files to disk"); time(&start_time); - for (i = 0; i < parray_num(filelist); i++) - { - pgFile *file = (pgFile *) parray_get(filelist, i); - - /* TODO: sync directory ? - * - at first glance we can rely on fs journaling, - * which is enabled by default on most platforms - * - but PG itself is not relying on fs, its durable_sync - * includes directory sync - */ - if (S_ISDIR(file->mode) || file->excluded) - continue; - - Assert(file->external_dir_num == 0); - join_path_components(fullpath, pgdata_path, file->rel_path); - if (fio_sync(fullpath, location) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); - } - - /* - * sync pg_control file - */ - join_path_components(fullpath, pgdata_path, pg_control_file->rel_path); - if (fio_sync(fullpath, location) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); + err = $i(pioSyncTree, drive, pgdata_path); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Syncing files"); time(&end_time); pretty_time_interval(difftime(end_time, start_time), @@ -620,28 +595,36 @@ int do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files, parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list) { - PGconn *source_conn = NULL; - PGNodeInfo source_node_info; - bool backup_logs = false; - parray *source_filelist = NULL; - pgFile *source_pg_control_file = NULL; - parray *dest_filelist = NULL; - char dest_xlog_path[MAXPGPATH]; - - RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; - PGStopBackupResult stop_backup_result; - bool catchup_isok = true; + pioDrive_i local_location = pioDriveForLocation(FIO_LOCAL_HOST); + pioDrive_i db_location = pioDriveForLocation(FIO_DB_HOST); + PGconn *source_conn = NULL; + PGNodeInfo source_node_info; + parray *source_filelist = NULL; + parray *source_filehash = NULL; + pgFile *source_pg_control_file = NULL; + parray *dest_filelist = NULL; + parray *dest_filehash = NULL; + char dest_xlog_path[MAXPGPATH]; + + RedoParams dest_redo = {0, InvalidXLogRecPtr, 0}; + PGStopBackupResult stop_backup_result; + bool catchup_isok = true; - int i; + int i; /* for fancy reporting */ - time_t start_time, end_time; - ssize_t transfered_datafiles_bytes = 0; - ssize_t transfered_walfiles_bytes = 0; - char pretty_source_bytes[20]; + time_t start_time, end_time; + int64_t transfered_datafiles_bytes = 0; + int64_t transfered_walfiles_bytes = 0; + char pretty_source_bytes[20]; + err_i err = $noerr(); - source_conn = catchup_init_state(&source_node_info, source_pgdata, dest_pgdata); - catchup_preflight_checks(&source_node_info, source_conn, source_pgdata, dest_pgdata); + + source_conn = catchup_init_state(db_location, &source_node_info, + source_pgdata, dest_pgdata); + catchup_preflight_checks(db_location, local_location, + &source_node_info, source_conn, source_pgdata, + dest_pgdata); /* we need to sort --exclude_path's for future searching */ if (exclude_absolute_paths_list != NULL) @@ -654,13 +637,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, if (current.backup_mode != BACKUP_MODE_FULL) { dest_filelist = parray_new(); - dir_list_file(dest_filelist, dest_pgdata, - true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); - filter_filelist(dest_filelist, dest_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Destination"); + db_list_dir(dest_filelist, dest_pgdata, true, false, 0, FIO_LOCAL_HOST); + filter_filelist(dest_filelist, dest_pgdata, exclude_absolute_paths_list, + exclude_relative_paths_list, "Destination"); // fill dest_redo.lsn and dest_redo.tli - get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); - elog(INFO, "syncLSN = %X/%X", (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn); + get_redo(local_location, dest_pgdata, &dest_redo); + elog(INFO, "syncLSN = %X/%X", (uint32) (dest_redo.lsn >> 32), + (uint32) dest_redo.lsn); /* * Future improvement to catch partial catchup: @@ -677,43 +661,53 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, */ if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(source_conn, &source_node_info); + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(source_conn, + &source_node_info); if (ptrack_lsn > dest_redo.lsn || ptrack_lsn == InvalidXLogRecPtr) - elog(ERROR, "LSN from ptrack_control in source %X/%X is greater than checkpoint LSN in destination %X/%X.\n" - "You can perform only FULL catchup.", - (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), - (uint32) (dest_redo.lsn >> 32), - (uint32) (dest_redo.lsn)); + elog(ERROR, + "LSN from ptrack_control in source %X/%X is greater than checkpoint LSN in destination %X/%X.\n" + "You can perform only FULL catchup.", + (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), + (uint32) (dest_redo.lsn >> 32), + (uint32) (dest_redo.lsn)); } { - char label[1024]; + char label[1024]; /* notify start of backup to PostgreSQL server */ time2iso(label, lengthof(label), current.start_time, false); strncat(label, " with pg_probackup", lengthof(label) - - strlen(" with pg_probackup")); + strlen(" with pg_probackup")); /* Call pg_start_backup function in PostgreSQL connect */ - pg_start_backup(label, smooth_checkpoint, ¤t, &source_node_info, source_conn); - elog(INFO, "pg_start_backup START LSN %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + pg_start_backup(label, smooth_checkpoint, ¤t, &source_node_info, + source_conn); + elog(INFO, "pg_start_backup START LSN %X/%X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); } /* Sanity: source cluster must be "in future" relatively to dest cluster */ if (current.backup_mode != BACKUP_MODE_FULL && dest_redo.lsn > current.start_lsn) - elog(ERROR, "Current START LSN %X/%X is lower than SYNC LSN %X/%X, " - "it may indicate that we are trying to catchup with PostgreSQL instance from the past", - (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), - (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn)); + elog(ERROR, "Current START LSN %X/%X is lower than SYNC LSN %X/%X, " + "it may indicate that we are trying to catchup with PostgreSQL instance from the past", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn)); /* Start stream replication */ join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR); if (!dry_run) { - fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST); - start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt, - current.start_lsn, current.tli, false); + err = $i(pioMakeDir, local_location, .path = dest_xlog_path, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create WAL directory: %s", $errmsg(err)); + } + start_WAL_streaming(source_conn, dest_xlog_path, + &instance_config.conn_opt, + current.start_lsn, current.tli, false); } else elog(INFO, "WAL streaming skipping with --dry-run option"); @@ -721,12 +715,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, source_filelist = parray_new(); /* list files with the logical path. omit $PGDATA */ - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(source_filelist, source_pgdata, - true, true, false, backup_logs, true, 0); - else - dir_list_file(source_filelist, source_pgdata, - true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + db_list_dir(source_filelist, source_pgdata, true, false, 0, FIO_DB_HOST); //REVIEW FIXME. Let's fix that before release. // TODO what if wal is not a dir (symlink to a dir)? @@ -751,29 +740,36 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, * extractPageMap(), make_pagemap_from_ptrack(). */ parray_qsort(source_filelist, pgFileCompareRelPathWithExternal); + source_filehash = make_filelist_hashtable(source_filelist); + //REVIEW Do we want to do similar calculation for dest? //REVIEW_ANSWER what for? { - ssize_t source_bytes = 0; - char pretty_bytes[20]; + ssize_t source_bytes = 0; + char pretty_bytes[20]; source_bytes += calculate_datasize_of_filelist(source_filelist); /* Extract information about files in source_filelist parsing their names:*/ parse_filelist_filenames(source_filelist, source_pgdata); - filter_filelist(source_filelist, source_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Source"); + filter_filelist(source_filelist, source_pgdata, + exclude_absolute_paths_list, + exclude_relative_paths_list, "Source"); current.pgdata_bytes += calculate_datasize_of_filelist(source_filelist); - pretty_size(current.pgdata_bytes, pretty_source_bytes, lengthof(pretty_source_bytes)); - pretty_size(source_bytes - current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); - elog(INFO, "Source PGDATA size: %s (excluded %s)", pretty_source_bytes, pretty_bytes); + pretty_size(current.pgdata_bytes, pretty_source_bytes, + lengthof(pretty_source_bytes)); + pretty_size(source_bytes - current.pgdata_bytes, pretty_bytes, + lengthof(pretty_bytes)); + elog(INFO, "Source PGDATA size: %s (excluded %s)", pretty_source_bytes, + pretty_bytes); } elog(INFO, "Start LSN (source): %X/%X, TLI: %X", - (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), - current.tli); + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + current.tli); if (current.backup_mode != BACKUP_MODE_FULL) elog(INFO, "LSN in destination: %X/%X, TLI: %X", (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn), @@ -787,9 +783,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* Build the page map from ptrack information */ make_pagemap_from_ptrack_2(source_filelist, source_conn, - source_node_info.ptrack_schema, - source_node_info.ptrack_version_num, - dest_redo.lsn); + source_node_info.ptrack_schema, + source_node_info.ptrack_version_num, + dest_redo.lsn); time(&end_time); elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec", difftime(end_time, start_time)); @@ -807,10 +803,10 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, */ for (i = 0; i < parray_num(source_filelist); i++) { - pgFile *file = (pgFile *) parray_get(source_filelist, i); + pgFile *file = (pgFile *) parray_get(source_filelist, i); char parent_dir[MAXPGPATH]; - if (!S_ISDIR(file->mode) || file->excluded) + if (file->kind != PIO_KIND_DIRECTORY || file->excluded) continue; /* @@ -825,54 +821,71 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, if (strcmp(parent_dir, PG_TBLSPC_DIR) != 0) { /* if the entry is a regular directory, create it in the destination */ - char dirpath[MAXPGPATH]; + char dirpath[MAXPGPATH]; join_path_components(dirpath, dest_pgdata, file->rel_path); elog(LOG, "Create directory '%s'", dirpath); if (!dry_run) - fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST); + { + err = $i(pioMakeDir, local_location, .path = dirpath, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create directory: %s", $errmsg(err)); + } + } } else { /* this directory located in pg_tblspc */ const char *linked_path = NULL; - char to_path[MAXPGPATH]; + char to_path[MAXPGPATH]; // TODO perform additional check that this is actually symlink? { /* get full symlink path and map this path to new location */ - char source_full_path[MAXPGPATH]; - char symlink_content[MAXPGPATH]; - join_path_components(source_full_path, source_pgdata, file->rel_path); - fio_readlink(source_full_path, symlink_content, sizeof(symlink_content), FIO_DB_HOST); + char source_full_path[MAXPGPATH]; + char symlink_content[MAXPGPATH]; + join_path_components(source_full_path, source_pgdata, + file->rel_path); + fio_readlink(FIO_DB_HOST, source_full_path, symlink_content, + sizeof(symlink_content)); /* we checked that mapping exists in preflight_checks for local catchup */ linked_path = get_tablespace_mapping(symlink_content); - elog(INFO, "Map tablespace full_path: \"%s\" old_symlink_content: \"%s\" new_symlink_content: \"%s\"\n", - source_full_path, - symlink_content, - linked_path); + elog(INFO, + "Map tablespace full_path: \"%s\" old_symlink_content: \"%s\" new_symlink_content: \"%s\"\n", + source_full_path, + symlink_content, + linked_path); } if (!is_absolute_path(linked_path)) - elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", - linked_path); + elog(ERROR, + "Tablespace directory path must be an absolute path: %s\n", + linked_path); join_path_components(to_path, dest_pgdata, file->rel_path); elog(INFO, "Create directory \"%s\" and symbolic link \"%s\"", - linked_path, to_path); + linked_path, to_path); if (!dry_run) { /* create tablespace directory */ - if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0) - elog(ERROR, "Could not create tablespace directory \"%s\": %s", - linked_path, strerror(errno)); + err = $i(pioMakeDir, local_location, .path = linked_path, + .mode = file->mode, .strict = false); + if ($haserr(err)) + { + elog(ERROR, + "Could not create tablespace directory \"%s\": \"%s\"", + linked_path, $errmsg(err)); + } /* create link to linked_path */ - if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0) - elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s", - linked_path, to_path, strerror(errno)); + if (fio_symlink(FIO_LOCAL_HOST, linked_path, to_path, true) < 0) + elog(ERROR, + "Could not create symbolic link \"%s\" -> \"%s\": %s", + to_path, linked_path, strerror(errno)); } } } @@ -888,10 +901,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */ search_key.rel_path = XLOG_CONTROL_FILE; search_key.external_dir_num = 0; - control_file_elem_index = parray_bsearch_index(source_filelist, &search_key, pgFileCompareRelPathWithExternal); - if(control_file_elem_index < 0) - elog(ERROR, "\"%s\" not found in \"%s\"\n", XLOG_CONTROL_FILE, source_pgdata); - source_pg_control_file = parray_remove(source_filelist, control_file_elem_index); + control_file_elem_index = parray_bsearch_index(source_filelist, + &search_key, + pgFileCompareRelPathWithExternal); + if (control_file_elem_index < 0) + elog(ERROR, "\"%s\" not found in \"%s\"\n", XLOG_CONTROL_FILE, + source_pgdata); + source_pg_control_file = parray_remove(source_filelist, + control_file_elem_index); } /* TODO before public release: must be more careful with pg_control. @@ -917,18 +934,18 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, parray_qsort(dest_filelist, pgFileCompareRelPathWithExternalDesc); for (i = 0; i < parray_num(dest_filelist); i++) { - bool redundant = true; - pgFile *file = (pgFile *) parray_get(dest_filelist, i); - pgFile **src_file = NULL; + bool redundant = true; + pgFile *file = (pgFile *) parray_get(dest_filelist, i); + pgFile *src_file = NULL; //TODO optimize it and use some merge-like algorithm //instead of bsearch for each file. - src_file = (pgFile **) parray_bsearch(source_filelist, file, pgFileCompareRelPathWithExternal); + src_file = search_file_in_hashtable(source_filehash, file); - if (src_file!= NULL && !(*src_file)->excluded && file->excluded) - (*src_file)->excluded = true; + if (src_file != NULL && !src_file->excluded && file->excluded) + src_file->excluded = true; - if (src_file!= NULL || file->excluded) + if (src_file != NULL || file->excluded) redundant = false; /* pg_filenode.map are always copied, because it's crc cannot be trusted */ @@ -939,14 +956,20 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* if file does not exists in destination list, then we can safely unlink it */ if (redundant) { - char fullpath[MAXPGPATH]; + char fullpath[MAXPGPATH]; join_path_components(fullpath, dest_pgdata, file->rel_path); if (!dry_run) { - fio_delete(file->mode, fullpath, FIO_LOCAL_HOST); + if (fio_remove(FIO_LOCAL_HOST, fullpath, false) == 0) + elog(LOG, "Deleted file \"%s\"", fullpath); + else + elog(ERROR, + "Cannot delete redundant file in destination \"%s\": %s", + fullpath, strerror(errno)); } - elog(LOG, "Deleted file \"%s\"", fullpath); + else + elog(LOG, "Deleted file \"%s\"", fullpath); /* shrink dest pgdata list */ pgFileFree(file); @@ -964,14 +987,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* Sort the array for binary search */ if (dest_filelist) - parray_qsort(dest_filelist, pgFileCompareRelPathWithExternal); + dest_filehash = make_filelist_hashtable(dest_filelist); /* run copy threads */ elog(INFO, "Start transferring data files"); time(&start_time); transfered_datafiles_bytes = catchup_multithreaded_copy(num_threads, &source_node_info, source_pgdata, dest_pgdata, - source_filelist, dest_filelist, + source_filelist, dest_filehash, dest_redo.lsn, current.backup_mode); catchup_isok = transfered_datafiles_bytes != -1; @@ -982,8 +1005,8 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, char to_fullpath[MAXPGPATH]; join_path_components(from_fullpath, source_pgdata, source_pg_control_file->rel_path); join_path_components(to_fullpath, dest_pgdata, source_pg_control_file->rel_path); - copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, - to_fullpath, FIO_LOCAL_HOST, source_pg_control_file); + copy_pgcontrol_file(db_location, from_fullpath, + local_location, to_fullpath, source_pg_control_file); transfered_datafiles_bytes += source_pg_control_file->size; } @@ -1013,58 +1036,26 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, pg_silent_client_messages(source_conn); /* Execute pg_stop_backup using PostgreSQL connection */ - pg_stop_backup_send(source_conn, source_node_info.server_version, current.from_replica, exclusive_backup, &stop_backup_query_text); + pg_stop_backup_send(source_conn, source_node_info.server_version, current.from_replica, &stop_backup_query_text); /* * Wait for the result of pg_stop_backup(), but no longer than * archive_timeout seconds */ - pg_stop_backup_consume(source_conn, source_node_info.server_version, exclusive_backup, timeout, stop_backup_query_text, &stop_backup_result); + pg_stop_backup_consume(source_conn, source_node_info.server_version, timeout, stop_backup_query_text, &stop_backup_result); /* Cleanup */ pg_free(stop_backup_query_text); } - if (!dry_run) - wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, ¤t); - -#if PG_VERSION_NUM >= 90600 - /* Write backup_label */ - Assert(stop_backup_result.backup_label_content != NULL); - if (!dry_run) - { - pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label", - stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, - NULL); - } - free(stop_backup_result.backup_label_content); - stop_backup_result.backup_label_content = NULL; - stop_backup_result.backup_label_content_len = 0; - - /* tablespace_map */ - if (stop_backup_result.tablespace_map_content != NULL) - { - // TODO what if tablespace is created during catchup? - /* Because we have already created symlinks in pg_tblspc earlier, - * we do not need to write the tablespace_map file. - * So this call is unnecessary: - * pg_stop_backup_write_file_helper(dest_pgdata, PG_TABLESPACE_MAP_FILE, "tablespace map", - * stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, - * NULL); - */ - free(stop_backup_result.tablespace_map_content); - stop_backup_result.tablespace_map_content = NULL; - stop_backup_result.tablespace_map_content_len = 0; - } -#endif - /* wait for end of wal streaming and calculate wal size transfered */ if (!dry_run) { parray *wal_files_list = NULL; wal_files_list = parray_new(); - if (wait_WAL_streaming_end(wal_files_list)) + if (wait_WAL_streaming_end(wal_files_list, dest_xlog_path, + stop_backup_result.lsn, ¤t)) elog(ERROR, "WAL streaming failed"); for (i = 0; i < parray_num(wal_files_list); i++) @@ -1078,13 +1069,36 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, wal_files_list = NULL; } - /* - * In case of backup from replica >= 9.6 we must fix minRecPoint - */ - if (current.from_replica && !exclusive_backup) + /* Write backup_label */ + Assert(stop_backup_result.backup_label_content.len != 0); + if (!dry_run) { - set_min_recovery_point(source_pg_control_file, dest_pgdata, current.stop_lsn); + pg_stop_backup_write_file_helper(local_location, + dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label", + stop_backup_result.backup_label_content, NULL); } + ft_str_free(&stop_backup_result.backup_label_content); + + /* tablespace_map */ + if (stop_backup_result.tablespace_map_content.len != 0) + { + // TODO what if tablespace is created during catchup? + /* Because we have already created symlinks in pg_tblspc earlier, + * we do not need to write the tablespace_map file. + * So this call is unnecessary: + * pg_stop_backup_write_file_helper(dest_pgdata, PG_TABLESPACE_MAP_FILE, "tablespace map", + * stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, + * NULL); + */ + } + ft_str_free(&stop_backup_result.tablespace_map_content); + + /* + * In case of backup from replica we must fix minRecPoint + */ + if (current.from_replica) + set_min_recovery_point(db_location, local_location, + source_pg_control_file, dest_pgdata, current.stop_lsn); /* close ssh session in main thread */ fio_disconnect(); @@ -1112,7 +1126,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* Sync all copied files unless '--no-sync' flag is used */ if (sync_dest_files && !dry_run) - catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file); + catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST); else elog(WARNING, "Files are not synced to disk"); @@ -1122,8 +1136,10 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, parray_walk(dest_filelist, pgFileFree); } parray_free(dest_filelist); + parray_free(dest_filehash); parray_walk(source_filelist, pgFileFree); parray_free(source_filelist); + parray_free(source_filehash); pgFileFree(source_pg_control_file); return 0; diff --git a/src/checkdb.c b/src/checkdb.c index 1133a7b5d..b1d6bd41d 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -16,7 +16,6 @@ #include "pg_probackup.h" -#include #include #include @@ -148,7 +147,7 @@ check_files(void *arg) elog(ERROR, "interrupted during checkdb"); /* No need to check directories */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) continue; if (!pg_atomic_test_set_flag(&file->lock)) @@ -162,7 +161,7 @@ check_files(void *arg) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, n_files_list, from_fullpath); - if (S_ISREG(file->mode)) + if (file->kind == PIO_KIND_REGULAR) { /* check only uncompressed by cfs datafiles */ if (file->is_datafile && !file->is_cfs) @@ -172,8 +171,7 @@ check_files(void *arg) * uses global variables to set connections. * Need refactoring. */ - if (!check_data_file(&(arguments->conn_arg), - file, from_fullpath, + if (!check_data_file(file, from_fullpath, arguments->checksum_version)) arguments->ret = 2; /* corruption found */ } @@ -208,8 +206,7 @@ do_block_validation(char *pgdata, uint32 checksum_version) files_list = parray_new(); /* list files with the logical path. omit $PGDATA */ - dir_list_file(files_list, pgdata, true, true, - false, false, true, 0, FIO_DB_HOST); + db_list_dir(files_list, pgdata, true, false, 0, FIO_DB_HOST); /* * Sort pathname ascending. @@ -230,7 +227,7 @@ do_block_validation(char *pgdata, uint32 checksum_version) } /* Sort by size for load balancing */ - parray_qsort(files_list, pgFileCompareSize); + parray_qsort(files_list, pgFileCompareSizeDesc); /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); @@ -293,7 +290,8 @@ check_indexes(void *arg) int i; check_indexes_arg *arguments = (check_indexes_arg *) arg; int n_indexes = 0; - my_thread_num = arguments->thread_num; + + set_my_thread_num(arguments->thread_num); if (arguments->index_list) n_indexes = parray_num(arguments->index_list); @@ -735,7 +733,7 @@ do_amcheck(ConnectionOptions conn_opt, PGconn *conn) /* Entry point of pg_probackup CHECKDB subcommand */ void -do_checkdb(bool need_amcheck, +do_checkdb(pioDrive_i drive, bool need_amcheck, ConnectionOptions conn_opt, char *pgdata) { PGNodeInfo nodeInfo; @@ -757,7 +755,7 @@ do_checkdb(bool need_amcheck, cur_conn = pgdata_basic_setup(conn_opt, &nodeInfo); /* ensure that conn credentials and pgdata are consistent */ - check_system_identifiers(cur_conn, pgdata); + check_system_identifiers(drive, cur_conn, pgdata); /* * we don't need this connection anymore. diff --git a/src/compatibility/file_compat.c b/src/compatibility/file_compat.c new file mode 100644 index 000000000..d465c53a1 --- /dev/null +++ b/src/compatibility/file_compat.c @@ -0,0 +1,152 @@ +#include +#include +#include + +#include "logging.h" +#include "file_compat.h" +/*vvs*/ +/* + * fsync_fname -- Try to fsync a file or directory + * + * Ignores errors trying to open unreadable files, or trying to fsync + * directories on systems where that isn't allowed/required. All other errors + * are fatal. + */ +int +fsync_fname_compat(const char* fname, bool isdir) +{ + int fd; + int flags; + int returncode; + + /* + * Some OSs require directories to be opened read-only whereas other + * systems don't allow us to fsync files opened read-only; so we need both + * cases here. Using O_RDWR will cause us to fail to fsync files that are + * not writable by our userid, but we assume that's OK. + */ + flags = PG_BINARY; + if (!isdir) + flags |= O_RDWR; + else + flags |= O_RDONLY; + + /* + * Open the file, silently ignoring errors about unreadable files (or + * unsupported operations, e.g. opening a directory under Windows), and + * logging others. + */ + fd = open(fname, flags, 0); + if (fd < 0) + { + if (errno == EACCES || (isdir && errno == EISDIR)) + return 0; + pg_log_error("could not open file \"%s\": %m", fname); + return -1; + } + + returncode = fsync(fd); + + /* + * Some OSes don't allow us to fsync directories at all, so we can ignore + * those errors. Anything else needs to be reported. + */ + if (returncode != 0 && !(isdir && (errno == EBADF || errno == EINVAL))) + { + pg_log_fatal("could not fsync file \"%s\": %m", fname); + (void)close(fd); + exit(EXIT_FAILURE); + } + + (void)close(fd); + return 0; +} + +/* + * fsync_parent_path -- fsync the parent path of a file or directory + * + * This is aimed at making file operations persistent on disk in case of + * an OS crash or power failure. + */ +int +fsync_parent_path_compat(const char* fname) +{ + char parentpath[MAXPGPATH]; + + strlcpy(parentpath, fname, MAXPGPATH); + get_parent_directory(parentpath); + + /* + * get_parent_directory() returns an empty string if the input argument is + * just a file name (see comments in path.c), so handle that as being the + * current directory. + */ + if (strlen(parentpath) == 0) + strlcpy(parentpath, ".", MAXPGPATH); + + if (fsync_fname_compat(parentpath, true) != 0) + return -1; + + return 0; +} + +/* + * durable_rename -- rename(2) wrapper, issuing fsyncs required for durability + * + * Wrapper around rename, similar to the backend version. + */ +int +durable_rename_compat(const char* oldfile, const char* newfile) +{ + int fd; + + /* + * First fsync the old and target path (if it exists), to ensure that they + * are properly persistent on disk. Syncing the target file is not + * strictly necessary, but it makes it easier to reason about crashes; + * because it's then guaranteed that either source or target file exists + * after a crash. + */ + if (fsync_fname_compat(oldfile, false) != 0) + return -1; + + fd = open(newfile, PG_BINARY | O_RDWR, 0); + if (fd < 0) + { + if (errno != ENOENT) + { + pg_log_error("could not open file \"%s\": %m", newfile); + return -1; + } + } + else + { + if (fsync(fd) != 0) + { + pg_log_fatal("could not fsync file \"%s\": %m", newfile); + close(fd); + exit(EXIT_FAILURE); + } + close(fd); + } + + /* Time to do the real deal... */ + if (rename(oldfile, newfile) != 0) + { + pg_log_error("could not rename file \"%s\" to \"%s\": %m", + oldfile, newfile); + return -1; + } + + /* + * To guarantee renaming the file is persistent, fsync the file with its + * new name, and its containing directory. + */ + if (fsync_fname_compat(newfile, false) != 0) + return -1; + + if (fsync_parent_path_compat(newfile) != 0) + return -1; + + return 0; +} diff --git a/src/compatibility/file_compat.h b/src/compatibility/file_compat.h new file mode 100644 index 000000000..1000e8e92 --- /dev/null +++ b/src/compatibility/file_compat.h @@ -0,0 +1,32 @@ +#ifndef FILE_COMPAT_H +#define FILE_COMPAT_H + +#include +#include "datatype/timestamp.h" + + +#if PG_VERSION_NUM >= 120000 +#include "common/logging.h" +#else +#include "logging.h" +#endif + + +extern int fsync_parent_path_compat(const char* fname); +extern int fsync_fname_compat(const char* fname, bool isdir); +extern int durable_rename_compat(const char* oldfile, const char* newfile); + + +#if PG_VERSION_NUM < 110000 +#include "file_compat10.h" +#else +#include "common/file_perm.h" +#include "access/xlog_internal.h" +#endif + + + +#endif /* FILE_COMPAT_H */ + + + diff --git a/src/compatibility/file_compat10.c b/src/compatibility/file_compat10.c new file mode 100644 index 000000000..e125bf13d --- /dev/null +++ b/src/compatibility/file_compat10.c @@ -0,0 +1,84 @@ +#include + +#if PG_VERSION_NUM < 110000 + +#include +#include +#include "logging.h" + +#include "file_compat.h" + +/* Modes for creating directories and files in the data directory */ +int pg_dir_create_mode = PG_DIR_MODE_OWNER; +int pg_file_create_mode = PG_FILE_MODE_OWNER; +/* + * Mode mask to pass to umask(). This is more of a preventative measure since + * all file/directory creates should be performed using the create modes above. + */ +int pg_mode_mask = PG_MODE_MASK_OWNER; + + +/* + * Set create modes and mask to use when writing to PGDATA based on the data + * directory mode passed. If group read/execute are present in the mode, then + * create modes and mask will be relaxed to allow group read/execute on all + * newly created files and directories. + */ +void +SetDataDirectoryCreatePerm(int dataDirMode) +{ + /* If the data directory mode has group access */ + if ((PG_DIR_MODE_GROUP & dataDirMode) == PG_DIR_MODE_GROUP) + { + pg_dir_create_mode = PG_DIR_MODE_GROUP; + pg_file_create_mode = PG_FILE_MODE_GROUP; + pg_mode_mask = PG_MODE_MASK_GROUP; + } + /* Else use default permissions */ + else + { + pg_dir_create_mode = PG_DIR_MODE_OWNER; + pg_file_create_mode = PG_FILE_MODE_OWNER; + pg_mode_mask = PG_MODE_MASK_OWNER; + } +} + + + +/* + * Get the create modes and mask to use when writing to PGDATA by examining the + * mode of the PGDATA directory and calling SetDataDirectoryCreatePerm(). + * + * Errors are not handled here and should be reported by the application when + * false is returned. + * + * Suppress when on Windows, because there may not be proper support for Unix-y + * file permissions. + */ +bool +GetDataDirectoryCreatePerm(const char* dataDir) +{ +#if !defined(WIN32) && !defined(__CYGWIN__) + struct stat statBuf; + + /* + * If an error occurs getting the mode then return false. The caller is + * responsible for generating an error, if appropriate, indicating that we + * were unable to access the data directory. + */ + if (stat(dataDir, &statBuf) == -1) + return false; + + /* Set permissions */ + SetDataDirectoryCreatePerm(statBuf.st_mode); + return true; +#else /* !defined(WIN32) && !defined(__CYGWIN__) */ + /* + * On Windows, we don't have anything to do here since they don't have + * Unix-y permissions. + */ + return true; +#endif +} + +#endif diff --git a/src/compatibility/file_compat10.h b/src/compatibility/file_compat10.h new file mode 100644 index 000000000..c40dda133 --- /dev/null +++ b/src/compatibility/file_compat10.h @@ -0,0 +1,172 @@ +#ifndef FILE_COMPAT10_H +#define FILE_COMPAT10_H + +//for PG10 + +//#include "pg_bswap.h" + + +#ifndef DEFAULT_XLOG_SEG_SIZE +#define DEFAULT_XLOG_SEG_SIZE (16*1024*1024) +#endif + + +#ifndef PG_FILE_MODE_OWNER + +/* + * Mode mask for data directory permissions that only allows the owner to + * read/write directories and files. + * + * This is the default. + */ +#define PG_MODE_MASK_OWNER (S_IRWXG | S_IRWXO) + + /* + * Mode mask for data directory permissions that also allows group read/execute. + */ +#define PG_MODE_MASK_GROUP (S_IWGRP | S_IRWXO) + + + +#define PG_FILE_MODE_OWNER (S_IRUSR | S_IWUSR) +//#define pg_file_create_mode PG_FILE_MODE_OWNER + +/* Default mode for creating directories */ +#define PG_DIR_MODE_OWNER S_IRWXU + +/* Mode for creating directories that allows group read/execute */ +#define PG_DIR_MODE_GROUP (S_IRWXU | S_IRGRP | S_IXGRP) + +/* Default mode for creating files */ +#define PG_FILE_MODE_OWNER (S_IRUSR | S_IWUSR) + +/* Mode for creating files that allows group read */ +#define PG_FILE_MODE_GROUP (S_IRUSR | S_IWUSR | S_IRGRP) + +/* Modes for creating directories and files in the data directory */ +extern int pg_dir_create_mode; +extern int pg_file_create_mode; + +/* Mode mask to pass to umask() */ +extern int pg_mode_mask; + +/* Set permissions and mask based on the provided mode */ +extern void SetDataDirectoryCreatePerm(int dataDirMode); + +/* Set permissions and mask based on the mode of the data directory */ +extern bool GetDataDirectoryCreatePerm(const char* dataDir); + +#endif + +/* Set permissions and mask based on the provided mode */ +extern void SetDataDirectoryCreatePerm(int dataDirMode); + +/* Set permissions and mask based on the mode of the data directory */ +extern bool GetDataDirectoryCreatePerm(const char *dataDir); + + +/* wal_segment_size can range from 1MB to 1GB */ +#define WalSegMinSize 1024 * 1024 +#define WalSegMaxSize 1024 * 1024 * 1024 + + +#define XLogSegmentOffset(xlogptr, wal_segsz_bytes) \ + ((xlogptr) & ((wal_segsz_bytes) - 1)) + +/* check that the given size is a valid wal_segment_size */ +#define IsPowerOf2(x) (x > 0 && ((x) & ((x)-1)) == 0) + +#define IsValidWalSegSize(size) \ + (IsPowerOf2(size) && \ + ((size) >= WalSegMinSize && (size) <= WalSegMaxSize)) + + + +/* From access/xlog_internal.h */ + +#undef XLByteToSeg +#undef XLogFileName +#undef XLogSegmentsPerXLogId +#undef XLogFromFileName +#undef XLogSegNoOffsetToRecPtr +#undef XLByteInSeg + +#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) \ + logSegNo = (xlrp) / (wal_segsz_bytes) + +#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + snprintf(fname, MAXFNAMELEN, "%08X%08X%08X", tli, \ + (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \ + (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes))) + +#define XLogSegmentsPerXLogId(wal_segsz_bytes) \ + (UINT64CONST(0x100000000) / (wal_segsz_bytes)) + + +/* + * The XLog directory and control file (relative to $PGDATA) + */ +#define XLOGDIR "pg_wal" +#define XLOG_CONTROL_FILE "global/pg_control" + +/* + * These macros encapsulate knowledge about the exact layout of XLog file + * names, timeline history file names, and archive-status file names. + */ +#define MAXFNAMELEN 64 + +/* Length of XLog file name */ +#define XLOG_FNAME_LEN 24 + +/* + * Generate a WAL segment file name. Do not use this macro in a helper + * function allocating the result generated. + */ +#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + snprintf(fname, MAXFNAMELEN, "%08X%08X%08X", tli, \ + (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \ + (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes))) + +#define XLogFileNameById(fname, tli, log, seg) \ + snprintf(fname, MAXFNAMELEN, "%08X%08X%08X", tli, log, seg) + +#define IsXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN) + + +#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + do { \ + uint32 log; \ + uint32 seg; \ + sscanf(fname, "%08X%08X%08X", tli, &log, &seg); \ + *logSegNo = (uint64) log * XLogSegmentsPerXLogId(wal_segsz_bytes) + seg; \ + } while (0) + +#define XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest) \ + (dest) = (segno) * (wal_segsz_bytes) + (offset) +/* + * Is an XLogRecPtr within a particular XLOG segment? + * + * For XLByteInSeg, do the computation at face value. For XLByteInPrevSeg, + * a boundary byte is taken to be in the previous segment. + */ +#define XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) \ + (((xlrp) / (wal_segsz_bytes)) == (logSegNo)) + + + +/* logs restore point */ +/* +typedef struct xl_restore_point +{ + TimestampTz rp_time; + char rp_name[MAXFNAMELEN]; +} xl_restore_point; +*/ + + +#endif /* FILE_COMPAT10_H */ + + + diff --git a/src/compatibility/logging.h b/src/compatibility/logging.h new file mode 100644 index 000000000..4fbf427dc --- /dev/null +++ b/src/compatibility/logging.h @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------- + * Logging framework for frontend programs + * + * Copyright (c) 2018-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 2021-2022, Postgres Professional + * + * src/include/common/logging.h + * + *------------------------------------------------------------------------- + */ +#ifndef COMMON_LOGGING_COMPAT_H +#define COMMON_LOGGING_COMPAT_H + +#if PG_VERSION_NUM >= 120000 && PG_VERSION_NUM < 150000 +#include "common/logging.h" + + +#else + +#include +#include "logger.h" + + +#define pg_log_fatal(...) elog(ERROR, __VA_ARGS__); + +#if PG_VERSION_NUM < 150000 +#define pg_log_error(...) elog(ERROR, __VA_ARGS__); +#define pg_log_warning(...) elog(WARNING, __VA_ARGS__); +#define pg_log_info(...) elog(INFO, __VA_ARGS__); +#endif + +#endif + +#endif /* COMMON_LOGGING_COMPAT_H */ + + + diff --git a/src/compatibility/pg-11.c b/src/compatibility/pg-11.c new file mode 100644 index 000000000..52f4b551c --- /dev/null +++ b/src/compatibility/pg-11.c @@ -0,0 +1,98 @@ +/*------------------------------------------------------------------------- + * + * pg-11.c + * PostgreSQL <= 11 compatibility + * + * Portions Copyright (c) 2022, Postgres Professional + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + *------------------------------------------------------------------------- + */ + +#include + +#if PG_VERSION_NUM < 120000 + +#include "c.h" +#include "utils/pg_crc.h" + +/* From postgresql src/backend/utils/hash/pg_crc.c */ + +/* + * Lookup table for calculating CRC-32 using Sarwate's algorithm. + * + * This table is based on the polynomial + * x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. + * (This is the same polynomial used in Ethernet checksums, for instance.) + * Using Williams' terms, this is the "normal", not "reflected" version. + */ + +const uint32 pg_crc32_table[256] = { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D +}; + +#endif diff --git a/src/compatibility/pg-11.h b/src/compatibility/pg-11.h new file mode 100644 index 000000000..63a83070a --- /dev/null +++ b/src/compatibility/pg-11.h @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------- + * + * pg-11.h + * PostgreSQL <= 11 compatibility + * + * Copyright (c) 2022, Postgres Professional + * + * When PG-11 reaches the end of support, we will need to remove + * *_CRC32_COMPAT macros and use *_CRC32C instead. + * And this file will be removed. + *------------------------------------------------------------------------- + */ + +#ifndef PG11_COMPAT_H +#define PG11_COMPAT_H + +#include "utils/pgut.h" + +#if PG_VERSION_NUM >= 120000 + +#define INIT_CRC32_COMPAT(backup_version, crc) \ +do { \ + Assert(backup_version >= 20025); \ + INIT_CRC32C(crc); \ +} while (0) + +#define COMP_CRC32_COMPAT(backup_version, crc, data, len) \ +do { \ + Assert(backup_version >= 20025); \ + COMP_CRC32C((crc), (data), (len)); \ +} while (0) + +#define FIN_CRC32_COMPAT(backup_version, crc) \ +do { \ + Assert(backup_version >= 20025); \ + FIN_CRC32C(crc); \ +} while (0) + +#else /* PG_VERSION_NUM < 120000 */ + +#define INIT_CRC32_COMPAT(backup_version, crc) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + INIT_CRC32C(crc); \ + else \ + INIT_TRADITIONAL_CRC32(crc); \ +} while (0) + +#define COMP_CRC32_COMPAT(backup_version, crc, data, len) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + COMP_CRC32C((crc), (data), (len)); \ + else \ + COMP_TRADITIONAL_CRC32(crc, data, len); \ +} while (0) + +#define FIN_CRC32_COMPAT(backup_version, crc) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + FIN_CRC32C(crc); \ + else \ + FIN_TRADITIONAL_CRC32(crc); \ +} while (0) + +#endif /* PG_VERSION_NUM < 120000 */ + +#endif /* PG11_COMPAT_H */ diff --git a/src/compatibility/receivelog.c b/src/compatibility/receivelog.c new file mode 100644 index 000000000..c5972ea1e --- /dev/null +++ b/src/compatibility/receivelog.c @@ -0,0 +1,1227 @@ +/*------------------------------------------------------------------------- + * + * receivelog.c - receive WAL files using the streaming + * replication protocol. + * + * Author: Magnus Hagander + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/receivelog.c + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "postgres_fe.h" + +#include +#include +#ifdef HAVE_SYS_SELECT_H +#include +#endif + +#include "common/file_utils.h" +#include "logging.h" +#include "libpq-fe.h" +#include "receivelog.h" +#include "streamutil.h" + +#include "file_compat.h" + + + /* + * Handy macro for printing XLogRecPtr in conventional format, e.g., + * + * printf("%X/%X", LSN_FORMAT_ARGS(lsn)); + */ +#ifndef LSN_FORMAT_ARGS +#define LSN_FORMAT_ARGS(lsn) (AssertVariableIsOfTypeMacro((lsn), XLogRecPtr), (uint32) ((lsn) >> 32)), ((uint32) (lsn)) +#endif + +/* fd and filename for currently open WAL file */ +static Walfile *walfile = NULL; +static char current_walfile_name[MAXPGPATH] = ""; +static bool reportFlushPosition = false; +static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr; + +static bool still_sending = true; /* feedback still needs to be sent? */ + +static PGresult *HandleCopyStream(PGconn *conn, StreamCtl *stream, + XLogRecPtr *stoppos); +static int CopyStreamPoll(PGconn *conn, long timeout_ms, pgsocket stop_socket); +static int CopyStreamReceive(PGconn *conn, long timeout, pgsocket stop_socket, + char **buffer); +static bool ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, + int len, XLogRecPtr blockpos, TimestampTz *last_status); +static bool ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, + XLogRecPtr *blockpos); +static PGresult *HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf, + XLogRecPtr blockpos, XLogRecPtr *stoppos); +static bool CheckCopyStreamStop(PGconn *conn, StreamCtl *stream, XLogRecPtr blockpos); +static long CalculateCopyStreamSleeptime(TimestampTz now, int standby_message_timeout, + TimestampTz last_status); + +static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos, + uint32 *timeline); + + +/* + * Open a new WAL file in the specified directory. + * + * Returns true if OK; on failure, returns false after printing an error msg. + * On success, 'walfile' is set to the FD for the file, and the base filename + * (without partial_suffix) is stored in 'current_walfile_name'. + * + * The file will be padded to 16Mb with zeroes. + */ +static bool +open_walfile(StreamCtl *stream, XLogRecPtr startpoint) +{ + Walfile *f; + char *fn; + ssize_t size; + XLogSegNo segno; + + XLByteToSeg(startpoint, segno, WalSegSz); + XLogFileName(current_walfile_name, stream->timeline, segno, WalSegSz); + + /* Note that this considers the compression used if necessary */ + fn = stream->walmethod->get_file_name(current_walfile_name); + + /* + * When streaming to files, if an existing file exists we verify that it's + * either empty (just created), or a complete WalSegSz segment (in which + * case it has been created and padded). Anything else indicates a corrupt + * file. Compressed files have no need for padding, so just ignore this + * case. + * + * When streaming to tar, no file with this name will exist before, so we + * never have to verify a size. + */ + if (stream->walmethod->compression() == 0 && + stream->walmethod->existsfile(fn)) + { + size = stream->walmethod->get_file_size(fn); + if (size < 0) + { + elog(ERROR, "could not get size of write-ahead log file \"%s\": %s", + fn, stream->walmethod->getlasterror()); + pg_free(fn); + return false; + } + if (size == WalSegSz) + { + /* Already padded file. Open it for use */ + f = stream->walmethod->open_for_write(current_walfile_name, false); + if (f == NULL) + { + elog(ERROR, "could not open existing write-ahead log file \"%s\": %s", + fn, stream->walmethod->getlasterror()); + pg_free(fn); + return false; + } + + /* fsync file in case of a previous crash */ + if (stream->walmethod->sync(f) != 0) + { + elog(ERROR, "could not fsync existing write-ahead log file \"%s\": %s", + fn, stream->walmethod->getlasterror()); + stream->walmethod->close(f, CLOSE_UNLINK); + exit(1); + } + + walfile = f; + pg_free(fn); + return true; + } + if (size != 0) + { + /* if write didn't set errno, assume problem is no disk space */ + if (errno == 0) + errno = ENOSPC; + elog(ERROR, ngettext("write-ahead log file \"%s\" has %d byte, should be 0 or %d", + "write-ahead log file \"%s\" has %d bytes, should be 0 or %d", + size), + fn, (int) size, WalSegSz); + pg_free(fn); + return false; + } + /* File existed and was empty, so fall through and open */ + } + + /* No file existed, so create one */ + + f = stream->walmethod->open_for_write(current_walfile_name, false); + if (f == NULL) + { + elog(ERROR, "could not open write-ahead log file \"%s\": %s", + fn, stream->walmethod->getlasterror()); + pg_free(fn); + return false; + } + + pg_free(fn); + walfile = f; + return true; +} + +/* + * Close the current WAL file (if open), and rename it to the correct + * filename if it's complete. On failure, prints an error message to stderr + * and returns false, otherwise returns true. + */ +static bool +close_walfile(StreamCtl *stream, XLogRecPtr pos) +{ + off_t currpos; + int r; + + if (walfile == NULL) + return true; + + currpos = stream->walmethod->get_current_pos(walfile); + if (currpos == -1) + { + elog(ERROR, "could not determine seek position in file \"%s\": %s", + current_walfile_name, stream->walmethod->getlasterror()); + stream->walmethod->close(walfile, CLOSE_UNLINK); + walfile = NULL; + + return false; + } + /* + * Pad file to WalSegSz size by zero bytes + */ + if (currpos < WalSegSz) + { + char *tempbuf = pgut_malloc0(XLOG_BLCKSZ); + int needWrite = WalSegSz - currpos; + int cnt; + while (needWrite > 0) + { + + cnt = needWrite > XLOG_BLCKSZ ? XLOG_BLCKSZ : needWrite; + if (stream->walmethod->write(walfile, tempbuf, cnt) != cnt) + { + elog(ERROR, "failed to append file \"%s\": %s", + current_walfile_name, stream->walmethod->getlasterror()); + stream->walmethod->close(walfile, CLOSE_NORMAL); + walfile = NULL; + pgut_free(tempbuf); + return false; + } + needWrite -= cnt; + } + pgut_free(tempbuf); + } + r = stream->walmethod->close(walfile, CLOSE_NORMAL); + + walfile = NULL; + + if (r != 0) + { + elog(ERROR, "could not close file \"%s\": %s", + current_walfile_name, stream->walmethod->getlasterror()); + return false; + } + + lastFlushPosition = pos; + return true; +} + + +/* + * Check if a timeline history file exists. + */ +static bool +existsTimeLineHistoryFile(StreamCtl *stream) +{ + char histfname[MAXFNAMELEN]; + + /* + * Timeline 1 never has a history file. We treat that as if it existed, + * since we never need to stream it. + */ + if (stream->timeline == 1) + return true; + + TLHistoryFileName(histfname, stream->timeline); + + return stream->walmethod->existsfile(histfname); +} + +static bool +writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) +{ + int size = strlen(content); + char histfname[MAXFNAMELEN]; + Walfile *f; + + /* + * Check that the server's idea of how timeline history files should be + * named matches ours. + */ + TLHistoryFileName(histfname, stream->timeline); + if (strcmp(histfname, filename) != 0) + { + pg_log_error("server reported unexpected history file name for timeline %u: %s", + stream->timeline, filename); + return false; + } + + f = stream->walmethod->open_for_write(histfname, true); + if (f == NULL) + { + pg_log_error("could not create timeline history file \"%s\": %s", + histfname, stream->walmethod->getlasterror()); + return false; + } + + if ((int) stream->walmethod->write(f, content, size) != size) + { + pg_log_error("could not write timeline history file \"%s\": %s", + histfname, stream->walmethod->getlasterror()); + + /* + * If we fail to make the file, delete it to release disk space + */ + stream->walmethod->close(f, CLOSE_UNLINK); + + return false; + } + + if (stream->walmethod->close(f, CLOSE_NORMAL) != 0) + { + pg_log_error("could not close file \"%s\": %s", + histfname, stream->walmethod->getlasterror()); + return false; + } + + return true; +} + +/* + * Send a Standby Status Update message to server. + */ +static bool +sendFeedback(PGconn *conn, XLogRecPtr blockpos, TimestampTz now, bool replyRequested) +{ + char replybuf[1 + 8 + 8 + 8 + 8 + 1]; + int len = 0; + + replybuf[len] = 'r'; + len += 1; + fe_sendint64(blockpos, &replybuf[len]); /* write */ + len += 8; + if (reportFlushPosition) + fe_sendint64(lastFlushPosition, &replybuf[len]); /* flush */ + else + fe_sendint64(InvalidXLogRecPtr, &replybuf[len]); /* flush */ + len += 8; + fe_sendint64(InvalidXLogRecPtr, &replybuf[len]); /* apply */ + len += 8; + fe_sendint64(now, &replybuf[len]); /* sendTime */ + len += 8; + replybuf[len] = replyRequested ? 1 : 0; /* replyRequested */ + len += 1; + + if (PQputCopyData(conn, replybuf, len) <= 0 || PQflush(conn)) + { + pg_log_error("could not send feedback packet: %s", + PQerrorMessage(conn)); + return false; + } + + return true; +} + +/* + * Check that the server version we're connected to is supported by + * ReceiveXlogStream(). + * + * If it's not, an error message is printed to stderr, and false is returned. + */ +bool +CheckServerVersionForStreaming(PGconn *conn) +{ + int minServerMajor, + maxServerMajor; + int serverMajor; + + /* + * The message format used in streaming replication changed in 9.3, so we + * cannot stream from older servers. And we don't support servers newer + * than the client; it might work, but we don't know, so err on the safe + * side. + */ + minServerMajor = 903; + maxServerMajor = PG_VERSION_NUM / 100; + serverMajor = PQserverVersion(conn) / 100; + if (serverMajor < minServerMajor) + { + const char *serverver = PQparameterStatus(conn, "server_version"); + + pg_log_error("incompatible server version %s; client does not support streaming from server versions older than %s", + serverver ? serverver : "'unknown'", + "9.3"); + return false; + } + else if (serverMajor > maxServerMajor) + { + const char *serverver = PQparameterStatus(conn, "server_version"); + + pg_log_error("incompatible server version %s; client does not support streaming from server versions newer than %s", + serverver ? serverver : "'unknown'", + PG_VERSION); + return false; + } + return true; +} + +/* + * Receive a log stream starting at the specified position. + * + * Individual parameters are passed through the StreamCtl structure. + * + * If sysidentifier is specified, validate that both the system + * identifier and the timeline matches the specified ones + * (by sending an extra IDENTIFY_SYSTEM command) + * + * All received segments will be written to the directory + * specified by basedir. This will also fetch any missing timeline history + * files. + * + * The stream_stop callback will be called every time data + * is received, and whenever a segment is completed. If it returns + * true, the streaming will stop and the function + * return. As long as it returns false, streaming will continue + * indefinitely. + * + * If stream_stop() checks for external input, stop_socket should be set to + * the FD it checks. This will allow such input to be detected promptly + * rather than after standby_message_timeout (which might be indefinite). + * Note that signals will interrupt waits for input as well, but that is + * race-y since a signal received while busy won't interrupt the wait. + * + * standby_message_timeout controls how often we send a message + * back to the primary letting it know our progress, in milliseconds. + * Zero means no messages are sent. + * This message will only contain the write location, and never + * flush or replay. + * + * If 'partial_suffix' is not NULL, files are initially created with the + * given suffix, and the suffix is removed once the file is finished. That + * allows you to tell the difference between partial and completed files, + * so that you can continue later where you left. + * + * If 'synchronous' is true, the received WAL is flushed as soon as written, + * otherwise only when the WAL file is closed. + * + * Note: The WAL location *must* be at a log segment start! + */ +bool +ReceiveXlogStream(PGconn *conn, StreamCtl *stream) +{ + char query[128]; + char slotcmd[128]; + PGresult *res; + XLogRecPtr stoppos; + + /* + * The caller should've checked the server version already, but doesn't do + * any harm to check it here too. + */ + if (!CheckServerVersionForStreaming(conn)) + return false; + + /* + * Decide whether we want to report the flush position. If we report the + * flush position, the primary will know what WAL we'll possibly + * re-request, and it can then remove older WAL safely. We must always do + * that when we are using slots. + * + * Reporting the flush position makes one eligible as a synchronous + * replica. People shouldn't include generic names in + * synchronous_standby_names, but we've protected them against it so far, + * so let's continue to do so unless specifically requested. + */ + if (stream->replication_slot != NULL) + { + reportFlushPosition = true; + sprintf(slotcmd, "SLOT \"%s\" ", stream->replication_slot); + } + else + { + slotcmd[0] = 0; + } + + if (stream->sysidentifier != NULL) + { + /* Validate system identifier hasn't changed */ + res = PQexec(conn, "IDENTIFY_SYSTEM"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + pg_log_error("could not send replication command \"%s\": %s", + "IDENTIFY_SYSTEM", PQerrorMessage(conn)); + PQclear(res); + return false; + } + if (PQntuples(res) != 1 || PQnfields(res) < 3) + { + pg_log_error("could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields", + PQntuples(res), PQnfields(res), 1, 3); + PQclear(res); + return false; + } + if (strcmp(stream->sysidentifier, PQgetvalue(res, 0, 0)) != 0) + { + pg_log_error("system identifier does not match between base backup and streaming connection"); + PQclear(res); + return false; + } + if (stream->timeline > atoi(PQgetvalue(res, 0, 1))) + { + pg_log_error("starting timeline %u is not present in the server", + stream->timeline); + PQclear(res); + return false; + } + PQclear(res); + } + + /* + * initialize flush position to starting point, it's the caller's + * responsibility that that's sane. + */ + lastFlushPosition = stream->startpos; + stream->currentpos = 0; + + while (1) + { + /* + * Fetch the timeline history file for this timeline, if we don't have + * it already. When streaming log to tar, this will always return + * false, as we are never streaming into an existing file and + * therefore there can be no pre-existing timeline history file. + */ + if (!existsTimeLineHistoryFile(stream)) + { + snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", stream->timeline); + res = PQexec(conn, query); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + /* FIXME: we might send it ok, but get an error */ + pg_log_error("could not send replication command \"%s\": %s", + "TIMELINE_HISTORY", PQresultErrorMessage(res)); + PQclear(res); + return false; + } + + /* + * The response to TIMELINE_HISTORY is a single row result set + * with two fields: filename and content + */ + if (PQnfields(res) != 2 || PQntuples(res) != 1) + { + pg_log_warning("unexpected response to TIMELINE_HISTORY command: got %d rows and %d fields, expected %d rows and %d fields", + PQntuples(res), PQnfields(res), 1, 2); + } + + /* Write the history file to disk */ + writeTimeLineHistoryFile(stream, + PQgetvalue(res, 0, 0), + PQgetvalue(res, 0, 1)); + + PQclear(res); + } + + /* + * Before we start streaming from the requested location, check if the + * callback tells us to stop here. + */ + if (stream->stream_stop(stream->startpos, stream->timeline, false)) + return true; + + /* Initiate the replication stream at specified location */ + snprintf(query, sizeof(query), "START_REPLICATION %s%X/%X TIMELINE %u", + slotcmd, + LSN_FORMAT_ARGS(stream->startpos), + stream->timeline); + res = PQexec(conn, query); + if (PQresultStatus(res) != PGRES_COPY_BOTH) + { + pg_log_error("could not send replication command \"%s\": %s", + "START_REPLICATION", PQresultErrorMessage(res)); + PQclear(res); + return false; + } + PQclear(res); + + /* Stream the WAL */ + res = HandleCopyStream(conn, stream, &stoppos); + if (res == NULL) + goto error; + + /* + * Streaming finished. + * + * There are two possible reasons for that: a controlled shutdown, or + * we reached the end of the current timeline. In case of + * end-of-timeline, the server sends a result set after Copy has + * finished, containing information about the next timeline. Read + * that, and restart streaming from the next timeline. In case of + * controlled shutdown, stop here. + */ + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + /* + * End-of-timeline. Read the next timeline's ID and starting + * position. Usually, the starting position will match the end of + * the previous timeline, but there are corner cases like if the + * server had sent us half of a WAL record, when it was promoted. + * The new timeline will begin at the end of the last complete + * record in that case, overlapping the partial WAL record on the + * old timeline. + */ + uint32 newtimeline; + bool parsed; + + parsed = ReadEndOfStreamingResult(res, &stream->startpos, &newtimeline); + PQclear(res); + if (!parsed) + goto error; + + /* Sanity check the values the server gave us */ + if (newtimeline <= stream->timeline) + { + pg_log_error("server reported unexpected next timeline %u, following timeline %u", + newtimeline, stream->timeline); + goto error; + } + if (stream->startpos > stoppos) + { + pg_log_error("server stopped streaming timeline %u at %X/%X, but reported next timeline %u to begin at %X/%X", + stream->timeline, LSN_FORMAT_ARGS(stoppos), + newtimeline, LSN_FORMAT_ARGS(stream->startpos)); + goto error; + } + + /* Read the final result, which should be CommandComplete. */ + res = PQgetResult(conn); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + pg_log_error("unexpected termination of replication stream: %s", + PQresultErrorMessage(res)); + PQclear(res); + goto error; + } + PQclear(res); + + /* + * Loop back to start streaming from the new timeline. Always + * start streaming at the beginning of a segment. + */ + stream->timeline = newtimeline; + stream->startpos = stream->startpos - + XLogSegmentOffset(stream->startpos, WalSegSz); + continue; + } + else if (PQresultStatus(res) == PGRES_COMMAND_OK) + { + PQclear(res); + + /* + * End of replication (ie. controlled shut down of the server). + * + * Check if the callback thinks it's OK to stop here. If not, + * complain. + */ + if (stream->stream_stop(stoppos, stream->timeline, false)) + return true; + else + { + pg_log_error("replication stream was terminated before stop point"); + goto error; + } + } + else + { + /* Server returned an error. */ + pg_log_error("unexpected termination of replication stream: %s", + PQresultErrorMessage(res)); + PQclear(res); + goto error; + } + } + +error: + if (walfile != NULL && stream->walmethod->close(walfile, CLOSE_NO_RENAME) != 0) + pg_log_error("could not close file \"%s\": %s", + current_walfile_name, stream->walmethod->getlasterror()); + walfile = NULL; + return false; +} + +/* + * Helper function to parse the result set returned by server after streaming + * has finished. On failure, prints an error to stderr and returns false. + */ +static bool +ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos, uint32 *timeline) +{ + uint32 startpos_xlogid, + startpos_xrecoff; + + /*---------- + * The result set consists of one row and two columns, e.g: + * + * next_tli | next_tli_startpos + * ----------+------------------- + * 4 | 0/9949AE0 + * + * next_tli is the timeline ID of the next timeline after the one that + * just finished streaming. next_tli_startpos is the WAL location where + * the server switched to it. + *---------- + */ + if (PQnfields(res) < 2 || PQntuples(res) != 1) + { + pg_log_error("unexpected result set after end-of-timeline: got %d rows and %d fields, expected %d rows and %d fields", + PQntuples(res), PQnfields(res), 1, 2); + return false; + } + + *timeline = atoi(PQgetvalue(res, 0, 0)); + if (sscanf(PQgetvalue(res, 0, 1), "%X/%X", &startpos_xlogid, + &startpos_xrecoff) != 2) + { + pg_log_error("could not parse next timeline's starting point \"%s\"", + PQgetvalue(res, 0, 1)); + return false; + } + *startpos = ((uint64) startpos_xlogid << 32) | startpos_xrecoff; + + return true; +} + +/* + * The main loop of ReceiveXlogStream. Handles the COPY stream after + * initiating streaming with the START_REPLICATION command. + * + * If the COPY ends (not necessarily successfully) due a message from the + * server, returns a PGresult and sets *stoppos to the last byte written. + * On any other sort of error, returns NULL. + */ +static PGresult * +HandleCopyStream(PGconn *conn, StreamCtl *stream, + XLogRecPtr *stoppos) +{ + char *copybuf = NULL; + TimestampTz last_status = -1; + XLogRecPtr blockpos = stream->startpos; + + still_sending = true; + + while (1) + { + int r; + TimestampTz now; + long sleeptime; + + /* + * Check if we should continue streaming, or abort at this point. + */ + if (!CheckCopyStreamStop(conn, stream, blockpos)) + goto error; + + now = feGetCurrentTimestamp(); + + /* + * Potentially send a status message to the primary + */ + if (still_sending && stream->standby_message_timeout > 0 && + feTimestampDifferenceExceeds(last_status, now, + stream->standby_message_timeout)) + { + /* Time to send feedback! */ + if (!sendFeedback(conn, blockpos, now, false)) + goto error; + last_status = now; + } + + /* + * Calculate how long send/receive loops should sleep + */ + sleeptime = CalculateCopyStreamSleeptime(now, stream->standby_message_timeout, + last_status); + + r = CopyStreamReceive(conn, sleeptime, stream->stop_socket, ©buf); + while (r != 0) + { + if (r == -1) + goto error; + if (r == -2) + { + PGresult *res = HandleEndOfCopyStream(conn, stream, copybuf, blockpos, stoppos); + + if (res == NULL) + goto error; + else + return res; + } + + /* Check the message type. */ + if (copybuf[0] == 'k') + { + if (!ProcessKeepaliveMsg(conn, stream, copybuf, r, blockpos, + &last_status)) + goto error; + } + else if (copybuf[0] == 'w') + { + if (!ProcessXLogDataMsg(conn, stream, copybuf, r, &blockpos)) + goto error; + + stream->currentpos = blockpos; + + /* + * Check if we should continue streaming, or abort at this + * point. + */ + if (!CheckCopyStreamStop(conn, stream, blockpos)) + goto error; + } + else + { + pg_log_error("unrecognized streaming header: \"%c\"", + copybuf[0]); + goto error; + } + + /* + * Process the received data, and any subsequent data we can read + * without blocking. + */ + r = CopyStreamReceive(conn, 0, stream->stop_socket, ©buf); + } + } + +error: + if (copybuf != NULL) + PQfreemem(copybuf); + return NULL; +} + +/* + * Wait until we can read a CopyData message, + * or timeout, or occurrence of a signal or input on the stop_socket. + * (timeout_ms < 0 means wait indefinitely; 0 means don't wait.) + * + * Returns 1 if data has become available for reading, 0 if timed out + * or interrupted by signal or stop_socket input, and -1 on an error. + */ +static int +CopyStreamPoll(PGconn *conn, long timeout_ms, pgsocket stop_socket) +{ + int ret; + fd_set input_mask; + int connsocket; + int maxfd; + struct timeval timeout; + struct timeval *timeoutptr; + + connsocket = PQsocket(conn); + if (connsocket < 0) + { + pg_log_error("invalid socket: %s", PQerrorMessage(conn)); + return -1; + } + + FD_ZERO(&input_mask); + FD_SET(connsocket, &input_mask); + maxfd = connsocket; + if (stop_socket != PGINVALID_SOCKET) + { + FD_SET(stop_socket, &input_mask); + maxfd = Max(maxfd, stop_socket); + } + + if (timeout_ms < 0) + timeoutptr = NULL; + else + { + timeout.tv_sec = timeout_ms / 1000L; + timeout.tv_usec = (timeout_ms % 1000L) * 1000L; + timeoutptr = &timeout; + } + + ret = select(maxfd + 1, &input_mask, NULL, NULL, timeoutptr); + + if (ret < 0) + { + if (errno == EINTR) + return 0; /* Got a signal, so not an error */ + pg_log_error("%s() failed: %m", "select"); + return -1; + } + if (ret > 0 && FD_ISSET(connsocket, &input_mask)) + return 1; /* Got input on connection socket */ + + return 0; /* Got timeout or input on stop_socket */ +} + +/* + * Receive CopyData message available from XLOG stream, blocking for + * maximum of 'timeout' ms. + * + * If data was received, returns the length of the data. *buffer is set to + * point to a buffer holding the received message. The buffer is only valid + * until the next CopyStreamReceive call. + * + * Returns 0 if no data was available within timeout, or if wait was + * interrupted by signal or stop_socket input. + * -1 on error. -2 if the server ended the COPY. + */ +static int +CopyStreamReceive(PGconn *conn, long timeout, pgsocket stop_socket, + char **buffer) +{ + char *copybuf = NULL; + int rawlen; + + if (*buffer != NULL) + PQfreemem(*buffer); + *buffer = NULL; + + /* Try to receive a CopyData message */ + rawlen = PQgetCopyData(conn, ©buf, 1); + if (rawlen == 0) + { + int ret; + + /* + * No data available. Wait for some to appear, but not longer than + * the specified timeout, so that we can ping the server. Also stop + * waiting if input appears on stop_socket. + */ + ret = CopyStreamPoll(conn, timeout, stop_socket); + if (ret <= 0) + return ret; + + /* Now there is actually data on the socket */ + if (PQconsumeInput(conn) == 0) + { + pg_log_error("could not receive data from WAL stream: %s", + PQerrorMessage(conn)); + return -1; + } + + /* Now that we've consumed some input, try again */ + rawlen = PQgetCopyData(conn, ©buf, 1); + if (rawlen == 0) + return 0; + } + if (rawlen == -1) /* end-of-streaming or error */ + return -2; + if (rawlen == -2) + { + pg_log_error("could not read COPY data: %s", PQerrorMessage(conn)); + return -1; + } + + /* Return received messages to caller */ + *buffer = copybuf; + return rawlen; +} + +/* + * Process the keepalive message. + */ +static bool +ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, + XLogRecPtr blockpos, TimestampTz *last_status) +{ + int pos; + bool replyRequested; + TimestampTz now; + + /* + * Parse the keepalive message, enclosed in the CopyData message. We just + * check if the server requested a reply, and ignore the rest. + */ + pos = 1; /* skip msgtype 'k' */ + pos += 8; /* skip walEnd */ + pos += 8; /* skip sendTime */ + + if (len < pos + 1) + { + pg_log_error("streaming header too small: %d", len); + return false; + } + replyRequested = copybuf[pos]; + + /* If the server requested an immediate reply, send one. */ + if (replyRequested && still_sending) + { + if (reportFlushPosition && lastFlushPosition < blockpos && + walfile != NULL) + { + /* + * If a valid flush location needs to be reported, flush the + * current WAL file so that the latest flush location is sent back + * to the server. This is necessary to see whether the last WAL + * data has been successfully replicated or not, at the normal + * shutdown of the server. + */ + if (stream->walmethod->sync(walfile) != 0) + { + pg_log_fatal("could not fsync file \"%s\": %s", + current_walfile_name, stream->walmethod->getlasterror()); + exit(1); + } + lastFlushPosition = blockpos; + } + + now = feGetCurrentTimestamp(); + if (!sendFeedback(conn, blockpos, now, false)) + return false; + *last_status = now; + } + + return true; +} + +/* + * Process XLogData message. + */ +static bool +ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, + XLogRecPtr *blockpos) +{ + int xlogoff; + int bytes_left; + int bytes_written; + int hdr_len; + + /* + * Once we've decided we don't want to receive any more, just ignore any + * subsequent XLogData messages. + */ + if (!(still_sending)) + return true; + + /* + * Read the header of the XLogData message, enclosed in the CopyData + * message. We only need the WAL location field (dataStart), the rest of + * the header is ignored. + */ + hdr_len = 1; /* msgtype 'w' */ + hdr_len += 8; /* dataStart */ + hdr_len += 8; /* walEnd */ + hdr_len += 8; /* sendTime */ + if (len < hdr_len) + { + pg_log_error("streaming header too small: %d", len); + return false; + } + *blockpos = fe_recvint64(©buf[1]); + + /* Extract WAL location for this block */ + xlogoff = XLogSegmentOffset(*blockpos, WalSegSz); + + /* + * Verify that the initial location in the stream matches where we think + * we are. + */ + if (walfile == NULL) + { + /* No file open yet */ + if (xlogoff != 0) + { + pg_log_error("received write-ahead log record for offset %u with no file open", + xlogoff); + return false; + } + } + else + { + /* More data in existing segment */ + if (stream->walmethod->get_current_pos(walfile) != xlogoff) + { + pg_log_error("got WAL data offset %08x, expected %08x", + xlogoff, (int) stream->walmethod->get_current_pos(walfile)); + return false; + } + } + + bytes_left = len - hdr_len; + bytes_written = 0; + + while (bytes_left) + { + int bytes_to_write; + + /* + * If crossing a WAL boundary, only write up until we reach wal + * segment size. + */ + if (xlogoff + bytes_left > WalSegSz) + bytes_to_write = WalSegSz - xlogoff; + else + bytes_to_write = bytes_left; + + if (walfile == NULL) + { + if (!open_walfile(stream, *blockpos)) + { + /* Error logged by open_walfile */ + return false; + } + } + + if (stream->walmethod->write(walfile, copybuf + hdr_len + bytes_written, + bytes_to_write) != bytes_to_write) + { + pg_log_error("could not write %u bytes to WAL file \"%s\": %s", + bytes_to_write, current_walfile_name, + stream->walmethod->getlasterror()); + return false; + } + + /* Write was successful, advance our position */ + bytes_written += bytes_to_write; + bytes_left -= bytes_to_write; + *blockpos += bytes_to_write; + xlogoff += bytes_to_write; + + /* Did we reach the end of a WAL segment? */ + if (XLogSegmentOffset(*blockpos, WalSegSz) == 0) + { + if (!close_walfile(stream, *blockpos)) + /* Error message written in close_walfile() */ + return false; + + xlogoff = 0; + + if (still_sending && stream->stream_stop(*blockpos, stream->timeline, true)) + { + if (PQputCopyEnd(conn, NULL) <= 0 || PQflush(conn)) + { + pg_log_error("could not send copy-end packet: %s", + PQerrorMessage(conn)); + return false; + } + still_sending = false; + return true; /* ignore the rest of this XLogData packet */ + } + } + } + /* No more data left to write, receive next copy packet */ + + return true; +} + +/* + * Handle end of the copy stream. + */ +static PGresult * +HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf, + XLogRecPtr blockpos, XLogRecPtr *stoppos) +{ + PGresult *res = PQgetResult(conn); + + /* + * The server closed its end of the copy stream. If we haven't closed + * ours already, we need to do so now, unless the server threw an error, + * in which case we don't. + */ + if (still_sending) + { + if (!close_walfile(stream, blockpos)) + { + /* Error message written in close_walfile() */ + PQclear(res); + return NULL; + } + if (PQresultStatus(res) == PGRES_COPY_IN) + { + if (PQputCopyEnd(conn, NULL) <= 0 || PQflush(conn)) + { + pg_log_error("could not send copy-end packet: %s", + PQerrorMessage(conn)); + PQclear(res); + return NULL; + } + res = PQgetResult(conn); + } + still_sending = false; + } + if (copybuf != NULL) + PQfreemem(copybuf); + *stoppos = blockpos; + return res; +} + +/* + * Check if we should continue streaming, or abort at this point. + */ +static bool +CheckCopyStreamStop(PGconn *conn, StreamCtl *stream, XLogRecPtr blockpos) +{ + if (still_sending && stream->stream_stop(blockpos, stream->timeline, false)) + { + if (!close_walfile(stream, blockpos)) + { + /* Potential error message is written by close_walfile */ + return false; + } + if (PQputCopyEnd(conn, NULL) <= 0 || PQflush(conn)) + { + pg_log_error("could not send copy-end packet: %s", + PQerrorMessage(conn)); + return false; + } + still_sending = false; + } + + return true; +} + +/* + * Calculate how long send/receive loops should sleep + */ +static long +CalculateCopyStreamSleeptime(TimestampTz now, int standby_message_timeout, + TimestampTz last_status) +{ + TimestampTz status_targettime = 0; + long sleeptime; + + if (standby_message_timeout && still_sending) + status_targettime = last_status + + (standby_message_timeout - 1) * ((int64) 1000); + + if (status_targettime > 0) + { + long secs; + int usecs; + + feTimestampDifference(now, + status_targettime, + &secs, + &usecs); + /* Always sleep at least 1 sec */ + if (secs <= 0) + { + secs = 1; + usecs = 0; + } + + sleeptime = secs * 1000 + usecs / 1000; + } + else + sleeptime = -1; + + return sleeptime; +} \ No newline at end of file diff --git a/src/compatibility/receivelog.h b/src/compatibility/receivelog.h new file mode 100644 index 000000000..cd5a290c9 --- /dev/null +++ b/src/compatibility/receivelog.h @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------- + * + * receivelog.h + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/receivelog.h + *------------------------------------------------------------------------- + */ + +#ifndef RECEIVELOG_H +#define RECEIVELOG_H + +#include "access/xlogdefs.h" +#include "libpq-fe.h" +#include "walmethods.h" + +/* + * Called before trying to read more data or when a segment is + * finished. Return true to stop streaming. + */ +typedef bool (*stream_stop_callback) (XLogRecPtr segendpos, uint32 timeline, bool segment_finished); + +/* + * Global parameters when receiving xlog stream. For details about the individual fields, + * see the function comment for ReceiveXlogStream(). + */ +typedef struct StreamCtl +{ + XLogRecPtr startpos; /* Start position for streaming */ + volatile XLogRecPtr currentpos; /* current position */ + TimeLineID timeline; /* Timeline to stream data from */ + char *sysidentifier; /* Validate this system identifier and + * timeline */ + int standby_message_timeout; /* Send status messages this often */ + + stream_stop_callback stream_stop; /* Stop streaming when returns true */ + + pgsocket stop_socket; /* if valid, watch for input on this socket + * and check stream_stop() when there is any */ + + WalWriteMethod *walmethod; /* How to write the WAL */ + char *replication_slot; /* Replication slot to use, or NULL */ +} StreamCtl; + + + +extern bool CheckServerVersionForStreaming(PGconn *conn); +extern bool ReceiveXlogStream(PGconn *conn, + StreamCtl *stream); + +#endif /* RECEIVELOG_H */ diff --git a/src/compatibility/streamutil.c b/src/compatibility/streamutil.c new file mode 100644 index 000000000..14268503a --- /dev/null +++ b/src/compatibility/streamutil.c @@ -0,0 +1,282 @@ +/*------------------------------------------------------------------------- + * + * streamutil.c - utility functions for pg_basebackup, pg_receivewal and + * pg_recvlogical + * + * Author: Magnus Hagander + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/streamutil.c + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "postgres_fe.h" + +#include +#include + +#include "common/connect.h" +#include "common/fe_memutils.h" +#include "logging.h" +#include "datatype/timestamp.h" +#include "port/pg_bswap.h" +#include "pqexpbuffer.h" +#include "receivelog.h" +#include "streamutil.h" + + +#define ERRCODE_DUPLICATE_OBJECT "42710" + +uint32 WalSegSz; + +#include "simple_prompt.h" +#include "file_compat.h" + +#include + +/* + * From version 10, explicitly set wal segment size using SHOW wal_segment_size + * since ControlFile is not accessible here. + */ +bool +RetrieveWalSegSize(PGconn *conn) +{ + PGresult *res; + char xlog_unit[3]; + int xlog_val, + multiplier = 1; + + /* check connection existence */ + Assert(conn != NULL); + + res = PQexec(conn, "SHOW wal_segment_size"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + pg_log_error("could not send replication command \"%s\": %s", + "SHOW wal_segment_size", PQerrorMessage(conn)); + + PQclear(res); + return false; + } + if (PQntuples(res) != 1 || PQnfields(res) < 1) + { + pg_log_error("could not fetch WAL segment size: got %d rows and %d fields, expected %d rows and %d or more fields", + PQntuples(res), PQnfields(res), 1, 1); + + PQclear(res); + return false; + } + + /* fetch xlog value and unit from the result */ + if (sscanf(PQgetvalue(res, 0, 0), "%d%2s", &xlog_val, xlog_unit) != 2) + { + pg_log_error("WAL segment size could not be parsed"); + PQclear(res); + return false; + } + + PQclear(res); + xlog_unit[2] = 0; + /* set the multiplier based on unit to convert xlog_val to bytes */ + if (strcmp(xlog_unit, "MB") == 0) + multiplier = 1024 * 1024; + else if (strcmp(xlog_unit, "GB") == 0) + multiplier = 1024 * 1024 * 1024; + + /* convert and set WalSegSz */ + WalSegSz = xlog_val * multiplier; + + if (!IsValidWalSegSize(WalSegSz)) + { + pg_log_error(ngettext("WAL segment size must be a power of two between 1 MB and 1 GB, but the remote server reported a value of %d byte", + "WAL segment size must be a power of two between 1 MB and 1 GB, but the remote server reported a value of %d bytes", + WalSegSz), + WalSegSz); + return false; + } + + return true; +} + + +/* + * Create a replication slot for the given connection. This function + * returns true in case of success. + */ +bool +CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, + bool is_temporary, bool is_physical, bool reserve_wal, + bool slot_exists_ok) +{ + PQExpBuffer query; + PGresult *res; + + query = createPQExpBuffer(); + + Assert((is_physical && plugin == NULL) || + (!is_physical && plugin != NULL)); + Assert(slot_name != NULL); + + /* Build query */ + appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\"", slot_name); + if (is_temporary) + appendPQExpBufferStr(query, " TEMPORARY"); + if (is_physical) + { + appendPQExpBufferStr(query, " PHYSICAL"); + if (reserve_wal) + appendPQExpBufferStr(query, " RESERVE_WAL"); + } + else + { + appendPQExpBuffer(query, " LOGICAL \"%s\"", plugin); + if (PQserverVersion(conn) >= 100000) + /* pg_recvlogical doesn't use an exported snapshot, so suppress */ + appendPQExpBufferStr(query, " NOEXPORT_SNAPSHOT"); + } + + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE); + + if (slot_exists_ok && + sqlstate && + strcmp(sqlstate, ERRCODE_DUPLICATE_OBJECT) == 0) + { + destroyPQExpBuffer(query); + PQclear(res); + return true; + } + else + { + pg_log_error("could not send replication command \"%s\": %s", + query->data, PQerrorMessage(conn)); + + destroyPQExpBuffer(query); + PQclear(res); + return false; + } + } + + if (PQntuples(res) != 1 || PQnfields(res) != 4) + { + pg_log_error("could not create replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields", + slot_name, + PQntuples(res), PQnfields(res), 1, 4); + + destroyPQExpBuffer(query); + PQclear(res); + return false; + } + + destroyPQExpBuffer(query); + PQclear(res); + return true; +} + +/* + * Frontend version of GetCurrentTimestamp(), since we are not linked with + * backend code. + */ +TimestampTz +feGetCurrentTimestamp(void) +{ + TimestampTz result; + struct timeval tp; + + gettimeofday(&tp, NULL); + + result = (TimestampTz) tp.tv_sec - + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY); + result = (result * USECS_PER_SEC) + tp.tv_usec; + + return result; +} + +/* + * Frontend version of TimestampDifference(), since we are not linked with + * backend code. + */ +void +feTimestampDifference(TimestampTz start_time, TimestampTz stop_time, + long *secs, int *microsecs) +{ + TimestampTz diff = stop_time - start_time; + + if (diff <= 0) + { + *secs = 0; + *microsecs = 0; + } + else + { + *secs = (long) (diff / USECS_PER_SEC); + *microsecs = (int) (diff % USECS_PER_SEC); + } +} + +/* + * Frontend version of TimestampDifferenceExceeds(), since we are not + * linked with backend code. + */ +bool +feTimestampDifferenceExceeds(TimestampTz start_time, + TimestampTz stop_time, + int msec) +{ + TimestampTz diff = stop_time - start_time; + + return (diff >= msec * INT64CONST(1000)); +} + +/* + * Converts an int64 to network byte order. + */ + +void +fe_sendint64(int64 i, char *buf) +{ + + uint32 n32; + + /* High order half first, since we're doing MSB-first */ + n32 = (uint32) (i >> 32); + n32 = htonl(n32); + memcpy(&buf[0], &n32, 4); + + /* Now the low order half */ + n32 = (uint32) i; + n32 = htonl(n32); + memcpy(&buf[4], &n32, 4); +} + +/* + * Converts an int64 from network byte order to native format. + */ + +int64 +fe_recvint64(char *buf) +{ + int64 result; + uint32 h32; + uint32 l32; + + memcpy(&h32, buf, 4); + memcpy(&l32, buf + 4, 4); + h32 = ntohl(h32); + l32 = ntohl(l32); + + result = h32; + result <<= 32; + result |= l32; + + return result; + +} + + + diff --git a/src/compatibility/streamutil.h b/src/compatibility/streamutil.h new file mode 100644 index 000000000..3a2feaa39 --- /dev/null +++ b/src/compatibility/streamutil.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * streamutil.h + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/streamutil.h + *------------------------------------------------------------------------- + */ + +#ifndef STREAMUTIL_H +#define STREAMUTIL_H + +#include "access/xlogdefs.h" +#include "datatype/timestamp.h" +#include "libpq-fe.h" + +extern uint32 WalSegSz; + +/* Replication commands */ +extern bool CreateReplicationSlot(PGconn *conn, const char *slot_name, + const char *plugin, bool is_temporary, + bool is_physical, bool reserve_wal, + bool slot_exists_ok); +extern bool RetrieveWalSegSize(PGconn *conn); +extern TimestampTz feGetCurrentTimestamp(void); +extern void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time, + long *secs, int *microsecs); + +extern bool feTimestampDifferenceExceeds(TimestampTz start_time, TimestampTz stop_time, + int msec); +extern void fe_sendint64(int64 i, char *buf); +extern int64 fe_recvint64(char *buf); + +#endif /* STREAMUTIL_H */ diff --git a/src/compatibility/walmethods.c b/src/compatibility/walmethods.c new file mode 100644 index 000000000..e6f9edd0e --- /dev/null +++ b/src/compatibility/walmethods.c @@ -0,0 +1,438 @@ +/*------------------------------------------------------------------------- + * + * walmethods.c - implementations of different ways to write received wal + * + * NOTE! The caller must ensure that only one method is instantiated in + * any given program, and that it's only instantiated once! + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/walmethods.c + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "postgres_fe.h" + +#include +#include +#include +#ifdef HAVE_LIBZ +#include +#endif + +#include "common/file_utils.h" +#include "pgtar.h" +#include "receivelog.h" +#include "streamutil.h" + +/* Size of zlib buffer for .tar.gz */ +#define ZLIB_OUT_SIZE 4096 + +#include "file_compat.h" + +#ifndef unconstify +#define unconstify(underlying_type, expr) \ + ((underlying_type) (expr)) +#endif + +/*------------------------------------------------------------------------- + * WalDirectoryMethod - write wal to a directory looking like pg_wal + *------------------------------------------------------------------------- + */ + +/* + * Global static data for this method + */ +typedef struct DirectoryMethodData +{ + char *basedir; + int compression; + bool sync; + const char *lasterrstring; /* if set, takes precedence over lasterrno */ + int lasterrno; + pioDrive_i drive; +} DirectoryMethodData; +static DirectoryMethodData *dir_data = NULL; + +/* + * Local file handle + */ +typedef struct DirectoryMethodFile +{ + pioWriteCloser_i fd; + off_t currpos; + char *pathname; + char *fullpath; +#ifdef HAVE_LIBZ + gzFile gzfp; +#endif +} DirectoryMethodFile; + +#define dir_clear_error() \ + (dir_data->lasterrstring = NULL, dir_data->lasterrno = 0) +#define dir_set_error(msg) \ + (dir_data->lasterrstring = _(msg)) + +static const char * +dir_getlasterror(void) +{ + if (dir_data->lasterrstring) + return dir_data->lasterrstring; + return strerror(dir_data->lasterrno); +} + +static char * +dir_get_file_name(const char *pathname) +{ + char *filename = pg_malloc0(MAXPGPATH * sizeof(char)); + + snprintf(filename, MAXPGPATH, "%s%s", + pathname, dir_data->compression > 0 ? ".gz" : ""); + + return filename; +} + +static Walfile +dir_open_for_write(const char *pathname, bool use_temp) +{ + FOBJ_FUNC_ARP(); + char tmppath[MAXPGPATH]; + char *filename; + pioWriteCloser_i fd; + DirectoryMethodFile *f; + err_i err = $noerr(); + +#ifdef HAVE_LIBZ + gzFile gzfp = NULL; +#endif + + dir_clear_error(); + + filename = dir_get_file_name(pathname); + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, filename); + pg_free(filename); + + /* + * Open a file for non-compressed as well as compressed files. Tracking + * the file descriptor is important for dir_sync() method as gzflush() + * does not do any system calls to fsync() to make changes permanent on + * disk. + */ + fd = $i(pioOpenRewrite, dir_data->drive, tmppath, .err = &err, .use_temp = use_temp); + if ($haserr(err)) + { + dir_data->lasterrno = getErrno(err); + return NULL; + } + +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + { + /* vvs + gzfp = gzdopen(fd, "wb"); + if (gzfp == NULL) + { + dir_data->lasterrno = errno; + close(fd); + return NULL; + } + + if (gzsetparams(gzfp, dir_data->compression, + Z_DEFAULT_STRATEGY) != Z_OK) + { + dir_data->lasterrno = errno; + gzclose(gzfp); + return NULL; + } + */ + } +#endif + + /* + * fsync WAL file and containing directory, to ensure the file is + * persistently created and zeroed (if padded). That's particularly + * important when using synchronous mode, where the file is modified and + * fsynced in-place, without a directory fsync. + */ + if (dir_data->sync) + { + err = $i(pioWriteFinish, fd); + + if ($haserr(err)) + { + dir_data->lasterrno =getErrno(err); + $i(pioClose, fd); + return NULL; + } + /* vvs + if (fsync_fname_compat(tmppath, false) != 0 || + fsync_parent_path_compat(tmppath) != 0) + { + dir_data->lasterrno = errno; +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + gzclose(gzfp); + else +#endif + close(fd); + return NULL; + } + */ + } + + f = pg_malloc0(sizeof(DirectoryMethodFile)); +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + f->gzfp = gzfp; +#endif + f->fd = $iref(fd); + f->currpos = 0; + f->pathname = pg_strdup(pathname); + f->fullpath = pg_strdup(tmppath); + + return f; +} + +static ssize_t +dir_write(Walfile f, const void *buf, size_t count) +{ + FOBJ_FUNC_ARP(); + ssize_t r = 0; + DirectoryMethodFile *df = (DirectoryMethodFile *) f; + + Assert(f != NULL); + dir_clear_error(); + + ft_bytes_t fBuf; + err_i err = $noerr(); +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + { + /* vvs + errno = 0; + r = (ssize_t) gzwrite(df->gzfp, buf, count); + if (r != count) + { + dir_data->lasterrno = errno ? errno : ENOSPC; + } + */ + } + else +#endif + { + errno = 0; + fBuf = ft_bytes((void *)buf, count); + err = $i(pioWrite, df->fd, fBuf); + if ($haserr(err)) + { + dir_data->lasterrno = getErrno(err) ? getErrno(err) : ENOSPC; + } + else r = count; + + + } + if (r > 0) + df->currpos += r; + return r; +} + +static off_t +dir_get_current_pos(Walfile f) +{ + Assert(f != NULL); + dir_clear_error(); + + /* Use a cached value to prevent lots of reseeks */ + return ((DirectoryMethodFile *) f)->currpos; +} + +static int +dir_close(Walfile f, WalCloseMethod method) +{ + int r = 0; + DirectoryMethodFile *df = (DirectoryMethodFile *) f; + char tmppath[MAXPGPATH]; + err_i err = $noerr(); + + Assert(f != NULL); + dir_clear_error(); + +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + { + errno = 0; /* in case gzclose() doesn't set it */ + r = gzclose(df->gzfp); + } + else +#endif + err = $i(pioClose, df->fd); + if ($haserr(err)) + { + dir_data->lasterrno = getErrno(err); + r = -1; + } + else if (method == CLOSE_UNLINK) + { + char *filename; + + /* Unlink the file once it's closed */ + filename = dir_get_file_name(df->pathname); + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, filename); + pg_free(filename); + err = $i(pioRemove, dir_data->drive, tmppath, .missing_ok = false); + } + + if ($haserr(err)){ + dir_data->lasterrno = getErrno(err); + r = -1; + } + + pg_free(df->pathname); + pg_free(df->fullpath); + $idel(&df->fd); + pg_free(df); + + return r; +} + +static int +dir_sync(Walfile f) +{ + err_i err = $noerr(); + Assert(f != NULL); + dir_clear_error(); + + if (!dir_data->sync) + return 0; + +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + { + if (gzflush(((DirectoryMethodFile *) f)->gzfp, Z_SYNC_FLUSH) != Z_OK) + { + dir_data->lasterrno = errno; + return -1; + } + } +#endif + + err = $i(pioWriteFinish, ((DirectoryMethodFile *) f)->fd); + if ($haserr(err)) + { + dir_data->lasterrno = getErrno(err); + return -1; + } + return 0; +} + +static ssize_t +dir_get_file_size(const char *pathname) +{ + pio_stat_t statbuf; + char tmppath[MAXPGPATH]; + err_i err = $noerr(); + + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, pathname); + + statbuf = $i(pioStat, dir_data->drive, .path = pathname, + .follow_symlink = false, .err = &err); + if ($haserr(err)) + { + dir_data->lasterrno = getErrno(err); + return -1; + } + + return statbuf.pst_size; +} + +static int +dir_compression(void) +{ + return dir_data->compression; +} + +static bool +dir_existsfile(const char *pathname) +{ + char tmppath[MAXPGPATH]; + bool ret; + + err_i err = $noerr(); + + dir_clear_error(); + + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, pathname); + + ret = $i(pioExists, dir_data->drive, .path = tmppath, .err = &err); + if ($haserr(err)) + { + dir_data->lasterrno = getErrno(err); + } + + return ret; + +} + +static bool +dir_finish(void) +{ + dir_clear_error(); + + if (dir_data->sync) + { + /* + * Files are fsynced when they are closed, but we need to fsync the + * directory entry here as well. + */ + /* vvs temp + if (fsync_fname_compat(dir_data->basedir, true) != 0) + { + dir_data->lasterrno = errno; + return false; + } + */ + } + return true; +} + + +WalWriteMethod * +CreateWalDirectoryMethod(const char *basedir, int compression, bool sync, pioDrive_i drive) +{ + WalWriteMethod *method; + + method = pg_malloc0(sizeof(WalWriteMethod)); + method->open_for_write = dir_open_for_write; + method->write = dir_write; + method->get_current_pos = dir_get_current_pos; + method->get_file_size = dir_get_file_size; + method->get_file_name = dir_get_file_name; + method->compression = dir_compression; + method->close = dir_close; + method->sync = dir_sync; + method->existsfile = dir_existsfile; + method->finish = dir_finish; + method->getlasterror = dir_getlasterror; + + dir_data = pg_malloc0(sizeof(DirectoryMethodData)); + dir_data->compression = compression; + dir_data->basedir = pg_strdup(basedir); + dir_data->sync = sync; + dir_data->drive = drive; + + return method; +} + +void +FreeWalDirectoryMethod(void) +{ + pg_free(dir_data->basedir); + pg_free(dir_data); + dir_data = NULL; +} diff --git a/src/compatibility/walmethods.h b/src/compatibility/walmethods.h new file mode 100644 index 000000000..3aab03ec1 --- /dev/null +++ b/src/compatibility/walmethods.h @@ -0,0 +1,102 @@ +/*------------------------------------------------------------------------- + * + * walmethods.h + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/walmethods.h + *------------------------------------------------------------------------- + */ + + +typedef void *Walfile; + +typedef enum +{ + CLOSE_NORMAL, + CLOSE_UNLINK, + CLOSE_NO_RENAME +} WalCloseMethod; + +/* + * A WalWriteMethod structure represents the different methods used + * to write the streaming WAL as it's received. + * + * All methods that have a failure return indicator will set state + * allowing the getlasterror() method to return a suitable message. + * Commonly, errno is this state (or part of it); so callers must take + * care not to clobber errno between a failed method call and use of + * getlasterror() to retrieve the message. + */ +typedef struct WalWriteMethod WalWriteMethod; +struct WalWriteMethod +{ + /* + * Open a target file. Returns Walfile, or NULL if open failed. If a temp + * suffix is specified, a file with that name will be opened, and then + * automatically renamed in close(). If pad_to_size is specified, the file + * will be padded with NUL up to that size, if supported by the Walmethod. + */ + Walfile (*open_for_write) (const char *pathname, bool use_temp); + + /* + * Close an open Walfile, using one or more methods for handling automatic + * unlinking etc. Returns 0 on success, other values for error. + */ + int (*close) (Walfile f, WalCloseMethod method); + + /* Check if a file exist */ + bool (*existsfile) (const char *pathname); + + /* Return the size of a file, or -1 on failure. */ + ssize_t (*get_file_size) (const char *pathname); + + /* + * Return the name of the current file to work on in pg_malloc()'d string, + * without the base directory. This is useful for logging. + */ + char *(*get_file_name) (const char *pathname); + + /* Return the level of compression */ + int (*compression) (void); + + /* + * Write count number of bytes to the file, and return the number of bytes + * actually written or -1 for error. + */ + ssize_t (*write) (Walfile f, const void *buf, size_t count); + + /* Return the current position in a file or -1 on error */ + off_t (*get_current_pos) (Walfile f); + + /* + * fsync the contents of the specified file. Returns 0 on success. + */ + int (*sync) (Walfile f); + + /* + * Clean up the Walmethod, closing any shared resources. For methods like + * tar, this includes writing updated headers. Returns true if the + * close/write/sync of shared resources succeeded, otherwise returns false + * (but the resources are still closed). + */ + bool (*finish) (void); + + /* Return a text for the last error in this Walfile */ + const char *(*getlasterror) (void); + +}; + +/* + * Available WAL methods: + * - WalDirectoryMethod - write WAL to regular files in a standard pg_wal + * - WalTarMethod - write WAL to a tarfile corresponding to pg_wal + * (only implements the methods required for pg_basebackup, + * not all those required for pg_receivewal) + */ +WalWriteMethod *CreateWalDirectoryMethod(const char *basedir, + int compression, bool sync, pioDrive_i drive); + +/* Cleanup routines for previously-created methods */ +void FreeWalDirectoryMethod(void); diff --git a/src/configure.c b/src/configure.c index f7befb0c5..5d328ab20 100644 --- a/src/configure.c +++ b/src/configure.c @@ -2,7 +2,7 @@ * * configure.c: - manage backup catalog. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -38,13 +38,15 @@ static void show_configure_json(ConfigOption *opt); #define OPTION_INSTANCE_GROUP "Backup instance information" #define OPTION_CONN_GROUP "Connection parameters" -#define OPTION_REPLICA_GROUP "Replica parameters" #define OPTION_ARCHIVE_GROUP "Archive parameters" #define OPTION_LOG_GROUP "Logging parameters" #define OPTION_RETENTION_GROUP "Retention parameters" #define OPTION_COMPRESS_GROUP "Compression parameters" #define OPTION_REMOTE_GROUP "Remote access parameters" +/* dummy placeholder for obsolete options to store in following instance_options[] */ +static char *obsolete_option_placeholder = NULL; + /* * Short name should be non-printable ASCII character. */ @@ -94,31 +96,26 @@ ConfigOption instance_options[] = &instance_config.conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, - /* Replica options */ + /* Obsolete options */ { 's', 202, "master-db", - &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value + &obsolete_option_placeholder, SOURCE_FILE_STRICT, SOURCE_CONST, "", 0, option_get_value }, { 's', 203, "master-host", - &instance_config.master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value + &obsolete_option_placeholder, SOURCE_FILE_STRICT, SOURCE_CONST, "", 0, option_get_value }, { 's', 204, "master-port", - &instance_config.master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value + &obsolete_option_placeholder, SOURCE_FILE_STRICT, SOURCE_CONST, "", 0, option_get_value }, { 's', 205, "master-user", - &instance_config.master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value + &obsolete_option_placeholder, SOURCE_FILE_STRICT, SOURCE_CONST, "", 0, option_get_value }, { - 'u', 206, "replica-timeout", - &instance_config.replica_timeout, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, OPTION_UNIT_S, option_get_value + 's', 206, "replica-timeout", + &obsolete_option_placeholder, SOURCE_FILE_STRICT, SOURCE_CONST, "", 0, option_get_value }, /* Archive options */ { @@ -286,32 +283,48 @@ do_show_config(void) show_configure_end(); } +/* + * Get configuration from configuration file. + * Return number of parsed options. + */ +int +config_read_opt(pioDrive_i drive, const char *path, ConfigOption options[], int elevel, + bool strict, err_i *err) +{ + int parsed_options = 0; + ft_bytes_t config_file = {0}; + fobj_reset_err(err); + + if (!options) + return parsed_options; + + config_file = $i(pioReadFile, drive, .path = path, .binary = false, + .err = err); + if ($haserr(*err)) + return 0; + + parsed_options = config_parse_opt(config_file, path, options, elevel, strict); + + ft_bytes_free(&config_file); + + return parsed_options; +} + /* * Save configure options into BACKUP_CATALOG_CONF_FILE. Do not save default * values into the file. */ void -do_set_config(InstanceState *instanceState, bool missing_ok) +do_set_config(InstanceState *instanceState) { - char path_temp[MAXPGPATH]; - FILE *fp; + ft_strbuf_t buf = ft_strbuf_zero(); + err_i err = $noerr(); int i; - snprintf(path_temp, sizeof(path_temp), "%s.tmp", instanceState->instance_config_path); - - if (!missing_ok && !fileExists(instanceState->instance_config_path, FIO_LOCAL_HOST)) - elog(ERROR, "Configuration file \"%s\" doesn't exist", instanceState->instance_config_path); - - fp = fopen(path_temp, "wt"); - if (fp == NULL) - elog(ERROR, "Cannot create configuration file \"%s\": %s", - BACKUP_CATALOG_CONF_FILE, strerror(errno)); - current_group = NULL; for (i = 0; instance_options[i].type; i++) { - int rc = 0; ConfigOption *opt = &instance_options[i]; char *value; @@ -328,37 +341,26 @@ do_set_config(InstanceState *instanceState, bool missing_ok) if (current_group == NULL || strcmp(opt->group, current_group) != 0) { current_group = opt->group; - fprintf(fp, "# %s\n", current_group); + ft_strbuf_catf(&buf, "# %s\n", current_group); } if (strchr(value, ' ')) - rc = fprintf(fp, "%s = '%s'\n", opt->lname, value); + ft_strbuf_catf(&buf, "%s = '%s'\n", opt->lname, value); else - rc = fprintf(fp, "%s = %s\n", opt->lname, value); - - if (rc < 0) - elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp); + ft_strbuf_catf(&buf, "%s = %s\n", opt->lname, value); pfree(value); } - if (ferror(fp) || fflush(fp)) - elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp); - - if (fclose(fp)) - elog(ERROR, "Cannot close configuration file: \"%s\"", path_temp); + err = $i(pioWriteFile, instanceState->backup_location, + .path = instanceState->instance_config_path, + .content = ft_bytes(buf.ptr, buf.len), + .binary = false); - if (fio_sync(path_temp, FIO_LOCAL_HOST) != 0) - elog(ERROR, "Failed to sync temp configuration file \"%s\": %s", - path_temp, strerror(errno)); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Writting configuration file"); - if (rename(path_temp, instanceState->instance_config_path) < 0) - { - int errno_temp = errno; - unlink(path_temp); - elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", - path_temp, instanceState->instance_config_path, strerror(errno_temp)); - } + ft_strbuf_free(&buf); } void @@ -376,8 +378,6 @@ init_config(InstanceConfig *config, const char *instance_name) config->xlog_seg_size = XLOG_SEG_SIZE; #endif - config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; - config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; /* Copy logger defaults */ @@ -406,6 +406,7 @@ readInstanceConfigFile(InstanceState *instanceState) char *log_format_file = NULL; char *compress_alg = NULL; int parsed_options; + err_i err; ConfigOption instance_options[] = { @@ -453,32 +454,6 @@ readInstanceConfigFile(InstanceState *instanceState) &instance->conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, - /* Replica options */ - { - 's', 202, "master-db", - &instance->master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 203, "master-host", - &instance->master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 204, "master-port", - &instance->master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 205, "master-user", - &instance->master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 'u', 206, "replica-timeout", - &instance->replica_timeout, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, OPTION_UNIT_S, option_get_value - }, /* Archive options */ { 'u', 207, "archive-timeout", @@ -627,16 +602,17 @@ readInstanceConfigFile(InstanceState *instanceState) init_config(instance, instanceState->instance_name); - - if (fio_access(instanceState->instance_config_path, F_OK, FIO_BACKUP_HOST) != 0) + parsed_options = config_read_opt(instanceState->backup_location, + instanceState->instance_config_path, + instance_options, WARNING, true, &err); + if (getErrno(err) == ENOENT) { elog(WARNING, "Control file \"%s\" doesn't exist", instanceState->instance_config_path); pfree(instance); return NULL; } - - parsed_options = config_read_opt(instanceState->instance_config_path, - instance_options, WARNING, true, true); + else if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Control file"); if (parsed_options == 0) { diff --git a/src/data.c b/src/data.c index 490faf9b6..8a24ade40 100644 --- a/src/data.c +++ b/src/data.c @@ -16,7 +16,6 @@ #include "utils/file.h" #include -#include #ifdef HAVE_LIBZ #include @@ -24,6 +23,9 @@ #include "utils/thread.h" +/* for crc32_compat macros */ +#include "compatibility/pg-11.h" + /* Union to ease operations on relation pages */ typedef struct DataPage { @@ -31,8 +33,52 @@ typedef struct DataPage char data[BLCKSZ]; } DataPage; -static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader *bph, - pg_crc32 *crc, bool use_crc32c); +typedef struct backup_page_iterator { + /* arguments */ + const char* fullpath; + pioReader_i in; + BackupPageHeader2 *headers; + int n_headers; + uint32_t backup_version; + CompressAlg compress_alg; + + /* iterator value */ + int64_t cur_pos; + int64_t read_pos; + BlockNumber blknum; + XLogRecPtr page_lsn; + uint16_t page_crc; + uint32_t n_hdr; + bool truncated; + bool is_compressed; + ft_bytes_t whole_read; + ft_bytes_t read_to; + ft_bytes_t compressed; + DataPage page; +} backup_page_iterator; + +static err_i send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, + uint32 checksum_version, + BackupPageHeader2 **headers, BackupMode backup_mode, bool sync); + +static err_i copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr sync_lsn, uint32 checksum_version, + BackupMode backup_mode); + +static size_t restore_data_file_internal(pioReader_i in, pioDBWriter_i out, pgFile *file, uint32 backup_version, + const char *from_fullpath, const char *to_fullpath, int nblocks, + datapagemap_t *map, PageState *checksum_map, int checksum_version, + datapagemap_t *lsn_map, BackupPageHeader2 *headers); + +static void backup_non_data_file_internal(pioDrive_i drive_from, pioDrive_i drive_to, + const char *from_fullpath, + const char *to_fullpath, pgFile *file, + bool missing_ok, bool sync); + +static err_i send_file(pioDrive_i drive_from, pioDrive_i drive_to, + const char *to_fullpath, const char *from_path, + bool cut_zero_tail, pgFile *file, bool sync); #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ @@ -135,7 +181,7 @@ do_decompress(void *dst, size_t dst_size, void const *src, size_t src_size, * But at least we will do this check only for pages which will no pass validation step. */ static bool -page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) +page_may_be_compressed(Page page, CompressAlg alg) { PageHeader phdr; @@ -151,12 +197,6 @@ page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) phdr->pd_special <= BLCKSZ && phdr->pd_special == MAXALIGN(phdr->pd_special))) { - /* ... end only if it is invalid, then do more checks */ - if (backup_version >= 20023) - { - /* Versions 2.0.23 and higher don't have such bug */ - return false; - } #ifdef HAVE_LIBZ /* For zlib we can check page magic: * https://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like @@ -205,12 +245,12 @@ get_header_errormsg(Page page, char **errormsg) if (PageGetPageSize(phdr) != BLCKSZ) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "page size %lu is not equal to block size %u", + "page size %zu is not equal to block size %u", PageGetPageSize(phdr), BLCKSZ); else if (phdr->pd_lower < SizeOfPageHeaderData) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_lower %i is less than page header size %lu", + "pd_lower %i is less than page header size %zu", phdr->pd_lower, SizeOfPageHeaderData); else if (phdr->pd_lower > phdr->pd_upper) @@ -230,7 +270,7 @@ get_header_errormsg(Page page, char **errormsg) else if (phdr->pd_special != MAXALIGN(phdr->pd_special)) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_special %i is misaligned, expected %lu", + "pd_special %i is misaligned, expected %zu", phdr->pd_special, MAXALIGN(phdr->pd_special)); else if (phdr->pd_flags & ~PD_VALID_FLAG_BITS) @@ -257,217 +297,72 @@ get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) pg_checksum_page(page, absolute_blkno)); } -/* - * Retrieves a page taking the backup mode into account - * and writes it into argument "page". Argument "page" - * should be a pointer to allocated BLCKSZ of bytes. - * - * Prints appropriate warnings/errors/etc into log. - * Returns: - * PageIsOk(0) if page was successfully retrieved - * PageIsTruncated(-1) if the page was truncated - * SkipCurrentPage(-2) if we need to skip this page, - * only used for DELTA and PTRACK backup - * PageIsCorrupted(-3) if the page checksum mismatch - * or header corruption, - * only used for checkdb - * TODO: probably we should always - * return it to the caller - */ -static int32 -prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, - BlockNumber blknum, FILE *in, - BackupMode backup_mode, - Page page, bool strict, - uint32 checksum_version, - const char *from_fullpath, - PageState *page_st) -{ - int try_again = PAGE_READ_ATTEMPTS; - bool page_is_valid = false; - BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; - int rc = 0; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during page reading"); - - /* - * Read the page and verify its header and checksum. - * Under high write load it's possible that we've read partly - * flushed page, so try several times before throwing an error. - */ - while (!page_is_valid && try_again--) - { - /* read the block */ - int read_len = fio_pread(in, page, blknum * BLCKSZ); - - /* The block could have been truncated. It is fine. */ - if (read_len == 0) - { - elog(VERBOSE, "Cannot read block %u of \"%s\": " - "block truncated", blknum, from_fullpath); - return PageIsTruncated; - } - else if (read_len < 0) - elog(ERROR, "Cannot read block %u of \"%s\": %s", - blknum, from_fullpath, strerror(errno)); - else if (read_len != BLCKSZ) - elog(WARNING, "Cannot read block %u of \"%s\": " - "read %i of %d, try again", - blknum, from_fullpath, read_len, BLCKSZ); - else - { - /* We have BLCKSZ of raw data, validate it */ - rc = validate_one_page(page, absolute_blknum, - InvalidXLogRecPtr, page_st, - checksum_version); - switch (rc) - { - case PAGE_IS_ZEROED: - elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); - return PageIsOk; - - case PAGE_IS_VALID: - /* in DELTA or PTRACK modes we must compare lsn */ - if (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) - page_is_valid = true; - else - return PageIsOk; - break; - - case PAGE_HEADER_IS_INVALID: - elog(VERBOSE, "File: \"%s\" blknum %u have wrong page header, try again", - from_fullpath, blknum); - break; - - case PAGE_CHECKSUM_MISMATCH: - elog(VERBOSE, "File: \"%s\" blknum %u have wrong checksum, try again", - from_fullpath, blknum); - break; - default: - Assert(false); - } - } - /* avoid re-reading once buffered data, flushing on further attempts, see PBCKP-150 */ - fflush(in); - } - - /* - * If page is not valid after PAGE_READ_ATTEMPTS attempts to read it - * throw an error. - */ - if (!page_is_valid) - { - int elevel = ERROR; - char *errormsg = NULL; - - /* Get the details of corruption */ - if (rc == PAGE_HEADER_IS_INVALID) - get_header_errormsg(page, &errormsg); - else if (rc == PAGE_CHECKSUM_MISMATCH) - get_checksum_errormsg(page, &errormsg, - file->segno * RELSEG_SIZE + blknum); - - /* Error out in case of merge or backup without ptrack support; - * issue warning in case of checkdb or backup with ptrack support - */ - if (!strict) - elevel = WARNING; - - if (errormsg) - elog(elevel, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, blknum, errormsg); - else - elog(elevel, "Corruption detected in file \"%s\", block %u", - from_fullpath, blknum); - - pg_free(errormsg); - return PageIsCorrupted; - } - - /* Checkdb not going futher */ - if (!strict) - return PageIsOk; - - /* - * Skip page if page lsn is less than START_LSN of parent backup. - * Nullified pages must be copied by DELTA backup, just to be safe. - */ - if ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev && - page_st->lsn > 0 && - page_st->lsn < prev_backup_start_lsn) - { - elog(VERBOSE, "Skipping blknum %u in file: \"%s\", file->exists_in_prev: %s, page_st->lsn: %X/%X, prev_backup_start_lsn: %X/%X", - blknum, from_fullpath, - file->exists_in_prev ? "true" : "false", - (uint32) (page_st->lsn >> 32), (uint32) page_st->lsn, - (uint32) (prev_backup_start_lsn >> 32), (uint32) prev_backup_start_lsn); - return SkipCurrentPage; - } - - return PageIsOk; -} - -/* split this function in two: compress() and backup() */ -static int -compress_and_backup_page(pgFile *file, BlockNumber blknum, - FILE *in, FILE *out, pg_crc32 *crc, - int page_state, Page page, - CompressAlg calg, int clevel, - const char *from_fullpath, const char *to_fullpath) +int +compress_page(char *write_buffer, size_t buffer_size, BlockNumber blknum, void *page, + CompressAlg calg, int clevel, const char *from_fullpath) { - int compressed_size = 0; - size_t write_buffer_size = 0; - char write_buffer[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ - BackupPageHeader* bph = (BackupPageHeader*)write_buffer; const char *errormsg = NULL; + int compressed_size; /* Compress the page */ - compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), - sizeof(write_buffer) - sizeof(BackupPageHeader), - page, BLCKSZ, calg, clevel, + compressed_size = do_compress(write_buffer, buffer_size, page, BLCKSZ, calg, clevel, &errormsg); /* Something went wrong and errormsg was assigned, throw a warning */ if (compressed_size < 0 && errormsg != NULL) elog(WARNING, "An error occured during compressing block %u of file \"%s\": %s", blknum, from_fullpath, errormsg); - file->compress_alg = calg; /* TODO: wtf? why here? */ - - /* compression didn`t worked */ + /* Compression skip magic part 1: compression didn`t work + * compresssed_size == BLCKSZ is a flag which shows non-compressed state + */ if (compressed_size <= 0 || compressed_size >= BLCKSZ) { /* Do not compress page */ - memcpy(write_buffer + sizeof(BackupPageHeader), page, BLCKSZ); + memcpy(write_buffer, page, BLCKSZ); compressed_size = BLCKSZ; } - bph->block = blknum; - bph->compressed_size = compressed_size; - write_buffer_size = compressed_size + sizeof(BackupPageHeader); - /* Update CRC */ - COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); + return compressed_size; +} + +static size_t +backup_page(pioWrite_i out, BlockNumber blknum, ft_bytes_t page, + const char *to_fullpath, err_i *err) +{ + BackupPageHeader bph; + size_t n; + fobj_reset_err(err); - /* write data page */ - if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size) - elog(ERROR, "File: \"%s\", cannot write at block %u: %s", - to_fullpath, blknum, strerror(errno)); + bph.block = blknum; + bph.compressed_size = page.len; - file->write_size += write_buffer_size; - file->uncompressed_size += BLCKSZ; + *err = $i(pioWrite, out, .buf = ft_bytes(&bph, sizeof(bph))); + if ($haserr(*err)) + return 0; + n = sizeof(bph); - return compressed_size; + /* write data page */ + *err = $i(pioWrite, out, .buf = page); + if ($noerr(*err)) + n += page.len; + return n; } /* Write page as-is. TODO: make it fastpath option in compress_and_backup_page() */ static int -write_page(pgFile *file, FILE *out, Page page) +write_page(pgFile *file, pioDBWriter_i out, int blknum, Page page) { + err_i err = $noerr(); + off_t target = blknum * BLCKSZ; + + err = $i(pioSeek, out, target); + if ($haserr(err)) + ft_logerr(FT_ERROR, $errmsg(err), "write_page"); + /* write data page */ - if (fio_fwrite(out, page, BLCKSZ) != BLCKSZ) - return -1; + err = $i(pioWrite, out, .buf = ft_bytes(page, BLCKSZ)); + if ($haserr(err)) + ft_log(FT_INFO, $errmsg(err), "write_page"); file->write_size += BLCKSZ; file->uncompressed_size += BLCKSZ; @@ -487,14 +382,11 @@ void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - HeaderMap *hdr_map, bool is_merge) + HeaderMap *hdr_map, bool is_merge, bool sync) { - int rc; - bool use_pagemap; - char *errmsg = NULL; - BlockNumber err_blknum = 0; /* page headers */ BackupPageHeader2 *headers = NULL; + err_i err = $noerr(); /* sanity */ if (file->size % BLCKSZ != 0) @@ -505,7 +397,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat * NOTE This is a normal situation, if the file size has changed * since the moment we computed it. */ - file->n_blocks = file->size/BLCKSZ; + file->n_blocks = ft_div_i64u32_to_i32(file->size, BLCKSZ); /* * Skip unchanged file only if it exists in previous backup. @@ -529,7 +421,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat file->read_size = 0; file->write_size = 0; file->uncompressed_size = 0; - INIT_FILE_CRC32(true, file->crc); + file->crc = 0; /* crc of empty file is 0 */ /* * Read each page, verify checksum and write it to backup. @@ -541,81 +433,26 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat * Such files should be fully copied. */ - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || - file->pagemap_isabsent || !file->exists_in_prev || - !file->pagemap.bitmap) - use_pagemap = false; - else - use_pagemap = true; + /* send prev backup START_LSN */ + XLogRecPtr start_lsn = (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr; + /* TODO: stop handling errors internally */ + err = send_pages(to_fullpath, from_fullpath, file, start_lsn, + calg, clevel, checksum_version, + &headers, backup_mode, sync); - /* Remote mode */ - if (fio_is_remote(FIO_DB_HOST)) - { - rc = fio_send_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, - /* send pagemap if any */ - use_pagemap, - /* variables for error reporting */ - &err_blknum, &errmsg, &headers); - } - else + if ($haserr(err)) { - /* TODO: stop handling errors internally */ - rc = send_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, use_pagemap, - &headers, backup_mode); - } - - /* check for errors */ - if (rc == FILE_MISSING) - { - elog(is_merge ? ERROR : LOG, "File not found: \"%s\"", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - err_blknum, to_fullpath, strerror(errno)); - - else if (rc == PAGE_CORRUPTION) - { - if (errmsg) - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, err_blknum, errmsg); - else - elog(ERROR, "Corruption detected in file \"%s\", block %u", - from_fullpath, err_blknum); - } - /* OPEN_FAILED and READ_FAILED */ - else if (rc == OPEN_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot open file \"%s\"", from_fullpath); - } - else if (rc == READ_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot read file \"%s\"", from_fullpath); + if (getErrno(err) == ENOENT) + { + elog(is_merge ? ERROR : LOG, "File not found: \"%s\"", + from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; + } + ft_logerr(FT_FATAL, $errmsg(err), "Copying data file \"%s\"", file->rel_path); } - file->read_size = rc * BLCKSZ; - - /* refresh n_blocks for FULL and DELTA */ - if (backup_mode == BACKUP_MODE_FULL || - backup_mode == BACKUP_MODE_DIFF_DELTA) - file->n_blocks = file->read_size / BLCKSZ; - /* Determine that file didn`t changed in case of incremental backup */ if (backup_mode != BACKUP_MODE_FULL && file->exists_in_prev && @@ -627,13 +464,9 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat cleanup: - /* finish CRC calculation */ - FIN_FILE_CRC32(true, file->crc); - /* dump page headers */ write_page_headers(headers, file, hdr_map, is_merge); - pg_free(errmsg); pg_free(file->pagemap.bitmap); pg_free(headers); } @@ -648,19 +481,16 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr sync_lsn, BackupMode backup_mode, - uint32 checksum_version, size_t prev_size) + uint32 checksum_version, int64_t prev_size) { - int rc; - bool use_pagemap; - char *errmsg = NULL; - BlockNumber err_blknum = 0; + err_i err = $noerr(); /* * Compute expected number of blocks in the file. * NOTE This is a normal situation, if the file size has changed * since the moment we computed it. */ - file->n_blocks = file->size/BLCKSZ; + file->n_blocks = ft_div_i64u32_to_i32(file->size, BLCKSZ); /* * Skip unchanged file only if it exists in destination directory. @@ -683,83 +513,23 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa file->write_size = 0; file->uncompressed_size = 0; - /* - * If page map is empty or file is not present in destination directory, - * then copy backup all pages of the relation. - */ - - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || - file->pagemap_isabsent || !file->exists_in_prev || - !file->pagemap.bitmap) - use_pagemap = false; - else - use_pagemap = true; - - if (use_pagemap) - elog(LOG, "Using pagemap for file \"%s\"", file->rel_path); - - /* Remote mode */ - if (fio_is_remote(FIO_DB_HOST)) - { - rc = fio_copy_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr, - NONE_COMPRESS, 1, checksum_version, - /* send pagemap if any */ - use_pagemap, - /* variables for error reporting */ - &err_blknum, &errmsg); - } - else - { - /* TODO: stop handling errors internally */ - rc = copy_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr, - checksum_version, use_pagemap, backup_mode); - } - - /* check for errors */ - if (rc == FILE_MISSING) - { - elog(LOG, "File not found: \"%s\"", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - err_blknum, to_fullpath, strerror(errno)); - - else if (rc == PAGE_CORRUPTION) - { - if (errmsg) - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, err_blknum, errmsg); - else - elog(ERROR, "Corruption detected in file \"%s\", block %u", - from_fullpath, err_blknum); - } - /* OPEN_FAILED and READ_FAILED */ - else if (rc == OPEN_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot open file \"%s\"", from_fullpath); - } - else if (rc == READ_FAILED) + /* send prev backup START_LSN */ + XLogRecPtr start_lsn = ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr; + /* TODO: stop handling errors internally */ + err = copy_pages(to_fullpath, from_fullpath, file, start_lsn, + checksum_version, backup_mode); + if ($haserr(err)) { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot read file \"%s\"", from_fullpath); + if (getErrno(err) == ENOENT) + { + elog(LOG, "File not found: \"%s\"", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; + } + ft_logerr(FT_FATAL, $errmsg(err), "Copying file \"%s\"", file->rel_path); } - file->read_size = rc * BLCKSZ; - /* Determine that file didn`t changed in case of incremental catchup */ if (backup_mode != BACKUP_MODE_FULL && file->exists_in_prev && @@ -770,7 +540,6 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa } cleanup: - pg_free(errmsg); pg_free(file->pagemap.bitmap); } @@ -781,16 +550,19 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa * and make a decision about copying or skiping the file. */ void -backup_non_data_file(pgFile *file, pgFile *prev_file, +backup_non_data_file(pioDrive_i drive_from, pioDrive_i drive_to, + pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, - bool missing_ok) + bool missing_ok, bool sync) { + FOBJ_FUNC_ARP(); + err_i err; /* special treatment for global/pg_control */ if (file->external_dir_num == 0 && strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0) { - copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, - to_fullpath, FIO_BACKUP_HOST, file); + copy_pgcontrol_file(drive_from, from_fullpath, + drive_to, to_fullpath, file); return; } @@ -806,21 +578,29 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, * file could be deleted under our feets. * But then backup_non_data_file_internal will handle it safely */ - if (file->forkName != cfm) - file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true); - else - file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true); + file->crc = $i(pioGetCRC32, drive_from, .path = from_fullpath, + .truncated = file->forkName == cfm, .err = &err); + if (getErrno(err) == ENOENT) + { + elog(LOG, "File \"%s\" is not found", from_fullpath); + file->crc = 0; + file->read_size = 0; + file->write_size = 0; + file->uncompressed_size = 0; + file->write_size = FILE_NOT_FOUND; + return; + } /* ...and checksum is the same... */ - if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) + if (EQ_CRC32C(file->crc, prev_file->crc)) { file->write_size = BYTES_INVALID; return; /* ...skip copying file. */ } } - backup_non_data_file_internal(from_fullpath, FIO_DB_HOST, - to_fullpath, file, missing_ok); + backup_non_data_file_internal(drive_from, drive_to, from_fullpath, + to_fullpath, file, missing_ok, sync); } /* @@ -829,13 +609,15 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, * Apply changed blocks to destination file from every backup in parent chain. */ size_t -restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, +restore_data_file(parray *parent_chain, pgFile *dest_file, pioDBWriter_i out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool use_headers) { size_t total_write_len = 0; char *in_buf = pgut_malloc(STDIO_BUFSIZE); int backup_seq = 0; + err_i err; + pioDrive_i backup_drive = pioDriveForLocation(FIO_BACKUP_HOST); /* * FULL -> INCR -> DEST @@ -854,11 +636,11 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, // for (i = 0; i < parray_num(parent_chain); i++) while (backup_seq >= 0 && backup_seq < parray_num(parent_chain)) { + FOBJ_LOOP_ARP(); char from_root[MAXPGPATH]; char from_fullpath[MAXPGPATH]; - FILE *in = NULL; + pioReader_i in; - pgFile **res_file = NULL; pgFile *tmp_file = NULL; /* page headers */ @@ -872,8 +654,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, backup_seq--; /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); - tmp_file = (res_file) ? *res_file : NULL; + tmp_file = search_file_in_hashtable(backup->hashtable, dest_file); /* Destination file is not exists yet at this moment */ if (tmp_file == NULL) @@ -900,13 +681,9 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, join_path_components(from_root, backup->root_dir, DATABASE_DIR); join_path_components(from_fullpath, from_root, tmp_file->rel_path); - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, - strerror(errno)); - - /* set stdio buffering for input data file */ - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + in = $i(pioOpenRead, backup_drive, from_fullpath, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Open backup file"); /* get headers for this file */ if (use_headers && tmp_file->n_headers > 0) @@ -932,9 +709,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, backup->stop_lsn <= shift_lsn ? lsn_map : NULL, headers); - if (fclose(in) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, - strerror(errno)); + $i(pioClose, in); pg_free(headers); @@ -942,9 +717,156 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, } pg_free(in_buf); + if (dest_file->n_blocks > 0) /* old binary's backups didn't have n_blocks field */ + { + err = $i(pioTruncate, out, .size = (int64_t)dest_file->n_blocks * BLCKSZ); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Could not truncate datafile"); + } + return total_write_len; } +static bool +backup_page_next(backup_page_iterator *it) +{ + size_t read_len; + int32_t compressed_size; + err_i err; + + + it->truncated = false; + /* newer backups have headers in separate storage */ + if (it->headers) + { + BackupPageHeader2* hd; + uint32_t n_hdr = it->n_hdr; + if (n_hdr >= it->n_headers) + return false; + it->n_hdr++; + + hd = &it->headers[n_hdr]; + it->blknum = hd->block; + it->page_lsn = hd->lsn; + it->page_crc = hd->checksum; + + ft_assert(hd->pos >= 0); + ft_assert((hd+1)->pos > hd->pos + sizeof(BackupPageHeader)); + it->read_pos = hd->pos; + + /* calculate payload size by comparing current and next page positions */ + read_len = (hd+1)->pos - hd->pos; + it->read_to = ft_bytes(&it->page, read_len); + compressed_size = read_len - sizeof(BackupPageHeader); + ft_assert(compressed_size <= BLCKSZ); + it->whole_read = ft_bytes(&it->page, read_len); + } + else + { + /* We get into this branch either when restoring old backup + * or when merging something. Align read_len only when restoring + * or merging old backups. + */ + read_len = $i(pioRead, it->in, ft_bytes(&it->page.bph, sizeof(it->page.bph)), + .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Reading block header"); + if (read_len == 0) /* end of file */ + return false; + if (read_len != sizeof(it->page.bph)) + ft_log(FT_FATAL, "Cannot read header at offset %lld of \"%s\"", + (long long)it->cur_pos, it->fullpath); + if (it->page.bph.block == 0 && it->page.bph.compressed_size == 0) + ft_log(FT_FATAL, "Empty block in file \"%s\"", it->fullpath); + + it->cur_pos += sizeof(BackupPageHeader); + it->read_pos = it->cur_pos; + it->blknum = it->page.bph.block; + compressed_size = it->page.bph.compressed_size; + if (compressed_size == PageIsTruncated) + { + it->truncated = true; + compressed_size = 0; + } + ft_assert(compressed_size >= 0 && compressed_size <= BLCKSZ); + it->page_lsn = 0; + it->page_crc = 0; + + /* this has a potential to backfire when retrying merge of old backups, + * so we just forbid the retrying of failed merges between versions >= 2.4.0 and + * version < 2.4.0 + */ + if (it->backup_version >= 20400) + read_len = compressed_size; + else + /* For some unknown and possibly dump reason I/O operations + * in versions < 2.4.0 were always aligned to 8 bytes. + * Now we have to deal with backward compatibility. + */ + read_len = MAXALIGN(compressed_size); + it->read_to = ft_bytes(&it->page.data, read_len); + it->whole_read = ft_bytes(&it->page, + sizeof(BackupPageHeader) + read_len); + } + + it->compressed = ft_bytes(&it->page.data, compressed_size); + return true; +} + +static err_i +backup_page_read(backup_page_iterator *it) +{ + err_i err; + size_t read_len; + + if (it->read_pos != it->cur_pos) + { + err = $i(pioSeek, it->in, it->read_pos); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Cannot seek block %u", + it->blknum); + it->cur_pos = it->read_pos; + } + + read_len = $i(pioRead, it->in, it->read_to, &err); + if ($haserr(err)) + return $err(RT, "Cannot read block {blknum} of file {path}: {cause}", + blknum(it->blknum), cause(err.self), path(it->fullpath)); + if (read_len != it->read_to.len) + return $err(RT, "Short read of block {blknum} of file {path}", + blknum(it->blknum), path(it->fullpath)); + it->cur_pos += read_len; + + it->is_compressed = it->compressed.len != BLCKSZ; + /* + * Compression skip magic part 2: + * if page size is smaller than BLCKSZ, decompress the page. + * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. + * we have to check, whether it is compressed or not using + * page_may_be_compressed() function. + */ + if (!it->is_compressed && it->backup_version < 20023 && + page_may_be_compressed(it->compressed.ptr, it->compress_alg)) + { + it->is_compressed = true; + } + return $noerr(); +} + +static err_i +backup_page_skip(backup_page_iterator *it) +{ + if (it->headers != NULL) + return $noerr(); + + /* Backward compatibility kludge TODO: remove in 3.0 + * go to the next page. + */ + it->cur_pos += it->read_to.len; + it->read_pos = it->cur_pos; + return $i(pioSeek, it->in, it->cur_pos); +} + /* Restore block from "in" file to "out" file. * If "nblocks" is greater than zero, then skip restoring blocks, * whose position if greater than "nblocks". @@ -955,16 +877,23 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, * marked as already restored, then page is skipped. */ size_t -restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, +restore_data_file_internal(pioReader_i in, pioDBWriter_i out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, datapagemap_t *lsn_map, BackupPageHeader2 *headers) { - BlockNumber blknum = 0; - int n_hdr = -1; size_t write_len = 0; off_t cur_pos_out = 0; - off_t cur_pos_in = 0; + err_i err = $noerr(); + + backup_page_iterator iter = { + .fullpath = from_fullpath, + .in = in, + .headers = headers, + .n_headers = file->n_headers, + .backup_version = backup_version, + .compress_alg = file->compress_alg, + }; /* should not be possible */ Assert(!(backup_version >= 20400 && file->n_headers <= 0)); @@ -979,77 +908,18 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * a lot when blocks from incremental backup are restored, * but should never happen in case of blocks from FULL backup. */ - if (fio_fseek(out, cur_pos_out) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); + err = $i(pioSeek, out, cur_pos_out); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Cannot seek block %u"); - for (;;) + while (backup_page_next(&iter)) { off_t write_pos; - size_t len; - size_t read_len; - DataPage page; - int32 compressed_size = 0; - bool is_compressed = false; - - /* incremental restore vars */ - uint16 page_crc = 0; - XLogRecPtr page_lsn = InvalidXLogRecPtr; /* check for interrupt */ if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during data file restore"); - /* newer backups have headers in separate storage */ - if (headers) - { - n_hdr++; - if (n_hdr >= file->n_headers) - break; - - blknum = headers[n_hdr].block; - page_lsn = headers[n_hdr].lsn; - page_crc = headers[n_hdr].checksum; - /* calculate payload size by comparing current and next page positions, - * page header is not included */ - compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos - sizeof(BackupPageHeader); - - Assert(compressed_size > 0); - Assert(compressed_size <= BLCKSZ); - - read_len = compressed_size + sizeof(BackupPageHeader); - } - else - { - /* We get into this function either when restoring old backup - * or when merging something. Align read_len only when restoring - * or merging old backups. - */ - if (get_page_header(in, from_fullpath, &(page).bph, NULL, false)) - { - cur_pos_in += sizeof(BackupPageHeader); - - /* backward compatibility kludge TODO: remove in 3.0 */ - blknum = page.bph.block; - compressed_size = page.bph.compressed_size; - - /* this has a potential to backfire when retrying merge of old backups, - * so we just forbid the retrying of failed merges between versions >= 2.4.0 and - * version < 2.4.0 - */ - if (backup_version >= 20400) - read_len = compressed_size; - else - /* For some unknown and possibly dump reason I/O operations - * in versions < 2.4.0 were always aligned to 8 bytes. - * Now we have to deal with backward compatibility. - */ - read_len = MAXALIGN(compressed_size); - } - else - break; - } - /* * Backward compatibility kludge: in the good old days * n_blocks attribute was available only in DELTA backups. @@ -1062,44 +932,31 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * is not happening in the first place. * TODO: remove in 3.0.0 */ - if (compressed_size == PageIsTruncated) + if (iter.truncated) { /* * Block header contains information that this block was truncated. * We need to truncate file to this length. */ - elog(VERBOSE, "Truncate file \"%s\" to block %u", to_fullpath, blknum); - - /* To correctly truncate file, we must first flush STDIO buffers */ - if (fio_fflush(out) != 0) - elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno)); + elog(VERBOSE, "Truncate file \"%s\" to block %u", to_fullpath, iter.blknum); - /* Set position to the start of file */ - if (fio_fseek(out, 0) < 0) - elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno)); - - if (fio_ftruncate(out, blknum * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno)); + err = $i(pioTruncate, out, (uint64_t)iter.blknum * BLCKSZ); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), ""); break; } - Assert(compressed_size > 0); - Assert(compressed_size <= BLCKSZ); - /* no point in writing redundant data */ - if (nblocks > 0 && blknum >= nblocks) + if (nblocks > 0 && iter.blknum >= nblocks) break; - if (compressed_size > BLCKSZ) - elog(ERROR, "Size of a blknum %i exceed BLCKSZ: %i", blknum, compressed_size); - /* Incremental restore in LSN mode */ - if (map && lsn_map && datapagemap_is_set(lsn_map, blknum)) - datapagemap_add(map, blknum); + if (map && lsn_map && datapagemap_is_set(lsn_map, iter.blknum)) + datapagemap_add(map, iter.blknum); - if (map && checksum_map && checksum_map[blknum].checksum != 0) + if (map && checksum_map && checksum_map[iter.blknum].checksum != 0) { //elog(INFO, "HDR CRC: %u, MAP CRC: %u", page_crc, checksum_map[blknum].checksum); /* @@ -1107,71 +964,37 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * If page in backup has the same checksum and lsn as * page in backup, then page can be skipped. */ - if (page_crc == checksum_map[blknum].checksum && - page_lsn == checksum_map[blknum].lsn) + if (iter.page_crc == checksum_map[iter.blknum].checksum && + iter.page_lsn == checksum_map[iter.blknum].lsn) { - datapagemap_add(map, blknum); + datapagemap_add(map, iter.blknum); } } /* if this page is marked as already restored, then skip it */ - if (map && datapagemap_is_set(map, blknum)) + if (map && datapagemap_is_set(map, iter.blknum)) { - /* Backward compatibility kludge TODO: remove in 3.0 - * go to the next page. - */ - if (!headers && fseek(in, read_len, SEEK_CUR) != 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, from_fullpath, strerror(errno)); + backup_page_skip(&iter); continue; } - if (headers && - cur_pos_in != headers[n_hdr].pos) - { - if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to offset %u of \"%s\": %s", - headers[n_hdr].pos, from_fullpath, strerror(errno)); - - cur_pos_in = headers[n_hdr].pos; - } - - /* read a page from file */ - if (headers) - len = fread(&page, 1, read_len, in); - else - len = fread(page.data, 1, read_len, in); - - if (len != read_len) - elog(ERROR, "Cannot read block %u file \"%s\": %s", - blknum, from_fullpath, strerror(errno)); - - cur_pos_in += read_len; - - /* - * if page size is smaller than BLCKSZ, decompress the page. - * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. - * we have to check, whether it is compressed or not using - * page_may_be_compressed() function. - */ - if (compressed_size != BLCKSZ - || page_may_be_compressed(page.data, file->compress_alg, backup_version)) - { - is_compressed = true; - } + err = backup_page_read(&iter); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), ""); /* * Seek and write the restored page. * When restoring file from FULL backup, pages are written sequentially, * so there is no need to issue fseek for every page. */ - write_pos = blknum * BLCKSZ; + write_pos = iter.blknum * BLCKSZ; if (cur_pos_out != write_pos) { - if (fio_fseek(out, write_pos) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); + err = $i(pioSeek, out, write_pos); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Cannot seek block %u", + iter.blknum); cur_pos_out = write_pos; } @@ -1180,85 +1003,36 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * If page is compressed and restore is in remote mode, * send compressed page to the remote side. */ - if (is_compressed) - { - ssize_t rc; - rc = fio_fwrite_async_compressed(out, page.data, compressed_size, file->compress_alg); - - if (!fio_is_remote_file(out) && rc != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s, size: %u", - blknum, to_fullpath, strerror(errno), compressed_size); - } + if (iter.is_compressed) + err = $i(pioWriteCompressed, out, iter.compressed, + .compress_alg = file->compress_alg); else - { - if (fio_fwrite_async(out, page.data, BLCKSZ) != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); - } + err = $i(pioWrite, out, iter.compressed); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Cannot write block %u", + iter.blknum); write_len += BLCKSZ; cur_pos_out += BLCKSZ; /* update current write position */ /* Mark page as restored to avoid reading this page when restoring parent backups */ if (map) - datapagemap_add(map, blknum); + datapagemap_add(map, iter.blknum); } - elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); + elog(LOG, "Copied file \"%s\": %zu bytes", from_fullpath, write_len); return write_len; } -/* - * Copy file to backup. - * We do not apply compression to these files, because - * it is either small control file or already compressed cfs file. - */ -void -restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, - const char *from_fullpath, const char *to_fullpath) -{ - size_t read_len = 0; - char *buf = pgut_malloc(STDIO_BUFSIZE); /* 64kB buffer */ - - /* copy content */ - for (;;) - { - read_len = 0; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during non-data file restore"); - - read_len = fread(buf, 1, STDIO_BUFSIZE, in); - - if (ferror(in)) - elog(ERROR, "Cannot read backup file \"%s\": %s", - from_fullpath, strerror(errno)); - - if (read_len > 0) - { - if (fio_fwrite_async(out, buf, read_len) != read_len) - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, - strerror(errno)); - } - - if (feof(in)) - break; - } - - pg_free(buf); - - elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); -} - size_t -restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, - pgFile *dest_file, FILE *out, const char *to_fullpath, +restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, + pioDrive_i out_drive, pioDBWriter_i out, const char *to_fullpath, bool already_exists) { char from_root[MAXPGPATH]; char from_fullpath[MAXPGPATH]; - FILE *in = NULL; + pioReadStream_i in; + err_i err; pgFile *tmp_file = NULL; pgBackup *tmp_backup = NULL; @@ -1281,11 +1055,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, tmp_backup = dest_backup->parent_backup_link; while (tmp_backup) { - pgFile **res_file = NULL; - /* lookup file in intermediate backup */ - res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal); - tmp_file = (res_file) ? *res_file : NULL; + tmp_file = search_file_in_hashtable(tmp_backup->hashtable, dest_file); /* * It should not be possible not to find destination file in intermediate @@ -1302,9 +1073,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (tmp_file->write_size == 0) { /* In case of incremental restore truncate file just to be safe */ - if (already_exists && fio_ftruncate(out, 0)) - elog(ERROR, "Cannot truncate file \"%s\": %s", - to_fullpath, strerror(errno)); + if (already_exists) + { + err = $i(pioTruncate, out, 0); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), ""); + } return 0; } @@ -1325,9 +1099,9 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); if (tmp_file->write_size <= 0) - elog(ERROR, "Full copy of non-data file has invalid size: %li. " + elog(ERROR, "Full copy of non-data file has invalid size: %lli. " "Metadata corruption in backup %s in file: \"%s\"", - tmp_file->write_size, backup_id_of(tmp_backup), + (long long)tmp_file->write_size, backup_id_of(tmp_backup), to_fullpath); /* incremental restore */ @@ -1335,11 +1109,16 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, { /* compare checksums of already existing file and backup file */ pg_crc32 file_crc; - if (tmp_file->forkName == cfm && - tmp_file->uncompressed_size > tmp_file->write_size) - file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false); - else - file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false); + bool truncated; + + truncated = tmp_file->forkName == cfm && + tmp_file->uncompressed_size > tmp_file->write_size; + + file_crc = $i(pioGetCRC32, out_drive, .path = to_fullpath, + .truncated = truncated, .err = &err); + + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Non-data file CRC32"); if (file_crc == tmp_file->crc) { @@ -1349,9 +1128,9 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, } /* Checksum mismatch, truncate file and overwrite it */ - if (fio_ftruncate(out, 0)) - elog(ERROR, "Cannot truncate file \"%s\": %s", - to_fullpath, strerror(errno)); + err = $i(pioTruncate, out, 0); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), ""); } if (tmp_file->external_dir_num == 0) @@ -1366,20 +1145,18 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, join_path_components(from_fullpath, from_root, dest_file->rel_path); - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, - strerror(errno)); - - /* disable stdio buffering for non-data files */ - setvbuf(in, NULL, _IONBF, BUFSIZ); + in = $i(pioOpenReadStream, dest_backup->backup_location, from_fullpath, + .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Open backup file"); - /* do actual work */ - restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); + err = pioCopy($reduce(pioWriteFlush, out), $reduce(pioRead, in)); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Copying backup file"); - if (fclose(in) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, - strerror(errno)); + $i(pioClose, in); + elog(LOG, "Copied file \"%s\": %llu bytes", from_fullpath, + (long long)dest_file->write_size); return tmp_file->write_size; } @@ -1391,103 +1168,128 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, * TODO: optimize remote copying */ void -backup_non_data_file_internal(const char *from_fullpath, - fio_location from_location, +backup_non_data_file_internal(pioDrive_i drive_from, pioDrive_i drive_to, + const char *from_fullpath, const char *to_fullpath, pgFile *file, - bool missing_ok) + bool missing_ok, bool sync) { - FILE *out = NULL; - char *errmsg = NULL; - int rc; bool cut_zero_tail; + err_i err; + FOBJ_FUNC_ARP(); cut_zero_tail = file->forkName == cfm; - INIT_FILE_CRC32(true, file->crc); - /* reset size summary */ + file->crc = 0; file->read_size = 0; file->write_size = 0; file->uncompressed_size = 0; - /* open backup file for write */ - out = fopen(to_fullpath, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath, file->mode) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - - /* backup remote file */ - if (fio_is_remote(FIO_DB_HOST)) - rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg); - else - rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg); + /* backup non-data file */ + err = send_file(drive_from, drive_to, to_fullpath, from_fullpath, + cut_zero_tail, file, sync); /* handle errors */ - if (rc == FILE_MISSING) - { - /* maybe deleted, it's not error in case of backup */ - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); - else if (rc != SEND_OK) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath); + if($haserr(err)) { + if(getErrno(err) == ENOENT) { + if(missing_ok) { + elog(LOG, "File \"%s\" is not found", from_fullpath); + file->write_size = FILE_NOT_FOUND; + return; + } else + elog(ERROR, "File \"%s\" is not found", from_fullpath); + } else + elog(ERROR, "An error occured while copying %s: %s", + from_fullpath, $errmsg(err)); } +} - file->uncompressed_size = file->read_size; +static err_i +send_file(pioDrive_i db_drive, pioDrive_i backup_drive, const char *to_fullpath, const char *from_fullpath, bool cut_zero_tail, pgFile *file, bool sync) { + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + pioReadStream_i in; + pioWriteCloser_i out; -cleanup: - if (errmsg != NULL) - pg_free(errmsg); + /* open to_fullpath */ + out = $i(pioOpenRewrite, backup_drive, .path = to_fullpath, + .permissions = file->mode, .sync = sync, .err = &err); + + if($haserr(err)) + return $iresult(err); + + /* open from_fullpath */ + in = $i(pioOpenReadStream, db_drive, .path = from_fullpath, .err = &err); + + if($haserr(err)) + return $iresult(err); + + /* + * Copy content and calc CRC as it gets copied. Optionally pioZeroTail + * will be used. + */ + pioCRC32Counter *c = pioCRC32Counter_alloc(); + pioCutZeroTail *zt = pioCutZeroTail_alloc(); + pioFilter_i ztFlt = bind_pioFilter(zt); + pioFilter_i crcFlt = bind_pioFilter(c); + pioFilter_i fltrs[] = { ztFlt, crcFlt }; + + err = pioCopyWithFilters($reduce(pioWriteFlush, out), $reduce(pioRead, in), + cut_zero_tail ? fltrs : &fltrs[1], + cut_zero_tail ? 2 : 1, + NULL); + + if($haserr(err)) + return $iresult(err); + + if (file) { + file->crc = pioCRC32Counter_getCRC32(c); + file->read_size = pioCRC32Counter_getSize(c); + file->write_size = pioCRC32Counter_getSize(c); + if (cut_zero_tail) + file->uncompressed_size = pioCutZeroTail_getReadSize(zt); + else + file->uncompressed_size = file->read_size; + } - /* finish CRC calculation and store into pgFile */ - FIN_FILE_CRC32(true, file->crc); + $i(pioClose, in); + err = $i(pioClose, out); - if (out && fclose(out)) - elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno)); + return $iresult(err); } /* * Create empty file, used for partial restore */ bool -create_empty_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file) +create_empty_file(const char *to_root, fio_location to_location, pgFile *file) { + FOBJ_FUNC_ARP(); char to_path[MAXPGPATH]; - FILE *out; + pioDrive_i drive = pioDriveForLocation(to_location); + pioWriteCloser_i fl; + err_i err; /* open file for write */ join_path_components(to_path, to_root, file->rel_path); - out = fio_fopen(to_path, PG_BINARY_W, to_location); - - if (out == NULL) - elog(ERROR, "Cannot open destination file \"%s\": %s", - to_path, strerror(errno)); + /* + * TODO: possibly it is better to use pioWriteFile, but it doesn't have + * permissions parameter, and I don't want to introduce is just for one + * use case + */ + fl = $i(pioOpenRewrite, drive, + .path = to_path, + .permissions = file->mode, + .use_temp = false, + .err = &err); + if ($haserr(err)) + ft_logerr(FT_ERROR, $errmsg(err), "Creating empty file"); - /* update file permission */ - if (fio_chmod(to_path, file->mode, to_location) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_path, - strerror(errno)); + err = $i(pioWriteFinish, fl); + err = fobj_err_combine(err, $i(pioClose, fl)); - if (fio_fclose(out)) - elog(ERROR, "Cannot close \"%s\": %s", to_path, strerror(errno)); + if ($haserr(err)) + ft_logerr(FT_ERROR, $errmsg(err), "Closing empty file"); return true; } @@ -1557,66 +1359,54 @@ validate_one_page(Page page, BlockNumber absolute_blkno, * also returns true if the file was not found */ bool -check_data_file(ConnectionArgs *arguments, pgFile *file, - const char *from_fullpath, uint32 checksum_version) +check_data_file(pgFile *file, const char *from_fullpath, uint32 checksum_version) { - FILE *in; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; - int page_state; - char curr_page[BLCKSZ]; - bool is_valid = true; - - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - { + FOBJ_FUNC_ARP(); + pioDBDrive_i local_location = pioDBDriveForLocation(FIO_LOCAL_HOST); + pioPagesIterator_i pages; + bool is_valid = true; + err_i err; + + pages = doIteratePages(local_location, + .from_fullpath = from_fullpath, + .file = file, + .checksum_version = checksum_version, + .backup_mode = BACKUP_MODE_FULL, + .just_validate = true, + .err = &err); + if ($haserr(err)) + { + if (getErrno(err) == ENOENT) { elog(LOG, "File \"%s\" is not found", from_fullpath); return true; } - - elog(WARNING, "Cannot open file \"%s\": %s", - from_fullpath, strerror(errno)); + ft_logerr(FT_WARNING, $errmsg(err), "Cannot open file \"%s\"", from_fullpath); return false; } - if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); - - /* - * Compute expected number of blocks in the file. - * NOTE This is a normal situation, if the file size has changed - * since the moment we computed it. - */ - nblocks = file->size/BLCKSZ; - - for (blknum = 0; blknum < nblocks; blknum++) + while(true) { - PageState page_st; - page_state = prepare_page(file, InvalidXLogRecPtr, - blknum, in, BACKUP_MODE_FULL, - curr_page, false, checksum_version, - from_fullpath, &page_st); - - if (page_state == PageIsTruncated) + PageIteratorValue value; + err_i err = $i(pioNextPage, pages, &value); + if ($haserr(err)) { + ft_logerr(FT_FATAL, $errmsg(err), "Checking data file"); + return false; + } + if (value.page_result == PageIsTruncated) break; - if (page_state == PageIsCorrupted) + if (value.page_result == PageIsCorrupted) { /* Page is corrupted, no need to elog about it, * prepare_page() already done that + * + * Still check the rest of the pages too */ is_valid = false; continue; } } - fclose(in); return is_valid; } @@ -1625,195 +1415,139 @@ bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map) { - size_t read_len = 0; + FOBJ_FUNC_ARP(); bool is_valid = true; - FILE *in; pg_crc32 crc; - bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; - BackupPageHeader2 *headers = NULL; - int n_hdr = -1; - off_t cur_pos_in = 0; + pioDrive_i drive; + err_i err; + + backup_page_iterator iter = { + .fullpath = fullpath, + .n_headers = file->n_headers, + .backup_version = backup_version, + .compress_alg = file->compress_alg, + }; elog(LOG, "Validate relation blocks for file \"%s\"", fullpath); /* should not be possible */ Assert(!(backup_version >= 20400 && file->n_headers <= 0)); - in = fopen(fullpath, PG_BINARY_R); - if (in == NULL) - elog(ERROR, "Cannot open file \"%s\": %s", - fullpath, strerror(errno)); + drive = pioDriveForLocation(FIO_BACKUP_HOST); - headers = get_data_file_headers(hdr_map, file, backup_version, false); + iter.in = $i(pioOpenRead, drive, fullpath, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), ""); - if (!headers && file->n_headers > 0) + iter.headers = get_data_file_headers(hdr_map, file, backup_version, false); + + if (!iter.headers && file->n_headers > 0) { elog(WARNING, "Cannot get page headers for file \"%s\"", fullpath); return false; } /* calc CRC of backup file */ - INIT_FILE_CRC32(use_crc32c, crc); + INIT_CRC32_COMPAT(backup_version, crc); /* read and validate pages one by one */ - while (true) + while (backup_page_next(&iter)) { int rc = 0; - size_t len = 0; - DataPage compressed_page; /* used as read buffer */ - int compressed_size = 0; DataPage page; - BlockNumber blknum = 0; + ft_bytes_t uncompressed; PageState page_st; if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during data file validation"); - /* newer backups have page headers in separate storage */ - if (headers) - { - n_hdr++; - if (n_hdr >= file->n_headers) - break; - - blknum = headers[n_hdr].block; - /* calculate payload size by comparing current and next page positions, - * page header is not included. - */ - compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos - sizeof(BackupPageHeader); - - Assert(compressed_size > 0); - Assert(compressed_size <= BLCKSZ); - - read_len = sizeof(BackupPageHeader) + compressed_size; - - if (cur_pos_in != headers[n_hdr].pos) - { - if (fio_fseek(in, headers[n_hdr].pos) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, fullpath, strerror(errno)); - else - elog(VERBOSE, "Seek to %u", headers[n_hdr].pos); - - cur_pos_in = headers[n_hdr].pos; - } - } - /* old backups rely on header located directly in data file */ - else - { - if (get_page_header(in, fullpath, &(compressed_page).bph, &crc, use_crc32c)) - { - /* Backward compatibility kludge, TODO: remove in 3.0 - * for some reason we padded compressed pages in old versions - */ - blknum = compressed_page.bph.block; - compressed_size = compressed_page.bph.compressed_size; - read_len = MAXALIGN(compressed_size); - } - else - break; - } - /* backward compatibility kludge TODO: remove in 3.0 */ - if (compressed_size == PageIsTruncated) + if (iter.truncated) { elog(VERBOSE, "Block %u of \"%s\" is truncated", - blknum, fullpath); + iter.blknum, fullpath); continue; } - Assert(compressed_size <= BLCKSZ); - Assert(compressed_size > 0); - - if (headers) - len = fread(&compressed_page, 1, read_len, in); - else - len = fread(compressed_page.data, 1, read_len, in); + ft_assert(iter.read_pos == iter.cur_pos); - if (len != read_len) + err = backup_page_read(&iter); + if ($haserr(err)) { - elog(WARNING, "Cannot read block %u file \"%s\": %s", - blknum, fullpath, strerror(errno)); + ft_logerr(FT_WARNING, $errmsg(err), ""); return false; } - /* update current position */ - cur_pos_in += read_len; + COMP_CRC32_COMPAT(backup_version, crc, iter.whole_read.ptr, iter.whole_read.len); - if (headers) - COMP_FILE_CRC32(use_crc32c, crc, &compressed_page, read_len); - else - COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); - - if (compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg, - backup_version)) + if (iter.is_compressed) { int32 uncompressed_size = 0; const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, - compressed_page.data, - compressed_size, + iter.compressed.ptr, + iter.compressed.len, file->compress_alg, &errormsg); if (uncompressed_size < 0 && errormsg != NULL) { elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", - blknum, fullpath, errormsg); + iter.blknum, fullpath, errormsg); return false; } if (uncompressed_size != BLCKSZ) { - if (compressed_size == BLCKSZ) + elog(WARNING, "Page %u of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + iter.blknum, fullpath, uncompressed_size); + if (iter.compressed.len == BLCKSZ) { is_valid = false; continue; } - elog(WARNING, "Page %u of file \"%s\" uncompressed to %d bytes. != BLCKSZ", - blknum, fullpath, uncompressed_size); return false; } - rc = validate_one_page(page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_st, checksum_version); + uncompressed = ft_bytes(page.data, BLCKSZ); } else - rc = validate_one_page(compressed_page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_st, checksum_version); + uncompressed = iter.compressed; + + rc = validate_one_page(uncompressed.ptr, + file->segno * RELSEG_SIZE + iter.blknum, + stop_lsn, &page_st, checksum_version); switch (rc) { case PAGE_IS_NOT_FOUND: - elog(VERBOSE, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); + elog(VERBOSE, "File \"%s\", block %u, page is NULL", file->rel_path, iter.blknum); break; case PAGE_IS_ZEROED: - elog(VERBOSE, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); + elog(VERBOSE, "File: %s blknum %u, empty zeroed page", file->rel_path, iter.blknum); break; case PAGE_HEADER_IS_INVALID: - elog(WARNING, "Page header is looking insane: %s, block %i", file->rel_path, blknum); + elog(WARNING, "Page header is looking insane: %s, block %i", file->rel_path, iter.blknum); is_valid = false; break; case PAGE_CHECKSUM_MISMATCH: - elog(WARNING, "File: %s blknum %u have wrong checksum: %u", file->rel_path, blknum, page_st.checksum); + elog(WARNING, "File: %s blknum %u have wrong checksum: %u", file->rel_path, iter.blknum, page_st.checksum); is_valid = false; break; case PAGE_LSN_FROM_FUTURE: elog(WARNING, "File: %s, block %u, checksum is %s. " "Page is from future: pageLSN %X/%X stopLSN %X/%X", - file->rel_path, blknum, + file->rel_path, iter.blknum, checksum_version ? "correct" : "not enabled", (uint32) (page_st.lsn >> 32), (uint32) page_st.lsn, (uint32) (stop_lsn >> 32), (uint32) stop_lsn); + is_valid = false; break; } } - FIN_FILE_CRC32(use_crc32c, crc); - fclose(in); + FIN_CRC32_COMPAT(backup_version, crc); + $i(pioClose, iter.in); if (crc != file->crc) { @@ -1822,7 +1556,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, is_valid = false; } - pg_free(headers); + pg_free(iter.headers); return is_valid; } @@ -1965,45 +1699,6 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, return lsn_map; } -/* Every page in data file contains BackupPageHeader, extract it */ -bool -get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, - pg_crc32 *crc, bool use_crc32c) -{ - /* read BackupPageHeader */ - size_t read_len = fread(bph, 1, sizeof(BackupPageHeader), in); - - if (ferror(in)) - elog(ERROR, "Cannot read file \"%s\": %s", - fullpath, strerror(errno)); - - if (read_len != sizeof(BackupPageHeader)) - { - if (read_len == 0 && feof(in)) - return false; /* EOF found */ - else if (read_len != 0 && feof(in)) - elog(ERROR, - "Odd size page found at offset %ld of \"%s\"", - ftello(in), fullpath); - else - elog(ERROR, "Cannot read header at offset %ld of \"%s\": %s", - ftello(in), fullpath, strerror(errno)); - } - - /* In older versions < 2.4.0, when crc for file was calculated, header was - * not included in crc calculations. Now it is. And now we have - * the problem of backward compatibility for backups of old versions - */ - if (crc) - COMP_FILE_CRC32(use_crc32c, *crc, bph, read_len); - - if (bph->block == 0 && bph->compressed_size == 0) - elog(ERROR, "Empty block in file \"%s\"", fullpath); - - Assert(bph->compressed_size != 0); - return true; -} - /* Open local backup file for writing, set permissions and buffering */ FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size) @@ -2027,299 +1722,265 @@ open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size) return out; } +#define FT_SLICE bpph2 +#define FT_SLICE_TYPE BackupPageHeader2 +#include + /* backup local file */ -int -send_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, - uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode) +static err_i +send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, + uint32 checksum_version, BackupPageHeader2 **headers, + BackupMode backup_mode, bool sync) { - FILE *in = NULL; - FILE *out = NULL; - off_t cur_pos_out = 0; - char curr_page[BLCKSZ]; - int n_blocks_read = 0; - BlockNumber blknum = 0; - datapagemap_iterator_t *iter = NULL; - int compressed_size = 0; - BackupPageHeader2 *header = NULL; - parray *harray = NULL; - - /* stdio buffers */ - char *in_buf = NULL; - char *out_buf = NULL; - - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - return FILE_MISSING; + FOBJ_FUNC_ARP(); + pioDrive_i backup_location = pioDriveForLocation(FIO_BACKUP_HOST); + pioDBDrive_i db_location = pioDBDriveForLocation(FIO_DB_HOST); + pioPagesIterator_i pages; + pioWriteCloser_i out = $null(pioWriteCloser); + pioWriteFlush_i wrapped = $null(pioWriteFlush); + pioCRC32Counter *crc32 = NULL; + ft_arr_bpph2_t harray = ft_arr_init(); + err_i err = $noerr(); + + pages = doIteratePages(db_location, .from_fullpath = from_fullpath, .file = file, + .start_lsn = prev_backup_start_lsn, .calg = calg, .clevel = clevel, + .checksum_version = checksum_version, .backup_mode = backup_mode, + .err = &err); + if ($haserr(err)) + return $iresult(err); - elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); - } - - /* - * Enable stdio buffering for local input file, - * unless the pagemap is involved, which - * imply a lot of random access. - */ - - if (use_pagemap) - { - iter = datapagemap_iterate(&file->pagemap); - datapagemap_next(iter, &blknum); /* set first block */ - - setvbuf(in, NULL, _IONBF, BUFSIZ); - } - else - { - in_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - } - - harray = parray_new(); - - while (blknum < file->n_blocks) + while (true) { - PageState page_st; - int rc = prepare_page(file, prev_backup_start_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - from_fullpath, &page_st); - - if (rc == PageIsTruncated) + PageIteratorValue value; + err_i err = $i(pioNextPage, pages, &value); + if ($haserr(err)) + return $iresult(err); + if (value.page_result == PageIsTruncated) break; - else if (rc == PageIsOk) - { - /* lazily open backup file (useful for s3) */ - if (!out) - out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); - - header = pgut_new0(BackupPageHeader2); - *header = (BackupPageHeader2){ - .block = blknum, - .pos = cur_pos_out, - .lsn = page_st.lsn, - .checksum = page_st.checksum, - }; - - parray_append(harray, header); - - compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), - rc, curr_page, calg, clevel, - from_fullpath, to_fullpath); - cur_pos_out += compressed_size + sizeof(BackupPageHeader); - } + if (value.page_result == PageIsOk) { + if($isNULL(out)) + { + out = $i(pioOpenRewrite, backup_location, to_fullpath, + .sync = sync, .err = &err); + if ($haserr(err)) + return $iresult(err); + crc32 = pioCRC32Counter_alloc(); + wrapped = pioWrapWriteFilter($reduce(pioWriteFlush, out), + $bind(pioFilter, crc32), + BLCKSZ + sizeof(BackupPageHeader)); + file->compress_alg = calg; + } - n_blocks_read++; + ft_arr_bpph2_push(&harray, (BackupPageHeader2){ + .block = value.blknum, + .pos = file->write_size, + .lsn = value.state.lsn, + .checksum = value.state.checksum, + }); + + file->uncompressed_size += BLCKSZ; + file->write_size += backup_page($reduce(pioWrite, wrapped), value.blknum, + ft_bytes(value.compressed_page, value.compressed_size), + to_fullpath, &err); + if ($haserr(err)) + return $iresult(err); + } - /* next block */ - if (use_pagemap) + if (value.page_result == PageIsCorrupted) { - /* exit if pagemap is exhausted */ - if (!datapagemap_next(iter, &blknum)) - break; + err = $err(RT, "Page %d is corrupted", + blknum(value.blknum)); + return $iresult(err); } - else - blknum++; + file->read_size += BLCKSZ; } + file->n_blocks = $i(pioFinalPageN, pages); /* * Add dummy header, so we can later extract the length of last header * as difference between their offsets. */ - if (parray_num(harray) > 0) + if (harray.len > 0) { - size_t hdr_num = parray_num(harray); - size_t i; - - file->n_headers = (int) hdr_num; /* is it valid? */ - *headers = (BackupPageHeader2 *) pgut_malloc0((hdr_num + 1) * sizeof(BackupPageHeader2)); - for (i = 0; i < hdr_num; i++) - { - header = (BackupPageHeader2 *)parray_get(harray, i); - (*headers)[i] = *header; - pg_free(header); - } - (*headers)[hdr_num] = (BackupPageHeader2){.pos=cur_pos_out}; + file->n_headers = harray.len; + ft_arr_bpph2_push(&harray, (BackupPageHeader2){.pos=file->write_size}); + *headers = harray.ptr; } - parray_free(harray); - - /* cleanup */ - if (in && fclose(in)) - elog(ERROR, "Cannot close the source file \"%s\": %s", - to_fullpath, strerror(errno)); /* close local output file */ - if (out && fclose(out)) - elog(ERROR, "Cannot close the backup file \"%s\": %s", - to_fullpath, strerror(errno)); + if ($notNULL(out)) + { + err = $i(pioWriteFinish, wrapped); + if ($haserr(err)) + return $iresult(err); + file->crc = pioCRC32Counter_getCRC32(crc32); + ft_dbg_assert(file->write_size == pioCRC32Counter_getSize(crc32)); - pg_free(iter); - pg_free(in_buf); - pg_free(out_buf); + err = $i(pioClose, out); + if ($haserr(err)) + return $iresult(err); + } - return n_blocks_read; + return $noerr(); } /* - * Copy local data file just as send_pages but without attaching additional header and compression + * Copy data file just as send_pages but without attaching additional header and compression */ -int -copy_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr sync_lsn, - uint32 checksum_version, bool use_pagemap, - BackupMode backup_mode) +static err_i +copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr sync_lsn, uint32 checksum_version, + BackupMode backup_mode) { - FILE *in = NULL; - FILE *out = NULL; - char curr_page[BLCKSZ]; - int n_blocks_read = 0; - BlockNumber blknum = 0; - datapagemap_iterator_t *iter = NULL; - - /* stdio buffers */ - char *in_buf = NULL; - char *out_buf = NULL; + FOBJ_FUNC_ARP(); + pioDBDrive_i backup_location = pioDBDriveForLocation(FIO_BACKUP_HOST); + err_i err = $noerr(); + pioPagesIterator_i pages; + pioDBWriter_i out; + + pages = doIteratePages(backup_location, .from_fullpath = from_fullpath, + .file = file, .start_lsn = sync_lsn, + .checksum_version = checksum_version, + .backup_mode = backup_mode, .err = &err); + if ($haserr(err)) + return $iresult(err); + + out = $i(pioOpenWrite, backup_location, to_fullpath, + .permissions = file->mode, .err = &err); + if ($haserr(err)) + return $iresult(err); - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) + while (true) { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - return FILE_MISSING; + PageIteratorValue value; + err = $i(pioNextPage, pages, &value); + if ($haserr(err)) + return $iresult(err); - elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); - } + if (value.page_result == PageIsTruncated) + break; - /* - * Enable stdio buffering for local input file, - * unless the pagemap is involved, which - * imply a lot of random access. - */ + if(value.page_result == PageIsOk) { + Assert(value.compressed_size == BLCKSZ); /* Assuming NONE_COMPRESS above */ + write_page(file, out, value.blknum, value.compressed_page); + } - if (use_pagemap) - { - iter = datapagemap_iterate(&file->pagemap); - datapagemap_next(iter, &blknum); /* set first block */ + if (value.page_result == PageIsCorrupted) { + elog(WARNING, "Page %d of \"%s\" is corrupted", + value.blknum, file->rel_path); + } - setvbuf(in, NULL, _IONBF, BUFSIZ); - } - else - { - in_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + file->read_size += BLCKSZ; } - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); - if (out == NULL) - elog(ERROR, "Cannot open destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath, file->mode) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); + file->n_blocks = $i(pioFinalPageN, pages); + file->size = (int64_t)file->n_blocks * BLCKSZ; + err = $i(pioTruncate, out, file->size); + if ($haserr(err)) + return $iresult(err); - /* Enable buffering for output file */ - out_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); + err = $i(pioWriteFinish, out); + if ($haserr(err)) + return $iresult(err); - while (blknum < file->n_blocks) - { - PageState page_st; - int rc = prepare_page(file, sync_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - from_fullpath, &page_st); - if (rc == PageIsTruncated) - break; + err = $i(pioClose, out); + if ($haserr(err)) + return $iresult(err); - else if (rc == PageIsOk) - { - if (fseek(out, blknum * BLCKSZ, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to position %u in destination file \"%s\": %s", - blknum * BLCKSZ, to_fullpath, strerror(errno)); + return $noerr(); +} - if (write_page(file, out, curr_page) != BLCKSZ) - elog(ERROR, "File: \"%s\", cannot write at block %u: %s", - to_fullpath, blknum, strerror(errno)); - } +typedef struct header_map_cache_item header_map_cache_item_t; +struct header_map_cache_item { + ft_str_t path; + pioReader_i fl; + err_i err; - n_blocks_read++; + header_map_cache_item_t* next; +}; - /* next block */ - if (use_pagemap) - { - /* exit if pagemap is exhausted */ - if (!datapagemap_next(iter, &blknum)) - break; - } - else - blknum++; - } +typedef struct HeaderMapCache { + header_map_cache_item_t *first; +} HeaderMapCache; +#define kls__HeaderMapCache mth(fobjDispose) +fobj_klass(HeaderMapCache); - /* truncate output file if required */ - if (fseek(out, 0, SEEK_END) != 0) - elog(ERROR, "Cannot seek to end of file position in destination file \"%s\": %s", - to_fullpath, strerror(errno)); - { - long pos = ftell(out); +static __thread HeaderMapCache *header_map_cache = NULL; - if (pos < 0) - elog(ERROR, "Cannot get position in destination file \"%s\": %s", - to_fullpath, strerror(errno)); +/* + * Header_map_cache_init initializes header_map open files cache. + * It allocates object that will reside in AutoReleasePool. + * Therefore it should be called in threads top level function. */ +void +header_map_cache_init(void) +{ + header_map_cache = $alloc(HeaderMapCache); +} - if (pos != file->size) - { - if (fflush(out) != 0) - elog(ERROR, "Cannot flush destination file \"%s\": %s", - to_fullpath, strerror(errno)); +static pioReadSeek_i +header_map_cache_open(pioDrive_i drive, path_t path, err_i* err) +{ + ft_str_t pth = ft_cstr(path); + header_map_cache_item_t **item; + ft_assert(header_map_cache != NULL); - if (ftruncate(fileno(out), file->size) == -1) - elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", - to_fullpath, file->size, strerror(errno)); - } + item = &header_map_cache->first; + while (*item) + { + if (ft_streq((*item)->path, pth)) + break; + item = &(*item)->next; } + if ((*item) == NULL) + { + *item = ft_calloc(sizeof(header_map_cache_item_t)); + (*item)->path = ft_strdup(pth); + (*item)->fl = $iref($i(pioOpenRead, drive, .path = path, + .err = &(*item)->err)); + (*item)->err = $iref((*item)->err); + } + *err = (*item)->err; + return $reduce(pioReadSeek, (*item)->fl); +} - /* cleanup */ - if (fclose(in)) - elog(ERROR, "Cannot close the source file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* close output file */ - if (fclose(out)) - elog(ERROR, "Cannot close the destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - pg_free(iter); - pg_free(in_buf); - pg_free(out_buf); +static void +HeaderMapCache_fobjDispose(VSelf) +{ + Self(HeaderMapCache); + header_map_cache_item_t *it; + ft_assert(header_map_cache == self); + header_map_cache = NULL; - return n_blocks_read; + while (self->first) + { + it = self->first; + self->first = it->next; + ft_str_free(&it->path); + $i(pioClose, it->fl); + $idel(&it->fl); + $idel(&it->err); + ft_free(it); + } } +fobj_klass_handle(HeaderMapCache); + /* * Attempt to open header file, read content and return as * array of headers. * TODO: some access optimizations would be great here: * less fseeks, buffering, descriptor sharing, etc. + * + * Used for post 2.4.0 backups */ BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict) { + FOBJ_FUNC_ARP(); + pioDrive_i drive = pioDriveForLocation(FIO_BACKUP_HOST); bool success = false; - FILE *in = NULL; size_t read_len = 0; pg_crc32 hdr_crc; BackupPageHeader2 *headers = NULL; @@ -2327,6 +1988,9 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b int z_len = 0; char *zheaders = NULL; const char *errormsg = NULL; + pioReadSeek_i reader = {0}; + size_t rc; + err_i err = $noerr(); if (backup_version < 20400) return NULL; @@ -2335,17 +1999,15 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b return NULL; /* TODO: consider to make this descriptor thread-specific */ - in = fopen(hdr_map->path, PG_BINARY_R); - - if (!in) + reader = header_map_cache_open(drive, hdr_map->path, &err); + if ($haserr(err)) { elog(strict ? ERROR : WARNING, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); return NULL; } - /* disable buffering for header file */ - setvbuf(in, NULL, _IONBF, 0); - if (fseeko(in, file->hdr_off, SEEK_SET)) + err = $i(pioSeek, reader, file->hdr_off); + if ($haserr(err)) { elog(strict ? ERROR : WARNING, "Cannot seek to position %llu in page header map \"%s\": %s", file->hdr_off, hdr_map->path, strerror(errno)); @@ -2362,7 +2024,8 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b zheaders = pgut_malloc(file->hdr_size); memset(zheaders, 0, file->hdr_size); - if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) + rc = $i(pioRead, reader, .buf = ft_bytes(zheaders, file->hdr_size), .err = &err); + if ($haserr(err) || rc != file->hdr_size) { elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %llu len: %i \"%s\": %s", file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); @@ -2388,14 +2051,14 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b } /* validate checksum */ - INIT_FILE_CRC32(true, hdr_crc); - COMP_FILE_CRC32(true, hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, hdr_crc); + INIT_CRC32C(hdr_crc); + COMP_CRC32C(hdr_crc, headers, read_len); + FIN_CRC32C(hdr_crc); if (hdr_crc != file->hdr_crc) { elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" " - "offset: %llu, len: %lu, current: %u, expected: %u", + "offset: %llu, len: %zu, current: %u, expected: %u", file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); goto cleanup; } @@ -2405,9 +2068,6 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b cleanup: pg_free(zheaders); - if (in && fclose(in)) - elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); - if (!success) { pg_free(headers); @@ -2422,8 +2082,10 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge) { + FOBJ_FUNC_ARP(); + pioDrive_i drive = pioDriveForLocation(FIO_BACKUP_HOST); + err_i err = $noerr(); size_t read_len = 0; - char *map_path = NULL; /* header compression */ int z_len = 0; char *zheaders = NULL; @@ -2433,13 +2095,12 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, return; /* when running merge we must write headers into temp map */ - map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path; read_len = (file->n_headers + 1) * sizeof(BackupPageHeader2); /* calculate checksums */ - INIT_FILE_CRC32(true, file->hdr_crc); - COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, file->hdr_crc); + INIT_CRC32C(file->hdr_crc); + COMP_CRC32C(file->hdr_crc, headers, read_len); + FIN_CRC32C(file->hdr_crc); zheaders = pgut_malloc(read_len * 2); memset(zheaders, 0, read_len * 2); @@ -2451,23 +2112,18 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, /* writing to header map must be serialized */ pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */ - if (!hdr_map->fp) + if ($isNULL(hdr_map->fp)) { - elog(LOG, "Creating page header map \"%s\"", map_path); - - hdr_map->fp = fopen(map_path, PG_BINARY_A); - if (hdr_map->fp == NULL) - elog(ERROR, "Cannot open header file \"%s\": %s", - map_path, strerror(errno)); - - /* enable buffering for header file */ - hdr_map->buf = pgut_malloc(LARGE_CHUNK_SIZE); - setvbuf(hdr_map->fp, hdr_map->buf, _IOFBF, LARGE_CHUNK_SIZE); + elog(LOG, "Creating page header map \"%s\"", hdr_map->path); - /* update file permission */ - if (chmod(map_path, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", map_path, - strerror(errno)); + hdr_map->fp = $iref( $i(pioOpenRewrite, drive, .path = hdr_map->path, + .permissions = FILE_PERMISSION, .binary = true, + .use_temp = is_merge, .sync = true, + .err = &err) ); + if ($haserr(err)) + { + ft_logerr(FT_FATAL, $errmsg(err), "opening header map for write"); + } file->hdr_off = 0; } @@ -2487,8 +2143,9 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, elog(VERBOSE, "Writing headers for file \"%s\" offset: %llu, len: %i, crc: %u", file->rel_path, file->hdr_off, z_len, file->hdr_crc); - if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) - elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); + err = $i(pioWrite, hdr_map->fp, .buf = ft_bytes(zheaders, z_len)); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "writing header map"); file->hdr_size = z_len; /* save the length of compressed headers */ hdr_map->offset += z_len; /* update current offset in map */ @@ -2502,21 +2159,27 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, void init_header_map(pgBackup *backup) { - backup->hdr_map.fp = NULL; - backup->hdr_map.buf = NULL; + $setNULL(&backup->hdr_map.fp); + join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); - join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; } void cleanup_header_map(HeaderMap *hdr_map) { + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + /* cleanup descriptor */ - if (hdr_map->fp && fclose(hdr_map->fp)) - elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); - hdr_map->fp = NULL; + if ($notNULL(hdr_map->fp)) + { + err = $i(pioClose, hdr_map->fp); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "closing header map"); + $idel(&hdr_map->fp); + $setNULL(&hdr_map->fp); + } + hdr_map->offset = 0; - pg_free(hdr_map->buf); - hdr_map->buf = NULL; } diff --git a/src/datapagemap.c b/src/datapagemap.c index 7e4202a72..15f70893c 100644 --- a/src/datapagemap.c +++ b/src/datapagemap.c @@ -14,12 +14,6 @@ #include "datapagemap.h" -struct datapagemap_iterator -{ - datapagemap_t *map; - BlockNumber nextblkno; -}; - /***** * Public functions */ @@ -65,49 +59,32 @@ datapagemap_add(datapagemap_t *map, BlockNumber blkno) map->bitmap[offset] |= (1 << bitno); } -/* - * Start iterating through all entries in the page map. - * - * After datapagemap_iterate, call datapagemap_next to return the entries, - * until it returns false. After you're done, use pg_free() to destroy the - * iterator. - */ -datapagemap_iterator_t * -datapagemap_iterate(datapagemap_t *map) -{ - datapagemap_iterator_t *iter; - - iter = pg_malloc(sizeof(datapagemap_iterator_t)); - iter->map = map; - iter->nextblkno = 0; - - return iter; -} - bool -datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno) +datapagemap_first(datapagemap_t map, BlockNumber *start_and_result) { - datapagemap_t *map = iter->map; - + BlockNumber blk = *start_and_result; for (;;) { - BlockNumber blk = iter->nextblkno; int nextoff = blk / 8; int bitno = blk % 8; + unsigned char c; - if (nextoff >= map->bitmapsize) + if (nextoff >= map.bitmapsize) break; - iter->nextblkno++; - - if (map->bitmap[nextoff] & (1 << bitno)) + c = map.bitmap[nextoff] >> bitno; + if (c == 0) + blk += 8 - bitno; + else if (c&1) { - *blkno = blk; + *start_and_result = blk; return true; } + else + blk += ffs(c)-1; } /* no more set bits in this bitmap. */ + *start_and_result = UINT32_MAX; return false; -} - +} \ No newline at end of file diff --git a/src/datapagemap.h b/src/datapagemap.h index 6ad7a6204..33000392c 100644 --- a/src/datapagemap.h +++ b/src/datapagemap.h @@ -25,10 +25,8 @@ struct datapagemap }; typedef struct datapagemap datapagemap_t; -typedef struct datapagemap_iterator datapagemap_iterator_t; extern void datapagemap_add(datapagemap_t *map, BlockNumber blkno); -extern datapagemap_iterator_t *datapagemap_iterate(datapagemap_t *map); -extern bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno); +extern bool datapagemap_first(datapagemap_t map, BlockNumber *start_and_result); #endif /* DATAPAGEMAP_H */ diff --git a/src/delete.c b/src/delete.c index 3f299d78b..1b817ae32 100644 --- a/src/delete.c +++ b/src/delete.c @@ -3,7 +3,7 @@ * delete.c: delete backup files. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -725,11 +725,7 @@ do_retention_wal(InstanceState *instanceState, bool dry_run) void delete_backup_files(pgBackup *backup) { - size_t i; char timestamp[100]; - parray *files; - size_t num_files; - char full_path[MAXPGPATH]; /* * If the backup was deleted already, there is nothing to do. @@ -755,31 +751,8 @@ delete_backup_files(pgBackup *backup) */ write_backup_status(backup, BACKUP_STATUS_DELETING, false); - /* list files to be deleted */ - files = parray_new(); - dir_list_file(files, backup->root_dir, false, false, true, false, false, 0, FIO_BACKUP_HOST); - - /* delete leaf node first */ - parray_qsort(files, pgFileCompareRelPathWithExternalDesc); - num_files = parray_num(files); - for (i = 0; i < num_files; i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - join_path_components(full_path, backup->root_dir, file->rel_path); - - if (interrupted) - elog(ERROR, "interrupted during delete backup"); - - if (progress) - elog(INFO, "Progress: (%zd/%zd). Delete file \"%s\"", - i + 1, num_files, full_path); - - pgFileDelete(file->mode, full_path); - } - - parray_walk(files, pgFileFree); - parray_free(files); + pioDrive_i drive = pioDriveForLocation(FIO_BACKUP_HOST); + $i(pioRemoveDir, drive, .root = backup->root_dir, .root_as_well = true); backup->status = BACKUP_STATUS_DELETED; return; @@ -815,8 +788,8 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli char first_to_del_str[MAXFNAMELEN]; char oldest_to_keep_str[MAXFNAMELEN]; int i; - size_t wal_size_logical = 0; - size_t wal_size_actual = 0; + int64_t wal_size_logical = 0; + int64_t wal_size_actual = 0; char wal_pretty_size[20]; bool purge_all = false; @@ -858,7 +831,7 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli /* sanity */ if (OldestToKeepSegNo > FirstToDeleteSegNo) { - wal_size_logical = (OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size; + wal_size_logical = (int64_t)(OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size; /* In case of 'purge all' scenario OldestToKeepSegNo will be deleted too */ if (purge_all) @@ -904,7 +877,7 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); if (purge_all || wal_file->segno < OldestToKeepSegNo) - wal_size_actual += wal_file->file.size; + wal_size_actual += wal_file->size; } /* Report the actual size to delete */ @@ -932,7 +905,7 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli { char wal_fullpath[MAXPGPATH]; - join_path_components(wal_fullpath, instanceState->instance_wal_subdir_path, wal_file->file.name); + join_path_components(wal_fullpath, instanceState->instance_wal_subdir_path, wal_file->name.ptr); /* save segment from purging */ if (wal_file->keep) @@ -941,13 +914,11 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli continue; } - /* unlink segment */ - if (fio_unlink(wal_fullpath, FIO_BACKUP_HOST) < 0) + /* remove segment, missing file is not considered as error condition */ + if (fio_remove(FIO_BACKUP_HOST, wal_fullpath, true) < 0) { - /* Missing file is not considered as error condition */ - if (errno != ENOENT) - elog(ERROR, "Could not remove file \"%s\": %s", - wal_fullpath, strerror(errno)); + elog(ERROR, "Could not remove file \"%s\": %s", + wal_fullpath, strerror(errno)); } else { diff --git a/src/dir.c b/src/dir.c index 0a55c0f67..36d248d28 100644 --- a/src/dir.c +++ b/src/dir.c @@ -13,16 +13,14 @@ #include "utils/file.h" -#if PG_VERSION_NUM < 110000 -#include "catalog/catalog.h" -#endif -#include "catalog/pg_tablespace.h" - #include -#include #include #include "utils/configuration.h" +#include "catalog/pg_tablespace.h" +#if PG_VERSION_NUM < 110000 +#include "catalog/catalog.h" +#endif /* * The contents of these directories are removed or recreated during server @@ -83,11 +81,7 @@ static char *pgdata_exclude_files[] = "probackup_recovery.conf", "recovery.signal", "standby.signal", - NULL -}; -static char *pgdata_exclude_files_non_exclusive[] = -{ /*skip in non-exclusive backup */ "backup_label", "tablespace_map", @@ -124,73 +118,61 @@ typedef struct TablespaceCreatedList static char dir_check_file(pgFile *file, bool backup_logs); -static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, - bool exclude, bool follow_symlink, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); static void cleanup_tablespace(const char *path); -static void control_string_bad_format(const char* str); +static void control_string_bad_format(ft_bytes_t str); +static void print_database_map(ft_strbuf_t *buf, parray *database_list); + /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; /* Extra directories mapping */ static TablespaceList external_remap_list = {NULL, NULL}; -/* - * Create directory, also create parent directories if necessary. - * In strict mode treat already existing directory as error. - * Return values: - * 0 - ok - * -1 - error (check errno) - */ -int -dir_create_dir(const char *dir, mode_t mode, bool strict) +static void +pgFileSetStat(pgFile* file, pio_stat_t st) { - char parent[MAXPGPATH]; - - strncpy(parent, dir, MAXPGPATH); - get_parent_directory(parent); - - /* Create parent first */ - if (access(parent, F_OK) == -1) - dir_create_dir(parent, mode, false); - - /* Create directory */ - if (mkdir(dir, mode) == -1) - { - if (errno == EEXIST && !strict) /* already exist */ - return 0; - return -1; - } - - return 0; + file->size = st.pst_size; + file->kind = st.pst_kind; + file->mode = st.pst_mode; + file->mtime = st.pst_mtime; } pgFile * pgFileNew(const char *path, const char *rel_path, bool follow_symlink, - int external_dir_num, fio_location location) + bool crc, pioDrive_i drive) { - struct stat st; + FOBJ_FUNC_ARP(); + pio_stat_t st; pgFile *file; + err_i err; /* stat the file */ - if (fio_stat(path, &st, follow_symlink, location) < 0) - { - /* file not found is not an error case */ - if (errno == ENOENT) - return NULL; - elog(ERROR, "cannot stat file \"%s\": %s", path, - strerror(errno)); + st = $i(pioStat, drive, .path = path, .follow_symlink = follow_symlink, + .err = &err); + if ($haserr(err)) { + ft_logerr(FT_FATAL, $errmsg(err), "pgFileNew"); } file = pgFileInit(rel_path); - file->size = st.st_size; - file->mode = st.st_mode; - file->mtime = st.st_mtime; - file->external_dir_num = external_dir_num; + pgFileSetStat(file, st); + if (file->kind == PIO_KIND_REGULAR) + { + file->write_size = file->size; + file->uncompressed_size = file->size; + } + if (file->kind == PIO_KIND_REGULAR && crc) + { + file->crc = $i(pioGetCRC32, drive, .path = path, + .compressed = false, .err = &err); + if ($haserr(err)) { + pgFileFree(file); + ft_logerr(FT_FATAL, $errmsg(err), "pgFileNew"); + } + } return file; } @@ -201,8 +183,7 @@ pgFileInit(const char *rel_path) pgFile *file; char *file_name = NULL; - file = (pgFile *) pgut_malloc(sizeof(pgFile)); - MemSet(file, 0, sizeof(pgFile)); + file = (pgFile *) pgut_malloc0(sizeof(pgFile)); file->rel_path = pgut_strdup(rel_path); canonicalize_path(file->rel_path); @@ -230,38 +211,6 @@ pgFileInit(const char *rel_path) return file; } -/* - * Delete file pointed by the pgFile. - * If the pgFile points directory, the directory must be empty. - */ -void -pgFileDelete(mode_t mode, const char *full_path) -{ - if (S_ISDIR(mode)) - { - if (rmdir(full_path) == -1) - { - if (errno == ENOENT) - return; - else if (errno == ENOTDIR) /* could be symbolic link */ - goto delete_file; - - elog(ERROR, "Cannot remove directory \"%s\": %s", - full_path, strerror(errno)); - } - return; - } - -delete_file: - if (remove(full_path) == -1) - { - if (errno == ENOENT) - return; - elog(ERROR, "Cannot remove file \"%s\": %s", full_path, - strerror(errno)); - } -} - void pgFileFree(void *file) { @@ -364,24 +313,31 @@ pgFileCompareLinked(const void *f1, const void *f2) /* Compare two pgFile with their size */ int -pgFileCompareSize(const void *f1, const void *f2) +pgFileCompareSizeDesc(const void *f1, const void *f2) { pgFile *f1p = *(pgFile **)f1; pgFile *f2p = *(pgFile **)f2; - if (f1p->size > f2p->size) + if (f1p->size < f2p->size) return 1; - else if (f1p->size < f2p->size) + else if (f1p->size > f2p->size) return -1; else return 0; } -/* Compare two pgFile with their size in descending order */ +/* + * Compare files by offset in headers file. + * It really matters during restore. + */ int -pgFileCompareSizeDesc(const void *f1, const void *f2) +pgFileCompareByHdrOff(const void *f1, const void *f2) { - return -1 * pgFileCompareSize(f1, f2); + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return f1p->hdr_off < f2p->hdr_off ? -1 : + f1p->hdr_off > f2p->hdr_off; } int @@ -428,51 +384,6 @@ db_map_entry_free(void *entry) free(entry); } -/* - * List files, symbolic links and directories in the directory "root" and add - * pgFile objects to "files". We add "root" to "files" if add_root is true. - * - * When follow_symlink is true, symbolic link is ignored and only file or - * directory linked to will be listed. - * - * TODO: make it strictly local - */ -void -dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, - bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, - fio_location location) -{ - pgFile *file; - - file = pgFileNew(root, "", follow_symlink, external_dir_num, location); - if (file == NULL) - { - /* For external directory this is not ok */ - if (external_dir_num > 0) - elog(ERROR, "External directory is not found: \"%s\"", root); - else - return; - } - - if (!S_ISDIR(file->mode)) - { - if (external_dir_num > 0) - elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", - root); - else - elog(WARNING, "Skip \"%s\": unexpected file format", root); - return; - } - if (add_root) - parray_append(files, file); - - dir_list_file_internal(files, file, root, exclude, follow_symlink, - backup_logs, skip_hidden, external_dir_num, location); - - if (!add_root) - pgFileFree(file); -} - #define CHECK_FALSE 0 #define CHECK_TRUE 1 #define CHECK_EXCLUDE_FALSE 2 @@ -494,26 +405,14 @@ static char dir_check_file(pgFile *file, bool backup_logs) { int i; - int sscanf_res; + int sscanf_res; bool in_tablespace = false; in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path); /* Check if we need to exclude file by name */ - if (S_ISREG(file->mode)) + if (file->kind == PIO_KIND_REGULAR) { - if (!exclusive_backup) - { - for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++) - if (strcmp(file->rel_path, - pgdata_exclude_files_non_exclusive[i]) == 0) - { - /* Skip */ - elog(LOG, "Excluding file: %s", file->name); - return CHECK_FALSE; - } - } - for (i = 0; pgdata_exclude_files[i]; i++) if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0) { @@ -526,7 +425,7 @@ dir_check_file(pgFile *file, bool backup_logs) * If the directory name is in the exclude list, do not list the * contents. */ - else if (S_ISDIR(file->mode) && !in_tablespace && file->external_dir_num == 0) + else if (file->kind == PIO_KIND_DIRECTORY && !in_tablespace && file->external_dir_num == 0) { /* * If the item in the exclude list starts with '/', compare to @@ -558,7 +457,7 @@ dir_check_file(pgFile *file, bool backup_logs) * Do not copy tablespaces twice. It may happen if the tablespace is located * inside the PGDATA. */ - if (S_ISDIR(file->mode) && + if (file->kind == PIO_KIND_DIRECTORY && strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) { Oid tblspcOid; @@ -603,14 +502,14 @@ dir_check_file(pgFile *file, bool backup_logs) } /* Do not backup ptrack_init files */ - if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) + if (file->kind == PIO_KIND_REGULAR && strcmp(file->name, "ptrack_init") == 0) return CHECK_FALSE; /* * Check files located inside database directories including directory * 'global' */ - if (S_ISREG(file->mode) && file->tblspcOid != 0 && + if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 && file->name && file->name[0]) { if (strcmp(file->name, "pg_internal.init") == 0) @@ -638,78 +537,58 @@ dir_check_file(pgFile *file, bool backup_logs) } /* - * List files in parent->path directory. If "exclude" is true do not add into - * "files" files from pgdata_exclude_files and directories from - * pgdata_exclude_dir. - * - * TODO: should we check for interrupt here ? + * List files and directories in the directory "root" and add + * pgFile objects to "files". */ -static void -dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, - bool exclude, bool follow_symlink, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location) +void +db_list_dir(parray *files, const char* root, + bool exclude, bool backup_logs, + int external_dir_num, fio_location location) { - DIR *dir; - struct dirent *dent; + FOBJ_FUNC_ARP(); + pio_dirent_t dent; + pioDrive_i drive; + pioRecursiveDir* dir; + err_i err; - if (!S_ISDIR(parent->mode)) - elog(ERROR, "\"%s\" is not a directory", parent_dir); + drive = pioDriveForLocation(location); /* Open directory and list contents */ - dir = fio_opendir(parent_dir, location); + dir = pioRecursiveDir_alloc(drive, root, &err); if (dir == NULL) { - if (errno == ENOENT) + if (getErrno(err) == ENOENT && external_dir_num == 0) { /* Maybe the directory was removed */ + /* XXX: why the hell it is "ok" for non-external directories? */ return; } - elog(ERROR, "Cannot open directory \"%s\": %s", - parent_dir, strerror(errno)); + if (getErrno(err) == ENOENT && external_dir_num != 0) + { + ft_logerr(FT_FATAL, $errmsg(err), "External directory is not found"); + } + if (getErrno(err) == ENOTDIR && external_dir_num != 0) + { + elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", + root); + } + ft_logerr(FT_FATAL, $errmsg(err), "Listing directory"); } - errno = 0; - while ((dent = fio_readdir(dir))) + while ((dent = pioRecursiveDir_next(dir, &err)).stat.pst_kind) { pgFile *file; char child[MAXPGPATH]; - char rel_child[MAXPGPATH]; char check_res; - join_path_components(child, parent_dir, dent->d_name); - join_path_components(rel_child, parent->rel_path, dent->d_name); - - file = pgFileNew(child, rel_child, follow_symlink, external_dir_num, - location); - if (file == NULL) - continue; + join_path_components(child, root, dent.name.ptr); - /* Skip entries point current dir or parent dir */ - if (S_ISDIR(file->mode) && - (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) - { - pgFileFree(file); - continue; - } + file = pgFileInit(dent.name.ptr); + pgFileSetStat(file, dent.stat); + file->external_dir_num = external_dir_num; - /* skip hidden files and directories */ - if (skip_hidden && file->name[0] == '.') - { - elog(WARNING, "Skip hidden file: '%s'", child); - pgFileFree(file); - continue; - } - - /* - * Add only files, directories and links. Skip sockets and other - * unexpected file formats. - */ - if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) - { - elog(WARNING, "Skip '%s': unexpected file format", child); - pgFileFree(file); - continue; - } + /* pioDirIter will not return other kinds of entries */ + ft_assert(file->kind == PIO_KIND_REGULAR || file->kind == PIO_KIND_DIRECTORY); if (exclude) { @@ -717,11 +596,15 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, if (check_res == CHECK_FALSE) { /* Skip */ + if (file->kind == PIO_KIND_DIRECTORY) + pioRecursiveDir_dont_recurse_current(dir); pgFileFree(file); continue; } else if (check_res == CHECK_EXCLUDE_FALSE) { + ft_assert(file->kind == PIO_KIND_DIRECTORY); + pioRecursiveDir_dont_recurse_current(dir); /* We add the directory itself which content was excluded */ parray_append(files, file); continue; @@ -729,24 +612,11 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, } parray_append(files, file); - - /* - * If the entry is a directory call dir_list_file_internal() - * recursively. - */ - if (S_ISDIR(file->mode)) - dir_list_file_internal(files, file, child, exclude, follow_symlink, - backup_logs, skip_hidden, external_dir_num, location); } - if (errno && errno != ENOENT) - { - int errno_tmp = errno; - fio_closedir(dir); - elog(ERROR, "Cannot read directory \"%s\": %s", - parent_dir, strerror(errno_tmp)); - } - fio_closedir(dir); + pioRecursiveDir_close(dir); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Listing directory \"%s\"", root); } /* @@ -863,10 +733,13 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba bool extract_tablespaces, bool incremental, fio_location location, const char* waldir_path) { + pioDrive_i drive = pioDriveForLocation(location); int i; parray *links = NULL; mode_t pg_tablespace_mode = DIR_PERMISSION; char to_path[MAXPGPATH]; + err_i err = $noerr(); + if (waldir_path && !dir_is_empty(waldir_path, location)) { @@ -899,7 +772,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba { pgFile *file = (pgFile *) parray_get(dest_files, i); - if (!S_ISDIR(file->mode)) + if (file->kind != PIO_KIND_DIRECTORY) continue; /* skip external directory content */ @@ -932,7 +805,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba char parent_dir[MAXPGPATH]; pgFile *dir = (pgFile *) parray_get(dest_files, i); - if (!S_ISDIR(dir->mode)) + if (dir->kind != PIO_KIND_DIRECTORY) continue; /* skip external directory content */ @@ -948,10 +821,16 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba waldir_path, to_path); /* create tablespace directory from waldir_path*/ - fio_mkdir(waldir_path, pg_tablespace_mode, location); + err = $i(pioMakeDir, drive, .path = waldir_path, + .mode = pg_tablespace_mode, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create tablespace directory: %s", + $errmsg(err)); + } /* create link to linked_path */ - if (fio_symlink(waldir_path, to_path, incremental, location) < 0) + if (fio_symlink(location, waldir_path, to_path, incremental) < 0) elog(ERROR, "Could not create symbolic link \"%s\": %s", to_path, strerror(errno)); @@ -990,10 +869,16 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba linked_path, to_path); /* create tablespace directory */ - fio_mkdir(linked_path, pg_tablespace_mode, location); + err = $i(pioMakeDir, drive, .path = linked_path, + .mode = pg_tablespace_mode, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create tablespace directory: %s", + $errmsg(err)); + } /* create link to linked_path */ - if (fio_symlink(linked_path, to_path, incremental, location) < 0) + if (fio_symlink(location, linked_path, to_path, incremental) < 0) elog(ERROR, "Could not create symbolic link \"%s\": %s", to_path, strerror(errno)); @@ -1007,8 +892,13 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba join_path_components(to_path, data_dir, dir->rel_path); - // TODO check exit code - fio_mkdir(to_path, dir->mode, location); + err = $i(pioMakeDir, drive, .path = to_path, .mode = dir->mode, + .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create tablespace directory: %s", + $errmsg(err)); + } } if (extract_tablespaces) @@ -1025,51 +915,52 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba void read_tablespace_map(parray *links, const char *backup_dir) { - FILE *fp; char db_path[MAXPGPATH], map_path[MAXPGPATH]; - char buf[MAXPGPATH * 2]; + pioDrive_i drive; + ft_bytes_t content; + ft_bytes_t parse; + ft_bytes_t line; + err_i err = $noerr(); join_path_components(db_path, backup_dir, DATABASE_DIR); join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); - fp = fio_open_stream(map_path, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open tablespace map file \"%s\": %s", map_path, strerror(errno)); + drive = pioDriveForLocation(FIO_BACKUP_HOST); + + content = $i(pioReadFile, drive, .path = map_path, .binary = false, + .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Reading tablespace map"); - while (fgets(buf, lengthof(buf), fp)) + parse = content; + + while (parse.len) { - char link_name[MAXPGPATH]; - char *path; - int n = 0; + ft_bytes_t link_name; + ft_bytes_t path; pgFile *file; - int i = 0; - if (sscanf(buf, "%s %n", link_name, &n) != 1) - elog(ERROR, "invalid format found in \"%s\"", map_path); + line = ft_bytes_shift_line(&parse); - path = buf + n; + link_name = ft_bytes_split(&line, ft_bytes_notspnc(line, " ")); + ft_bytes_consume(&line, 1); + path = ft_bytes_split(&line, ft_bytes_notspnc(line, "\n\r")); - /* Remove newline character at the end of string if any */ - i = strcspn(path, "\n"); - if (strlen(path) > i) - path[i] = '\0'; + if (link_name.len == 0 || path.len == 0) + elog(ERROR, "invalid format found in \"%s\"", map_path); - file = pgut_new(pgFile); - memset(file, 0, sizeof(pgFile)); + file = pgut_new0(pgFile); /* follow the convention for pgFileFree */ - file->name = pgut_strdup(link_name); - file->linked = pgut_strdup(path); + file->name = ft_strdup_bytes(link_name).ptr; + file->linked = ft_strdup_bytes(path).ptr; canonicalize_path(file->linked); parray_append(links, file); } - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", map_path); - - fio_close_stream(fp); + ft_bytes_free(&content); } /* @@ -1116,6 +1007,11 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pg if (tablespace_dirs.head != NULL) elog(ERROR, "Backup %s has no tablespaceses, nothing to remap " "via \"--tablespace-mapping\" option", backup_id_of(backup)); + + free(tmp_file); + parray_walk(links, pgFileFree); + parray_free(links); + return NoTblspc; } @@ -1319,160 +1215,183 @@ get_external_remap(char *current_dir) return current_dir; } -/* Parsing states for get_control_value_str() */ -#define CONTROL_WAIT_NAME 1 -#define CONTROL_INNAME 2 -#define CONTROL_WAIT_COLON 3 -#define CONTROL_WAIT_VALUE 4 -#define CONTROL_INVALUE 5 -#define CONTROL_WAIT_NEXT_NAME 6 - /* - * Get value from json-like line "str" of backup_content.control file. + * Parse values from json-like line "str" of backup_content.control file. * * The line has the following format: * {"name1":"value1", "name2":"value2"} - * - * The value will be returned in "value_int64" as int64. - * - * Returns true if the value was found in the line and parsed. */ -bool -get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory) -{ - char buf_int64[32]; +typedef struct pb_control_line_kv { + uint32_t key_hash; + uint32_t key_start; + uint32_t key_len; + uint32_t val_len; +} pb_control_line_kv; - assert(value_int64); +#define FT_SLICE clkv +#define FT_SLICE_TYPE pb_control_line_kv +#include "ft_array.inc.h" - /* Set default value */ - *value_int64 = 0; +void +init_pb_control_line(pb_control_line* pb_line) +{ + pb_line->kvs = ft_malloc(sizeof(ft_arr_clkv_t)); + *pb_line->kvs = (ft_arr_clkv_t)ft_arr_init(); + pb_line->strbuf = ft_strbuf_zero(); +} - if (!get_control_value_str(str, name, buf_int64, sizeof(buf_int64), is_mandatory)) - return false; +void +parse_pb_control_line(pb_control_line* pb_line, ft_bytes_t line) +{ + pb_control_line_kv kv = {0}; + ft_strbuf_t *strbuf = &pb_line->strbuf; + ft_bytes_t parse; + + pb_line->line = line; + ft_arr_clkv_reset_for_reuse(pb_line->kvs); + ft_strbuf_reset_for_reuse(&pb_line->strbuf); - if (!parse_int64(buf_int64, value_int64, 0)) + parse = line; + ft_bytes_consume(&parse, ft_bytes_spnc(parse, "{ \t")); + while (parse.len) { - /* We assume that too big value is -1 */ - if (errno == ERANGE) - *value_int64 = BYTES_INVALID; - else - control_string_bad_format(str); - return false; + ft_bytes_t name; + ft_bytes_t value; + + ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES)); + /* name in quotes */ + if (!ft_bytes_starts_withc(parse, "\"")) + control_string_bad_format(line); + ft_bytes_consume(&parse, 1); /* skip quote */ + + name = ft_bytes_split(&parse, ft_bytes_notspnc(parse, "\"")); + if (!ft_bytes_starts_withc(parse, "\"")) + control_string_bad_format(line); + kv.key_start = strbuf->len; + kv.key_len = name.len; + ft_strbuf_catbytes(strbuf, name); + ft_strbuf_cat1(strbuf, '\0'); + kv.key_hash = ft_small_cstr_hash(strbuf->ptr + kv.key_start); + + ft_bytes_consume(&parse, 1); /* skip quote */ + ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES)); + if (!ft_bytes_starts_withc(parse, ":")) + control_string_bad_format(line); + ft_bytes_consume(&parse, 1); /* skip colon */ + ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES)); + + /* value in quotes */ + if (!ft_bytes_starts_withc(parse, "\"")) + control_string_bad_format(line); + ft_bytes_consume(&parse, 1); /* skip quote */ + + value = ft_bytes_split(&parse, ft_bytes_notspnc(parse, "\"")); + if (!ft_bytes_starts_withc(parse, "\"")) + control_string_bad_format(line); + kv.val_len = value.len; + ft_strbuf_catbytes(strbuf, value); + ft_strbuf_cat1(strbuf, '\0'); + ft_arr_clkv_push(pb_line->kvs, kv); + + ft_bytes_consume(&parse, 1); /* skip quote */ + ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES)); + if (ft_bytes_starts_withc(parse, ",")) + { + ft_bytes_consume(&parse, 1); + continue; + } + break; } - return true; + if (!ft_bytes_starts_withc(parse, "}")) + control_string_bad_format(line); + ft_bytes_consume(&parse, 1); + ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES)); + if (parse.len != 0) + control_string_bad_format(line); } -/* - * Get value from json-like line "str" of backup_content.control file. - * - * The line has the following format: - * {"name1":"value1", "name2":"value2"} - * - * The value will be returned to "value_str" as string. - * - * Returns true if the value was found in the line. - */ +void +deinit_pb_control_line(pb_control_line *pb_line) +{ + ft_arr_clkv_free(pb_line->kvs); + ft_free(pb_line->kvs); + pb_line->kvs = NULL; + ft_strbuf_free(&pb_line->strbuf); +} bool -get_control_value_str(const char *str, const char *name, - char *value_str, size_t value_str_size, bool is_mandatory) +pb_control_line_try_str(pb_control_line *pb_line, const char *name, ft_str_t *value) { - int state = CONTROL_WAIT_NAME; - char *name_ptr = (char *) name; - char *buf = (char *) str; - char *const value_str_start = value_str; + pb_control_line_kv kv; + ft_str_t key; + uint32_t i; + uint32_t key_hash = ft_small_cstr_hash(name); - assert(value_str); - assert(value_str_size > 0); + for (i = 0; i < pb_line->kvs->len; i++) + { + kv = ft_arr_clkv_at(pb_line->kvs, i); + if (kv.key_hash != key_hash) + continue; + key = ft_str(pb_line->strbuf.ptr + kv.key_start, kv.key_len); + if (!ft_streqc(key, name)) + continue; + *value = ft_str(pb_line->strbuf.ptr + kv.key_start + kv.key_len + 1, + kv.val_len); + return true; + } + *value = ft_str("", 0); + return false; +} - /* Set default value */ - *value_str = '\0'; +bool +pb_control_line_try_int64(pb_control_line *pb_line, const char *name, int64 *value) +{ + ft_str_t val; - while (*buf) - { - switch (state) - { - case CONTROL_WAIT_NAME: - if (*buf == '"') - state = CONTROL_INNAME; - else if (IsAlpha(*buf)) - control_string_bad_format(str); - break; - case CONTROL_INNAME: - /* Found target field. Parse value. */ - if (*buf == '"') - state = CONTROL_WAIT_COLON; - /* Check next field */ - else if (*buf != *name_ptr) - { - name_ptr = (char *) name; - state = CONTROL_WAIT_NEXT_NAME; - } - else - name_ptr++; - break; - case CONTROL_WAIT_COLON: - if (*buf == ':') - state = CONTROL_WAIT_VALUE; - else if (!IsSpace(*buf)) - control_string_bad_format(str); - break; - case CONTROL_WAIT_VALUE: - if (*buf == '"') - { - state = CONTROL_INVALUE; - } - else if (IsAlpha(*buf)) - control_string_bad_format(str); - break; - case CONTROL_INVALUE: - /* Value was parsed, exit */ - if (*buf == '"') - { - *value_str = '\0'; - return true; - } - else - { - /* verify if value_str not exceeds value_str_size limits */ - if (value_str - value_str_start >= value_str_size - 1) { - elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", - name, str, DATABASE_FILE_LIST); - } - *value_str = *buf; - value_str++; - } - break; - case CONTROL_WAIT_NEXT_NAME: - if (*buf == ',') - state = CONTROL_WAIT_NAME; - break; - default: - /* Should not happen */ - break; - } + *value = 0; + if (!pb_control_line_try_str(pb_line, name, &val)) + return false; - buf++; + if (!parse_int64(val.ptr, value, 0)) + { + /* We assume that too big value is -1 */ + if (errno == ERANGE) + *value = BYTES_INVALID; + else + control_string_bad_format(pb_line->line); + return false; } - /* There is no close quotes */ - if (state == CONTROL_INNAME || state == CONTROL_INVALUE) - control_string_bad_format(str); + return true; +} - /* Did not find target field */ - if (is_mandatory) - elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", - name, str, DATABASE_FILE_LIST); - return false; +ft_str_t +pb_control_line_get_str(pb_control_line *pb_line, const char *name) +{ + ft_str_t res; + if (!pb_control_line_try_str(pb_line, name, &res)) + elog(ERROR, "field \"%s\" is not found in the line %.*s of the file %s", + name, (int)pb_line->line.len, pb_line->line.ptr, DATABASE_FILE_LIST); + return res; +} + +int64_t +pb_control_line_get_int64(pb_control_line *pb_line, const char *name) +{ + int64_t res; + if (!pb_control_line_try_int64(pb_line, name, &res)) + elog(ERROR, "field \"%s\" is not found in the line %.*s of the file %s", + name, (int)pb_line->line.len, pb_line->line.ptr, DATABASE_FILE_LIST); + return res; } static void -control_string_bad_format(const char* str) +control_string_bad_format(ft_bytes_t str) { - elog(ERROR, "%s file has invalid format in line %s", - DATABASE_FILE_LIST, str); + elog(ERROR, "%s file has invalid format in line %.*s", + DATABASE_FILE_LIST, (int)str.len, str.ptr); } /* @@ -1481,36 +1400,15 @@ control_string_bad_format(const char* str) bool dir_is_empty(const char *path, fio_location location) { - DIR *dir; - struct dirent *dir_ent; + pioDrive_i drive = pioDriveForLocation(location); + err_i err; + bool is_empty; - dir = fio_opendir(path, location); - if (dir == NULL) - { - /* Directory in path doesn't exist */ - if (errno == ENOENT) - return true; - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); - } + is_empty = $i(pioIsDirEmpty, drive, path, &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Checking dir is empty"); - errno = 0; - while ((dir_ent = fio_readdir(dir))) - { - /* Skip entries point current dir or parent dir */ - if (strcmp(dir_ent->d_name, ".") == 0 || - strcmp(dir_ent->d_name, "..") == 0) - continue; - - /* Directory is not empty */ - fio_closedir(dir); - return false; - } - if (errno) - elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno)); - - fio_closedir(dir); - - return true; + return is_empty; } /* @@ -1519,25 +1417,14 @@ dir_is_empty(const char *path, fio_location location) bool fileExists(const char *path, fio_location location) { - struct stat buf; + FOBJ_FUNC_ARP(); + err_i err; + bool exists; - if (fio_stat(path, &buf, true, location) == -1 && errno == ENOENT) - return false; - else if (!S_ISREG(buf.st_mode)) - return false; - else - return true; -} - -size_t -pgFileSize(const char *path) -{ - struct stat buf; - - if (stat(path, &buf) == -1) - elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + exists = $i(pioExists, pioDriveForLocation(location), .path = path, + .err = &err); - return buf.st_size; + return exists; } /* @@ -1619,8 +1506,8 @@ backup_contains_external(const char *dir, parray *dirs_list) /* * Print database_map */ -void -print_database_map(FILE *out, parray *database_map) +static void +print_database_map(ft_strbuf_t *buf, parray *database_map) { int i; @@ -1628,7 +1515,7 @@ print_database_map(FILE *out, parray *database_map) { db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, i); - fio_fprintf(out, "{\"dbOid\":\"%u\", \"datname\":\"%s\"}\n", + ft_strbuf_catf(buf, "{\"dbOid\":\"%u\", \"datname\":\"%s\"}\n", db_entry->dbOid, db_entry->datname); } @@ -1641,34 +1528,29 @@ print_database_map(FILE *out, parray *database_map) void write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_list) { - FILE *fp; + ft_strbuf_t buf; pgFile *file; char database_dir[MAXPGPATH]; char database_map_path[MAXPGPATH]; + pioDrive_i drive = pioDriveForLocation(FIO_BACKUP_HOST); + err_i err; join_path_components(database_dir, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, database_dir, DATABASE_MAP); - fp = fio_fopen(database_map_path, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open database map \"%s\": %s", database_map_path, - strerror(errno)); + buf = ft_strbuf_zero(); + print_database_map(&buf, database_map); - print_database_map(fp, database_map); - if (fio_fflush(fp) || fio_fclose(fp)) - { - fio_unlink(database_map_path, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write database map \"%s\": %s", - database_map_path, strerror(errno)); - } + err = $i(pioWriteFile, drive, .path = database_map_path, + .content = ft_str2bytes(ft_strbuf_ref(&buf))); - /* Add metadata to backup_content.control */ - file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, - FIO_BACKUP_HOST); - file->crc = pgFileGetCRC(database_map_path, true, false); - file->write_size = file->size; - file->uncompressed_size = file->size; + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Writting database map"); + + ft_strbuf_free(&buf); + /* Add metadata to backup_content.control */ + file = pgFileNew(database_map_path, DATABASE_MAP, true, true, drive); parray_append(backup_files_list, file); } @@ -1678,47 +1560,50 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ parray * read_database_map(pgBackup *backup) { - FILE *fp; parray *database_map; - char buf[MAXPGPATH]; char path[MAXPGPATH]; char database_map_path[MAXPGPATH]; + pioDrive_i drive; + err_i err = $noerr(); + ft_bytes_t content; + ft_bytes_t parse; + ft_bytes_t line; + pb_control_line pb_line; join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); - fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST); - if (fp == NULL) - { - /* It is NOT ok for database_map to be missing at this point, so - * we should error here. - * It`s a job of the caller to error if database_map is not empty. - */ - elog(ERROR, "Cannot open \"%s\": %s", database_map_path, strerror(errno)); - } + drive = pioDriveForLocation(FIO_BACKUP_HOST); + + content = $i(pioReadFile, drive, .path = database_map_path, .binary = false, + .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Reading database_map"); database_map = parray_new(); - while (fgets(buf, lengthof(buf), fp)) + init_pb_control_line(&pb_line); + + parse = content; + while (parse.len > 0) { - char datname[MAXPGPATH]; + ft_str_t datname; int64 dbOid; + db_map_entry *db_entry = pgut_new0(db_map_entry); - db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); + line = ft_bytes_shift_line(&parse); + parse_pb_control_line(&pb_line, line); - get_control_value_int64(buf, "dbOid", &dbOid, true); - get_control_value_str(buf, "datname", datname, sizeof(datname), true); + dbOid = pb_control_line_get_int64(&pb_line, "dbOid"); + datname = pb_control_line_get_str(&pb_line, "datname"); db_entry->dbOid = dbOid; - db_entry->datname = pgut_strdup(datname); + db_entry->datname = ft_strdup(datname).ptr; parray_append(database_map, db_entry); } - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", database_map_path); - - fio_close_stream(fp); + deinit_pb_control_line(&pb_line); /* Return NULL if file is empty */ if (parray_num(database_map) == 0) @@ -1738,27 +1623,8 @@ read_database_map(pgBackup *backup) void cleanup_tablespace(const char *path) { - int i; - char fullpath[MAXPGPATH]; - parray *files = parray_new(); - - fio_list_dir(files, path, false, false, false, false, false, 0); - - /* delete leaf node first */ - parray_qsort(files, pgFileCompareRelPathWithExternalDesc); - - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - join_path_components(fullpath, path, file->rel_path); - - fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(LOG, "Deleted file \"%s\"", fullpath); - } - - parray_walk(files, pgFileFree); - parray_free(files); + pioDrive_i drive = pioDriveForLocation(FIO_DB_HOST); + $i(pioRemoveDir, drive, .root = path, .root_as_well = false); } /* diff --git a/src/extra.h b/src/extra.h new file mode 100644 index 000000000..89b8474e7 --- /dev/null +++ b/src/extra.h @@ -0,0 +1,32 @@ +#ifndef __EXTRA_H__ +#define __EXTRA_H__ + +typedef enum CompressAlg +{ + NOT_DEFINED_COMPRESS = 0, + NONE_COMPRESS, + PGLZ_COMPRESS, + ZLIB_COMPRESS, +} CompressAlg; + +typedef struct PageState +{ + uint16 checksum; + XLogRecPtr lsn; +} PageState; + +typedef enum BackupMode +{ + BACKUP_MODE_INVALID = 0, + BACKUP_MODE_DIFF_PAGE, /* incremental page backup */ + BACKUP_MODE_DIFF_PTRACK, /* incremental page backup with ptrack system */ + BACKUP_MODE_DIFF_DELTA, /* incremental page backup with lsn comparison */ + BACKUP_MODE_FULL /* full backup */ +} BackupMode; + +typedef struct pgFile pgFile; + +int compress_page(char *write_buffer, size_t buffer_size, BlockNumber blknum, void *page, + CompressAlg calg, int clevel, const char *from_fullpath); + +#endif /* __EXTRA_H__ */ diff --git a/src/fetch.c b/src/fetch.c deleted file mode 100644 index bef30dac6..000000000 --- a/src/fetch.c +++ /dev/null @@ -1,108 +0,0 @@ -/*------------------------------------------------------------------------- - * - * fetch.c - * Functions for fetching files from PostgreSQL data directory - * - * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include - -/* - * Read a file into memory. The file to be read is /. - * The file contents are returned in a malloc'd buffer, and *filesize - * is set to the length of the file. - * - * The returned buffer is always zero-terminated; the size of the returned - * buffer is actually *filesize + 1. That's handy when reading a text file. - * This function can be used to read binary files as well, you can just - * ignore the zero-terminator in that case. - * - */ -char * -slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fio_location location) -{ - int fd; - char *buffer; - struct stat statbuf; - char fullpath[MAXPGPATH]; - int len; - - join_path_components(fullpath, datadir, path); - - if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, location)) == -1) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not open file \"%s\" for reading: %s", - fullpath, strerror(errno)); - } - - if (fio_stat(fullpath, &statbuf, true, location) < 0) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not stat file \"%s\": %s", - fullpath, strerror(errno)); - } - - len = statbuf.st_size; - buffer = pg_malloc(len + 1); - - if (fio_read(fd, buffer, len) != len) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not read file \"%s\": %s\n", - fullpath, strerror(errno)); - } - - fio_close(fd); - - /* Zero-terminate the buffer. */ - buffer[len] = '\0'; - - if (filesize) - *filesize = len; - return buffer; -} - -/* - * Receive a single file as a malloc'd buffer. - */ -char * -fetchFile(PGconn *conn, const char *filename, size_t *filesize) -{ - PGresult *res; - char *result; - const char *params[1]; - int len; - - params[0] = filename; - res = pgut_execute_extended(conn, "SELECT pg_catalog.pg_read_binary_file($1)", - 1, params, false, false); - - /* sanity check the result set */ - if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) - elog(ERROR, "unexpected result set while fetching remote file \"%s\"", - filename); - - /* Read result to local variables */ - len = PQgetlength(res, 0, 0); - result = pg_malloc(len + 1); - memcpy(result, PQgetvalue(res, 0, 0), len); - result[len] = '\0'; - - PQclear(res); - *filesize = len; - - return result; -} diff --git a/src/fu_util/CMakeLists.txt b/src/fu_util/CMakeLists.txt new file mode 100644 index 000000000..a5426df42 --- /dev/null +++ b/src/fu_util/CMakeLists.txt @@ -0,0 +1,66 @@ +cmake_minimum_required(VERSION 3.16) +project(fu_utils VERSION 0.1 LANGUAGES C) + +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_EXTENSIONS true) + +include(CheckCSourceCompiles) +include(CheckFunctionExists) + +add_library(fu_utils impl/ft_impl.c impl/fo_impl.c) + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) +target_link_libraries(fu_utils PRIVATE Threads::Threads) + +if(CMAKE_USE_PTHREADS_INIT) + target_compile_definitions(fu_utils PRIVATE USE_PTHREADS) +else() + message(FATAL_ERROR "Need pthread support to build") +endif() + +CHECK_FUNCTION_EXISTS(strerror_r HAVE_STRERROR_R) + +# Detect for installed beautiful https://github.com/ianlancetaylor/libbacktrace +include_directories(.) +if(NOT CMAKE_C_COMPILER MATCHES tcc) + find_library(LIBBACKTRACE backtrace) + if(LIBBACKTRACE) + set(CMAKE_REQUIRED_LIBRARIES backtrace) + target_link_libraries(fu_utils PRIVATE backtrace) + check_c_source_compiles(" + #include + int main(void) { + struct backtrace_state *st = backtrace_create_state(NULL, 0, NULL, NULL); + return 0; + } + " HAVE_LIBBACKTRACE) + if (HAVE_LIBBACKTRACE) + target_compile_definitions(fu_utils PRIVATE HAVE_LIBBACKTRACE) + endif() + endif() +endif() +check_include_file(execinfo.h HAVE_EXECINFO_H) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fexceptions") + +if(HAVE_EXECINFO_H) + target_compile_definitions(fu_utils PRIVATE HAVE_EXECINFO_H) +endif() +if(HAVE_STRERROR_R) + target_compile_definitions(fu_utils PRIVATE HAVE_STRERROR_R) +endif() + + +configure_file(fu_utils_cfg.h.in fu_utils_cfg.h) +target_include_directories(fu_utils INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}") +target_include_directories(fu_utils PRIVATE "${PROJECT_BINARY_DIR}") + +install(TARGETS fu_utils DESTINATION lib) +install(FILES fm_util.h ft_util.h fo_obj.h + ft_sort.inc.h ft_ss_examples.h ft_search.inc.h ft_array.inc.h + ft_ar_examples.h "${PROJECT_BINARY_DIR}/fu_utils_cfg.h" + DESTINATION include/fu_utils) +install(FILES impl/ft_impl.h impl/fo_impl.h DESTINATION include/fu_utils/impl) + +add_subdirectory(test) diff --git a/src/fu_util/LICENSE b/src/fu_util/LICENSE new file mode 100644 index 000000000..d59b83d19 --- /dev/null +++ b/src/fu_util/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2022-2022 Postgres Professional +Copyright (c) 2022-2022 Yura Sokolov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/fu_util/README.obj.ru.md b/src/fu_util/README.obj.ru.md new file mode 100644 index 000000000..78b8d142b --- /dev/null +++ b/src/fu_util/README.obj.ru.md @@ -0,0 +1,923 @@ +# Интерфейсно-Объектная библиотечка fobj - funny objects. + +Библиотечка призвана предоставить решение проблеме полиформизма: +- иметь множество реализаций одного поведения (методов), +- и прозрачное инициирование поведения не зависимо от реализации (вызов + метода). + +Реализует концепцию динамического связывания: +- объект аллоцируется со скрытым хедером, содержащим идентификатор класса +- во время вызова метода, реализация метода ищется в рантайме используя + идентификатор класса из хедера объекта и идентификатор метода. + +Плюс, библиотека предоставляет управление временем жизни объектов +посредством счётчика ссылок. + +### Пример + +Рассмотрим на примере создания объекта с состоянием в виде переменной +double и паре методов: "умножить и прибавить" и "прибавить и умножить". + +Без "магии" это могло выглядеть так: +```c + typedef struct dstate { double st; } dstate; + double mult_and_add(dstate *st, double mult, double add) { + st->st = st->st * mult + add; + return st->st; + } + + double add_and_mult(dstate *st, double add, double mult) { + st->st = (st->st + add) * mult; + return st->st; + } + + int main (void) { + dstate mystate = { 1 }; + printf("%f\n", mult_and_add(&mystate, 2, 1)); + printf("%f\n", add_and_mult(&mystate, 2, 1)); + printf("%f\n", mult_and_add(&mystate, 5, 8)); + printf("%f\n", add_and_mult(&mystate, 5, 8)); + } +``` + +## Метод + +Метод - главный персонаж библиотеки. Определяет поведение. + +Метод - "дженерик" функция с диспатчингом по первому аргументу. +Первый аргумент не типизирован. Им может быть "произвольный" объект. +Типы второго и следующего аргумента фиксированные. + +Аргуметны могут быть "обязательными" и "опциональными". У опционального +аргумента может быть значение по умолчанию. + +Диспатчинг происходит в рантайме. Если обнаружится, что на объекте +метод не определён, произойдёт FATAL ошибка с абортом. Если обязательный +аргумент не передан, тоже. + +Имена методов строго уникальны, потому именовать метод рекомендуется +с абревиатурой неймспейса. Также рекомендуется использовать +theCammelCase. + +Чтобы создать метод, нужно: + +- объявить имя, сигнатуру метода. + Обычно это делается в заголовочных файлах (.h) + Первый аргумент (объект) явно объявлять не нужно. +```c + #define mth__hotMultAndAdd double, (double, mult), (double, add) + #define mth__hotAddAndMult double, (double, add), (double, mult) + #define mth__hotGetState double +``` +- если есть опциональные аргументы, то добавить их объявление. + (если нет, то макрос объявлять не нужно) +```c + #define mth__hotMultAndAdd_optional() (mult, 1), (add, 0) +``` +- позвать генерирующий макрос (так же в .h файле, если это метод должен + быть виден из-вне) +```c + fobj_method(hotMultAndAdd); + fobj_method(hotAddAndMult); + fobj_method(hotGetState); +``` + + Макрос генерит следующие объявления, используемые пользователем: + +```c + // Функция вызова метода без "магии" именованных/опциональных + // параметров. + static inline double hotMultAndAdd(fobj_t obj, double mult, double add); + + // Интерфейс для одного метода + typedef union hotMultAndAdd_i { + fobj_t self; + uintptr_t has_hotMultAndAdd; + } hotMultAndAdd_i; + + // Биндинг интерфейса для объекта + static inline hotMultAndAdd_i bind_hotMultAndAdd(fobj_t obj); + + // Биндинг интерфейса + увеличение счётчика ссылок на объекте. + static inline hotMultAndAdd_i bindref_hotAddAndMult(fobj_t obj); + + // Проверка реализации интерфейса + static inline bool implements_hotAddAndMult(fobj_t obj, hotMultAndAdd_i *iface); +``` + + Последующие объявления пользователем непосредственно не используются. + Можете не читать. + +```c + // Хэндл метода + static inline fobj_method_handle_t hotMultAndAdd__mh(void); + + // Тип функции - реализации + typedef double (*hotMultAndAdd__impl)(fobj_t obj, double mult, double mult); + + // Связанная реализация + typedef struct hotMultAndAdd_cb { + fobj_t self; + hotMultAndAdd_impl impl; + } hotMultAndAdd_cb; + + // Получение связанной реализации + static inline hotMultAndAdd_cb + fobj_fetch_hotMultAndAdd(fobj_t obj, fobj_klass_handle_t parent); + + // Регистрация реализации + static inline void + fobj__register_hotMultAndAdd(fobj_klass_handle_t klass, hotMultAndAdd_impl impl); + + // Валидация существования метода для класса + // (В случае ошибки, будет FATAL) + static inline void + fobj__klass_validate_hotMultAndAdd(fobj_klass_handle_t klass); + + // Тип для именованных параметров + typedef struct hotMultAndAdd__params_t { + fobj__dumb_t _dumb_first_param; + + double mult; + fobj__dumb_t mult__given; + + double add; + fobj__dumb_t add__given; + } hotMultAndAdd__params_t; + + // Функция вызова связанной реализации с параметрами. + static inline double + fobj__invoke_hotMultAndAdd(hotMultAndAdd__cb cb, hotMultAndAdd__params params); +``` + +------ + +## Класс + +Класс определяет связывание методов с конкретными объектами. + +Объявляя класс, можно указать: +- список методов +- родителя (опционально) +- интерфейсы, которые нужно провалидировать (опционально) +- расширяемая ли аллокация (опционально) + +Чтобы создать класс, нужно: +- объявить тело класса +```c + typedef struct hotDState { + double st; + } hotDState; + + // И "наследника" + typedef struct hotCState { + hotDState p; // parent + double add_acc; // аккумулятор для слагаемых + } hotCState; +``` + +- обявить сигнатуру класса +```c + #define kls__hotDState mth(hotMultAndAdd, hotGetState), \ + iface(hotState) + #define kls__hotCState inherits(hotDState), \ + mth(hotMultAndAdd, hotAddAndMult), + iface(hotState) +``` + Примечания: + - мы не объявили hotAddAndMult на hotDState, но он удовлетворяет + hotState, т.к. hotAddAndMult не обязателен. + - мы не объявляли hotGetState на hotCState, но он удовлетворяет + hotState, т.к. этот метод он наследует от hotDState. + +- позвать генерирующие макросы +```c + fobj_klass(hotDState); + fobj_klass(hotCState); +``` + На самом деле, это всего лишь генерит методы хэндл-ов. +```c + extern fobj_khandle_method_t hotDState__kh(void); + extern fobj_khandle_method_t hotCState__kh(void); +``` + +- объявить реализации методов: +```c + static double + hotDState_hotMultAndAdd(VSelf, double mult, double add) { + Self(hotDState); + self->st = self->st * mult + add; + return self->st; + } + + static double + hotDState_hotGetState(VSelf) { + Self(hotDState); + return self->st; + } + + static double + hotCState_hotMultAndAdd(VSelf, double mult, double add) { + Self(hotCState); + // Вызов метода на родителе + $super(hotMultAndAdd, self, .mult = mult, .add = add); + self->add_acc += add; + return self->st; + } + + static double + hotCState_hotAddAndMult(VSelf, double add, double mult) { + Self(hotCState); + $(hotMultAndAdd, self, .add = add); + $(hotMultAndAdd, self, .mult = mult); + return self->st; + } +``` + +- После всех реализаций (или хотя бы их прототипов) в одном .c файле нужно создать реализацию хэндла класса: +```c + fobj_klass_handle(hotDState); + fobj_klass_handle(hotCState); +``` + +- Опционально, можно инициализировать класс. + Этот шаг не обязателен чаще всего, но может потребоваться, если вы собираетесь + заморозить рантайм (`fobj_freeze`), или захотите поискать класс по имени. +```c + void + libarary_initialization(void) { + /*...*/ + fobj_klass_init(hotDState); + fobj_klass_init(hotCState); + /*...*/ + } +``` + +### Деструктор + +Когда объект уничтожается, выполняется стандартный метод `fobjDispose` +(если определён на объекте). +```c + typedef struct myStore { + fobj_t field; + } myStore; + + #define kls__myKlass mth(fobjDispose) + + static void + myKlass_fobjDispose(VSelf) { + Self(myKlass); + $del(&self->field); + } +``` + +Методы `fobjDispose` вызываются для всех классов в цепочке наследования, +для которых они определены, по порядку *от потомков к первому родителю*. +Т.е. сперва вызвается `fobjDispose` вашего класса, потом его +непосредственного родителя, потом родителя родителя и т.д. + +Нет способа вернуть ошибку из `fobjDispose` (т.к. не куда). Он обязан +сам со всем разобраться. + +Иногда нужно "отключить объект" не дожидаясь, пока он сам уничтожится. +Т.е. хочется позвать `fobjDispose`. Но явный вызов `fobjDispose` запрещён. +Для этого нужно воспользоваться обёрткой `fobj_dispose`. + +Обёртка гарантирует, что `fobjDispose` будет позван только один раз. +Кроме того, она запоминает, что вызов `fobjDispose` завершился, и после +этого любой вызов метода (вызванный без хаков) на этом объекте будет +падать в FATAL. + +### Методы класса + +Долго думал, и решил, что нет проблем, решаемых методами класса, +и не решаемых другими методами. + +Методы класса играют обычно роли: +- неймспейса для статических функций. + - Но в С можно просто звать глобальные функции. +- синглтон объектов, связанных с множеством объектов. + - Если объекту очень нужен синглтон, можно объявить метод, + возвращающий такой синглтон. Но большинство объектов не требует + связанного с ним синглтона. +- фабрик для создания объектов + - Что не отличается от кейса со статическими функциями. + +В общем, пока не появится очевидная необходимость в методах класса, +делать их не буду. Ибо тогда потребуется создавать мета-классы и их +иерархию. Что будет серьёзным усложнением рантайма. + +### Конструктор + +И тут вероятно вы зададите мне вопрос: +- Эй, уважаемый, что за омлет? А где же яйца? + (с) Дискотека Авария. + +В смысле: до сих пор ни слова про конструкторы. + +И не будет. Применяется подход "Конструктор бедного человека": +- конструированием объекта должен заниматься не объект. +- объект конструирует либо глобальная функция, либо метод другого объекта. + - либо всё нужное для работы объекта готовится перед аллокацией и + передаётся непосредственно в значения полей в момен аллокации, + - либо у объекта есть некий метод `initializeThisFu__ingObject` + который зовётся после аллокации. + (название выдуманно, такого стандартного метода нет), + +Этот подход применён в Go, и в целом, его можно понять и принять. +Сделать семантику конструктора с корректной обработкой возможных ошибок, +вызова родительского конструктора и прочим не просто. И не сказать, +чтобы языки, имеющие конструкторы, справляются с ними без проблем. + +В библиотечке так же наложились проблемы: +- с сохранением где-нибудь имени метода(?) или функции(?) конструктора + и передачи ему параметров. +- перегрузки конструкторов в зависимости от параметров(?) +- необходимость уникальных имён методов для каждого набора параметров. +- необходимость куда-то возвращать ошибки? +- отсутствие методов класса. + +В общем, пораскинув мозгами, я решил, что простота Go рулит, и усложнять +это место не особо нужно. +Тем более, что зачастую объекты создаются в методах других объектах. + +----- + +## Объекты + +Объекты - это экземпляры/инстансы классов. + +### Aллокация. +```c + hotDState* dst; + hotCState* cst; + + // По умолчанию, аллоцируется, зачищенное нулями. + dst = $alloc(DState); + + // Но можно указать значения. + cst = $alloc(CState, .p.st = 100, .add_acc = 0.1); +``` + +На что нужно обратит внимание: +- если вы пользуетесь передачей начальных значений в `$alloc` +- и освобождаете что-то в `fobjDispose` +- то передавать в `$alloc` нужно то, что можно в `fobjDispose` освободить. + +Т.е. +- если вы передаёте объект, то его нужно `$ref(obj)` (см.ниже) +- если вы передаёте строку, то её нужно `ft_strdup(str)` +- и т.д. + +### Вызов метода + +В вызове метода можно указывать аргументы по порядку или используя +имена параметров. + +Опциональные параметры можно пропускать. После пропущенного опционального +параметра можно использовать только именованные параметры. + +```c + // "Классический" + printf("%f\n", $(hotMultAndAdd, dst, 2, 3)); + printf("%f\n", $(hotMultAndAdd, cst, 3, 4)); + printf("%f\n", $(hotGetState, dst)); + printf("%f\n", $(hotGetState, cst)); + printf("%f\n", $(hotAddAndMult, cst, 5, 6)); + + // С именованными параметрами + printf("%f\n", $(hotMultAndAdd, dst, .mult = 2, .add = 3)); + printf("%f\n", $(hotMultAndAdd, cst, .add = 3, .mult = 4)); + printf("%f\n", $(hotGetState, dst)); // нет параметров. + printf("%f\n", $(hotAddAndMult, cst, .add = 5, .mult = 6)); + printf("%f\n", $(hotAddAndMult, cst, .mult = 5, .add = 6)); + + // С дефолтными параметрами + printf("%f\n", $(hotMultAndAdd, dst, .mult = 2)); + printf("%f\n", $(hotMultAndAdd, cst, .add = 3)); + printf("%f\n", $(hotMultAndAdd, cst)); + // А вот это упадёт с FATAL, т.к. у hotAddAndMult не имеет + // опциональных аргументов + // printf("%f\n", $(hotAddAndMult, cst, .add = 5)); + // printf("%f\n", $(hotAddAndMult, cst, .mult = 5)); + // printf("%f\n", $(hotAddAndMult, cst)); +``` + +Можно использовать метод непосредственно как С функцию, но аргументы +придётся тогда указывать все и по порядку. Именнованные аргументы +при этом указать не получится, и пропустить опциональные - тоже. + +```c + printf("%f\n", hotMultAndAdd(dst, 2, 3)); + printf("%f\n", hotMultAndAdd(cst, 3, 4)); + printf("%f\n", hotGetState(dst)); + printf("%f\n", hotGetState(cst)); + printf("%f\n", hotAddAndMult(cst, 5, 6)); + // а вот это свалится с FATAL + // printf("%f\n", hotAddAndMult(dst, 6, 7)); +``` + +### Условные вызов метода. + +Доступна конструкция вызова метода только в случае, если он определён: + +```c + double v; + if ($ifdef(v =, hotMultAndAdd, dst, .mult = 1)) { + printf("dst responds to hotMultAndAdd: %f\n", v); + } + + if ($ifdef(, hotGetStatus, cst)) { + printf("cst responds to hotGetStatus.\n" + "Result assignment could be ommitted. " + "Although compiler could warn on this."); + } +``` + +### Проверка реализации метода + +Можно проверить, определён ли метод на объекте, с помощью макроса +`$implement(Method, obj)` + +```c + if ($implements(hotGetState, dst)) { + workWithObject(dst); + } + + hotGetState_i hgs; + if ($implements(hotGetState, dst, &hgs)) { + $i(hotGetState, hgs); + } +``` + +(На самом деле, используется механизм определения реализации интерфейса, +сгенерённого для метода). + +------- + +## Интерфейс + +Интерфейс - это формальный набор методов. + +Служит для цели проверки согласованности реализации объекта/класса, и +улучшает самодокументируемость сигнатур методов. + +Для каждого метода сразу создаётся интерфейс, содержащий один обязательный +метод. Потому создавать ещё раз интерфейс для одного метода не требуется. + +Чтобы создать интерфейс, нужно: + +- объявить интерфейс +```c + #define iface__hotState mth(hotMultAndAdd, hotGetState), \ + opt(hotAddAndMult) +``` + Здесь мы объявили два обязательных и один опциональный метод. + Количество секций mth и opt - произвольное. Количество методов в них - + тоже. + (Произвольное - в пределах разумного - ~ до 16) + +- позвать генерирующий макрос +```c + fobj_iface(hotState); +``` + + Макрос генерирует объявления: +```c + // Структура интерфейса с реализациями методов. + typedef union hotState_i { + fobj_t self; + uintptr_t has_hotMultAndAdd; + uintptr_t has_hotGetState; + uintptr_t has_hotAddAndMult; + } hotState_i; + + // Биндинг интерфейса для объекта + static inline hotState_i bind_hotState(fobj_t obj); + // Биндинг интерфейса + увеличение счётчика ссылок на объекте + static inline hotState_i bindref_hotState(fobj_t obj); + // Проверка реализации интерфейса + static inline bool implements_hotState(fobj_t obj, hotState_i *iface); +``` + И "скрытое объявление" +``` + // Проверка объявления интерфейса + static inline void + fobj__klass_validate_hotState(fobj__klass_handle_t klass); +``` + +### Биндинг метода/интерфейса + +По сути, это всегда биндинг интерфейса. Просто каждый метод определяет +интерфейс с одним этим методом. + +```c + hotMultAndAdd_i hmad = bind_hotMultAndAdd(dst); + hotMultAndAdd_i hmac = bind_hotMultAndAdd(cst); + hotState_i hstd = bind_hotState(dst); + hotState_i hstc = bind_hotState(cst); +``` + +### Вызов метода на интерфейсе + +Заметьте, тут интерфейс передаётся по значению, а не по поинтеру. +Сделал так после того, как один раз ошибся: вместо `$i()` написал `$()`, +и компилятор радостно скомпилировал, т.к. `$()` принимает `void*`. + +```c + printf("%f\n", $i(hotMultAndAdd, hmaa, .mult = 1)); + printf("%f\n", $i(hotMultAndAdd, hmac, .add = 2)); + + printf("%f\n", $i(hotMultAndAdd, hstd)); + printf("%f\n", $i(hotMultAndAdd, hstc)); + printf("%f\n", $i(hotGetState, hstd)); + printf("%f\n", $i(hotGetState, hstc)); + + printf("%f\n", $i(hotAddAndMult, hstc, .mult = 4, .add = 7)); + // Проверка на обязательность аргументов тут работает так же. + // Потому след.вызовы упадут с FATAL: + // $i(hotAddAndMult, hstd, .mult = 1); + // $i(hotAddAndMult, hstd, .add = 1); + // $i(hotAddAndMult, hstd); +``` + +A вот на `hstd` так просто `hotAddAndMult` позвать нельзя: +- `hotDState` этот метод не определял +- `hotAddAndMult` является опциональным методом интерфейса +- потому в `hstd` этот метод остался не заполненным. +Нужно проверять: + +```c + if ($ifilled(hotAddAndMult, hstd)) { + printf("This wont be printed: %f\n", + $i(hotAddAndMult, hstd, .mult=1, .add=2)); + } + if (fobj_iface_filled(hotAddAndMult, hstd)) { /*...*/ } +``` + +Или воспользоваться условным вызовом метода: +```c + if ($iifdef(v =, hotAddAndMult, hstd, .mult = 1, .add = 2)) { + printf("This wont be printed: %f\n", v); + } +``` + +### Проверка реализации интерфейса + +Вызов `bind_someInterface` упадёт с FATAL, если интерфейс не реализован. + +Проверить, реализован ли интерфейс, можно с помощью `$implements()`: + +```c + if ($implements(hotState, dst)) { + workWithObject(dst); + } + + if ($implements(hotState, dst, &hstd)) { + $i(hotGetState, hstd); + } +``` + +### Накладные расходы. + +Интерфейс служит только для типизации, и реализован в виде union размером в 1 +поинтер. В целом, накладные расходы не отличаются от поинтера на объект. + +-------- + +## Время жизни объекта. + +Время жизни объекта управляется методом подсчёта ссылок. + +Когда счётчик доходит до 0, объект уничтожается (и вызывается его fobjDispose, если определён). + +### Счётчик ссылок + +#### Инкремент счётчика ссылок + +Если объект `a` начинает ссылается на объект `b`, то счётчик ссылок `b` +должен быть увеличен на 1. Для этого служит `$ref(fobj)`/`fobj_ref(obj)`. + +```c + // Увеличить счётчик на объекте. + + store->field = $ref(obj); + + // Увеличить счётчик на объекте, возвращённом из метода. + + store->field = $ref($(createObject, fabric)); + +``` + +То же самое, когда вы аллоцируете объект и передаёте ему ссылку на другой +объект + +```c + store = $alloc(Store, .field = $ref(obj)); + + store = $alloc(Store, .field = $ref($(createObject, fabric))); +``` + +#### Декремент счётчика ссылок + +Если объект `a` перестаёт ссылаться на объект `b`, то нужно уменьшить +счётчик ссылок `b` на 1. + +```c + $del(&store->field); +``` + +То же нужно делать и в деструкторе (`fobjDispose`): + +```c + void + Store_fobjDispose(Vself) { + Self(Store); + $del(&Store->field); + } +``` + +#### Корректное перезапись ссылки + +Если требуется переписать ссылку объекта `a` с объекта `b1` на объект +`b2`, то нужно не забыть декремент счётчика на `b1`. Но это нужно сделать +после инкремента счётчика на `b2`, т.к. это может быть один и тот же +объект. + +Чтобы избежать ошибок, используйте: + +```c + $set(&store->field, obj); + + $set(&store->field, $(createObject, fabric)); +``` + +Заметьте: явно звать `$ref` или `$del` *не нужно*. + +### AutoRelease Pool (ARP) + +Для облегчения жизни программиста, используется концепция AutoRelase Pool: +- в начале крупных функций и в теле долгих циклов нужно объявить пул. + Также можно объявить пул для блока, если нужно ограничить время жизни + объектов. +```c + fobj_t + longRunningFunction() + { + FOBJ_FUNC_ARP(); + /*...*/ + for (i = 0; i < 1000000000; i++) { + FOBJ_LOOP_ARP(); + /*...*/ + } + /*...*/ + { + FOBJ_BLOCK_ARP(); + /*...*/ + } + } +``` +- AutoRelease Pool очищается при выходе из скоупа (функции, блока, одной + итерации цикла). Для этого используется расширение GCC (поддержанное + так же clang, tcc, lcc (Эльбрус), и возможно другими) + `__attribute__((cleanup(func))`. + При этом к каждому объекту, помещённым в ARP, применяется `$del` + столько раз, сколько раз объект помещён в пул. + +Все вновь созданные объекты имеют refcnt = 1 и уже помещены в "ближайший" +ARP. Если к новому объекту не применено `$ref`, то он будет уничтожен. + +#### Разлинковка объектов + +Если требуется вернуть объект `b`, с которым объект `a` теряет связь, +то нужно не делать декремент счётчика, а помещать объект в ARP. + +Для этого служит `$unref(b)`: + +```c + fobj_t b = store->field; + store->field = NULL; + return $unref(b); +``` + +Для перезаписи ссылки и возврата предыдущего значения служит `$swap`. +При этом возвращаемое значение помещается в ARP: + +```c + return $swap(&store->field, b2); +``` + +Этим же удобно пользоваться для однострочной разлинковки: + +```c + reutrn $swap(&store->field, NULL); +``` + +#### Спасение объекта + +Как уже говорилось, при выходе из скоупа, для которого объявлен +ARP, объект может быть уничтожен (т.к. счётчик ссылок у него станет +равным 0). + +Можно "спасать" объект, вручную увеличивая счётчик ссылок с помощью +`$ref(obj)`, и потом помещая в ARP пул другого скоупа с помощью +`$unref(obj)`. + +Но для удобства сделаны макросы `$save(obj)`, `$result(obj)` и +`$return(obj)`. + +`$save(obj)` сохраняет объект в случае выхода из блока/цикла. Он берёт +пул, объявленный с помощью `FOBJ_BLOCK_ARP()` или `FOBJ_LOOP_ARP()`, +узнаёт из него ссылку на предыдущий, и сохраняет объект в этом предыдущем +ARP пуле (предварительно сделав инкремент счётчика ссылок). + +`$result(obj)` делает то же самое, но с пулом, объявленным в скоупе +функции с помощью `FOBJ_FUNC_ARP()`. Таким образом, объект можно будет +передать в качестве результата, не опасаясь, что он будет тут же +уничтожен. + +`$return(obj)` разворачивается просто в `return $result(obj)`. + +#### Интерфейсы + +Такие же макросы есть для работы с интерфейсами: +- `$iref(iface)` +- `$iunref(iface)` +- `$iset(&iface_var, iface)` +- `$iswap(&iface_var, iface)` +- `$idel(&iface_var)` +- `$isave(iface)` +- `$iresult(iface)` +- `$ireturn(iface)` + +### Пример + +#### Cвязи между объектами + +```c + typedef struct myKlass2 { + fobj_t someObj; + char* someStr; + } myKlass2; + + #define mth__setSomeObj void, (fobj_t, so) + fobj_method(setSomeObj) + + #define mth__delSomeObj void + fobj_method(delSomeObj) + + #define mth__popSomeObj fobj_t + fobj_method(popSomeObj) + + #define mth__replaceSomeObj void, (fobj_t, so) + fobj_method(replaceSomeObj) + + #define mth__setSomeStr void, (char*, ss) + fobj_method(setSomeStr) + + #define kls__myKlass2 mth(fobjDispose), \ + mth(setSomeObj, delSomeObj, popSomeObj, replaceSomeObj), \ + mth(setSomeStr) + fobj_klass(MyKlass2) + + /* Корректно освобождаем ресурсы */ + myKlass2_fobjDispose(VSelf) { + Self(myKlass2); + $del(&self->someObj); + ft_free(self->someStr); + } + + void + myKlass2_setSomeObj(VSelf, fobj_t so) { + Self(myKlass2); + $set(&self->someObj, so); + } + + void + myKlass2_delSomeObj(VSelf) { + Self(myKlass2); + $del(&self->someObj); + } + + void + myKlass2_popSomeObj(VSelf) { + Self(myKlass2); + return $swap(&self->someObj, NULL); + // Or + // fobj_t old = self->someObj; + // self->someObj = NULL; + // return $unref(old); + } + + void + myKlass2_replaceSomeObj(VSelf, fobj_t so) { + Self(myKlass2); + return $swap(&self->someObj, so); + // Or + // fobj_t old = self->someObj; + // self->someObj = $ref(so); + // return $unref(old); + } + + myKlass2_resetSomeObj(VSelf, fobj_t so) { + Self(myKlass2); + const char *old = self->someStr; + $set(&self->someObj, so); + self->someStr = ft_strdup(ss); + ft_free(old); + } + + myKlass2* + make_MyKlass2(fobj_t so, char *ss) { + return $alloc(myKlass2, + .someObj = $ref(so), + .someStr = ft_strdup(ss)); + } + + myKlass2* + make_set_MyKlass2(fobj_t so, char *ss) { + MyKlass2* mk = $alloc(myKlass2); + mk->someObj = $ref(so); + mk->someStr = ft_strdup(ss); + return mk; + } +``` + +#### Работа с ARP пулом + +```c + // Нужно вернуть объект + fobj_t + doSomethingAndReturn(/*...*/, fobjErr **err) { + FOBJ_FUNC_ARP(); // AutoRelease Pool для функции + fobj_t result; + fobj_t loop_result = NULL; + // Проверим, что err != NULL, и присвоим *err = NULL + fobj_reset_err(err); + + for(/*...*/) { + FOBJ_LOOP_ARP(); // AutoRelease Pool для каждой итерации цикла + fobj_t some = findsomewhere(/*...*/); + + if (isGood(some)) { + // Если не сделать $save(some), то он (возможно) + // уничтожится при выходе из цикла. + loop_result = $save(some); + break; + } + if (tooBad(some)) { + // нужно "вернуть" err + *err = fobj_error("SHIT HAPPENED"); + // Без этого *err будет уничтожен при выходе из функции + $result(*err); + return NULL; + } + } + + result = createKlass(loop_result); + $return(result); + + // Если сделать просто `return result`, то объект уничтожится + // при выходе из функции. + } +``` + +Для быстрого выхода из вложенных ARP пулов можно вручную позвать +`$ref` + `$unref`: + +```c + fobj_t + doSomethingAndReturn(/*...*/) { + FOBJ_FUNC_ARP(); // AutoRelease Pool для функции + fobj_t result; + fobj_t loop_result = NULL; + + { + FOBJ_BLOCK_ARP(); + /*...*/ + for (/*...*/) { + FOBJ_LOOP_ARP(); + /*...*/ + if (/*...*/) { + loop_result = $ref(some); + goto quick_exit; + } + } + } + quick_exit: + // Не забыть поместить в ARP + $unref(loop_result) + /*...*/ + } +``` + +## Инициализация + +В главном исполняемом файле где-нибудь в начале функции `main` нужно позвать: +```c + fobj_init(); +``` +До этого момента создание новых классов (`fobj_klass_init`) будет падать с +FATAL ошибкой. + +Метод подготавливает рантайм и определяет некоторые базовые классы и методы. diff --git a/src/fu_util/fm_util.h b/src/fu_util/fm_util.h new file mode 100644 index 000000000..ccd19f1a9 --- /dev/null +++ b/src/fu_util/fm_util.h @@ -0,0 +1,254 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=0 */ +#ifndef FM_UTIL_H +#define FM_UTIL_H + +#define fm_cat(x, y) fm__cat(x, y) +#define fm__cat(x, y) x##y +#define fm_cat3(x, y, z) fm__cat3(x, y, z) +#define fm__cat3(x, y, z) x##y##z +#define fm_cat4(w, x, y, z) fm__cat4(w, x, y, z) +#define fm__cat4(w, x, y, z) w##x##y##z +#define fm_str(...) fm__str(__VA_ARGS__) +#define fm__str(...) #__VA_ARGS__ +#define fm_uniq(x) fm_cat(_##x##_, __COUNTER__) + +#define fm_expand(...) __VA_ARGS__ +#define fm_empty(...) + +#define fm_comma(...) , +#define fm__comma , + +#define fm_apply(macro, ...) \ + macro(__VA_ARGS__) + +/****************************************/ +// LOGIC + +#define fm_true 1 +#define fm_false 0 + +#define fm_compl(v) fm_cat(fm_compl_, v) +#define fm_compl_0 1 +#define fm_compl_1 0 +#define fm_and(x, y) fm_cat3(fm__and_, x, y) +#define fm__and_00 0 +#define fm__and_01 0 +#define fm__and_10 0 +#define fm__and_11 1 +#define fm_or(x, y) fm_cat3(fm__or_, x, y) +#define fm__or_00 0 +#define fm__or_01 1 +#define fm__or_10 1 +#define fm__or_11 1 +#define fm_nand(x, y) fm_cat3(fm__nand_, x, y) +#define fm__nand_00 1 +#define fm__nand_01 1 +#define fm__nand_10 1 +#define fm__nand_11 0 +#define fm_nor(x, y) fm_cat3(fm__nor_, x, y) +#define fm__nor_00 1 +#define fm__nor_01 0 +#define fm__nor_10 0 +#define fm__nor_11 0 +#define fm_xor(x, y) fm_cat3(fm__xor_, x, y) +#define fm__xor_00 0 +#define fm__xor_01 1 +#define fm__xor_10 1 +#define fm__xor_11 0 + +#define fm_if(x, y, ...) fm_cat(fm__if_, x)(y, __VA_ARGS__) +#define fm__if_1(y, ...) y +#define fm__if_0(y, ...) __VA_ARGS__ +#define fm_when(x) fm_cat(fm__when_, x) +#define fm__when_1(...) __VA_ARGS__ +#define fm__when_0(...) +#define fm_iif(x) fm_cat(fm__iif_, x) +#define fm__iif_1(...) __VA_ARGS__ fm_empty +#define fm__iif_0(...) fm_expand + +/****************************************/ +// COMPARISON + +#define fm_equal(x, y) \ + fm_compl(fm_not_equal(x, y)) +#define fm_not_equal(x, y) \ + fm_if(fm_and(fm__is_comparable(x),fm__is_comparable(y)), fm__primitive_compare, 1 fm_empty)(x, y) +#define fm__primitive_compare(x, y) fm_is_tuple(COMPARE_##x(COMPARE_##y)(())) +#define fm__is_comparable(x) fm_is_tuple(fm_cat(COMPARE_,x)(())) + +/****************************************/ +// __VA_ARGS__ + +#define fm_head(...) fm__head(__VA_ARGS__) +#define fm__head(x, ...) x +#define fm_tail(...) fm__tail(__VA_ARGS__) +#define fm__tail(x, ...) __VA_ARGS__ + +#define fm_va_single(...) fm__va_single(__VA_ARGS__, fm__comma) +#define fm_va_many(...) fm__va_many(__VA_ARGS__, fm__comma) +#define fm__va_single(x, y, ...) fm__va_result(y, 1, 0) +#define fm__va_many(x, y, ...) fm__va_result(y, 0, 1) +#define fm__va_result(...) fm__va_result_fin(__VA_ARGS__) +#define fm__va_result_fin(x, y, res, ...) res + +#define fm_no_va fm_is_empty +#define fm_va_01 fm_isnt_empty + +#ifndef FM_USE_STRICT + #if defined(__STRICT_ANSI__) || defined(_MSC_VER) /* well, clang-cl doesn't allow to distinguish std mode */ + #define FM_USE_STRICT + #endif +#endif + +#ifndef FM_USE_STRICT +#define fm_is_empty(...) fm__is_empty(__VA_ARGS__) +#define fm__is_empty(...) fm_va_single(~, ##__VA_ARGS__) +#define fm_isnt_empty(...) fm__isnt_empty(__VA_ARGS__) +#define fm__isnt_empty(...) fm_va_many(~, ##__VA_ARGS__) + +#define fm_va_01n(...) fm_cat3(fm__va_01n_, fm__isnt_empty(__VA_ARGS__), fm_va_many(__VA_ARGS__)) +#define fm__va_01n_00 0 +#define fm__va_01n_10 1 +#define fm__va_01n_11 n + +#define fm_when_isnt_empty(...) fm_cat(fm__when_, fm__isnt_empty(__VA_ARGS__)) +#else +#define fm_is_empty(...) fm_and(fm__is_emptyfirst(__VA_ARGS__), fm_va_single(__VA_ARGS__)) +#define fm_isnt_empty(...) fm_nand(fm__is_emptyfirst(__VA_ARGS__), fm_va_single(__VA_ARGS__)) + +#define fm__is_emptyfirst(x, ...) fm_iif(fm_is_tuple(x))(0)(fm__is_emptyfirst_impl(x)) +#define fm__is_emptyfirst_impl(x,...) fm__va_result(\ + fm__is_emptyfirst_do1 x (fm__is_emptyfirst_do2), 1, 0) +#define fm__is_emptyfirst_do1(F) F() +#define fm__is_emptyfirst_do2(...) , + +#define fm_when_isnt_empty(...) fm_cat(fm__when_, fm_isnt_empty(__VA_ARGS__)) + +#define fm_va_01n(...) fm_cat3(fm__va_01n_, fm__is_emptyfirst(__VA_ARGS__), fm_va_many(__VA_ARGS__)) +#define fm__va_01n_10 0 +#define fm__va_01n_00 1 +#define fm__va_01n_01 n +#define fm__va_01n_11 n +#endif + +#define fm_or_default(...) \ + fm_iif(fm_va_01(__VA_ARGS__))(__VA_ARGS__) + +#define fm_va_comma(...) \ + fm_when_isnt_empty(__VA_ARGS__)(fm__comma) +#define fm_va_comma_fun(...) \ + fm_if(fm_va_01(__VA_ARGS__), fm_comma, fm_empty) + + +/****************************************/ +// Tuples + +#define fm_is_tuple(x, ...) fm__is_tuple_(fm__is_tuple_help x, 1, 0) +#define fm__is_tuple_choose(a,b,x,...) x +#define fm__is_tuple_help(...) , +#define fm__is_tuple_(...) fm__is_tuple_choose(__VA_ARGS__) + +/****************************************/ +// Iteration + +/* recursion engine */ +#define fm_eval(...) fm__eval_0(__VA_ARGS__) +#ifdef FU_LONG_EVAL +#define fm__eval_0(...) fm__eval_1(fm__eval_1(fm__eval_1(fm__eval_1(__VA_ARGS__)))) +#else +#define fm__eval_0(...) fm__eval_1(fm__eval_1(__VA_ARGS__)) +#endif +#define fm__eval_1(...) fm__eval_2(fm__eval_2(__VA_ARGS__)) +#define fm__eval_2(...) fm__eval_3(fm__eval_3(__VA_ARGS__)) +#define fm__eval_3(...) __VA_ARGS__ + +// recursion handle : delay macro expansion to next recursion iteration +#define fm_recurs(id) id fm_empty fm_empty() () +#define fm_recurs2(a,b) fm_cat fm_empty() (a,b) +#define fm_defer(id) id fm_empty() + +#define fm_foreach_join(join, macro, ...) \ + fm_cat(fm_foreach_join_, fm_va_01n(__VA_ARGS__))(fm_empty, join, macro, __VA_ARGS__) +#define fm_foreach_join_0(join1, join2, macro, ...) +#define fm_foreach_join_1(join1, join2, macro, x) \ + join1() macro(x) +#define fm_foreach_join_n(join1, join2, macro, x, y, ...) \ + join1() macro(x) \ + join2() macro(y) \ + fm_recurs2(fm_foreach_join_, fm_va_01n(__VA_ARGS__))(join2, join2, macro, __VA_ARGS__) + +#define fm_foreach(macro, ...) \ + fm_foreach_join(fm_empty, macro, __VA_ARGS__) +#define fm_foreach_comma(macro, ...) \ + fm_foreach_join(fm_comma, macro, __VA_ARGS__) + +#define fm_foreach_arg_join(join, macro, arg, ...) \ + fm_cat(fm_foreach_arg_join_, fm_va_01n(__VA_ARGS__))(fm_empty, join, macro, arg, __VA_ARGS__) +#define fm_foreach_arg_join_0(join1, join2, macro, ...) +#define fm_foreach_arg_join_1(join1, join2, macro, arg, x) \ + join1() macro(arg, x) +#define fm_foreach_arg_join_n(join1, join2, macro, arg, x, y, ...) \ + join1() macro(arg, x) \ + join2() macro(arg, y) \ + fm_recurs2(fm_foreach_arg_join_, fm_va_01n(__VA_ARGS__))(join1, join2, macro, arg, __VA_ARGS__) + +#define fm_foreach_arg(macro, arg, ...) \ + fm_foreach_arg_join(fm_empty, macro, arg, __VA_ARGS__) +#define fm_foreach_arg_comma(macro, arg, ...) \ + fm_foreach_arg_join(fm_comma, macro, arg, __VA_ARGS__) + +#define fm_foreach_tuple_join(join, macro, ...) \ + fm_cat(fm_foreach_tuple_join_, fm_va_01n(__VA_ARGS__))(fm_empty, join, macro, __VA_ARGS__) +#define fm_foreach_tuple_join_0(join1, join2, macro, ...) +#define fm_foreach_tuple_join_1(join1, join2, macro, x) \ + join1() macro x +#define fm_foreach_tuple_join_n(join1, join2, macro, x, y, ...) \ + join1() macro x \ + join2() macro y \ + fm_recurs2(fm_foreach_tuple_join_, fm_va_01n(__VA_ARGS__))(join2, join2, macro, __VA_ARGS__) + +#define fm_foreach_tuple(macro, ...) \ + fm_foreach_tuple_join(fm_empty, macro, __VA_ARGS__) +#define fm_foreach_tuple_comma(macro, ...) \ + fm_foreach_tuple_join(fm_comma, macro, __VA_ARGS__) + +#define fm_foreach_tuple_arg_join(join, macro, arg, ...) \ + fm_cat(fm_foreach_tuple_arg_join_, fm_va_01n(__VA_ARGS__))(fm_empty, join, macro, arg, __VA_ARGS__) +#define fm_foreach_tuple_arg_join_0(join1, join2, macro, ...) +#define fm_foreach_tuple_arg_join_1(join1, join2, macro, arg, x) \ + join1() fm_apply(macro, arg, fm_expand x) +#define fm_foreach_tuple_arg_join_n(join1, join2, macro, arg, x, y, ...) \ + join1() fm_apply(macro, arg, fm_expand x) \ + join2() fm_apply(macro, arg, fm_expand y) \ + fm_recurs2(fm_foreach_tuple_arg_join_, fm_va_01n(__VA_ARGS__))(join2, join2, macro, arg, __VA_ARGS__) + +#define fm_foreach_tuple_arg(macro, arg, ...) \ + fm_foreach_tuple_arg_join(fm_empty, macro, arg, __VA_ARGS__) +#define fm_foreach_tuple_arg_comma(macro, arg, ...) \ + fm_foreach_tuple_arg_join(fm_comma, macro, arg, __VA_ARGS__) + +#define fm_eval_foreach(macro, ...) \ + fm_eval(fm_foreach(macro, __VA_ARGS__)) + +#define fm_eval_foreach_comma(macro, ...) \ + fm_eval(fm_foreach_comma(macro, __VA_ARGS__)) + +#define fm_eval_foreach_arg(macro, arg, ...) \ + fm_eval(fm_foreach_arg(macro, arg, __VA_ARGS__)) + +#define fm_eval_tuples(macro, ...) \ + fm_eval(fm_foreach_tuple(macro, __VA_ARGS__)) + +#define fm_eval_tuples_arg(macro, arg, ...) \ + fm_eval(fm_foreach_tuple_arg(macro, arg, __VA_ARGS__)) + +#define fm_eval_tuples_comma(macro, ...) \ + fm_eval(fm_foreach_tuple_comma(macro, __VA_ARGS__)) + +#define fm_eval_tuples_arg_comma(macro, arg, ...) \ + fm_eval(fm_foreach_tuple_arg_comma(macro, arg, __VA_ARGS__)) + +#define fm__dumb_require_semicolon \ + struct __dumb_struct_declaration_for_semicolon + +#endif diff --git a/src/fu_util/fo_obj.h b/src/fu_util/fo_obj.h new file mode 100644 index 000000000..49a6c6999 --- /dev/null +++ b/src/fu_util/fo_obj.h @@ -0,0 +1,620 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FOBJ_OBJ_H +#define FOBJ_OBJ_H + +#include + +typedef void* fobj_t; + +#include + +/* + * Pointer to "object*. + * In fact, it is just 'void *'. + */ +/* + * First argument, representing method receiver. + * Unfortunately, function couldn't have arbitrary typed receiver without issueing + * compiller warning. + * Use Self(Klass) to convert to concrete type. + */ +#define VSelf fobj_t Vself +/* + * Self(Klass) initiate "self" variable with casted pointer. + */ +#define Self(Klass) Self_impl(Klass) + +extern void fobj_init(void); +/* + * fobj_freeze forbids further modifications to runtime. + * It certainly should be called before additional threads are created. + */ +extern void fobj_freeze(void); + +#define fobj_self_klass 0 + +#include "./impl/fo_impl.h" + +/* Generate all method boilerplate. */ +#define fobj_method(method) fobj__define_method(method) + +/* Generates runtime privately called method boilerplate */ +#define fobj_special_method(meth) fobj__special_method(meth) + +/* + * Ensure method initialized. + * Calling fobj_method_init is not required, + * unless you want search method by string name or use `fobj_freeze` + */ +#define fobj_method_init(method) fobj__method_init(method) + +/* Declare klass handle */ +#define fobj_klass(klass) fobj__klass_declare(klass) +/* + * Implement klass handle. + * Here all the binding are done, therefore it should be called + * after method implementions or at least prototypes. + * Additional declarations could be passed here. + */ +#define fobj_klass_handle(klass, ...) fobj__klass_handle(klass, __VA_ARGS__) +/* + * Calling fobj_klass_init is not required, + * unless you want search klass by string name or use `fobj_freeze` + */ +#define fobj_klass_init(klass) fobj__klass_init(klass) +#define fobj_add_methods(klass, ...) fobj__add_methods(klass, __VA_ARGS__) + +#define fobj_iface(iface) fobj__iface_declare(iface) + +/* + * Allocate klass instance, and optionally copy fields. + * + * $alloc(klass) + * fobj_alloc(klass) + * allocates instance + * $alloc(klass, .field1 = val1, .field2 = val2) - + * fobj_alloc(klass, .field1 = val1, .field2 = val2) - + * allocates instance + * copies `(klass){.field1 = val1, .field2 = val2}` + */ +#define fobj_alloc(klass, ...) \ + fobj__alloc(klass, __VA_ARGS__) +#define $alloc(klass, ...) \ + fobj__alloc(klass, __VA_ARGS__) + +/* + * Allocate variable sized instance with additional size. + * Size should be set in bytes, not variable sized field elements count. + * Don't pass variable sized fields as arguments, they will not be copied. + * Fill variable sized fields later. + * + * fobj_alloc_sized(Klass, size) + * allocates instance with custom additional `size` + * returns obj + * fobj_alloc_sized(Klass, size, .field1 = val1, .field2 = val2) + * allocates instance with custom additional `size` + * copies `(klass){.field1 = val1, .field2 = val2}` + * returns obj + */ +#define fobj_alloc_sized(klass, size, ...) \ + fobj__alloc_sized(klass, size, __VA_ARGS__) + +/* + * Object lifetime. + * + * $ref(obj) + * Add reference to object. + * Manually increments reference count. It will prevent object's destruction. + * $unref(obj) + * Forget reference to object, but keep object alive (for some time). + * Will put object to AutoRelease Pool, so it will be destroyed later. + * $del(&var) + * Drop reference to object and (probably) destroy it. + * Manually decrement reference count and clear variable. + * It will destroy object, if its reference count become zero. + * $set(&var, obj) + * Replace value, pointed by first argument, with new value. + * New value will be passed to `$ref` and assigned to ptr. + * Then old value will be passed to `$del`, so it could be destroyed at this + * moment. + * $swap(&var, obj) + * Replace value, pointed by first argument, with new value, and return old + * value. + * New value will be passed to `$ref` and assigned to ptr. + * Then old value will be passed to `$unref`, so preserved in ARP. + * + * Same routines for interfaces: + * + * $iref(obj) + * $iunref(obj) + * $idel(&var) + * $iset(&var, iface) + * $iswap(&var, iface) + * + * AutoRelease Pool. + * + * AutoRelease Pool holds references to objects "about to be destroyed". + * + * AutoRelease Pool is drained on scope exit using GCC's __attribute__((cleanup)), + * and all objects stored in ARP are passed to $del. + * + * Newly created objects are always registered in current Autorelelease Pool, + * and if no $ref(obj), $set(var, obj) or $swap is called with them, they will be + * automatially destroyed on scope exit. + * + * As well, if you replace/delete object in some container and return old value, it + * should be put into AutoRelease Pool to be preserved for value acceptor. + * + * FOBJ_FUNC_ARP() + * Declare autorelease pool for function scope. + * FOBJ_LOOP_ARP() + * Declare autorelease pool for loop body. + * FOBJ_BLOCK_ARP() + * Declare autorelease pool for block body. + * + * $save(obj), $isave(iface) + * Increment reference and store object in parent autorelease pool. + * It is used to preserve object on loop or block exit with autorelease pool + * declared (`FOBJ_LOOP_ARP()` or `FOBJ_BLOCK_ARP()`). + * $result(obj), $iresult(iface) + * Increment reference and store object in autorelease pool parent to + * function's one. + * It is used to preserve object on exit from function with autorelease pool + * declared (`FOBJ_FUNC_ARP()`). + * $return(obj), $ireturn(obj) + * is just `return $result(obj)` + */ +#define $ref(obj) fobj_ref(obj) +#define $unref(obj) fobj_unref(obj) +#define $del(var) $set(var, NULL) +#define $set(var, obj) fobj__set_impl((var), (obj)) +#define $swap(var, obj) fobj__swap_impl((var), (obj)) + +extern fobj_t fobj_ref(fobj_t obj); +extern fobj_t fobj_unref(fobj_t obj); +#define fobj_del(var) fobj_set(var, NULL) +extern void fobj_set(fobj_t* var, fobj_t newval); +extern fobj_t fobj_swap(fobj_t* var, fobj_t newval); + +#define $iref(iface) fobj__iref(iface) +#define $iunref(iface) fobj__iunref(iface) +#define $idel(iface) fobj__idel(iface) +#define $iset(ptr, iface) fobj__iset((ptr), (iface)) +#define $iswap(ptr, iface) fobj__iswap((ptr), (iface)) + +#define FOBJ_FUNC_ARP() FOBJ_ARP_POOL(fobj__func_ar_pool) +#define FOBJ_LOOP_ARP() FOBJ_ARP_POOL(fobj__block_ar_pool) +#define FOBJ_BLOCK_ARP() FOBJ_ARP_POOL(fobj__block_ar_pool) + +#define $save(obj) fobj_store_to_parent_pool($ref(obj), &fobj__block_ar_pool) +#define $result(obj) fobj_store_to_parent_pool($ref(obj), &fobj__func_ar_pool) +#define $return(obj) return $result(obj) + +#define $isave(iface) fobj__isave(iface) +#define $iresult(iface) fobj__iresult(iface) +#define $ireturn(iface) fobj__ireturn(iface) + +/* + * fobjDispose should finish all object's activity and release resources. + * It is called automatically before destroying object. + */ +#define mth__fobjDispose void +fobj_special_method(fobjDispose); + +/* + * returns globally allocated klass name. + * DO NOT modify it. + */ +extern const char *fobj_klass_name(fobj_klass_handle_t klass); + +/* + * Return real klass of object. + * + * Note: `fobjKlass` is a method, so it could return something dirrefent from + * real klass. But if you have to cast pointer, you'd probably need real klass. + * + * But in other cases you'd better not abuse this function. + */ +extern fobj_klass_handle_t fobj_real_klass_of(fobj_t); + +/* + * Call method with named/optional args. + * + * $(someMethod, object) + * $(someMethod, object, v1, v2) + * $(someMethod, object, .arg1=v1, .arg2=v2) + * $(someMethod, object, .arg2=v2, .arg1=v1) + * // Skip optional .arg3 + * $(someMethod, object, v1, v2, .arg4=v4) + * $(someMethod, object, .arg1=v1, .arg2=v2, .arg4=v4) + * // Order isn't important with named args. + * $(someMethod, object, .arg4=v4, .arg1=v1, .arg2=v2) + * $(someMethod, object, .arg4=v4, .arg2=v2, .arg1=v1) + * + * fobj_call(someMethod, object) + * fobj_call(someMethod, object, v1, v2) + * fobj_call(someMethod, object, .arg1=v1, .arg2=v2) + * fobj_call(someMethod, object, v1, v2, .arg4=v4) + * fobj_call(someMethod, object, .arg1=v1, .arg2=v2, .arg4=v4) + */ +#define $(meth, self, ...) \ + fobj_call(meth, self, __VA_ARGS__) + +/* + * Call parent klass method implementation with named/optional args. + * + * $super(someMethod, object) + * $super(someMethod, object, v1, v2, .arg4=v4) + * $super(someMethod, object, .arg1=v1, .arg2=v2, .arg4=v4) + * fobj_call_super(someMethod, object) + * fobj_call_super(someMethod, object, v1, v2) + * fobj_call_super(someMethod, object, v1, v2, .arg4=v4) + * fobj_call_super(someMethod, object, .arg1=v1, .arg2=v2, .arg4=v4) + * + * It uses variable set inside of Self(klass) statement. + */ +#define $super(meth, self, ...) \ + fobj_call_super(meth, fobj__klassh, self, __VA_ARGS__) + +/* + * Call method stored in the interface struct. + * Interface is passed by value, not pointer. + * + * SomeIface_i someIface = bind_SomeIface(obj); + * $i(someMethod, someIface) + * $i(someMethod, someIface, v1, v2, .arg4=v4) + * $i(someMethod, someIface, .arg1=v1, .arg2=v2, .arg4=v4) + * fobj_iface_call(someMethod, someIface) + * fobj_iface_call(someMethod, someIface, v1, v2) + * fobj_iface_call(someMethod, someIface, v1, v2, .arg4=v4) + * fobj_iface_call(someMethod, someIface, .arg1=v1, .arg2=v2, .arg4=v4) + */ +#define $i(meth, iface, ...) \ + fobj_iface_call(meth, iface, __VA_ARGS__) + +/* + * Determine if object implements interface. + * + * if ($implements(someIface, object, &iface_var)) { + * $i(someMethod, iface_var); + * } + * + * if ($implements(someIface, object)) { + * workWith(object); + * } + * + * if (fobj_implements(iface, object, &iface_var)) { + * fobj_iface_call(someMethod, iface_var); + * } + * + * if (fobj_implements(iface, object)) { + * workWith(object); + * } + * + * And without macroses: + * + * if (implements_someIface(object, &iface_var)) { + * $i(someMethod, iface_var); + * } + * + * if (implements_someIface(object, NULL)) { + * workWith(object); + * } + */ +#define $implements(iface, obj, ...) \ + fobj__implements(iface, obj, __VA_ARGS__) +#define fobj_implements(iface, obj, ...) \ + fobj__implements(iface, obj, __VA_ARGS__) + +/* + * Determine if optional method is filled in interface. + * Note: required methods are certainly filled. + * + * if ($ifilled(someMethod, iface)) { + * $i(someMethod, iface); + * } + * + * if (fobj_iface_filled(someMethod, iface)) { + * fobj_iface_call(someMethod, iface); + * } + */ +#define $ifilled(meth, iface) \ + fobj_iface_filled(meth, iface) + +/* + * Call method if it is defined, and assign result. + * + * value_t val; + * if ($ifdef(val =, someMethod, self, v1, v2, .arg4=v4)) { + * ... + * } + * + * or doesn't assign result + * + * if ($ifdef(, someMethod, self, v1, v2, .arg4=v4)) { + * ... + * } + */ +#define $ifdef(assignment, meth, self, ...) \ + fobj_ifdef(assignment, meth, (self), __VA_ARGS__) + +#define $bind(iface_type, obj) fobj_bind(iface_type, (obj)) +#define $reduce(newiface, iface) fobj_reduce(newiface, (iface)) + +#define $isNULL(iface) ((iface).self == NULL) +#define $notNULL(iface) ((iface).self != NULL) +#define $setNULL(ifacep) ((ifacep)->self = NULL) +#define $null(iface_type) ((iface_type##_i){NULL}) + +/* + * Base type + */ +#define iface__fobj mth(fobjKlass, fobjRepr) +/* hardcoded instantiation because of fobj_iface always include iface__fobj */ +fobj__iface_declare_i(fobj, (mth, fobjKlass, fobjRepr)); + +#define mth__fobjRepr union fobjStr* +fobj__define_base_method(fobjRepr); +#define mth__fobjKlass fobj_klass_handle_t +fobj__define_base_method(fobjKlass); + +#define $repr(obj) fobj_getstr(fobjRepr(obj)).ptr +#define $irepr(iface) fobj_getstr(fobjRepr((iface).self)).ptr + +typedef struct fobjBase { + char fobj__base[0]; +} fobjBase; +#define kls__fobjBase mth(fobjKlass, fobjRepr) +fobj_klass(fobjBase); + +/* + * fobjFormat should be defined for pretty printing + */ +#define mth__fobjFormat void, (ft_strbuf_t*, out), (const char*, fmt, NULL) +fobj_method(fobjFormat); + +/********************************* + * String + */ + +typedef union fobjStr fobjStr; + +ft_inline fobjStr* fobj_str(const char* s); +ft_inline fobjStr* fobj_str_const(const char* s); +#define $S(s) (__builtin_constant_p(s) ? fobj_str_const(s) : fobj_str(s)) +enum FOBJ_STR_ALLOC { + FOBJ_STR_GIFTED, + FOBJ_STR_CONST, + FOBJ_STR_COPY, +}; +extern fobjStr* fobj_newstr(ft_str_t str, enum FOBJ_STR_ALLOC ownership); +ft_inline ft_str_t fobj_getstr(fobjStr *str); + +/* + * Steal if buffer is allocated, or copy otherwise. + * Buffer is zeroed and should be re-initialized. + */ +ft_inline fobjStr* fobj_strbuf_steal(ft_strbuf_t *buf); + +ft_gnu_printf(1, 2) +extern fobjStr* fobj_sprintf(const char* fmt, ...); +extern fobjStr* fobj_strcat(fobjStr *ostr, ft_str_t str); +extern fobjStr* fobj_strcat2(fobjStr *ostr, ft_str_t str1, ft_str_t str2); +ft_inline fobjStr* fobj_strcatc(fobjStr *ostr, const char *str); +ft_inline fobjStr* fobj_strcatc2(fobjStr *ostr, const char *str1, const char *str2); +ft_inline fobjStr* fobj_stradd(fobjStr *ostr, fobjStr *other); +ft_gnu_printf(2, 3) +extern fobjStr* fobj_strcatf(fobjStr *str, const char* fmt, ...); + +/* String comparison */ +ft_inline bool fobj_streq(fobjStr* self, fobjStr *oth); +ft_inline FT_CMP_RES fobj_strcmp(fobjStr* self, fobjStr *oth); +ft_inline bool fobj_streq_str(fobjStr* self, ft_str_t oth); +ft_inline FT_CMP_RES fobj_strcmp_str(fobjStr* self, ft_str_t oth); +ft_inline bool fobj_streq_c(fobjStr* self, const char *oth); +ft_inline FT_CMP_RES fobj_strcmp_c(fobjStr* self, const char *oth); + +/* turn object to string using fobjFormat */ +extern fobjStr* fobj_tostr(fobj_t obj, const char* fmt); +#define $tostr(obj, ...) fobj_getstr(fobj_tostr((obj), fm_or_default(__VA_ARGS__)(NULL))).ptr +#define $itostr(obj, ...) fobj_getstr(fobj_tostr((obj).self, fm_or_default(__VA_ARGS__)(NULL))).ptr + +#define kls__fobjStr mth(fobjRepr, fobjFormat) +fobj_klass(fobjStr); + +/********************************** + * Int + */ + +typedef struct fobjInt { + int64_t i; +} fobjInt; + +ft_inline fobjInt* fobj_int(int64_t i); +#define $I(i) fobj_int(i) + +#define kls__fobjInt mth(fobjRepr, fobjFormat) +fobj_klass(fobjInt); + +/********************************** + * UInt + */ + +typedef struct fobjUInt { + uint64_t u; +} fobjUInt; + +ft_inline fobjUInt* fobj_uint(uint64_t u); +#define $U(i) fobj_uint(i) + +#define kls__fobjUInt mth(fobjRepr, fobjFormat) +fobj_klass(fobjUInt); + +/********************************** + * Float + */ + +typedef struct fobjFloat { + double f; +} fobjFloat; + +ft_inline fobjFloat* fobj_float(double f); +#define $F(f) fobj_float(f) + +#define kls__fobjFloat mth(fobjRepr, fobjFormat) +fobj_klass(fobjFloat); + +/********************************** + * Bool + */ + +typedef struct fobjBool { + bool b; +} fobjBool; + +extern fobjBool* fobj_bool(bool f); +#define $B(f) fobj_bool(f) + +#define kls__fobjBool mth(fobjRepr, fobjFormat) +fobj_klass(fobjBool); + +/* + * Allocate temporary blob. + * It could be anything, and it will be automatically released. + */ +static inline void* fobj_alloc_temp(size_t buf_size); +/* get object pointer for temporary blob */ +static inline fobj_t fobj_temp2obj(void* temp); +#define fobj_temp_save(ptr) $save(fobj_temp2obj(ptr)) +#define fobj_temp_result(ptr) $result(fobj_temp2obj(ptr)) +#define fobj_temp_return(ptr) $return(fobj_temp2obj(ptr)) + +/********************************** + * kv + */ +typedef struct fobj_kv { + const char * key; + fobj_t value; +} fobj_kv; + +#define FT_SLICE fokv +#define FT_SLICE_TYPE fobj_kv +#include +#define FT_SEARCH fokv +#define FT_SEARCH_TYPE fobj_kv +#define FT_SEARCH_PATTERN const char* +#include +ft_inline FT_CMP_RES fobj_fokv_cmpc(fobj_kv kv, const char* nm) { + return strcmp(kv.key, nm); +} + +extern fobjStr* fobj_printkv(const char *fmt, ft_slc_fokv_t kv); +#define $fmt(fmt, ...) fobj__printkv(fmt, __VA_ARGS__) + + +/********************************** + * ERRORS + */ + +#define mth___fobjErr_marker_DONT_IMPLEMENT_ME void +fobj_special_method(_fobjErr_marker_DONT_IMPLEMENT_ME); + +#define iface__err mth(_fobjErr_marker_DONT_IMPLEMENT_ME) +fobj_iface(err); + +#define fobj_error_kind(err) fobj__error_kind(err) +#define fobj_error_flag_key(key) fobj__error_flag_key(key) +#define fobj_error_int_key(key) fobj__error_int_key(key) +#define fobj_error_uint_key(key) fobj__error_uint_key(key) +#define fobj_error_cstr_key(key) fobj__error_cstr_key(key) +#define fobj_error_float_key(key) fobj__error_float_key(key) +#define fobj_error_bool_key(key) fobj__error_bool_key(key) +#define fobj_error_object_key(key) fobj__error_object_key(key) + +extern ft_arg_t fobj_err_getkv(err_i err, const char *key, ft_arg_t dflt, bool *found); + +fobj_error_kind(RT); +fobj_error_kind(SysErr); + +fobj_error_object_key(cause); +fobj_error_cstr_key(causeStr); +fobj_error_int_key(errNo); +fobj_error_int_key(intCode); +fobj_error_cstr_key(errNoStr); +#define fobj_errno_keys(errno) (errNo, errno), (errNoStr, ft_strerror(errno)) +fobj_error_cstr_key(path); +fobj_error_cstr_key(old_path); +fobj_error_cstr_key(new_path); + +/* special key for raw appending to error message */ +fobj_error_cstr_key(__msgSuffix); + +/* + * $err(Type) + * $err(Type, "some error") + * $err(Type, "Some bad thing happens at {path}", (path, filename)) + */ +#define $err(type, ...) fobj_make_err(type, __VA_ARGS__) +/* + * $noerr() - empty error + * $noerr(err) - true, if $isNULL(err) + */ +#define $noerr(...) fm_if(fm_va_01(__VA_ARGS__), $isNULL(__VA_ARGS__), $null(err)) +/* + * $haserr(err) - true if $notNULL(err) + */ +#define $haserr(err) $notNULL(err) +/* + * $err_has_kind(kind, err) + */ +#define $err_has_kind(kind, err) fobj_err_has_kind(kind, err) + +/* + * $syserr(errno) + * $syserr(errno, "allocation error") + * $syserr(errno, "Could not open file {path}", (path, filename)) + */ +#define $syserr(erno, ...) fobj_make_syserr((erno), __VA_ARGS__) + +/* fetch key back */ +#define $errkey(key, err, ...) fobj__err_getkey(key, err, __VA_ARGS__) +/* + * Get errno stored in `errNo` error key + */ +ft_inline int getErrno(err_i err); +/* + * Get errno string stored in `errNoStr` error key + */ +ft_inline const char* getErrnoStr(err_i err); + +/* + * Get error type + */ +#define $errkind(err) fobj_errkind(err) + +/* + * Get error message + */ +#define $errmsg(err) fobj_errmsg(err) +/* + * Get error location + */ +#define $errsrc(err) fobj_errsrc(err) + +#define kls__fobjErr mth(fobjDispose, fobjRepr, fobjFormat) +fobj_klass(fobjErr); + +/* + * Combines two error by placing second into single linked list of siblings. + * If either of error is NULL, other error is returned. + * If both errors are NULL, then NULL is returned. + * If second already has siblings, first's list of siblings is appended to + * second's list, then second becames first sibling of first. + */ +extern err_i fobj_err_combine(err_i first, err_i second); + +#define fobj_reset_err(err) do { ft_dbg_assert(err != NULL); *err = (err_i){NULL}; } while(0) + +#include "./impl/fo_impl2.h" + +#endif diff --git a/src/fu_util/ft_ar_examples.h b/src/fu_util/ft_ar_examples.h new file mode 100644 index 000000000..12cbd3b8e --- /dev/null +++ b/src/fu_util/ft_ar_examples.h @@ -0,0 +1,71 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FT_AR_EXAMPLES_H +#define FT_AR_EXAMPLES_H + +/* + * Slices and arrays for int. + * Defines slice: + * + * typedef struct { int *ptr; size_t len; } ft_slc_int_t; + * + * ft_slc_int_t ft_slc_int_make(int *ptr, size_t len); + * ft_slc_int_t ft_slc_int_alloc(int *ptr, size_t len); + * + * int ft_slc_int_at(ft_slc_int_t *ptr, ssize_t at); + * int ft_slc_int_set(ft_slc_int_t *ptr, ssize_t at, int v); + * + * ft_slc_int_t ft_slc_int_slice(ft_slc_int_t *ptr, ssize_t start, ssize_t end) + * + * void ft_slc_int_each(ft_slc_int_t *, void (*each)(int)); + * void ft_slc_int_each_r(ft_slc_int_t *, void (*each)(int, ft_arg_t), ft_arg_t); + * + * Defines array: + * + * typedef struct { int *ptr; size_t len; size_t cap; } ft_arr_int_t; + * ft_arr_int_t ft_arr_int_alloc(int *ptr, size_t len); + * + * int ft_arr_int_at (ft_arr_int_t *ptr, ssize_t at); + * int ft_arr_int_set(ft_arr_int_t *ptr, ssize_t at, int v); + * + * ft_slc_int_t ft_arr_int_slice(ft_arr_int_t *ptr, ssize_t start, ssize_t end) + * + * void ft_arr_int_each (ft_arr_int_t *, void (*each)(int)); + * void ft_arr_int_each_r(ft_arr_int_t *, void (*each)(int, ft_arg_t), ft_arg_t); + * + * void ft_arr_int_ensure(ft_arr_int_t *, size_t addcapa); + * void ft_arr_int_recapa(ft_arr_int_t *, size_t newcapa); + * void ft_arr_int_resize(ft_arr_int_t *, size_t newsize); + * + * void ft_arr_int_insert_at(ft_arr_int_t *, ssize_t at, int el); + * void ft_arr_int_insert_n (ft_arr_int_t *, ssize_t at, int *el, size_t n); + * void ft_arr_int_push (ft_arr_int_t *, int el); + * void ft_arr_int_append (ft_arr_int_t *, int *el, size_t n); + * + * int ft_arr_int_del_at (ft_arr_int_t *, ssize_t at); + * int ft_arr_int_pop (ft_arr_int_t *); + * void ft_arr_int_del_slice(ft_arr_int_t *, ssize_t start, ssize_t end); + * + * void ft_array_walk (ft_arr_int_t *, + * FT_WALK_ACT (*walk)(intl)) + * void ft_array_walk_r(ft_arr_int_t *, + * FT_WALK_ACT (*walk)(int, ft_arg_t), ft_arg_t) + */ +#define FT_SLICE int +#define FT_SLICE_TYPE int +#include + +/* + * Slices and arrays for C strings + */ +#define FT_SLICE cstr +#define FT_SLICE_TYPE const char* +#include + +/* + * Slices and arrays for C strings + */ +#define FT_SLICE void +#define FT_SLICE_TYPE void* +#include + +#endif diff --git a/src/fu_util/ft_array.inc.h b/src/fu_util/ft_array.inc.h new file mode 100644 index 000000000..aaeeb2471 --- /dev/null +++ b/src/fu_util/ft_array.inc.h @@ -0,0 +1,604 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FU_UTIL_H +#error "ft_util.h should be included" +#endif + +/* + * Accepts 2 macroses: + * - FT_SLICE - name suffix + * - FT_SLICE_TYPE - element type. + * + * Produces: + * typedef struct { + * FT_SLICE_TYPE* ptr; + * size_t len; + * } ft_slc_FT_SLICE_t; + * + * typedef struct { + * FT_SLICE_TYPE* ptr; + * size_t len; + * size_t cap; + * } ft_arr_FT_SLICE_t; + * + * - create slice struct + * + * ft_slc_FT_SLICE_t + * ft_slc_FT_SLICE_make(FT_SLICE_TYPE_t *ptr, size_t len); + * + * - allocate memory and copy data + * + * ft_slc_FT_SLICE_t + * ft_slc_FT_SLICE_alloc(FT_SLICE_TYPE_t *ptr, size_t len); + * + * ft_arr_FT_SLICE_t + * ft_arr_FT_SLICE_alloc(FT_SLICE_TYPE_t *ptr, size_t len); + * + * - take an element. + * if `at < 0`, then takes at `len - at` + * + * FT_SLICE_TYPE + * ft_slc_FT_SLICE_at(ft_slc_FT_SLICE_t *sl, ssize_t at); + * + * FT_SLICE_TYPE + * ft_arr_FT_SLICE_at(ft_arr_FT_SLICE_t *sl, ssize_t at); + * + * - set an element. + * if `at < 0`, then sets at `len - at` + * + * FT_SLICE_TYPE + * ft_slc_FT_SLICE_set(ft_slc_FT_SLICE_t *sl, ssize_t at, FT_SLICE_TYPE el); + * + * FT_SLICE_TYPE + * ft_arr_FT_SLICE_set(ft_arr_FT_SLICE_t *sl, ssize_t at, FT_SLICE_TYPE el); + * + * - take subslice + * `start` and `end` are normalized as index in `*_at`, `*_set` functions. + * Additionally, FT_SLICE_END could be used as a slice end. + * + * ft_slc_FT_SLICE_t + * ft_slc_FT_SLICE_slice(ft_slc_FT_SLICE_t *sl, ssize_t start, ssize end); + * + * ft_slc_FT_SLICE_t + * ft_arr_FT_SLICE_slice(ft_arr_FT_SLICE_t *sl, ssize_t start, ssize end); + * + * - call function for each element by value + * + * void + * ft_slc_FT_SLICE_each(ft_slc_FT_SLICE_t *, void (*)(FT_SLICE_TYPE)); + * + * void + * ft_slc_FT_SLICE_each_r(ft_slc_FT_SLICE_t *, + * void (*)(FT_SLICE_TYPE, ft_arg_t), ft_arg_t); + * + * void + * ft_arr_FT_SLICE_each(ft_arr_FT_SLICE_t *, void (*)(FT_SLICE_TYPE)); + * + * void + * ft_arr_FT_SLICE_each_r(ft_arr_FT_SLICE_t *, + * void (*)(FT_SLICE_TYPE, ft_arg_t), ft_arg_t); + * + * Following are only for ARRAY: + * + * - ensure array's capacity for additional elements + * + * void + * ft_arr_FT_SLICE_ensure(ft_arr_FT_SLICE_t *arr, size_t addelems); + * + * - ensure array's total capacity (or decrease it) + * It rounds up capacity to power of 2. + * It will panic if new capacity is less than lenght. + * + * void + * ft_arr_FT_SLICE_recapa(ft_arr_FT_SLICE_t *arr, size_t newcapa); + * + * - truncate or zero extend array. + * + * void + * ft_arr_FT_SLICE_resize(ft_arr_FT_SLICE_t *arr, size_t newsize); + * + * - push one or many elements to end of array + * + * void + * ft_arr_FT_SLICE_push(ft_arr_FT_SLICE_t *arr, FT_SLICE_TYPE el); + * + * void + * ft_arr_FT_SLICE_append(ft_arr_FT_SLICE_t *arr, FT_SLICE_TYPE *el, size_t n); + * + * - insert one or many elements into middle of array + * + * void + * ft_arr_FT_SLICE_insert_at(ft_arr_FT_SLICE_t *arr, size_t at, FT_SLICE_TYPE el); + * + * void + * ft_arr_FT_SLICE_insert_n(ft_arr_FT_SLICE_t *arr, size_t at, FT_SLICE_TYPE *el, size_t n); + * + * - delete one or many elements + * + * FT_SLICE_TYPE + * ft_arr_FT_SLICE_pop(ft_arr_FT_SLICE_t *arr); + * + * FT_SLICE_TYPE + * ft_arr_FT_SLICE_del_at(ft_arr_FT_SLICE_t *arr, size_t at); + * + * void + * ft_arr_FT_SLICE_del_slice(ft_arr_FT_SLICE_T *arr, ssize_t start, ssize_t end); + * + * - controllable array iteration. + * Callback may tell what to do: + * FT_WALK_CONT - continue + * FT_WALK_BREAK - break + * FT_WALK_DEL - delete current element and continue + * FT_WALK_DEL_BREAK - delete current element and break + * + * void + * ft_arr_FT_SLICE_walk(ft_arr_FT_SLICE_T *arr, + * FT_WALK_ACT (*walk)(FT_SLICE_TYPE *el)) + * + * void + * ft_arr_FT_SLICE_walk_r(ft_arr_FT_SLICE_T *arr, + * FT_WALK_ACT (*walk)(FT_SLICE_TYPE *el, ft_arg_t arg), + * ft_arg_t arg) + */ + +#define ft_slice_pref fm_cat(ft_slc_, FT_SLICE) +#define ft_array_pref fm_cat(ft_arr_, FT_SLICE) +#define ft_slice_type fm_cat(ft_slice_pref,_t) +#define ft_array_type fm_cat(ft_array_pref,_t) + +#define ft_slice_make fm_cat(ft_slice_pref, _make) +#define ft_slice_alloc fm_cat(ft_slice_pref, _alloc) +#define ft_slice_at fm_cat(ft_slice_pref, _at) +#define ft_slice_set fm_cat(ft_slice_pref, _set) +#define ft_slice_slice fm_cat(ft_slice_pref, _slice) +#define ft_slice_each fm_cat(ft_slice_pref, _each) +#define ft_slice_each_r fm_cat(ft_slice_pref, _each_r) + +#define ft_array_alloc fm_cat(ft_array_pref, _alloc) +#define ft_array_at fm_cat(ft_array_pref, _at) +#define ft_array_set fm_cat(ft_array_pref, _set) +#define ft_array_slice fm_cat(ft_array_pref, _slice) +#define ft_array_each fm_cat(ft_array_pref, _each) +#define ft_array_each_r fm_cat(ft_array_pref, _each_r) + +#define ft_array_ensure fm_cat(ft_array_pref, _ensure) +#define ft_array_recapa fm_cat(ft_array_pref, _recapa) +#define ft_array_resize fm_cat(ft_array_pref, _resize) +#define ft_array_reset_for_reuse fm_cat(ft_array_pref, _reset_for_reuse) +#define ft_array_free fm_cat(ft_array_pref, _free) + +#define ft_array_insert_at fm_cat(ft_array_pref, _insert_at) +#define ft_array_insert_n fm_cat(ft_array_pref, _insert_n) +#define ft_array_push fm_cat(ft_array_pref, _push) +#define ft_array_push2 fm_cat(ft_array_pref, _push2) +#define ft_array_append fm_cat(ft_array_pref, _append) + +#define ft_array_del_at fm_cat(ft_array_pref, _del_at) +#define ft_array_del_slice fm_cat(ft_array_pref, _del_slice) +#define ft_array_pop fm_cat(ft_array_pref, _pop) + +#define ft_array_walk fm_cat(ft_array_pref, _walk) +#define ft_array_walk_r fm_cat(ft_array_pref, _walk_r) + +#if __SIZEOF_SIZE_T__ < 8 +#define HUGE_SIZE ((size_t)UINT_MAX >> 2) +#else +#define HUGE_SIZE ((size_t)UINT_MAX << 16) +#endif + +#ifndef NDEBUG +/* try to catch uninitialized vars */ +#define ft_slice_invariants(slc) \ + ft_dbg_assert(ft_mul_size(sizeof(FT_SLICE_TYPE), slc->len) < HUGE_SIZE); \ + ft_dbg_assert((slc->len == 0) || (slc->ptr != NULL)) +#define ft_array_invariants(arr) \ + ft_dbg_assert(ft_mul_size(sizeof(FT_SLICE_TYPE), arr->len) < HUGE_SIZE); \ + ft_dbg_assert(ft_mul_size(sizeof(FT_SLICE_TYPE), arr->len) < HUGE_SIZE); \ + ft_dbg_assert(arr->cap >= arr->len); \ + ft_dbg_assert((arr->cap == 0) || (arr->ptr != NULL)) +#else +#define ft_slice_invariants(slc) \ + ft_dbg_assert((slc->len == 0) || (slc->ptr != NULL)) +#define ft_array_invariants(arr) \ + ft_dbg_assert(arr->cap >= arr->len); \ + ft_dbg_assert((arr->cap == 0) || (arr->ptr != NULL)) +#endif + +typedef struct ft_slice_type { + FT_SLICE_TYPE *ptr; + size_t len; +} ft_slice_type; + +typedef struct ft_array_type { + FT_SLICE_TYPE *ptr; + size_t len; + size_t cap; +} ft_array_type; + +ft_inline ft_slice_type +ft_slice_make(FT_SLICE_TYPE *ptr, size_t len) { + return (ft_slice_type){.ptr = ptr, .len = len}; +} + +ft_inline ft_slice_type +ft_slice_alloc(FT_SLICE_TYPE *ptr, size_t len) { + FT_SLICE_TYPE *newptr = ft_malloc_arr(sizeof(FT_SLICE_TYPE), len); + memcpy(newptr, ptr, sizeof(FT_SLICE_TYPE) * len); + return (ft_slice_type){.ptr = newptr, .len = len}; +} + +ft_inline FT_SLICE_TYPE +ft_slice_at(const ft_slice_type *sl, ssize_t at) { + ft_slice_invariants(sl); + at = ft__index_unify(at, sl->len); + return sl->ptr[at]; +} + +ft_inline FT_SLICE_TYPE +ft_slice_set(const ft_slice_type *sl, ssize_t at, FT_SLICE_TYPE val) { + ft_slice_invariants(sl); + at = ft__index_unify(at, sl->len); + sl->ptr[at] = val; + return val; +} + +ft_inline ft_slice_type +ft_slice_slice(const ft_slice_type *sl, ssize_t start, ssize_t end) { + ft_slice_invariants(sl); + start = ft__slcindex_unify(start, sl->len); + end = ft__slcindex_unify(end, sl->len); + ft_assert(start <= end); + return (ft_slice_type){.ptr = sl->ptr + start, .len = end - start}; +} + +ft_inline void +ft_slice_each_r(const ft_slice_type *sl, + void (*each)(FT_SLICE_TYPE el, ft_arg_t arg), + ft_arg_t arg) { + size_t i; + ft_slice_invariants(sl); + for (i = 0; i < sl->len; i++) { + each(sl->ptr[i], arg); + } +} + +ft_inline void +ft_slice_each(const ft_slice_type *sl, void (*each)(FT_SLICE_TYPE el)) { + size_t i; + ft_slice_invariants(sl); + for (i = 0; i < sl->len; i++) { + each(sl->ptr[i]); + } +} + +/* ARRAY */ + +ft_inline FT_SLICE_TYPE +ft_array_at(const ft_array_type *arr, ssize_t at) { + ft_array_invariants(arr); + at = ft__index_unify(at, arr->len); + return arr->ptr[at]; +} + +ft_inline FT_SLICE_TYPE +ft_array_set(const ft_array_type *arr, ssize_t at, FT_SLICE_TYPE val) { + ft_array_invariants(arr); + at = ft__index_unify(at, arr->len); + arr->ptr[at] = val; + return val; +} + +ft_inline ft_slice_type +ft_array_slice(const ft_array_type *arr, ssize_t start, ssize_t end) { + ft_array_invariants(arr); + start = ft__slcindex_unify(start, arr->len); + end = ft__slcindex_unify(end, arr->len); + ft_assert(start <= end); + return (ft_slice_type){.ptr = arr->ptr + start, .len = end - start}; +} + +ft_inline void +ft_array_each_r(const ft_array_type *arr, + void (*each)(FT_SLICE_TYPE el, ft_arg_t arg), + ft_arg_t arg) { + size_t i; + ft_array_invariants(arr); + for (i = 0; i < arr->len; i++) { + each(arr->ptr[i], arg); + } +} + +ft_inline void +ft_array_each(const ft_array_type *arr, void (*each)(FT_SLICE_TYPE el)) { + size_t i; + ft_array_invariants(arr); + for (i = 0; i < arr->len; i++) { + each(arr->ptr[i]); + } +} + +ft_inline void +ft_array_ensure(ft_array_type *arr, size_t sz) { + size_t newcap; + size_t newlen; + + ft_array_invariants(arr); + ft_assert(SIZE_MAX/2 - 1 - arr->len >= sz); + + newlen = arr->len + sz; + if (arr->cap >= newlen) + return; + + newcap = arr->cap ? arr->cap : 4; + while (newcap < newlen) + newcap *= 2; + + arr->ptr = ft_realloc_arr(arr->ptr, sizeof(FT_SLICE_TYPE), arr->cap, newcap); + arr->cap = newcap; +} + +ft_inline void +ft_array_recapa(ft_array_type *arr, size_t cap) { + size_t newcap; + + ft_array_invariants(arr); + ft_assert(cap >= arr->len); + ft_dbg_assert(SIZE_MAX/2 - 1 >= cap); + + newcap = (arr->cap && arr->cap <= cap) ? arr->cap : 4; + while (newcap < cap) + newcap *= 2; + + if (newcap == cap) + return; + + arr->ptr = ft_realloc_arr(arr->ptr, sizeof(FT_SLICE_TYPE), arr->cap, newcap); + arr->cap = newcap; +} + +ft_inline void +ft_array_resize(ft_array_type *arr, size_t len) { + ft_array_invariants(arr); + + if (len > arr->cap) + ft_array_recapa(arr, len); + + if (len < arr->len) { + memset(&arr->ptr[len], 0, sizeof(FT_SLICE_TYPE) * (arr->len - len)); + } else if (len > arr->len) { + memset(&arr->ptr[arr->len], 0, sizeof(FT_SLICE_TYPE) * (len - arr->len)); + } + + arr->len = len; + + if (arr->len < arr->cap / 4) + ft_array_recapa(arr, arr->len); +} + +ft_inline void +ft_array_reset_for_reuse(ft_array_type *arr) { + arr->len = 0; +} + +ft_inline ft_array_type +ft_array_alloc(FT_SLICE_TYPE *ptr, size_t len) { + ft_array_type arr = {NULL, 0, 0}; + + if (len > 0) { + ft_array_ensure(&arr, len); + memcpy(arr.ptr, ptr, sizeof(FT_SLICE_TYPE) * len); + arr.len = len; + } + return arr; +} + +ft_inline void +ft_array_free(ft_array_type *arr) { + ft_array_invariants(arr); + ft_free(arr->ptr); + arr->ptr = 0; + arr->len = 0; + arr->cap = 0; +} + +ft_inline FT_SLICE_TYPE +ft_array_del_at(ft_array_type *arr, ssize_t at) { + FT_SLICE_TYPE el; + ft_array_invariants(arr); + + at = ft__index_unify(at, arr->len); + el = arr->ptr[at]; + if (at+1 < arr->len) { + memmove(&arr->ptr[at], &arr->ptr[at+1], sizeof(FT_SLICE_TYPE)*(arr->len-at-1)); + } + memset(&arr->ptr[arr->len-1], 0, sizeof(FT_SLICE_TYPE)); + arr->len--; + + if (arr->len < arr->cap / 4) + ft_array_recapa(arr, arr->len); + + return el; +} + +ft_inline void +ft_array_del_slice(ft_array_type *arr, ssize_t start, ssize_t end) { + ft_array_invariants(arr); + + start = ft__slcindex_unify(start, arr->len); + end = ft__slcindex_unify(end, arr->len); + ft_assert(end >= start); + if (end == start) + return; + + if (end < arr->len) { + memmove(&arr->ptr[start], &arr->ptr[end], sizeof(FT_SLICE_TYPE)*(arr->len-end)); + } + + memset(&arr->ptr[arr->len-(end-start)], 0, sizeof(FT_SLICE_TYPE)*(end-start)); + arr->len -= end-start; + + if (arr->len < arr->cap / 4) + ft_array_recapa(arr, arr->len); +} + +ft_inline FT_SLICE_TYPE +ft_array_pop(ft_array_type *arr) { + FT_SLICE_TYPE el; + ft_array_invariants(arr); + + el = arr->ptr[arr->len-1]; + memset(&arr->ptr[arr->len-1], 0, sizeof(FT_SLICE_TYPE)); + arr->len--; + + if (arr->len < arr->cap / 4) + ft_array_recapa(arr, arr->len); + + return el; +} + +ft_inline void +ft_array_insert_at(ft_array_type *arr, ssize_t at, FT_SLICE_TYPE el) { + ft_array_invariants(arr); + at = ft__slcindex_unify(at, arr->len); + ft_array_ensure(arr, 1); + if (at != arr->len) + memmove(&arr->ptr[at+1], &arr->ptr[at], sizeof(FT_SLICE_TYPE) * (arr->len - at)); + arr->ptr[at] = el; + arr->len++; +} + +ft_inline void +ft_array_push(ft_array_type *arr, FT_SLICE_TYPE el) { + ft_array_invariants(arr); + ft_array_ensure(arr, 1); + arr->ptr[arr->len] = el; + arr->len++; +} + +ft_inline void +ft_array_push2(ft_array_type *arr, FT_SLICE_TYPE el1, FT_SLICE_TYPE el2) { + ft_array_invariants(arr); + ft_array_ensure(arr, 2); + arr->ptr[arr->len+0] = el1; + arr->ptr[arr->len+1] = el2; + arr->len+=2; +} + +ft_inline void +ft_array_insert_n(ft_array_type *arr, ssize_t at, FT_SLICE_TYPE *el, size_t n) { + bool alloced = false; + ft_array_invariants(arr); + + at = ft__slcindex_unify(at, arr->len); + + if (el + n >= arr->ptr && el < arr->ptr + arr->len) { + /* oops, overlap. + * Since we could reallocate array, we have to copy to allocated place */ + FT_SLICE_TYPE *cpy; + cpy = ft_malloc_arr(sizeof(FT_SLICE_TYPE), n); + memcpy(cpy, el, sizeof(FT_SLICE_TYPE) * n); + el = cpy; + alloced = true; + } + + ft_array_ensure(arr, n); + + if (at != arr->len) + memmove(&arr->ptr[at+n], &arr->ptr[at], + sizeof(FT_SLICE_TYPE) * (arr->len - at)); + memmove(&arr->ptr[at], el, sizeof(FT_SLICE_TYPE) * n); + arr->len += n; + + if (alloced) + ft_free(el); +} + +ft_inline void +ft_array_append(ft_array_type *arr, FT_SLICE_TYPE *el, size_t n) { + ft_array_invariants(arr); + ft_array_insert_n(arr, arr->len, el, n); +} + +ft_inline void +ft_array_walk_r(ft_array_type *arr, + FT_WALK_ACT (*walk)(FT_SLICE_TYPE *el, ft_arg_t arg), + ft_arg_t arg) { + size_t i, j = 0; + FT_WALK_ACT act = FT_WALK_CONT; + ft_array_invariants(arr); + + for (i = 0; i < arr->len && (act & FT_WALK_BREAK) == 0; i++) { + act = walk(&arr->ptr[i], arg); + if ((act & FT_WALK_DEL) == 0) { + if (i != j) + arr->ptr[j] = arr->ptr[i]; + j++; + } + } + /* move tail */ + if (i != arr->len) { + if (i != j) { + memmove(&arr->ptr[j], &arr->ptr[i], sizeof(FT_SLICE_TYPE)*(arr->len - i)); + } + j += arr->len - i; + } + + /* set length */ + if (j != arr->len) { + memset(&arr->ptr[j], 0, sizeof(FT_SLICE_TYPE)*(arr->len - j)); + arr->len = j; + if (arr->len < arr->cap / 4) + ft_array_recapa(arr, arr->len); + } +} + +ft_inline void +ft_array_walk(ft_array_type *arr, FT_WALK_ACT (*walk)(FT_SLICE_TYPE *el)) { + ft_array_walk_r(arr, (FT_WALK_ACT (*)(FT_SLICE_TYPE*, ft_arg_t))(void*)walk, ft_mka_z()); +} + +#undef FT_SLICE +#undef FT_SLICE_TYPE + +#undef ft_slice_pref +#undef ft_array_pref +#undef ft_slice_type +#undef ft_array_type + +#undef ft_slice_make +#undef ft_slice_alloc +#undef ft_slice_at +#undef ft_slice_set +#undef ft_slice_slice +#undef ft_slice_each +#undef ft_slice_each_r + +#undef ft_array_alloc +#undef ft_array_at +#undef ft_array_set +#undef ft_array_slice +#undef ft_array_each +#undef ft_array_each_r + +#undef ft_array_ensure +#undef ft_array_recapa +#undef ft_array_resize +#undef ft_array_reset_for_reuse +#undef ft_array_free + +#undef ft_array_insert_at +#undef ft_array_insert_n +#undef ft_array_push +#undef ft_array_push2 +#undef ft_array_append + +#undef ft_array_del_at +#undef ft_array_del_slice +#undef ft_array_pop + +#undef ft_array_walk +#undef ft_array_walk_r + +#undef HUGE_SIZE +#undef ft_slice_invariants +#undef ft_array_invariants + diff --git a/src/fu_util/ft_search.inc.h b/src/fu_util/ft_search.inc.h new file mode 100644 index 000000000..149874cd6 --- /dev/null +++ b/src/fu_util/ft_search.inc.h @@ -0,0 +1,116 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FU_UTIL_H +#error "ft_util.h should be included" +#endif + +/* + * Sort template. + * Accepts four macrosses: + * - FT_SEARCH - suffix for functions + * - FT_SEARCH_TYPE - type of array element + * - FT_SEARCH_PATTERN - optional type of key element. Defined to FT_SEARCH_TYPE if not present + * + * Produces: + * + * + * + * - binary search function + * It returns index of first element that is equal of greater to pattern in + * a sorted array. + * + * ft_bsres_t + * ft_bsearch_FT_SEARCH( + * FT_SEARCH_TYPE *arr, size_t len, FT_SEARCH_PATTERN pat, + * FT_CMP_RES (*cmp)(FT_SEARCH_TYPE el, FT_SEARCH_PATTERN pat)); + * ft_bsres_t + * ft_bsearch_FT_SEARCH_r( + * FT_SEARCH_TYPE *arr, size_t len, FT_SEARCH_PATTERN pat, + * FT_CMP_RES (*cmp)(FT_SEARCH_TYPE el, FT_SEARCH_PATTERN pat, ft_arg_t arg), + * ft_arg_t arg); + * + * - linear search function + * It returns index of first element that matches predicate, or len. + * + * size_t + * ft_search_FT_SEARCH( + * FT_SEARCH_TYPE *arr, size_t len, FT_SEARCH_PATTERN pat, + * FT_CMP_RES (*eq)(FT_SEARCH_TYPE el, FT_SEARCH_PATTERN pat)) + * or + * size_t + * ft_search_FT_SEARCH( + * FT_SEARCH_TYPE *arr, size_t len, FT_SEARCH_PATTERN pat, + * FT_CMP_RES (*eq)(FT_SEARCH_TYPE el, FT_SEARCH_PATTERN pat, ft_arg_t arg), + * ft_arg_t arg) + * + */ + +#define ft_func_bsearch fm_cat(ft_bsearch_, FT_SEARCH) +#define ft_func_bsearch_r fm_cat3(ft_bsearch_, FT_SEARCH, _r) +#define ft_func_search fm_cat(ft_search_, FT_SEARCH) +#define ft_func_search_r fm_cat3(ft_search_, FT_SEARCH, _r) +#ifndef FT_SEARCH_PATTERN +#define FT_SEARCH_PATTERN FT_SEARCH_TYPE +#endif + +#define _ft_cmp_def_r(x) \ + FT_CMP_RES (*x)(FT_SEARCH_TYPE a, FT_SEARCH_PATTERN b, ft_arg_t arg) +#define _ft_cmp_def(x) \ + FT_CMP_RES (*x)(FT_SEARCH_TYPE a, FT_SEARCH_PATTERN b) + +ft_inline ft_bsres_t +ft_func_bsearch_r(FT_SEARCH_TYPE *arr, size_t len, FT_SEARCH_PATTERN pat, + _ft_cmp_def_r(cmp), ft_arg_t arg) { + ft_bsres_t res = {len, false}; + size_t l, r, m; + int cmpres; + l = 0; + r = len; + while (l < r) { + m = l + (r - l) / 2; + cmpres = cmp(arr[m], pat, arg); + if (cmpres >= FT_CMP_EQ) { + r = m; + res.eq = cmpres == FT_CMP_EQ; + } else { + l = m + 1; + } + } + res.ix = l; + return res; +} + +ft_inline ft_bsres_t +ft_func_bsearch(FT_SEARCH_TYPE *arr, size_t len, FT_SEARCH_PATTERN pat, + _ft_cmp_def(cmp)) { + return ft_func_bsearch_r(arr, len, pat, (_ft_cmp_def_r())(void*) cmp, ft_mka_z()); +} + +ft_inline size_t +ft_func_search_r(FT_SEARCH_TYPE *arr, size_t len, FT_SEARCH_PATTERN pat, + _ft_cmp_def_r(cmp), ft_arg_t arg) { + size_t i; + for (i = 0; i < len; i++) { + if (cmp(arr[i], pat, arg) == FT_CMP_EQ) + break; + } + return i; +} + +ft_inline size_t +ft_func_search(FT_SEARCH_TYPE *arr, size_t len, FT_SEARCH_PATTERN pat, + _ft_cmp_def(cmp)) { + return ft_func_search_r(arr, len, pat, (_ft_cmp_def_r())(void*) cmp, ft_mka_z()); +} + +#undef FT_SEARCH +#undef FT_SEARCH_TYPE +#undef FT_SEARCH_PATTERN +#ifdef FT_SEARCH_ARG +#undef FT_SEARCH_ARG +#endif +#undef ft_func_bsearch +#undef ft_func_bsearch_r +#undef ft_func_search +#undef ft_func_search_r +#undef _ft_cmp_def_r +#undef _ft_cmp_def diff --git a/src/fu_util/ft_sort.inc.h b/src/fu_util/ft_sort.inc.h new file mode 100644 index 000000000..a55093908 --- /dev/null +++ b/src/fu_util/ft_sort.inc.h @@ -0,0 +1,174 @@ +/* + * Sort template. + * Accepts three macrosses: + * - FT_SORT - suffix for functions + * - FT_SORT_TYPE - type of array element + * - FT_SORT_ARG - optionally - argument to comparison function. + * + * Produces: + * + * - shell sort function + * void ft_shsort_FT_SORT(FT_SORT_TYPE *arr, size_t len, + * int (*cmp)(FT_SORT_TYPE a, FT_SORT_TYPE b)) + * void ft_shsort_FT_SORT_r(FT_SORT_TYPE *arr, size_t len, + * int (*cmp)(FT_SORT_TYPE a, FT_SORT_TYPE b, ft_arg_t arg), + * ft_arg_t arg) + * + * - quick sort function + * void ft_qsort_FT_SORT(FT_SORT_TYPE *arr, size_t len, + * int (*cmp)(FT_SORT_TYPE a, FT_SORT_TYPE b)) + * void ft_qsort_FT_SORT_r(FT_SORT_TYPE *arr, size_t len, + * int (*cmp)(FT_SORT_TYPE a, FT_SORT_TYPE b, ft_arg_t arg), + * ft_arg_t arg) + */ + +#include + +#ifndef FT_SORT +#error "FT_SORT should be defined" +#endif + +#ifndef FT_SORT_TYPE +#error "FT_SORT_TYPE should be defined" +#endif + +#define ft_func_shsort fm_cat(ft_shsort_, FT_SORT) +#define ft_func_shsort_r fm_cat3(ft_shsort_, FT_SORT, _r) +#define ft_func_qsort fm_cat(ft_qsort_, FT_SORT) +#define ft_func_qsort_r fm_cat3(ft_qsort_, FT_SORT, _r) + +#define _ft_cmp_def_r(x) int (*x)(FT_SORT_TYPE a, FT_SORT_TYPE b, ft_arg_t arg) +#define _ft_cmp_def(x) int (*x)(FT_SORT_TYPE a, FT_SORT_TYPE b) + +ft_inline ft_optimize3 void +ft_func_shsort_r(FT_SORT_TYPE *arr, size_t len, _ft_cmp_def_r(cmp), ft_arg_t arg) { + FT_SORT_TYPE el; + size_t m, n, d; + ft_dbg_assert((ssize_t)len >= 0); + if (len < 2) {} + else if (len == 2) { + if (cmp(arr[1], arr[0], arg) < 0) { + ft_swap(&arr[1], &arr[0]); + } + } else { + d = (size_t)(len / 1.4142135) | 1; + for (;;) { + for (m = d; m < len; m++) { + n = m; + el = arr[n]; + for (; n >= d && cmp(el, arr[n - d], arg) < 0; n -= d) { + arr[n] = arr[n-d]; + } + arr[n] = el; + } + if (d == 1) break; + else if (d < 10) d = 1; + else if (d <= 24) d = (size_t)(d / 2.221); + else d = (size_t)(d / 2.7182818) | 1; + } + } +} + +ft_inline ft_optimize3 void +ft_func_shsort(FT_SORT_TYPE *arr, size_t len, _ft_cmp_def(cmp)) { + ft_func_shsort_r(arr, len, (_ft_cmp_def_r())(void*) cmp, ft_mka_z()); +} + +ft_inline ft_optimize3 void +ft_func_qsort_r(FT_SORT_TYPE *arr_, size_t len_, _ft_cmp_def_r(cmp), ft_arg_t arg) { + FT_SORT_TYPE *arr = arr_; + FT_SORT_TYPE pivot; + size_t len = len_; + size_t m, n, mid[5]; +#define STSZ 32 + int const stsz = STSZ; + int sttop = 0; + struct { FT_SORT_TYPE *ar; size_t ln; } stack[STSZ]; +#undef STSZ + + ft_dbg_assert((ssize_t)len >= 0); + + stack[sttop].ar = arr; stack[sttop].ln = len; sttop++; + while (sttop > 0) { + sttop--; + arr = stack[sttop].ar; len = stack[sttop].ln; + /* check for fallback to shell sort */ + if (len < 24 || (sttop == stsz-1)) { + ft_func_shsort_r(arr, len, cmp, arg); + continue; + } + else { + m = 1; + while (m < len && cmp(arr[m-1], arr[m], arg) < 0) m++; + if (m == len) + continue; + } + /* find a pivot as median of 5 */ + mid[0] = 0; + mid[2] = len/2; + mid[1] = 1 + ft_randn(mid[2]-2); + mid[3] = mid[2] + 1 + ft_randn(mid[2]-2); + mid[4] = len-1; + /* fast median of 5 */ + { + static int const ix[] = {0, 1, 3, 4, 0, 3, 1, 4, 1, 2, 2, 3, 1, 2}; + for (int i = 0; i < ft_arrsz(ix); i += 2) { + if (cmp(arr[mid[ix[i]]], arr[mid[ix[i+1]]], arg) < 0) + ft_swap(&mid[ix[i]], &mid[ix[i+1]]); + } + } + /* make a[i] <= a[l] if i < m */ + pivot = arr[mid[2]]; + m = 0; n = len; + for (;;) { + while (m < n && cmp(pivot, arr[m], arg) >= 0) m++; + while (m < n && cmp(pivot, arr[n-1], arg) < 0) n--; + if (m == n) break; + ft_swap(&arr[m], &arr[n-1]); + m++; n--; + } + if (m < len) { + /* lgr - left greater */ + bool lgr = m > len - m; + stack[sttop+(1-lgr)].ar = arr; + stack[sttop+(1-lgr)].ln = m; + stack[sttop+lgr].ar = arr + m; + stack[sttop+lgr].ln = len - m; + sttop += 2; + } else { + /* all <= pivot */ + /* make a[i] < a[l] if i < m*/ + ft_dbg_assert(n == len); + for (;m > 0 && cmp(arr[m-1], pivot, arg) >= 0; m--); + n = m; + for (;m > 0;m--) { + if (cmp(arr[m-1], pivot, arg) >= 0) { + if (m < n) { + ft_swap(&arr[m-1], &arr[n-1]); + } + n--; + } + } + if (n > 0) { + stack[sttop].ar = arr; stack[sttop].ln = n; + sttop++; + } + } + } +} + +ft_inline ft_optimize3 void +ft_func_qsort(FT_SORT_TYPE *arr, size_t len, _ft_cmp_def(cmp)) { + ft_func_qsort_r(arr, len, (_ft_cmp_def_r())(void*) cmp, ft_mka_z()); +} + + +#undef FT_SORT +#undef FT_SORT_TYPE +#undef ft_func_shsort +#undef ft_func_shsort_r +#undef ft_func_qsort +#undef ft_func_qsort_r +#undef _ft_cmp_def_r +#undef _ft_cmp_def + diff --git a/src/fu_util/ft_ss_examples.h b/src/fu_util/ft_ss_examples.h new file mode 100644 index 000000000..a44c38856 --- /dev/null +++ b/src/fu_util/ft_ss_examples.h @@ -0,0 +1,131 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FT_SS_EXAMPLES_H +#define FT_SS_EXAMPLES_H + +/* + * Sort for integers. + * Defines: + * void + * ft_shsort_int (int *arr, size_t len, int (*cmp)(int, int)); + * void + * ft_qsort_int (int *arr, size_t len, int (*cmp)(int, int)); + * void + * ft_shsort_int_r(int *arr, size_t len, + * int (*cmp)(int, int, ft_arg_t), + * ft_arg_t); + * void + * ft_qsort_int_r (int *arr, size_t len, + * int (*cmp)(int, int, ft_arg_t), + * ft_arg_t); + */ +#define FT_SORT int +#define FT_SORT_TYPE int +#include +ft_inline FT_CMP_RES ft_int_cmp(int a, int b) { return ft_cmp(a, b); } + +/* + * Sort for strings. + * Defines: + * void + * ft_shsort_cstr (const char **arr, size_t len, + * int (*cmp)(const char*, const char*)); + * void + * ft_qsort_cstr (const char **arr, size_t len, + * int (*cmp)(const char*, const char*, ft_arg_t), + * ft_arg_t); + * void + * ft_shsort_cstr_r(const char **arr, size_t len, + * int (*cmp)(const char*, const char*)); + * void + * ft_qsort_cstr_r (const char **arr, size_t len, + * int (*cmp)(const char*, const char*, ft_arg_t), + * ft_arg_t); + */ +#define FT_SORT cstr +#define FT_SORT_TYPE const char* +#include +/* + * While we could pass raw strcmp to sort and search functions, + * lets define wrapper for clarity + */ +ft_inline FT_CMP_RES ft_cstr_cmp(const char *a, const char *b) { + return ft_cmp(strcmp(a, b), 0); +} + +/* + * Sort for void*. + * Defines: + * void + * ft_shsort_void (void **arr, size_t len, + * int (*cmp)(void*, void*)); + * void + * ft_qsort_void (void **arr, size_t len, + * int (*cmp)(void*, void*, ft_arg_t), + * ft_arg_t); + * void + * ft_shsort_void_r(void **arr, size_t len, + * int (*cmp)(void*, void*)); + * void + * ft_qsort_void_r (void **arr, size_t len, + * int (*cmp)(void*, void*, ft_arg_t), + * ft_arg_t); + */ +#define FT_SORT void +#define FT_SORT_TYPE void* +#include + +/* + * Search for integers. + * Defines: + * ft_bsres_t + * ft_bsearch_int (int *arr, size_t len, + * int (*cmp)(int, int)) + * ft_bsres_t + * ft_bsearch_int_r(int *arr, size_t len, + * int (*cmp)(int, int, ft_arg_t), + * ft_arg_t) + * size_t + * ft_qsort_int (int *arr, size_t len, + * bool (*eq)(int, int)) + * ft_qsort_int_r(int *arr, size_t len, + * bool (*eq)(int, int, ft_arg_t), + * ft_arg_t) + */ +#define FT_SEARCH int +#define FT_SEARCH_TYPE int +#include + +/* + * Search for strings. + * Defines: + * ft_bsres_t + * ft_bsearch_cstr (const char **arr, size_t len, + * int (*cmp)(const char*, const char*)) + * ft_bsres_t + * ft_bsearch_cstr_r(const char **arr, size_t len, + * int (*cmp)(const char*, const char*, ft_arg_t), + * ft_arg_t) + * size_t + * ft_qsort_cstr (const char **arr, size_t len, + * int (*eq)(const char*, const char*)) + * ft_qsort_cstr_r(const char **arr, size_t len, + * int (*eq)(const char*, const char*, ft_arg_t), + * ft_arg_t) + */ +#define FT_SEARCH cstr +#define FT_SEARCH_TYPE const char* +#include + +/* + * Search for void*. + * Defines: + * ft_bsres_t ft_bsearch_void(void **arr, size_t len, + * int (*cmp)(void*, void*)) + * ft_bsres_t ft_search_void(void **arr, size_t len, + * bool (*eq)(void*, void*)) + */ +#define FT_SEARCH void +#define FT_SEARCH_TYPE void* +#include + +#endif /* FT_SS_EXAMPLES_H */ diff --git a/src/fu_util/ft_util.h b/src/fu_util/ft_util.h new file mode 100644 index 000000000..fa37fbb69 --- /dev/null +++ b/src/fu_util/ft_util.h @@ -0,0 +1,541 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FU_UTIL_H +#define FU_UTIL_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include +/* trick to find ssize_t even on windows and strict ansi mode */ +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#define SSIZE_MAX ((ssize_t)((SIZE_MAX) >> 1)) + +#if !defined(WIN32) && defined(_WIN32) +#define WIN32 _WIN32 +#endif + +#endif +#include +#include +#include + +#ifdef __GNUC__ +#define ft_gcc_const __attribute__((const)) +#define ft_gcc_pure __attribute__((pure)) +#if __GNUC__ > 10 && !defined(__clang__) +#define ft_gcc_malloc(free, idx) __attribute__((malloc, malloc(free, idx))) +#else +#define ft_gcc_malloc(free, idx) __attribute__((malloc)) +#endif +#define ft_unused __attribute__((unused)) +#if !defined(__clang__) +#define ft_gnu_printf(fmt, arg) __attribute__((format(gnu_printf,fmt,arg))) +#else +#define ft_gnu_printf(fmt, arg) __attribute__((format(printf,fmt,arg))) +#endif +#define ft_likely(x) __builtin_expect(!!(x), 1) +#define ft_unlikely(x) __builtin_expect(!!(x), 0) +#define ft_always_inline __attribute__((always_inline)) +#define ft_cleanup(func) __attribute__((cleanup(func))) +#else +#define ft_gcc_const +#define ft_gcc_pure +#define ft_gcc_malloc(free, idx) +#define ft_unused +#define ft_gnu_printf(fmt, arg) +#define ft_likely(x) (x) +#define ft_unlikely(x) (x) +#define ft_always_inline +#endif +#define ft_static static ft_unused +#define ft_inline static ft_unused inline + +#if defined(__GNUC__) && !defined(__clang__) +#define ft_optimize3 __attribute__((optimize(3))) +#else +#define ft_optimize3 +#endif + +#if __STDC_VERSION__ >= 201112L +#elif defined(__GNUC__) && !defined(_Noreturn) +#define _Noreturn __attribute__((__noreturn__)) +#elif !defined(_Noreturn) +#define _Noreturn +#endif + +/* Logging and asserts */ + +#if defined(__GNUC__) && !defined(__clang__) +#define ft_FUNC __PRETTY_FUNCTION__ +#else +#define ft_FUNC __func__ +#endif + +typedef struct ft_source_position { + const char *file; + int line; + const char *func; +} ft_source_position_t; + +#define ft__srcpos() ((ft_source_position_t){.file=__FILE__,.line=__LINE__,.func=ft_FUNC}) + +enum FT_LOG_LEVEL { + FT_UNINITIALIZED = -100, + FT_DEBUG = -2, + FT_LOG = -1, + FT_INFO = 0, + FT_WARNING = 1, + FT_ERROR = 2, + FT_OFF = 3, + FT_FATAL = 98, + FT_TRACE = 100 /* for active debugging only */ +}; + +enum FT_ASSERT_LEVEL { FT_ASSERT_RUNTIME = 0, FT_ASSERT_ALL }; + +ft_inline const char* ft_log_level_str(enum FT_LOG_LEVEL level); + +/* + * Hook type to plugin external logging. + * Default loggin writes to stderr only. + */ +typedef void ft_gnu_printf(4, 0) (*ft_log_hook_t)(enum FT_LOG_LEVEL, + ft_source_position_t srcpos, + const char* error, + const char *fmt, + va_list args); +/* + * Initialize logging in main executable file. + * Pass custom hook or NULL. + * In MinGW if built with libbacktrace, pass executable path (argv[0]). + */ +#define ft_init_log(hook) ft__init_log(hook, __FILE__) + +/* Reset log level for all files */ +extern void ft_log_level_reset(enum FT_LOG_LEVEL level); +extern void ft_assert_level_reset(enum FT_ASSERT_LEVEL level); +/* Adjust log level for concrete file or all files */ +extern void ft_log_level_set(const char *file, enum FT_LOG_LEVEL level); +extern void ft_assert_level_set(const char *file, enum FT_ASSERT_LEVEL level); + +/* truncates filename to source root */ +const char* ft__truncate_log_filename(const char *file); + +/* register source for fine tuned logging */ +#define ft_register_source() ft__register_source_impl() + +/* log simple message */ +#define ft_log(level, fmt_or_msg, ...) \ + ft__log_impl(level, NULL, fmt_or_msg, __VA_ARGS__) +/* log message with error. Error will be appended as ": %s". */ +#define ft_logerr(level, error, fmt_or_msg, ...) \ + ft__log_impl(level, error, fmt_or_msg, __VA_ARGS__) + +/* + * Assertions uses standard logging for output. + * Assertions are runtime enabled: + * - ft_assert is enabled always. + * - ft_dbg_assert is disabled be default, but will be enabled if `ft_assert_level` is set positive. + */ + +#define ft_dbg_enabled() ft__dbg_enabled() +#define ft_dbg_assert(x, ...) ft__dbg_assert(x, #x, __VA_ARGS__) +#define ft_assert(x, ...) ft__assert(x, #x, ##__VA_ARGS__) +#define ft_assyscall(syscall, ...) ft__assyscall(syscall, fm_uniq(res), __VA_ARGS__) + +/* threadsafe strerror */ +extern const char* ft__strerror(int eno, char *buf, size_t len); +#ifndef __TINYC__ +extern const char* ft_strerror(int eno); +#else +#define ft_strerror(eno) ft__strerror(eno, (char[256]){0}, 256) +#endif + +// Memory + +// Standartize realloc(p, 0) +// Realloc implementations differ in handling newsz == 0 case: +// some returns NULL, some returns unique allocation. +// This version always returns NULL. +extern void* ft_realloc(void* ptr, size_t new_sz); +extern void* ft_calloc(size_t sz); +extern void* ft_realloc_arr(void* ptr, size_t elem_sz, size_t old_elems, size_t new_elems); + +extern void* ft_malloc(size_t sz); +extern void* ft_malloc_arr(size_t sz, size_t cnt); +extern void ft_free(void* ptr); +extern void* ft_calloc_arr(size_t sz, size_t cnt); + + +extern void ft_set_allocators(void *(*_realloc)(void *, size_t), + void (*_free)(void*)); + +/* overflow checking size addition and multiplication */ +ft_inline size_t ft_add_size(size_t a, size_t b); +ft_inline size_t ft_mul_size(size_t a, size_t b); + +/* division 64->32 bit */ +ft_inline int32_t ft_div_i64u32_to_i32(int64_t a, uint32_t b); + +#define ft_new(type) ft_calloc(sizeof(type)) +#define ft_newar(type, cnt) ft_calloc(ft_mul_size(sizeof(type), (cnt))) + +// Function to clear freshly allocated memory +extern void ft_memzero(void* ptr, size_t sz); + +// Comparison + +/* ft_max - macro-safe calculation of maximum */ +#define ft_max(a_, b_) ft__max((a_), (b_), fm_uniq(a), fm_uniq(b)) +/* ft_min - macro-safe calculation of minimum */ +#define ft_min(a_, b_) ft__min((a_), (b_), fm_uniq(a), fm_uniq(b)) + +/* Well, it is a bit fake enum. */ +typedef enum FT_CMP_RES { + FT_CMP_LT = -1, + FT_CMP_EQ = 0, + FT_CMP_GT = 1, + FT_CMP_NE = 2, +} FT_CMP_RES; +/* ft_cmp - macro-safe comparison */ +#define ft_cmp(a_, b_) ft__cmp((a_), (b_), fm_uniq(a), fm_uniq(b)) +/* ft_swap - macro-safe swap of variables */ +#define ft_swap(a_, b_) ft__swap((a_), (b_), fm_uniq(ap), fm_uniq(bp), fm_uniq(t)) + +/* ft_arrsz - geterminze size of static array */ +#define ft_arrsz(ar) (sizeof(ar)/sizeof(ar[0])) + +/* used in ft_*_foreach iterations to close implicit scope */ +#define ft_end_foreach } while(0) + +// Some Numeric Utils + +ft_inline uint32_t ft_rol32(uint32_t x, unsigned n); +ft_inline uint32_t ft_ror32(uint32_t x, unsigned n); +ft_inline size_t ft_nextpow2(size_t sz); + +/* + * Simple inline murmur hash implementation hashing a 32 bit integer, for + * performance. + */ +ft_inline uint32_t ft_mix32(uint32_t data); + + +/* Dumb quality random */ +extern uint32_t ft_rand(void); +/* Dumb quality random 0<=rptr); + *bytes = ft_bytes(NULL, 0); +} + +ft_inline void ft_bytes_consume(ft_bytes_t *bytes, size_t cut); +ft_inline size_t ft_bytes_move(ft_bytes_t *dest, ft_bytes_t *src); +ft_inline ft_bytes_t ft_bytes_split(ft_bytes_t *bytes, size_t n); + +extern ft_bytes_t ft_bytes_shift_line(ft_bytes_t *bytes); +ft_inline bool ft_bytes_shift_to(ft_bytes_t *bytes, ft_bytes_t to); +ft_inline void ft_bytes_shift_must(ft_bytes_t *bytes, ft_bytes_t to); + +extern size_t ft_bytes_find_bytes(ft_bytes_t haystack, ft_bytes_t needle); +ft_inline size_t ft_bytes_find_cstr(ft_bytes_t haystack, const char *needle); +ft_inline bool ft_bytes_has_cstr(ft_bytes_t haystack, const char *needle); + +ft_inline bool ft_bytes_starts_with(ft_bytes_t haystack, ft_bytes_t needle); +ft_inline bool ft_bytes_starts_withc(ft_bytes_t haystack, const char* needle); +ft_inline bool ft_bytes_ends_with(ft_bytes_t haystack, ft_bytes_t needle); +ft_inline bool ft_bytes_ends_withc(ft_bytes_t haystack, const char* needle); + +ft_inline size_t ft_bytes_spn(ft_bytes_t bytes, ft_bytes_t chars); +ft_inline size_t ft_bytes_notspn(ft_bytes_t bytes, ft_bytes_t chars); +ft_inline size_t ft_bytes_spnc(ft_bytes_t bytes, const char* chars); +ft_inline size_t ft_bytes_notspnc(ft_bytes_t bytes, const char* chars); + +// String utils +extern size_t ft_strlcpy(char *dest, const char* src, size_t dest_size); +/* + * Concat strings regarding destination buffer size. + * Note: if dest already full and doesn't contain \0n character, then fatal log is issued. + */ +extern size_t ft_strlcat(char *dest, const char* src, size_t dest_size); + +/* dup string using ft_malloc */ +ft_inline char * ft_cstrdup(const char *str); +ft_inline char * ft_cstrdupn(const char *str, size_t n); + +/**************** + * String + */ + +typedef struct ft_str_t { + char* ptr; + size_t len; +} ft_str_t; + +ft_inline ft_str_t ft_str(const char* ptr, size_t len) { + return (ft_str_t){.ptr = (char*)ptr, .len = len}; +} + +ft_inline ft_str_t ft_cstr(const char* ptr) { + return (ft_str_t){.ptr = (char*)ptr, .len = ptr ? strlen(ptr) : 0}; +} + +ft_inline ft_bytes_t ft_str2bytes(ft_str_t str) { + return ft_bytes(str.ptr, str.len); +} + +ft_inline ft_bytes_t ft_str2bytes_withzb(ft_str_t str) { + return ft_bytes(str.ptr, str.len+1); +} + +ft_inline ft_str_t ft_strdup(ft_str_t str); +ft_inline ft_str_t ft_strdupc(const char* str); +ft_inline ft_str_t ft_strdup_bytes(ft_bytes_t bytes); +/* use only if string was allocated */ +ft_inline void ft_str_free(ft_str_t *str); + +ft_inline ft_str_t ft_str_steal(ft_str_t *str) { + ft_str_t res = *str; + *str = ft_str(NULL, 0); + return res; +} + +/* print string into ft_malloc-ed buffer */ +extern ft_str_t ft_asprintf(const char *fmt, ...) ft_gnu_printf(1,2); +extern ft_str_t ft_vasprintf(const char *fmt, va_list args) ft_gnu_printf(1,0); + +ft_inline bool ft_streq (ft_str_t str, ft_str_t oth); +ft_inline FT_CMP_RES ft_strcmp (ft_str_t str, ft_str_t oth); +ft_inline bool ft_streqc (ft_str_t str, const char* oth); +ft_inline FT_CMP_RES ft_strcmpc(ft_str_t str, const char* oth); + +ft_inline void ft_str_consume(ft_str_t *str, size_t cut); + +ft_inline size_t ft_str_spnc(ft_str_t str, const char* chars); +ft_inline bool ft_str_ends_withc(ft_str_t str, const char* needle); +ft_inline size_t ft_str_find_cstr(ft_str_t haystack, const char *needle); + +ft_inline void ft_str_chop1(ft_str_t *str); + +/* shift zero-terminated string. Will assert if no zero-byte found and it is not last */ +extern ft_str_t ft_bytes_shift_zt(ft_bytes_t *bytes); + +/* + * String buffer. + * It could be growable or fixed. + * Note: it is limited by 4GB-1 bytes. + */ +typedef struct ft_strbuf_t ft_strbuf_t; +struct ft_strbuf_t { + char* ptr; + /* len doesn't encount last zero byte */ + uint32_t len; + /* cap is 1 byte less than real cap for zero bytes */ + uint32_t cap; + /* could buffer grow? + * It could be set on initialization, of if buffer reaches 4GB limit */ + bool fixed; + bool overflowed; + /* does ptr points to malloced place? */ + /* if so, then ft_strbuf_finish would not strdup */ + bool alloced; +}; + +/* empty growable buffer */ +ft_inline ft_strbuf_t ft_strbuf_zero(void); +/* + * Give buffer some non-freeable place, so it could use it until grows over. + * It will not prevent buffer from growing. In this case, buffer will be allocated. + * ft_strbuf_finish will duplicate string if it wasn't allocated. + * `capa` will be decreased by 1 to ensure there's room for zero-termination. + */ +ft_inline ft_strbuf_t ft_strbuf_init_stack(char* buf, size_t capa); +/* + * Give buffer some non-freeable place. + * Buffer will not grow more than that, and will remember if was overflowed. + * `capa` will be decreased by 1 to ensure there's room for zero-termination. + */ +ft_inline ft_strbuf_t ft_strbuf_init_fixed(char* buf, size_t capa); +/* + * Give buffer some non-freeable place for possible concatenation. + * `len` and `cap` are set to `str.len`, therefore it will reallocate on first addition. + */ +ft_inline ft_strbuf_t ft_strbuf_init_str(ft_str_t str); + +/* + * Ensure space for future addition. + * Returns false if buffer is fixed and there's no enough space. + */ +ft_inline bool ft_strbuf_ensure(ft_strbuf_t *buf, size_t n); + +/* All functions below returns false if fixed buffer was overflowed */ +ft_inline bool ft_strbuf_may (ft_strbuf_t *buf); +ft_inline bool ft_strbuf_cat (ft_strbuf_t *buf, ft_str_t s); +/* cat string together with zero-terminated byte */ +ft_inline bool ft_strbuf_cat_zt(ft_strbuf_t *buf, ft_str_t s); +ft_inline bool ft_strbuf_catbytes(ft_strbuf_t *buf, ft_bytes_t b); +ft_inline bool ft_strbuf_cat1 (ft_strbuf_t *buf, char c); +ft_inline bool ft_strbuf_cat2 (ft_strbuf_t *buf, char c1, char c2); +ft_inline bool ft_strbuf_catc (ft_strbuf_t *buf, const char *s); +ft_inline bool ft_strbuf_catc_zt(ft_strbuf_t *buf, const char *s); +ft_gnu_printf(2, 3) +extern bool ft_strbuf_catf (ft_strbuf_t *buf, const char *fmt, ...); +ft_gnu_printf(2, 0) +extern bool ft_strbuf_vcatf (ft_strbuf_t *buf, const char *fmt, va_list args); +/* + * err is filled with true, if vsnprintf returns error. + * Use it if format string comes from user. + */ +ft_gnu_printf(3, 0) +extern bool ft_strbuf_vcatf_err (ft_strbuf_t *buf, bool err[1], + const char *fmt, va_list args); +/* + * Returns string which points into the buffer. + * Buffer still owns content. + * Useful if `buf->alloced = false` and you will duplicate string in your own way. + */ +ft_inline ft_str_t ft_strbuf_ref(ft_strbuf_t *buf); + +/* + * Reset buffer's len to 0 without deallocation. + */ +ft_inline void ft_strbuf_reset_for_reuse(ft_strbuf_t *buf); + +/* + * Free buffer's buffer, if it was allocated + */ +ft_inline void ft_strbuf_free(ft_strbuf_t *buf); + +/* + * Always return allocated string. + * If buffer wasn't empty, returns it's ptr intact. + * If buffer was empty, allocate 1 bytes string with zero end + * Meaningless with fixed non-allocated buffer if you don't want to allocate. + * + * Buffer fields are cleared, therefore it will be unusable after. + * You will have to initialize it again. + */ +ft_inline ft_str_t ft_strbuf_steal(ft_strbuf_t *buf); + +#include "./impl/ft_impl.h" + +/* Include some examples for search and sort usages */ +//#include "./ft_ss_examples.h" +//#include "./ft_ar_examples.h" + +#endif diff --git a/src/fu_util/fu_utils_cfg.h b/src/fu_util/fu_utils_cfg.h new file mode 100644 index 000000000..e9b0ec5ca --- /dev/null +++ b/src/fu_util/fu_utils_cfg.h @@ -0,0 +1,9 @@ +#ifndef FU_UTILS_FU_UTILS_CFG_H +#define FU_UTILS_FU_UTILS_CFG_H + +#define FU_UTILS_VERSION_MAJOR 0 +#define FU_UTILS_VERSION_MINOR 1 + +//#undef HAVE_LIBBACKTRACE + +#endif //FU_UTILS_FU_UTILS_CFG_H diff --git a/src/fu_util/fu_utils_cfg.h.in b/src/fu_util/fu_utils_cfg.h.in new file mode 100644 index 000000000..f5b3c30af --- /dev/null +++ b/src/fu_util/fu_utils_cfg.h.in @@ -0,0 +1,9 @@ +#ifndef FU_UTILS_FU_UTILS_H_IN +#define FU_UTILS_FU_UTILS_H_IN + +#define FU_UTILS_VERSION_MAJOR @fu_utils_VERSION_MAJOR@ +#define FU_UTILS_VERSION_MINOR @fu_utils_VERSION_MINOR@ + +//#cmakedefine HAVE_LIBBACKTRACE + +#endif diff --git a/src/fu_util/impl/fo_impl.c b/src/fu_util/impl/fo_impl.c new file mode 100644 index 000000000..7e87c9f95 --- /dev/null +++ b/src/fu_util/impl/fo_impl.c @@ -0,0 +1,1532 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#include +#include +#include +#include +#include + +#include + +#include + +/* + * We limits total number of methods, klasses and method implementations. + * Restricted number allows to use uint16_t for id and doesn't bother with + * smart structures for hashes. + * If you need more, you have to change the way they are stored. + */ +#define FOBJ_OBJ_MAX_KLASSES (1<<10) +#define FOBJ_OBJ_MAX_METHODS (1<<10) +#define FOBJ_OBJ_MAX_METHOD_IMPLS (1<<15) + +enum { FOBJ_DISPOSING = 1, FOBJ_DISPOSED = 2 }; + +typedef enum { + FOBJ_RT_NOT_INITIALIZED, + FOBJ_RT_INITIALIZED, + FOBJ_RT_FROZEN +} FOBJ_GLOBAL_STATE; + +typedef struct fobj_header { +#ifndef NDEBUG +#define FOBJ_HEADER_MAGIC UINT64_C(0x1234567890abcdef) + uint64_t magic; +#endif + volatile uint32_t rc; + volatile uint16_t flags; + fobj_klass_handle_t klass; +} fobj_header_t; + +#define METHOD_PARTITIONS (16) + +typedef struct fobj_klass_registration { + const char *name; + uint32_t hash; + uint32_t hash_next; + + ssize_t size; + fobj_klass_handle_t parent; + + uint32_t nmethods; + + /* common methods */ + fobj__nm_impl_t(fobjDispose) dispose; + + volatile uint16_t method_lists[METHOD_PARTITIONS]; +} fobj_klass_registration_t; + +typedef struct fobj_method_registration { + const char *name; + uint32_t hash; + uint32_t hash_next; + + uint32_t nklasses; + volatile uint32_t first; +} fobj_method_registration_t; + +typedef struct fobj_method_impl { + uint16_t method; + uint16_t next_for_klass; + uint16_t klass; + uint16_t next_for_method; + void* impl; +} fobj_method_impl_t; + + +static fobj_klass_registration_t fobj_klasses[1<<10] = {{0}}; +static fobj_method_registration_t fobj_methods[1<<10] = {{0}}; +#define FOBJ_OBJ_HASH_SIZE (FOBJ_OBJ_MAX_METHODS/4) +static volatile uint16_t fobj_klasses_hash[FOBJ_OBJ_HASH_SIZE] = {0}; +static volatile uint16_t fobj_methods_hash[FOBJ_OBJ_HASH_SIZE] = {0}; +static fobj_method_impl_t fobj_method_impl[FOBJ_OBJ_MAX_METHOD_IMPLS] = {{0}}; +static volatile uint32_t fobj_klasses_n = 0; +static volatile uint32_t fobj_methods_n = 0; +static volatile uint32_t fobj_impls_n = 0; + +static fobj_t fobj_autorelease(fobj_t obj, fobj_autorelease_pool *pool); +static void fobj_release(fobj_t self); +static fobj_autorelease_pool** fobj_AR_current_ptr(void); + +static pthread_mutex_t fobj_runtime_mutex = PTHREAD_MUTEX_INITIALIZER; +static volatile uint32_t fobj_global_state = FOBJ_RT_NOT_INITIALIZED; + +#define pth_assert(...) do { \ + int rc = __VA_ARGS__; \ + ft_assert(!rc, "fobj_runtime_mutex: %s", ft_strerror(rc)); \ +} while(0) + +#define atload(v) __atomic_load_n((v), __ATOMIC_ACQUIRE) + +bool +fobj_method_init_impl(volatile fobj_method_handle_t *meth, const char *name) { + uint32_t hash, mh; + fobj_method_registration_t *reg; + + ft_dbg_assert(meth); + + pth_assert(pthread_mutex_lock(&fobj_runtime_mutex)); + if ((mh = *meth) != 0) { + reg = &fobj_methods[mh]; + pth_assert(pthread_mutex_unlock(&fobj_runtime_mutex)); + ft_assert(mh <= atload(&fobj_methods_n)); + ft_assert(strcmp(reg->name, name) == 0); + return true; + } + + + hash = ft_small_cstr_hash(name); + mh = fobj_methods_hash[hash % FOBJ_OBJ_HASH_SIZE]; + for (; mh != 0; mh = reg->hash_next) { + reg = &fobj_methods[mh]; + if (reg->hash == hash && strcmp(reg->name, name) == 0) { + __atomic_store_n(meth, mh, __ATOMIC_RELEASE); + pth_assert(pthread_mutex_unlock(&fobj_runtime_mutex)); + return true; + } + } + + ft_assert(fobj_global_state == FOBJ_RT_INITIALIZED); + + mh = fobj_methods_n + 1; + ft_dbg_assert(mh > 0); + ft_assert(*meth < FOBJ_OBJ_MAX_METHODS, "Too many methods defined"); + reg = &fobj_methods[mh]; + reg->name = name; + reg->hash = hash; + reg->hash_next = fobj_methods_hash[hash % FOBJ_OBJ_HASH_SIZE]; + fobj_methods_hash[hash % FOBJ_OBJ_HASH_SIZE] = mh; + + __atomic_store_n(&fobj_methods_n, mh, __ATOMIC_RELEASE); + __atomic_store_n(meth, mh, __ATOMIC_RELEASE); + + pth_assert(pthread_mutex_unlock(&fobj_runtime_mutex)); + + return false; +} + +static inline void* +fobj_search_impl(fobj_method_handle_t meth, fobj_klass_handle_t klass) { + uint32_t i; + + i = atload(&fobj_klasses[klass].method_lists[meth%METHOD_PARTITIONS]); + while (i != 0) { + if (fobj_method_impl[i].method == meth) + return fobj_method_impl[i].impl; + i = fobj_method_impl[i].next_for_klass; + } + + return NULL; +} + +void* +fobj_klass_method_search(fobj_klass_handle_t klass, fobj_method_handle_t meth) { + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + ft_dbg_assert(meth > 0 && meth <= atload(&fobj_methods_n)); + ft_dbg_assert(meth != fobj__nm_mhandle(fobjDispose)()); + ft_dbg_assert(klass > 0 && klass <= atload(&fobj_klasses_n)); + + do { + void *impl = fobj_search_impl(meth, klass); + if (impl) + return impl; + klass = fobj_klasses[klass].parent; + } while (klass != 0); + return NULL; +} + + +fobj__method_callback_t +fobj_method_search(const fobj_t self, fobj_method_handle_t meth, fobj_klass_handle_t for_child, bool validate) { + fobj_header_t *h; + fobj_klass_handle_t klass; + fobj_klass_handle_t for_klass; + fobj__method_callback_t cb = {self, NULL}; + + if (ft_unlikely(ft_dbg_enabled())) { + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + ft_assert(meth > 0 && meth <= atload(&fobj_methods_n)); + ft_assert(meth != fobj__nm_mhandle(fobjDispose)()); + } + + if (self == NULL) { + if (validate) + ft_assert(self != NULL, "Call '%s' on NULL object", fobj_methods[meth].name); + return cb; + } + + h = ((fobj_header_t*)self - 1); + assert(h->magic == FOBJ_HEADER_MAGIC); + klass = h->klass; + if (ft_unlikely(ft_dbg_enabled())) { + ft_assert(klass > 0 && klass <= atload(&fobj_klasses_n)); + ft_assert((h->flags & FOBJ_DISPOSED) == 0, "Call '%s' on disposed object '%s'", + fobj_methods[meth].name, fobj_klasses[klass].name); + } + + if (for_child != 0) { + if (ft_unlikely(ft_dbg_enabled())) { + while (klass && klass != for_child) { + klass = fobj_klasses[klass].parent; + } + ft_assert(klass == for_child); + } else { + klass = for_child; + } + klass = fobj_klasses[klass].parent; + } + + for_klass = klass; + + do { + cb.impl = fobj_search_impl(meth, klass); + if (cb.impl != NULL) + return cb; + + klass = fobj_klasses[klass].parent; + } while (klass); + if (validate) + ft_assert(cb.impl != NULL, "Klass '%s' has no method '%s'", + fobj_klasses[for_klass].name, + fobj_methods[meth].name); + cb.self = NULL; + return cb; +} + +bool +fobj_method_implements(const fobj_t self, fobj_method_handle_t meth) { + fobj_header_t *h; + fobj_klass_handle_t klass; + + if (self == NULL) + return false; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + if (ft_dbg_enabled()) { + ft_assert(meth > 0 && meth <= atload(&fobj_methods_n)); + ft_assert(meth != fobj__nm_mhandle(fobjDispose)()); + } + + h = ((fobj_header_t*)self - 1); + assert(h->magic == FOBJ_HEADER_MAGIC); + klass = h->klass; + ft_dbg_assert(klass > 0 && klass <= atload(&fobj_klasses_n)); + + do { + if (fobj_search_impl(meth, klass) != NULL) + return true; + + klass = fobj_klasses[klass].parent; + } while (klass); + return false; +} + +_Noreturn +void fobj__validate_arg(const char* file, int line, const char *arg) { + ft_log(FT_FATAL, "%s:%d: missing argument %s", file, line, arg); +} + +const char * +fobj_klass_name(fobj_klass_handle_t klass) { + fobj_klass_registration_t *reg; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + ft_dbg_assert(klass && klass <= atload(&fobj_klasses_n)); + + reg = &fobj_klasses[klass]; + + return reg->name; +} + +fobj_klass_handle_t +fobj_real_klass_of(fobj_t self) { + fobj_header_t *h; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + ft_assert(self != NULL); + + h = ((fobj_header_t*)self - 1); + assert(h->magic == FOBJ_HEADER_MAGIC); + return h->klass; +} + +static void fobj_method_register_priv(fobj_klass_handle_t klass, + fobj_method_handle_t meth, + void* impl); + +bool +fobj_klass_init_impl(volatile fobj_klass_handle_t *klass, + ssize_t size, + fobj_klass_handle_t parent, + fobj__method_impl_box_t *methods, + const char *name) { + uint32_t hash, kl; + fobj_klass_registration_t *reg; + + ft_assert(fobj_global_state == FOBJ_RT_INITIALIZED); + ft_dbg_assert(klass); + + pth_assert(pthread_mutex_lock(&fobj_runtime_mutex)); + + if ((kl = *klass) != 0) { + reg = &fobj_klasses[kl]; + pth_assert(pthread_mutex_unlock(&fobj_runtime_mutex)); + ft_assert(kl <= atload(&fobj_klasses_n)); + ft_assert(strcmp(reg->name, name) == 0); + ft_assert(reg->size == size); + ft_assert(reg->parent == parent); + return true; + } + + hash = ft_small_cstr_hash(name); + kl = fobj_klasses_hash[hash % FOBJ_OBJ_HASH_SIZE]; + for (; kl != 0; kl = reg->hash_next) { + reg = &fobj_klasses[kl]; + if (reg->hash == hash && strcmp(reg->name, name) == 0) { + __atomic_store_n(klass, kl, __ATOMIC_RELEASE); + pth_assert(pthread_mutex_unlock(&fobj_runtime_mutex)); + ft_assert(reg->size == size); + ft_assert(reg->parent == parent); + return true; + } + } + + kl = fobj_klasses_n + 1; + ft_dbg_assert(kl > 0); + ft_assert(*klass < FOBJ_OBJ_MAX_KLASSES, "Too many klasses defined"); + reg = &fobj_klasses[kl]; + reg->size = size; + reg->name = name; + reg->parent = parent; + reg->hash = hash; + reg->hash_next = fobj_klasses_hash[hash % FOBJ_OBJ_HASH_SIZE]; + fobj_klasses_hash[hash % FOBJ_OBJ_HASH_SIZE] = kl; + + __atomic_store_n(&fobj_klasses_n, kl, __ATOMIC_RELEASE); + /* declare methods before store klass */ + while (methods->meth != 0) { + fobj_method_register_priv(kl, methods->meth, methods->impl); + methods++; + } + + __atomic_store_n(klass, kl, __ATOMIC_RELEASE); + + pth_assert(pthread_mutex_unlock(&fobj_runtime_mutex)); + + return false; +} + +static void +fobj_method_register_priv(fobj_klass_handle_t klass, fobj_method_handle_t meth, void* impl) { + fobj_method_registration_t *mreg; + fobj_klass_registration_t *kreg; + void *existed; + uint32_t nom; + + mreg = &fobj_methods[meth]; + kreg = &fobj_klasses[klass]; + + existed = fobj_search_impl(meth, klass); + ft_dbg_assert(existed == NULL || existed == impl, + "Method %s.%s is redeclared with different implementation", + kreg->name, mreg->name); + + if (existed == impl) { + return; + } + + nom = fobj_impls_n + 1; + ft_assert(nom < FOBJ_OBJ_MAX_METHOD_IMPLS); + fobj_method_impl[nom].method = meth; + fobj_method_impl[nom].klass = klass; + fobj_method_impl[nom].next_for_method = mreg->first; + fobj_method_impl[nom].next_for_klass = kreg->method_lists[meth%METHOD_PARTITIONS]; + fobj_method_impl[nom].impl = impl; + __atomic_store_n(&mreg->first, nom, __ATOMIC_RELEASE); + __atomic_store_n(&kreg->method_lists[meth%METHOD_PARTITIONS], nom, + __ATOMIC_RELEASE); + + if (meth == fobj__nm_mhandle(fobjDispose)()) + kreg->dispose = (fobj__nm_impl_t(fobjDispose)) impl; + + __atomic_store_n(&fobj_impls_n, nom, __ATOMIC_RELEASE); +} + +void +fobj_method_register_impl(fobj_klass_handle_t klass, fobj_method_handle_t meth, void* impl) { + ft_assert(fobj_global_state == FOBJ_RT_INITIALIZED); + ft_dbg_assert(meth > 0 && meth <= atload(&fobj_methods_n)); + ft_dbg_assert(klass > 0 && klass <= atload(&fobj_klasses_n)); + + pth_assert(pthread_mutex_lock(&fobj_runtime_mutex)); + + fobj_method_register_priv(klass, meth, impl); + + pth_assert(pthread_mutex_unlock(&fobj_runtime_mutex)); +} + +void* +fobj__allocate(fobj_klass_handle_t klass, void *init, ssize_t size) { + fobj_klass_registration_t *kreg; + fobj_header_t *hdr; + fobj_t self; + ssize_t copy_size; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + ft_dbg_assert(klass > 0 && klass <= atload(&fobj_klasses_n)); + + kreg = &fobj_klasses[klass]; + copy_size = kreg->size >= 0 ? kreg->size : -1-kreg->size; + if (size < 0) { + size = copy_size; + } else { + ft_assert(kreg->size < 0); + size += copy_size; + } + hdr = ft_calloc(sizeof(fobj_header_t) + size); +#ifndef NDEBUG + hdr->magic = FOBJ_HEADER_MAGIC; +#endif + hdr->klass = klass; + hdr->rc = 1; + self = (fobj_t)(hdr + 1); + if (init != NULL) + memcpy(self, init, copy_size); + fobj_autorelease(self, *fobj_AR_current_ptr()); + return self; +} + +fobj_t +fobj_ref(fobj_t self) { + fobj_header_t *h; + if (self == NULL) + return NULL; + h = ((fobj_header_t*)self - 1); + assert(h->magic == FOBJ_HEADER_MAGIC); + ft_assert(h->klass > 0 && h->klass <= atload(&fobj_klasses_n)); + __atomic_fetch_add(&h->rc, 1, __ATOMIC_ACQ_REL); + return self; +} + +void +fobj_set(fobj_t *ptr, fobj_t val) { + fobj_t oldval = *ptr; + *ptr = val ? fobj_ref(val) : NULL; + if (oldval) fobj_release(oldval); +} + +fobj_t +fobj_swap(fobj_t *ptr, fobj_t val) { + fobj_t oldval = *ptr; + *ptr = val ? fobj_ref(val) : NULL; + return oldval ? fobj_autorelease(oldval, *fobj_AR_current_ptr()) : NULL; +} + +fobj_t +fobj_unref(fobj_t val) { + return fobj_autorelease(val, *fobj_AR_current_ptr()); +} + +static void +fobj__dispose_req(fobj_t self, fobj_klass_registration_t *kreg) { + if (kreg->dispose) + kreg->dispose(self); + if (kreg->parent) { + fobj_klass_registration_t *preg; + + preg = &fobj_klasses[kreg->parent]; + fobj__dispose_req(self, preg); + } +} + +static void +fobj__do_dispose(fobj_t self, fobj_header_t *h, fobj_klass_registration_t *kreg) { + uint32_t old = __atomic_fetch_or(&h->flags, FOBJ_DISPOSING, __ATOMIC_ACQ_REL); + if (old & FOBJ_DISPOSING) + return; + fobj__dispose_req(self, kreg); + __atomic_fetch_or(&h->flags, FOBJ_DISPOSED, __ATOMIC_ACQ_REL); + + if (atload(&h->rc) == 0) + { + *h = (fobj_header_t){0}; + ft_free(h); + } +} + +static void +fobj_release(fobj_t self) { + fobj_header_t *h; + fobj_klass_handle_t klass; + fobj_klass_registration_t *kreg; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + + if (self == NULL) + return; + + h = ((fobj_header_t*)self - 1); + assert(h->magic == FOBJ_HEADER_MAGIC); + klass = h->klass; + ft_dbg_assert(klass > 0 && klass <= atload(&fobj_klasses_n)); + kreg = &fobj_klasses[klass]; + + + if (__atomic_sub_fetch(&h->rc, 1, __ATOMIC_ACQ_REL) != 0) + return; + if ((atload(&h->flags) & FOBJ_DISPOSING) != 0) + return; + fobj__do_dispose(self, h, kreg); +} + +#if 0 +void +fobj_dispose(fobj_t self) { + fobj_header_t *h; + fobj_klass_handle_t klass; + fobj_klass_registration_t *kreg; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + + if (self == NULL) + return; + + h = ((fobj_header_t*)self - 1); + assert(h->magic == FOBJ_HEADER_MAGIC); + klass = h->klass; + ft_dbg_assert(klass > 0 && klass <= atload(&fobj_klasses_n)); + kreg = &fobj_klasses[klass]; + + fobj__do_dispose(self, h, kreg); +} + +bool +fobj_disposing(fobj_t self) { + fobj_header_t *h; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + ft_assert(self != NULL); + + h = ((fobj_header_t*)self - 1); + assert(h->magic == FOBJ_HEADER_MAGIC); + return (atload(&h->flags) & FOBJ_DISPOSING) != 0; +} + +bool +fobj_disposed(fobj_t self) { + fobj_header_t *h; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + ft_assert(self != NULL); + + h = ((fobj_header_t*)self - 1); + assert(h->magic == FOBJ_HEADER_MAGIC); + return (atload(&h->flags) & FOBJ_DISPOSED) != 0; +} + +#endif + +static fobj_klass_handle_t +fobjBase_fobjKlass(fobj_t self) { + return fobj_real_klass_of(self); +} + +static fobjStr* +fobjBase_fobjRepr(VSelf) { + Self(fobjBase); + fobj_klass_handle_t klass = fobjKlass(self); + return fobj_sprintf("%s@%p", fobj_klass_name(klass), self); +} + +err_i +fobj_err_combine(err_i fst, err_i scnd) { + fobjErr* first = (fobjErr*)fst.self; + fobjErr* second = (fobjErr*)scnd.self; + fobjErr **tail; + if (first == NULL) + return scnd; + if (second == NULL) + return fst; + ft_assert(fobj_real_klass_of(first) == fobjErr__kh()); + ft_assert(fobj_real_klass_of(second) == fobjErr__kh()); + if (first->sibling != NULL) { + tail = &second->sibling; + while (*tail != NULL) tail = &(*tail)->sibling; + /* ownership is also transferred */ + *tail = first->sibling; + } + first->sibling = $ref(second); + return fst; +} + +static fobjStr* +fobj_reservestr(size_t size) { + fobjStr *str; +#if __SIZEOF_POINTER__ < 8 + ft_assert(size < (1<<30)-2); +#else + ft_assert(size < UINT32_MAX-2); +#endif + if (size < FOBJ_STR_SMALL_SIZE) { + if (size < FOBJ_STR_FREE_SPACE) + str = fobj_alloc(fobjStr); + else { + size_t diff = size + 1 - FOBJ_STR_FREE_SPACE; + str = fobj_alloc_sized(fobjStr, diff); + } + str->small.type = FOBJ_STR_SMALL; + str->small.len = size; + str->small.buf[size] = '\0'; + } else { + str = fobj_alloc_sized(fobjStr, size + 1); + str->ptr.type = FOBJ_STR_UNOWNED; // abuse it because we don't need separate deallocation + str->ptr.len = size; + str->ptr.ptr = (char*)(str+1); + str->ptr.ptr[size] = '\0'; + } + return str; +} + +fobjStr* +fobj_newstr(ft_str_t s, enum FOBJ_STR_ALLOC ownership) { + fobjStr *str; +#if __SIZEOF_POINTER__ < 8 + ft_assert(s.len < (1<<30)-2); +#else + ft_assert(s.len < UINT32_MAX-2); +#endif + if (s.len >= FOBJ_STR_FREE_SPACE && + (ownership == FOBJ_STR_GIFTED || ownership == FOBJ_STR_CONST)) { + str = fobj_alloc(fobjStr); + str->ptr.type = ownership == FOBJ_STR_GIFTED ? FOBJ_STR_PTR : FOBJ_STR_UNOWNED; + str->ptr.len = s.len; + str->ptr.ptr = s.ptr; + return str; + } + str = fobj_reservestr(s.len); + memcpy(fobj_getstr(str).ptr, s.ptr, s.len); + if (ownership == FOBJ_STR_GIFTED) + ft_str_free(&s); + return str; +} + +ft_inline fobjStr* fobj_str_const(const char* s); + +static void +fobjStr_fobjDispose(VSelf) { + Self(fobjStr); + if (self->type == FOBJ_STR_PTR) { + ft_free(self->ptr.ptr); + } +} + +fobjStr* +fobj_strcat(fobjStr *self, ft_str_t s) { + fobjStr *newstr; + ft_str_t news; + ft_str_t selfs = fobj_getstr(self); + size_t alloc_len = selfs.len + s.len + 1; + ft_assert(alloc_len < UINT32_MAX-2); + + if (s.len == 0) + return self; + + newstr = fobj_reservestr(alloc_len-1); + news = fobj_getstr(newstr); + memcpy(news.ptr, selfs.ptr, selfs.len); + memcpy(news.ptr + selfs.len, s.ptr, s.len); + return newstr; +} + +fobjStr* +fobj_strcat2(fobjStr *self, ft_str_t s1, ft_str_t s2) { + fobjStr *newstr; + ft_str_t news; + ft_str_t selfs = fobj_getstr(self); + size_t alloc_len = selfs.len + s1.len + s2.len + 1; + ft_assert(alloc_len < UINT32_MAX-2); + + if (s1.len + s2.len == 0) + return self; + + newstr = fobj_reservestr(alloc_len-1); + news = fobj_getstr(newstr); + memcpy(news.ptr, selfs.ptr, selfs.len); + memcpy(news.ptr + selfs.len, s1.ptr, s1.len); + memcpy(news.ptr + selfs.len + s1.len, s2.ptr, s2.len); + return newstr; +} + +fobjStr* +fobj_sprintf(const char *fmt, ...) { + char buffer[256] = {0}; + ft_strbuf_t buf = ft_strbuf_init_stack(buffer, 256); + va_list args; + + va_start(args, fmt); + ft_strbuf_vcatf(&buf, fmt, args); + va_end(args); + + return fobj_strbuf_steal(&buf); +} + +fobjStr* +fobj_strcatf(fobjStr *ostr, const char *fmt, ...) { + ft_strbuf_t buf = ft_strbuf_init_str(fobj_getstr(ostr)); + bool err; + va_list args; + + va_start(args, fmt); + ft_strbuf_vcatf_err(&buf, &err, fmt, args); + va_end(args); + + if (err) { + ft_log(FT_ERROR, "error printing format '%s'", fmt); + return NULL; + } + + /* empty print? */ + if (ft_strbuf_ref(&buf).ptr == fobj_getstr(ostr).ptr) { + return ostr; + } + return fobj_strbuf_steal(&buf); +} + +fobjStr* +fobj_tostr(fobj_t obj, const char *fmt) { + char buffer[32]; + ft_strbuf_t buf = ft_strbuf_init_stack(buffer, 32); + + if (obj == NULL) { + return fobj_str(""); + } + + if (fobj_real_klass_of(obj) == fobjStr__kh() && (fmt == NULL || fmt[0] == '\0')) { + return obj; + } + + if (!$ifdef(, fobjFormat, obj, &buf, fmt)) { + /* fallback to Repr */ + return $(fobjRepr, obj); + } + return fobj_strbuf_steal(&buf); +} + +static void +fobj_format_string(ft_strbuf_t *buf, ft_str_t str, const char *fmt) { + int i; + char c; + + if (fmt == NULL || fmt[0] == '\0') { + ft_strbuf_cat(buf, str); + return; + } else if (strcmp(fmt, "q") != 0) { + char realfmt[32] = "%"; + + ft_assert(ft_strlcat(realfmt, fmt, 32) < 32); + ft_strbuf_catf(buf, realfmt, str.ptr); + + return; + } + + /* Ok, we're asked for quoted representation */ + if (str.ptr == NULL) { + ft_strbuf_catc(buf, "NULL"); + } + + ft_strbuf_cat1(buf, '"'); + for (i = 0; i < str.len; i++) { + c = str.ptr[i]; + switch (c) { + case '\"': ft_strbuf_catc(buf, "\\\""); break; + case '\t': ft_strbuf_catc(buf, "\\t"); break; + case '\n': ft_strbuf_catc(buf, "\\n"); break; + case '\r': ft_strbuf_catc(buf, "\\r"); break; + case '\a': ft_strbuf_catc(buf, "\\a"); break; + case '\b': ft_strbuf_catc(buf, "\\b"); break; + case '\f': ft_strbuf_catc(buf, "\\f"); break; + case '\v': ft_strbuf_catc(buf, "\\v"); break; + case '\\': ft_strbuf_catc(buf, "\\\\"); break; + default: + if (c < 0x20) { + ft_strbuf_catc(buf, "\\x"); + ft_strbuf_cat2(buf, '0'+(c>>4), ((c&0xf)<=9?'0':'a')+(c&0xf)); + } else { + ft_strbuf_cat1(buf, c); + } + } + } + ft_strbuf_cat1(buf, '"'); +} + +static fobjStr* +fobjStr_fobjRepr(VSelf) { + Self(fobjStr); + char buffer[32] = {0}; + ft_strbuf_t buf = ft_strbuf_init_stack(buffer, 32); + + ft_strbuf_catc(&buf, "$S("); + fobj_format_string(&buf, fobj_getstr(self), "q"); + ft_strbuf_cat1(&buf, ')'); + + return fobj_strbuf_steal(&buf); +} + +static void +fobjStr_fobjFormat(VSelf, ft_strbuf_t *out, const char *fmt) { + Self(fobjStr); + fobj_format_string(out, fobj_getstr(self), fmt); +} + +static fobjStr* +fobjInt_fobjRepr(VSelf) { + Self(fobjInt); + return fobj_sprintf("$I(%"PRIi64")", self->i); +} + +static void +fobj_format_int(ft_strbuf_t *buf, uint64_t i, bool _signed, const char *fmt) { + char tfmt[32] = "%"; + char base; + size_t fmtlen; + + + if (fmt == NULL || fmt[0] == 0) { + if (_signed) { + ft_strbuf_catf(buf, "%"PRIi64, (int64_t)i); + } else { + ft_strbuf_catf(buf, "%"PRIu64, (uint64_t)i); + } + return; + } + + /* need to clean length specifiers ('l', 'll', 'z') */ + fmtlen = ft_strlcat(tfmt, fmt, 32); + ft_assert(fmtlen<28); + base = tfmt[fmtlen-1]; + ft_assert(base=='x' || base=='X' || base=='o' || base=='u' || + base=='d' || base=='i'); + do fmtlen--; + while (tfmt[fmtlen-1] == 'l' || tfmt[fmtlen-1] == 'z'); + tfmt[fmtlen] = '\0'; + + /* now add real suitable format */ + switch (base) { + case 'x': ft_strlcat(tfmt, PRIx64, sizeof(tfmt)); break; + case 'X': ft_strlcat(tfmt, PRIX64, sizeof(tfmt)); break; + case 'o': ft_strlcat(tfmt, PRIo64, sizeof(tfmt)); break; + case 'u': ft_strlcat(tfmt, PRIu64, sizeof(tfmt)); break; + case 'd': ft_strlcat(tfmt, PRId64, sizeof(tfmt)); break; + default: + case 'i': ft_strlcat(tfmt, PRIi64, sizeof(tfmt)); break; + } + + switch (base) { + case 'd': case 'i': + ft_strbuf_catf(buf, tfmt, (int64_t)i); + break; + default: + ft_strbuf_catf(buf, tfmt, (uint64_t)i); + break; + } +} + +static void +fobjInt_fobjFormat(VSelf, ft_strbuf_t *buf, const char *fmt) { + Self(fobjInt); + fobj_format_int(buf, self->i, true, fmt); +} + +static fobjStr* +fobjUInt_fobjRepr(VSelf) { + Self(fobjUInt); + return fobj_sprintf("$U(%"PRIu64")", self->u); +} + +static void +fobjUInt_fobjFormat(VSelf, ft_strbuf_t *buf, const char *fmt) { + Self(fobjUInt); + fobj_format_int(buf, self->u, false, fmt); +} + +static fobjStr* +fobjFloat_fobjRepr(VSelf) { + Self(fobjFloat); + return fobj_sprintf("$F(%f)", self->f); +} + +static void +fobj_format_float(ft_strbuf_t *buf, double f, const char *fmt) { + char tfmt[32] = "%"; + + if (fmt == NULL || fmt[0] == 0) { + ft_strbuf_catf(buf, "%f", f); + return; + } + ft_strlcat(tfmt, fmt, 32); + ft_strbuf_catf(buf, tfmt, f); +} + +static void +fobjFloat_fobjFormat(VSelf, ft_strbuf_t *buf, const char *fmt) { + Self(fobjFloat); + fobj_format_float(buf, self->f, fmt); +} + +static fobjBool* fobjTrue = NULL; +static fobjBool* fobjFalse = NULL; +static fobjStr* trueRepr = NULL; +static fobjStr* falseRepr = NULL; + +fobjBool* +fobj_bool(bool b) { + return b ? fobjTrue : fobjFalse; +} + +static fobjStr* +fobjBool_fobjRepr(VSelf) { + Self(fobjBool); + return self->b ? trueRepr : falseRepr; +} + +static void +fobj_format_bool(ft_strbuf_t *buf, bool b, const char *fmt) { + char tfmt[32] = "%"; + size_t fmtlen; + const char *repr = NULL; + + if (fmt == NULL || fmt[0] == 0) { + if (b) + ft_strbuf_catc(buf, "true"); + else + ft_strbuf_catc(buf, "false"); + return; + } + fmtlen = ft_strlcat(tfmt, fmt, 32); + switch (tfmt[fmtlen-1]) { + case 'B': repr = b ? "TRUE" : "FALSE"; break; + case 'b': repr = b ? "true" : "false"; break; + case 'P': repr = b ? "True" : "False"; break; + case 'Y': repr = b ? "Yes" : "No"; break; + case 'y': repr = b ? "yes" : "no"; break; + } + if (repr != NULL) { + tfmt[fmtlen-1] = 's'; + ft_strbuf_catf(buf, tfmt, repr); + } else { + ft_strbuf_catf(buf, tfmt, b); + } +} + +static void +fobjBool_fobjFormat(VSelf, ft_strbuf_t *buf, const char *fmt) { + Self(fobjBool); + fobj_format_bool(buf, self->b, fmt); +} + +static void +fobj_format_arg(ft_strbuf_t *out, ft_arg_t arg, const char *fmt) { + switch (ft_arg_type(arg)) { + case 'i': + fobj_format_int(out, (uint64_t)arg.v.i, true, fmt); + break; + case 'u': + fobj_format_int(out, arg.v.i, false, fmt); + break; + case 'f': + fobj_format_float(out, arg.v.f, fmt); + break; + case 's': + fobj_format_string(out, ft_cstr(arg.v.s), fmt); + break; + case 'b': + fobj_format_bool(out, arg.v.b, fmt); + break; + case 'o': + if (arg.v.o == NULL) { + ft_strbuf_catc(out, "(null)"); + } else if (!$ifdef(, fobjFormat, arg.v.o, out, fmt)) { + fobjStr* repr = $(fobjRepr, arg.v.o); + ft_strbuf_cat(out, fobj_getstr(repr)); + } + break; + default: + ft_assert(false, "Could not format arg of type '%c'", ft_arg_type(arg)); + } +} + +static void +fobj_repr_arg(ft_strbuf_t *out, ft_arg_t arg) { + fobjStr* repr; + switch (ft_arg_type(arg)) { + case 'i': + fobj_format_int(out, (uint64_t)arg.v.i, true, "i"); + break; + case 'u': + fobj_format_int(out, arg.v.u, false, NULL); + break; + case 'f': + fobj_format_float(out, arg.v.f, NULL); + break; + case 's': + fobj_format_string(out, ft_cstr(arg.v.s), "q"); + break; + case 'b': + fobj_format_bool(out, arg.v.b, NULL); + break; + case 'o': + if (arg.v.o == NULL) { + ft_strbuf_catc(out, "NULL"); + } else { + repr = $(fobjRepr, arg.v.o); + ft_strbuf_cat(out, fobj_getstr(repr)); + } + break; + default: + ft_assert(false, "Could not represent arg of type '%c'", ft_arg_type(arg)); + } +} + +static const char* +fobj__format_errmsg(const char* msg, fobj_err_kv_t *kvs) { + char buf[128]; + ft_strbuf_t out = ft_strbuf_init_stack(buf, 128); + bool found; + const char* cur; + char* closebrace; + char* formatdelim; + size_t identlen; + size_t formatlen; + char ident[32]; + char format[32]; + fobj_err_kv_t* kv; + + if (strchr(msg, '{') == NULL || strchr(msg, '}') == NULL) + return ft_cstrdup(msg); + + for (cur = msg; *cur; cur++) { + if (*cur != '{') { + ft_strbuf_cat1(&out, *cur); + continue; + } + if (cur[1] == '{') { + ft_strbuf_cat1(&out, '{'); + cur++; + continue; + } + cur++; + closebrace = strchr(cur, '}'); + ft_assert(closebrace, "error format string braces unbalanced"); + formatdelim = memchr(cur, ':', closebrace - cur); + identlen = (formatdelim ?: closebrace) - cur; + ft_assert(identlen <= 31, + "ident is too long in message \"%s\"", msg); + ft_assert(formatdelim == NULL || closebrace - formatdelim <= 31, + "format is too long in message \"%s\"", msg); + memcpy(ident, cur, identlen); + ident[identlen] = 0; + formatlen = formatdelim ? closebrace - (formatdelim+1) : 0; + if (formatlen > 0) { + memcpy(format, formatdelim + 1, formatlen); + } + format[formatlen] = 0; + kv = kvs; + found = false; + for (;kv->key != NULL; kv++) { + if (strcmp(kv->key, ident) == 0) { + found = true; + fobj_format_arg(&out, kv->val, format); + break; + } + } + ft_dbg_assert(found, "ident '%s' is not found (message \"%s\")", ident, msg); + cur = closebrace; + } + + return ft_strbuf_steal(&out).ptr; +} + +extern err_i +fobj__make_err(const char *type, + ft_source_position_t src, + const char *msg, + fobj_err_kv_t *kvs, + size_t kvn) { + fobjErr* err; + fobj_err_kv_t* kv; + fobj_err_kv_t* cpy; + ft_strbuf_t nmsg; + + err = fobj_alloc_sized(fobjErr, + ft_mul_size(sizeof(*kvs), kvn+1), + .type = type ?: "RT", + .src = src); + err->src.file = ft__truncate_log_filename(err->src.file); + msg = msg ?: err->type ?: "Unspecified Error"; + nmsg = ft_strbuf_init_str(ft_cstr(msg)); + /* search for suffix */ + if (kvn > 0) { + memcpy(err->kv, kvs, sizeof(*kvs)*kvn); + cpy = err->kv; + for (kv = err->kv; kv->key; kv++) { + if (strcmp(kv->key, "__msgSuffix") == 0) { + ft_strbuf_catc(&nmsg, ft_arg_s(kv->val)); + continue; + } + switch (ft_arg_type(kv->val)) { + case 'o': + $ref(ft_arg_o(kv->val)); + break; + case 's': + kv->val.v.s = kv->val.v.s ? ft_cstrdup(kv->val.v.s) : NULL; + break; + } + if (cpy != kv) + *cpy = *kv; + cpy++; + } + if (cpy != kv) + *cpy = (fobj_err_kv_t){NULL, ft_mka_z()}; + } + err->message = fobj__format_errmsg(ft_strbuf_ref(&nmsg).ptr, err->kv); + ft_strbuf_free(&nmsg); + return bind_err(err); +} + +err_i +fobj__alloc_err(const char *type, + ft_source_position_t src, + const char *msg, + fobj_err_kv_t *kvs, + size_t kvn) { + fobjErr* err; + fobj_err_kv_t* kv; + + src.func = ft_cstrdup(src.func); + src.file = ft_cstrdup(src.file); + err = fobj_alloc_sized(fobjErr, + ft_mul_size(sizeof(*kvs), kvn+1), + .type = ft_cstrdup(type), + .message = ft_cstrdup(msg), + .src = src, + .free_type_and_src = true, + ); + memcpy(err->kv, kvs, sizeof(*kvs)*kvn); + /* search for suffix */ + for (kv = err->kv; kv->key; kv++) { + switch (ft_arg_type(kv->val)) { + case 'o': + $ref(ft_arg_o(kv->val)); + break; + case 's': + kv->val.v.s = kv->val.v.s ? ft_cstrdup(kv->val.v.s) : NULL; + break; + } + } + return bind_err(err); +} + +static void +fobjErr__fobjErr_marker_DONT_IMPLEMENT_ME(VSelf) { +} + +static void +fobjErr_fobjDispose(VSelf) { + Self(fobjErr); + fobj_err_kv_t *kv; + for (kv = self->kv; kv->key != NULL; kv++) { + switch (ft_arg_type(kv->val)) { + case 'o': + $del(&kv->val.v.o); + break; + case 's': + ft_free(kv->val.v.s); + break; + } + } + if (self->free_type_and_src) + { + ft_free((void*)self->type); + ft_free((void*)self->src.file); + ft_free((void*)self->src.func); + } + ft_free((void*)self->message); + $del(&self->sibling); +} + +static fobjStr* +fobjErr_fobjRepr(VSelf) { + Self(fobjErr); + char buffer[256]; + ft_strbuf_t buf = ft_strbuf_init_stack(buffer, 256); + fobj_err_kv_t* kv = self->kv; + + ft_strbuf_catc(&buf, "$err("); + ft_strbuf_catc(&buf, self->type); + ft_strbuf_catc(&buf, ", "); + fobj_format_string(&buf, ft_cstr(self->message), "q"); + for (;kv->key; kv++) { + ft_strbuf_catc(&buf, ", ("); + ft_strbuf_catc(&buf, kv->key); + ft_strbuf_catc(&buf, ", "); + fobj_repr_arg(&buf, kv->val); + ft_strbuf_cat1(&buf, ')'); + } + ft_strbuf_cat1(&buf, ')'); + return fobj_strbuf_steal(&buf); +} + +static void +fobjErr_fobjFormat(VSelf, ft_strbuf_t *buf, const char *fmt) { + Self(fobjErr); + const char* c; + fobj_err_kv_t* kv = self->kv; + + if (fmt == NULL || fmt[0] == 0) { + // fmt = "$T: $M ($F@$f:$l)"; + ft_strbuf_catf(buf, "%s: %s (%s@%s:%d)", + self->type, self->message, + self->src.func, self->src.file, self->src.line); + return; + } + + for (c = fmt; *c; c++) { + if (*c != '$') { + ft_strbuf_cat1(buf, *c); + continue; + } + c++; + switch (*c) { + case 0: c--; break; + case '$': ft_strbuf_cat1(buf, '$'); break; + case 'T': ft_strbuf_catc(buf, self->type); break; + case 'M': ft_strbuf_catc(buf, self->message); break; + case 'F': ft_strbuf_catc(buf, self->src.func); break; + case 'f': ft_strbuf_catc(buf, self->src.file); break; + case 'l': ft_strbuf_catf(buf, "%d", self->src.line); break; + case 'K': + ft_strbuf_cat1(buf, '{'); + for (kv = self->kv; kv->key; kv++) { + if (kv != self->kv) + ft_strbuf_catc(buf, ", "); + fobj_format_string(buf, ft_cstr(kv->key), NULL); + ft_strbuf_catc(buf, ": "); + fobj_format_arg(buf, kv->val, NULL); + } + ft_strbuf_cat1(buf, '}'); + break; + default: + ft_log(FT_ERROR, "Unknown error format character '%c'", *c); + } + } +} + +ft_arg_t +fobj_err_getkv(err_i err, const char *key, ft_arg_t dflt, bool *found) { + fobjErr* oerr = (fobjErr*)(err.self); + fobj_err_kv_t* kv; + if (oerr == NULL) return dflt; + ft_assert(fobj_real_klass_of(oerr) == fobjErr__kh()); \ + kv = oerr->kv; + for (;kv->key != NULL; kv++) { + if (strcmp(kv->key, key) == 0) { + if (found) *found = true; + return kv->val; + } + } + if (found) found = false; + return dflt; +} + +fobjStr* +fobj_printkv(const char *fmt, ft_slc_fokv_t kvs) { + char buf[128]; + ft_strbuf_t out = ft_strbuf_init_stack(buf, 128); + size_t i; + const char* cur; + char* closebrace; + char* formatdelim; + size_t identlen; + size_t formatlen; + char ident[32]; + char format[32]; + + if (strchr(fmt, '{') == NULL || strchr(fmt, '}') == NULL) { + return fobj_str(fmt); + } + + for (cur = fmt; *cur; cur++) { + if (*cur != '{') { + ft_strbuf_cat1(&out, *cur); + continue; + } + if (cur[1] == '{') { + ft_strbuf_cat1(&out, '{'); + cur++; + continue; + } + cur++; + closebrace = strchr(cur, '}'); + ft_assert(closebrace, "format string braces unbalanced"); + formatdelim = memchr(cur, ':', closebrace - cur); + identlen = (formatdelim ?: closebrace) - cur; + ft_assert(identlen <= 31, + "ident is too long in format \"%s\"", fmt); + ft_assert(formatdelim == NULL || closebrace - formatdelim <= 31, + "format is too long in format \"%s\"", fmt); + memcpy(ident, cur, identlen); + ident[identlen] = 0; + formatlen = formatdelim ? closebrace - (formatdelim+1) : 0; + if (formatlen > 0) { + memcpy(format, formatdelim + 1, formatlen); + } + format[formatlen] = 0; + i = ft_search_fokv(kvs.ptr, kvs.len, ident, fobj_fokv_cmpc); + if (ft_unlikely(i >= kvs.len)) { + ft_log(FT_WARNING, "ident '%s' is not found (fmt \"%s\")", ident, fmt); + } else if (kvs.ptr[i].value == NULL) { + ft_strbuf_catc(&out, "NULL"); + } else if (!$ifdef(, fobjFormat, kvs.ptr[i].value, &out, format)) { + /* fallback to repr */ + ft_strbuf_cat(&out, fobj_getstr(fobjRepr(kvs.ptr[i].value))); + } + cur = closebrace; + } + + return fobj_strbuf_steal(&out); +} + +#ifndef WIN32 +static pthread_key_t fobj_AR_current_key = 0; +static void fobj_destroy_thread_AR(void *arg); +#endif + +/* Custom fobjBase implementation */ +fobj_klass_handle_t +fobjBase__kh(void) { + static volatile fobj_klass_handle_t hndl = 0; + fobj_klass_handle_t khandle = hndl; + ssize_t kls_size = sizeof(fobjBase); + if (khandle) return khandle; + { + fobj__method_impl_box_t methods[] = { + fobj__klass_decl_methods(fobjBase, fobj__map_params(kls__fobjBase)) + { 0, NULL } + }; + if (fobj_klass_init_impl(&hndl, kls_size, 0, methods, "fobjBase")) + return hndl; + } + khandle = hndl; + return khandle; +} + +fobj_klass_handle(fobjErr, mth(fobjRepr, _fobjErr_marker_DONT_IMPLEMENT_ME), varsized(kv)); +fobj_klass_handle(fobjStr, mth(fobjDispose), varsized()); +fobj_klass_handle(fobjInt); +fobj_klass_handle(fobjUInt); +fobj_klass_handle(fobjFloat); +fobj_klass_handle(fobjBool); +fobj_klass_handle(fobjTempBuffer); + +void +fobj_init(void) { + ft_assert(fobj_global_state == FOBJ_RT_NOT_INITIALIZED); + +#ifndef WIN32 + { + int res = pthread_key_create(&fobj_AR_current_key, fobj_destroy_thread_AR); + if (res != 0) { + fprintf(stderr, "could not initialize autorelease thread key: %s", + strerror(res)); + abort(); + } + } +#endif + + fobj_global_state = FOBJ_RT_INITIALIZED; + + fobj__consume(fobjDispose__mh()); + fobj_klass_init(fobjBase); + fobj_klass_init(fobjErr); + fobj_klass_init(fobjStr); + fobj_klass_init(fobjInt); + fobj_klass_init(fobjUInt); + fobj_klass_init(fobjFloat); + fobj_klass_init(fobjBool); + + FOBJ_FUNC_ARP(); + + fobjTrue = $alloc(fobjBool, .b = true); + fobjFalse = $alloc(fobjBool, .b = false); + falseRepr = $ref($S("$B(false)")); + trueRepr = $ref($S("$B(true)")); +} + +void +fobj_freeze(void) { + fobj_global_state = FOBJ_RT_FROZEN; +} + +/* Without this function clang could commit initialization of klass without methods */ +volatile uint16_t fobj__FAKE__x; +void +fobj__consume(uint16_t _) { + fobj__FAKE__x += _; +} + +// AUTORELEASE POOL + +static void fobj_autorelease_pool_release_till(fobj_autorelease_pool **from, fobj_autorelease_pool *till); + +#ifndef __TINYC__ +static __thread fobj_autorelease_pool *fobj_AR_current = NULL; +#ifndef WIN32 +static __thread bool fobj_AR_current_set = false; +#endif +static inline fobj_autorelease_pool** +fobj_AR_current_ptr(void) { + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + +#ifndef WIN32 + if (!fobj_AR_current_set) + pthread_setspecific(fobj_AR_current_key, &fobj_AR_current); +#endif + return &fobj_AR_current; +} +#ifndef WIN32 +static void +fobj_destroy_thread_AR(void *arg) { + ft_assert(arg == &fobj_AR_current); + fobj_autorelease_pool_release_till(&fobj_AR_current, NULL); +} +#endif +#else +static fobj_autorelease_pool** +fobj_AR_current_ptr(void) { + fobj_autorelease_pool **current; + + ft_assert(fobj_global_state != FOBJ_RT_NOT_INITIALIZED); + + current = pthread_getspecific(fobj_AR_current_key); + if (current == NULL) { + current = ft_calloc(sizeof(fobj_autorelease_pool*)); + pthread_setspecific(fobj_AR_current_key, current); + } + return current; +} + +static void +fobj_destroy_thread_AR(void *arg) { + fobj_autorelease_pool **current = arg; + + fobj_autorelease_pool_release_till(current, NULL); + ft_free(current); +} +#endif + +fobj__autorelease_pool_ref +fobj_autorelease_pool_init(fobj_autorelease_pool *pool) { + fobj_autorelease_pool **parent = fobj_AR_current_ptr(); + pool->ref.parent = *parent; + pool->ref.root = parent; + pool->last = &pool->first; + pool->first.prev = NULL; + pool->first.cnt = 0; + *parent = pool; + return pool->ref; +} + +void +fobj_autorelease_pool_release(fobj_autorelease_pool *pool) { + fobj_autorelease_pool_release_till(pool->ref.root, pool->ref.parent); +} + +static void +fobj_autorelease_pool_release_till(fobj_autorelease_pool **from, fobj_autorelease_pool *till) { + fobj_autorelease_pool *current; + fobj_autorelease_chunk *chunk; + + while (*from != till) { + current = *from; + while (current->last != ¤t->first || current->last->cnt != 0) { + chunk = current->last; + if (chunk->cnt == 0) { + current->last = chunk->prev; + ft_free(chunk); + continue; + } + fobj_del(&chunk->refs[--chunk->cnt]); + } + ft_assert(*from == current); + *from = (*from)->ref.parent; + } +} + +static fobj_t +fobj_autorelease(fobj_t obj, fobj_autorelease_pool *pool) { + fobj_autorelease_chunk *chunk, *new_chunk; + + if (obj == NULL) + return NULL; + + ft_assert(pool != NULL); + + chunk = pool->last; + if (chunk->cnt == FOBJ_AR_CHUNK_SIZE) { + new_chunk = ft_calloc(sizeof(fobj_autorelease_chunk)); + new_chunk->prev = chunk; + pool->last = chunk = new_chunk; + } + chunk->refs[chunk->cnt] = obj; + chunk->cnt++; + return obj; +} + +fobj_t +fobj_store_to_parent_pool(fobj_t obj, fobj_autorelease_pool *child_pool_or_null) { + if (obj == NULL) + return NULL; + return fobj_autorelease(obj, + (child_pool_or_null ?: *fobj_AR_current_ptr())->ref.parent); +} + +ft_register_source(); diff --git a/src/fu_util/impl/fo_impl.h b/src/fu_util/impl/fo_impl.h new file mode 100644 index 000000000..76f353a91 --- /dev/null +++ b/src/fu_util/impl/fo_impl.h @@ -0,0 +1,639 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FOBJ_OBJ_PRIV_H +#define FOBJ_OBJ_PRIV_H + +#define Self_impl(Klass) \ + Klass * self ft_unused = Vself; fobj_klass_handle_t fobj__klassh ft_unused = fobj__nm_khandle(Klass)() + +typedef uint16_t fobj_klass_handle_t; +typedef uint16_t fobj_method_handle_t; + +#define FOBJ_ARGS_COMPLEX + +typedef struct fobj__missing_argument_detector { + char is_set; +} fobj__missing_argument_detector; +#define fobj__dumb_arg ((fobj__missing_argument_detector){1}) +#define fobj__check_arg(name) fobj__nm_given(name).is_set + +typedef struct { + fobj_method_handle_t meth; + void* impl; +} fobj__method_impl_box_t; + +/* params coversions */ + +/* map params to tuples */ +#define fobj__map_params(...) \ + fm_eval_foreach_comma(fobj__map_param, __VA_ARGS__) +#define fobj__map_params_(...) \ + fm_foreach_comma(fobj__map_param, __VA_ARGS__) +#define fobj__map_param(param) \ + fm_cat(fobj__map_param_, param) +#define fobj__map_param_varsized(...) (varsized, __VA_ARGS__) +#define fobj__map_param_mth(...) (mth, __VA_ARGS__) +#define fobj__map_param_iface(...) (iface, __VA_ARGS__) +#define fobj__map_param_inherits(parent) (inherits, parent) + +/* filter and flatten methods */ +#define fobj__flat_methods(...) \ + fm_tail(fm_eval_tuples(fobj__fetch_methods, __VA_ARGS__)) +#define fobj__fetch_methods(tag, ...) fobj__fetch_methods_##tag(__VA_ARGS__) +#define fobj__fetch_methods_mth(...) , __VA_ARGS__ +#define fobj__fetch_methods_iface(...) +#define fobj__fetch_methods_inherits(...) +#define fobj__fetch_methods_varsized(...) + +/* filter and flatten interfaces */ +#define fobj__flat_ifaces(...) \ + fm_tail(fm_eval_tuples(fobj__fetch_ifaces, __VA_ARGS__)) +#define fobj__fetch_ifaces(tag, ...) \ + fobj__fetch_ifaces_##tag(__VA_ARGS__) +#define fobj__fetch_ifaces_mth(...) +#define fobj__fetch_ifaces_iface(...) , __VA_ARGS__ +#define fobj__fetch_ifaces_inherits(...) +#define fobj__fetch_ifaces_varsized(...) + +/* Standard naming */ + +#define fobj__nm_mth(meth) mth__##meth +#define fobj__nm_mthdflt(meth) mth__##meth##__optional +#define fobj__nm_kls(klass) kls__##klass +#define fobj__nm_iface(iface) iface__##iface +#define fobj__nm_mhandle(meth) meth##__mh +#define fobj__nm_do(meth) meth##__do +#define fobj__nm_params_t(meth) meth##__params_t +#define fobj__nm_impl_t(meth) meth##__impl +#define fobj__nm_cb(meth) meth##__fetch_cb +#define fobj__nm_cb_t(meth) meth##__cb_t +#define fobj__nm_register(meth) meth##__register +#define fobj__nm_wrap_decl(meth) meth##__wrap_decl +#define fobj__nm_meth_i(meth) meth##_i +#define fobj__nm_has(m) has_##m +#define fobj__nm_bind(m_or_i) bind_##m_or_i +#define fobj__nm_bindref(m_or_i) bindref_##m_or_i +#define fobj__nm_implements(m_or_i) implements_##m_or_i +#define fobj__nm_khandle(klass) klass##__kh +#define fobj__nm_klass_meth(klass, meth) klass##_##meth +#define fobj__nm_iface_i(iface) iface##_i +#define fobj__nm_given(param) param##__given +#define fobj__nm_kvalidate(m_or_i) fobj__klass_validate_##m_or_i + +/* Method definition */ +#define fobj__predefine_method(method) \ + ft_static ft_gcc_const fobj_method_handle_t fobj__nm_mhandle(method)(void) + +#define fobj__define_method(meth) \ + fobj__method_declare_i(meth, fobj__nm_mth(meth)) \ + fobj__iface_declare_i(meth, fobj__map_params(mth(meth), iface__fobj)) \ + fm__dumb_require_semicolon +#define fobj__define_base_method(meth) \ + fobj__method_declare_i(meth, fobj__nm_mth(meth)) \ + fobj__iface_declare_i(meth, iface__fobj) \ + fm__dumb_require_semicolon +#define fobj__method_declare_i(meth, ...) \ + fobj__method_declare(meth, __VA_ARGS__) +#define fobj__method_declare(meth, res, ...) \ + fobj__method_declare_impl(meth, \ + fobj__nm_mhandle(meth), \ + fobj__nm_params_t(meth), \ + fobj__nm_do(meth), \ + fobj__nm_impl_t(meth), \ + fobj__nm_cb(meth), \ + fobj__nm_cb_t(meth), \ + fobj__nm_register(meth), \ + fobj__nm_wrap_decl(meth), \ + fobj__nm_meth_i(meth), \ + fobj__nm_bind(meth), \ + fobj__nm_bindref(meth), \ + fobj__nm_implements(meth), \ + fobj__nm_kvalidate(meth), \ + fm_va_comma_fun(__VA_ARGS__), \ + res, __VA_ARGS__) + +#define fobj__special_method(meth) \ + fobj__special_method_declare_i(meth, fobj__nm_mth(meth)) \ + fm__dumb_require_semicolon +#define fobj__special_method_declare_i(meth, ...) \ + fobj__special_method_declare(meth, __VA_ARGS__) +#define fobj__special_method_declare(meth, res, ...) \ + fobj__method_common(meth, \ + fobj__nm_mhandle(meth), \ + fobj__nm_impl_t(meth), \ + fobj__nm_register(meth), \ + fobj__nm_wrap_decl(meth), \ + fm_va_comma_fun(__VA_ARGS__), \ + res, __VA_ARGS__) + +#define fobj__method_declare_impl(meth, handle, \ + params_t, \ + meth_do, \ + impl_meth_t, \ + cb_meth, cb_meth_t, \ + register_meth, wrap_decl, \ + meth_i, bind_meth, bindref_meth, \ + implements_meth, \ + kvalidate, comma, res, ...) \ + \ + fobj__method_common(meth, handle, impl_meth_t, register_meth, \ + wrap_decl, comma, res, __VA_ARGS__) \ + \ + typedef struct params_t { \ + fobj__mapArgs_toFields(__VA_ARGS__) \ + } params_t; \ + \ + typedef struct cb_meth_t { \ + fobj_t self; \ + impl_meth_t impl; \ + } cb_meth_t; \ + \ + ft_inline ft_always_inline cb_meth_t \ + cb_meth(fobj_t self, fobj_klass_handle_t parent, bool validate) { \ + fobj__method_callback_t fnd = {NULL, NULL}; \ + fnd = fobj_method_search(self, handle(), parent, validate); \ + return (cb_meth_t){fnd.self, fnd.impl}; \ + } \ + \ + ft_static res \ + meth_do(fobj_t self, fobj_klass_handle_t parent comma() fobj__mapArgs_toArgs(__VA_ARGS__)) { \ + cb_meth_t cb = cb_meth(self, parent, true); \ + return cb.impl(cb.self comma() fobj__mapArgs_toNames(__VA_ARGS__)); \ + } \ + \ + ft_inline ft_always_inline res \ + meth(fobj_t self comma() fobj__mapArgs_toArgs(__VA_ARGS__)) { \ + return meth_do(self, fobj_self_klass comma() fobj__mapArgs_toNames(__VA_ARGS__)); \ + } + +#define fobj__method_common(meth, handle, impl_meth_t, register_meth, \ + wrap_decl, comma, res, ...) \ + \ + ft_inline ft_gcc_const fobj_method_handle_t handle(void) { \ + static volatile fobj_method_handle_t hndl = 0; \ + fobj_method_handle_t h = hndl; \ + if (ft_likely(h)) return h; \ + fobj_method_init_impl(&hndl, fm_str(meth)); \ + return hndl; \ + } \ + \ + typedef res (* impl_meth_t)(fobj_t self comma() fobj__mapArgs_toArgs(__VA_ARGS__)); \ + \ + ft_inline void \ + register_meth(fobj_klass_handle_t klass, impl_meth_t cb) { \ + fobj_method_register_impl(klass, handle(), (void *)cb); \ + } \ + \ + ft_inline fobj__method_impl_box_t \ + wrap_decl(impl_meth_t cb) { \ + return (fobj__method_impl_box_t) { handle(), cb }; \ + } + +#define fobj__mapArgs_toArgs(...) \ + fm_eval_tuples_comma(fobj__mapArgs_toArgs_do, __VA_ARGS__) +#define fobj__mapArgs_toArgs_do(x, y, ...) x y + +#define fobj__mapArgs_toFields(...) \ + fm_eval_tuples(fobj__mapArgs_toFields_do, __VA_ARGS__) +#define fobj__mapArgs_toFields_do(x, y, ...) \ + x y; \ + fobj__missing_argument_detector fobj__nm_given(y); + +#define fobj__mapArgs_toNames(...) \ + fm_eval_tuples_comma(fobj__mapArgs_toNames_do, __VA_ARGS__) +#define fobj__mapArgs_toNames_do(x, y, ...) y + +#define fobj__mapArgs_toNamedParams(params, ...) \ + fm_eval_tuples_arg_comma(fobj__mapArgs_toNamedParams_do, params, __VA_ARGS__) +#define fobj__mapArgs_toNamedParams_do(params, x, y, ...) params.y + +#define fobj__params_defaultsEach(params, ...) \ + fm_eval_tuples_arg(fobj__params_defaultsEach_do, params, __VA_ARGS__) +#define fobj__params_defaultsEach_do(params, x, y, ...) \ + if (!fobj__check_arg(params.y)) { \ + fm_if(fm_is_empty(__VA_ARGS__), \ + fobj__validate_arg(__FILE__, __LINE__, #y), \ + params.y = __VA_ARGS__); \ + } + +/* Klass declarations */ + +#define fobj__klass_declare(klass) \ + extern fobj_klass_handle_t fobj__nm_khandle(klass)(void) ft_gcc_const; \ + fm__dumb_require_semicolon + +#define fobj__klass_handle(klass, ...) \ + fobj__klass_handle_i(klass, \ + fobj__map_params(fobj__nm_kls(klass) fm_when_isnt_empty(__VA_ARGS__)(fm__comma __VA_ARGS__))) +#define fobj__klass_handle_i(klass, ...) \ + fobj__klass_handle_impl(klass, __VA_ARGS__) +#define fobj__klass_handle_impl(klass, ...) \ + fobj_klass_handle_t fobj__nm_khandle(klass) (void) { \ + static volatile fobj_klass_handle_t hndl = 0; \ + fobj_klass_handle_t khandle = hndl; \ + fobj_klass_handle_t kparent = fobjBase__kh(); \ + ssize_t kls_size = sizeof(klass); \ + if (khandle) return khandle; \ + fm_eval_tuples_arg(fobj__klass_detect_size, klass, __VA_ARGS__) \ + { \ + fobj__method_impl_box_t methods[] = { \ + fobj__klass_decl_methods(klass, __VA_ARGS__) \ + { 0, NULL } \ + }; \ + if (fobj_klass_init_impl(&hndl, kls_size, kparent, methods, fm_str(klass))) \ + return hndl; \ + } \ + khandle = hndl; \ + fobj__klass_check_ifaces(klass, __VA_ARGS__) \ + return khandle; \ + } + +#define fobj__klass_detect_size(klass, tag, ...) \ + fobj__klass_detect_size_##tag (klass, __VA_ARGS__) +#define fobj__klass_detect_size_inherits(klass, parent) \ + kparent = fobj__nm_khandle(parent)(); +#define fobj__klass_detect_size_varsized(klass, ...) \ + fm_iif(fm_va_01(__VA_ARGS__)) \ + ( kls_size = -1-offsetof(klass,fm_head(__VA_ARGS__)); ) \ + ( kls_size = -1-sizeof(klass); ) +#define fobj__klass_detect_size_mth(...) +#define fobj__klass_detect_size_iface(...) + +#define fobj__klass_decl_methods(klass, ...) \ + fm_eval_foreach_arg(fobj__klass_decl_method, klass, fobj__flat_methods(__VA_ARGS__)) +#define fobj__klass_decl_method(klass, meth) \ + fobj__nm_wrap_decl(meth)(fobj__nm_klass_meth(klass, meth)), + +#define fobj__klass_check_ifaces(klass, ...) \ + fm_eval_foreach_arg(fobj__klass_check_iface, klass, fobj__flat_ifaces(__VA_ARGS__)) +#define fobj__klass_check_iface(klass, iface) \ + fobj__nm_kvalidate(iface)(khandle); + +#define fobj__method_init(meth) \ + fobj__consume(fobj__nm_mhandle(meth)()) +#define fobj__klass_init(klass) \ + fobj__consume(fobj__nm_khandle(klass)()) + +/* add methods after class declaration */ + +#define fobj__add_methods(klass, ...) do { \ + fobj_klass_handle_t khandle = fobj__nm_khandle(klass)(); \ + fm_eval_foreach_arg(fobj__add_methods_do, klass, __VA_ARGS__) \ +} while (0) +#define fobj__add_methods_do(klass, meth) \ + fobj__nm_register(meth)(khandle, fobj__nm_klass_meth(klass, meth)); + +/* Instance creation */ +#define fobj__alloc(klass, ...) \ + fm_cat(fobj__alloc_, fm_va_01(__VA_ARGS__))(klass, fobj__nm_khandle(klass), -1, __VA_ARGS__) +#define fobj__alloc_sized(klass, size, ...) \ + fm_cat(fobj__alloc_, fm_va_01(__VA_ARGS__))(\ + klass, fobj__nm_khandle(klass), (size), __VA_ARGS__) +#define fobj__alloc_0(klass, khandle, size, ...) \ + ((klass *)fobj__allocate(khandle(), NULL, size)) +#define fobj__alloc_1(klass, khandle, size, ...) \ + ((klass *)fobj__allocate(khandle(), &(klass){__VA_ARGS__}, size)) + +/* Interface declaration */ + +#define fobj__iface_declare(iface) \ + fobj__iface_declare_i(iface, fobj__map_params(fobj__nm_iface(iface), iface__fobj)) \ + fm__dumb_require_semicolon + +#define fobj__iface_declare_i(iface, ...) \ + fobj__iface_declare_impl(iface, \ + fobj__nm_iface_i(iface), fobj__nm_bind(iface), \ + fobj__nm_bindref(iface), fobj__nm_implements(iface), \ + fobj__nm_kvalidate(iface), (fobj__flat_methods(__VA_ARGS__))) + +#define fobj__iface_declare_impl(iface, iface_i, \ + bind_iface, bindref_iface, implements_iface, \ + kvalidate, methods) \ + fobj__mapMethods_toHandlers methods \ + typedef union iface_i { \ + fobj_t self; \ + fobj__mapMethods_toFields methods \ + } iface_i; \ + \ + ft_inline iface_i \ + bind_iface(fobj_t self) { \ + iface_i _iface = (iface_i){ .self = self }; \ + fobj__mapMethods_toSetters methods \ + return _iface; \ + } \ + \ + ft_inline bool \ + implements_iface(fobj_t self, iface_i *ifacep) { \ + iface_i _iface = (iface_i){ .self = self }; \ + bool all_ok = true; \ + fobj__mapMethods_toIfSetters methods \ + if (ifacep != NULL) \ + *ifacep = all_ok ? _iface : (iface_i){NULL}; \ + return all_ok; \ + } \ + \ + ft_inline iface_i \ + bindref_iface(fobj_t self) { \ + iface_i _iface = bind_iface(self); \ + fobj_ref(_iface.self); \ + return _iface; \ + } \ + \ + ft_inline void \ + kvalidate(fobj_klass_handle_t khandle) { \ + fobj__kvalidateMethods methods \ + } + +#define fobj__mapMethods_toHandlers(...) \ + fm_eval_foreach(fobj__mapMethods_toHandlers_do, __VA_ARGS__) +#define fobj__mapMethods_toHandlers_do(m) \ + fobj__predefine_method(m); + +#define fobj__mapMethods_toFields(...) \ + fm_eval_foreach(fobj__mapMethods_toFields_do, __VA_ARGS__) +#define fobj__mapMethods_toFields_do(m) \ + uintptr_t fobj__nm_has(m); + +#define fobj__mapMethods_toSetters(...) \ + fm_eval_foreach(fobj__mapMethods_toSetters_do, __VA_ARGS__) +#define fobj__mapMethods_toSetters_do(meth) \ + ft_assert(fobj_method_implements(self, fobj__nm_mhandle(meth)())); + +#define fobj__mapMethods_toIfSetters(...) \ + fm_eval_foreach(fobj__mapMethods_toIfSetters_do, __VA_ARGS__) +#define fobj__mapMethods_toIfSetters_do(meth) \ + all_ok &= fobj_method_implements(self, fobj__nm_mhandle(meth)()); + +#define fobj__kvalidateMethods(...) \ + fm_eval_foreach(fobj__kvalidateMethods_do, __VA_ARGS__) +#define fobj__kvalidateMethods_do(meth) \ + ft_assert(fobj_klass_method_search(khandle, fobj__nm_mhandle(meth)()) != NULL); + +#ifndef NDEBUG +#define fobj_reduce(newifacetype, oldiface) ({ \ + if (0) { \ + __typeof(oldiface) _old_iface_ ft_unused = {NULL}; \ + fobj__nm_iface_i(newifacetype) _new_iface_ ft_unused = {NULL}; \ + fobj__mapMethods_toCopyChecks(newifacetype) \ + } \ + ((fobj__nm_iface_i(newifacetype)){.self = (oldiface).self}); \ +}) +#else +#define fobj_reduce(newifacetype, oldiface) \ + ((fobj__nm_iface_i(newifacetype)){.self = (oldiface).self}) +#endif + +#define fobj__mapMethods_toCopyChecks(iface) \ + fobj__mapMethods_toCopyChecks_i( \ + fm_iif(fobj__macroIsIface(iface)) \ + (fobj__map_params(fobj__nm_iface(iface))) \ + ((mth, iface))) +#define fobj__mapMethods_toCopyChecks_i(...) \ + fm_eval_foreach(fobj__mapMethods_toCopyChecks_do, fobj__flat_methods(__VA_ARGS__)) +#define fobj__mapMethods_toCopyChecks_do(meth) \ + _new_iface_.fobj__nm_has(meth) = _old_iface_.fobj__nm_has(meth); + +#define fobj__macroIsIface(iface) \ + fm_is_empty(fobj__macroIsIface_i(fobj__nm_iface(iface))) +#define fobj__macroIsIface_i(...) \ + fm_eval_foreach(fobj__macroIsIface_do, __VA_ARGS__) +#define fobj__macroIsIface_do(x) \ + fobj__macroIsIface_##x +#define fobj__macroIsIface_mth(...) + +/* Method invocation */ + +#define fobj_call(meth, self, ...) fobj__call_1(meth, self, fobj_self_klass, fm_uniq(params), __VA_ARGS__) +#define fobj__call_1(meth, self, parent, params, ...) ({\ + ft_unused fobj__nm_params_t(meth) params = fobj_pass_params(meth, __VA_ARGS__); \ + fobj__call_2(meth, (self), parent, params, fobj__nm_mth(meth)) \ + }) +#define fobj__call_2(meth, self, parent, params, ...) \ + fobj__call_3(meth, self, parent, params, __VA_ARGS__) +#define fobj__call_3(meth, self, parent, params, res, ...) \ + fobj__params_defaultsEach(params, __VA_ARGS__); \ + fobj__nm_do(meth)(self, parent fm_va_comma(__VA_ARGS__) fobj__mapArgs_toNamedParams(params, __VA_ARGS__)); + +#define fobj_call_super(meth, _klassh, self, ...) \ + fobj__call_1(meth, self, _klassh, fm_uniq(params), __VA_ARGS__) + +#define fobj_iface_call(meth, iface, ...) \ + fobj_call(meth, (fobj_t)(iface).fobj__nm_has(meth), __VA_ARGS__) + +#define fobj_cb_fastcall(cb, ...) \ + (cb).impl((cb).self, __VA_ARGS__) + +#define fobj__implements(iface, self, ...) \ + (fobj__nm_implements(iface)(self, fm_if(fm_no_va(__VA_ARGS__), NULL, __VA_ARGS__))) + +#define fobj_iface_filled(meth, iface) \ + (fobj__nm_implements(meth)((fobj_t)(iface).fobj__nm_has(meth), NULL)) + +#define fobj_ifdef(assignment, meth, self, ...) \ + fobj__ifdef_impl(assignment, meth, (self), \ + fm_uniq(cb), fm_uniq(_self), fobj__nm_cb(meth), \ + fobj__nm_cb_t(meth), __VA_ARGS__) +#define fobj__ifdef_impl(assignment, meth, self_, cb, self, \ + cb_meth, cb_meth_t, ...) ({ \ + fobj_t self = (self_); \ + cb_meth_t cb = cb_meth(self, fobj_self_klass, false); \ + if (cb.impl != NULL) { \ + assignment fobj_call(meth, self, __VA_ARGS__); \ + } \ + cb.impl != NULL; \ + }) + +/* Named params passing hazzles with optional and defaults */ + +#define fobj_pass_params(meth, ...) \ + ((fobj__nm_params_t(meth)){fm_eval_foreach_comma(fobj__pass_params_each, __VA_ARGS__)}) + +#define fobj__pass_params_each(param) \ + param, fobj__dumb_arg + +#define fobj_bind(iface, obj) fobj__nm_bind(iface)(obj) + +/* Declarations "private" implementation functions */ +extern bool fobj_method_init_impl(volatile fobj_method_handle_t *meth, + const char *name); +extern void fobj_method_register_impl(fobj_klass_handle_t klass, + fobj_method_handle_t meth, + void* impl); +extern bool fobj_klass_init_impl(volatile fobj_klass_handle_t *klass, + ssize_t size, + fobj_klass_handle_t parent, + fobj__method_impl_box_t *methods, + const char *name); +extern void* fobj__allocate(fobj_klass_handle_t klass, + void *init, + ssize_t size); + +/* helper function to consume value to disable compiler optimizations */ +extern void fobj__consume(uint16_t); + +typedef struct fobj__method_callback { + fobj_t self; + void* impl; +} fobj__method_callback_t; +extern fobj__method_callback_t fobj_method_search(fobj_t self, + fobj_method_handle_t meth, + fobj_klass_handle_t for_child_take_parent, + bool validate); + +extern bool fobj_method_implements(fobj_t self, + fobj_method_handle_t meth); + +extern void* fobj_klass_method_search(fobj_klass_handle_t klass, + fobj_method_handle_t meth); + +extern _Noreturn +#if __OPTIMIZE__ || defined(__clang__) +__attribute__((error("missing argument"))) +#endif +void fobj__validate_arg(const char* file, int line, const char *arg); + +/* Variable set helpers */ + +#ifndef NDEBUG +#define fobj__set_impl(ptr, obj) do { \ + __typeof(&(**ptr)) fm_uniq(_validate_ptrptr_) ft_unused = NULL; \ + fobj_set((void**)(ptr), (obj)); \ +} while(0) +#define fobj__swap_impl(ptr, obj) ({ \ + __typeof(&(**ptr)) fm_uniq(_validate_ptrptr_) ft_unused = NULL; \ + fobj_swap((void**)(ptr), (obj)); \ +}) +#define fobj__del_impl(ptr) do { \ + __typeof(&(**ptr)) fm_uniq(_validate_ptrptr_) ft_unused = NULL; \ + fobj_del((void**)(ptr)); \ +} while (0) +#else +#define fobj__set_impl(ptr, obj) fobj_set((void**)(ptr), (obj)) +#define fobj__swap_impl(ptr, obj) fobj_swap((void**)(ptr), (obj)) +#define fobj__del_impl(ptr) fobj_del((void**)(ptr)) +#endif + +#define fobj__iref(iface) ({__typeof(iface) t = (iface); t.self=fobj_ref(t.self); t;}) +#define fobj__iunref(iface) ({__typeof(iface) t = (iface); t.self=fobj_unref(t.self); t;}) +#ifndef NDEBUG +#define fobj__iset(ptr, iface) do { \ + __typeof(*(ptr)) fm_uniq(_validate_ptr_) ft_unused = (__typeof(iface)){}; \ + fobj_set(&(ptr)->self, (iface).self); \ +} while (0) +#define fobj__iswap(ptr, iface) ({ \ + __typeof(*(ptr)) fm_uniq(_validate_ptr_) ft_unused = (__typeof(iface)){}; \ + (__typeof(iface)){.self=fobj_swap(&(ptr)->self, (iface).self)}; \ +}) +#else +#define fobj__iset(ptr, iface) \ + fobj_set(&(ptr)->self, (iface).self) +#define fobj__iswap(ptr, iface) \ + ((__typeof(iface)){.self=fobj_swap(&(ptr)->self, (iface).self)}) +#endif +#define fobj__idel(iface) fobj_del((void*)&(iface)->self) + +#define fobj__isave(iface) ({__typeof(iface) t=(iface); $save(t.self); t;}) +#define fobj__iresult(iface) ({__typeof(iface) t=(iface); $result(t.self); t;}) +#define fobj__ireturn(iface) return $iresult(iface) + +/* Autorelease pool handling */ + +#define FOBJ_AR_CHUNK_SIZE 14 +typedef struct fobj_autorelease_chunk fobj_autorelease_chunk; +struct fobj_autorelease_chunk { + fobj_autorelease_chunk *prev; + uint32_t cnt; + fobj_t refs[FOBJ_AR_CHUNK_SIZE]; +}; +typedef struct fobj__autorelease_pool_ref fobj__autorelease_pool_ref; +typedef struct fobj_autorelease_pool fobj_autorelease_pool; +struct fobj__autorelease_pool_ref { + fobj_autorelease_pool *parent; + fobj_autorelease_pool **root; +}; +struct fobj_autorelease_pool { + struct fobj__autorelease_pool_ref ref; + fobj_autorelease_chunk *last; + fobj_autorelease_chunk first; +}; + +extern fobj__autorelease_pool_ref fobj_autorelease_pool_init(fobj_autorelease_pool *pool); +extern void fobj_autorelease_pool_release(fobj_autorelease_pool *pool); +extern fobj_t fobj_store_to_parent_pool(fobj_t obj, + fobj_autorelease_pool *child_pool_or_null); + +#define FOBJ_ARP_POOL(name) \ + fobj_autorelease_pool __attribute__((cleanup(fobj_autorelease_pool_release))) \ + name = {fobj_autorelease_pool_init(&name), &name.first} + + +/******************************** + * ERROR + */ +typedef struct fobj_err_kv { + const char *key; + ft_arg_t val; +} fobj_err_kv_t; + +#define fobj__error_kind(err) \ + ft_inline const char* fobj_error_kind_##err(void) { return #err; } + +#define fobj__error_flag_key(key) \ + ft_inline fobj_err_kv_t fobj__err_mkkv_##key(void) { \ + return (fobj_err_kv_t){#key, ft_mka_z()}; \ + } \ + ft_inline bool fobj__err_getkv_##key(err_i err, bool *found) { \ + bool fnd; \ + fobj_err_getkv(err, #key, ft_mka_z(), &fnd)); \ + if (found) *found = fnd; \ + return fnd; \ + } + +#define fobj__error_int_key(key) \ + ft_inline fobj_err_kv_t fobj__err_mkkv_##key(int64_t v) { \ + return (fobj_err_kv_t){#key, ft_mka_i(v)}; \ + } \ + ft_inline int64_t fobj__err_getkv_##key(err_i err, bool *found) { \ + return ft_arg_i(fobj_err_getkv(err, #key, ft_mka_i(0), found)); \ + } + +#define fobj__error_uint_key(key) \ + ft_inline fobj_err_kv_t fobj__err_mkkv_##key(uint64_t v) { \ + return (fobj_err_kv_t){#key, ft_mka_u(v)}; \ + } \ + ft_inline uint64_t fobj__err_getkv_##key(err_i err, bool *found) { \ + return ft_arg_u(fobj_err_getkv(err, #key, ft_mka_u(0), found)); \ + } + +#define fobj__error_cstr_key(key) \ + ft_inline fobj_err_kv_t fobj__err_mkkv_##key(const char* v) { \ + return (fobj_err_kv_t){#key, ft_mka_s((char*)v)}; \ + } \ + ft_inline const char* fobj__err_getkv_##key(err_i err, bool *found) { \ + return ft_arg_s(fobj_err_getkv(err, #key, ft_mka_s(NULL), found)); \ + } + +#define fobj__error_float_key(key) \ + ft_inline fobj_err_kv_t fobj__err_mkkv_##key(double v) { \ + return (fobj_err_kv_t){#key, ft_mka_f(v)}; \ + } \ + ft_inline double fobj__err_getkv_##key(err_i err, bool *found) { \ + return ft_arg_f(fobj_err_getkv(err, #key, ft_mka_f(0), found)); \ + } + +#define fobj__error_bool_key(key) \ + ft_inline fobj_err_kv_t fobj__err_mkkv_##key(bool v) { \ + return (fobj_err_kv_t){#key, ft_mka_b(v)}; \ + } \ + ft_inline bool fobj__err_getkv_##key(err_i err, bool *found) { \ + return ft_arg_b(fobj_err_getkv(err, #key, ft_mka_b(false), found)); \ + } + +#define fobj__error_object_key(key) \ + ft_inline fobj_err_kv_t fobj__err_mkkv_##key(fobj_t v) { \ + return (fobj_err_kv_t){#key, ft_mka_o(v)}; \ + } \ + ft_inline fobj_t fobj__err_getkv_##key(err_i err, bool *found) { \ + return ft_arg_o(fobj_err_getkv(err, #key, ft_mka_o(NULL), found)); \ + } + +#endif diff --git a/src/fu_util/impl/fo_impl2.h b/src/fu_util/impl/fo_impl2.h new file mode 100644 index 000000000..16e0a1921 --- /dev/null +++ b/src/fu_util/impl/fo_impl2.h @@ -0,0 +1,273 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FOBJ_OBJ_PRIV2_H +#define FOBJ_OBJ_PRIV2_H + +enum fobjStrType { + FOBJ_STR_SMALL = 1, + FOBJ_STR_UNOWNED, + FOBJ_STR_PTR, +}; +#define FOBJ_STR_SMALL_SIZE ((1<<14)-1) +#define FOBJ_STR_FREE_SPACE (sizeof(fobjStr) - offsetof(fobjStr, small.buf)) + +union fobjStr { + struct { + uint16_t type:2; + }; + struct { + uint16_t type:2; + uint16_t len:14; + char buf[]; + } small; + struct { + uint16_t type:2; + uint32_t len; + char* ptr; + } ptr; +}; + +ft_inline fobjStr* +fobj_str(const char* s) { + return fobj_newstr(ft_cstr(s), FOBJ_STR_COPY); +} + +ft_inline fobjStr* +fobj_str_const(const char* s) { + return fobj_newstr(ft_cstr(s), FOBJ_STR_CONST); +} + +ft_inline fobjStr* +fobj_strbuf_steal(ft_strbuf_t *buf) { + if (buf->len < FOBJ_STR_FREE_SPACE && !buf->alloced) + return fobj_newstr(ft_strbuf_ref(buf), FOBJ_STR_COPY); + return fobj_newstr(ft_strbuf_steal(buf), FOBJ_STR_GIFTED); +} + +ft_inline ft_str_t +fobj_getstr(fobjStr *str) { + switch (str->type) { + case FOBJ_STR_SMALL: + return ft_str(str->small.buf, str->small.len); + case FOBJ_STR_PTR: + case FOBJ_STR_UNOWNED: + return ft_str(str->ptr.ptr, str->ptr.len); + default: + ft_log(FT_FATAL, "Unknown fobj_str type %d", str->type); + return ft_str(NULL, 0); + } +} + +ft_inline fobjStr* +fobj_strcatc(fobjStr *ostr, const char *str) { + return fobj_strcat(ostr, ft_cstr(str)); +} + +ft_inline fobjStr* +fobj_strcatc2(fobjStr *ostr, const char *str1, const char *str2) { + /* a bit lazy to do it in a fast way */ + return fobj_strcat2(ostr, ft_cstr(str1), ft_cstr(str2)); +} + +ft_inline fobjStr* +fobj_stradd(fobjStr *ostr, fobjStr *other) { + return fobj_strcat(ostr, fobj_getstr(other)); +} + +ft_inline bool +fobj_streq(fobjStr* self, fobjStr *oth) { + return ft_streq(fobj_getstr(self), fobj_getstr(oth)); +} + +ft_inline FT_CMP_RES +fobj_strcmp(fobjStr* self, fobjStr *oth) { + return ft_strcmp(fobj_getstr(self), fobj_getstr(oth)); +} + +ft_inline bool +fobj_streq_str(fobjStr* self, ft_str_t oth) { + return ft_streq(fobj_getstr(self), oth); +} + +ft_inline FT_CMP_RES +fobj_strcmp_str(fobjStr* self, ft_str_t oth) { + return ft_strcmp(fobj_getstr(self), oth); +} + +ft_inline bool +fobj_streq_c(fobjStr* self, const char *oth) { + return ft_streqc(fobj_getstr(self), oth); +} + +ft_inline FT_CMP_RES +fobj_strcmp_c(fobjStr* self, const char *oth) { + return ft_strcmpc(fobj_getstr(self), oth); +} + +ft_inline fobjInt* +fobj_int(int64_t i) { + return $alloc(fobjInt, .i = i); +} + +ft_inline fobjUInt* +fobj_uint(uint64_t u) { + return $alloc(fobjUInt, .u = u); +} + +ft_inline fobjFloat* +fobj_float(double f) { + return $alloc(fobjFloat, .f = f); +} + +typedef struct fobjErr fobjErr; +struct fobjErr { + const char* type; + const char* message; + ft_source_position_t src; + fobjErr* sibling; /* sibling error */ + bool free_type_and_src; + fobj_err_kv_t kv[]; +}; + +#define fobj_make_err(type, ...) \ + fm_cat(fobj_make_err_, fm_va_01n(__VA_ARGS__))(type, __VA_ARGS__) +#define fobj_make_err_0(type, ...) ({ \ + fobj__make_err(fobj_error_kind_##type(), \ + ft__srcpos(), "Unspecified Error", NULL, 0); \ +}) +#define fobj_make_err_1(type, msg) ({ \ + fobj__make_err(fobj_error_kind_##type(), \ + ft__srcpos(), msg, NULL, 0); \ +}) +#define fobj_make_err_n(type, msg, ...) ({ \ + fobj_err_kv_t kvs[] = { \ + fobj__err_transform_kv(__VA_ARGS__) \ + }; \ + fobj__make_err(fobj_error_kind_##type(), \ + ft__srcpos(), msg, \ + kvs, ft_arrsz(kvs)); \ +}) + +#define fobj_make_syserr(erno_, ...) \ + fm_cat(fobj_make_syserr_, fm_va_01(__VA_ARGS__))((erno_), fm_uniq(erno), __VA_ARGS__) +#define fobj_make_syserr_0(erno_, erno, ...) ({ \ + int erno = erno_; \ + fobj_err_kv_t kvs[] = { \ + {"errNo", ft_mka_i(erno)}, \ + {"errNoStr", ft_mka_s((char*)ft_strerror(erno))}, \ + }; \ + fobj__make_err(fobj_error_kind_SysErr(), \ + ft__srcpos(), "System Error: {errNoStr}", \ + kvs, ft_arrsz(kvs));\ +}) +#define fobj_make_syserr_1(erno_, erno, msg, ...) ({ \ + int erno = erno_; \ + fobj_err_kv_t kvs[] = { \ + {"errNo", ft_mka_i(erno)}, \ + {"errNoStr", ft_mka_s((char*)ft_strerror(erno))}, \ + {"__msgSuffix", ft_mka_s((char*)": {errNoStr}")}, \ + fobj__err_transform_kv(__VA_ARGS__) \ + }; \ + fobj__make_err(fobj_error_kind_SysErr(), \ + ft__srcpos(), msg, \ + kvs, ft_arrsz(kvs));\ +}) + +extern err_i fobj__make_err(const char *type, + ft_source_position_t src, + const char *msg, + fobj_err_kv_t *kvs, + size_t kvn); +extern err_i fobj__alloc_err(const char *type, + ft_source_position_t src, + const char *msg, + fobj_err_kv_t *kvs, + size_t kvn); + +#define fobj_err_has_kind(kind, err) \ + (!$noerr(err) && strcmp(fobj_error_kind_##kind(), $errkind(err)) == 0) + +#define fobj__err_transform_kv_do(v) \ + fobj__err_mkkv_##v +#define fobj__err_transform_kv(...) \ + fm_eval_foreach_comma(fobj__err_transform_kv_do, __VA_ARGS__) + +#define fobj__err_getkey(key, err, ...) \ + fobj__err_getkv_##key(err, fm_or_default(__VA_ARGS__)(NULL)) + +ft_inline int +getErrno(err_i err) { + return $errkey(errNo, err); +} + +ft_inline const char* +getErrnoStr(err_i err) { + return $errkey(errNoStr, err); +} + +ft_inline const char* +fobj_errkind(err_i err) { + fobjErr* self = (fobjErr*)(err.self); + ft_assert(fobj_real_klass_of(self) == fobjErr__kh()); \ + return self->type ? self->type : "RT"; +} + +ft_inline const char* +fobj_errmsg(err_i err) { + fobjErr* self = (fobjErr*)(err.self); + ft_assert(fobj_real_klass_of(self) == fobjErr__kh()); \ + return self->message ? self->message : "Unspecified Error"; +} + +ft_inline ft_source_position_t +fobj_errsrc(err_i err) { + fobjErr* self = (fobjErr*)(err.self); + ft_assert(fobj_real_klass_of(self) == fobjErr__kh()); \ + return self->src; +} + +#define fobj__printkv(fmt, ...) \ + fm_cat(fobj__printkv_, fm_va_01(__VA_ARGS__))(fmt, __VA_ARGS__) + +#define fobj__printkv_0(fmt, ...) \ + fobj_printkv(fmt, ft_slc_fokv_make(NULL, 0)) + +#define fobj__printkv_1(fmt, ...) ({ \ + fobj_kv kvs[] = { \ + fobj__transform_fokv(__VA_ARGS__) \ + }; \ + fobj_printkv(fmt, ft_slc_fokv_make(kvs, ft_arrsz(kvs))); \ +}) + +#define fobj__transform_fokv_do(key, val) \ + { #key, val } +#define fobj__transform_fokv(...) \ + fm_eval_tuples_comma(fobj__transform_fokv_do, __VA_ARGS__) + +/********************************** + * Temp Buffer - "anything" you want to be automatically cleared + */ +typedef struct fobjTempBuffer { + char buf[1]; +} fobjTempBuffer; +#define kls__fobjTempBuffer varsized(buf) +fobj_klass(fobjTempBuffer); + + +static inline void* +fobj_alloc_temp(size_t buf_size) +{ + return fobj_alloc_sized(fobjTempBuffer, buf_size)->buf; +} + +static inline fobj_t +fobj_temp2obj(void* temp) +{ + /* + * It looks dumb for the moment. + * But in future object header will not be hidden, therefore + * it will be meaningful. + */ + return (fobj_t)((char*)temp - offsetof(fobjTempBuffer, buf)); +} + +#endif // FOBJ_OBJ_PRIV2_H diff --git a/src/fu_util/impl/ft_impl.c b/src/fu_util/impl/ft_impl.c new file mode 100644 index 000000000..10a6f4812 --- /dev/null +++ b/src/fu_util/impl/ft_impl.c @@ -0,0 +1,781 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#include + +#include +#include +#include +#if !defined(WIN32) || defined(__MINGW64__) || defined(__MINGW32__) +#include +#include +#else +#define WIN32_LEAN_AND_MEAN + +#include +#include +#include +#undef small +#include +#include +#include +#undef near +#endif + +#ifdef HAVE_LIBBACKTRACE +#include +#if defined(__MINGW32__) || defined(__MINGW64__) +#include +#endif +#elif HAVE_EXECINFO_H +#include +#endif + +#include + +#define FT_LOG_MAX_FILES (1<<12) + +static void * (*_ft_realloc) (void *, size_t) = realloc; +static void (*_ft_free) (void *) = free; + +void ft_set_allocators( + void *(*_realloc)(void *, size_t), + void (*_free)(void*)) { + _ft_realloc = _realloc ? _realloc : realloc; + _ft_free = _free ? _free : free; +} + +void* +ft_calloc(size_t size) { + void * res = ft_malloc(size); + ft_memzero(res, size); + return res; +} + +void* +ft_realloc(void *oldptr, size_t size) { + if (size) { + void *res = _ft_realloc(oldptr, size); + ft_assert(res, "ft_realloc failed: oldptr=%p size=%zd", oldptr, size); + return res; + } + if (oldptr) + _ft_free(oldptr); + return NULL; +} + +void* +ft_malloc(size_t sz) +{ + return ft_realloc(NULL, sz); +} + +void* +ft_malloc_arr(size_t sz, size_t cnt) +{ + return ft_realloc(NULL, ft_mul_size(sz, cnt)); +} + +void +ft_free(void* ptr) +{ + ft_realloc(ptr, 0); +} + +void* +ft_calloc_arr(size_t sz, size_t cnt) +{ + return ft_calloc(ft_mul_size(sz, cnt)); +} + +void* +ft_realloc_arr(void* ptr, size_t elem_sz, size_t old_elems, size_t new_elems) { + ptr = ft_realloc(ptr, ft_mul_size(elem_sz, new_elems)); + if (new_elems > old_elems) + ft_memzero((char*)ptr + elem_sz * old_elems, + elem_sz * (new_elems - old_elems)); + return ptr; +} + +#ifdef OPTIMIZE_FT_MEMZERO +#define MEMZERO_BLOCK 4096 +static const uint8_t zero[4096] = {0}; +void +ft_memzero(void *_ptr, size_t sz) { + uint8_t* ptr = _ptr; + uintptr_t ptri = (uintptr_t)ptr; + uintptr_t diff; + + if (ptri & (MEMZERO_BLOCK-1)) { + diff = MEMZERO_BLOCK - (ptri & (MEMZERO_BLOCK-1)); + if (diff > sz) + diff = sz; + memset(ptr, 0, diff); + ptr += diff; + sz -= diff; + } + + /* Do not dirty page if it clear */ + while (sz >= MEMZERO_BLOCK) { + if (memcmp(ptr, zero, MEMZERO_BLOCK) != 0) { + memset(ptr, 0, MEMZERO_BLOCK); + } + ptr += MEMZERO_BLOCK; + sz -= MEMZERO_BLOCK; + } + + if (sz) + memset(ptr, 0, sz); +} +#else +void +ft_memzero(void *ptr, size_t sz) { + memset(ptr, 0, sz); +} +#endif + +/* String utils */ + +size_t +ft_strlcat(char *dest, const char* src, size_t dest_size) { + char* dest_null = memchr(dest, 0, dest_size); + size_t dest_len = dest_null ? dest_null - dest : dest_size; + ft_assert(dest_null, "destination has no zero byte"); + if (dest_len < dest_size-1) { + size_t cpy_len = dest_size - dest_len - 1; + cpy_len = ft_min(cpy_len, strlen(src)); + memcpy(dest+dest_len, src, cpy_len); + dest[dest_len + cpy_len] = '\0'; + } + return dest_len + strlen(src); +} + +size_t +ft_strlcpy(char *dest, const char* src, size_t dest_size) { + size_t cpy_len = dest_size - 1; + cpy_len = ft_min(cpy_len, strlen(src)); + memcpy(dest, src, cpy_len); + dest[cpy_len] = '\0'; + return strlen(src); +} + +ft_str_t +ft_vasprintf(const char *fmt, va_list args) { + ft_strbuf_t buf = ft_strbuf_zero(); + bool err; + + ft_strbuf_vcatf_err(&buf, &err, fmt, args); + + if (err) { + ft_strbuf_free(&buf); + return ft_str(NULL, 0); + } + return ft_strbuf_steal(&buf); +} + +ft_str_t +ft_asprintf(const char *fmt, ...) { + ft_strbuf_t buf = ft_strbuf_zero(); + bool err; + va_list args; + + va_start(args, fmt); + ft_strbuf_vcatf_err(&buf, &err, fmt, args); + va_end(args); + + if (err) { + ft_strbuf_free(&buf); + return ft_str(NULL, 0); + } + return ft_strbuf_steal(&buf); +} + +bool +ft__strbuf_ensure(ft_strbuf_t *buf, size_t n) { + size_t new_len; + size_t new_cap; + bool overflowed = false; + ft_assert(!buf->fixed); + ft_assert(buf->cap < ft_add_size(buf->len, n)); + /* 4GB string limit */ + ft_assert(buf->len + n <= UINT32_MAX); + new_len = buf->len + n; + if (new_len > UINT32_MAX) { + new_len = UINT32_MAX; + overflowed = true; + } + new_cap = ft_nextpow2(new_len); + if (buf->alloced) + buf->ptr = ft_realloc(buf->ptr, new_cap); + else { + char* newbuf = ft_malloc(new_cap); + if (buf->ptr != NULL) + memcpy(newbuf, buf->ptr, (size_t)buf->len+1); + else { + ft_assert(buf->len == 0); + newbuf[0] = '\0'; + } + buf->ptr = newbuf; + } + buf->cap = new_cap-1; + buf->alloced = true; + buf->fixed = overflowed; + buf->overflowed = overflowed; + return !overflowed; +} + +extern bool +ft_strbuf_vcatf_err(ft_strbuf_t *buf, bool err[1], const char *fmt, va_list args) { + int save_errno = errno; + char localbuf[256] = ""; + char *str = NULL; + size_t init_len = buf->len; + ssize_t len, need_len; + bool overflowed = false; + va_list argcpy; + + if (!ft_strbuf_may(buf)) + return false; + + err[0] = false; + + va_copy(argcpy, args); + need_len = vsnprintf(localbuf, ft_arrsz(localbuf), fmt, argcpy); + va_end(argcpy); + + if (need_len < 0) { + err[0] = true; + return true; + } + + if (need_len < ft_arrsz(localbuf)) { + return ft_strbuf_cat(buf, ft_str(localbuf, need_len)); + } + + for (;;) { + len = need_len; + if (!ft_strbuf_ensure(buf, len)) { + len = buf->cap - buf->len; + overflowed = true; + } + str = buf->ptr + init_len; + + errno = save_errno; + va_copy(argcpy, args); + need_len = vsnprintf(str, len+1, fmt, argcpy); + va_end(argcpy); + + if (need_len < 0) { + buf->ptr[buf->len] = '0'; + err[0] = true; + return true; + } + + if (need_len <= len) { + buf->len += need_len; + return ft_strbuf_may(buf); + } + if (overflowed) { + buf->len = buf->cap; + buf->overflowed = true; + return false; + } + } +} + +bool +ft_strbuf_vcatf(ft_strbuf_t *buf, const char *fmt, va_list args) { + bool err = false; + bool may_continue = ft_strbuf_vcatf_err(buf, &err, fmt, args); + if (err) + ft_log(FT_ERROR, "error printing format '%s'", fmt); + return may_continue; +} + +bool +ft_strbuf_catf(ft_strbuf_t *buf, const char *fmt, ...) { + bool err = false; + bool may_continue; + va_list args; + + va_start(args, fmt); + may_continue = ft_strbuf_vcatf_err(buf, &err, fmt, args); + va_end(args); + + if (err) + ft_log(FT_ERROR, "error printing format '%s'", fmt); + + return may_continue; +} + +/* Time */ +double +ft_time(void) { + struct timeval tv = {0, 0}; + ft_assyscall(gettimeofday(&tv, NULL)); + return (double)tv.tv_sec + (double)tv.tv_usec/1e6; +} + +/* Logging */ + +/* +static _Noreturn void +ft_quick_exit(const char* msg) { + write(STDERR_FILENO, msg, strlen(msg)); + abort(); +} +*/ + +static const char *ft_log_main_file = __FILE__; +const char* +ft__truncate_log_filename(const char *file) { + const char *me = ft_log_main_file; + const char *he = file; + for (;*he && *me && *he==*me;he++, me++) { +#ifndef WIN32 + if (*he == '/') + file = he+1; +#else + if (*he == '/' || *he == '\\') + file = he+1; +#endif + } + return file; +} + +static const char* +ft__base_log_filename(const char *file) { + const char *he = file; + for (;*he;he++) { +#ifndef WIN32 + if (*he == '/') + file = he+1; +#else + if (*he == '/' || *he == '\\') + file = he+1; +#endif + } + return file; +} + +#ifdef HAVE_LIBBACKTRACE +static struct backtrace_state * volatile ft_btstate = NULL; +static pthread_once_t ft_btstate_once = PTHREAD_ONCE_INIT; + + +static void +ft_backtrace_err(void *data, const char *msg, int errnum) +{ + fprintf(stderr, "ft_backtrace_err %s %d\n", msg, errnum); +} + +static void +ft_backtrace_init(void) { + const char *app = NULL; +#if defined(__MINGW32__) || defined(__MINGW64__) + static char appbuf[2048] = {0}; + /* 2048 should be enough, don't check error */ + GetModuleFileNameA(0, appbuf, sizeof(appbuf)-1); + app = appbuf; +#endif + __atomic_store_n(&ft_btstate, backtrace_create_state(app, 1, ft_backtrace_err, NULL), + __ATOMIC_RELEASE); +} + +static int +ft_backtrace_add(void *data, uintptr_t pc, + const char* filename, int lineno, + const char *function) { + struct ft_strbuf_t *buf = data; + ssize_t sz; + if (filename == NULL) + return 0; + return !ft_strbuf_catf(buf, "\n\t%s:%-4d\t%s", + ft__truncate_log_filename(filename), lineno, function ? function : "(unknown)"); +} +#endif + +static void ft_gnu_printf(4,0) +ft_default_log(enum FT_LOG_LEVEL level, ft_source_position_t srcpos, + const char* error, const char *fmt, va_list args) { +#define LOGMSG_SIZE (1<<12) + char buffer[LOGMSG_SIZE] = {0}; + ft_strbuf_t buf = ft_strbuf_init_fixed(buffer, LOGMSG_SIZE); + bool err; + double now; + + now = ft_time(); + ft_strbuf_catf(&buf, "%.3f %d [%s]", now, getpid(), ft_log_level_str(level)); + + if (level <= FT_DEBUG || level >= FT_ERROR) { + ft_strbuf_catf(&buf, " (%s@%s:%d)", srcpos.func, srcpos.file, srcpos.line); + } + + ft_strbuf_catc(&buf, " > "); + ft_strbuf_vcatf_err(&buf, &err, fmt, args); + if (err) { + ft_strbuf_catc(&buf, "<>"); + } + + if (error != NULL) { + ft_strbuf_catc(&buf, ": "); + ft_strbuf_catc(&buf, error); + } + + if (!ft_strbuf_may(&buf)) + goto done; + + if (level == FT_ERROR || level == FT_FATAL) { +#ifdef HAVE_LIBBACKTRACE + if (__atomic_load_n(&ft_btstate, __ATOMIC_ACQUIRE) == NULL) + pthread_once(&ft_btstate_once, ft_backtrace_init); + if (ft_btstate) + backtrace_full(ft_btstate, 0, ft_backtrace_add, NULL, &buf); +#elif defined(HAVE_EXECINFO_H) + void *backtr[32] = {0}; + char **syms = NULL; + int i, n; + n = backtrace(backtr, 32); + syms = backtrace_symbols(backtr, n); + if (syms != NULL) { + for (i = 1; i < n; i++) { + ft_strbuf_cat1(&buf, '\n'); + ft_strbuf_catc(&buf, syms[i]); + } + free(syms); + } +#endif + } + +done: + if (!ft_strbuf_may(&buf)) { + buf.ptr[buf.len-3] = '.'; + buf.ptr[buf.len-2] = '.'; + buf.ptr[buf.len-1] = '.'; + } + + fprintf(stderr, "%s\n", buffer); +} + +static ft_gnu_printf(4,0) +void (*ft_log_hook)(enum FT_LOG_LEVEL, ft_source_position_t, const char*, const char *fmt, va_list args) = ft_default_log; + +void +ft__init_log(ft_log_hook_t hook, const char *file) { + ft_log_hook = hook == NULL ? ft_default_log : hook; + ft_log_main_file = file == NULL ? __FILE__ : file; +} + +void +ft__log(enum FT_LOG_LEVEL level, ft_source_position_t srcpos, + const char* error, const char *fmt, ...) { + va_list args; + srcpos.file = ft__truncate_log_filename(srcpos.file); + va_start(args, fmt); + ft_log_hook(level, srcpos, error, fmt, args); + va_end(args); +} + +extern _Noreturn void +ft__log_fatal(ft_source_position_t srcpos, const char* error, + const char *fmt, ...) { + va_list args; + va_start(args, fmt); + ft_log_hook(FT_FATAL, srcpos, error, fmt, args); + va_end(args); + abort(); +} + +const char* +ft__strerror(int eno, char *buf, size_t len) { +#ifndef HAVE_STRERROR_R + char *sbuf = strerror(eno); + + if (sbuf == NULL) /* can this still happen anywhere? */ + return NULL; + /* To minimize thread-unsafety hazard, copy into caller's buffer */ + ft_strlcpy(buf, sbuf, len); + return buf; +#elif !_GNU_SOURCE && (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) + int saveno = errno; + int e = strerror_r(eno, buf, len); + if (e != 0) { + if (e == -1) { + e = errno; + } + if (e == EINVAL) { + snprintf(buf, len, "Wrong errno %d", eno); + } else if (e == ERANGE) { + snprintf(buf, len, "errno = %d has huge message", eno); + } + } + errno = saveno; + return buf; +#else + return strerror_r(eno, buf, len); +#endif +} + +#ifndef __TINYC__ +const char* +ft_strerror(int eno) { + static __thread char buf[256]; + return ft__strerror(eno, buf, sizeof(buf)); +} +#endif + +struct ft_log_and_assert_level ft_log_assert_levels = { + .log_level = FT_INFO, +#ifndef NDEBUG + .assert_level = FT_ASSERT_ALL, +#else + .assert_level = FT_ASSERT_RUNTIME, +#endif +}; + +typedef struct { + const char *file; + uint32_t next; + struct ft_log_and_assert_level local_levels; +} ft_log_file_registration; + +#define FT_LOG_FILES_HASH (FT_LOG_MAX_FILES/4) +static ft_log_file_registration ft_log_file_regs[FT_LOG_MAX_FILES] = {{0}}; +static uint32_t ft_log_file_reg_hash[FT_LOG_FILES_HASH] = {0}; +static uint32_t ft_log_file_n = 0; + +extern void +ft__register_source( + const char *file, + struct ft_log_and_assert_level **local_levels) { + ft_log_file_registration *reg; + uint32_t hash; + + ft_assert(ft_log_file_n < FT_LOG_MAX_FILES); + ft_dbg_assert(file != NULL); + + reg = &ft_log_file_regs[ft_log_file_n++]; + + reg->file = file; + reg->local_levels = ft_log_assert_levels; + + *local_levels = ®->local_levels; + + hash = ft_small_cstr_hash(ft__base_log_filename(reg->file)); + reg->next = ft_log_file_reg_hash[hash%FT_LOG_FILES_HASH]; + ft_log_file_reg_hash[hash%FT_LOG_FILES_HASH] = ft_log_file_n; +} + +static void +ft__log_level_reset(int what, int level) { + uint32_t i; + + if (what) + ft_log_assert_levels.log_level = level; + else + ft_log_assert_levels.assert_level = level; + + for (i = 0; i < ft_log_file_n; i++) { + if (what) + ft_log_file_regs[i].local_levels.log_level = level; + else + ft_log_file_regs[i].local_levels.assert_level = level; + } +} + +static void +ft__log_level_set(const char *file, int what, int level) { + ft_log_file_registration *reg; + uint32_t hash, i; + bool found = false; + size_t len = strlen(file); + + ft_dbg_assert(file != NULL); + + if (strcmp(file, "ALL") == 0) { + ft__log_level_reset(what, level); + return; + } + + hash = ft_small_cstr_hash(ft__base_log_filename(file)); + i = ft_log_file_reg_hash[hash%FT_LOG_FILES_HASH]; + while (i) { + size_t reglen; + reg = &ft_log_file_regs[i-1]; + ft_dbg_assert(reg->file != NULL); + reglen = strlen(reg->file); + if (reglen >= len && strcmp(reg->file + (reglen-len), file) == 0) { + if (what) + reg->local_levels.log_level = level; + else + reg->local_levels.assert_level = level; + found = true; + } + i = reg->next; + } + if (found) + return; + /* + * ooops... not found... pity... + * ok, lets set global one, but without per-file setting + */ + if (what) + ft_log_assert_levels.log_level = level; + else + ft_log_assert_levels.assert_level = level; +} + +void +ft_log_level_reset(enum FT_LOG_LEVEL level) { + ft__log_level_reset(1, level); +} + +void +ft_assert_level_reset(enum FT_ASSERT_LEVEL level) { + ft__log_level_reset(0, level); +} + +void +ft_log_level_set(const char *file, enum FT_LOG_LEVEL level) { + ft__log_level_set(file, 1, level); +} + +void +ft_assert_level_set(const char *file, enum FT_ASSERT_LEVEL level) { + ft__log_level_set(file, 0, level); +} + +uint32_t +ft_rand(void) { + static volatile uint32_t rstate = 0xbeaf1234; + uint32_t rand = __atomic_fetch_add(&rstate, 0x11, __ATOMIC_RELAXED); + rand = ft_mix32(rand); + return rand; +} + +uint32_t +ft_small_cstr_hash(const char *key) { + unsigned char *str = (unsigned char *)key; + uint32_t h1 = 0x3b00; + uint32_t h2 = 0; + for (;str[0]; str++) { + h1 += str[0]; + h1 *= 9; + h2 += h1; + h2 = ft_rol32(h2, 7); + h2 *= 5; + } + h1 ^= h2; + h1 += ft_rol32(h2, 14); + h2 ^= h1; h2 += ft_ror32(h1, 6); + h1 ^= h2; h1 += ft_rol32(h2, 5); + h2 ^= h1; h2 += ft_ror32(h1, 8); + return h2; +} + +// bytes + +ft_bytes_t +ft_bytes_shift_line(ft_bytes_t *bytes) +{ + size_t i; + char *p = bytes->ptr; + + for (i = 0; i < bytes->len; i++) { + if (p[i] == '\r' || p[i] == '\n') { + if (p[i] == '\r' && i+1 < bytes->len && p[i+1] == '\n') + i++; + ft_bytes_consume(bytes, i+1); + return ft_bytes(p, i+1); + } + } + + ft_bytes_consume(bytes, bytes->len); + return ft_bytes(p, i); +} + +ft_str_t +ft_bytes_shift_zt(ft_bytes_t *bytes) +{ + size_t i; + char *p = bytes->ptr; + + for (i = 0; i < bytes->len; i++) { + if (p[i] == '\0') { + ft_bytes_consume(bytes, i+1); + return ft_str(p, i); + } + } + + ft_assert(bytes->len == 0, "ft_bytes_shift_str have to be on zero-terminated bytes"); + return ft_str(NULL, 0); +} + +size_t +ft_bytes_find_bytes(ft_bytes_t haystack, ft_bytes_t needle) +{ + // TODO use memmem if present + size_t i; + char first; + + if (needle.len == 0) + return 0; + if (needle.len > haystack.len) + return haystack.len; + + first = needle.ptr[0]; + for (i = 0; i < haystack.len - needle.len; i++) + { + if (haystack.ptr[i] != first) + continue; + if (memcmp(haystack.ptr + i, needle.ptr, needle.len) == 0) + return i; + } + + return haystack.len; +} + +size_t +ft_bytes_spn_impl(ft_bytes_t bytes, ft_bytes_t chars, bool include) +{ + /* 32*8 = 256 bit */ + uint32_t mask[8] = {0}; + size_t i; + unsigned char c; + + if (chars.len == 0) + return 0; + + if (chars.len == 1 && include) { + c = chars.ptr[0]; + for (i = 0; i < bytes.len; i++) + if (bytes.ptr[i] != c) + return i; + return bytes.len; + } else if (chars.len == 1 && !include) { + c = chars.ptr[0]; + for (i = 0; i < bytes.len; i++) + if (bytes.ptr[i] == c) + return i; + return bytes.len; + } + + for (i = 0; i < chars.len; i++) { + c = chars.ptr[i]; + mask[c/32] |= 1 << (c&31); + } + + if (include) { + for (i = 0; i < bytes.len; i++) { + c = bytes.ptr[i]; + if ((mask[c / 32] & (1 << (c & 31))) == 0) + return i; + } + } else { + for (i = 0; i < bytes.len; i++) { + c = bytes.ptr[i]; + if ((mask[c / 32] & (1 << (c & 31))) != 0) + return i; + } + } + + return bytes.len; +} \ No newline at end of file diff --git a/src/fu_util/impl/ft_impl.h b/src/fu_util/impl/ft_impl.h new file mode 100644 index 000000000..096093f71 --- /dev/null +++ b/src/fu_util/impl/ft_impl.h @@ -0,0 +1,676 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 : */ +#ifndef FT_IMPL_H +#define FT_IMPL_H + +#ifdef __TINYC__ + +#if defined(__attribute__) +#undef __attribute__ +#define __attribute__ __attribute__ +#endif + +#include +#define __atomic_add_fetch(x, y, z) ft__atomic_add_fetch((x), (y), z, fm_uniq(y)) +#define ft__atomic_add_fetch(x, y_, z, y) ({ \ + __typeof(y_) y = y_; \ + __atomic_fetch_add((x), y, z) + y; \ +}) +#define __atomic_sub_fetch(x, y, z) ft__atomic_sub_fetch((x), (y), z, fm_uniq(y)) +#define ft__atomic_sub_fetch(x, y_, z, y) ({ \ + __typeof(y_) y = y_; \ + __atomic_fetch_sub((x), y, z) - y; \ +}) +#define __atomic_load_n(x, z) __atomic_load((x), z) +#define __atomic_store_n(x, y, z) __atomic_store((x), (y), z) + +#endif /* __TINYC__ */ + +/* Memory */ + +/* Logging */ + +static ft_unused inline const char* +ft_log_level_str(enum FT_LOG_LEVEL level) { + switch (level) { + case FT_DEBUG: return "DEBUG"; + case FT_LOG: return "LOG"; + case FT_INFO: return "INFO"; + case FT_WARNING: return "WARNING"; + case FT_ERROR: return "ERROR"; + case FT_FATAL: return "FATAL"; + case FT_OFF: return "OFF"; + case FT_TRACE: return "TRACE"; + default: return "UNKNOWN"; + } +} + +extern void ft__init_log(ft_log_hook_t hook, const char *file); + +struct ft_log_and_assert_level { + enum FT_LOG_LEVEL log_level; + enum FT_ASSERT_LEVEL assert_level; +}; + +extern struct ft_log_and_assert_level ft_log_assert_levels; + +/* this variable is duplicated in every source as static variable */ +static ft_unused +struct ft_log_and_assert_level *ft_local_lgas_levels = &ft_log_assert_levels; + +#define ft_will_log(level) (level >= ft_local_lgas_levels->log_level) +extern void ft__register_source(const char *file, + struct ft_log_and_assert_level **local_levels); + +#if defined(__GNUC__) || defined(__TINYC__) +#define ft__register_source_impl() \ + static __attribute__((constructor)) void \ + ft__register_source_(void) { \ + ft__register_source(__FILE__, &ft_local_lgas_levels); \ + } \ + fm__dumb_require_semicolon +#else +#define ft_register_source_impl() fm__dumb_require_semicolon +#endif + +#define COMPARE_FT_FATAL(x) x +#define ft__log_impl(level, error, fmt_or_msg, ...) \ + fm_if(fm_equal(level, FT_FATAL), \ + ft__log_fatal(ft__srcpos(), error, ft__log_fmt_msg(fmt_or_msg, __VA_ARGS__)), \ + ft__log_common(level, error, fmt_or_msg, __VA_ARGS__)) + +#define ft__log_common(level, error, fmt_or_msg, ...) do {\ + if (level >= FT_ERROR || ft_unlikely(ft_will_log(level))) \ + ft__log(level, ft__srcpos(), error, ft__log_fmt_msg(fmt_or_msg, __VA_ARGS__)); \ +} while(0) + +#define ft__log_fmt_msg(fmt, ...) \ + fm_iif(fm_no_va(__VA_ARGS__))("%s", fmt)(fmt, __VA_ARGS__) + +extern ft_gnu_printf(4, 5) +void ft__log(enum FT_LOG_LEVEL level, ft_source_position_t srcpos, const char* error, const char *fmt, ...); +extern _Noreturn ft_gnu_printf(3, 4) +void ft__log_fatal(ft_source_position_t srcpos, const char* error, const char *fmt, ...); + +ft_inline bool ft__dbg_enabled(void) { + return ft_unlikely(ft_local_lgas_levels->assert_level >= FT_ASSERT_ALL); +} + +#define ft__dbg_assert(x, xs, ...) do { \ + if (ft__dbg_enabled() && ft_unlikely(!(x))) \ + ft__log_fatal(ft__srcpos(), xs, ft__assert_arg(__VA_ARGS__)); \ +} while(0) + +#define ft__assert(x, xs, ...) do { \ + if (ft_unlikely(!(x))) \ + ft__log_fatal(ft__srcpos(), xs, ft__assert_arg(__VA_ARGS__)); \ +} while(0) + +#define ft__assert_arg(...) \ + fm_if(fm_no_va(__VA_ARGS__), "Asserion failed", \ + ft__log_fmt_msg(__VA_ARGS__)) + +#define ft__assyscall(syscall, res, ...) ({ \ + __typeof(syscall) res = (syscall); \ + ft__assert(res >= 0, ft_strerror(errno), #syscall __VA_ARGS__); \ + res; \ + }) + +/* Comparison */ + +#define ft__max(a_, b_, a, b) ({ \ + __typeof(a_) a = (a_); \ + __typeof(b_) b = (b_); \ + a < b ? b : a ; \ + }) + +#define ft__min(a_, b_, a, b) ({ \ + __typeof(a_) a = (a_); \ + __typeof(b_) b = (b_); \ + a > b ? b : a ; \ + }) + +#define ft__cmp(a_, b_, a, b) ({ \ + __typeof(a_) a = (a_); \ + __typeof(b_) b = (b_); \ + a < b ? FT_CMP_LT : (a > b ? FT_CMP_GT : FT_CMP_EQ); \ + }) + +#define ft__swap(a_, b_, ap, bp, t) do { \ + __typeof(a_) ap = a_; \ + __typeof(a_) bp = b_; \ + __typeof(*ap) t = *ap; \ + *ap = *bp; \ + *bp = t; \ +} while (0) + +#if defined(__has_builtin) || defined(__clang__) +# if __has_builtin(__builtin_add_overflow) && __has_builtin(__builtin_mul_overflow) +# define ft__has_builtin_int_overflow +# endif +#elif __GNUC__ > 4 && !defined(__clang__) && !defined(__LCC__) +# define ft__has_builtin_int_overflow +#endif + +ft_inline size_t ft_add_size(size_t a, size_t b) { + size_t r; +#ifdef ft__has_builtin_int_overflow + if (ft_unlikely(__builtin_add_overflow(a, b, &r))) + ft_assert(r >= a && r >= b); +#else + r = a + b; + ft_assert(r >= a && r >= b); +#endif + return r; +} + +ft_inline size_t ft_mul_size(size_t a, size_t b) { + size_t r; +#ifdef ft__has_builtin_int_overflow + if (ft_unlikely(__builtin_mul_overflow(a, b, &r))) + ft_assert(r / a == b); +#else + r = a * b; + ft_assert(r / a == b); +#endif + return r; +} + +/* division 64->32 bit */ +ft_inline int32_t ft_div_i64u32_to_i32(int64_t a, uint32_t b) { + int64_t r; + ft_assert(a >= 0); + r = a / b; + ft_assert(r <= INT32_MAX); + return (int32_t)r; +} + +extern ft_gcc_malloc(ft_realloc, 1) void* ft_realloc(void* ptr, size_t new_sz); +extern ft_gcc_malloc(ft_realloc, 1) void* ft_calloc(size_t sz); + +// Some Numeric Utils + +ft_inline uint32_t +ft_rol32(uint32_t x, unsigned n) { + return n == 0 ? x : n >= 32 ? 0 : (x << n) | (x >> (32 - n)); +} + +ft_inline uint32_t +ft_ror32(uint32_t x, unsigned n) { + return n == 0 ? x : n >= 32 ? 0 : (x << (32 - n)) | (x >> n); +} + +ft_inline size_t +ft_nextpow2(size_t sz) { + sz |= sz >> 1; + sz |= sz >> 2; + sz |= sz >> 4; + sz |= sz >> 8; + sz |= sz >> 16; +#if !defined(__SIZEOF_SIZE_T__) + if (sizeof(sz) > 4) + sz |= sz >> 32; +#elif __SIZEOF_SIZE_T__ > 4 + sz |= sz >> 32; +#endif + return ft_add_size(sz, 1); +} + +ft_inline uint32_t +ft_mix32(uint32_t h) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + +ft_inline uint32_t +ft_fast_randmod(uint32_t v, uint32_t mod) { + return (uint32_t)(((uint64_t)v * mod) >> 32); +} + +ft_inline uint32_t ft_randn(uint32_t mod) { + return ft_fast_randmod(ft_rand(), mod); +} + +ft_inline uint32_t ft_rand32(uint32_t* state, uint32_t mod) { + uint32_t x = *state; + uint32_t r = ft_rol32(x, 15); + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + r += x; + return mod ? ft_fast_randmod(r, mod) : r; +} + +/* ft_val_t */ +struct ft_arg { + union { + void *p; + char *s; + int64_t i; + uint64_t u; + double f; + bool b; +#ifdef FOBJ_OBJ_H + fobj_t o; +#endif + } v; + char t; +}; + +ft_inline ft_arg_t ft_mka_z(void) { return (ft_arg_t){.v={.u = 0}, .t='z'};} +ft_inline ft_arg_t ft_mka_p(void* p) { return (ft_arg_t){.v={.p = p}, .t='p'};} +ft_inline ft_arg_t ft_mka_s(char* s) { return (ft_arg_t){.v={.s = s}, .t='s'};} +ft_inline ft_arg_t ft_mka_i(int64_t i) { return (ft_arg_t){.v={.i = i}, .t='i'};} +ft_inline ft_arg_t ft_mka_u(uint64_t u) { return (ft_arg_t){.v={.u = u}, .t='u'};} +ft_inline ft_arg_t ft_mka_f(double f) { return (ft_arg_t){.v={.f = f}, .t='f'};} +ft_inline ft_arg_t ft_mka_b(bool b) { return (ft_arg_t){.v={.b = b}, .t='b'};} +#ifdef FOBJ_OBJ_H +ft_inline ft_arg_t ft_mka_o(fobj_t o) { return (ft_arg_t){.v={.o = o}, .t='o'};} +#endif + +ft_inline char ft_arg_type(ft_arg_t v) { return v.t; } + +ft_inline void ft_arg_z(ft_arg_t v) { ft_dbg_assert(v.t=='z'); } +ft_inline void* ft_arg_p(ft_arg_t v) { ft_dbg_assert(v.t=='p'); return v.v.p; } +ft_inline char* ft_arg_s(ft_arg_t v) { ft_dbg_assert(v.t=='s'); return v.v.s; } +ft_inline int64_t ft_arg_i(ft_arg_t v) { ft_dbg_assert(v.t=='i'); return v.v.i; } +ft_inline uint64_t ft_arg_u(ft_arg_t v) { ft_dbg_assert(v.t=='u'); return v.v.u; } +ft_inline double ft_arg_f(ft_arg_t v) { ft_dbg_assert(v.t=='f'); return v.v.f; } +ft_inline bool ft_arg_b(ft_arg_t v) { ft_dbg_assert(v.t=='b'); return v.v.b; } +#ifdef FOBJ_OBJ_H +ft_inline fobj_t ft_arg_o(ft_arg_t v) { ft_dbg_assert(v.t=='o'); return v.v.o; } +#endif + +/* slices and arrays */ + +ft_inline size_t +ft__index_unify(ssize_t at, size_t len) { + if (at >= 0) { + ft_assert(at < len); + return at; + } else { + ft_assert((size_t)(-at) <= len); + return (size_t)(len - (size_t)(-at)); + } +} + +ft_inline size_t +ft__slcindex_unify(ssize_t end, size_t len) { + if (end >= 0) { + ft_assert(end <= len); + return end; + } else if (end == FT_SLICE_END) { + return len; + } else { + ft_assert((size_t)(-end) <= len); + return (size_t)(len - (size_t)(-end)); + } +} + +// Bytes + +ft_inline ft_bytes_t +ft_bytes_split(ft_bytes_t *bytes, size_t n) { + ft_dbg_assert(n <= bytes->len); + ft_bytes_t head = ft_bytes(bytes->ptr, n); + bytes->ptr += n; + bytes->len -= n; + return head; +} + +ft_inline void +ft_bytes_consume(ft_bytes_t *bytes, size_t cut) { + ft_dbg_assert(cut <= bytes->len); + bytes->ptr += cut; + bytes->len -= cut; +} + +ft_inline size_t +ft_bytes_move(ft_bytes_t *dest, ft_bytes_t *src) { + size_t len = ft_min(dest->len, src->len); + memmove(dest->ptr, src->ptr, len); + ft_bytes_consume(dest, len); + ft_bytes_consume(src, len); + + return len; +} + +ft_inline bool +ft_bytes_shift_to(ft_bytes_t *bytes, ft_bytes_t to) +{ + if (bytes->len < to.len) + return false; + memmove(to.ptr, bytes->ptr, to.len); + ft_bytes_consume(bytes, to.len); + return true; +} + +ft_inline void +ft_bytes_shift_must(ft_bytes_t *bytes, ft_bytes_t to) +{ + ft_dbg_assert(to.len <= bytes->len); + memmove(to.ptr, bytes->ptr, to.len); + ft_bytes_consume(bytes, to.len); +} + +ft_inline bool +ft_bytes_starts_with(ft_bytes_t haystack, ft_bytes_t needle) +{ + return haystack.len >= needle.len && + memcmp(haystack.ptr, needle.ptr, needle.len) == 0; +} + +ft_inline bool +ft_bytes_starts_withc(ft_bytes_t haystack, const char* needle) +{ + return ft_bytes_starts_with(haystack, ft_bytesc(needle)); +} + +ft_inline bool +ft_bytes_ends_with(ft_bytes_t haystack, ft_bytes_t needle) +{ + return haystack.len >= needle.len && + memcmp(haystack.ptr + haystack.len - needle.len, needle.ptr, needle.len) == 0; +} + +ft_inline bool +ft_bytes_ends_withc(ft_bytes_t haystack, const char* needle) +{ + return ft_bytes_ends_with(haystack, ft_bytesc(needle)); +} + +ft_inline size_t +ft_bytes_find_cstr(ft_bytes_t haystack, const char* needle) +{ + return ft_bytes_find_bytes(haystack, ft_str2bytes(ft_cstr(needle))); +} + +ft_inline bool +ft_bytes_has_cstr(ft_bytes_t haystack, const char* needle) +{ + size_t pos = ft_bytes_find_cstr(haystack, needle); + return pos != haystack.len; +} + +extern size_t ft_bytes_spn_impl(ft_bytes_t bytes, ft_bytes_t chars, bool include); + +ft_inline size_t +ft_bytes_spn(ft_bytes_t bytes, ft_bytes_t chars) +{ + return ft_bytes_spn_impl(bytes, chars, true); +} + +ft_inline size_t +ft_bytes_notspn(ft_bytes_t bytes, ft_bytes_t chars) +{ + return ft_bytes_spn_impl(bytes, chars, false); +} + +ft_inline size_t +ft_bytes_spnc(ft_bytes_t bytes, const char* chars) +{ + return ft_bytes_spn(bytes, ft_bytesc(chars)); +} + +ft_inline size_t +ft_bytes_notspnc(ft_bytes_t bytes, const char* chars) +{ + return ft_bytes_notspn(bytes, ft_bytesc(chars)); +} + +// String utils + +ft_inline char * +ft_cstrdup(const char *str) { + return (char*)ft_strdupc(str).ptr; +} + +ft_inline char * +ft_cstrdupn(const char *str, size_t n) { + return (char*)ft_strdup_bytes(ft_bytes((char*)str, n)).ptr; +} + +ft_inline ft_str_t +ft_strdup(ft_str_t str) { + return ft_strdup_bytes(ft_bytes(str.ptr, str.len)); +} + +ft_inline ft_str_t +ft_strdup_bytes(ft_bytes_t str) { + char *mem = ft_malloc(str.len + 1); + if (str.ptr != NULL) + memcpy(mem, str.ptr, str.len); + mem[str.len] = '\0'; + return ft_str(mem, str.len); +} + +ft_inline ft_str_t +ft_strdupc(const char *str) { + return ft_strdup(ft_cstr(str)); +} + +ft_inline void +ft_str_free(ft_str_t *str) { + ft_free(str->ptr); + str->ptr = NULL; + str->len = 0; +} + +ft_inline bool +ft_streq(ft_str_t str, ft_str_t oth) { + return str.len == oth.len && strncmp(str.ptr, oth.ptr, str.len) == 0; +} + +ft_inline FT_CMP_RES +ft_strcmp(ft_str_t str, ft_str_t oth) { + size_t m = ft_min(str.len, oth.len); + return strncmp(str.ptr, oth.ptr, m) ?: ft_cmp(str.len, oth.len); +} + +ft_inline bool +ft_streqc(ft_str_t str, const char* oth) { + return ft_streq(str, ft_cstr(oth)); +} + +ft_inline FT_CMP_RES +ft_strcmpc(ft_str_t str, const char* oth) { + return ft_strcmp(str, ft_cstr(oth)); +} + +ft_inline void +ft_str_consume(ft_str_t *str, size_t cut) { + ft_dbg_assert(cut <= str->len); + str->ptr = str->ptr + cut; + str->len -= cut; +} + +ft_inline size_t +ft_str_spnc(ft_str_t str, const char* chars) { + return ft_bytes_spnc(ft_str2bytes(str), chars); +} + +ft_inline bool +ft_str_ends_withc(ft_str_t str, const char* needle) { + return ft_bytes_ends_withc(ft_str2bytes(str), needle); +} + +ft_inline size_t +ft_str_find_cstr(ft_str_t haystack, const char *needle) { + return ft_bytes_find_cstr(ft_str2bytes(haystack), needle); +} + +ft_inline void +ft_str_chop1(ft_str_t *str) { + ft_assert(str->len >= 1); + str->ptr[str->len-1] = 0; + str->len--; +} + +ft_inline ft_strbuf_t +ft_strbuf_zero(void) { + return (ft_strbuf_t){.ptr = "", .len = 0, .cap = 0}; +} + +ft_inline ft_strbuf_t +ft_strbuf_init_stack(char *buf, size_t capa) { + if (capa == 0) + return (ft_strbuf_t){.ptr = "", .len = 0, .cap = 0}; + ft_assert(capa <= UINT32_MAX); + buf[0] = '\0'; + return (ft_strbuf_t){.ptr = buf, .len = 0, .cap = capa-1}; +} + +ft_inline ft_strbuf_t +ft_strbuf_continue(char *buf, size_t capa) { + if (capa == 0) + return (ft_strbuf_t){.ptr = "", .len = 0, .cap = 0}; + ft_assert(capa <= UINT32_MAX); + buf[0] = '\0'; + return (ft_strbuf_t){.ptr = buf, .len = 0, .cap = capa-1}; +} + +ft_inline ft_strbuf_t +ft_strbuf_init_fixed(char *buf, size_t capa) { + ft_assert(capa > 0 && capa <= UINT32_MAX); + buf[0] = '\0'; + return (ft_strbuf_t){.ptr = buf, .len = 0, .cap = capa-1, .fixed = true}; +} + +ft_inline ft_strbuf_t +ft_strbuf_init_str(ft_str_t str) { + ft_assert(str.len <= UINT32_MAX); + return (ft_strbuf_t){.ptr = str.ptr, .len = str.len, .cap = str.len}; +} + +/* + * always allocates space for 1 zero ending byte. + * Returns false, if buffer reaches 4GB limit. + */ +extern bool ft__strbuf_ensure(ft_strbuf_t *buf, size_t n); + +ft_inline bool +ft_strbuf_may(ft_strbuf_t *buf) { + return !buf->fixed || buf->len < buf->cap; +} + +ft_inline bool +ft_strbuf_ensure(ft_strbuf_t *buf, size_t n) { + if ((size_t)buf->cap < ft_add_size(buf->len, n)) { + if (buf->fixed) + return false; + return ft__strbuf_ensure(buf, n); + } + return true; +} + +ft_inline bool +ft_strbuf_cat(ft_strbuf_t *buf, ft_str_t s) { + /* we could actually reuse ft_strbuf_catbytes */ + return ft_strbuf_catbytes(buf, ft_bytes(s.ptr, s.len)); +} + +ft_inline bool +ft_strbuf_cat_zt(ft_strbuf_t *buf, ft_str_t s) { + /* we could actually reuse ft_strbuf_catbytes */ + if (s.len == 0) + return ft_strbuf_cat1(buf, 0); + return ft_strbuf_catbytes(buf, ft_bytes(s.ptr, s.len+1)); +} + +ft_inline bool +ft_strbuf_catbytes(ft_strbuf_t *buf, ft_bytes_t s) { + if (!ft_strbuf_may(buf)) + return false; + if (s.len == 0) + return true; + if (!ft_strbuf_ensure(buf, s.len)) { + s.len = buf->cap - buf->len; + buf->overflowed = true; + } + if (s.len > 0) { + memmove(buf->ptr + buf->len, s.ptr, s.len); + buf->len += s.len; + buf->ptr[buf->len] = '\0'; + } + return ft_strbuf_may(buf); +} + +ft_inline bool +ft_strbuf_cat1(ft_strbuf_t *buf, char c) { + if (!ft_strbuf_may(buf)) + return false; + if (ft_strbuf_ensure(buf, 1)) { + buf->ptr[buf->len+0] = c; + buf->ptr[buf->len+1] = '\0'; + buf->len++; + } else { + buf->overflowed = true; + } + return ft_strbuf_may(buf); +} + +ft_inline bool +ft_strbuf_cat2(ft_strbuf_t *buf, char c1, char c2) { + if (!ft_strbuf_may(buf)) + return false; + if (ft_strbuf_ensure(buf, 2)) { + buf->ptr[buf->len+0] = c1; + buf->ptr[buf->len+1] = c1; + buf->ptr[buf->len+2] = '\0'; + buf->len+=2; + } else if (ft_strbuf_ensure(buf, 1)){ + buf->ptr[buf->len+0] = c1; + buf->ptr[buf->len+1] = '\0'; + buf->len++; + buf->overflowed = true; + } else { + buf->overflowed = true; + } + return ft_strbuf_may(buf); +} + +ft_inline bool +ft_strbuf_catc(ft_strbuf_t *buf, const char *s) { + return ft_strbuf_cat(buf, ft_cstr(s)); +} + +ft_inline bool +ft_strbuf_catc_zt(ft_strbuf_t *buf, const char *s) { + return ft_strbuf_cat_zt(buf, ft_cstr(s)); +} + +ft_inline void +ft_strbuf_reset_for_reuse(ft_strbuf_t *buf) { + buf->len = 0; + buf->overflowed = false; +} + +ft_inline void +ft_strbuf_free(ft_strbuf_t *buf) { + if (buf->alloced) { + ft_free(buf->ptr); + } + *buf = (ft_strbuf_t){NULL}; +} + +ft_inline ft_str_t +ft_strbuf_ref(ft_strbuf_t *buf) { + return ft_str(buf->ptr, buf->len); +} + +ft_inline ft_str_t +ft_strbuf_steal(ft_strbuf_t *buf) { + ft_str_t res = ft_str(buf->ptr, buf->len); + if (!buf->alloced) { + res = ft_strdup(res); + } + *buf = (ft_strbuf_t){NULL}; + return res; +} + +#endif diff --git a/src/fu_util/test/CMakeLists.txt b/src/fu_util/test/CMakeLists.txt new file mode 100644 index 000000000..3752ecf18 --- /dev/null +++ b/src/fu_util/test/CMakeLists.txt @@ -0,0 +1,39 @@ +cmake_minimum_required(VERSION 3.11) + +add_executable(fm fm.c) +add_executable(fm1 fm.c) + +target_compile_options(fm1 PRIVATE -DFM_USE_STRICT=1) + +add_executable(array array.c) +target_link_libraries(array fu_utils) + +add_executable(bsearch bsearch.c) +target_link_libraries(bsearch fu_utils) + +add_executable(fuprintf fuprintf.c) +target_link_libraries(fuprintf fu_utils) + +add_executable(sort sort.c) +target_link_libraries(sort fu_utils) + +add_executable(sort_p sort_p.c) +target_link_libraries(sort_p fu_utils) + +add_executable(obj1 obj1.c) +target_link_libraries(obj1 fu_utils) + +add_executable(thread thread.c) +target_link_libraries(thread fu_utils) + +enable_testing() + +add_test(NAME fm COMMAND fm) +add_test(NAME fm1 COMMAND fm1) +add_test(NAME array COMMAND array) +add_test(NAME bsearch COMMAND bsearch) +add_test(NAME fuprintf COMMAND fuprintf) +add_test(NAME sort COMMAND sort) +add_test(NAME sort_p COMMAND sort_p) +add_test(NAME obj1 COMMAND obj1) +add_test(NAME thread COMMAND thread) diff --git a/src/fu_util/test/array.c b/src/fu_util/test/array.c new file mode 100644 index 000000000..aeb108e58 --- /dev/null +++ b/src/fu_util/test/array.c @@ -0,0 +1,135 @@ +#include +#include +#include +#define FU_MALLOC_RAW +#include "../ft_util.h" +#include "../ft_ss_examples.h" +#include "../ft_ar_examples.h" +#include + +static void +check_equal_fun(int *a, int *b, int len) +{ + for (len--; len >= 0; len--) + ft_assert(a[len] == b[len]); +} + +#define check_equal(_a_, ...) do { \ + int _cmp_[] = {__VA_ARGS__}; \ + int _len_ = ft_arrsz(_cmp_); \ + ft_assert((_a_)->len == _len_); \ + check_equal_fun((_a_)->ptr, _cmp_, _len_); \ +} while (0) + +static int wlkcnt = 0; +static FT_WALK_ACT +walk_simple(int *el) { + wlkcnt++; + if (*el > 8) + return FT_WALK_BREAK; + return FT_WALK_CONT; +} + +static FT_WALK_ACT +walk_del(int *el, ft_arg_t v) { + wlkcnt++; + if (*el == ft_arg_i(v)) + return FT_WALK_DEL; + return FT_WALK_CONT; +} + +static FT_WALK_ACT +walk_del2(int *el, ft_arg_t v) { + wlkcnt++; + if (*el == ft_arg_i(v)) + return FT_WALK_DEL_BREAK; + return FT_WALK_CONT; +} + + +int +main(void) { + ft_arr_int_t arr = ft_arr_init(); + int v, i; + ft_bsres_t bsres; + + ft_arr_int_push(&arr, 1); + check_equal(&arr, 1); + + ft_arr_int_push(&arr, 10); + ft_arr_int_push(&arr, 5); + ft_arr_int_push(&arr, 25); + ft_arr_int_push(&arr, 15); + ft_arr_int_push(&arr, 2); + + check_equal(&arr, 1, 10, 5, 25, 15, 2); + + ft_arr_int_resize(&arr, 1); + check_equal(&arr, 1); + ft_arr_int_append(&arr, ((int[]){10, 5, 25, 15, 2}), 5); + check_equal(&arr, 1, 10, 5, 25, 15, 2); + + ft_assert(ft_arr_int_at(&arr, 1) == 10); + ft_assert(ft_arr_int_at(&arr, 5) == 2); + + ft_shsort_int(ft_2ptrlen(arr), ft_int_cmp); + check_equal(&arr, 1, 2, 5, 10, 15, 25); + ft_assert(ft_arr_int_at(&arr, 2) == 5); + ft_assert(ft_arr_int_at(&arr, 5) == 25); + + ft_arr_int_set(&arr, 2, 8); + check_equal(&arr, 1, 2, 8, 10, 15, 25); + + bsres = ft_bsearch_int(ft_2ptrlen(arr), 14, ft_int_cmp); + ft_assert(bsres.ix == 4); + ft_assert(!bsres.eq); + bsres = ft_bsearch_int(ft_2ptrlen(arr), 2, ft_int_cmp); + ft_assert(bsres.ix == 1); + ft_assert(bsres.eq); + + i = ft_search_int(ft_2ptrlen(arr), 2, ft_int_cmp); + ft_assert(i == 1); + i = ft_search_int(ft_2ptrlen(arr), 3, ft_int_cmp); + ft_assert(i == 6); + + v = ft_arr_int_pop(&arr); + ft_assert(v == 25); + check_equal(&arr, 1, 2, 8, 10, 15); + + v = ft_arr_int_del_at(&arr, 1); + ft_assert(v == 2); + check_equal(&arr, 1, 8, 10, 15); + + ft_arr_int_insert_at(&arr, 3, 11); + check_equal(&arr, 1, 8, 10, 11, 15); + ft_arr_int_insert_at(&arr, 5, 20); + check_equal(&arr, 1, 8, 10, 11, 15, 20); + + ft_arr_int_del_slice(&arr, 3, 5); + check_equal(&arr, 1, 8, 10, 20); + + ft_arr_int_insert_n(&arr, 1, (int[]){7, 7, 9, 9}, 4); + check_equal(&arr, 1, 7, 7, 9, 9, 8, 10, 20); + + ft_arr_int_del_slice(&arr, -2, FT_SLICE_END); + check_equal(&arr, 1, 7, 7, 9, 9, 8); + + wlkcnt = 0; + ft_arr_int_walk(&arr, walk_simple); + ft_assert(wlkcnt == 4); + + wlkcnt = 0; + ft_arr_int_walk_r(&arr, walk_del, ft_mka_i(9)); + ft_assert(wlkcnt == 6); + check_equal(&arr, 1, 7, 7, 8); + + wlkcnt = 0; + ft_arr_int_walk_r(&arr, walk_del2, ft_mka_i(7)); + ft_assert(wlkcnt == 2); + check_equal(&arr, 1, 7, 8); + + ft_arr_int_free(&arr); + ft_assert(arr.len == 0); + ft_assert(arr.ptr == NULL); + +} diff --git a/src/fu_util/test/bsearch.c b/src/fu_util/test/bsearch.c new file mode 100644 index 000000000..9d33fd806 --- /dev/null +++ b/src/fu_util/test/bsearch.c @@ -0,0 +1,46 @@ +#include +#include +#include +#include +#include +#include + +int +main(void) { + int ex[] = {1, 3, 5, 7, 8, 9}; + ft_bsres_t bs; + + bs = ft_bsearch_int(ex, 6, 0, ft_int_cmp); + ft_assert(bs.ix == 0); + ft_assert(!bs.eq); + bs = ft_bsearch_int(ex, 6, 1, ft_int_cmp); + ft_assert(bs.ix == 0); + ft_assert(bs.eq); + bs = ft_bsearch_int(ex, 6, 2, ft_int_cmp); + ft_assert(bs.ix == 1); + ft_assert(!bs.eq); + bs = ft_bsearch_int(ex, 6, 3, ft_int_cmp); + ft_assert(bs.ix == 1); + ft_assert(bs.eq); + bs = ft_bsearch_int(ex, 6, 4, ft_int_cmp); + ft_assert(bs.ix == 2); + ft_assert(!bs.eq); + bs = ft_bsearch_int(ex, 6, 5, ft_int_cmp); + ft_assert(bs.ix == 2); + ft_assert(bs.eq); + bs = ft_bsearch_int(ex, 6, 6, ft_int_cmp); + ft_assert(bs.ix == 3); + ft_assert(!bs.eq); + bs = ft_bsearch_int(ex, 6, 7, ft_int_cmp); + ft_assert(bs.ix == 3); + ft_assert(bs.eq); + bs = ft_bsearch_int(ex, 6, 8, ft_int_cmp); + ft_assert(bs.ix == 4); + ft_assert(bs.eq); + bs = ft_bsearch_int(ex, 6, 9, ft_int_cmp); + ft_assert(bs.ix == 5); + ft_assert(bs.eq); + bs = ft_bsearch_int(ex, 6, 10, ft_int_cmp); + ft_assert(bs.ix == 6); + ft_assert(!bs.eq); +} diff --git a/src/fu_util/test/fm.c b/src/fu_util/test/fm.c new file mode 100644 index 000000000..f42693802 --- /dev/null +++ b/src/fu_util/test/fm.c @@ -0,0 +1,316 @@ +#include + +#ifndef __TINYC__ + #define AssertEq(x, v, name) _Static_assert(x == v, name) + #define AssertEqStr(x, str, name) _Static_assert(__builtin_strcmp(x, str) == 0, name) + #define AssertNqStr(x, str, name) _Static_assert(__builtin_strcmp(x, str) != 0, name) +#else + #ifdef NDEBUG + #undef NDEBUG + #endif + #include + #include + #define AssertEq(x, v, name) assert(x == v) + #define AssertEqStr(x, str, name) assert(strcmp(x, str) == 0) + #define AssertNqStr(x, str, name) assert(strcmp(x, str) != 0) +#endif + +int main(void) { + +#define asdfhjkl 99 +AssertEq(fm_cat(asdf,hjkl), 99, "fm_cat"); +AssertEq(fm_cat(fm_cat(as,df),fm_cat(hj,kl)), 99, "fm_cat(fm_cat)"); +AssertEq(fm_cat3(as,dfhj,kl), 99, "fm_cat3"); +AssertEq(fm_cat4(as,df,hj,kl), 99, "fm_cat4"); + +AssertEqStr(fm_str(1), "1", "fm_str"); +AssertEqStr(fm_str(1,2), "1,2", "fm_str"); +AssertEqStr(fm_str(1, 2), "1, 2", "fm_str"); +AssertEqStr(fm_str(1, 2), "1, 2", "fm_str"); +AssertEqStr(fm_str(1 , 2), "1 , 2", "fm_str"); +AssertEqStr(fm_str(1, 2 ), "1, 2", "fm_str"); +AssertEqStr(fm_str(fm_cat(1,2)), "12", "fm_str"); +AssertEqStr(fm_str(fm_cat(1, 2 )), "12", "fm_str"); +AssertEqStr(fm_str(fm_cat(x, y )), "xy", "fm_str"); +AssertEqStr(fm_str(fm_cat(x , y )), "xy", "fm_str"); +AssertEqStr(fm_str(fm_cat3(1,2,3)), "123", "fm_str"); +AssertEqStr(fm_str(fm_cat3(1, 2 , 3)), "123", "fm_str"); +AssertEqStr(fm_str(fm_cat3(x, y , z)), "xyz", "fm_str"); +AssertEqStr(fm_str(fm_cat3(x , y ,z)), "xyz", "fm_str"); + +AssertNqStr(fm_str(fm_uniq(x)), fm_str(fm_uniq(x)), "fm_uniq"); + +AssertEqStr(fm_str(fm_expand()), "", "fm_expand"); +AssertEqStr(fm_str(fm_expand(1)), "1", "fm_expand"); +AssertEqStr(fm_str(fm_expand( 1 )), "1", "fm_expand"); +AssertEqStr(fm_str( fm_expand( 1 ) ), "1", "fm_expand"); +AssertEqStr(fm_str(fm_expand(1,2)), "1,2", "fm_expand"); +AssertEqStr(fm_str(fm_expand( 1 , 2 )), "1 , 2", "fm_expand"); +AssertEqStr(fm_str( fm_expand( 1 , 2) ), "1 , 2", "fm_expand"); + +AssertEqStr(fm_str(fm_empty()), "", "fm_empty"); +AssertEqStr(fm_str(fm_empty(1)), "", "fm_empty"); +AssertEqStr(fm_str(fm_empty(1 , 3)), "", "fm_empty"); + +AssertEqStr(fm_str(fm_comma), "fm_comma", "fm_comma"); +AssertEqStr(fm_str(fm_comma()), ",", "fm_comma"); +AssertEqStr(fm_str(fm_comma(xx,xx)), ",", "fm_comma"); +AssertEqStr(fm_str(fm__comma), ",", "fm_comma"); + +AssertEqStr(fm_str(fm_apply(fm_expand)), "", "fm_apply"); +AssertEqStr(fm_str(fm_apply(fm_expand, 1)), "1", "fm_apply"); +AssertEqStr(fm_str(fm_apply(fm_expand, 1, 2)), "1, 2", "fm_apply"); +AssertEqStr(fm_str(fm_apply(fm_expand, 1,2)), "1,2", "fm_apply"); +AssertEqStr(fm_str(fm_apply(fm_expand, 1 ,2)), "1 ,2", "fm_apply"); +AssertEqStr(fm_str(fm_apply(fm_cat, 1 ,2)), "12", "fm_apply"); +AssertEqStr(fm_str(fm_apply(fm_cat, 1, 2)), "12", "fm_apply"); +AssertEqStr(fm_str(fm_apply(fm_cat3, x, y, z)), "xyz", "fm_apply"); +AssertEqStr(fm_str(fm_apply(fm_comma, ())), ",", "fm_apply"); + +AssertEq(fm_compl(1), 0, "fm_compl"); +AssertEq(fm_compl(0), 1, "fm_compl"); +AssertEq(fm_compl(fm_true), 0, "fm_compl"); +AssertEq(fm_compl(fm_false), 1, "fm_compl"); + +AssertEq(fm_and(0, 0), 0, "fm_and"); +AssertEq(fm_and(0, 1), 0, "fm_and"); +AssertEq(fm_and(1, 0), 0, "fm_and"); +AssertEq(fm_and(1, 1), 1, "fm_and"); +AssertEq(fm_and(fm_false, fm_false), fm_false, "fm_and"); +AssertEq(fm_and(fm_false, fm_true), fm_false, "fm_and"); +AssertEq(fm_and(fm_true, fm_false), fm_false, "fm_and"); +AssertEq(fm_and(fm_true, fm_true), fm_true, "fm_and"); + +AssertEq(fm_or(0, 0), 0, "fm_or"); +AssertEq(fm_or(0, 1), 1, "fm_or"); +AssertEq(fm_or(1, 0), 1, "fm_or"); +AssertEq(fm_or(1, 1), 1, "fm_or"); +AssertEq(fm_or(fm_false, fm_false), fm_false, "fm_or"); +AssertEq(fm_or(fm_false, fm_true), fm_true, "fm_or"); +AssertEq(fm_or(fm_true, fm_false), fm_true, "fm_or"); +AssertEq(fm_or(fm_true, fm_true), fm_true, "fm_or"); + +AssertEq(fm_nand(0, 0), 1, "fm_nand"); +AssertEq(fm_nand(0, 1), 1, "fm_nand"); +AssertEq(fm_nand(1, 0), 1, "fm_nand"); +AssertEq(fm_nand(1, 1), 0, "fm_nand"); +AssertEq(fm_nand(fm_false, fm_false), fm_true, "fm_nand"); +AssertEq(fm_nand(fm_false, fm_true), fm_true, "fm_nand"); +AssertEq(fm_nand(fm_true, fm_false), fm_true, "fm_nand"); +AssertEq(fm_nand(fm_true, fm_true), fm_false, "fm_nand"); + +AssertEq(fm_nor(0, 0), 1, "fm_nor"); +AssertEq(fm_nor(0, 1), 0, "fm_nor"); +AssertEq(fm_nor(1, 0), 0, "fm_nor"); +AssertEq(fm_nor(1, 1), 0, "fm_nor"); +AssertEq(fm_nor(fm_false, fm_false), fm_true, "fm_nor"); +AssertEq(fm_nor(fm_false, fm_true), fm_false, "fm_nor"); +AssertEq(fm_nor(fm_true, fm_false), fm_false, "fm_nor"); +AssertEq(fm_nor(fm_true, fm_true), fm_false, "fm_nor"); + +AssertEq(fm_xor(0, 0), 0, "fm_xor"); +AssertEq(fm_xor(0, 1), 1, "fm_xor"); +AssertEq(fm_xor(1, 0), 1, "fm_xor"); +AssertEq(fm_xor(1, 1), 0, "fm_xor"); +AssertEq(fm_xor(fm_false, fm_false), fm_false, "fm_xor"); +AssertEq(fm_xor(fm_false, fm_true), fm_true, "fm_xor"); +AssertEq(fm_xor(fm_true, fm_false), fm_true, "fm_xor"); +AssertEq(fm_xor(fm_true, fm_true), fm_false, "fm_xor"); + +AssertEq(fm_if(fm_true, 3, 4), 3, "fm_if"); +AssertEq(fm_if(fm_false, 3, 4), 4, "fm_if"); +AssertEqStr(fm_str(fm_if(fm_false, 3, 4)), "4", "fm_if"); +AssertEqStr(fm_str(fm_if(fm_false, 3, 4, 5)), "4, 5", "fm_if"); + +AssertEqStr(fm_str(fm_when(fm_true)(3, 4)), "3, 4", "fm_when"); +AssertEqStr(fm_str(fm_when(fm_false)(3, 4)), "", "fm_when"); + +AssertEqStr(fm_str(fm_iif(fm_true)(3, 4)(5, 6)), "3, 4", "fm_iif"); +AssertEqStr(fm_str(fm_iif(fm_false)(3, 4)(5, 6)), "5, 6", "fm_iif"); + +#define COMPARE_FOO(x) x +#define COMPARE_BAR(x) x +AssertEq(fm_equal(FOO, FOO), 1, "fm_equal"); +AssertEq(fm_equal(BAR, BAR), 1, "fm_equal"); +AssertEq(fm_equal(FOO, BAR), 0, "fm_equal"); +AssertEq(fm_equal(BAR, FOO), 0, "fm_equal"); +AssertEq(fm_equal(BAR, BAZ), 0, "fm_equal"); +AssertEq(fm_equal(BAZ, BAR), 0, "fm_equal"); +AssertEq(fm_equal(BAZ, BAD), 0, "fm_equal"); + +AssertEq(fm_not_equal(FOO, FOO), 0, "fm_not_equal"); +AssertEq(fm_not_equal(BAR, BAR), 0, "fm_not_equal"); +AssertEq(fm_not_equal(FOO, BAR), 1, "fm_not_equal"); +AssertEq(fm_not_equal(BAR, FOO), 1, "fm_not_equal"); +AssertEq(fm_not_equal(BAR, BAZ), 1, "fm_not_equal"); +AssertEq(fm_not_equal(BAZ, BAR), 1, "fm_not_equal"); +AssertEq(fm_not_equal(BAZ, BAD), 1, "fm_not_equal"); + +AssertEq(fm_head(2), 2, "fm_head"); +AssertEq(fm_head(2, 3), 2, "fm_head"); +AssertEq(fm_head(2, 3, 4), 2, "fm_head"); +AssertEq(fm_head(2, 3, 4), 2, "fm_head"); +AssertEqStr(fm_str(fm_head()), "", "fm_head"); +AssertEqStr(fm_str(fm_head(, 1)), "", "fm_head"); +AssertEqStr(fm_str(fm_head(fm__comma)), "", "fm_head"); +AssertEqStr(fm_str(fm_head(fm__comma, 1)), "", "fm_head"); +AssertEqStr(fm_str(fm_head(fm_comma(), 1)), "", "fm_head"); +AssertEqStr(fm_str(fm_tail(2)), "", "fm_head"); +AssertEqStr(fm_str(fm_tail(2, 3)), "3", "fm_head"); +AssertEqStr(fm_str(fm_tail(2, 3, 4)), "3, 4", "fm_head"); + +AssertEq(fm_va_single(), 1, "fm_va_single"); +AssertEq(fm_va_single(1), 1, "fm_va_single"); +AssertEq(fm_va_single(,), 0, "fm_va_single"); +AssertEq(fm_va_single(fm_expand()), 1, "fm_va_single"); +AssertEq(fm_va_single(fm_expand(1)), 1, "fm_va_single"); +AssertEq(fm_va_single(fm_expand(,)), 0, "fm_va_single"); + +AssertEq(fm_va_many(), 0, "fm_va_many"); +AssertEq(fm_va_many(1), 0, "fm_va_many"); +AssertEq(fm_va_many(,), 1, "fm_va_many"); +AssertEq(fm_va_many(fm_expand()), 0, "fm_va_many"); +AssertEq(fm_va_many(fm_expand(1)), 0, "fm_va_many"); +AssertEq(fm_va_many(fm_expand(,)), 1, "fm_va_many"); + +AssertEq(fm_no_va(), 1, "fm_no_va"); +AssertEq(fm_no_va(fm_empty()), 1, "fm_no_va"); +AssertEq(fm_no_va(fm_empty(1)), 1, "fm_no_va"); +AssertEq(fm_no_va(1), 0, "fm_no_va"); +AssertEq(fm_no_va(,), 0, "fm_no_va"); +AssertEq(fm_no_va(,1), 0, "fm_no_va"); + +AssertEq(fm_va_01(), 0, "fm_va_01"); +AssertEq(fm_va_01(fm_empty()), 0, "fm_va_01"); +AssertEq(fm_va_01(fm_empty(1)), 0, "fm_va_01"); +AssertEq(fm_va_01(1), 1, "fm_va_01"); +AssertEq(fm_va_01(,), 1, "fm_va_01"); +AssertEq(fm_va_01(,1), 1, "fm_va_01"); + +AssertEq(fm_va_01n(), 0, "fm_va_01n"); +AssertEq(fm_va_01n(fm_empty()), 0, "fm_va_01n"); +AssertEq(fm_va_01n(x), 1, "fm_va_01n"); +AssertEq(fm_va_01n(fm_cat(x, y)), 1, "fm_va_01n"); +AssertEq(fm_va_01n(fm_head(x, y)), 1, "fm_va_01n"); +AssertEq(fm_va_01n(fm_tail(x)), 0, "fm_va_01n"); +AssertEq(fm_va_01n(fm_tail(x, y)), 1, "fm_va_01n"); +AssertEqStr(fm_str(fm_va_01n(,)), "n", "fm_va_01n"); +AssertEqStr(fm_str(fm_va_01n(x,)), "n", "fm_va_01n"); +AssertEqStr(fm_str(fm_va_01n(1,2)), "n", "fm_va_01n"); +AssertEqStr(fm_str(fm_va_01n(fm_tail(1,2,3))), "n", "fm_va_01n"); + +AssertEq(fm_or_default()(5), 5, "fm_or_default"); +AssertEq(fm_or_default(4)(5), 4, "fm_or_default"); +AssertEqStr(fm_str(fm_or_default()(5, 6)), "5, 6", "fm_or_default"); +AssertEqStr(fm_str(fm_or_default(3, 4)(5, 6)), "3, 4", "fm_or_default"); + +AssertEqStr(fm_str(fm_when_isnt_empty()(5)), "", "fm_when_isnt_empty"); +AssertEqStr(fm_str(fm_when_isnt_empty(fm_empty())(5)), "", "fm_when_isnt_empty"); +AssertEqStr(fm_str(fm_when_isnt_empty(1)(5)), "5", "fm_when_isnt_empty"); +AssertEqStr(fm_str(fm_when_isnt_empty(1)(5, 6)), "5, 6", "fm_when_isnt_empty"); + +AssertEq(fm_is_tuple(), 0, "fm_is_tuple"); +AssertEq(fm_is_tuple(1), 0, "fm_is_tuple"); +AssertEq(fm_is_tuple(1, 2), 0, "fm_is_tuple"); +AssertEq(fm_is_tuple(,), 0, "fm_is_tuple"); +AssertEq(fm_is_tuple(()), 1, "fm_is_tuple"); +AssertEq(fm_is_tuple((,)), 1, "fm_is_tuple"); +AssertEq(fm_is_tuple((fm_comma)), 1, "fm_is_tuple"); + +#define add_x(y) y##x +#define add_ax(a, y) y##a##x +AssertEqStr(fm_str(fm_eval_foreach(add_x)), + "", "fm_eval_foreach"); +AssertEqStr(fm_str(fm_eval_foreach(add_x, a)), + "ax", "fm_eval_foreach"); +AssertEqStr(fm_str(fm_eval_foreach(add_x, a, b)), + "ax bx", "fm_eval_foreach"); +AssertEqStr(fm_str(fm_eval_foreach(add_x, a, b, c)), + "ax bx cx", "fm_eval_foreach"); +AssertEqStr(fm_str(fm_eval_foreach(add_x, a, b, c, d)), + "ax bx cx dx", "fm_eval_foreach"); +AssertEqStr(fm_str(fm_eval_foreach(add_x, a, b, c, d, e)), + "ax bx cx dx ex", "fm_eval_foreach"); + +AssertEqStr(fm_str(fm_eval_foreach_comma(add_x)), + "", "fm_eval_foreach_comma"); +AssertEqStr(fm_str(fm_eval_foreach_comma(add_x, a)), + "ax", "fm_eval_foreach_comma"); +AssertEqStr(fm_str(fm_eval_foreach_comma(add_x, a, b)), + "ax , bx", "fm_eval_foreach_comma"); +AssertEqStr(fm_str(fm_eval_foreach_comma(add_x, a, b, c)), + "ax , bx , cx", "fm_eval_foreach_comma"); +AssertEqStr(fm_str(fm_eval_foreach_comma(add_x, a, b, c, d)), + "ax , bx , cx , dx", "fm_eval_foreach_comma"); +AssertEqStr(fm_str(fm_eval_foreach_comma(add_x, a, b, c, d, e)), + "ax , bx , cx , dx , ex", "fm_eval_foreach_comma"); + +AssertEqStr(fm_str(fm_eval_foreach_arg(add_ax, Z)), + "", "fm_eval_foreach_arg"); +AssertEqStr(fm_str(fm_eval_foreach_arg(add_ax, Z, a)), + "aZx", "fm_eval_foreach_arg"); +AssertEqStr(fm_str(fm_eval_foreach_arg(add_ax, Z, a, b)), + "aZx bZx", "fm_eval_foreach_arg"); +AssertEqStr(fm_str(fm_eval_foreach_arg(add_ax, Z, a, b, c)), + "aZx bZx cZx", "fm_eval_foreach_arg"); +AssertEqStr(fm_str(fm_eval_foreach_arg(add_ax, Z, a, b, c, d)), + "aZx bZx cZx dZx", "fm_eval_foreach_arg"); +AssertEqStr(fm_str(fm_eval_foreach_arg(add_ax, Z, a, b, c, d, e)), + "aZx bZx cZx dZx eZx", "fm_eval_foreach_arg"); + +#define map_tuple(t, ...) map_tuple_##t(__VA_ARGS__) +#define map_tuple_k(x) fm_cat(x, K) +#define map_tuple_n(x, y) fm_cat3(x, y, N) + +#define map_tuple_a(a, t, ...) map_tuple_a##t(a, __VA_ARGS__) +#define map_tuple_ak(a, x) fm_cat3(x, a, K) +#define map_tuple_an(a, x, y) fm_cat4(x, a, y, N) + +AssertEqStr(fm_str(fm_eval_tuples(map_tuple)), + "", "fm_eval_tuples"); +AssertEqStr(fm_str(fm_eval_tuples(map_tuple, (k, a))), + "aK", "fm_eval_tuples"); +AssertEqStr(fm_str(fm_eval_tuples(map_tuple, (n, a, b))), + "abN", "fm_eval_tuples"); +AssertEqStr(fm_str(fm_eval_tuples(map_tuple, (k, a), (n, a, b))), + "aK abN", "fm_eval_tuples"); +AssertEqStr(fm_str(fm_eval_tuples(map_tuple, (k, a), (n, a, b), (k, a))), + "aK abN aK", "fm_eval_tuples"); +AssertEqStr(fm_str(fm_eval_tuples(map_tuple, (k, a), (n, a, b), (k, c))), + "aK abN cK", "fm_eval_tuples"); +AssertEqStr(fm_str(fm_eval_tuples(map_tuple, (k, a), (n, a, b), (k, c), (n, c, d))), + "aK abN cK cdN", "fm_eval_tuples"); +AssertEqStr(fm_str(fm_eval_tuples(map_tuple, (k, a), (n, a, b), (k, c), (n, c, d), (k, e))), + "aK abN cK cdN eK", "fm_eval_tuples"); + +AssertEqStr(fm_str(fm_eval_tuples_comma(map_tuple)), + "", "fm_eval_tuples_comma"); +AssertEqStr(fm_str(fm_eval_tuples_comma(map_tuple, (k, a))), + "aK", "fm_eval_tuples_comma"); +AssertEqStr(fm_str(fm_eval_tuples_comma(map_tuple, (n, a, b))), + "abN", "fm_eval_tuples_comma"); +AssertEqStr(fm_str(fm_eval_tuples_comma(map_tuple, (k, a), (n, a, b))), + "aK , abN", "fm_eval_tuples_comma"); +AssertEqStr(fm_str(fm_eval_tuples_comma(map_tuple, (k, a), (n, a, b), (k, c))), + "aK , abN , cK", "fm_eval_tuples_comma"); +AssertEqStr(fm_str(fm_eval_tuples_comma(map_tuple, (k, a), (n, a, b), (k, c), (n, c, d))), + "aK , abN , cK , cdN", "fm_eval_tuples_comma"); +AssertEqStr(fm_str(fm_eval_tuples_comma(map_tuple, (k, a), (n, a, b), (k, c), (n, c, d), (k, e))), + "aK , abN , cK , cdN , eK", "fm_eval_tuples_comma"); + +AssertEqStr(fm_str(fm_eval_tuples_arg(map_tuple_a, Y)), + "", "fm_eval_tuples_arg"); +AssertEqStr(fm_str(fm_eval_tuples_arg(map_tuple_a, Y, (k, a))), + "aYK", "fm_eval_tuples_arg"); +AssertEqStr(fm_str(fm_eval_tuples_arg(map_tuple_a, Y, (n, a, b))), + "aYbN", "fm_eval_tuples_arg"); +AssertEqStr(fm_str(fm_eval_tuples_arg(map_tuple_a, Y, (k, a), (n, a, b))), + "aYK aYbN", "fm_eval_tuples_arg"); +AssertEqStr(fm_str(fm_eval_tuples_arg(map_tuple_a, Y, (k, a), (n, a, b), (k, c))), + "aYK aYbN cYK", "fm_eval_tuples_arg"); +AssertEqStr(fm_str(fm_eval_tuples_arg(map_tuple_a, Y, (k, a), (n, a, b), (k, c), (n, c, d))), + "aYK aYbN cYK cYdN", "fm_eval_tuples_arg"); +AssertEqStr(fm_str(fm_eval_tuples_arg(map_tuple_a, Y, (k, a), (n, a, b), (k, c), (n, c, d), (k, e))), + "aYK aYbN cYK cYdN eYK", "fm_eval_tuples_arg"); + +} \ No newline at end of file diff --git a/src/fu_util/test/fuprintf.c b/src/fu_util/test/fuprintf.c new file mode 100644 index 000000000..73f680da7 --- /dev/null +++ b/src/fu_util/test/fuprintf.c @@ -0,0 +1,26 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#include +#include +#include + +int main(void) { + ft_str_t msg = ft_asprintf("asdf %d asdf\n", 123456); + const char *cmp = "asdf 123456 asdf\n"; + + ft_assert(msg.ptr != NULL); + ft_assert(msg.len == strlen(cmp)); + ft_assert(strcmp(msg.ptr, cmp) == 0); + + for (int i = 0; i < 10; i++) { + ft_str_t newmsg = ft_asprintf("%s%s", msg.ptr, msg.ptr); + ft_free((char*)msg.ptr); + msg = newmsg; + ft_assert(msg.ptr != NULL); + ft_assert(msg.len == strlen(cmp) * (1 << (i+1))); + } + + ft_free((char*)msg.ptr); + + return 0; +} + diff --git a/src/fu_util/test/obj1.c b/src/fu_util/test/obj1.c new file mode 100644 index 000000000..e47115048 --- /dev/null +++ b/src/fu_util/test/obj1.c @@ -0,0 +1,291 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#include +#include + +#include +#include + +static int verbose = 0; +#define logf(...) ft_log(FT_DEBUG, __VA_ARGS__) + +#define mth__ioRead ssize_t, (void *, buf), (size_t, count, 4) +#define mth__ioClose int +#define mth__ioStatus int +#define mth__fobjGetError err_i +fobj_method(ioRead); +fobj_method(ioClose); +fobj_method(ioStatus); +fobj_method(fobjGetError); + +#define iface__ioReader mth(ioRead) +#define iface__ioReadCloser iface__ioReader, mth(ioClose, ioStatus) +#define iface__obj +fobj_iface(ioReadCloser); +fobj_iface(ioReader); +fobj_iface(obj); + +#define kls__Klass0 mth(fobjDispose), \ + iface__ioReader, mth(fobjGetError) +#define kls__KlassA inherits(Klass0), \ + iface__ioReadCloser, \ + mth(ioStatus), iface(ioReadCloser, ioReader, ioRead) + +fobj_klass(Klass0); +fobj_klass(KlassA); + +typedef struct Klass0 { + int x; +} Klass0; + +typedef struct KlassA { + Klass0 p; + size_t offset; +} KlassA; + +static void +Klass0_fobjDispose(VSelf) { + Self(Klass0); + logf("{.x = %d}", self->x); +} + +static ssize_t +Klass0_ioRead(VSelf, void *buf, size_t count) { + Self(Klass0); + logf("{.x = %d}, .count = %zd", self->x, count); + self->x += 1; + return count; +} + +fobj_error_int_key(myx); +fobj_error_float_key(myy); + +static err_i +Klass0_fobjGetError(VSelf) { + Self(Klass0); + return $err(RT, "WTF ERROR {myx:05d} {myy:9.4f}", myx(self->x), myy(100.001)); +} + +static int +KlassA_ioClose(VSelf) { + //Self(KlassA); + return 0; +} + +static ssize_t +KlassA_ioRead(VSelf, void *buf, size_t count) { + Self(KlassA); + logf("p{.offset = %zd}, .count = %zd", + self->offset, count); + self->offset += count; + $super(ioRead, self, buf, count); + return count; +} + +static int +KlassA_ioStatus(VSelf) { + Self(KlassA); + logf("{.offset = %zd}", self->offset); + return (int)self->offset; +} + +static void +KlassA_fobjDispose(VSelf) { + Self(KlassA); + logf("{.offset = %zd}", self->offset); +} + +fobj_klass_handle(KlassA, mth(fobjDispose), iface(obj)); +fobj_klass_handle(Klass0); + +int main(int argc, char** argv) { + ft_init_log(NULL); + fobj_init(); + + FOBJ_FUNC_ARP(); + ft_assert(fobj__func_ar_pool.last != NULL); + + char b[1024]; + int benchmode = 0, benchcnt = 0; + int i; + + verbose = atoi(getenv("VERBOSE") ?: "0"); + benchcnt = atoi(getenv("BENCHCNT") ?: "0"); + benchmode = atoi(getenv("BENCHMODE") ?: "0"); + + if (verbose) { + //ft_log_level_reset(FT_LOG); + ft_log_level_set(__FILE__, FT_DEBUG); + } else { + ft_log_level_set("ALL", FT_ERROR); + } + + fobj_klass_init(Klass0); + fobj_klass_init(KlassA); + fobj_add_methods(KlassA, ioRead); + + fobj_freeze(); + + KlassA *a = $alloc(KlassA, .offset = 1, .p.x = 2); + logf("a=%s", $repr(a)); + + logf("Before block 1 enter"); + { + FOBJ_BLOCK_ARP(); + KlassA *d; + fobj_t e; + logf("Before block 2 enter"); + { + FOBJ_BLOCK_ARP(); + KlassA *c = $alloc(KlassA, .p.x = 55555); + d = $alloc(KlassA, .p.x = 12345); + e = $alloc(KlassA, .p.x = 67890); + $unref($ref(c)); /* incref and store in current ARP */ + $save(d); /* store in outter ARP */ + $ref(e); /* explicit reference increment */ + logf("Before block 2 exits"); + } + logf("After block 2 exited"); + /* $set is needed only if variable is explicitely managed with $ref/$del */ + $set(&e, $alloc(KlassA, .p.x = 67891)); + $swap(&e, $alloc(KlassA, .p.x = 78912)); + $del(&e); /* explicit reference decrement */ + logf("Before block 1 exits"); + } + logf("After block 1 exited"); + + ioRead_i aird = bind_ioRead(a); + $i(ioRead, aird, b, 100); + $i(ioRead, aird, b); + // will fail in runtime with param.buf__given != NULL + //$i(ioRead, aird, .count = 100); + $i(ioRead, aird, .buf = b, .count = 100); + $i(ioRead, aird, .buf = b); + + ioReader_i ard = bind_ioReader(a); + $i(ioRead, ard, b, 100); + $i(ioRead, ard, .buf = b, .count = 100); + + ard = $bind(ioReader, a); + aird = $bind(ioRead, ard.self); + aird = $reduce(ioRead, ard); + ard = $reduce(ioReader, aird); + + ioReadCloser_i ardcl = bind_ioReadCloser(a); + ardcl = $reduce(ioReadCloser, ardcl); + ard = $reduce(ioReader, ardcl); + aird = $reduce(ioRead, ardcl); + + ioRead(a, b, 100); + $(ioRead, a, b, 100); + $(ioRead, a, .buf = b, .count = 100); + + $(ioStatus, a); + + aird = (ioRead_i){NULL}; + ard = (ioReader_i){NULL}; + + err_i err = $err(RT, "ha"); + + ft_assert(!$implements(ioRead, err.self)); + ft_assert(!$implements(ioRead, err.self, &aird)); + ft_assert(!$ifilled(ioRead, aird)); + ft_assert(!$implements(ioReader, err.self)); + ft_assert(!$implements(ioReader, err.self, &ard)); + ft_assert(!$ifilled(ioRead, ard)); + + ft_assert($implements(ioRead, a)); + ft_assert($implements(ioRead, a, &aird)); + ft_assert($ifilled(ioRead, aird)); + ft_assert($implements(ioReader, a)); + ft_assert($implements(ioReader, a, &ard)); + ft_assert($ifilled(ioRead, ard)); + + i = ioStatus(a) - 1; + ft_assert($ifdef(,ioStatus, a)); + ft_assert(i != ioStatus(a)); + ft_assert($ifdef(i =, ioStatus, a)); + ft_assert(i == ioStatus(a)); + ft_assert(!$ifdef(,fobjFormat, a, NULL)); + + err = $(fobjGetError, a); + logf("Error: %s", $errmsg(err)); + logf("Error: %s", $itostr(err, NULL)); + logf("Error: %s", $itostr(err, "$T $M $K")); + ioRead(a, b, strlen($errmsg(err))); + $(ioRead, a, b, strlen($errmsg(err))); + $(ioRead, a, b, $(ioRead, a, b, $(ioStatus, a))); + logf("Error: %s", $errmsg($(fobjGetError, a))); + + err = $syserr(ENOENT); + logf("Error: %s", $errmsg(err)); + logf("Error: %s", $irepr(err)); + errno = ENOENT; + err = $syserr(errno, "Opening file"); + logf("Error: %s", $errmsg(err)); + logf("Error: %s", $irepr(err)); + err = $syserr(ENOENT, "Opening file {path}", path("folder/read.me")); + logf("Error: %s", $errmsg(err)); + logf("Error: %s", $irepr(err)); + logf("Errno: %d", getErrno(err)); + + Klass0 *k0 = $alloc(Klass0); + aird = bind_ioRead(k0); + ioRead__cb_t k0_ioRead = ioRead__fetch_cb(k0, fobj_self_klass, true); + for (i = 0; i < benchcnt; i++) { + switch (benchmode) { + case 0: ioRead(k0, b, 100); break; + case 1: $(ioRead, k0, b, 100); break; + case 2: $i(ioRead, aird, b, 100); break; + case 3: fobj_cb_fastcall(k0_ioRead, b, 100); break; + } + } + + $ref(a); + { fobj_t b = a; $del(&b); } + $(ioStatus, a); + + { + ioRead_i bird = $null(ioRead); + $iset(&bird, aird); + $iswap(&bird, aird); + $iref(bird); + $iunref(bird); + $idel(&bird); + } + + fobjStr *stra = $S("this is string a"); + fobjStr *strb = $S("this is b"); + + ft_assert(fobj_streq_c(stra, "this is string a")); + ft_assert(fobj_streq_c(strb, "this is b")); + + fobjStr *strc = fobj_strcatc(stra, "??????"); + fobjStr *strd = fobj_strcatc(strb, "!!"); + + ft_assert(fobj_streq_c(strc, "this is string a??????")); + ft_assert(fobj_streq_c(strd, "this is b!!")); + + fobjStr *stre = fobj_stradd(strc, strd); + + ft_assert(fobj_streq_c(stre, "this is string a??????this is b!!")); + + stre = fobj_sprintf("%s:%d", "hello", 1); + + ft_assert(fobj_streq_c(stre, "hello:1")); + + stre = fobj_strcatf(stre, "/%d/%s", 100, "goodbye"); + + ft_assert(fobj_streq_c(stre, "hello:1/100/goodbye")); + + fobjStr *strf = $fmt("Some {usual:8s} things cost > $${money:-8.4f}$$"); + ft_assert(fobj_streq_c(strf, "Some things cost > $$$$")); + strf = $fmt("Some {usual:8s} things cost > $${money:-8.4f}$$", + (usual, $S("scary")), (money, $F(12.48))); + ft_assert(fobj_streq_c(strf, "Some scary things cost > $$12.4800 $$"), + "String is '%s'", $tostr(strf)); + + ft_log(FT_ERROR, "and try backtrace"); + logf("BEFORE EXIT"); +} + +ft_register_source(); diff --git a/src/fu_util/test/qsort/qsort.inc.c b/src/fu_util/test/qsort/qsort.inc.c new file mode 100644 index 000000000..2a53ae93f --- /dev/null +++ b/src/fu_util/test/qsort/qsort.inc.c @@ -0,0 +1,248 @@ +/* Copyright (C) 1991-2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Written by Douglas C. Schmidt (schmidt@ics.uci.edu). + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* If you consider tuning this algorithm, you should consult first: + Engineering a sort function; Jon Bentley and M. Douglas McIlroy; + Software - Practice and Experience; Vol. 23 (11), 1249-1265, 1993. */ + +#include +#include +#include + +/* Byte-wise swap two items of size SIZE. */ +#define SWAP(a, b, size) \ + do \ + { \ + size_t __size = (size); \ + char *__a = (a), *__b = (b); \ + do \ + { \ + char __tmp = *__a; \ + *__a++ = *__b; \ + *__b++ = __tmp; \ + } while (--__size > 0); \ + } while (0) + +/* Discontinue quicksort algorithm when partition gets below this size. + This particular magic number was chosen to work best on a Sun 4/260. */ +#define MAX_THRESH 4 + +/* Stack node declarations used to store unfulfilled partition obligations. */ +typedef struct + { + char *lo; + char *hi; + } stack_node; + +/* The next 4 #defines implement a very fast in-line stack abstraction. */ +/* The stack needs log (total_elements) entries (we could even subtract + log(MAX_THRESH)). Since total_elements has type size_t, we get as + upper bound for log (total_elements): + bits per byte (CHAR_BIT) * sizeof(size_t). */ +#define STACK_SIZE (CHAR_BIT * sizeof(size_t)) +#define PUSH(low, high) ((void) ((top->lo = (low)), (top->hi = (high)), ++top)) +#define POP(low, high) ((void) (--top, (low = top->lo), (high = top->hi))) +#define STACK_NOT_EMPTY (stack < top) + + +/* Order size using quicksort. This implementation incorporates + four optimizations discussed in Sedgewick: + + 1. Non-recursive, using an explicit stack of pointer that store the + next array partition to sort. To save time, this maximum amount + of space required to store an array of SIZE_MAX is allocated on the + stack. Assuming a 32-bit (64 bit) integer for size_t, this needs + only 32 * sizeof(stack_node) == 256 bytes (for 64 bit: 1024 bytes). + Pretty cheap, actually. + + 2. Chose the pivot element using a median-of-three decision tree. + This reduces the probability of selecting a bad pivot value and + eliminates certain extraneous comparisons. + + 3. Only quicksorts TOTAL_ELEMS / MAX_THRESH partitions, leaving + insertion sort to order the MAX_THRESH items within each partition. + This is a big win, since insertion sort is faster for small, mostly + sorted array segments. + + 4. The larger of the two sub-partitions is always pushed onto the + stack first, with the algorithm then concentrating on the + smaller partition. This *guarantees* no more than log (total_elems) + stack size is needed (actually O(1) in this case)! */ + +void +_quicksort (void *const pbase, size_t total_elems, size_t size, + int (*cmp)(const void *, const void*, void*), void *arg) +{ + char *base_ptr = (char *) pbase; + + const size_t max_thresh = MAX_THRESH * size; + + if (total_elems == 0) + /* Avoid lossage with unsigned arithmetic below. */ + return; + + if (total_elems > MAX_THRESH) + { + char *lo = base_ptr; + char *hi = &lo[size * (total_elems - 1)]; + stack_node stack[STACK_SIZE]; + stack_node *top = stack; + + PUSH (NULL, NULL); + + while (STACK_NOT_EMPTY) + { + char *left_ptr; + char *right_ptr; + + /* Select median value from among LO, MID, and HI. Rearrange + LO and HI so the three values are sorted. This lowers the + probability of picking a pathological pivot value and + skips a comparison for both the LEFT_PTR and RIGHT_PTR in + the while loops. */ + + char *mid = lo + size * ((hi - lo) / size >> 1); + + if ((*cmp) ((void *) mid, (void *) lo, arg) < 0) + SWAP (mid, lo, size); + if ((*cmp) ((void *) hi, (void *) mid, arg) < 0) + SWAP (mid, hi, size); + else + goto jump_over; + if ((*cmp) ((void *) mid, (void *) lo, arg) < 0) + SWAP (mid, lo, size); + jump_over:; + + left_ptr = lo + size; + right_ptr = hi - size; + + /* Here's the famous ``collapse the walls'' section of quicksort. + Gotta like those tight inner loops! They are the main reason + that this algorithm runs much faster than others. */ + do + { + while ((*cmp) ((void *) left_ptr, (void *) mid, arg) < 0) + left_ptr += size; + + while ((*cmp) ((void *) mid, (void *) right_ptr, arg) < 0) + right_ptr -= size; + + if (left_ptr < right_ptr) + { + SWAP (left_ptr, right_ptr, size); + if (mid == left_ptr) + mid = right_ptr; + else if (mid == right_ptr) + mid = left_ptr; + left_ptr += size; + right_ptr -= size; + } + else if (left_ptr == right_ptr) + { + left_ptr += size; + right_ptr -= size; + break; + } + } + while (left_ptr <= right_ptr); + + /* Set up pointers for next iteration. First determine whether + left and right partitions are below the threshold size. If so, + ignore one or both. Otherwise, push the larger partition's + bounds on the stack and continue sorting the smaller one. */ + + if ((size_t) (right_ptr - lo) <= max_thresh) + { + if ((size_t) (hi - left_ptr) <= max_thresh) + /* Ignore both small partitions. */ + POP (lo, hi); + else + /* Ignore small left partition. */ + lo = left_ptr; + } + else if ((size_t) (hi - left_ptr) <= max_thresh) + /* Ignore small right partition. */ + hi = right_ptr; + else if ((right_ptr - lo) > (hi - left_ptr)) + { + /* Push larger left partition indices. */ + PUSH (lo, right_ptr); + lo = left_ptr; + } + else + { + /* Push larger right partition indices. */ + PUSH (left_ptr, hi); + hi = right_ptr; + } + } + } + + /* Once the BASE_PTR array is partially sorted by quicksort the rest + is completely sorted using insertion sort, since this is efficient + for partitions below MAX_THRESH size. BASE_PTR points to the beginning + of the array to sort, and END_PTR points at the very last element in + the array (*not* one beyond it!). */ + +#define min(x, y) ((x) < (y) ? (x) : (y)) + + { + char *const end_ptr = &base_ptr[size * (total_elems - 1)]; + char *tmp_ptr = base_ptr; + char *thresh = min(end_ptr, base_ptr + max_thresh); + char *run_ptr; + + /* Find smallest element in first threshold and place it at the + array's beginning. This is the smallest array element, + and the operation speeds up insertion sort's inner loop. */ + + for (run_ptr = tmp_ptr + size; run_ptr <= thresh; run_ptr += size) + if ((*cmp) ((void *) run_ptr, (void *) tmp_ptr, arg) < 0) + tmp_ptr = run_ptr; + + if (tmp_ptr != base_ptr) + SWAP (tmp_ptr, base_ptr, size); + + /* Insertion sort, running from left-hand-side up to right-hand-side. */ + + run_ptr = base_ptr + size; + while ((run_ptr += size) <= end_ptr) + { + tmp_ptr = run_ptr - size; + while ((*cmp) ((void *) run_ptr, (void *) tmp_ptr, arg) < 0) + tmp_ptr -= size; + + tmp_ptr += size; + if (tmp_ptr != run_ptr) + { + char *trav; + + trav = run_ptr + size; + while (--trav >= run_ptr) + { + char c = *trav; + char *hi, *lo; + + for (hi = lo = trav; (lo -= size) >= tmp_ptr; hi = lo) + *hi = *lo; + *hi = c; + } + } + } + } +} diff --git a/src/fu_util/test/qsort/qsort_pg.inc.c b/src/fu_util/test/qsort/qsort_pg.inc.c new file mode 100644 index 000000000..0d3221aff --- /dev/null +++ b/src/fu_util/test/qsort/qsort_pg.inc.c @@ -0,0 +1,21 @@ +/* + * qsort.c: standard quicksort algorithm + */ +#include + +#define ST_SORT pg_qsort +#define ST_ELEMENT_TYPE_VOID +#define ST_COMPARE_RUNTIME_POINTER +#define ST_SCOPE +#define ST_DECLARE +#define ST_DEFINE +#include "./sort_template.h" + +/* + * qsort comparator wrapper for strcmp. + */ +int +pg_qsort_strcmp(const void *a, const void *b) +{ + return strcmp(*(const char *const *) a, *(const char *const *) b); +} diff --git a/src/fu_util/test/qsort/sort_template.h b/src/fu_util/test/qsort/sort_template.h new file mode 100644 index 000000000..f204bf835 --- /dev/null +++ b/src/fu_util/test/qsort/sort_template.h @@ -0,0 +1,435 @@ +/*------------------------------------------------------------------------- + * + * sort_template.h + * + * A template for a sort algorithm that supports varying degrees of + * specialization. + * + * Copyright (c) 2021-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1992-1994, Regents of the University of California + * + * Usage notes: + * + * To generate functions specialized for a type, the following parameter + * macros should be #define'd before this file is included. + * + * - ST_SORT - the name of a sort function to be generated + * - ST_ELEMENT_TYPE - type of the referenced elements + * - ST_DECLARE - if defined the functions and types are declared + * - ST_DEFINE - if defined the functions and types are defined + * - ST_SCOPE - scope (e.g. extern, static inline) for functions + * - ST_CHECK_FOR_INTERRUPTS - if defined the sort is interruptible + * + * Instead of ST_ELEMENT_TYPE, ST_ELEMENT_TYPE_VOID can be defined. Then + * the generated functions will automatically gain an "element_size" + * parameter. This allows us to generate a traditional qsort function. + * + * One of the following macros must be defined, to show how to compare + * elements. The first two options are arbitrary expressions depending + * on whether an extra pass-through argument is desired, and the third + * option should be defined if the sort function should receive a + * function pointer at runtime. + * + * - ST_COMPARE(a, b) - a simple comparison expression + * - ST_COMPARE(a, b, arg) - variant that takes an extra argument + * - ST_COMPARE_RUNTIME_POINTER - sort function takes a function pointer + * + * To say that the comparator and therefore also sort function should + * receive an extra pass-through argument, specify the type of the + * argument. + * + * - ST_COMPARE_ARG_TYPE - type of extra argument + * + * The prototype of the generated sort function is: + * + * void ST_SORT(ST_ELEMENT_TYPE *data, size_t n, + * [size_t element_size,] + * [ST_SORT_compare_function compare,] + * [ST_COMPARE_ARG_TYPE *arg]); + * + * ST_SORT_compare_function is a function pointer of the following type: + * + * int (*)(const ST_ELEMENT_TYPE *a, const ST_ELEMENT_TYPE *b, + * [ST_COMPARE_ARG_TYPE *arg]) + * + * HISTORY + * + * Modifications from vanilla NetBSD source: + * - Add do ... while() macro fix + * - Remove __inline, _DIAGASSERTs, __P + * - Remove ill-considered "swap_cnt" switch to insertion sort, in favor + * of a simple check for presorted input. + * - Take care to recurse on the smaller partition, to bound stack usage + * - Convert into a header that can generate specialized functions + * + * IDENTIFICATION + * src/include/lib/sort_template.h + * + *------------------------------------------------------------------------- + */ + +/* $NetBSD: qsort.c,v 1.13 2003/08/07 16:43:42 agc Exp $ */ + +/*- + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Qsort routine based on J. L. Bentley and M. D. McIlroy, + * "Engineering a sort function", + * Software--Practice and Experience 23 (1993) 1249-1265. + * + * We have modified their original by adding a check for already-sorted + * input, which seems to be a win per discussions on pgsql-hackers around + * 2006-03-21. + * + * Also, we recurse on the smaller partition and iterate on the larger one, + * which ensures we cannot recurse more than log(N) levels (since the + * partition recursed to is surely no more than half of the input). Bentley + * and McIlroy explicitly rejected doing this on the grounds that it's "not + * worth the effort", but we have seen crashes in the field due to stack + * overrun, so that judgment seems wrong. + */ +#define CppConcat(x, y) x##y +#define pg_noinline __attribute__((noinline)) +#define Min(a, b) ((a) < (b) ? (a) : (b)) + +#define ST_MAKE_PREFIX(a) CppConcat(a,_) +#define ST_MAKE_NAME(a,b) ST_MAKE_NAME_(ST_MAKE_PREFIX(a),b) +#define ST_MAKE_NAME_(a,b) CppConcat(a,b) + +/* + * If the element type is void, we'll also need an element_size argument + * because we don't know the size. + */ +#ifdef ST_ELEMENT_TYPE_VOID +#define ST_ELEMENT_TYPE void +#define ST_SORT_PROTO_ELEMENT_SIZE , size_t element_size +#define ST_SORT_INVOKE_ELEMENT_SIZE , element_size +#else +#define ST_SORT_PROTO_ELEMENT_SIZE +#define ST_SORT_INVOKE_ELEMENT_SIZE +#endif + +/* + * If the user wants to be able to pass in compare functions at runtime, + * we'll need to make that an argument of the sort and med3 functions. + */ +#ifdef ST_COMPARE_RUNTIME_POINTER +/* + * The type of the comparator function pointer that ST_SORT will take, unless + * you've already declared a type name manually and want to use that instead of + * having a new one defined. + */ +#ifndef ST_COMPARATOR_TYPE_NAME +#define ST_COMPARATOR_TYPE_NAME ST_MAKE_NAME(ST_SORT, compare_function) +#endif +#define ST_COMPARE compare +#ifndef ST_COMPARE_ARG_TYPE +#define ST_SORT_PROTO_COMPARE , ST_COMPARATOR_TYPE_NAME compare +#define ST_SORT_INVOKE_COMPARE , compare +#else +#define ST_SORT_PROTO_COMPARE , ST_COMPARATOR_TYPE_NAME compare +#define ST_SORT_INVOKE_COMPARE , compare +#endif +#else +#define ST_SORT_PROTO_COMPARE +#define ST_SORT_INVOKE_COMPARE +#endif + +/* + * If the user wants to use a compare function or expression that takes an + * extra argument, we'll need to make that an argument of the sort, compare and + * med3 functions. + */ +#ifdef ST_COMPARE_ARG_TYPE +#define ST_SORT_PROTO_ARG , ST_COMPARE_ARG_TYPE *arg +#define ST_SORT_INVOKE_ARG , arg +#else +#define ST_SORT_PROTO_ARG +#define ST_SORT_INVOKE_ARG +#endif + +#ifdef ST_DECLARE + +#ifdef ST_COMPARE_RUNTIME_POINTER +typedef int (*ST_COMPARATOR_TYPE_NAME) (const ST_ELEMENT_TYPE *, + const ST_ELEMENT_TYPE * ST_SORT_PROTO_ARG); +#endif + +/* Declare the sort function. Note optional arguments at end. */ +ST_SCOPE void ST_SORT(ST_ELEMENT_TYPE * first, size_t n + ST_SORT_PROTO_ELEMENT_SIZE + ST_SORT_PROTO_COMPARE + ST_SORT_PROTO_ARG); + +#endif + +#ifdef ST_DEFINE + +/* sort private helper functions */ +#define ST_MED3 ST_MAKE_NAME(ST_SORT, med3) +#define ST_SWAP ST_MAKE_NAME(ST_SORT, swap) +#define ST_SWAPN ST_MAKE_NAME(ST_SORT, swapn) + +/* Users expecting to run very large sorts may need them to be interruptible. */ +#ifdef ST_CHECK_FOR_INTERRUPTS +#define DO_CHECK_FOR_INTERRUPTS() CHECK_FOR_INTERRUPTS() +#else +#define DO_CHECK_FOR_INTERRUPTS() +#endif + +/* + * Create wrapper macros that know how to invoke compare, med3 and sort with + * the right arguments. + */ +#ifdef ST_COMPARE_RUNTIME_POINTER +#define DO_COMPARE(a_, b_) ST_COMPARE((a_), (b_) ST_SORT_INVOKE_ARG) +#elif defined(ST_COMPARE_ARG_TYPE) +#define DO_COMPARE(a_, b_) ST_COMPARE((a_), (b_), arg) +#else +#define DO_COMPARE(a_, b_) ST_COMPARE((a_), (b_)) +#endif +#define DO_MED3(a_, b_, c_) \ + ST_MED3((a_), (b_), (c_) \ + ST_SORT_INVOKE_COMPARE \ + ST_SORT_INVOKE_ARG) +#define DO_SORT(a_, n_) \ + ST_SORT((a_), (n_) \ + ST_SORT_INVOKE_ELEMENT_SIZE \ + ST_SORT_INVOKE_COMPARE \ + ST_SORT_INVOKE_ARG) + +/* + * If we're working with void pointers, we'll use pointer arithmetic based on + * uint8, and use the runtime element_size to step through the array and swap + * elements. Otherwise we'll work with ST_ELEMENT_TYPE. + */ +#ifndef ST_ELEMENT_TYPE_VOID +#define ST_POINTER_TYPE ST_ELEMENT_TYPE +#define ST_POINTER_STEP 1 +#define DO_SWAPN(a_, b_, n_) ST_SWAPN((a_), (b_), (n_)) +#define DO_SWAP(a_, b_) ST_SWAP((a_), (b_)) +#else +#define ST_POINTER_TYPE uint8_t +#define ST_POINTER_STEP element_size +#define DO_SWAPN(a_, b_, n_) ST_SWAPN((a_), (b_), (n_)) +#define DO_SWAP(a_, b_) DO_SWAPN((a_), (b_), element_size) +#endif + +/* + * Find the median of three values. Currently, performance seems to be best + * if the comparator is inlined here, but the med3 function is not inlined + * in the qsort function. + */ +static pg_noinline ST_ELEMENT_TYPE * +ST_MED3(ST_ELEMENT_TYPE * a, + ST_ELEMENT_TYPE * b, + ST_ELEMENT_TYPE * c + ST_SORT_PROTO_COMPARE + ST_SORT_PROTO_ARG) +{ + return DO_COMPARE(a, b) < 0 ? + (DO_COMPARE(b, c) < 0 ? b : (DO_COMPARE(a, c) < 0 ? c : a)) + : (DO_COMPARE(b, c) > 0 ? b : (DO_COMPARE(a, c) < 0 ? a : c)); +} + +static inline void +ST_SWAP(ST_POINTER_TYPE * a, ST_POINTER_TYPE * b) +{ + ST_POINTER_TYPE tmp = *a; + + *a = *b; + *b = tmp; +} + +static inline void +ST_SWAPN(ST_POINTER_TYPE * a, ST_POINTER_TYPE * b, size_t n) +{ + for (size_t i = 0; i < n; ++i) + ST_SWAP(&a[i], &b[i]); +} + +/* + * Sort an array. + */ +ST_SCOPE void +ST_SORT(ST_ELEMENT_TYPE * data, size_t n + ST_SORT_PROTO_ELEMENT_SIZE + ST_SORT_PROTO_COMPARE + ST_SORT_PROTO_ARG) +{ + ST_POINTER_TYPE *a = (ST_POINTER_TYPE *) data, + *pa, + *pb, + *pc, + *pd, + *pl, + *pm, + *pn; + size_t d1, + d2; + int r, + presorted; + +loop: + DO_CHECK_FOR_INTERRUPTS(); + if (n < 7) + { + for (pm = a + ST_POINTER_STEP; pm < a + n * ST_POINTER_STEP; + pm += ST_POINTER_STEP) + for (pl = pm; pl > a && DO_COMPARE(pl - ST_POINTER_STEP, pl) > 0; + pl -= ST_POINTER_STEP) + DO_SWAP(pl, pl - ST_POINTER_STEP); + return; + } + presorted = 1; + for (pm = a + ST_POINTER_STEP; pm < a + n * ST_POINTER_STEP; + pm += ST_POINTER_STEP) + { + DO_CHECK_FOR_INTERRUPTS(); + if (DO_COMPARE(pm - ST_POINTER_STEP, pm) > 0) + { + presorted = 0; + break; + } + } + if (presorted) + return; + pm = a + (n / 2) * ST_POINTER_STEP; + if (n > 7) + { + pl = a; + pn = a + (n - 1) * ST_POINTER_STEP; + if (n > 40) + { + size_t d = (n / 8) * ST_POINTER_STEP; + + pl = DO_MED3(pl, pl + d, pl + 2 * d); + pm = DO_MED3(pm - d, pm, pm + d); + pn = DO_MED3(pn - 2 * d, pn - d, pn); + } + pm = DO_MED3(pl, pm, pn); + } + DO_SWAP(a, pm); + pa = pb = a + ST_POINTER_STEP; + pc = pd = a + (n - 1) * ST_POINTER_STEP; + for (;;) + { + while (pb <= pc && (r = DO_COMPARE(pb, a)) <= 0) + { + if (r == 0) + { + DO_SWAP(pa, pb); + pa += ST_POINTER_STEP; + } + pb += ST_POINTER_STEP; + DO_CHECK_FOR_INTERRUPTS(); + } + while (pb <= pc && (r = DO_COMPARE(pc, a)) >= 0) + { + if (r == 0) + { + DO_SWAP(pc, pd); + pd -= ST_POINTER_STEP; + } + pc -= ST_POINTER_STEP; + DO_CHECK_FOR_INTERRUPTS(); + } + if (pb > pc) + break; + DO_SWAP(pb, pc); + pb += ST_POINTER_STEP; + pc -= ST_POINTER_STEP; + } + pn = a + n * ST_POINTER_STEP; + d1 = Min(pa - a, pb - pa); + DO_SWAPN(a, pb - d1, d1); + d1 = Min(pd - pc, pn - pd - ST_POINTER_STEP); + DO_SWAPN(pb, pn - d1, d1); + d1 = pb - pa; + d2 = pd - pc; + if (d1 <= d2) + { + /* Recurse on left partition, then iterate on right partition */ + if (d1 > ST_POINTER_STEP) + DO_SORT(a, d1 / ST_POINTER_STEP); + if (d2 > ST_POINTER_STEP) + { + /* Iterate rather than recurse to save stack space */ + /* DO_SORT(pn - d2, d2 / ST_POINTER_STEP) */ + a = pn - d2; + n = d2 / ST_POINTER_STEP; + goto loop; + } + } + else + { + /* Recurse on right partition, then iterate on left partition */ + if (d2 > ST_POINTER_STEP) + DO_SORT(pn - d2, d2 / ST_POINTER_STEP); + if (d1 > ST_POINTER_STEP) + { + /* Iterate rather than recurse to save stack space */ + /* DO_SORT(a, d1 / ST_POINTER_STEP) */ + n = d1 / ST_POINTER_STEP; + goto loop; + } + } +} +#endif + +#undef DO_CHECK_FOR_INTERRUPTS +#undef DO_COMPARE +#undef DO_MED3 +#undef DO_SORT +#undef DO_SWAP +#undef DO_SWAPN +#undef ST_CHECK_FOR_INTERRUPTS +#undef ST_COMPARATOR_TYPE_NAME +#undef ST_COMPARE +#undef ST_COMPARE_ARG_TYPE +#undef ST_COMPARE_RUNTIME_POINTER +#undef ST_ELEMENT_TYPE +#undef ST_ELEMENT_TYPE_VOID +#undef ST_MAKE_NAME +#undef ST_MAKE_NAME_ +#undef ST_MAKE_PREFIX +#undef ST_MED3 +#undef ST_POINTER_STEP +#undef ST_POINTER_TYPE +#undef ST_SCOPE +#undef ST_SORT +#undef ST_SORT_INVOKE_ARG +#undef ST_SORT_INVOKE_COMPARE +#undef ST_SORT_INVOKE_ELEMENT_SIZE +#undef ST_SORT_PROTO_ARG +#undef ST_SORT_PROTO_COMPARE +#undef ST_SORT_PROTO_ELEMENT_SIZE +#undef ST_SWAP +#undef ST_SWAPN diff --git a/src/fu_util/test/sort.c b/src/fu_util/test/sort.c new file mode 100644 index 000000000..41472755c --- /dev/null +++ b/src/fu_util/test/sort.c @@ -0,0 +1,257 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=0 */ +#include +#include +#include +#include +#include +#include +#include "./qsort/qsort_pg.inc.c" + +static void +check_sorted(int *a, int len) { + for (; len > 1 ; len--) { + ft_assert(a[len-2] <= a[len-1]); + } +} + +static void +fill_ascending(int *a, int len) { + int i = 0; + for (; i < len ; i++) + a[i] = i; +} + +static void +fill_descending(int *a, int len) { + int i = 0; + for (; i < len ; i++) + a[i] = len - i; +} + +static void +fill_saw_1(int *a, int len) { + int i = 0; + for (; i < len/2 ; i++) + a[i] = i; + for (; i < len ; i++) + a[i] = len-i; +} + +static void +fill_saw_2(int *a, int len) { + int i = 0; + for (; i < len/2 ; i++) + a[i] = len-i; + for (; i < len ; i++) + a[i] = i; +} + +#define rand_init(len) \ + uint32_t r, rand = 0xdeadbeef ^ (uint32_t)len +#define rand_step do { \ + r = rand; \ + rand = rand * 0xcafedead + 0xbeef; \ + r = (r ^ (rand >> 16)) * 0x51235599; \ +} while(0) + + +static void +fill_flip(int *a, int len) { + int i = 0; + rand_init(len); + for (; i < len ; i++) + { + rand_step; + a[i] = r >> 31; + } +} + +static void +fill_several(int *a, int len) { + int i = 0; + rand_init(len); + for (; i < len ; i++) + { + rand_step; + a[i] = r >> 28; + } +} + +static void +fill_rand(int *a, int len) { + int i = 0; + uint32_t max = (uint32_t)len; + rand_init(len); + for (; i < len ; i++) + { + rand_step; + a[i] = ((uint64_t)r * max) >> 32; + } +} + +static void +fill_rand_div5(int *a, int len) { + int i = 0; + uint32_t max = (uint32_t)len / 5 + 1; + rand_init(len); + for (; i < len ; i++) + { + rand_step; + a[i] = ((uint64_t)r * max) >> 32; + } +} + +static void +fill_asc_swap_tail4(int *a, int len) { + int i = 0, j; + rand_init(len); + fill_ascending(a, len); + if (len < 8) + return; + for (; i < 4 ; i++) + { + rand_step; + j = ((uint64_t)r * (uint32_t)(len-4)) >> 32; + ft_swap(&a[len - 1 - i], &a[j]); + } +} + +static void +fill_asc_swap_head4(int *a, int len) { + int i = 0, j; + rand_init(len); + fill_ascending(a, len); + if (len < 8) + return; + for (; i < 4 ; i++) + { + rand_step; + j = ((uint64_t)r * (uint32_t)(len-5)) >> 32; + ft_swap(&a[i], &a[4+j]); + } +} + +static uint64_t ncomp = 0; + +static int +int_cmp_raw2(int a, int b) { + ncomp++; + return ft_cmp(a, b); +} + +static void ft_unused +sort_shell(int *a, int len) { + ft_shsort_int(a, len, int_cmp_raw2); +} + +static void ft_unused +sort_quick(int *a, int len) { + ft_qsort_int(a, len, int_cmp_raw2); +} + +static int +compare_int(const void *pa, const void *pb) { + int a = *(const int *)pa; + int b = *(const int *)pb; + ncomp++; + return a < b ? -1 : a > b; +} + +static void ft_unused +sort_qsort(int *a, int len) { + qsort(a, len, sizeof(len), compare_int); +} + +static void ft_unused +sort_qsort_pg(int *a, int len) { + pg_qsort(a, len, sizeof(len), compare_int); +} + +#define ST_SORT sort_qsort_pg2_ +#define ST_ELEMENT_TYPE int +#define ST_COMPARE(a, b) (ncomp++, *(a) < *(b) ? -1 : *(a) > *(b)) +#define ST_SCOPE static +#define ST_DECLARE +#define ST_DEFINE +#include "./qsort/sort_template.h" + +static void +sort_qsort_pg2(int *a, int len) { + sort_qsort_pg2_(a, len); +} + + +typedef void (*tfiller)(int *, int); +typedef void (*tsorter)(int *, int); + +static double +mtime(void) { + struct timespec ts = {0, 0}; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec/1e9; +} + + +int +main(void) { + int verbose = getenv("VERBOSE") ? atoi(getenv("VERBOSE")) : 0; + int ex[] = {8, 4, 0, 2, 6, 32, 12}; + ft_shsort_int(ex, 7, ft_int_cmp); + check_sorted(ex, 7); + + const char *sex[] = {"hi", "ho", "no", "yes", "obhs", "dump", "vamp"}; + ft_shsort_cstr(sex, 7, strcmp); + for (int i = 0; i < 6; i++) + ft_assert(strcmp(sex[i], sex[i+1]) < 0); + +#define VS(v) {v, #v} + struct { tfiller f; const char *name; } fillers[] = { + VS(fill_ascending), + VS(fill_descending), + VS(fill_rand), + VS(fill_rand_div5), + VS(fill_several), + VS(fill_flip), + VS(fill_saw_1), + VS(fill_saw_2), + VS(fill_asc_swap_head4), + VS(fill_asc_swap_tail4), + }; + struct { tsorter sorter; const char* name; } sorters[] = { + VS(sort_shell), + VS(sort_quick), + VS(sort_qsort), + VS(sort_qsort_pg), + VS(sort_qsort_pg2), + }; + int sizes[] = {1, 2, 3, 5, 10, 20, 50, 100, 500, 1000, 2000, 100000}; + int sz, fl, srt; + int *ar, *cp; + for (sz = 0; sz < ft_arrsz(sizes); sz++) { + if (verbose) + printf("sz: %d\n", sizes[sz]); + ar = calloc(sizeof(int), sizes[sz]); + cp = calloc(sizeof(int), sizes[sz]); + for(fl = 0; fl < ft_arrsz(fillers); fl++) { + fillers[fl].f(ar, sizes[sz]); + if (verbose) + printf(" filler: %s\n", fillers[fl].name); + for (srt = 0; srt < ft_arrsz(sorters); srt++) { + double tend, tstart; + ncomp = 0; + memcpy(cp, ar, sizeof(int)*sizes[sz]); + tstart = mtime(); + sorters[srt].sorter(cp, sizes[sz]); + tend = mtime(); + check_sorted(cp, sizes[sz]); + if (verbose) + printf(" %s: %.6f\tcmp: %llu\n", + sorters[srt].name, + tend - tstart, + (unsigned long long)ncomp); + } + } + free(ar); + free(cp); + } +} diff --git a/src/fu_util/test/sort_p.c b/src/fu_util/test/sort_p.c new file mode 100644 index 000000000..f4b10cb3c --- /dev/null +++ b/src/fu_util/test/sort_p.c @@ -0,0 +1,271 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=0 */ +#include +#include +#include +#include +#include +#include +#include "./qsort/qsort.inc.c" +#include "./qsort/qsort_pg.inc.c" + +static void +fill_ascending(int *a, int len) { + int i = 0; + for (; i < len ; i++) + a[i] = i; +} + +static void +fill_descending(int *a, int len) { + int i = 0; + for (; i < len ; i++) + a[i] = len - i; +} + +static void +fill_saw_1(int *a, int len) { + int i = 0; + for (; i < len/2 ; i++) + a[i] = i; + for (; i < len ; i++) + a[i] = len-i; +} + +static void +fill_saw_2(int *a, int len) { + int i = 0; + for (; i < len/2 ; i++) + a[i] = len-i; + for (; i < len ; i++) + a[i] = i; +} + +#define rand_init(len) \ + uint32_t r, rand = 0xdeadbeef ^ (uint32_t)len +#define rand_step do { \ + r = rand; \ + rand = rand * 0xcafedead + 0xbeef; \ + r = (r ^ (rand >> 16)) * 0x51235599; \ +} while(0) + +static void +fill_flip(int *a, int len) { + int i = 0; + rand_init(len); + for (; i < len ; i++) + { + rand_step; + a[i] = r >> 31; + } +} + +static void +fill_several(int *a, int len) { + int i = 0; + rand_init(len); + for (; i < len ; i++) + { + rand_step; + a[i] = r >> 28; + } +} + +static void +fill_rand(int *a, int len) { + int i = 0; + uint32_t max = (uint32_t)len; + rand_init(len); + for (; i < len ; i++) + { + rand_step; + a[i] = ((uint64_t)r * max) >> 32; + } +} + +static void +fill_rand_div5(int *a, int len) { + int i = 0; + uint32_t max = (uint32_t)len / 5 + 1; + rand_init(len); + for (; i < len ; i++) + { + rand_step; + a[i] = ((uint64_t)r * max) >> 32; + } +} + +static void +fill_asc_swap_tail4(int *a, int len) { + int i = 0, j; + rand_init(len); + fill_ascending(a, len); + if (len < 16) + return; + for (; i < 8 ; i++) + { + rand_step; + j = ((uint64_t)r * (uint32_t)(len-9)) >> 32; + ft_swap(&a[len - 1 - i], &a[j]); + } +} + +static void +fill_asc_swap_head4(int *a, int len) { + int i = 0, j; + rand_init(len); + fill_ascending(a, len); + if (len < 16) + return; + for (; i < 8 ; i++) + { + rand_step; + j = ((uint64_t)r * (uint32_t)(len-9)) >> 32; + ft_swap(&a[i], &a[8+j]); + } +} + +static const char **ref = NULL; + +static void +fill_ref(int len) { + int i = 0, ignore ft_unused; + ref = calloc(sizeof(char*), len); + for (; i < len; i++) + { + ref[i] = ft_asprintf("%08x", i).ptr; + } +} + +static void +clear_ref(int len) { + int i = 0; + ref = calloc(sizeof(char*), len); + for (; i < len; i++) + free((void*)ref[i]); + free(ref); + ref = NULL; +} + +static uint64_t ncomp = 0; + +static FT_CMP_RES +compare_int_raw(int a, int b) { + ncomp++; + return ft_cstr_cmp(ref[a], ref[b]); +} + +static void ft_unused +sort_shell(int *a, int len) { + ft_shsort_int(a, len, compare_int_raw); +} + +static void ft_unused +sort_quick(int *a, int len) { + ft_qsort_int(a, len, compare_int_raw); +} + +static int +compare_int(const void *pa, const void *pb) { + int a = *(const int *)pa; + int b = *(const int *)pb; + ncomp++; + return strcmp(ref[a], ref[b]); +} + +static int +compare_int_v(const void *pa, const void *pb, void *p) { + int a = *(const int *)pa; + int b = *(const int *)pb; + ncomp++; + return strcmp(ref[a], ref[b]); +} + +static void ft_unused +sort_qsort(int *a, int len) { + qsort(a, len, sizeof(len), compare_int); +} + +static void ft_unused +sort_qsort_cpy(int *a, int len) { + _quicksort(a, len, sizeof(len), compare_int_v, NULL); +} + +static void ft_unused +sort_qsort_pg(int *a, int len) { + pg_qsort(a, len, sizeof(len), compare_int); +} + +static void +check_sorted(int *a, int len) { + for (; len > 1 ; len--) { + ft_assert(strcmp(ref[a[len-2]], ref[a[len-1]]) <= 0); + } +} + +typedef void (*tfiller)(int *, int); +typedef void (*tsorter)(int *, int); + +static double +mtime(void) { + struct timespec ts = {0, 0}; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec/1e9; +} + + +int +main(void) { + int verbose = getenv("VERBOSE") ? atoi(getenv("VERBOSE")) : 0; +#define VS(v) {v, #v} + struct { tfiller f; const char *name; } fillers[] = { + VS(fill_ascending), + VS(fill_descending), + VS(fill_rand), + VS(fill_rand_div5), + VS(fill_several), + VS(fill_flip), + VS(fill_saw_1), + VS(fill_saw_2), + VS(fill_asc_swap_head4), + VS(fill_asc_swap_tail4), + }; + struct { tsorter sorter; const char* name; } sorters[] = { + VS(sort_shell), + VS(sort_quick), + VS(sort_qsort), + VS(sort_qsort_cpy), + VS(sort_qsort_pg), + }; + int sizes[] = {1, 2, 3, 5, 10, 20, 50, 100, 500, 1000, 2000, 100000}; + int sz, fl, srt; + int *ar, *cp; + for (sz = 0; sz < ft_arrsz(sizes); sz++) { + if (verbose) + printf("sz: %d\n", sizes[sz]); + ar = calloc(sizeof(int), sizes[sz]); + cp = calloc(sizeof(int), sizes[sz]); + fill_ref(ft_max(sizes[sz]+2, 32)); + for(fl = 0; fl < ft_arrsz(fillers); fl++) { + fillers[fl].f(ar, sizes[sz]); + if (verbose) + printf(" filler: %s\n", fillers[fl].name); + for (srt = 0; srt < ft_arrsz(sorters); srt++) { + double tend, tstart; + ncomp = 0; + memcpy(cp, ar, sizeof(int)*sizes[sz]); + tstart = mtime(); + sorters[srt].sorter(cp, sizes[sz]); + tend = mtime(); + check_sorted(cp, sizes[sz]); + if (verbose) + printf(" %s: %.6f\tcmp: %llu\n", + sorters[srt].name, + tend - tstart, + (unsigned long long)ncomp); + } + } + free(ar); + free(cp); + clear_ref(ft_max(sizes[sz]+2, 32)); + } +} diff --git a/src/fu_util/test/thread.c b/src/fu_util/test/thread.c new file mode 100644 index 000000000..1058fbc52 --- /dev/null +++ b/src/fu_util/test/thread.c @@ -0,0 +1,69 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#include +#include + +#include +#include + +#include + + +typedef struct FlagShip { + bool *flag; +} FlagShip; +#define kls__FlagShip mth(fobjDispose) +fobj_klass(FlagShip); + +static bool theFlag1 = false; +static bool theFlag2 = false; +static bool theFlag3 = false; + +static void +FlagShip_fobjDispose(VSelf) +{ + Self(FlagShip); + *self->flag = true; +} + +fobj_klass_handle(FlagShip); + +static +int thr_func3(FlagShip *f) +{ + FOBJ_FUNC_ARP(); + $unref($ref(f)); + $alloc(FlagShip, .flag = &theFlag3); + pthread_exit(NULL); + return 1; +} + +static +int thr_func2(FlagShip *f) +{ + FOBJ_FUNC_ARP(); + $unref($ref(f)); + return theFlag2 + thr_func3($alloc(FlagShip, .flag = &theFlag2)); +} + +static +void* thr_func1(void *arg) +{ + FOBJ_FUNC_ARP(); + printf("%d\n", theFlag1 + thr_func2($alloc(FlagShip, .flag = &theFlag1))); + return NULL; +} + +int +main(int argc, char** argv) +{ + pthread_t th; + void* res; + fobj_init(); + if (pthread_create(&th, NULL, thr_func1, NULL)) + ft_log(FT_FATAL, "Can't create\n"); + if (pthread_join(th, &res)) + ft_log(FT_FATAL, "Can't join\n"); + ft_assert(theFlag1); + ft_assert(theFlag2); + ft_assert(theFlag3); +} \ No newline at end of file diff --git a/src/help.c b/src/help.c index 116a0711c..d1002f063 100644 --- a/src/help.c +++ b/src/help.c @@ -2,7 +2,7 @@ * * help.c * - * Copyright (c) 2017-2021, Postgres Professional + * Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -428,13 +428,6 @@ help_backup(void) printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n")); - - printf(_("\n Replica options:\n")); - printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); - printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); - printf(_(" --master-host=host_name database server host of master (deprecated)\n")); - printf(_(" --master-port=port database server port of master (deprecated)\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n\n")); } static void @@ -931,13 +924,6 @@ help_set_config(void) printf(_(" --archive-host=destination address or hostname for ssh connection to archive host\n")); printf(_(" --archive-port=port port for ssh connection to archive host (default: 22)\n")); printf(_(" --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n")); - - printf(_("\n Replica options:\n")); - printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); - printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); - printf(_(" --master-host=host_name database server host of master (deprecated)\n")); - printf(_(" --master-port=port database server port of master (deprecated)\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n\n")); } static void diff --git a/src/init.c b/src/init.c index 8773016b5..849c8535b 100644 --- a/src/init.c +++ b/src/init.c @@ -11,7 +11,6 @@ #include "pg_probackup.h" #include -#include /* * Initialize backup catalog. @@ -19,27 +18,44 @@ int do_init(CatalogState *catalogState) { - int results; + pioDrive_i backup_location = pioDriveForLocation(FIO_BACKUP_HOST); + bool empty; + err_i err; - results = pg_check_dir(catalogState->catalog_path); + empty = $i(pioIsDirEmpty, backup_location,.path = catalogState->catalog_path, + .err = &err); - if (results == 4) /* exists and not empty*/ + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "cannot open backup catalog directory"); + if (!empty) elog(ERROR, "backup catalog already exist and it's not empty"); - else if (results == -1) /*trouble accessing directory*/ - { - int errno_tmp = errno; - elog(ERROR, "cannot open backup catalog directory \"%s\": %s", - catalogState->catalog_path, strerror(errno_tmp)); - } /* create backup catalog root directory */ - dir_create_dir(catalogState->catalog_path, DIR_PERMISSION, false); + err = $i(pioMakeDir, backup_location, .path = catalogState->catalog_path, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create backup catalog root directory: %s", + $errmsg(err)); + } /* create backup catalog data directory */ - dir_create_dir(catalogState->backup_subdir_path, DIR_PERMISSION, false); + err = $i(pioMakeDir, backup_location, .path = catalogState->backup_subdir_path, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create backup catalog data directory: %s", + $errmsg(err)); + } /* create backup catalog wal directory */ - dir_create_dir(catalogState->wal_subdir_path, DIR_PERMISSION, false); + err = $i(pioMakeDir, backup_location, .path = catalogState->wal_subdir_path, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create backup catalog WAL directory: %s", + $errmsg(err)); + } elog(INFO, "Backup catalog '%s' successfully inited", catalogState->catalog_path); return 0; @@ -48,8 +64,12 @@ do_init(CatalogState *catalogState) int do_add_instance(InstanceState *instanceState, InstanceConfig *instance) { - struct stat st; + pioDrive_i backup_location = instanceState->backup_location; + pioDrive_i db_location = instanceState->database_location; CatalogState *catalogState = instanceState->catalog_state; + err_i err; + bool exists; + int i; /* PGDATA is always required */ if (instance->pgdata == NULL) @@ -57,37 +77,59 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) "(-D, --pgdata)"); /* Read system_identifier from PGDATA */ - instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false); + instance->system_identifier = get_system_identifier(db_location, instance->pgdata, false); /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ - instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); + instance->xlog_seg_size = get_xlog_seg_size(db_location, instance->pgdata); /* Ensure that all root directories already exist */ /* TODO maybe call do_init() here instead of error?*/ - if (access(catalogState->catalog_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", catalogState->catalog_path); - - if (access(catalogState->backup_subdir_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", catalogState->backup_subdir_path); - - if (access(catalogState->wal_subdir_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", catalogState->wal_subdir_path); - - if (stat(instanceState->instance_backup_subdir_path, &st) == 0 && S_ISDIR(st.st_mode)) - elog(ERROR, "Instance '%s' backup directory already exists: '%s'", - instanceState->instance_name, instanceState->instance_backup_subdir_path); + { + const char *paths[] = { + catalogState->catalog_path, + catalogState->backup_subdir_path, + catalogState->wal_subdir_path}; + for (i = 0; i < ft_arrsz(paths); i++) + { + exists = $i(pioExists, backup_location, .path = paths[i], + .expected_kind = PIO_KIND_DIRECTORY, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Check instance"); + if (!exists) + elog(ERROR, "Directory does not exist: '%s'", paths[i]); + } + } - /* - * Create directory for wal files of this specific instance. - * Existence check is extra paranoid because if we don't have such a - * directory in data dir, we shouldn't have it in wal as well. - */ - if (stat(instanceState->instance_wal_subdir_path, &st) == 0 && S_ISDIR(st.st_mode)) - elog(ERROR, "Instance '%s' WAL archive directory already exists: '%s'", - instanceState->instance_name, instanceState->instance_wal_subdir_path); + { + const char *paths[][2] = { + {"backup", instanceState->instance_backup_subdir_path}, + {"WAL", instanceState->instance_wal_subdir_path}, + }; + for (i = 0; i < ft_arrsz(paths); i++) + { + exists = !$i(pioIsDirEmpty, backup_location, .path = paths[i][1], + .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Check instance"); + if (exists) + elog(ERROR, "Instance '%s' %s directory already exists: '%s'", + instanceState->instance_name, paths[i][0], paths[i][1]); + } + } /* Create directory for data files of this specific instance */ - dir_create_dir(instanceState->instance_backup_subdir_path, DIR_PERMISSION, false); - dir_create_dir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, false); + err = $i(pioMakeDir, backup_location, .path = instanceState->instance_backup_subdir_path, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create instance backup directory: %s", + $errmsg(err)); + } + err = $i(pioMakeDir, backup_location, .path = instanceState->instance_wal_subdir_path, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create instance WAL directory: %s", $errmsg(err)); + } /* * Write initial configuration file. @@ -119,7 +161,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) SOURCE_DEFAULT); /* pgdata was set through command line */ - do_set_config(instanceState, true); + do_set_config(instanceState); elog(INFO, "Instance '%s' successfully inited", instanceState->instance_name); return 0; diff --git a/src/main.c b/src/main.c new file mode 100644 index 000000000..5099d7682 --- /dev/null +++ b/src/main.c @@ -0,0 +1,18 @@ +/*------------------------------------------------------------------------- + * + * main.c: proxy main + * + * To allow linking pg_probackup.c with tests we have to have proxy `main` + * in separate file to call real `pbk_main` function + * + * Copyright (c) 2018-2022, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +int +main(int argc, char** argv) +{ + return pbk_main(argc, argv); +} \ No newline at end of file diff --git a/src/merge.c b/src/merge.c index 0017c9e9c..70ee282a6 100644 --- a/src/merge.c +++ b/src/merge.c @@ -9,13 +9,13 @@ #include "pg_probackup.h" -#include #include #include "utils/thread.h" typedef struct { + InstanceState *state; parray *merge_filelist; parray *parent_chain; @@ -435,6 +435,7 @@ merge_chain(InstanceState *instanceState, parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, bool no_validate, bool no_sync) { + FOBJ_FUNC_ARP(); int i; char full_external_prefix[MAXPGPATH]; char full_database_dir[MAXPGPATH]; @@ -456,6 +457,8 @@ merge_chain(InstanceState *instanceState, /* in-place merge flags */ bool compression_match = false; bool program_version_match = false; + err_i err = $noerr(); + /* It's redundant to check block checksumms during merge */ skip_block_validation = true; @@ -579,6 +582,7 @@ merge_chain(InstanceState *instanceState, backup->files = get_backup_filelist(backup, true); parray_qsort(backup->files, pgFileCompareRelPathWithExternal); + backup->hashtable = make_filelist_hashtable(backup->files); /* Set MERGING status for every member of the chain */ if (backup->backup_mode == BACKUP_MODE_FULL) @@ -594,6 +598,8 @@ merge_chain(InstanceState *instanceState, else write_backup_status(backup, BACKUP_STATUS_MERGING, true); } + /* attempt to speedup headers reading at least for dest backup */ + parray_qsort(dest_backup->files, pgFileCompareByHdrOff); /* Construct path to database dir: /backup_dir/instance_name/FULL/database */ join_path_components(full_database_dir, full_backup->root_dir, DATABASE_DIR); @@ -626,7 +632,7 @@ merge_chain(InstanceState *instanceState, pgFile *file = (pgFile *) parray_get(dest_backup->files, i); /* if the entry was an external directory, create it in the backup */ - if (file->external_dir_num && S_ISDIR(file->mode)) + if (file->external_dir_num && file->kind == PIO_KIND_DIRECTORY) { char dirpath[MAXPGPATH]; char new_container[MAXPGPATH]; @@ -634,7 +640,13 @@ merge_chain(InstanceState *instanceState, makeExternalDirPathByNum(new_container, full_external_prefix, file->external_dir_num); join_path_components(dirpath, new_container, file->rel_path); - dir_create_dir(dirpath, DIR_PERMISSION, false); + err = $i(pioMakeDir, dest_backup->backup_location, .path = dirpath, + .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create backup external directory: %s", + $errmsg(err)); + } } pg_atomic_init_flag(&file->lock); @@ -697,19 +709,9 @@ merge_chain(InstanceState *instanceState, pretty_time); /* If temp header map is open, then close it and make rename */ - if (full_backup->hdr_map.fp) + if ($notNULL(full_backup->hdr_map.fp)) { cleanup_header_map(&(full_backup->hdr_map)); - - /* sync new header map to disk */ - if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync temp header map \"%s\": %s", - full_backup->hdr_map.path_tmp, strerror(errno)); - - /* Replace old header map with new one */ - if (rename(full_backup->hdr_map.path_tmp, full_backup->hdr_map.path)) - elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", - full_backup->hdr_map.path_tmp, full_backup->hdr_map.path, strerror(errno)); } /* Close page header maps */ @@ -797,8 +799,10 @@ merge_chain(InstanceState *instanceState, /* We need full path, file object has relative path */ join_path_components(full_file_path, full_database_dir, full_file->rel_path); - pgFileDelete(full_file->mode, full_file_path); - elog(LOG, "Deleted \"%s\"", full_file_path); + if (fio_remove(FIO_BACKUP_HOST, full_file_path, true) == 0) + elog(LOG, "Deleted \"%s\"", full_file_path); + else + elog(ERROR, "Cannot delete file or directory \"%s\": %s", full_file_path, strerror(errno)); } } @@ -907,6 +911,7 @@ merge_chain(InstanceState *instanceState, { parray_walk(backup->files, pgFileFree); parray_free(backup->files); + parray_free(backup->hashtable); } } } @@ -917,12 +922,16 @@ merge_chain(InstanceState *instanceState, static void * merge_files(void *arg) { + FOBJ_FUNC_ARP(); int i; merge_files_arg *arguments = (merge_files_arg *) arg; size_t n_files = parray_num(arguments->dest_backup->files); + header_map_cache_init(); + for (i = 0; i < n_files; i++) { + FOBJ_LOOP_ARP(); pgFile *dest_file = (pgFile *) parray_get(arguments->dest_backup->files, i); pgFile *tmp_file; bool in_place = false; /* keep file as it is */ @@ -935,6 +944,7 @@ merge_files(void *arg) continue; tmp_file = pgFileInit(dest_file->rel_path); + tmp_file->kind = dest_file->kind; tmp_file->mode = dest_file->mode; tmp_file->is_datafile = dest_file->is_datafile; tmp_file->is_cfs = dest_file->is_cfs; @@ -942,10 +952,10 @@ merge_files(void *arg) tmp_file->dbOid = dest_file->dbOid; /* Directories were created before */ - if (S_ISDIR(dest_file->mode)) + if (dest_file->kind == PIO_KIND_DIRECTORY) goto done; - elog(progress ? INFO : LOG, "Progress: (%d/%lu). Merging file \"%s\"", + elog(progress ? INFO : LOG, "Progress: (%d/%zu). Merging file \"%s\"", i + 1, n_files, dest_file->rel_path); if (dest_file->is_datafile && !dest_file->is_cfs) @@ -1001,14 +1011,12 @@ merge_files(void *arg) for (i = parray_num(arguments->parent_chain) - 1; i >= 0; i--) { - pgFile **res_file = NULL; pgFile *file = NULL; pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, i); /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); - file = (res_file) ? *res_file : NULL; + file = search_file_in_hashtable(backup->hashtable, dest_file); /* Destination file is not exists yet, * in-place merge is impossible @@ -1039,11 +1047,8 @@ merge_files(void *arg) */ if (in_place) { - pgFile **res_file = NULL; pgFile *file = NULL; - res_file = parray_bsearch(arguments->full_backup->files, dest_file, - pgFileCompareRelPathWithExternal); - file = (res_file) ? *res_file : NULL; + file = search_file_in_hashtable(arguments->full_backup->hashtable, dest_file); /* If file didn`t changed in any way, then in-place merge is possible */ if (file && @@ -1115,31 +1120,6 @@ merge_files(void *arg) return NULL; } -/* Recursively delete a directory and its contents */ -static void -remove_dir_with_files(const char *path) -{ - parray *files = parray_new(); - int i; - char full_path[MAXPGPATH]; - - dir_list_file(files, path, false, false, true, false, false, 0, FIO_LOCAL_HOST); - parray_qsort(files, pgFileCompareRelPathWithExternalDesc); - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - join_path_components(full_path, path, file->rel_path); - - pgFileDelete(file->mode, full_path); - elog(LOG, "Deleted \"%s\"", full_path); - } - - /* cleanup */ - parray_walk(files, pgFileFree); - parray_free(files); -} - /* Get index of external directory */ static int get_external_index(const char *key, const parray *list) @@ -1173,7 +1153,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, { char old_path[MAXPGPATH]; makeExternalDirPathByNum(old_path, externaldir_template, i + 1); - remove_dir_with_files(old_path); + $i(pioRemoveDir, to_backup->backup_location, .root = old_path, .root_as_well = true); } else if (from_num != i + 1) { @@ -1199,11 +1179,11 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, const char *full_database_dir, bool use_bitmap, bool is_retry, bool no_sync) { - FILE *out = NULL; - char *buffer = pgut_malloc(STDIO_BUFSIZE); char to_fullpath[MAXPGPATH]; char to_fullpath_tmp1[MAXPGPATH]; /* used for restore */ - char to_fullpath_tmp2[MAXPGPATH]; /* used for backup */ + pioDBDrive_i drive = pioDBDriveForLocation(FIO_BACKUP_HOST); + pioDBWriter_i out; + err_i err; /* The next possible optimization is copying "as is" the file * from intermediate incremental backup, that didn`t changed in @@ -1213,25 +1193,21 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, /* set fullpath of destination file and temp files */ join_path_components(to_fullpath, full_database_dir, tmp_file->rel_path); snprintf(to_fullpath_tmp1, MAXPGPATH, "%s_tmp1", to_fullpath); - snprintf(to_fullpath_tmp2, MAXPGPATH, "%s_tmp2", to_fullpath); /* open temp file */ - out = fopen(to_fullpath_tmp1, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open merge target file \"%s\": %s", - to_fullpath_tmp1, strerror(errno)); - setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE); + out = $i(pioOpenWrite, drive, to_fullpath_tmp1, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Open merge target file"); /* restore file into temp file */ tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, use_bitmap, NULL, InvalidXLogRecPtr, NULL, /* when retrying merge header map cannot be trusted */ is_retry ? false : true); - if (fclose(out) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", - to_fullpath_tmp1, strerror(errno)); - pg_free(buffer); + err = $i(pioClose, out); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Closing target file"); /* tmp_file->size is greedy, even if there is single 8KB block in file, * that was overwritten twice during restore_data_file, we would assume that its size is @@ -1241,11 +1217,11 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, * 2 backups of old versions, where n_blocks is missing. */ - backup_data_file(tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, + backup_data_file(tmp_file, to_fullpath_tmp1, to_fullpath, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, dest_backup->checksum_version, - &(full_backup->hdr_map), true); + &(full_backup->hdr_map), true, !no_sync); /* drop restored temp file */ if (unlink(to_fullpath_tmp1) == -1) @@ -1267,16 +1243,6 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, if (tmp_file->write_size == 0) return; - /* sync second temp file to disk */ - if (!no_sync && fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync merge temp file \"%s\": %s", - to_fullpath_tmp2, strerror(errno)); - - /* Do atomic rename from second temp file to destination file */ - if (rename(to_fullpath_tmp2, to_fullpath) == -1) - elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", - to_fullpath_tmp2, to_fullpath, strerror(errno)); - /* drop temp file */ unlink(to_fullpath_tmp1); } @@ -1294,7 +1260,6 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, { int i; char to_fullpath[MAXPGPATH]; - char to_fullpath_tmp[MAXPGPATH]; /* used for backup */ char from_fullpath[MAXPGPATH]; pgBackup *from_backup = NULL; pgFile *from_file = NULL; @@ -1310,8 +1275,6 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, else join_path_components(to_fullpath, full_database_dir, dest_file->rel_path); - snprintf(to_fullpath_tmp, MAXPGPATH, "%s_tmp", to_fullpath); - /* * Iterate over parent chain starting from direct parent of destination * backup to oldest backup in chain, and look for the first @@ -1321,12 +1284,10 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, */ for (i = 0; i < parray_num(parent_chain); i++) { - pgFile **res_file = NULL; from_backup = (pgBackup *) parray_get(parent_chain, i); /* lookup file in intermediate backup */ - res_file = parray_bsearch(from_backup->files, dest_file, pgFileCompareRelPathWithExternal); - from_file = (res_file) ? *res_file : NULL; + from_file = search_file_in_hashtable(from_backup->hashtable, dest_file); /* * It should not be possible not to find source file in intermediate @@ -1370,19 +1331,9 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, } /* Copy file to FULL backup directory into temp file */ - backup_non_data_file(tmp_file, NULL, from_fullpath, - to_fullpath_tmp, BACKUP_MODE_FULL, 0, false); - - /* sync temp file to disk */ - if (!no_sync && fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync merge temp file \"%s\": %s", - to_fullpath_tmp, strerror(errno)); - - /* Do atomic rename from second temp file to destination file */ - if (rename(to_fullpath_tmp, to_fullpath) == -1) - elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", - to_fullpath_tmp, to_fullpath, strerror(errno)); - + backup_non_data_file(full_backup->backup_location, dest_backup->backup_location, + tmp_file, NULL, from_fullpath, + to_fullpath, BACKUP_MODE_FULL, 0, false, !no_sync); } /* diff --git a/src/parsexlog.c b/src/parsexlog.c index 7c4b5b349..f8adece24 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -5,7 +5,7 @@ * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -32,11 +32,8 @@ #if PG_VERSION_NUM >= 150000 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ name, -#elif PG_VERSION_NUM >= 100000 -#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ - name, #else -#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \ +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ name, #endif @@ -111,13 +108,9 @@ typedef struct XLogReaderData bool need_switch; - int xlogfile; + pioDrive_i drive; + pioReader_i xlogfile; char xlogpath[MAXPGPATH]; - -#ifdef HAVE_LIBZ - gzFile gz_xlogfile; - char gz_xlogpath[MAXPGPATH]; -#endif } XLogReaderData; /* Function to process a WAL record */ @@ -141,9 +134,6 @@ typedef struct */ bool got_target; - /* Should we read record, located at endpoint position */ - bool inclusive_endpoint; - /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. @@ -174,8 +164,7 @@ static bool RunXLogThreads(const char *archivedir, XLogRecPtr startpoint, XLogRecPtr endpoint, bool consistent_read, xlog_record_function process_record, - XLogRecTarget *last_rec, - bool inclusive_endpoint); + XLogRecTarget *last_rec); //static XLogReaderState *InitXLogThreadRead(xlog_thread_arg *arg); static bool SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg); @@ -256,8 +245,8 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, /* easy case */ extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, InvalidXLogRecPtr, end_tli, wal_seg_size, - startpoint, endpoint, false, extractPageInfo, - NULL, true); + startpoint, endpoint, true, extractPageInfo, + NULL); else { /* We have to process WAL located on several different xlog intervals, @@ -336,22 +325,12 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, for (i = parray_num(interval_list) - 1; i >= 0; i--) { - bool inclusive_endpoint; lsnInterval *tmp_interval = (lsnInterval *) parray_get(interval_list, i); - /* In case of replica promotion, endpoints of intermediate - * timelines can be unreachable. - */ - inclusive_endpoint = false; - - /* ... but not the end timeline */ - if (tmp_interval->tli == end_tli) - inclusive_endpoint = true; - extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, InvalidXLogRecPtr, tmp_interval->tli, wal_seg_size, tmp_interval->begin_lsn, tmp_interval->end_lsn, - false, extractPageInfo, NULL, inclusive_endpoint); + true, extractPageInfo, NULL); if (!extract_isok) break; @@ -380,7 +359,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, got_endpoint = RunXLogThreads(archivedir, 0, InvalidTransactionId, InvalidXLogRecPtr, tli, xlog_seg_size, backup->start_lsn, backup->stop_lsn, - false, NULL, NULL, true); + true, NULL, NULL); if (!got_endpoint) { @@ -414,13 +393,14 @@ validate_wal(pgBackup *backup, const char *archivedir, char last_timestamp[100], target_timestamp[100]; bool all_wal = false; + err_i err; if (!XRecOffIsValid(backup->start_lsn)) elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), backup_id_of(backup)); - if (!XRecOffIsValid(backup->stop_lsn)) + if (!XRecEndLooksGood(backup->stop_lsn, wal_seg_size)) elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), backup_id_of(backup)); @@ -454,7 +434,7 @@ validate_wal(pgBackup *backup, const char *archivedir, * recovery target time or xid. */ if (!TransactionIdIsValid(target_xid) && target_time == 0 && - !XRecOffIsValid(target_lsn)) + XLogRecPtrIsInvalid(target_lsn)) { /* Recovery target is not given so exit */ elog(INFO, "Backup %s WAL segments are valid", backup_id_of(backup)); @@ -465,8 +445,10 @@ validate_wal(pgBackup *backup, const char *archivedir, * If recovery target is provided, ensure that archive files exist in * archive directory. */ - if (dir_is_empty(archivedir, FIO_LOCAL_HOST)) + if ($i(pioIsDirEmpty, backup->backup_location, .path = archivedir, .err = &err)) elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); + else if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Check WAL archive dir"); /* * Check if we have in archive all files needed to restore backup @@ -483,13 +465,13 @@ validate_wal(pgBackup *backup, const char *archivedir, if ((TransactionIdIsValid(target_xid) && target_xid == last_rec.rec_xid) || (target_time != 0 && backup->recovery_time >= target_time) - || (XRecOffIsValid(target_lsn) && last_rec.rec_lsn >= target_lsn)) + || (!XLogRecPtrIsInvalid(target_lsn) && last_rec.rec_lsn >= target_lsn)) all_wal = true; all_wal = all_wal || RunXLogThreads(archivedir, target_time, target_xid, target_lsn, tli, wal_seg_size, backup->stop_lsn, - InvalidXLogRecPtr, true, validateXLogRecord, &last_rec, true); + InvalidXLogRecPtr, true, validateXLogRecord, &last_rec); if (last_rec.rec_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), timestamptz_to_time_t(last_rec.rec_time), false); @@ -517,12 +499,13 @@ validate_wal(pgBackup *backup, const char *archivedir, else if (target_time != 0) elog(ERROR, "Not enough WAL records to time %s", target_timestamp); - else if (XRecOffIsValid(target_lsn)) + else if (!XLogRecPtrIsInvalid(target_lsn)) elog(ERROR, "Not enough WAL records to lsn %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); } } +#define STEPBACK_CHUNK (1024*1024) /* * Read from archived WAL segments latest recovery time and xid. All necessary * segments present at archive folder. We waited **stop_lsn** in @@ -533,7 +516,8 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, time_t *recovery_time) { - XLogRecPtr startpoint = stop_lsn; + XLogRecPtr startpoint = stop_lsn - (stop_lsn % STEPBACK_CHUNK); + XLogRecPtr endpoint = stop_lsn; XLogReaderState *xlogreader; XLogReaderData reader_data; bool res; @@ -542,7 +526,7 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, elog(ERROR, "Invalid start_lsn value %X/%X", (uint32) (start_lsn >> 32), (uint32) (start_lsn)); - if (!XRecOffIsValid(stop_lsn)) + if (!XRecEndLooksGood(stop_lsn, wal_seg_size)) elog(ERROR, "Invalid stop_lsn value %X/%X", (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); @@ -552,248 +536,77 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, /* Read records from stop_lsn down to start_lsn */ do { + XLogRecPtr trypoint; + XLogRecPtr curpoint; + XLogRecPtr prevpoint = 0; XLogRecord *record; TimestampTz last_time = 0; char *errormsg; -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif + trypoint = startpoint; + if (trypoint < start_lsn) + trypoint = start_lsn; - record = WalReadRecord(xlogreader, startpoint, &errormsg); - if (record == NULL) - { - XLogRecPtr errptr; + curpoint = XLogFindNextRecord(xlogreader, trypoint); - errptr = startpoint ? startpoint : xlogreader->EndRecPtr; - - if (errormsg) - elog(ERROR, "Could not read WAL record at %X/%X: %s", - (uint32) (errptr >> 32), (uint32) (errptr), - errormsg); - else - elog(ERROR, "Could not read WAL record at %X/%X", - (uint32) (errptr >> 32), (uint32) (errptr)); - } - - /* Read previous record */ - startpoint = record->xl_prev; - - if (getRecordTimestamp(xlogreader, &last_time)) + if (XLogRecPtrIsInvalid(curpoint)) { - *recovery_time = timestamptz_to_time_t(last_time); - - /* Found timestamp in WAL record 'record' */ - res = true; - goto cleanup; - } - } while (startpoint >= start_lsn); - - /* Didn't find timestamp from WAL records between start_lsn and stop_lsn */ - res = false; - -cleanup: - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - return res; -} - -/* - * Check if there is a WAL segment file in 'archivedir' which contains - * 'target_lsn'. - */ -bool -wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, - TimeLineID target_tli, uint32 wal_seg_size) -{ - XLogReaderState *xlogreader; - XLogReaderData reader_data; - char *errormsg; - bool res; - - if (!XRecOffIsValid(target_lsn)) - elog(ERROR, "Invalid target_lsn value %X/%X", - (uint32) (target_lsn >> 32), (uint32) (target_lsn)); - - xlogreader = InitXLogPageRead(&reader_data, archivedir, target_tli, - wal_seg_size, false, false, true); - - if (xlogreader == NULL) - elog(ERROR, "Out of memory"); - - xlogreader->system_identifier = instance_config.system_identifier; - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(target_lsn)) - target_lsn = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, target_lsn); -#endif - - res = WalReadRecord(xlogreader, target_lsn, &errormsg) != NULL; - /* Didn't find 'target_lsn' and there is no error, return false */ - - if (errormsg) - elog(WARNING, "Could not read WAL record at %X/%X: %s", - (uint32) (target_lsn >> 32), (uint32) (target_lsn), errormsg); - - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - return res; -} - -/* - * Get LSN of a first record within the WAL segment with number 'segno'. - */ -XLogRecPtr -get_first_record_lsn(const char *archivedir, XLogSegNo segno, - TimeLineID tli, uint32 wal_seg_size, int timeout) -{ - XLogReaderState *xlogreader; - XLogReaderData reader_data; - XLogRecPtr record = InvalidXLogRecPtr; - XLogRecPtr startpoint; - char wal_segment[MAXFNAMELEN]; - int attempts = 0; - - if (segno <= 1) - elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); - - GetXLogFileName(wal_segment, tli, segno, instance_config.xlog_seg_size); - - xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, - false, false, true); - if (xlogreader == NULL) - elog(ERROR, "Out of memory"); - xlogreader->system_identifier = instance_config.system_identifier; - - /* Set startpoint to 0 in segno */ - GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif - - while (attempts <= timeout) - { - record = XLogFindNextRecord(xlogreader, startpoint); - - if (XLogRecPtrIsInvalid(record)) - record = InvalidXLogRecPtr; - else - { - elog(LOG, "First record in WAL segment \"%s\": %X/%X", wal_segment, - (uint32) (record >> 32), (uint32) (record)); - break; + if (trypoint == start_lsn) + { + elog(ERROR, "There is no valid log between %X/%X and %X/%X", + (uint32_t)(start_lsn>>32), (uint32_t)start_lsn, + (uint32_t)(stop_lsn>>32), (uint32_t)stop_lsn); + break; + } + endpoint = startpoint; + startpoint--; + startpoint = startpoint - (startpoint % STEPBACK_CHUNK); + continue; } - attempts++; - sleep(1); - } - - /* cleanup */ - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - return record; -} - - -/* - * Get LSN of the record next after target lsn. - */ -XLogRecPtr -get_next_record_lsn(const char *archivedir, XLogSegNo segno, - TimeLineID tli, uint32 wal_seg_size, int timeout, - XLogRecPtr target) -{ - XLogReaderState *xlogreader; - XLogReaderData reader_data; - XLogRecPtr startpoint, found; - XLogRecPtr res = InvalidXLogRecPtr; - char wal_segment[MAXFNAMELEN]; - int attempts = 0; - - if (segno <= 1) - elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); - - GetXLogFileName(wal_segment, tli, segno, instance_config.xlog_seg_size); - - xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, - false, false, true); - if (xlogreader == NULL) - elog(ERROR, "Out of memory"); - xlogreader->system_identifier = instance_config.system_identifier; - - /* Set startpoint to 0 in segno */ - GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif - - found = XLogFindNextRecord(xlogreader, startpoint); + do { + record = WalReadRecord(xlogreader, curpoint, &errormsg); + if (prevpoint == 0) + prevpoint = record->xl_prev; + if (record == NULL) + { + XLogRecPtr errptr; - if (XLogRecPtrIsInvalid(found)) - { - if (xlogreader->errormsg_buf[0] != '\0') - elog(WARNING, "Could not read WAL record at %X/%X: %s", - (uint32) (startpoint >> 32), (uint32) (startpoint), - xlogreader->errormsg_buf); - else - elog(WARNING, "Could not read WAL record at %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - PrintXLogCorruptionMsg(&reader_data, ERROR); - } - startpoint = found; + errptr = curpoint ? curpoint : xlogreader->EndRecPtr; - while (attempts <= timeout) - { - XLogRecord *record; - char *errormsg; + if (errormsg) + elog(ERROR, "Could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(ERROR, "Could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + } - if (interrupted) - elog(ERROR, "Interrupted during WAL reading"); + if (curpoint < endpoint && getRecordTimestamp(xlogreader, &last_time)) + { + *recovery_time = timestamptz_to_time_t(last_time); - record = WalReadRecord(xlogreader, startpoint, &errormsg); + /* Found timestamp in WAL record 'record' */ + res = true; + } - if (record == NULL) - { - XLogRecPtr errptr; + /* for compatibility with Pg < 13 */ + curpoint = InvalidXLogRecPtr; + } while (xlogreader->EndRecPtr < endpoint); - errptr = XLogRecPtrIsInvalid(startpoint) ? xlogreader->EndRecPtr : - startpoint; + if (res) + goto cleanup; - if (errormsg) - elog(WARNING, "Could not read WAL record at %X/%X: %s", - (uint32) (errptr >> 32), (uint32) (errptr), - errormsg); - else - elog(WARNING, "Could not read WAL record at %X/%X", - (uint32) (errptr >> 32), (uint32) (errptr)); - PrintXLogCorruptionMsg(&reader_data, ERROR); - } + /* Goto previous megabyte */ + endpoint = startpoint; + startpoint = prevpoint - (prevpoint % STEPBACK_CHUNK); + } while (endpoint > start_lsn); - if (xlogreader->ReadRecPtr >= target) - { - elog(LOG, "Record %X/%X is next after target LSN %X/%X", - (uint32) (xlogreader->ReadRecPtr >> 32), (uint32) (xlogreader->ReadRecPtr), - (uint32) (target >> 32), (uint32) (target)); - res = xlogreader->ReadRecPtr; - break; - } - else - startpoint = InvalidXLogRecPtr; - } + /* Didn't find timestamp from WAL records between start_lsn and stop_lsn */ + res = false; - /* cleanup */ +cleanup: CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); @@ -814,8 +627,7 @@ get_next_record_lsn(const char *archivedir, XLogSegNo segno, */ XLogRecPtr get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, - XLogRecPtr stop_lsn, TimeLineID tli, bool seek_prev_segment, - uint32 wal_seg_size) + XLogRecPtr stop_lsn, TimeLineID tli, uint32 wal_seg_size) { XLogReaderState *xlogreader; XLogReaderData reader_data; @@ -829,7 +641,7 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, if (segno <= 1) elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); - if (seek_prev_segment) + if (stop_lsn % wal_seg_size == 0) segno = segno - 1; xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, @@ -844,26 +656,15 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, * Calculate startpoint. Decide: we should use 'start_lsn' or offset 0. */ GetXLogSegNo(start_lsn, start_segno, wal_seg_size); - if (start_segno == segno) - { - startpoint = start_lsn; -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif - } - else + + for (;;) { XLogRecPtr found; - GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif + if (start_segno == segno) + startpoint = start_lsn; + else + GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); found = XLogFindNextRecord(xlogreader, startpoint); @@ -876,9 +677,28 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, else elog(WARNING, "Could not read WAL record at %X/%X", (uint32) (startpoint >> 32), (uint32) (startpoint)); + if (segno > start_segno) + { + segno--; + continue; + } PrintXLogCorruptionMsg(&reader_data, ERROR); } startpoint = found; + break; + } + + /* This check doesn't make much value. But let it be. */ + if (start_segno == segno && + XRecPtrLooksGood(start_lsn, wal_seg_size) && + startpoint != start_lsn) + { + elog(ERROR, "Start lsn %X/%X wasn't found despite it looks good." + "Got lsn %X/%X instead. " + "(in archive dir %s)", + (uint32_t)(start_lsn>>32), (uint32_t)start_lsn, + (uint32_t)(startpoint>>32), (uint32_t)startpoint, + archivedir); } while (true) @@ -927,24 +747,6 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, return res; } -#ifdef HAVE_LIBZ -/* - * Show error during work with compressed file - */ -static const char * -get_gz_error(gzFile gzf) -{ - int errnum; - const char *errmsg; - - errmsg = fio_gzerror(gzf, &errnum); - if (errnum == Z_ERRNO) - return strerror(errno); - else - return errmsg; -} -#endif - /* XLogreader callback function, to read a WAL page */ static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, @@ -956,6 +758,9 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, { XLogReaderData *reader_data; uint32 targetPageOff; + FOBJ_FUNC_ARP(); + err_i err; + size_t rd; reader_data = (XLogReaderData *) xlogreader->private_data; targetPageOff = targetPagePtr % wal_seg_size; @@ -1012,63 +817,70 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (!reader_data->xlogexists) { char xlogfname[MAXFNAMELEN]; + char gz_file[MAXPGPATH]; char partial_file[MAXPGPATH]; + err_i err2 = $noerr(); GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, wal_seg_size); join_path_components(reader_data->xlogpath, wal_archivedir, xlogfname); - snprintf(reader_data->gz_xlogpath, MAXPGPATH, "%s.gz", reader_data->xlogpath); /* We fall back to using .partial segment in case if we are running * multi-timeline incremental backup right after standby promotion. * TODO: it should be explicitly enabled. */ snprintf(partial_file, MAXPGPATH, "%s.partial", reader_data->xlogpath); + snprintf(gz_file, MAXPGPATH, "%s.gz", reader_data->xlogpath); /* If segment do not exists, but the same * segment with '.partial' suffix does, use it instead */ - if (!fileExists(reader_data->xlogpath, FIO_LOCAL_HOST) && - fileExists(partial_file, FIO_LOCAL_HOST)) + if (!$i(pioExists, reader_data->drive, reader_data->xlogpath, .err=&err) && + $i(pioExists, reader_data->drive, partial_file, .err=&err2)) { - snprintf(reader_data->xlogpath, MAXPGPATH, "%s", partial_file); + ft_strlcpy(reader_data->xlogpath, partial_file, MAXPGPATH); + } + else if ($haserr(err) || $haserr(err2)) + { + ft_logerr(FT_WARNING, $errmsg(fobj_err_combine(err, err2)), + "Thread [%d]: Looking for WAL segment"); + return -1; } - if (fileExists(reader_data->xlogpath, FIO_LOCAL_HOST)) + if ($i(pioExists, reader_data->drive, reader_data->xlogpath, .err=&err)) { elog(LOG, "Thread [%d]: Opening WAL segment \"%s\"", reader_data->thread_num, reader_data->xlogpath); reader_data->xlogexists = true; - reader_data->xlogfile = fio_open(reader_data->xlogpath, - O_RDONLY | PG_BINARY, FIO_LOCAL_HOST); - - if (reader_data->xlogfile < 0) - { - elog(WARNING, "Thread [%d]: Could not open WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->xlogpath, - strerror(errno)); - return -1; - } + reader_data->xlogfile = $iref($i(pioOpenRead, reader_data->drive, + .path = reader_data->xlogpath, .err = &err)); } #ifdef HAVE_LIBZ /* Try to open compressed WAL segment */ - else if (fileExists(reader_data->gz_xlogpath, FIO_LOCAL_HOST)) + else if ($noerr(err) && + $i(pioExists, reader_data->drive, gz_file, .err=&err)) { + pioReader_i reader; + ft_strlcpy(reader_data->xlogpath, gz_file, MAXPGPATH); + elog(LOG, "Thread [%d]: Opening compressed WAL segment \"%s\"", - reader_data->thread_num, reader_data->gz_xlogpath); + reader_data->thread_num, reader_data->xlogpath); reader_data->xlogexists = true; - reader_data->gz_xlogfile = fio_gzopen(reader_data->gz_xlogpath, - "rb", -1, FIO_LOCAL_HOST); - if (reader_data->gz_xlogfile == NULL) + reader = $i(pioOpenRead, reader_data->drive, + .path = reader_data->xlogpath, .err = &err); + if ($noerr(err)) { - elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->gz_xlogpath, - strerror(errno)); - return -1; + reader = pioWrapForReSeek(reader, pioGZDecompressWrapper(false)); + reader_data->xlogfile = $iref(reader); } } #endif + if ($haserr(err)) + { + ft_logerr(FT_WARNING, $errmsg(err), "Thread [%d]: Open WAL segment"); + return -1; + } /* Exit without error if WAL segment doesn't exist */ if (!reader_data->xlogexists) return -1; @@ -1088,52 +900,41 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, memcpy(readBuf, reader_data->page_buf, XLOG_BLCKSZ); #if PG_VERSION_NUM < 130000 *pageTLI = reader_data->tli; +#else + xlogreader->seg.ws_tli = reader_data->tli; #endif return XLOG_BLCKSZ; } /* Read the requested page */ - if (reader_data->xlogfile != -1) + err = $i(pioSeek, reader_data->xlogfile, targetPageOff); + if ($haserr(err)) { - if (fio_seek(reader_data->xlogfile, (off_t) targetPageOff) < 0) - { - elog(WARNING, "Thread [%d]: Could not seek in WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->xlogpath, strerror(errno)); - return -1; - } + ft_logerr(FT_WARNING, $errmsg(err), "Thread [%d]: Seek in WAL segment", + reader_data->thread_num); + return -1; + } - if (fio_read(reader_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) - { - elog(WARNING, "Thread [%d]: Could not read from WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->xlogpath, strerror(errno)); - return -1; - } + rd = $i(pioRead, reader_data->xlogfile, ft_bytes(readBuf, XLOG_BLCKSZ), + .err = &err); + if ($noerr(err) && rd != XLOG_BLCKSZ) + { + err = $err(RT, "Short read from {path}: {size} < XLOG_BLCKSZ", + path(reader_data->xlogpath), size(rd)); } -#ifdef HAVE_LIBZ - else + if ($haserr(err)) { - if (fio_gzseek(reader_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) - { - elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->gz_xlogpath, - get_gz_error(reader_data->gz_xlogfile)); - return -1; - } - - if (fio_gzread(reader_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) - { - elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->gz_xlogpath, - get_gz_error(reader_data->gz_xlogfile)); - return -1; - } + ft_logerr(FT_WARNING, $errmsg(err), "Thread [%d]: Read from WAL segment", + reader_data->thread_num); + return -1; } -#endif memcpy(reader_data->page_buf, readBuf, XLOG_BLCKSZ); reader_data->prev_page_off = targetPageOff; #if PG_VERSION_NUM < 130000 *pageTLI = reader_data->tli; +#else + xlogreader->seg.ws_tli = reader_data->tli; #endif return XLOG_BLCKSZ; } @@ -1155,7 +956,8 @@ InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, MemSet(reader_data, 0, sizeof(XLogReaderData)); reader_data->tli = tli; - reader_data->xlogfile = -1; + reader_data->drive = pioDriveForLocation(FIO_BACKUP_HOST); + $setNULL(&reader_data->xlogfile); if (allocate_reader) { @@ -1190,7 +992,7 @@ RunXLogThreads(const char *archivedir, time_t target_time, TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, uint32 segment_size, XLogRecPtr startpoint, XLogRecPtr endpoint, bool consistent_read, xlog_record_function process_record, - XLogRecTarget *last_rec, bool inclusive_endpoint) + XLogRecTarget *last_rec) { pthread_t *threads; xlog_thread_arg *thread_args; @@ -1199,7 +1001,8 @@ RunXLogThreads(const char *archivedir, time_t target_time, XLogSegNo endSegNo = 0; bool result = true; - if (!XRecOffIsValid(startpoint) && !XRecOffIsNull(startpoint)) + /* we actually can start from record end */ + if (!XRecEndLooksGood(startpoint, segment_size)) elog(ERROR, "Invalid startpoint value %X/%X", (uint32) (startpoint >> 32), (uint32) (startpoint)); @@ -1211,19 +1014,12 @@ RunXLogThreads(const char *archivedir, time_t target_time, if (!XLogRecPtrIsInvalid(endpoint)) { -// if (XRecOffIsNull(endpoint) && !inclusive_endpoint) - if (XRecOffIsNull(endpoint)) - { - GetXLogSegNo(endpoint, endSegNo, segment_size); + GetXLogSegNo(endpoint, endSegNo, segment_size); + if (endpoint % segment_size == 0) endSegNo--; - } - else if (!XRecOffIsValid(endpoint)) - { + else if (!XRecEndLooksGood(endpoint, segment_size)) elog(ERROR, "Invalid endpoint value %X/%X", (uint32) (endpoint >> 32), (uint32) (endpoint)); - } - else - GetXLogSegNo(endpoint, endSegNo, segment_size); } /* Initialize static variables for workers */ @@ -1258,7 +1054,6 @@ RunXLogThreads(const char *archivedir, time_t target_time, arg->startpoint = startpoint; arg->endpoint = endpoint; arg->endSegNo = endSegNo; - arg->inclusive_endpoint = inclusive_endpoint; arg->got_target = false; /* By default there is some error */ arg->ret = 1; @@ -1370,11 +1165,10 @@ XLogThreadWorker(void *arg) elog(ERROR, "Thread [%d]: out of memory", reader_data->thread_num); xlogreader->system_identifier = instance_config.system_identifier; -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(thread_arg->startpoint)) - thread_arg->startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, thread_arg->startpoint); -#endif + elog(LOG, "Thread [%d]: Starting LSN: %X/%X , end: %X/%X", + reader_data->thread_num, + (uint32) (thread_arg->startpoint >> 32), (uint32) (thread_arg->startpoint), + (uint32) (thread_arg->endpoint >> 32), (uint32) (thread_arg->endpoint)); found = XLogFindNextRecord(xlogreader, thread_arg->startpoint); @@ -1405,11 +1199,6 @@ XLogThreadWorker(void *arg) thread_arg->startpoint = found; - elog(VERBOSE, "Thread [%d]: Starting LSN: %X/%X", - reader_data->thread_num, - (uint32) (thread_arg->startpoint >> 32), - (uint32) (thread_arg->startpoint)); - while (need_read) { XLogRecord *record; @@ -1493,10 +1282,9 @@ XLogThreadWorker(void *arg) (uint32) (errptr >> 32), (uint32) (errptr)); /* In we failed to read record located at endpoint position, - * and endpoint is not inclusive, do not consider this as an error. + * do not consider this as an error. */ - if (!thread_arg->inclusive_endpoint && - errptr == thread_arg->endpoint) + if (errptr == thread_arg->endpoint) { elog(LOG, "Thread [%d]: Endpoint %X/%X is not inclusive, switch to the next timeline", reader_data->thread_num, @@ -1718,18 +1506,11 @@ CleanupXLogPageRead(XLogReaderState *xlogreader) XLogReaderData *reader_data; reader_data = (XLogReaderData *) xlogreader->private_data; - if (reader_data->xlogfile >= 0) + if (!$isNULL(reader_data->xlogfile)) { - fio_close(reader_data->xlogfile); - reader_data->xlogfile = -1; + $i(pioClose, reader_data->xlogfile); + $idel(&reader_data->xlogfile); } -#ifdef HAVE_LIBZ - else if (reader_data->gz_xlogfile != NULL) - { - fio_gzclose(reader_data->gz_xlogfile); - reader_data->gz_xlogfile = NULL; - } -#endif reader_data->prev_page_off = 0; reader_data->xlogexists = false; } @@ -1746,16 +1527,10 @@ PrintXLogCorruptionMsg(XLogReaderData *reader_data, int elevel) if (!reader_data->xlogexists) elog(elevel, "Thread [%d]: WAL segment \"%s\" is absent", reader_data->thread_num, reader_data->xlogpath); - else if (reader_data->xlogfile != -1) + else elog(elevel, "Thread [%d]: Possible WAL corruption. " "Error has occured during reading WAL segment \"%s\"", reader_data->thread_num, reader_data->xlogpath); -#ifdef HAVE_LIBZ - else if (reader_data->gz_xlogfile != NULL) - elog(elevel, "Thread [%d]: Possible WAL corruption. " - "Error has occured during reading WAL segment \"%s\"", - reader_data->thread_num, reader_data->gz_xlogpath); -#endif } else { @@ -1879,7 +1654,7 @@ validateXLogRecord(XLogReaderState *record, XLogReaderData *reader_data, timestamptz_to_time_t(reader_data->cur_rec.rec_time) >= wal_target_time) *stop_reading = true; /* Check target lsn */ - else if (XRecOffIsValid(wal_target_lsn) && + else if (!XLogRecPtrIsInvalid(wal_target_lsn) && reader_data->cur_rec.rec_lsn >= wal_target_lsn) *stop_reading = true; } @@ -1938,7 +1713,7 @@ bool validate_wal_segment(TimeLineID tli, XLogSegNo segno, const char *prefetch_ rc = RunXLogThreads(prefetch_dir, 0, InvalidTransactionId, InvalidXLogRecPtr, tli, wal_seg_size, - startpoint, endpoint, false, NULL, NULL, true); + startpoint, endpoint, false, NULL, NULL); num_threads = tmp_num_threads; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ed48178b4..f57d12799 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -35,7 +35,7 @@ * which includes info about pgdata directory and connection. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -47,8 +47,6 @@ #include "streamutil.h" #include "utils/file.h" -#include - #include "utils/configuration.h" #include "utils/thread.h" #include @@ -75,14 +73,11 @@ bool no_color = false; bool show_color = true; bool is_archive_cmd = false; pid_t my_pid = 0; -__thread int my_thread_num = 1; bool progress = false; bool no_sync = false; time_t start_time = INVALID_BACKUP_ID; -#if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; bool temp_slot = false; -#endif bool perm_slot = false; /* backup options */ @@ -207,9 +202,7 @@ static ConfigOption cmd_options[] = { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, -#if PG_VERSION_NUM >= 100000 { 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, -#endif { 'b', 'P', "perm-slot", &perm_slot, SOURCE_CMD_STRICT }, { 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, @@ -294,10 +287,16 @@ static ConfigOption cmd_options[] = * Entry point of pg_probackup command. */ int -main(int argc, char *argv[]) +pbk_main(int argc, char *argv[]) { char *command = NULL; ProbackupSubcmd backup_subcmd = NO_CMD; + err_i err; + + ft_init_log(elog_ft_log); + fobj_init(); + FOBJ_FUNC_ARP(); + init_pio_objects(); PROGRAM_NAME_FULL = argv[0]; @@ -305,7 +304,7 @@ main(int argc, char *argv[]) init_console(); /* Initialize current backup */ - pgBackupInit(¤t); + pgBackupInit(¤t, $null(pioDrive)); /* Initialize current instance configuration */ //TODO get git of this global variable craziness @@ -318,6 +317,7 @@ main(int argc, char *argv[]) // Setting C locale for numeric values in order to impose dot-based floating-point representation memorize_environment_locale(); setlocale(LC_NUMERIC, "C"); + setlocale(LC_TIME, "C"); /* Get current time */ current_time = time(NULL); @@ -325,13 +325,11 @@ main(int argc, char *argv[]) my_pid = getpid(); //set_pglocale_pgservice(argv[0], "pgscripts"); -#if PG_VERSION_NUM >= 110000 /* * Reset WAL segment size, we will retreive it using RetrieveWalSegSize() * later. */ WalSegSz = 0; -#endif /* * Save main thread's tid. It is used call exit() in case of errors. @@ -477,12 +475,7 @@ main(int argc, char *argv[]) if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); - catalogState = pgut_new(CatalogState); - strncpy(catalogState->catalog_path, backup_path, MAXPGPATH); - join_path_components(catalogState->backup_subdir_path, - catalogState->catalog_path, BACKUPS_DIR); - join_path_components(catalogState->wal_subdir_path, - catalogState->catalog_path, WAL_SUBDIR); + catalogState = catalog_new(backup_path); } /* backup_path is required for all pg_probackup commands except help, version, checkdb and catchup */ @@ -509,17 +502,7 @@ main(int argc, char *argv[]) } else { - instanceState = pgut_new(InstanceState); - instanceState->catalog_state = catalogState; - - strncpy(instanceState->instance_name, instance_name, MAXPGPATH); - join_path_components(instanceState->instance_backup_subdir_path, - catalogState->backup_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_wal_subdir_path, - catalogState->wal_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_config_path, - instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); - + instanceState = makeInstanceState(catalogState, instance_name); } /* ===== instanceState (END) ======*/ @@ -538,24 +521,22 @@ main(int argc, char *argv[]) if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && backup_subcmd != ARCHIVE_GET_CMD) { - struct stat st; + bool exists = $i(pioExists, catalogState->backup_location, + .path = instanceState->instance_backup_subdir_path, + .expected_kind = PIO_KIND_DIRECTORY, + .err = &err); - if (fio_stat(instanceState->instance_backup_subdir_path, - &st, true, FIO_BACKUP_HOST) != 0) + if ($haserr(err)) { - elog(WARNING, "Failed to access directory \"%s\": %s", - instanceState->instance_backup_subdir_path, strerror(errno)); + ft_logerr(FT_WARNING, $errmsg(err), "Failed to access directory \"%s\"", + instanceState->instance_backup_subdir_path); // TODO: redundant message, should we get rid of it? - elog(ERROR, "Instance '%s' does not exist in this backup catalog", - instance_name); - } - else - { - /* Ensure that backup_path is a path to a directory */ - if (!S_ISDIR(st.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); + elog(ERROR, "-B, --backup-path must be a path to directory"); } + if (!exists) + elog(ERROR, "Instance '%s' does not exist in this backup catalog", + instance_name); } } @@ -573,10 +554,15 @@ main(int argc, char *argv[]) if (backup_subcmd != ADD_INSTANCE_CMD && backup_subcmd != ARCHIVE_GET_CMD) { - if (backup_subcmd == CHECKDB_CMD) - config_read_opt(instanceState->instance_config_path, instance_options, ERROR, true, true); - else - config_read_opt(instanceState->instance_config_path, instance_options, ERROR, true, false); + config_read_opt(instanceState->backup_location, instanceState->instance_config_path, + instance_options, ERROR, true, &err); + + if (getErrno(err) == ENOENT && backup_subcmd == CHECKDB_CMD) + { + /* ignore */ + } + else if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Reading instance control"); /* * We can determine our location only after reading the configuration file, @@ -591,6 +577,16 @@ main(int argc, char *argv[]) config_get_opt_env(instance_options); } + /* reset, since it could be changed in setMyLocation above */ + if (catalogState) + catalogState->backup_location = pioDriveForLocation(FIO_BACKUP_HOST); + if (instanceState) + { + instanceState->backup_location = pioDriveForLocation(FIO_BACKUP_HOST); + instanceState->database_location = pioDriveForLocation(FIO_DB_HOST); + } + current.backup_location = pioDriveForLocation(FIO_BACKUP_HOST); + /* * Disable logging into file for archive-push and archive-get. * Note, that we should NOT use fio_is_remote() here, @@ -718,14 +714,6 @@ main(int argc, char *argv[]) if (!instance_config.conn_opt.pghost && instance_config.remote.host) instance_config.conn_opt.pghost = instance_config.remote.host; - /* Setup stream options. They are used in streamutil.c. */ - if (instance_config.conn_opt.pghost != NULL) - dbhost = pstrdup(instance_config.conn_opt.pghost); - if (instance_config.conn_opt.pgport != NULL) - dbport = pstrdup(instance_config.conn_opt.pgport); - if (instance_config.conn_opt.pguser != NULL) - dbuser = pstrdup(instance_config.conn_opt.pguser); - if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) { /* @@ -883,7 +871,7 @@ main(int argc, char *argv[]) if (wal_file_path == NULL) { /* 1st case */ - system_id = get_system_identifier(current_dir, FIO_DB_HOST, false); + system_id = get_system_identifier(instanceState->database_location, current_dir, false); join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR); } else @@ -899,10 +887,11 @@ main(int argc, char *argv[]) */ char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name); join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR); - if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST)) + if ($i(pioFilesAreSame, instanceState->database_location, + .file1 = stripped_wal_file_path, .file2 = archive_push_xlog_dir)) { /* 2nd case */ - system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false); + system_id = get_system_identifier(instanceState->database_location, instance_config.pgdata, false); /* archive_push_xlog_dir already have right value */ } else @@ -912,7 +901,7 @@ main(int argc, char *argv[]) else elog(ERROR, "Value specified to --wal_file_path is too long"); - system_id = get_system_identifier(current_dir, FIO_DB_HOST, true); + system_id = get_system_identifier(instanceState->database_location, current_dir, true); /* 3rd case if control file present -- i.e. system_id != 0 */ if (system_id == 0) @@ -938,14 +927,13 @@ main(int argc, char *argv[]) wal_file_name, instanceState->instance_name, instance_config.system_identifier, system_id); } -#if PG_VERSION_NUM >= 100000 if (temp_slot && perm_slot) elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option"); /* if slot name was not provided for temp slot, use default slot name */ if (!replication_slot && temp_slot) replication_slot = DEFAULT_TEMP_SLOT_NAME; -#endif + if (!replication_slot && perm_slot) replication_slot = DEFAULT_PERMANENT_SLOT_NAME; @@ -973,7 +961,11 @@ main(int argc, char *argv[]) case DELETE_INSTANCE_CMD: return do_delete_instance(instanceState); case INIT_CMD: - return do_init(catalogState); + { + int err = 0; + err = do_init(catalogState); + return err; + } case BACKUP_CMD: { current.stream = stream_wal; @@ -1043,7 +1035,7 @@ main(int argc, char *argv[]) do_show_config(); break; case SET_CONFIG_CMD: - do_set_config(instanceState, false); + do_set_config(instanceState); break; case SET_BACKUP_CMD: if (!backup_id_string) @@ -1051,7 +1043,7 @@ main(int argc, char *argv[]) do_set_backup(instanceState, current.backup_id, set_backup_params); break; case CHECKDB_CMD: - do_checkdb(need_amcheck, + do_checkdb(pioDriveForLocation(FIO_DB_HOST), need_amcheck, instance_config.conn_opt, instance_config.pgdata); break; case NO_CMD: @@ -1201,4 +1193,4 @@ opt_exclude_path(ConfigOption *opt, const char *arg) opt_parser_add_to_parray_helper(&exclude_absolute_paths_list, arg); else opt_parser_add_to_parray_helper(&exclude_relative_paths_list, arg); -} +} \ No newline at end of file diff --git a/src/pg_probackup.h b/src/pg_probackup.h index fa3bc4123..78ff15ab0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -10,12 +10,17 @@ #ifndef PG_PROBACKUP_H #define PG_PROBACKUP_H +/* real main */ +extern int pbk_main(int argc, char** argv); + +#include #include "postgres_fe.h" #include "libpq-fe.h" -#include "libpq-int.h" #include "access/xlog_internal.h" +#include "file_compat.h" + #include "utils/pg_crc.h" #include "catalog/pg_control.h" @@ -31,6 +36,7 @@ #include #endif +#include "extra.h" #include "utils/configuration.h" #include "utils/logger.h" #include "utils/remote.h" @@ -43,17 +49,8 @@ #include "pg_probackup_state.h" - -#ifdef WIN32 -#define __thread __declspec(thread) -#else -#include -#endif - -#if PG_VERSION_NUM >= 150000 -// _() is explicitly undefined in libpq-int.h -// https://github.com/postgres/postgres/commit/28ec316787674dd74d00b296724a009b6edc2fb0 -#define _(s) gettext(s) +#if defined(WIN32) && !(defined(_UCRT) && defined(__MINGW64__)) +#error Windows port requires compilation in MinGW64 UCRT environment #endif /* Wrap the code that we're going to delete after refactoring in this define*/ @@ -70,13 +67,8 @@ extern const char *PROGRAM_EMAIL; #define DATABASE_DIR "database" #define BACKUPS_DIR "backups" #define WAL_SUBDIR "wal" -#if PG_VERSION_NUM >= 100000 #define PG_XLOG_DIR "pg_wal" #define PG_LOG_DIR "log" -#else -#define PG_XLOG_DIR "pg_xlog" -#define PG_LOG_DIR "pg_log" -#endif #define PG_TBLSPC_DIR "pg_tblspc" #define PG_GLOBAL_DIR "global" #define BACKUP_CONTROL_FILE "backup.control" @@ -98,15 +90,10 @@ extern const char *PROGRAM_EMAIL; /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 -#define REPLICA_TIMEOUT_DEFAULT 300 #define LOCK_TIMEOUT 60 #define LOCK_STALE_TIMEOUT 30 #define LOG_FREQ 10 -/* Directory/File permission */ -#define DIR_PERMISSION (0700) -#define FILE_PERMISSION (0600) - /* 64-bit xid support for PGPRO_EE */ #ifndef PGPRO_EE #define XID_FMT "%u" @@ -121,8 +108,11 @@ extern const char *PROGRAM_EMAIL; #define STDIO_BUFSIZE 65536 #define ERRMSG_MAX_LEN 2048 +#define SMALL_CHUNK_SIZE (32 * 1024) #define CHUNK_SIZE (128 * 1024) +#define MEDIUM_CHUNK_SIZE (512 * 1024) #define LARGE_CHUNK_SIZE (4 * 1024 * 1024) +#define IN_BUF_SIZE (512 * 1024) #define OUT_BUF_SIZE (512 * 1024) /* retry attempts */ @@ -135,6 +125,32 @@ extern const char *PROGRAM_EMAIL; #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) +ft_inline bool +XRecPtrLooksGood(XLogRecPtr xlrp, uint64_t segsize) +{ + uint64_t off = xlrp % segsize; + + /* + * record start is good if + * - if it points after page header + * -- but remember: first segments' page's header is longer + */ + return (off >= SizeOfXLogLongPHD && XRecOffIsValid(off)); +} + +ft_inline bool +XRecEndLooksGood(XLogRecPtr xlrp, uint64_t segsize) +{ + uint64_t off = xlrp % segsize; + + /* + * record end is good if + * - it points to valid record start + * - or it points to block/segment start (actually, end of previous) + */ + return XRecOffIsNull(off) || XRecPtrLooksGood(xlrp, segsize); +} + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ #define base36bufsize 14 @@ -161,12 +177,6 @@ typedef struct RedoParams uint32 checksum_version; } RedoParams; -typedef struct PageState -{ - uint16 checksum; - XLogRecPtr lsn; -} PageState; - typedef struct db_map_entry { Oid dbOid; @@ -207,14 +217,6 @@ typedef enum RecoverySettingsMode * if not explicitly forbidden */ } RecoverySettingsMode; -typedef enum CompressAlg -{ - NOT_DEFINED_COMPRESS = 0, - NONE_COMPRESS, - PGLZ_COMPRESS, - ZLIB_COMPRESS, -} CompressAlg; - typedef enum ForkName { none, @@ -225,28 +227,6 @@ typedef enum ForkName ptrack } ForkName; -#define INIT_FILE_CRC32(use_crc32c, crc) \ -do { \ - if (use_crc32c) \ - INIT_CRC32C(crc); \ - else \ - INIT_TRADITIONAL_CRC32(crc); \ -} while (0) -#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \ -do { \ - if (use_crc32c) \ - COMP_CRC32C((crc), (data), (len)); \ - else \ - COMP_TRADITIONAL_CRC32(crc, data, len); \ -} while (0) -#define FIN_FILE_CRC32(use_crc32c, crc) \ -do { \ - if (use_crc32c) \ - FIN_CRC32C(crc); \ - else \ - FIN_TRADITIONAL_CRC32(crc); \ -} while (0) - #define pg_off_t unsigned long long @@ -254,18 +234,21 @@ do { \ typedef struct pgFile { char *name; /* file or directory name */ - mode_t mode; /* protection (file type and permission) */ - size_t size; /* size of the file */ - time_t mtime; /* file st_mtime attribute, can be used only - during backup */ - size_t read_size; /* size of the portion read (if only some pages are + + pio_file_kind_e kind; /* kind of file */ + uint32_t mode; /* protection (permission) */ + int64_t size; /* size of the file */ + + int64_t read_size; /* size of the portion read (if only some pages are backed up, it's different from size) */ - int64 write_size; /* size of the backed-up file. BYTES_INVALID means + int64_t write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ - size_t uncompressed_size; /* size of the backed-up file before compression + int64_t uncompressed_size; /* size of the backed-up file before compression * and adding block headers. */ + time_t mtime; /* file st_mtime attribute, can be used only + during backup */ /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ char *rel_path; /* relative path of the file */ @@ -292,6 +275,10 @@ typedef struct pgFile pg_off_t hdr_off; /* offset in header map */ int hdr_size; /* length of headers */ bool excluded; /* excluded via --exclude-path option */ + + /* hash table entry fields */ + uint32_t hash; + struct pgFile* next; } pgFile; typedef struct page_map_entry @@ -326,15 +313,6 @@ typedef enum BackupStatus BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ } BackupStatus; -typedef enum BackupMode -{ - BACKUP_MODE_INVALID = 0, - BACKUP_MODE_DIFF_PAGE, /* incremental page backup */ - BACKUP_MODE_DIFF_PTRACK, /* incremental page backup with ptrack system */ - BACKUP_MODE_DIFF_DELTA, /* incremental page backup with lsn comparison */ - BACKUP_MODE_FULL /* full backup */ -} BackupMode; - typedef enum ShowFormat { SHOW_PLAIN, @@ -347,11 +325,11 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.11" +#define PROGRAM_VERSION "2.6.0" /* update when remote agent API or behaviour changes */ -#define AGENT_PROTOCOL_VERSION 20509 -#define AGENT_PROTOCOL_VERSION_STR "2.5.9" +#define AGENT_PROTOCOL_VERSION 20600 +#define AGENT_PROTOCOL_VERSION_STR "2.6.0" /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" @@ -391,9 +369,6 @@ typedef struct InstanceConfig char *external_dir_str; ConnectionOptions conn_opt; - ConnectionOptions master_conn_opt; - - uint32 replica_timeout; //Deprecated. Not used anywhere /* Wait timeout for WAL segment archiving */ uint32 archive_timeout; @@ -444,9 +419,7 @@ typedef struct PGNodeInfo typedef struct HeaderMap { char path[MAXPGPATH]; - char path_tmp[MAXPGPATH]; /* used only in merge */ - FILE *fp; /* used only for writing */ - char *buf; /* buffer */ + pioWriteCloser_i fp; /* used only for writing */ pg_off_t offset; /* current position in fp */ pthread_mutex_t mutex; @@ -528,6 +501,7 @@ struct pgBackup backup_path/instance_name/backup_id/database */ parray *files; /* list of files belonging to this backup * must be populated explicitly */ + parray *hashtable; /* hash table for faster file search */ char *note; pg_crc32 content_crc; @@ -535,6 +509,8 @@ struct pgBackup /* map used for access to page headers */ HeaderMap hdr_map; + pioDrive_i backup_location; /* Where to save to/read from */ + char backup_id_encoded[base36bufsize]; }; @@ -600,13 +576,14 @@ typedef struct pgSetBackupParams typedef struct { PGNodeInfo *nodeInfo; + struct InstanceState *instanceState; const char *from_root; const char *to_root; const char *external_prefix; parray *files_list; - parray *prev_filelist; + parray *prev_filehash; parray *external_dirs; XLogRecPtr prev_start_lsn; @@ -635,7 +612,7 @@ struct timelineInfo { XLogSegNo end_segno; /* last present segment in this timeline */ size_t n_xlog_files; /* number of segments (only really existing) * does not include lost segments */ - size_t size; /* space on disk taken by regular WAL files */ + int64_t size; /* space on disk taken by regular WAL files */ parray *backups; /* array of pgBackup sturctures with info * about backups belonging to this timeline */ parray *xlog_filelist; /* array of ordinary WAL segments, '.partial' @@ -671,7 +648,8 @@ typedef enum xlogFileType typedef struct xlogFile { - pgFile file; + ft_str_t name; + int64_t size; XLogSegNo segno; xlogFileType type; bool keep; /* Used to prevent removal of WAL segments @@ -722,8 +700,6 @@ typedef struct StopBackupCallbackParams strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) -#if PG_VERSION_NUM >= 110000 - #define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ XLogSegmentOffset(xlogptr, wal_segsz_bytes) #define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ @@ -744,59 +720,33 @@ typedef struct StopBackupCallbackParams #define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) -#else -#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ - ((xlogptr) & ((XLogSegSize) - 1)) -#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ - XLByteToSeg(xlrp, logSegNo) -#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ - XLogSegNoOffsetToRecPtr(segno, offset, dest) -#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ - XLogFileName(fname, tli, logSegNo) -#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ - XLByteInSeg(xlrp, logSegNo) -#define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \ - snprintf(fname, 20, "%08X%08X",\ - (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \ - (uint32) ((logSegNo) % XLogSegmentsPerXLogId)) - -#define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \ - logSegNo = (uint64) log * XLogSegmentsPerXLogId + seg - -#define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ - XLogFromFileName(fname, tli, logSegNo) -#endif #define IsPartialCompressXLogFileName(fname) \ - (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.partial") && \ - strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ - strcmp((fname) + XLOG_FNAME_LEN, ".gz.partial") == 0) + ((fname).len == XLOG_FNAME_LEN + strlen(".gz.partial") && \ + ft_str_spnc((fname), "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + ft_str_ends_withc((fname), ".gz.partial")) #define IsTempXLogFileName(fname) \ - (strlen(fname) == XLOG_FNAME_LEN + strlen(".part") && \ - strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ - strcmp((fname) + XLOG_FNAME_LEN, ".part") == 0) + ((fname).len == XLOG_FNAME_LEN + strlen("~tmp") + 6 && \ + ft_str_spnc((fname), "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + ft_str_find_cstr((fname), "~tmp") == XLOG_FNAME_LEN) #define IsTempCompressXLogFileName(fname) \ - (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.part") && \ - strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ - strcmp((fname) + XLOG_FNAME_LEN, ".gz.part") == 0) + ((fname).len == XLOG_FNAME_LEN + strlen(".gz~tmp") + 6 && \ + ft_str_spnc((fname), "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + ft_str_find_cstr((fname), ".gz~tmp") == XLOG_FNAME_LEN) #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) /* common options */ extern pid_t my_pid; -extern __thread int my_thread_num; extern int num_threads; extern bool stream_wal; extern bool show_color; extern bool progress; extern bool is_archive_cmd; /* true for archive-{get,push} */ -/* In pre-10 'replication_slot' is defined in receivelog.h */ extern char *replication_slot; -#if PG_VERSION_NUM >= 100000 extern bool temp_slot; -#endif extern bool perm_slot; /* backup options */ @@ -805,8 +755,6 @@ extern bool smooth_checkpoint; /* remote probackup options */ extern bool remote_agent; -extern bool exclusive_backup; - /* delete options */ extern bool delete_wal; extern bool delete_expired; @@ -836,6 +784,9 @@ typedef struct InstanceState //TODO split into some more meaningdul parts InstanceConfig *config; + + pioDrive_i database_location; + pioDrive_i backup_location; } InstanceState; /* ===== instanceState (END) ===== */ @@ -857,12 +808,15 @@ extern char** commands_args; /* in backup.c */ extern int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, bool no_validate, bool no_sync, bool backup_logs, time_t start_time); -extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, +extern void do_checkdb(pioDrive_i drive, bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); +extern void check_external_for_tablespaces(parray *external_list, + PGconn *backup_conn); +extern void backup_cleanup(bool fatal, void *userdata); /* in catchup.c */ extern int do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files, @@ -892,8 +846,6 @@ extern void reset_backup_id(pgBackup *backup); extern parray *get_backup_filelist(pgBackup *backup, bool strict); extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict); extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); -extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode); /* in remote.c */ extern void check_remote_agent_compatibility(int agent_version, @@ -921,8 +873,10 @@ extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instanc char *wal_file_name, int batch_size, bool validate_wal); /* in configure.c */ +extern int config_read_opt(pioDrive_i drive, const char *path, ConfigOption options[], int elevel, + bool strict, err_i *err); extern void do_show_config(void); -extern void do_set_config(InstanceState *instanceState, bool missing_ok); +extern void do_set_config(InstanceState *instanceState); extern void init_config(InstanceConfig *config, const char *instance_name); extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState); @@ -940,14 +894,6 @@ extern int do_delete_instance(InstanceState *instanceState); extern void do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, const char *status); -/* in fetch.c */ -extern char *slurpFile(const char *datadir, - const char *path, - size_t *filesize, - bool safe, - fio_location location); -extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); - /* in help.c */ extern void help_print_version(void); extern void help_pg_probackup(void); @@ -973,7 +919,9 @@ extern parray* get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli #define PAGE_LSN_FROM_FUTURE (-6) /* in catalog.c */ -extern pgBackup *read_backup(const char *root_dir); +extern CatalogState * catalog_new(const char *backup_path); +extern pgBackup *read_backup(pioDrive_i drive, const char *root_dir); +extern pgBackup *readBackupControlFile(pioDrive_i drive, const char *path); extern void write_backup(pgBackup *backup, bool strict); extern void write_backup_status(pgBackup *backup, BackupStatus status, bool strict); @@ -1002,14 +950,15 @@ extern void do_set_backup(InstanceState *instanceState, time_t backup_id, extern void pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params); extern void add_note(pgBackup *target_backup, char *note); -extern void pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc); +extern ft_str_t pgBackupWriteControl(pgBackup *backup, bool utc); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync); +extern InstanceState* makeInstanceState(CatalogState* catalogState, const char* name); extern void pgBackupInitDir(pgBackup *backup, const char *backup_instance_path); extern void pgNodeInit(PGNodeInfo *node); -extern void pgBackupInit(pgBackup *backup); +extern void pgBackupInit(pgBackup *backup, pioDrive_i drive); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); extern int pgBackupCompareIdDesc(const void *f1, const void *f2); @@ -1036,12 +985,24 @@ extern CompressAlg parse_compress_alg(const char *arg); extern const char* deparse_compress_alg(int alg); /* in dir.c */ -extern bool get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory); -extern bool get_control_value_str(const char *str, const char *name, - char *value_str, size_t value_str_size, bool is_mandatory); -extern void dir_list_file(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location); +typedef struct ft_arr_clkv_t ft_arr_clkv_t; +typedef struct pb_control_line pb_control_line; +struct pb_control_line { + ft_bytes_t line; + ft_arr_clkv_t *kvs; + ft_strbuf_t strbuf; +}; +extern void init_pb_control_line(pb_control_line *pb_line); +extern void parse_pb_control_line(pb_control_line *pb_line, ft_bytes_t line); +extern void deinit_pb_control_line(pb_control_line *pb_line); +extern int64_t pb_control_line_get_int64(pb_control_line *pb_line, const char *name); +extern ft_str_t pb_control_line_get_str(pb_control_line *pb_line, const char *name); +extern bool pb_control_line_try_int64(pb_control_line *pb_line, const char *name, int64 *value); +extern bool pb_control_line_try_str(pb_control_line *pb_line, const char *name, ft_str_t *value); + +extern void db_list_dir(parray *files, const char *root, bool exclude, + bool backup_logs, + int external_dir_num, fio_location location); extern const char *get_tablespace_mapping(const char *dir); extern void create_data_directories(parray *dest_files, @@ -1059,7 +1020,6 @@ extern int check_tablespace_mapping(pgBackup *backup, bool incremental, bool fo extern void check_external_dir_mapping(pgBackup *backup, bool incremental); extern char *get_external_remap(char *current_dir); -extern void print_database_map(FILE *out, parray *database_list); extern void write_database_map(pgBackup *backup, parray *database_list, parray *backup_file_list); extern void db_map_entry_free(void *map); @@ -1073,25 +1033,15 @@ extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path, const int dir_num); extern bool backup_contains_external(const char *dir, parray *dirs_list); -extern int dir_create_dir(const char *path, mode_t mode, bool strict); extern bool dir_is_empty(const char *path, fio_location location); extern bool fileExists(const char *path, fio_location location); -extern size_t pgFileSize(const char *path); extern pgFile *pgFileNew(const char *path, const char *rel_path, - bool follow_symlink, int external_dir_num, - fio_location location); + bool follow_symlink, bool crc, pioDrive_i drive); extern pgFile *pgFileInit(const char *rel_path); -extern void pgFileDelete(mode_t mode, const char *full_path); -extern void fio_pgFileDelete(pgFile *file, const char *full_path); - extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); -extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok); -extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok); - extern int pgFileMapComparePath(const void *f1, const void *f2); extern int pgFileCompareName(const void *f1, const void *f2); extern int pgFileCompareNameWithString(const void *f1, const void *f2); @@ -1099,49 +1049,41 @@ extern int pgFileCompareRelPathWithString(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); -extern int pgFileCompareSize(const void *f1, const void *f2); extern int pgFileCompareSizeDesc(const void *f1, const void *f2); +extern int pgFileCompareByHdrOff(const void *f1, const void *f2); extern int pgCompareString(const void *str1, const void *str2); extern int pgPrefixCompareString(const void *str1, const void *str2); extern int pgCompareOid(const void *f1, const void *f2); extern void pfilearray_clear_locks(parray *file_list); extern bool set_forkname(pgFile *file); +extern parray* make_filelist_hashtable(parray* files); +extern pgFile* search_file_in_hashtable(parray* buckets, pgFile* file); + /* in data.c */ -extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, - const char *from_fullpath, uint32 checksum_version); +extern bool check_data_file(pgFile *file, const char *from_fullpath, uint32 checksum_version); extern void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr sync_lsn, BackupMode backup_mode, - uint32 checksum_version, size_t prev_size); + uint32 checksum_version, int64_t prev_size); extern void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - HeaderMap *hdr_map, bool missing_ok); -extern void backup_non_data_file(pgFile *file, pgFile *prev_file, + HeaderMap *hdr_map, bool missing_ok, bool sync); +extern void backup_non_data_file(pioDrive_i from, pioDrive_i to, + pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, - bool missing_ok); -extern void backup_non_data_file_internal(const char *from_fullpath, - fio_location from_location, - const char *to_fullpath, pgFile *file, - bool missing_ok); + bool missing_ok, bool sync); -extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, +extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, pioDBWriter_i out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool use_headers); -extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map, PageState *checksum_map, int checksum_version, - datapagemap_t *lsn_map, BackupPageHeader2 *headers); -extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, - pgFile *dest_file, FILE *out, const char *to_fullpath, +extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, + pioDrive_i out_drive, pioDBWriter_i out, const char *to_fullpath, bool already_exists); -extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, - const char *from_fullpath, const char *to_fullpath); -extern bool create_empty_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file); +extern bool create_empty_file(const char *to_root, fio_location to_location, pgFile *file); extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); @@ -1150,6 +1092,7 @@ extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); +extern void header_map_cache_init(void); extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict); extern void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge); extern void init_header_map(pgBackup *backup); @@ -1169,37 +1112,28 @@ extern bool read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, time_t *recovery_time); -extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, - TimeLineID target_tli, uint32 seg_size); extern XLogRecPtr get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, - XLogRecPtr stop_lsn, TimeLineID tli, - bool seek_prev_segment, uint32 seg_size); - -extern XLogRecPtr get_first_record_lsn(const char *archivedir, XLogRecPtr start_lsn, - TimeLineID tli, uint32 wal_seg_size, int timeout); -extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, TimeLineID tli, - uint32 wal_seg_size, int timeout, XLogRecPtr target); + XLogRecPtr stop_lsn, TimeLineID tli, uint32 seg_size); /* in util.c */ extern TimeLineID get_current_timeline(PGconn *conn); -extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe); extern XLogRecPtr get_checkpoint_location(PGconn *conn); -extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe); +extern uint64 get_system_identifier(pioDrive_i drive, const char *pgdata_path, bool safe); extern uint64 get_remote_system_identifier(PGconn *conn); -extern uint32 get_data_checksum_version(bool safe); -extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); -extern uint32 get_xlog_seg_size(const char *pgdata_path); -extern void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo); -extern void set_min_recovery_point(pgFile *file, const char *backup_path, +extern pg_crc32c get_pgcontrol_checksum(pioDrive_i drive, const char *pgdata_path); +extern uint32 get_xlog_seg_size(pioDrive_i drive, const char *pgdata_path); +extern void get_redo(pioDrive_i drive, const char *pgdata_path, RedoParams *redo); +extern void set_min_recovery_point(pioDrive_i drive_from, pioDrive_i drive_to, + pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); -extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, - const char *to_fullpath, fio_location to_location, pgFile *file); +extern void copy_pgcontrol_file(pioDrive_i drive_from, const char *from_fullpath, + pioDrive_i drive_to, const char *to_fullpath, pgFile *file); extern void time2iso(char *buf, size_t len, time_t time, bool utc); extern const char *status2str(BackupStatus status); const char *status2str_color(BackupStatus status); extern BackupStatus str2status(const char *status); -extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]); +extern const char *base36enc_to(long unsigned int value, char buf[static base36bufsize]); /* Abuse C99 Compound Literal's lifetime */ #define base36enc(value) (base36enc_to((value), (char[base36bufsize]){0})) extern long unsigned int base36dec(const char *text); @@ -1215,7 +1149,7 @@ extern void pretty_size(int64 size, char *buf, size_t len); extern void pretty_time_interval(double time, char *buf, size_t len); extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); -extern void check_system_identifiers(PGconn *conn, const char *pgdata); +extern void check_system_identifiers(pioDrive_i drive, PGconn *conn, const char *pgdata); extern void parse_filelist_filenames(parray *files, const char *root); /* in ptrack.c */ @@ -1233,51 +1167,13 @@ extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack /* open local file to writing */ extern FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size); -extern int send_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, - uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode); -extern int copy_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr prev_backup_start_lsn, - uint32 checksum_version, bool use_pagemap, - BackupMode backup_mode); - /* FIO */ -extern void setMyLocation(ProbackupSubcmd const subcmd); -extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); -extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, - XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber *err_blknum, char **errormsg, - BackupPageHeader2 **headers); -extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, - XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber *err_blknum, char **errormsg); -/* return codes for fio_send_pages */ -extern int fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg); -extern int fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail, - pgFile *file, char **errormsg); -extern int fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, - pgFile *file, char **errormsg); - -extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, - bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num); - extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); extern void pgut_setenv(const char *key, const char *val); extern void pgut_unsetenv(const char *key); -extern PageState *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, - XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location); - -extern datapagemap_t *fio_get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno, - fio_location location); -extern pid_t fio_check_postmaster(const char *pgdata, fio_location location); - -extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg); - -/* return codes for fio_send_pages() and fio_send_file() */ +/* return codes for fio_send_file() */ #define SEND_OK (0) #define FILE_MISSING (-1) #define OPEN_FAILED (-2) @@ -1287,10 +1183,6 @@ extern int32 fio_decompress(void* dst, void const* src, size_t size, int compres #define REMOTE_ERROR (-6) #define PAGE_CORRUPTION (-8) -/* Check if specified location is local for current node */ -extern bool fio_is_remote(fio_location location); -extern bool fio_is_remote_simple(fio_location location); - extern void get_header_errormsg(Page page, char **errormsg); extern void get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno); @@ -1302,12 +1194,13 @@ extern void datapagemap_print_debug(datapagemap_t *map); /* in stream.c */ -extern XLogRecPtr stop_backup_lsn; extern void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, XLogRecPtr startpos, TimeLineID starttli, bool is_backup); -extern int wait_WAL_streaming_end(parray *backup_files_list); +extern void wait_WAL_streaming_starts(void); +extern int wait_WAL_streaming_end(parray *backup_files_list, const char* xlog_path, + XLogRecPtr stop_lsn, pgBackup* backup); extern parray* parse_tli_history_buffer(char *history, TimeLineID tli); /* external variables and functions, implemented in backup.c */ @@ -1323,10 +1216,8 @@ typedef struct PGStopBackupResult * Fields that store pg_catalog.pg_stop_backup() result */ XLogRecPtr lsn; - size_t backup_label_content_len; - char *backup_label_content; - size_t tablespace_map_content_len; - char *tablespace_map_content; + ft_str_t backup_label_content; + ft_str_t tablespace_map_content; } PGStopBackupResult; extern bool backup_in_progress; @@ -1334,18 +1225,28 @@ extern parray *backup_files_list; extern void pg_start_backup(const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn); +extern void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); extern void pg_silent_client_messages(PGconn *conn); extern void pg_create_restore_point(PGconn *conn, time_t backup_start_time); -extern void pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text); +extern void pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, char **query_text); extern void pg_stop_backup_consume(PGconn *conn, int server_version, - bool is_exclusive, uint32 timeout, const char *query_text, + uint32 timeout, const char *query_text, PGStopBackupResult *result); -extern void pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, - const void *data, size_t len, parray *file_list); +extern void pg_stop_backup_write_file_helper(pioDrive_i drive, const char *path, const char *filename, const char *error_msg_filename, + ft_str_t data, parray *file_list); extern XLogRecPtr wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, - bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir); -extern void wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup); + bool segment_only, int timeout_elevel); extern int64 calculate_datasize_of_filelist(parray *filelist); +/* + * Slices and arrays for C strings + */ +#define FT_SLICE cstr +#define FT_SLICE_TYPE char* +#include + +#define FT_SLICE str +#define FT_SLICE_TYPE ft_str_t +#include + #endif /* PG_PROBACKUP_H */ diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h index 56d852537..51f5dd092 100644 --- a/src/pg_probackup_state.h +++ b/src/pg_probackup_state.h @@ -19,7 +19,9 @@ typedef struct CatalogState char backup_subdir_path[MAXPGPATH]; /* $BACKUP_PATH/wal */ char wal_subdir_path[MAXPGPATH]; // previously global var arclog_path -} CatalogState; + + pioDrive_i backup_location; +} CatalogState; /* ====== CatalogState (END) ======= */ diff --git a/src/restore.c b/src/restore.c index 6c0e1881f..11928020f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -12,7 +12,6 @@ #include "access/timeline.h" -#include #include #include "utils/thread.h" @@ -41,10 +40,10 @@ typedef struct static void -print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, +print_recovery_settings(ft_strbuf_t *buf, InstanceState *instanceState, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt); static void -print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params); +print_standby_settings_common(ft_strbuf_t *buf, pgBackup *backup, pgRestoreParams *params); #if PG_VERSION_NUM >= 120000 static void @@ -63,11 +62,16 @@ static void create_recovery_conf(InstanceState *instanceState, time_t backup_id, static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); -static void restore_chain(pgBackup *dest_backup, parray *parent_chain, +static void restore_chain(InstanceState *instanceState, + pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, const char *pgdata_path, bool no_sync, bool cleanup_pgdata, bool backup_has_tblspc); +static DestDirIncrCompatibility check_incremental_compatibility(pioDrive_i dbdrive, + const char *pgdata, + uint64 system_identifier, + IncrRestoreMode incremental_mode); /* * Iterate over backup list to find all ancestors of the broken parent_backup * and update their status to BACKUP_STATUS_ORPHAN @@ -154,7 +158,8 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", instance_config.pgdata); - rc = check_incremental_compatibility(instance_config.pgdata, + rc = check_incremental_compatibility(instanceState->database_location, + instance_config.pgdata, instance_config.system_identifier, params->incremental_mode); if (rc == POSTMASTER_IS_RUNNING) @@ -480,7 +485,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg { RedoParams redo; parray *timelines = NULL; - get_redo(instance_config.pgdata, FIO_DB_HOST, &redo); + get_redo(instanceState->database_location, instance_config.pgdata, &redo); if (redo.checksum_version == 0) elog(ERROR, "Incremental restore in 'lsn' mode require " @@ -666,7 +671,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg backup_id_of(dest_backup), dest_backup->server_version); - restore_chain(dest_backup, parent_chain, dbOid_exclude_list, params, + restore_chain(instanceState, dest_backup, parent_chain, dbOid_exclude_list, params, instance_config.pgdata, no_sync, cleanup_pgdata, backup_has_tblspc); //TODO rename and update comment @@ -693,7 +698,8 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg * Flag 'cleanup_pgdata' demands the removing of already existing content in PGDATA. */ void -restore_chain(pgBackup *dest_backup, parray *parent_chain, +restore_chain(InstanceState *instanceState, + pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, const char *pgdata_path, bool no_sync, bool cleanup_pgdata, bool backup_has_tblspc) @@ -716,6 +722,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, size_t total_bytes = 0; char pretty_time[20]; time_t start_time, end_time; + err_i err = $noerr(); /* Preparations for actual restoring */ time2iso(timestamp, lengthof(timestamp), dest_backup->start_time, false); @@ -765,6 +772,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, * using bsearch. */ parray_qsort(backup->files, pgFileCompareRelPathWithExternal); + backup->hashtable = make_filelist_hashtable(backup->files); } /* If dest backup version is older than 2.4.0, then bitmap optimization @@ -812,8 +820,17 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(LOG, "Restore external directories"); for (i = 0; i < parray_num(external_dirs); i++) - fio_mkdir(parray_get(external_dirs, i), - DIR_PERMISSION, FIO_DB_HOST); + { + char *dirpath = parray_get(external_dirs, i); + + err = $i(pioMakeDir, instanceState->database_location, + .path = dirpath, .mode = DIR_PERMISSION, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not restore external directory: %s", + $errmsg(err)); + } + } } /* @@ -823,11 +840,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, { pgFile *file = (pgFile *) parray_get(dest_files, i); - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) total_bytes += 4096; if (!params->skip_external_dirs && - file->external_dir_num && S_ISDIR(file->mode)) + file->external_dir_num && file->kind == PIO_KIND_DIRECTORY) { char *external_path; char dirpath[MAXPGPATH]; @@ -839,7 +856,13 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, join_path_components(dirpath, external_path, file->rel_path); elog(LOG, "Create external directory \"%s\"", dirpath); - fio_mkdir(dirpath, file->mode, FIO_DB_HOST); + err = $i(pioMakeDir, instanceState->database_location, .path = dirpath, + .mode = file->mode, .strict = false); + if ($haserr(err)) + { + elog(ERROR, "Can not create backup external directory: %s", + $errmsg(err)); + } } } @@ -854,7 +877,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(INFO, "Extracting the content of destination directory for incremental restore"); time(&start_time); - fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); + db_list_dir(pgdata_files, pgdata_path, false, false, 0, FIO_DB_HOST); /* * TODO: @@ -874,8 +897,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, char *external_path = parray_get(external_dirs, i); parray *external_files = parray_new(); - fio_list_dir(external_files, external_path, - false, true, false, false, true, i+1); + db_list_dir(external_files, external_path, + false, false, i + 1, FIO_DB_HOST); parray_concat(pgdata_files, external_files); parray_free(external_files); @@ -898,7 +921,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, bool redundant = true; pgFile *file = (pgFile *) parray_get(pgdata_files, i); - if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal)) + if (search_file_in_hashtable(dest_backup->hashtable, file)) redundant = false; /* pg_filenode.map are always restored, because it's crc cannot be trusted */ @@ -907,7 +930,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, redundant = true; /* do not delete the useful internal directories */ - if (S_ISDIR(file->mode) && !redundant) + if (file->kind == PIO_KIND_DIRECTORY && !redundant) continue; /* if file does not exists in destination list, then we can safely unlink it */ @@ -917,8 +940,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, join_path_components(fullpath, pgdata_path, file->rel_path); - fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(LOG, "Deleted file \"%s\"", fullpath); + if (fio_remove(FIO_DB_HOST, fullpath, false) == 0) + elog(LOG, "Deleted file \"%s\"", fullpath); + else + elog(ERROR, "Cannot delete redundant file \"%s\": %s", fullpath, strerror(errno)); /* shrink pgdata list */ pgFileFree(file); @@ -962,6 +987,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, time(&start_time); thread_interrupted = false; + parray_qsort(dest_files, pgFileCompareByHdrOff); + /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { @@ -1027,40 +1054,20 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(WARNING, "Restored files are not synced to disk"); else { + pioDBDrive_i dbdrive = $bind(pioDBDrive, instanceState->database_location.self); elog(INFO, "Syncing restored files to disk"); time(&start_time); - for (i = 0; i < parray_num(dest_files); i++) - { - char to_fullpath[MAXPGPATH]; - pgFile *dest_file = (pgFile *) parray_get(dest_files, i); - - if (S_ISDIR(dest_file->mode)) - continue; - - /* skip external files if ordered to do so */ - if (dest_file->external_dir_num > 0 && - params->skip_external_dirs) - continue; - - /* construct fullpath */ - if (dest_file->external_dir_num == 0) - { - if (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) - continue; - if (strcmp(DATABASE_MAP, dest_file->rel_path) == 0) - continue; - join_path_components(to_fullpath, pgdata_path, dest_file->rel_path); - } - else - { - char *external_path = parray_get(external_dirs, dest_file->external_dir_num - 1); - join_path_components(to_fullpath, external_path, dest_file->rel_path); - } + err = $i(pioSyncTree, dbdrive, .root = pgdata_path); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Syncing pgdata"); - /* TODO: write test for case: file to be synced is missing */ - if (fio_sync(to_fullpath, FIO_DB_HOST) != 0) - elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno)); + for (i = 0; i < parray_num(external_dirs); i++) + { + char *external_path = parray_get(external_dirs, i); + err = $i(pioSyncTree, dbdrive, .root = external_path); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Syncin external dir %d", i); } time(&end_time); @@ -1088,6 +1095,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray_walk(backup->files, pgFileFree); parray_free(backup->files); + parray_free(backup->hashtable); } } @@ -1097,26 +1105,33 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, static void * restore_files(void *arg) { + FOBJ_FUNC_ARP(); int i; - uint64 n_files; + size_t n_files; char to_fullpath[MAXPGPATH]; - FILE *out = NULL; - char *out_buf = pgut_malloc(STDIO_BUFSIZE); + + pioDBWriter_i out; + pioDBDrive_i db_drive; restore_files_arg *arguments = (restore_files_arg *) arg; - n_files = (unsigned long) parray_num(arguments->dest_files); + header_map_cache_init(); + + n_files = parray_num(arguments->dest_files); + + db_drive = pioDBDriveForLocation(FIO_DB_HOST); for (i = 0; i < parray_num(arguments->dest_files); i++) { + FOBJ_LOOP_ARP(); bool already_exists = false; PageState *checksum_map = NULL; /* it should take ~1.5MB at most */ datapagemap_t *lsn_map = NULL; /* it should take 16kB at most */ - char *errmsg = NULL; /* remote agent error message */ pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); + err_i err = $noerr(); /* Directories were created before */ - if (S_ISDIR(dest_file->mode)) + if (dest_file->kind == PIO_KIND_DIRECTORY) continue; if (!pg_atomic_test_set_flag(&dest_file->lock)) @@ -1126,7 +1141,7 @@ restore_files(void *arg) if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during restore"); - elog(progress ? INFO : LOG, "Progress: (%d/%lu). Restore file \"%s\"", + elog(progress ? INFO : LOG, "Progress: (%d/%zu). Restore file \"%s\"", i + 1, n_files, dest_file->rel_path); /* Only files from pgdata can be skipped by partial restore */ @@ -1140,8 +1155,7 @@ restore_files(void *arg) * We cannot simply skip the file, because it may lead to * failure during WAL redo; hence, create empty file. */ - create_empty_file(FIO_BACKUP_HOST, - arguments->to_root, FIO_DB_HOST, dest_file); + create_empty_file(arguments->to_root, FIO_DB_HOST, dest_file); elog(LOG, "Skip file due to partial restore: \"%s\"", dest_file->rel_path); @@ -1197,40 +1211,31 @@ restore_files(void *arg) { if (arguments->incremental_mode == INCR_LSN) { - lsn_map = fio_get_lsn_map(to_fullpath, arguments->dest_backup->checksum_version, + lsn_map = fio_get_lsn_map(FIO_DB_HOST, to_fullpath, + arguments->dest_backup->checksum_version, dest_file->n_blocks, arguments->shift_lsn, - dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); + dest_file->segno * RELSEG_SIZE); } else if (arguments->incremental_mode == INCR_CHECKSUM) { - checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, + checksum_map = fio_get_checksum_map(FIO_DB_HOST, to_fullpath, + arguments->dest_backup->checksum_version, dest_file->n_blocks, arguments->dest_backup->stop_lsn, - dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); + dest_file->segno * RELSEG_SIZE); } } - /* - * Open dest file and truncate it to zero, if destination - * file already exists and dest file size is zero, or - * if file do not exist - */ - if ((already_exists && dest_file->write_size == 0) || !already_exists) - out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); - /* - * If file already exists and dest size is not zero, - * then open it for reading and writing. - */ - else - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_DB_HOST); + out = $i(pioOpenWrite, db_drive, .path = to_fullpath, + .permissions = dest_file->mode, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Cannot open restore target file"); - if (out == NULL) - elog(ERROR, "Cannot open restore target file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); + if (already_exists && dest_file->write_size == 0) + { + err = $i(pioTruncate, out, .size = 0); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Cannot truncate datafile"); + } if (!dest_file->is_datafile || dest_file->is_cfs) elog(LOG, "Restoring non-data file: \"%s\"", to_fullpath); @@ -1244,9 +1249,6 @@ restore_files(void *arg) /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) { - /* enable stdio buffering for local destination data file */ - if (!fio_is_remote_file(out)) - setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); /* Destination file is data file */ arguments->restored_bytes += restore_data_file(arguments->parent_chain, dest_file, out, to_fullpath, @@ -1255,24 +1257,18 @@ restore_files(void *arg) } else { - /* disable stdio buffering for local destination non-data file */ - if (!fio_is_remote_file(out)) - setvbuf(out, NULL, _IONBF, BUFSIZ); /* Destination file is non-data file */ arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, - arguments->dest_backup, dest_file, out, to_fullpath, - already_exists); + arguments->dest_backup, dest_file, + $reduce(pioDrive, db_drive), out, + to_fullpath, already_exists); } done: - /* Writing is asynchronous in case of restore in remote mode, so check the agent status */ - if (fio_check_error_file(out, &errmsg)) - elog(ERROR, "Cannot write to the remote file \"%s\": %s", to_fullpath, errmsg); - /* close file */ - if (fio_fclose(out) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, - strerror(errno)); + err = $i(pioClose, out); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Close restored file"); /* free pagemap used for restore optimization */ pg_free(dest_file->pagemap.bitmap); @@ -1284,8 +1280,6 @@ restore_files(void *arg) pg_free(checksum_map); } - free(out_buf); - /* ssh connection to longer needed */ fio_disconnect(); @@ -1356,20 +1350,25 @@ create_recovery_conf(InstanceState *instanceState, time_t backup_id, /* TODO get rid of using global variables: instance_config */ static void -print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, +print_recovery_settings(ft_strbuf_t *buf, InstanceState *instanceState, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt) { - char restore_command_guc[16384]; - fio_fprintf(fp, "## recovery settings\n"); + char restore_command_buf[1024]; + ft_strbuf_t restore_command_guc; + + restore_command_guc = ft_strbuf_init_stack(restore_command_buf, + sizeof(restore_command_buf)); + + ft_strbuf_catf(buf, "## recovery settings\n"); /* If restore_command is provided, use it. Otherwise construct it from scratch. */ if (instance_config.restore_command && (pg_strcasecmp(instance_config.restore_command, "none") != 0)) - sprintf(restore_command_guc, "%s", instance_config.restore_command); + ft_strbuf_catc(&restore_command_guc, instance_config.restore_command); else { /* default cmdline, ok for local restore */ - sprintf(restore_command_guc, "\"%s\" archive-get -B \"%s\" --instance \"%s\" " + ft_strbuf_catf(&restore_command_guc, "\"%s\" archive-get -B \"%s\" --instance \"%s\" " "--wal-file-path=%%p --wal-file-name=%%f", PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, /* TODO What is going on here? Why do we use catalog path as wal-file-path? */ @@ -1378,20 +1377,20 @@ print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup /* append --remote-* parameters provided via --archive-* settings */ if (instance_config.archive.host) { - strcat(restore_command_guc, " --remote-host="); - strcat(restore_command_guc, instance_config.archive.host); + ft_strbuf_catc(&restore_command_guc, " --remote-host="); + ft_strbuf_catc(&restore_command_guc, instance_config.archive.host); } if (instance_config.archive.port) { - strcat(restore_command_guc, " --remote-port="); - strcat(restore_command_guc, instance_config.archive.port); + ft_strbuf_catc(&restore_command_guc, " --remote-port="); + ft_strbuf_catc(&restore_command_guc, instance_config.archive.port); } if (instance_config.archive.user) { - strcat(restore_command_guc, " --remote-user="); - strcat(restore_command_guc, instance_config.archive.user); + ft_strbuf_catc(&restore_command_guc, " --remote-user="); + ft_strbuf_catc(&restore_command_guc, instance_config.archive.user); } } @@ -1400,26 +1399,26 @@ print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup * exclusive options is specified, so the order of calls is insignificant. */ if (rt->target_name) - fio_fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); + ft_strbuf_catf(buf, "recovery_target_name = '%s'\n", rt->target_name); if (rt->time_string) - fio_fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); + ft_strbuf_catf(buf, "recovery_target_time = '%s'\n", rt->time_string); if (rt->xid_string) - fio_fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); + ft_strbuf_catf(buf, "recovery_target_xid = '%s'\n", rt->xid_string); if (rt->lsn_string) - fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); + ft_strbuf_catf(buf, "recovery_target_lsn = '%s'\n", rt->lsn_string); if (rt->target_stop && (strcmp(rt->target_stop, "immediate") == 0)) - fio_fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); + ft_strbuf_catf(buf, "recovery_target = '%s'\n", rt->target_stop); if (rt->inclusive_specified) - fio_fprintf(fp, "recovery_target_inclusive = '%s'\n", + ft_strbuf_catf(buf, "recovery_target_inclusive = '%s'\n", rt->target_inclusive ? "true" : "false"); if (rt->target_tli) - fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); + ft_strbuf_catf(buf, "recovery_target_timeline = '%u'\n", rt->target_tli); else { #if PG_VERSION_NUM >= 120000 @@ -1429,31 +1428,33 @@ print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup * is extremely risky. Explicitly preserve old behavior of recovering to current * timneline for PG12. */ - fio_fprintf(fp, "recovery_target_timeline = 'current'\n"); + ft_strbuf_catf(buf, "recovery_target_timeline = 'current'\n"); #endif } if (rt->target_action) - fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); + ft_strbuf_catf(buf, "recovery_target_action = '%s'\n", rt->target_action); else /* default recovery_target_action is 'pause' */ - fio_fprintf(fp, "recovery_target_action = '%s'\n", "pause"); + ft_strbuf_catf(buf, "recovery_target_action = '%s'\n", "pause"); - elog(LOG, "Setting restore_command to '%s'", restore_command_guc); - fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); + elog(LOG, "Setting restore_command to '%s'", restore_command_guc.ptr); + ft_strbuf_catf(buf, "restore_command = '%s'\n", restore_command_guc.ptr); + + ft_strbuf_free(&restore_command_guc); } static void -print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params) +print_standby_settings_common(ft_strbuf_t *buf, pgBackup *backup, pgRestoreParams *params) { - fio_fprintf(fp, "\n## standby settings\n"); + ft_strbuf_catf(buf, "\n## standby settings\n"); if (params->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", params->primary_conninfo); + ft_strbuf_catf(buf, "primary_conninfo = '%s'\n", params->primary_conninfo); else if (backup->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + ft_strbuf_catf(buf, "primary_conninfo = '%s'\n", backup->primary_conninfo); if (params->primary_slot_name != NULL) - fio_fprintf(fp, "primary_slot_name = '%s'\n", params->primary_slot_name); + ft_strbuf_catf(buf, "primary_slot_name = '%s'\n", params->primary_slot_name); } #if PG_VERSION_NUM < 120000 @@ -1461,8 +1462,9 @@ static void update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt) { - FILE *fp; char path[MAXPGPATH]; + ft_strbuf_t buf = ft_strbuf_zero(); + err_i err; /* * If PITR is not requested and instance is not restored as replica, @@ -1474,33 +1476,27 @@ update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backu return; } - elog(LOG, "update recovery settings in recovery.conf"); - join_path_components(path, instance_config.pgdata, "recovery.conf"); - - fp = fio_fopen(path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); - - if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno)); - - fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", + ft_strbuf_catf(&buf, "# recovery.conf generated by pg_probackup %s\n", PROGRAM_VERSION); if (params->recovery_settings_mode == PITR_REQUESTED) - print_recovery_settings(instanceState, fp, backup, params, rt); + print_recovery_settings(&buf, instanceState, backup, params, rt); if (params->restore_as_replica) { - print_standby_settings_common(fp, backup, params); - fio_fprintf(fp, "standby_mode = 'on'\n"); + print_standby_settings_common(&buf, backup, params); + ft_strbuf_catc(&buf, "standby_mode = 'on'\n"); } - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, - strerror(errno)); + elog(LOG, "update recovery settings in recovery.conf"); + + join_path_components(path, instance_config.pgdata, "recovery.conf"); + + err = $i(pioWriteFile, instanceState->database_location, .path = path, + .content = ft_str2bytes(ft_strbuf_ref(&buf)), .binary = false); + + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "writting recovery settings"); } #endif @@ -1516,17 +1512,17 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, { char postgres_auto_path[MAXPGPATH]; - char postgres_auto_path_tmp[MAXPGPATH]; char path[MAXPGPATH]; - FILE *fp = NULL; - FILE *fp_tmp = NULL; - struct stat st; char current_time_str[100]; /* postgresql.auto.conf parsing */ - char line[16384] = "\0"; - char *buf = NULL; - int buf_len = 0; - int buf_len_max = 16384; + ft_bytes_t old_content; + ft_bytes_t parse; + ft_bytes_t line; + ft_bytes_t zero = ft_bytes(NULL, 0); + ft_strbuf_t content; + err_i err; + + content = ft_strbuf_zero(); elog(LOG, "update recovery settings in postgresql.auto.conf"); @@ -1534,100 +1530,49 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, join_path_components(postgres_auto_path, instance_config.pgdata, "postgresql.auto.conf"); - if (fio_stat(postgres_auto_path, &st, false, FIO_DB_HOST) < 0) - { - /* file not found is not an error case */ - if (errno != ENOENT) - elog(ERROR, "cannot stat file \"%s\": %s", postgres_auto_path, - strerror(errno)); - st.st_size = 0; - } + old_content = $i(pioReadFile, instanceState->database_location, + .path = postgres_auto_path, .err = &err, .binary = false); - /* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */ - if (st.st_size > 0) + /* file not found is not an error case */ + if ($haserr(err) && getErrno(err) != ENOENT) { - fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); + ft_logerr(FT_FATAL, $errmsg(err), ""); } - sprintf(postgres_auto_path_tmp, "%s.tmp", postgres_auto_path); - fp_tmp = fio_fopen(postgres_auto_path_tmp, "w", FIO_DB_HOST); - if (fp_tmp == NULL) - elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path_tmp, strerror(errno)); + parse = old_content; /* copy since ft_bytes_shift_line mutates bytes */ - while (fp && fgets(line, lengthof(line), fp)) + while (parse.len > 0) { + line = ft_bytes_shift_line(&parse); /* ignore "include 'probackup_recovery.conf'" directive */ - if (strstr(line, "include") && - strstr(line, "probackup_recovery.conf")) + if (ft_bytes_has_cstr(line, "include") && + ft_bytes_has_cstr(line, "probackup_recovery.conf")) { continue; } /* ignore already existing recovery options */ - if (strstr(line, "restore_command") || - strstr(line, "recovery_target")) + if (ft_bytes_has_cstr(line, "restore_command") || + ft_bytes_has_cstr(line, "recovery_target")) { continue; } - if (!buf) - buf = pgut_malloc(buf_len_max); - - /* avoid buffer overflow */ - if ((buf_len + strlen(line)) >= buf_len_max) - { - buf_len_max += (buf_len + strlen(line)) *2; - buf = pgut_realloc(buf, buf_len_max); - } - - buf_len += snprintf(buf+buf_len, sizeof(line), "%s", line); + ft_strbuf_catbytes(&content, line); } - /* close input postgresql.auto.conf */ - if (fp) - fio_close_stream(fp); - - /* Write data to postgresql.auto.conf.tmp */ - if (buf_len > 0 && - (fio_fwrite(fp_tmp, buf, buf_len) != buf_len)) - elog(ERROR, "Cannot write to \"%s\": %s", - postgres_auto_path_tmp, strerror(errno)); - - if (fio_fflush(fp_tmp) != 0 || - fio_fclose(fp_tmp)) - elog(ERROR, "Cannot write file \"%s\": %s", postgres_auto_path_tmp, - strerror(errno)); - pg_free(buf); - - if (fio_rename(postgres_auto_path_tmp, postgres_auto_path, FIO_DB_HOST) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - postgres_auto_path_tmp, postgres_auto_path, strerror(errno)); - - if (fio_chmod(postgres_auto_path, FILE_PERMISSION, FIO_DB_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", postgres_auto_path, strerror(errno)); + ft_bytes_free(&old_content); if (params) { - fp = fio_fopen(postgres_auto_path, "a", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", postgres_auto_path, - strerror(errno)); - - fio_fprintf(fp, "\n# recovery settings added by pg_probackup restore of backup %s at '%s'\n", + ft_strbuf_catf(&content, "\n# recovery settings added by pg_probackup restore of backup %s at '%s'\n", backup_id_of(backup), current_time_str); if (params->recovery_settings_mode == PITR_REQUESTED) - print_recovery_settings(instanceState, fp, backup, params, rt); + print_recovery_settings(&content, instanceState, backup, params, rt); if (params->restore_as_replica) - print_standby_settings_common(fp, backup, params); - - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", postgres_auto_path, - strerror(errno)); + print_standby_settings_common(&content, backup, params); /* * Create "recovery.signal" to mark this recovery as PITR for PostgreSQL. @@ -1644,15 +1589,11 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, elog(LOG, "creating recovery.signal file"); join_path_components(path, instance_config.pgdata, "recovery.signal"); - fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); + err = $i(pioWriteFile, instanceState->database_location, .path = path, + .content = zero); - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, - strerror(errno)); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "writting recovery.signal"); } if (params->restore_as_replica) @@ -1660,17 +1601,21 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, elog(LOG, "creating standby.signal file"); join_path_components(path, instance_config.pgdata, "standby.signal"); - fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); + err = $i(pioWriteFile, instanceState->database_location, .path = path, + .content = zero); - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, - strerror(errno)); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "writting standby.signal"); } } + + /* Write data to postgresql.auto.conf.tmp */ + err = $i(pioWriteFile, instanceState->database_location, + .path = postgres_auto_path, + .content = ft_str2bytes(ft_strbuf_ref(&content))); + ft_strbuf_free(&content); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "writting recovery options"); } #endif @@ -2001,7 +1946,6 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, { int i; int j; -// pg_crc32 crc; parray *database_map = NULL; parray *dbOid_exclude_list = NULL; pgFile *database_map_file = NULL; @@ -2031,13 +1975,6 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); - /* check database_map CRC */ -// crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST); -// -// if (crc != database_map_file->crc) -// elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X", -// database_map_file->path, crc, database_map_file->crc); - /* get database_map from file */ database_map = read_database_map(backup); @@ -2140,7 +2077,8 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, * TODO: add PG_CONTROL_IS_MISSING */ DestDirIncrCompatibility -check_incremental_compatibility(const char *pgdata, uint64 system_identifier, +check_incremental_compatibility(pioDrive_i dbdrive, const char *pgdata, + uint64 system_identifier, IncrRestoreMode incremental_mode) { uint64 system_id_pgdata; @@ -2152,7 +2090,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, char backup_label[MAXPGPATH]; /* check postmaster pid */ - pid = fio_check_postmaster(pgdata, FIO_DB_HOST); + pid = fio_check_postmaster(FIO_DB_HOST, pgdata); if (pid == 1) /* postmaster.pid is mangled */ { @@ -2165,8 +2103,8 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, } else if (pid > 1) /* postmaster is up */ { - elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"", - pid, pgdata); + elog(WARNING, "Postmaster with pid %llu is running in destination directory \"%s\"", + (long long)pid, pgdata); success = false; postmaster_is_up = true; } @@ -2184,14 +2122,14 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, */ elog(LOG, "Trying to read pg_control file in destination directory"); - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); + system_id_pgdata = get_system_identifier(dbdrive, pgdata, false); - if (system_id_pgdata == instance_config.system_identifier) + if (system_id_pgdata == system_identifier) system_id_match = true; else - elog(WARNING, "Backup catalog was initialized for system id %lu, " - "but destination directory system id is %lu", - system_identifier, system_id_pgdata); + elog(WARNING, "Backup catalog was initialized for system id %llu, " + "but destination directory system id is %llu", + (long long)system_identifier, (long long)system_id_pgdata); /* * TODO: maybe there should be some other signs, pointing to pg_control @@ -2199,8 +2137,10 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, */ if (incremental_mode == INCR_LSN) { + err_i err = $noerr(); + join_path_components(backup_label, pgdata, "backup_label"); - if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) + if($i(pioExists, dbdrive, .path = backup_label, .err = &err)) { elog(WARNING, "Destination directory contains \"backup_control\" file. " "This does NOT mean that you should delete this file and retry, only that " diff --git a/src/show.c b/src/show.c index 2e06582ed..71ca0a0a8 100644 --- a/src/show.c +++ b/src/show.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "utils/json.h" @@ -72,7 +71,8 @@ static PQExpBufferData show_buf; static bool first_instance = true; static int32 json_level = 0; -static const char* lc_env_locale; +static const char* lc_env_locale_numeric; +static const char* lc_env_locale_time; typedef enum { LOCALE_C, // Used for formatting output to unify the dot-based floating point representation LOCALE_ENV // Default environment locale @@ -82,11 +82,14 @@ typedef enum { static locale_t env_locale, c_locale; #endif void memorize_environment_locale() { - lc_env_locale = (const char *)getenv("LC_NUMERIC"); - lc_env_locale = lc_env_locale != NULL ? lc_env_locale : "C"; + lc_env_locale_numeric = (const char *)getenv("LC_NUMERIC"); + lc_env_locale_numeric = lc_env_locale_numeric != NULL ? lc_env_locale_numeric : "C"; + lc_env_locale_time = (const char *)getenv("LC_TIME"); + lc_env_locale_time = lc_env_locale_time != NULL ? lc_env_locale_time : "C"; #ifdef HAVE_USELOCALE - env_locale = newlocale(LC_NUMERIC_MASK, lc_env_locale, (locale_t)0); - c_locale = newlocale(LC_NUMERIC_MASK, "C", (locale_t)0); + env_locale = newlocale(LC_NUMERIC_MASK, lc_env_locale_numeric, (locale_t)0); + env_locale = newlocale(LC_TIME_MASK, lc_env_locale_time, env_locale); + c_locale = newlocale(LC_NUMERIC_MASK|LC_TIME_MASK, "C", (locale_t)0); #else #ifdef HAVE__CONFIGTHREADLOCALE _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); @@ -105,7 +108,8 @@ static void set_output_numeric_locale(output_numeric_locale loc) { #ifdef HAVE_USELOCALE uselocale(loc == LOCALE_C ? c_locale : env_locale); #else - setlocale(LC_NUMERIC, loc == LOCALE_C ? "C" : lc_env_locale); + setlocale(LC_NUMERIC, loc == LOCALE_C ? "C" : lc_env_locale_numeric); + setlocale(LC_TIME, loc == LOCALE_C ? "C" : lc_env_locale_time); #endif } @@ -519,7 +523,13 @@ show_backup(InstanceState *instanceState, time_t requested_backup_id) } if (show_format == SHOW_PLAIN) - pgBackupWriteControl(stdout, backup, false); + { + ft_str_t buf = pgBackupWriteControl(backup, false); + + fwrite(buf.ptr, 1, buf.len, stdout); + + ft_str_free(&buf); + } else elog(ERROR, "Invalid show format %d", (int) show_format); @@ -911,7 +921,7 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, cur++; /* N files */ - snprintf(row->n_segments, lengthof(row->n_segments), "%lu", + snprintf(row->n_segments, lengthof(row->n_segments), "%zu", tlinfo->n_xlog_files); widths[cur] = Max(widths[cur], strlen(row->n_segments)); cur++; @@ -931,7 +941,7 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, cur++; /* N backups */ - snprintf(row->n_backups, lengthof(row->n_backups), "%lu", + snprintf(row->n_backups, lengthof(row->n_backups), "%zu", tlinfo->backups?parray_num(tlinfo->backups):0); widths[cur] = Max(widths[cur], strlen(row->n_backups)); cur++; @@ -1087,10 +1097,10 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, json_add_value(buf, "max-segno", tmp_buf, json_level, true); json_add_key(buf, "n-segments", json_level); - appendPQExpBuffer(buf, "%lu", tlinfo->n_xlog_files); + appendPQExpBuffer(buf, "%zu", tlinfo->n_xlog_files); json_add_key(buf, "size", json_level); - appendPQExpBuffer(buf, "%lu", tlinfo->size); + appendPQExpBuffer(buf, "%lld", (long long)tlinfo->size); json_add_key(buf, "zratio", json_level); diff --git a/src/stream.c b/src/stream.c index f7bbeae5a..22395e878 100644 --- a/src/stream.c +++ b/src/stream.c @@ -2,7 +2,7 @@ * * stream.c: pg_probackup specific code for WAL streaming * - * Portions Copyright (c) 2015-2020, Postgres Professional + * Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -26,7 +26,8 @@ static int standby_message_timeout = 10 * 1000; /* stop_backup_lsn is set by pg_stop_backup() to stop streaming */ -XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; +/* TODO: use atomic */ +static volatile XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; /* @@ -37,6 +38,8 @@ static uint32 stream_stop_timeout = 0; /* Time in which we started to wait for streaming end */ static time_t stream_stop_begin = 0; +static StreamCtl stream_ctl = {0}; + /* * We need to wait end of WAL streaming before execute pg_stop_backup(). */ @@ -174,10 +177,10 @@ checkpoint_timeout(PGconn *backup_conn) * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, * bool is_temporary, bool is_physical, bool reserve_wal, * bool slot_exists_ok) - * PG 9.5-10 + * PG 10 * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, * bool is_physical, bool slot_exists_ok) - * NOTE: PG 9.6 and 10 support reserve_wal in + * NOTE: PG 10 support reserve_wal in * pg_catalog.pg_create_physical_replication_slot(slot_name name [, immediately_reserve boolean]) * and * CREATE_REPLICATION_SLOT slot_name { PHYSICAL [ RESERVE_WAL ] | LOGICAL output_plugin } @@ -188,26 +191,8 @@ CreateReplicationSlot_compat(PGconn *conn, const char *slot_name, const char *pl bool is_temporary, bool is_physical, bool slot_exists_ok) { -#if PG_VERSION_NUM >= 150000 - return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, - /* reserve_wal = */ true, slot_exists_ok, /* two_phase = */ false); -#elif PG_VERSION_NUM >= 110000 return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, /* reserve_wal = */ true, slot_exists_ok); -#elif PG_VERSION_NUM >= 100000 - /* - * PG-10 doesn't support creating temp_slot by calling CreateReplicationSlot(), but - * it will be created by setting StreamCtl.temp_slot later in StreamLog() - */ - if (!is_temporary) - return CreateReplicationSlot(conn, slot_name, plugin, /*is_temporary,*/ is_physical, /*reserve_wal,*/ slot_exists_ok); - else - return true; -#else - /* these parameters not supported in PG < 10 */ - Assert(!is_temporary); - return CreateReplicationSlot(conn, slot_name, plugin, /*is_temporary,*/ is_physical, /*reserve_wal,*/ slot_exists_ok); -#endif } /* @@ -216,6 +201,7 @@ CreateReplicationSlot_compat(PGconn *conn, const char *slot_name, const char *pl static void * StreamLog(void *arg) { + FOBJ_FUNC_ARP(); StreamThreadArg *stream_arg = (StreamThreadArg *) arg; /* @@ -229,13 +215,8 @@ StreamLog(void *arg) stream_stop_begin = 0; /* Create repslot */ -#if PG_VERSION_NUM >= 100000 if (temp_slot || perm_slot) if (!CreateReplicationSlot_compat(stream_arg->conn, replication_slot, NULL, temp_slot, true, false)) -#else - if (perm_slot) - if (!CreateReplicationSlot_compat(stream_arg->conn, replication_slot, NULL, false, true, false)) -#endif { interrupted = true; elog(ERROR, "Couldn't create physical replication slot %s", replication_slot); @@ -248,83 +229,42 @@ StreamLog(void *arg) elog(LOG, "started streaming WAL at %X/%X (timeline %u) using%s slot %s", (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, stream_arg->starttli, -#if PG_VERSION_NUM >= 100000 temp_slot ? " temporary" : "", -#else - "", -#endif replication_slot); else elog(LOG, "started streaming WAL at %X/%X (timeline %u)", (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, stream_arg->starttli); -#if PG_VERSION_NUM >= 90600 { - StreamCtl ctl; - - MemSet(&ctl, 0, sizeof(ctl)); - - ctl.startpos = stream_arg->startpos; - ctl.timeline = stream_arg->starttli; - ctl.sysidentifier = NULL; - ctl.stream_stop = stop_streaming; - ctl.standby_message_timeout = standby_message_timeout; - ctl.partial_suffix = NULL; - ctl.synchronous = false; - ctl.mark_done = false; - -#if PG_VERSION_NUM >= 100000 -#if PG_VERSION_NUM >= 150000 - ctl.walmethod = CreateWalDirectoryMethod( - stream_arg->basedir, - PG_COMPRESSION_NONE, - 0, - false); -#else /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 150000 */ - ctl.walmethod = CreateWalDirectoryMethod( + stream_ctl.startpos = stream_arg->startpos; + stream_ctl.timeline = stream_arg->starttli; + stream_ctl.sysidentifier = NULL; + stream_ctl.stream_stop = stop_streaming; + stream_ctl.standby_message_timeout = standby_message_timeout; + + stream_ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, -// (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, 0, - false); -#endif /* PG_VERSION_NUM >= 150000 */ - ctl.replication_slot = replication_slot; - ctl.stop_socket = PGINVALID_SOCKET; - ctl.do_sync = false; /* We sync all files at the end of backup */ -// ctl.mark_done /* for future use in s3 */ -#if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 - /* StreamCtl.temp_slot used only for PG-10, in PG>10, temp_slots are created by calling CreateReplicationSlot() */ - ctl.temp_slot = temp_slot; -#endif /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 */ -#else /* PG_VERSION_NUM < 100000 */ - ctl.basedir = (char *) stream_arg->basedir; -#endif /* PG_VERSION_NUM >= 100000 */ - - if (ReceiveXlogStream(stream_arg->conn, &ctl) == false) + false, + pioDriveForLocation(FIO_BACKUP_HOST)); + + stream_ctl.replication_slot = replication_slot; + stream_ctl.stop_socket = PGINVALID_SOCKET; + + if (ReceiveXlogStream(stream_arg->conn, &stream_ctl) == false) { interrupted = true; elog(ERROR, "Problem in receivexlog"); } -#if PG_VERSION_NUM >= 100000 - if (!ctl.walmethod->finish()) + if (!stream_ctl.walmethod->finish()) { interrupted = true; elog(ERROR, "Could not finish writing WAL files: %s", strerror(errno)); } -#endif /* PG_VERSION_NUM >= 100000 */ - } -#else /* PG_VERSION_NUM < 90600 */ - /* PG-9.5 */ - if (ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, - NULL, (char *) stream_arg->basedir, stop_streaming, - standby_message_timeout, NULL, false, false) == false) - { - interrupted = true; - elog(ERROR, "Problem in receivexlog"); } -#endif /* PG_VERSION_NUM >= 90600 */ /* be paranoid and sort xlog_files_list, * so if stop_lsn segno is already in the list, @@ -671,20 +611,88 @@ start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOption pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } +void +wait_WAL_streaming_starts(void) +{ + int timeout; + int try_count; + + if (instance_config.archive_timeout > 0) + timeout = instance_config.archive_timeout; + else + timeout = ARCHIVE_TIMEOUT_DEFAULT; + + for (try_count = 0; try_count < timeout; try_count++) + { + if (stream_ctl.currentpos) + break; + + /* Inform user if WAL streaming didn't start at first attempt */ + if (try_count == 1) + elog(INFO, "Wait for WAL streaming to start"); + + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during waiting for WAL streaming"); + + sleep(1); + } +} + /* * Wait for the completion of stream * append list of streamed xlog files * into backup_files_list (if it is not NULL) */ int -wait_WAL_streaming_end(parray *backup_files_list) +wait_WAL_streaming_end(parray *backup_files_list, const char* xlog_path, + XLogRecPtr stop_lsn, pgBackup* backup) { + XLogRecPtr prev; + + stop_backup_lsn = stop_lsn; + pthread_join(stream_thread, NULL); if(backup_files_list != NULL) parray_concat(backup_files_list, xlog_files_list); parray_free(xlog_files_list); - return stream_thread_arg.ret; + if (!stream_thread_arg.ret) + { + elog(INFO, "stop_stream_lsn %X/%X currentpos %X/%X", + (uint32_t)(stop_stream_lsn>>32), (uint32_t)stop_stream_lsn, + (uint32_t)(stream_ctl.currentpos>>32), (uint32_t)stream_ctl.currentpos + ); + } + + if (stream_thread_arg.ret) + return stream_thread_arg.ret; + + /* + * Actually we don't need to check for stop_lsn since we already + * know streamer stopped after stop_lsn. + * Just do it for sanity. + */ + prev = get_prior_record_lsn(xlog_path, + backup->start_lsn, + stop_lsn, backup->tli, + instance_config.xlog_seg_size); + + if (!XLogRecPtrIsInvalid(prev)) + { + /* LSN of the prior record was found */ + elog(LOG, "Found prior LSN: %X/%X", + (uint32) (prev >> 32), (uint32) prev); + /* so write stop_lsn */ + backup->stop_lsn = stop_lsn; + } + + if (XLogRecPtrIsInvalid(backup->stop_lsn)) + { + elog(ERROR, "Couldn't find stop_lsn for stop_stream_lsn %X/%X", + (uint32) (stop_stream_lsn>>32), (uint32) stop_stream_lsn); + } + + return stream_thread_arg.ret; } /* Append streamed WAL segment to filelist */ @@ -697,6 +705,7 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos char wal_segment_fullpath[MAXPGPATH]; pgFile *file = NULL; pgFile **existing_file = NULL; + pioDrive_i drive = pioDriveForLocation(FIO_BACKUP_HOST); GetXLogSegNo(xlogpos, xlog_segno, xlog_seg_size); @@ -714,7 +723,10 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos join_path_components(wal_segment_fullpath, basedir, wal_segment_name); join_path_components(wal_segment_relpath, PG_XLOG_DIR, wal_segment_name); - file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, 0, FIO_BACKUP_HOST); + file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, do_crc, drive); + ft_assert(file->size == xlog_seg_size, + "Wal segment file '%s' is of unexpected size %lld", + wal_segment_fullpath, (long long)file->size); /* * Check if file is already in the list @@ -726,22 +738,15 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos if (existing_file) { - if (do_crc) - (*existing_file)->crc = pgFileGetCRC(wal_segment_fullpath, true, false); - (*existing_file)->write_size = xlog_seg_size; - (*existing_file)->uncompressed_size = xlog_seg_size; + (*existing_file)->crc = file->crc; + (*existing_file)->size = file->size; + (*existing_file)->write_size = file->write_size; + (*existing_file)->uncompressed_size = file->uncompressed_size; + + pgFileFree(file); return; } - - if (do_crc) - file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); - - /* Should we recheck it using stat? */ - file->write_size = xlog_seg_size; - file->uncompressed_size = xlog_seg_size; - - /* append file to filelist */ parray_append(filelist, file); } @@ -753,6 +758,7 @@ add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) char fullpath[MAXPGPATH]; char relpath[MAXPGPATH]; pgFile *file = NULL; + pioDrive_i drive = pioDriveForLocation(FIO_BACKUP_HOST); /* Timeline 1 does not have a history file */ if (timeline == 1) @@ -762,14 +768,6 @@ add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) join_path_components(fullpath, basedir, filename); join_path_components(relpath, PG_XLOG_DIR, filename); - file = pgFileNew(fullpath, relpath, false, 0, FIO_BACKUP_HOST); - - /* calculate crc */ - if (do_crc) - file->crc = pgFileGetCRC(fullpath, true, false); - file->write_size = file->size; - file->uncompressed_size = file->size; - - /* append file to filelist */ + file = pgFileNew(fullpath, relpath, false, do_crc, drive); parray_append(filelist, file); -} +} \ No newline at end of file diff --git a/src/util.c b/src/util.c index e371d2c6d..1b4cbd7ae 100644 --- a/src/util.c +++ b/src/util.c @@ -3,7 +3,7 @@ * util.c: log messages to log file or stderr, and misc code. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -31,8 +31,15 @@ static const char *statusName[] = "CORRUPT" }; +static err_i +get_control_file(pioDrive_i drive, path_t pgdata_path, path_t file, + ControlFileData *control, bool safe); + +static TimeLineID +get_current_timeline_from_control(pioDrive_i drive, const char *pgdata_path); + const char * -base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]) +base36enc_to(long unsigned int value, char buf[static base36bufsize]) { const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; char buffer[base36bufsize]; @@ -80,62 +87,28 @@ checkControlFile(ControlFileData *ControlFile) "the PostgreSQL installation would be incompatible with this data directory."); } -/* - * Verify control file contents in the buffer src, and copy it to *ControlFile. - */ -static void -digestControlFile(ControlFileData *ControlFile, char *src, size_t size) -{ -#if PG_VERSION_NUM >= 100000 - int ControlFileSize = PG_CONTROL_FILE_SIZE; -#else - int ControlFileSize = PG_CONTROL_SIZE; -#endif - - if (size != ControlFileSize) - elog(ERROR, "unexpected control file size %d, expected %d", - (int) size, ControlFileSize); - - memcpy(ControlFile, src, sizeof(ControlFileData)); - - /* Additional checks on control file */ - checkControlFile(ControlFile); -} - /* * Write ControlFile to pg_control */ static void -writeControlFile(ControlFileData *ControlFile, const char *path, fio_location location) +writeControlFile(pioDrive_i drive, const char *path, ControlFileData *ControlFile) { - int fd; char *buffer = NULL; + err_i err; -#if PG_VERSION_NUM >= 100000 int ControlFileSize = PG_CONTROL_FILE_SIZE; -#else - int ControlFileSize = PG_CONTROL_SIZE; -#endif /* copy controlFileSize */ buffer = pg_malloc0(ControlFileSize); memcpy(buffer, ControlFile, sizeof(ControlFileData)); - /* Write pg_control */ - fd = fio_open(path, - O_RDWR | O_CREAT | O_TRUNC | PG_BINARY, location); - - if (fd < 0) - elog(ERROR, "Failed to open file: %s", path); - - if (fio_write(fd, buffer, ControlFileSize) != ControlFileSize) - elog(ERROR, "Failed to overwrite file: %s", path); + err = $i(pioWriteFile, drive, .path = path, + .content = ft_bytes(buffer, ControlFileSize)); - if (fio_flush(fd) != 0) - elog(ERROR, "Failed to sync file: %s", path); - - fio_close(fd); pg_free(buffer); + + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Writting control file"); } /* @@ -145,104 +118,118 @@ writeControlFile(ControlFileData *ControlFile, const char *path, fio_location lo TimeLineID get_current_timeline(PGconn *conn) { - PGresult *res; TimeLineID tli = 0; char *val; + bool ok = false; + pioDrive_i drive; res = pgut_execute_extended(conn, "SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()", 0, NULL, true, true); if (PQresultStatus(res) == PGRES_TUPLES_OK) + { val = PQgetvalue(res, 0, 0); - else - return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); + ok = parse_uint32(val, &tli, 0); + if (!ok) + /* TODO 3.0 just error out */ + elog(WARNING, "Invalid value of timeline_id %s", val); + } + PQclear(res); - if (!parse_uint32(val, &tli, 0)) - { - PQclear(res); - elog(WARNING, "Invalid value of timeline_id %s", val); + if (ok) + return tli; + + /* or get timeline from control data */ + drive = pioDriveForLocation(FIO_DB_HOST); + return get_current_timeline_from_control(drive, instance_config.pgdata); +} - /* TODO 3.0 remove it and just error out */ - return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); +static err_i +get_control_file(pioDrive_i drive, path_t pgdata_path, path_t file, + ControlFileData *control, bool safe) +{ + char fullpath[MAXPGPATH]; + ft_bytes_t bytes; + err_i err; + + fobj_reset_err(&err); + + join_path_components(fullpath, pgdata_path, file); + + bytes = $i(pioReadFile, drive, .path = fullpath, .err = &err); + if ($haserr(err) && safe) + { + ft_logerr(FT_WARNING, $errmsg(err), "Could not get control file"); + memset(control, 0, sizeof(ControlFileData)); + return $noerr(); } + if ($haserr(err)) + return $err(RT, "Could not get control file: {cause}", + cause(err.self)); + + if (bytes.len != PG_CONTROL_FILE_SIZE) + return $err(RT, "unexpected control file size: {size}, expected {wantedSz}", + size(bytes.len), wantedSz(PG_CONTROL_FILE_SIZE)); - return tli; + memcpy(control, bytes.ptr, sizeof(ControlFileData)); + ft_bytes_free(&bytes); + + /* Additional checks on control file */ + checkControlFile(control); + + return $noerr(); } /* Get timeline from pg_control file */ -TimeLineID -get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe) +static TimeLineID +get_current_timeline_from_control(pioDrive_i drive, const char *pgdata_path) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, - safe, location); - if (safe && buffer == NULL) - return 0; + err_i err; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(drive, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Getting current timeline"); return ControlFile.checkPointCopy.ThisTimeLineID; } /* - * Get last check point record ptr from pg_tonrol. + * Get last check point record ptr from pg_control. */ XLogRecPtr get_checkpoint_location(PGconn *conn) { -#if PG_VERSION_NUM >= 90600 PGresult *res; uint32 lsn_hi; uint32 lsn_lo; XLogRecPtr lsn; -#if PG_VERSION_NUM >= 100000 res = pgut_execute(conn, "SELECT checkpoint_lsn FROM pg_catalog.pg_control_checkpoint()", 0, NULL); -#else - res = pgut_execute(conn, - "SELECT checkpoint_location FROM pg_catalog.pg_control_checkpoint()", - 0, NULL); -#endif XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); PQclear(res); /* Calculate LSN */ lsn = ((uint64) lsn_hi) << 32 | lsn_lo; return lsn; -#else - char *buffer; - size_t size; - ControlFileData ControlFile; - - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.checkPoint; -#endif } uint64 -get_system_identifier(const char *pgdata_path, fio_location location, bool safe) +get_system_identifier(pioDrive_i drive, const char *pgdata_path, bool safe) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; + err_i err; - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, safe, location); - if (safe && buffer == NULL) - return 0; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(drive, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, safe); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Getting system identifier"); return ControlFile.system_identifier; } @@ -250,7 +237,6 @@ get_system_identifier(const char *pgdata_path, fio_location location, bool safe) uint64 get_remote_system_identifier(PGconn *conn) { -#if PG_VERSION_NUM >= 90600 PGresult *res; uint64 system_id_conn; char *val; @@ -267,31 +253,20 @@ get_remote_system_identifier(PGconn *conn) PQclear(res); return system_id_conn; -#else - char *buffer; - size_t size; - ControlFileData ControlFile; - - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.system_identifier; -#endif } uint32 -get_xlog_seg_size(const char *pgdata_path) +get_xlog_seg_size(pioDrive_i drive, const char *pgdata_path) { #if PG_VERSION_NUM >= 110000 + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; + err_i err; - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(drive, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Trying to fetch segment size"); return ControlFile.xlog_seg_size; #else @@ -299,52 +274,32 @@ get_xlog_seg_size(const char *pgdata_path) #endif } -uint32 -get_data_checksum_version(bool safe) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, - safe, FIO_DB_HOST); - if (buffer == NULL) - return 0; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.data_checksum_version; -} - pg_crc32c -get_pgcontrol_checksum(const char *pgdata_path) +get_pgcontrol_checksum(pioDrive_i drive, const char *pgdata_path) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; + err_i err; - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_BACKUP_HOST); - - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(drive, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Getting pgcontrol checksum"); return ControlFile.crc; } void -get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo) +get_redo(pioDrive_i drive, const char *pgdata_path, RedoParams *redo) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, pgdata_location); + err_i err; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(drive, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Fetching redo lsn"); redo->lsn = ControlFile.checkPointCopy.redo; redo->tli = ControlFile.checkPointCopy.ThisTimeLineID; @@ -371,17 +326,19 @@ get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo * 'as-is' is not to be trusted. */ void -set_min_recovery_point(pgFile *file, const char *backup_path, +set_min_recovery_point(pioDrive_i drive_from, pioDrive_i drive_to, + pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; - char fullpath[MAXPGPATH]; + char fullpath[MAXPGPATH]; + err_i err; - /* First fetch file content */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - digestControlFile(&ControlFile, buffer, size); + err = get_control_file(drive_from, instance_config.pgdata, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Set min recovery point"); elog(LOG, "Current minRecPoint %X/%X", (uint32) (ControlFile.minRecoveryPoint >> 32), @@ -401,37 +358,34 @@ set_min_recovery_point(pgFile *file, const char *backup_path, /* overwrite pg_control */ join_path_components(fullpath, backup_path, XLOG_CONTROL_FILE); - writeControlFile(&ControlFile, fullpath, FIO_LOCAL_HOST); + writeControlFile(drive_to, fullpath, &ControlFile); /* Update pg_control checksum in backup_list */ file->crc = ControlFile.crc; - - pg_free(buffer); } /* * Copy pg_control file to backup. We do not apply compression to this file. */ void -copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, - const char *to_fullpath, fio_location to_location, pgFile *file) +copy_pgcontrol_file(pioDrive_i drive_from, const char *from_fullpath, + pioDrive_i drive_to, const char *to_fullpath, pgFile *file) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; - - buffer = slurpFile(from_fullpath, "", &size, false, from_location); + err_i err; - digestControlFile(&ControlFile, buffer, size); + err = get_control_file(drive_from, from_fullpath, "", + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Fetching control file"); file->crc = ControlFile.crc; - file->read_size = size; - file->write_size = size; - file->uncompressed_size = size; - - writeControlFile(&ControlFile, to_fullpath, to_location); + file->read_size = PG_CONTROL_FILE_SIZE; + file->write_size = PG_CONTROL_FILE_SIZE; + file->uncompressed_size = PG_CONTROL_FILE_SIZE; - pg_free(buffer); + writeControlFile(drive_to, to_fullpath, &ControlFile); } /* @@ -552,14 +506,10 @@ datapagemap_is_set(datapagemap_t *map, BlockNumber blkno) void datapagemap_print_debug(datapagemap_t *map) { - datapagemap_iterator_t *iter; - BlockNumber blocknum; + BlockNumber blocknum = 0; - iter = datapagemap_iterate(map); - while (datapagemap_next(iter, &blocknum)) + for (;datapagemap_first(*map, &blocknum); blocknum++) elog(VERBOSE, " block %u", blocknum); - - pg_free(iter); } const char* diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 193d1c680..0d760abac 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -3,7 +3,7 @@ * configuration.c: - function implementations to work with pg_probackup * configurations. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -92,7 +92,7 @@ static const unit_conversion time_unit_conversion_table[] = }; /* Order is important, keep it in sync with utils/configuration.h:enum ProbackupSubcmd declaration */ -static char const * const subcmd_names[] = +static const char * const subcmd_names[] = { "NO_CMD", "init", @@ -118,7 +118,7 @@ static char const * const subcmd_names[] = }; ProbackupSubcmd -parse_subcmd(char const * const subcmd_str) +parse_subcmd(const char * const subcmd_str) { struct { ProbackupSubcmd id; @@ -141,7 +141,7 @@ parse_subcmd(char const * const subcmd_str) return NO_CMD; } -char const * +const char * get_subcmd_name(ProbackupSubcmd const subcmd) { Assert((int)subcmd < sizeof(subcmd_names) / sizeof(subcmd_names[0])); @@ -224,6 +224,16 @@ longopts_to_optstring(const struct option opts[], const size_t len) return result; } +static inline char +key_char(char c) +{ + /* '-', '_' and ' ' are equal */ + if (c == '_' || c == ' ') + return '-'; + else + return ToLower(c); +} + /* * Compare two strings ignore cases and ignore. */ @@ -231,15 +241,8 @@ static bool key_equals(const char *lhs, const char *rhs) { for (; *lhs && *rhs; lhs++, rhs++) - { - if (strchr("-_ ", *lhs)) - { - if (!strchr("-_ ", *rhs)) - return false; - } - else if (ToLower(*lhs) != ToLower(*rhs)) + if (key_char(*lhs) != key_char(*rhs)) return false; - } return *lhs == '\0' && *rhs == '\0'; } @@ -356,110 +359,90 @@ assign_option(ConfigOption *opt, const char *optarg, OptionSource src) } } -static const char * -skip_space(const char *str, const char *line) -{ - while (IsSpace(*str)) { str++; } - return str; -} - -static const char * -get_next_token(const char *src, char *dst, const char *line) +static bool +get_next_token(ft_bytes_t *src, ft_strbuf_t *dest) { - const char *s; - int i; - int j; + ft_bytes_t val; - if ((s = skip_space(src, line)) == NULL) - return NULL; + ft_bytes_consume(src, ft_bytes_spnc(*src, SPACES)); /* parse quoted string */ - if (*s == '\'') + if (ft_bytes_starts_withc(*src, "\'")) { - s++; - for (i = 0, j = 0; s[i] != '\0'; i++) + bool seen_quote = false; + + ft_bytes_consume(src, 1); + while (src->len) { - if (s[i] == '\'') + char c = src->ptr[0]; + ft_bytes_consume(src, 1); + /* doubled quote becomes just one quote */ + if (c == '\'' && seen_quote) { - i++; - /* doubled quote becomes just one quote */ - if (s[i] == '\'') - dst[j] = s[i]; - else - break; + ft_strbuf_cat1(dest, '\''); + seen_quote = false; } + else if (c == '\'') + seen_quote = true; + else if (seen_quote) /* previous char was closing quote */ + return true; else - dst[j] = s[i]; - j++; + ft_strbuf_cat1(dest, c); } + /* last char was closing quote */ + return seen_quote; } else { - i = j = strcspn(s, "#\n\r\t\v"); - memcpy(dst, s, j); + val = ft_bytes_split(src, ft_bytes_notspnc(*src, "#"SPACES)); + ft_strbuf_catbytes(dest, val); } - dst[j] = '\0'; - return s + i; + return true; } -static bool -parse_pair(const char buffer[], char key[], char value[]) -{ - const char *start; - const char *end; +enum pair_result { + PAIR_OK, + PAIR_EMPTY, + PAIR_ERROR, +}; - key[0] = value[0] = '\0'; +static enum pair_result +parse_pair(ft_bytes_t buffer, ft_strbuf_t *keybuf, ft_strbuf_t *valuebuf) +{ + ft_bytes_t key; /* * parse key */ - start = buffer; - if ((start = skip_space(start, buffer)) == NULL) - return false; - - end = start + strcspn(start, "=# \n\r\t\v"); + ft_bytes_consume(&buffer, ft_bytes_spnc(buffer, SPACES)); + key = ft_bytes_split(&buffer, ft_bytes_notspnc(buffer, "=#"SPACES)); + ft_bytes_consume(&buffer, ft_bytes_spnc(buffer, SPACES)); - /* skip blank buffer */ - if (end - start <= 0) + if (key.len == 0) { - if (*start == '=') - elog(ERROR, "Syntax error in \"%s\"", buffer); - return false; + if (ft_bytes_starts_withc(buffer, "=")) + return PAIR_ERROR; + return PAIR_EMPTY; } - /* key found */ - strncpy(key, start, end - start); - key[end - start] = '\0'; - - /* find key and value split char */ - if ((start = skip_space(end, buffer)) == NULL) - return false; + if (!ft_bytes_starts_withc(buffer, "=")) + return PAIR_ERROR; - if (*start != '=') - { - elog(ERROR, "Syntax error in \"%s\"", buffer); - return false; - } + ft_strbuf_catbytes(keybuf, key); - start++; + ft_bytes_consume(&buffer, 1); - /* - * parse value - */ - if ((end = get_next_token(start, value, buffer)) == NULL) - return false; + /* take value */ + if (!get_next_token(&buffer, valuebuf)) + return PAIR_ERROR; - if ((start = skip_space(end, buffer)) == NULL) - return false; + ft_bytes_consume(&buffer, ft_bytes_spnc(buffer, SPACES)); - if (*start != '\0' && *start != '#') - { - elog(ERROR, "Syntax error in \"%s\"", buffer); - return false; - } + if (buffer.len != 0 && buffer.ptr[0] != '#') + return PAIR_ERROR; - return true; + return PAIR_OK; } /* @@ -549,57 +532,60 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], * Return number of parsed options. */ int -config_read_opt(const char *path, ConfigOption options[], int elevel, - bool strict, bool missing_ok) +config_parse_opt(ft_bytes_t content, const char *path, + ConfigOption options[], int elevel, bool strict) { - FILE *fp; - char buf[4096]; - char key[1024]; - char value[2048]; + ft_strbuf_t key = ft_strbuf_zero(); + ft_strbuf_t value = ft_strbuf_zero(); int parsed_options = 0; + int lno = 0; if (!options) return parsed_options; - if ((fp = pgut_fopen(path, "rt", missing_ok)) == NULL) - return parsed_options; - - while (fgets(buf, lengthof(buf), fp)) + while (content.len > 0) { size_t i; + ft_bytes_t line = ft_bytes_shift_line(&content); + enum pair_result pr; + + lno++; + pr = parse_pair(line, &key, &value); + if (pr == PAIR_EMPTY) + continue; - for (i = strlen(buf); i > 0 && IsSpace(buf[i - 1]); i--) - buf[i - 1] = '\0'; + if (pr == PAIR_ERROR) + elog(ERROR, "Syntax error on %s:%d: %.*s", + path, lno, (int)line.len, line.ptr); - if (parse_pair(buf, key, value)) + for (i = 0; options[i].type; i++) { - for (i = 0; options[i].type; i++) - { - ConfigOption *opt = &options[i]; + ConfigOption *opt = &options[i]; - if (key_equals(key, opt->lname)) + if (key_equals(key.ptr, opt->lname)) + { + if (opt->allowed < SOURCE_FILE && + opt->allowed != SOURCE_FILE_STRICT) + elog(elevel, "Option %s cannot be specified in file", + opt->lname); + else if (opt->source <= SOURCE_FILE) { - if (opt->allowed < SOURCE_FILE && - opt->allowed != SOURCE_FILE_STRICT) - elog(elevel, "Option %s cannot be specified in file", - opt->lname); - else if (opt->source <= SOURCE_FILE) - { - assign_option(opt, value, SOURCE_FILE); - parsed_options++; - } - break; + assign_option(opt, value.ptr, SOURCE_FILE); + parsed_options++; } + break; } - if (strict && !options[i].type) - elog(elevel, "Invalid option \"%s\" in file \"%s\"", key, path); } - } - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", path); + if (strict && !options[i].type) + elog(elevel, "Invalid option \"%s\" in file \"%s\"", key.ptr, path); + + ft_strbuf_reset_for_reuse(&key); + ft_strbuf_reset_for_reuse(&value); + } - fio_close_stream(fp); + ft_strbuf_free(&key); + ft_strbuf_free(&value); return parsed_options; } diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 2c6ea3eec..12c70e5be 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -101,12 +101,12 @@ struct ConfigOption #define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) -extern ProbackupSubcmd parse_subcmd(char const * const subcmd_str); -extern char const *get_subcmd_name(ProbackupSubcmd const subcmd); +extern ProbackupSubcmd parse_subcmd(const char * const subcmd_str); +extern const char *get_subcmd_name(ProbackupSubcmd const subcmd); extern int config_get_opt(int argc, char **argv, ConfigOption cmd_options[], ConfigOption options[]); -extern int config_read_opt(const char *path, ConfigOption options[], int elevel, - bool strict, bool missing_ok); +extern int config_parse_opt(ft_bytes_t content, const char *path, + ConfigOption options[], int elevel, bool strict); extern void config_get_opt_env(ConfigOption options[]); extern void config_set_opt(ConfigOption options[], void *var, OptionSource source); diff --git a/src/utils/file.c b/src/utils/file.c index c4ed9c721..c9ed0a8f6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2,17 +2,14 @@ #include #include "pg_probackup.h" -/* sys/stat.h must be included after pg_probackup.h (see problems with compilation for windows described in PGPRO-5750) */ -#include +#include #include "file.h" #include "storage/checksum.h" #define PRINTF_BUF_SIZE 1024 -#define FILE_PERMISSIONS 0600 -static __thread unsigned long fio_fdset = 0; -static __thread void* fio_stdin_buffer; +static __thread uint64_t fio_fdset = 0; static __thread int fio_stdout = 0; static __thread int fio_stdin = 0; static __thread int fio_stderr = 0; @@ -22,6 +19,8 @@ static char *async_errormsg = NULL; #define PAGE_ZEROSEARCH_FINE_GRANULARITY 64 static const char zerobuf[PAGE_ZEROSEARCH_COARSE_GRANULARITY] = {0}; +#define PIO_DIR_REMOTE_BATCH 100 + fio_location MyLocation; typedef struct @@ -36,22 +35,16 @@ typedef struct int path_len; } fio_send_request; -typedef struct -{ - char path[MAXPGPATH]; - bool exclude; - bool follow_symlink; - bool add_root; - bool backup_logs; - bool exclusive_backup; - bool skip_hidden; - int external_dir_num; -} fio_list_dir_request; +typedef struct { + char path[MAXPGPATH]; + bool root_as_well; +} fio_remove_dir_request; typedef struct { + pio_file_kind_e kind; mode_t mode; - size_t size; + int64_t size; time_t mtime; bool is_datafile; Oid tblspcOid; @@ -79,13 +72,33 @@ typedef struct uint32 checksumVersion; } fio_lsn_map_request; +typedef struct __attribute__((packed)) +{ + int32_t segno; + int32_t pagemaplen; + XLogRecPtr start_lsn; + CompressAlg calg; + int clevel; + uint32 checksum_version; + int just_validate; +} fio_iterate_pages_request; + +struct __attribute__((packed)) fio_req_open_rewrite { + uint32_t permissions; + bool binary; + bool use_temp; + bool sync; +}; -/* Convert FIO pseudo handle to index in file descriptor array */ -#define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) +struct __attribute__((packed)) fio_req_open_write { + uint32_t permissions; + bool exclusive; + bool sync; +}; #if defined(WIN32) -#undef open(a, b, c) -#undef fopen(a, b) +#undef open +#undef fopen #endif void @@ -97,13 +110,28 @@ setMyLocation(ProbackupSubcmd const subcmd) elog(ERROR, "Currently remote operations on Windows are not supported"); #endif - MyLocation = IsSshProtocol() - ? (subcmd == ARCHIVE_PUSH_CMD || subcmd == ARCHIVE_GET_CMD) - ? FIO_DB_HOST - : (subcmd == BACKUP_CMD || subcmd == RESTORE_CMD || subcmd == ADD_INSTANCE_CMD || subcmd == CATCHUP_CMD) - ? FIO_BACKUP_HOST - : FIO_LOCAL_HOST - : FIO_LOCAL_HOST; + if (!IsSshProtocol()) + { + MyLocation = FIO_LOCAL_HOST; + return; + } + + switch (subcmd) + { + case ARCHIVE_GET_CMD: + case ARCHIVE_PUSH_CMD: + MyLocation = FIO_DB_HOST; + break; + case BACKUP_CMD: + case RESTORE_CMD: + case ADD_INSTANCE_CMD: + case CATCHUP_CMD: + MyLocation = FIO_BACKUP_HOST; + break; + default: + MyLocation = FIO_LOCAL_HOST; + break; + } } /* Use specified file descriptors as stdin/stdout for FIO functions */ @@ -116,7 +144,7 @@ fio_redirect(int in, int out, int err) } void -fio_error(int rc, int size, char const* file, int line) +fio_error(int rc, int size, const char* file, int line) { if (remote_agent) { @@ -138,49 +166,7 @@ fio_error(int rc, int size, char const* file, int line) } } -/* Check if file descriptor is local or remote (created by FIO) */ -static bool -fio_is_remote_fd(int fd) -{ - return (fd & FIO_PIPE_MARKER) != 0; -} - #ifdef WIN32 - -#undef stat - -/* - * The stat() function in win32 is not guaranteed to update the st_size - * field when run. So we define our own version that uses the Win32 API - * to update this field. - */ -static int -fio_safestat(const char *path, struct stat *buf) -{ - int r; - WIN32_FILE_ATTRIBUTE_DATA attr; - - r = stat(path, buf); - if (r < 0) - return r; - - if (!GetFileAttributesEx(path, GetFileExInfoStandard, &attr)) - { - errno = ENOENT; - return -1; - } - - /* - * XXX no support for large files here, but we don't do that in general on - * Win32 yet. - */ - buf->st_size = attr.nFileSizeLow; - - return 0; -} - -#define stat(x, y) fio_safestat(x, y) - /* TODO: use real pread on Linux */ static ssize_t pread(int fd, void* buf, size_t size, off_t off) @@ -190,21 +176,29 @@ pread(int fd, void* buf, size_t size, off_t off) return -1; return read(fd, buf, size); } +#endif /* WIN32 */ +#ifdef WIN32 static int -remove_file_or_dir(char const* path) +remove_file_or_dir(const char* path) { int rc = remove(path); -#ifdef WIN32 - if (rc < 0 && errno == EACCESS) + + if (rc < 0 && errno == EACCES) rc = rmdir(path); -#endif return rc; } #else #define remove_file_or_dir(path) remove(path) #endif +static void +fio_ensure_remote(void) +{ + if (!fio_stdin && !launch_agent()) + elog(ERROR, "Failed to establish SSH connection: %s", strerror(errno)); +} + /* Check if specified location is local for current node */ bool fio_is_remote(fio_location location) @@ -227,6 +221,31 @@ fio_is_remote_simple(fio_location location) return is_remote; } +static int +find_free_handle(void) +{ + uint64_t m = fio_fdset; + int i; + for (i = 0; (m & 1); i++, m>>=1) {} + if (i == FIO_FDMAX) { + elog(ERROR, "Descriptor pool for remote files is exhausted, " + "probably too many remote directories are opened"); + } + return i; +} + +static void +set_handle(int i) +{ + fio_fdset |= 1 << i; +} + +static void +unset_handle(int i) +{ + fio_fdset &= ~(1 << i); +} + /* Try to read specified amount of bytes unless error or EOF are encountered */ static ssize_t fio_read_all(int fd, void* buf, size_t size) @@ -276,9 +295,7 @@ fio_write_all(int fd, void const* buf, size_t size) void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) { - fio_header hdr; - hdr.cop = FIO_AGENT_VERSION; - hdr.size = 0; + fio_header hdr = (fio_header){.cop = FIO_AGENT_VERSION}; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); @@ -291,208 +308,117 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size); } -/* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ -FILE* -fio_open_stream(char const* path, fio_location location) +pio_file_kind_e +pio_statmode2file_kind(mode_t mode, const char* path) { - FILE* f; - if (fio_is_remote(location)) - { - fio_header hdr; - hdr.cop = FIO_LOAD; - hdr.size = strlen(path) + 1; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND); - if (hdr.size > 0) - { - Assert(fio_stdin_buffer == NULL); - fio_stdin_buffer = pgut_malloc(hdr.size); - IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); -#ifdef WIN32 - f = tmpfile(); - IO_CHECK(fwrite(fio_stdin_buffer, 1, hdr.size, f), hdr.size); - SYS_CHECK(fseek(f, 0, SEEK_SET)); -#else - f = fmemopen(fio_stdin_buffer, hdr.size, "r"); + pio_file_kind_e kind = PIO_KIND_UNKNOWN; + if (S_ISREG(mode)) + kind = PIO_KIND_REGULAR; + else if (S_ISDIR(mode)) + kind = PIO_KIND_DIRECTORY; +#ifdef S_ISLNK + else if (S_ISLNK(mode)) + kind = PIO_KIND_SYMLINK; +#endif +#ifdef S_ISFIFO + else if (S_ISFIFO(mode)) + kind = PIO_KIND_FIFO; +#endif +#ifdef S_ISSOCK + else if (S_ISFIFO(mode)) + kind = PIO_KIND_SOCK; +#endif +#ifdef S_ISCHR + else if (S_ISCHR(mode)) + kind = PIO_KIND_CHARDEV; +#endif +#ifdef S_ISBLK + else if (S_ISBLK(mode)) + kind = PIO_KIND_BLOCKDEV; #endif - } - else - { - f = NULL; - } - } else - { - f = fopen(path, "rt"); - } - return f; -} - -/* Close input stream */ -int -fio_close_stream(FILE* f) -{ - if (fio_stdin_buffer) - { - free(fio_stdin_buffer); - fio_stdin_buffer = NULL; - } - return fclose(f); + elog(ERROR, "Unsupported file mode kind \"%x\" for file '%s'", + mode, path); + return kind; } -/* Open directory */ -DIR* -fio_opendir(char const* path, fio_location location) +pio_file_kind_e +pio_str2file_kind(const char* str, const char* path) { - DIR* dir; - if (fio_is_remote(location)) - { - int i; - fio_header hdr; - unsigned long mask; - - mask = fio_fdset; - for (i = 0; (mask & 1) != 0; i++, mask >>= 1); - if (i == FIO_FDMAX) { - elog(ERROR, "Descriptor pool for remote files is exhausted, " - "probably too many remote directories are opened"); - } - hdr.cop = FIO_OPENDIR; - hdr.handle = i; - hdr.size = strlen(path) + 1; - fio_fdset |= 1 << i; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - fio_fdset &= ~(1 << hdr.handle); - return NULL; - } - dir = (DIR*)(size_t)(i + 1); - } + pio_file_kind_e kind = PIO_KIND_UNKNOWN; + if (strncmp(str, "reg", 3) == 0) + kind = PIO_KIND_REGULAR; + else if (strncmp(str, "dir", 3) == 0) + kind = PIO_KIND_DIRECTORY; + else if (strncmp(str, "sym", 3) == 0) + kind = PIO_KIND_SYMLINK; + else if (strncmp(str, "fifo", 4) == 0) + kind = PIO_KIND_FIFO; + else if (strncmp(str, "sock", 4) == 0) + kind = PIO_KIND_SOCK; + else if (strncmp(str, "chdev", 5) == 0) + kind = PIO_KIND_CHARDEV; + else if (strncmp(str, "bldev", 5) == 0) + kind = PIO_KIND_BLOCKDEV; else - { - dir = opendir(path); - } - return dir; + elog(ERROR, "Unknown file kind \"%s\" for file '%s'", + str, path); + return kind; } -/* Get next directory entry */ -struct dirent* -fio_readdir(DIR *dir) +const char* +pio_file_kind2str(pio_file_kind_e kind, const char *path) { - if (fio_is_remote_file((FILE*)dir)) - { - fio_header hdr; - static __thread struct dirent entry; - - hdr.cop = FIO_READDIR; - hdr.handle = (size_t)dir - 1; - hdr.size = 0; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND); - if (hdr.size) { - Assert(hdr.size == sizeof(entry)); - IO_CHECK(fio_read_all(fio_stdin, &entry, sizeof(entry)), sizeof(entry)); - } - - return hdr.size ? &entry : NULL; - } - else + switch (kind) { - return readdir(dir); - } + case PIO_KIND_REGULAR: + return "reg"; + case PIO_KIND_DIRECTORY: + return "dir"; + case PIO_KIND_SYMLINK: + return "sym"; + case PIO_KIND_FIFO: + return "fifo"; + case PIO_KIND_SOCK: + return "sock"; + case PIO_KIND_CHARDEV: + return "chdev"; + case PIO_KIND_BLOCKDEV: + return "bldev"; + default: + elog(ERROR, "Unknown file kind \"%d\" for file '%s'", + kind, path); + } + return NULL; } -/* Close directory */ -int -fio_closedir(DIR *dir) -{ - if (fio_is_remote_file((FILE*)dir)) - { - fio_header hdr; - hdr.cop = FIO_CLOSEDIR; - hdr.handle = (size_t)dir - 1; - hdr.size = 0; - fio_fdset &= ~(1 << hdr.handle); - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - return 0; - } - else - { - return closedir(dir); - } -} +#ifndef S_ISGID +#define S_ISGID 0 +#endif +#ifndef S_ISUID +#define S_ISUID 0 +#endif +#ifndef S_ISVTX +#define S_ISVTX 0 +#endif -/* Open file */ -int -fio_open(char const* path, int mode, fio_location location) +mode_t +pio_limit_mode(mode_t mode) { - int fd; - if (fio_is_remote(location)) - { - int i; - fio_header hdr; - unsigned long mask; - - mask = fio_fdset; - for (i = 0; (mask & 1) != 0; i++, mask >>= 1); - if (i == FIO_FDMAX) - elog(ERROR, "Descriptor pool for remote files is exhausted, " - "probably too many remote files are opened"); - - hdr.cop = FIO_OPEN; - hdr.handle = i; - hdr.size = strlen(path) + 1; - hdr.arg = mode; -// hdr.arg = mode & ~O_EXCL; -// elog(INFO, "PATH: %s MODE: %i, %i", path, mode, O_EXCL); -// elog(INFO, "MODE: %i", hdr.arg); - fio_fdset |= 1 << i; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - fio_fdset &= ~(1 << hdr.handle); - return -1; - } - fd = i | FIO_PIPE_MARKER; - } + if (S_ISDIR(mode)) + mode &= 0x1ff | S_ISGID | S_ISUID | S_ISVTX; else - { - fd = open(path, mode, FILE_PERMISSIONS); - } - return fd; + mode &= 0x1ff; + return mode; } - /* Close ssh session */ void fio_disconnect(void) { if (fio_stdin) { - fio_header hdr; - hdr.cop = FIO_DISCONNECT; - hdr.size = 0; + fio_header hdr = (fio_header){.cop = FIO_DISCONNECT}; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_DISCONNECTED); @@ -506,3486 +432,5284 @@ fio_disconnect(void) } } -/* Open stdio file */ -FILE* -fio_fopen(char const* path, char const* mode, fio_location location) +/* Close remote file implementation */ +static void +fio_close_impl(int fd, int out) { - FILE *f = NULL; + fio_header hdr = { + .cop = FIO_CLOSE, + .handle = -1, + .size = 0, + .arg = 0, + }; - if (fio_is_remote(location)) - { - int flags = 0; - int fd; - if (strcmp(mode, PG_BINARY_W) == 0) { - flags = O_TRUNC|PG_BINARY|O_RDWR|O_CREAT; - } else if (strcmp(mode, "w") == 0) { - flags = O_TRUNC|O_RDWR|O_CREAT; - } else if (strcmp(mode, PG_BINARY_R) == 0) { - flags = O_RDONLY|PG_BINARY; - } else if (strcmp(mode, "r") == 0) { - flags = O_RDONLY; - } else if (strcmp(mode, PG_BINARY_R "+") == 0) { - /* stdio fopen("rb+") actually doesn't create unexisted file, but probackup frequently - * needs to open existed file or create new one if not exists. - * In stdio it can be done using two fopen calls: fopen("r+") and if failed then fopen("w"). - * But to eliminate extra call which especially critical in case of remote connection - * we change r+ semantic to create file if not exists. - */ - flags = O_RDWR|O_CREAT|PG_BINARY; - } else if (strcmp(mode, "r+") == 0) { /* see comment above */ - flags |= O_RDWR|O_CREAT; - } else if (strcmp(mode, "a") == 0) { - flags |= O_CREAT|O_RDWR|O_APPEND; - } else { - Assert(false); - } - fd = fio_open(path, flags, location); - if (fd >= 0) - f = (FILE*)(size_t)((fd + 1) & ~FIO_PIPE_MARKER); - } - else - { - f = fopen(path, mode); - if (f == NULL && strcmp(mode, PG_BINARY_R "+") == 0) - f = fopen(path, PG_BINARY_W); - } - return f; + if (close(fd) != 0) + hdr.arg = errno; + + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -/* Format output to file stream */ -int -fio_fprintf(FILE* f, char const* format, ...) +/* seek is asynchronous */ +static void +fio_seek_impl(int fd, off_t offs) { int rc; - va_list args; - va_start (args, format); - if (fio_is_remote_file(f)) - { - char buf[PRINTF_BUF_SIZE]; -#ifdef HAS_VSNPRINTF - rc = vsnprintf(buf, sizeof(buf), format, args); -#else - rc = vsprintf(buf, format, args); -#endif - if (rc > 0) { - fio_fwrite(f, buf, rc); - } - } - else - { - rc = vfprintf(f, format, args); - } - va_end (args); - return rc; -} -/* Flush stream data (does nothing for remote file) */ -int -fio_fflush(FILE* f) -{ - int rc = 0; - if (!fio_is_remote_file(f)) - rc = fflush(f); - return rc; -} + /* Quick exit for tainted agent */ + if (async_errormsg) + return; -/* Sync file to the disk (does nothing for remote file) */ -int -fio_flush(int fd) -{ - return fio_is_remote_fd(fd) ? 0 : fsync(fd); -} + rc = lseek(fd, offs, SEEK_SET); -/* Close output stream */ -int -fio_fclose(FILE* f) -{ - return fio_is_remote_file(f) - ? fio_close(fio_fileno(f)) - : fclose(f); + if (rc < 0) + { + async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); + } } -/* Close file */ -int -fio_close(int fd) +/* + * Write buffer to descriptor by calling write(), + * If size of written data is less than buffer size, + * then try to write what is left. + * We do this to get honest errno if there are some problems + * with filesystem, since writing less than buffer size + * is not considered an error. + */ +static ssize_t +durable_write(int fd, const char* buf, size_t size) { - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_CLOSE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - fio_fdset &= ~(1 << hdr.handle); - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + off_t current_pos = 0; + size_t bytes_left = size; - /* Wait for response */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + while (bytes_left > 0) + { + int rc = write(fd, buf + current_pos, bytes_left); - if (hdr.arg != 0) - { - errno = hdr.arg; - return -1; - } + if (rc <= 0) + return rc; - return 0; - } - else - { - return close(fd); + bytes_left -= rc; + current_pos += rc; } + + return size; } -/* Close remote file implementation */ static void -fio_close_impl(int fd, int out) +fio_write_impl(int fd, void const* buf, size_t size, int out) { - fio_header hdr; + fio_header hdr = { + .cop = FIO_WRITE, + .handle = -1, + .size = 0, + .arg = 0, + }; + int rc; - hdr.cop = FIO_CLOSE; - hdr.arg = 0; + rc = durable_write(fd, buf, size); - if (close(fd) != 0) + if (rc < 0) hdr.arg = errno; /* send header */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + + return; } -/* Truncate stdio file */ -int -fio_ftruncate(FILE* f, off_t size) -{ - return fio_is_remote_file(f) - ? fio_truncate(fio_fileno(f), size) - : ftruncate(fileno(f), size); -} - -/* Truncate file - * TODO: make it synchronous +/* + * Read value of a symbolic link + * this is a wrapper about readlink() syscall + * side effects: string truncation occur (and it + * can be checked by caller by comparing + * returned value >= valsiz) */ -int -fio_truncate(int fd, off_t size) +ssize_t +fio_readlink(fio_location location, const char *path, char *value, size_t valsiz) { - if (fio_is_remote_fd(fd)) + if (!fio_is_remote(location)) { - fio_header hdr; - - hdr.cop = FIO_TRUNCATE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = size; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - return 0; + /* readlink don't place trailing \0 */ + ssize_t len = readlink(path, value, valsiz); + value[len < valsiz ? len : valsiz] = '\0'; + return len; } else { - return ftruncate(fd, size); - } -} - - -/* - * Read file from specified location. - */ -int -fio_pread(FILE* f, void* buf, off_t offs) -{ - if (fio_is_remote_file(f)) - { - int fd = fio_fileno(f); fio_header hdr; + size_t path_len = strlen(path) + 1; - hdr.cop = FIO_PREAD; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = offs; + hdr.cop = FIO_READLINK; + hdr.handle = -1; + Assert(valsiz <= UINT_MAX); /* max value of fio_header.arg */ + hdr.arg = valsiz; + hdr.size = path_len; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND); - if (hdr.size != 0) - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - - /* TODO: error handling */ - - return hdr.arg; - } - else - { - /* For local file, opened by fopen, we should use stdio functions */ - int rc = fseek(f, offs, SEEK_SET); - - if (rc < 0) - return rc; - - return fread(buf, 1, BLCKSZ, f); + Assert(hdr.cop == FIO_READLINK); + Assert(hdr.size <= valsiz); + IO_CHECK(fio_read_all(fio_stdin, value, hdr.size), hdr.size); + value[hdr.size < valsiz ? hdr.size : valsiz] = '\0'; + return hdr.size; } } -/* Set position in stdio file */ -int -fio_fseek(FILE* f, off_t offs) -{ - return fio_is_remote_file(f) - ? fio_seek(fio_fileno(f), offs) - : fseek(f, offs, SEEK_SET); -} - -/* Set position in file */ -/* TODO: make it synchronous or check async error */ +/* Create symbolic link */ int -fio_seek(int fd, off_t offs) +fio_symlink(fio_location location, const char* target, const char* link_path, bool overwrite) { - if (fio_is_remote_fd(fd)) + if (fio_is_remote(location)) { - fio_header hdr; - - hdr.cop = FIO_SEEK; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = offs; + size_t target_len = strlen(target) + 1; + size_t link_path_len = strlen(link_path) + 1; + fio_header hdr = { + .cop = FIO_SYMLINK, + .handle = -1, + .size = target_len + link_path_len, + .arg = overwrite ? 1 : 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, target, target_len), target_len); + IO_CHECK(fio_write_all(fio_stdout, link_path, link_path_len), link_path_len); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_SYMLINK); + + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } return 0; } else { - return lseek(fd, offs, SEEK_SET); + if (overwrite) + remove_file_or_dir(link_path); + + return symlink(target, link_path); } } -/* seek is asynchronous */ static void -fio_seek_impl(int fd, off_t offs) +fio_symlink_impl(const char* target, const char* link_path, bool overwrite, int out) { - int rc; - - /* Quick exit for tainted agent */ - if (async_errormsg) - return; + fio_header hdr = { + .cop = FIO_SYMLINK, + .handle = -1, + .size = 0, + .arg = 0, + }; - rc = lseek(fd, offs, SEEK_SET); + if (overwrite) + remove_file_or_dir(link_path); - if (rc < 0) - { - async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); - } -} + if (symlink(target, link_path) != 0) + hdr.arg = errno; -/* Write data to stdio file */ -size_t -fio_fwrite(FILE* f, void const* buf, size_t size) -{ - if (fio_is_remote_file(f)) - return fio_write(fio_fileno(f), buf, size); - else - return fwrite(buf, 1, size, f); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -/* - * Write buffer to descriptor by calling write(), - * If size of written data is less than buffer size, - * then try to write what is left. - * We do this to get honest errno if there are some problems - * with filesystem, since writing less than buffer size - * is not considered an error. - */ -static ssize_t -durable_write(int fd, const char* buf, size_t size) +static void +fio_rename_impl(char const* old_path, const char* new_path, int out) { - off_t current_pos = 0; - size_t bytes_left = size; - - while (bytes_left > 0) - { - int rc = write(fd, buf + current_pos, bytes_left); - - if (rc <= 0) - return rc; - - bytes_left -= rc; - current_pos += rc; - } + fio_header hdr = { + .cop = FIO_RENAME, + .handle = -1, + .size = 0, + .arg = 0, + }; + + if (rename(old_path, new_path) != 0) + hdr.arg = errno; - return size; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -/* Write data to the file synchronously */ -ssize_t -fio_write(int fd, void const* buf, size_t size) +enum { + GET_CRC32_DECOMPRESS = 1, + GET_CRC32_TRUNCATED = 2 +}; + +/* Remove file */ +int +fio_remove(fio_location location, const char* path, bool missing_ok) { - if (fio_is_remote_fd(fd)) - { - fio_header hdr; + int result = 0; - hdr.cop = FIO_WRITE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = size; + if (fio_is_remote(location)) + { + fio_header hdr = { + .cop = FIO_REMOVE, + .handle = -1, + .size = strlen(path) + 1, + .arg = missing_ok ? 1 : 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - /* check results */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_REMOVE); - /* set errno */ - if (hdr.arg > 0) + if (hdr.arg != 0) { errno = hdr.arg; - return -1; + result = -1; } - - return size; } else { - return durable_write(fd, buf, size); + if (remove_file_or_dir(path) != 0) + { + if (!missing_ok || errno != ENOENT) + result = -1; + } } + return result; } static void -fio_write_impl(int fd, void const* buf, size_t size, int out) +fio_remove_impl(const char* path, bool missing_ok, int out) { - int rc; - fio_header hdr; - - rc = durable_write(fd, buf, size); - - hdr.arg = 0; - hdr.size = 0; - - if (rc < 0) - hdr.arg = errno; + fio_header hdr = { + .cop = FIO_REMOVE, + .handle = -1, + .size = 0, + .arg = 0, + }; + + if (remove_file_or_dir(path) != 0) + { + if (!missing_ok || errno != ENOENT) + hdr.arg = errno; + } - /* send header */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - - return; -} - -size_t -fio_fwrite_async(FILE* f, void const* buf, size_t size) -{ - return fio_is_remote_file(f) - ? fio_write_async(fio_fileno(f), buf, size) - : fwrite(buf, 1, size, f); } -/* Write data to the file */ -/* TODO: support async report error */ -ssize_t -fio_write_async(int fd, void const* buf, size_t size) +/* + * Create directory, also create parent directories if necessary. + * In strict mode treat already existing directory as error. + * Return values: + * 0 - ok + * -1 - error (check errno) + */ +static int +dir_create_dir(const char *dir, mode_t mode, bool strict) { - if (size == 0) - return 0; + char parent[MAXPGPATH]; - if (fio_is_remote_fd(fd)) - { - fio_header hdr; + strncpy(parent, dir, MAXPGPATH); + get_parent_directory(parent); - hdr.cop = FIO_WRITE_ASYNC; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = size; + /* Create parent first */ + if (access(parent, F_OK) == -1) + dir_create_dir(parent, mode, false); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + /* Create directory */ + if (mkdir(dir, mode) == -1) + { + if (errno == EEXIST && !strict) /* already exist */ + return 0; + return -1; } - else - return durable_write(fd, buf, size); - return size; + return 0; } +/* + * Executed by remote agent. + */ static void -fio_write_async_impl(int fd, void const* buf, size_t size, int out) +fio_mkdir_impl(const char* path, int mode, bool strict, int out) { - /* Quick exit if agent is tainted */ - if (async_errormsg) - return; + fio_header hdr = { + .cop = FIO_MKDIR, + .handle = -1, + .size = 0, + .arg = 0, + }; + + if (dir_create_dir(path, mode, strict) != 0) + hdr.arg = errno; - if (durable_write(fd, buf, size) <= 0) - { - async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); - } + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -int32 -fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg) +static void +fio_send_pio_err(int out, err_i err) { - const char *internal_errormsg = NULL; - int32 uncompressed_size = do_decompress(dst, BLCKSZ, - src, - size, - compress_alg, &internal_errormsg); - - if (uncompressed_size < 0 && internal_errormsg != NULL) - { - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, ERRMSG_MAX_LEN, "An error occured during decompressing block: %s", internal_errormsg); - return -1; - } - - if (uncompressed_size != BLCKSZ) - { - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, ERRMSG_MAX_LEN, "Page uncompressed to %d bytes != BLCKSZ", uncompressed_size); - return -1; - } - return uncompressed_size; -} + ft_strbuf_t load = ft_strbuf_zero(); + ft_source_position_t src = $errsrc(err); + fio_header hdr = { + .cop = FIO_PIO_ERROR, + .arg = (getErrno(err) & 0xff) | (src.line << 8), + }; -/* Write data to the file */ -ssize_t -fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg) -{ - if (fio_is_remote_file(f)) - { - fio_header hdr; - hdr.cop = FIO_WRITE_COMPRESSED_ASYNC; - hdr.handle = fio_fileno(f) & ~FIO_PIPE_MARKER; - hdr.size = size; - hdr.arg = compress_alg; + ft_strbuf_catc_zt(&load, $errkind(err)); + ft_strbuf_catc_zt(&load, $errmsg(err)); + ft_strbuf_catc_zt(&load, src.file); + ft_strbuf_catc_zt(&load, src.func); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + hdr.size = load.len; - return size; - } - else - { - char *errormsg = NULL; - char decompressed_buf[BLCKSZ]; - int32 decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &errormsg); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, load.ptr, load.len), load.len); - if (decompressed_size < 0) - elog(ERROR, "%s", errormsg); + /* We also need to send all the KVs */ + ft_strbuf_free(&load); +} - return fwrite(decompressed_buf, 1, decompressed_size, f); - } +static err_i +fio_receive_pio_err(fio_header *hdr) +{ + int pio_errno = hdr->arg & 0xff; + ft_bytes_t load = ft_bytes_alloc(hdr->size); + ft_bytes_t parse; + ft_strbuf_t rmsg = ft_strbuf_init_str(ft_cstr("(remote) ")); + ft_str_t kind; + ft_str_t msg; + ft_str_t file; + int line = hdr->arg >> 8; + ft_str_t func; + fobj_err_kv_t kvs[] = {{.key="errNo", .val=ft_mka_i(pio_errno)}}; + err_i err; + + IO_CHECK(fio_read_all(fio_stdin, load.ptr, load.len), load.len); + parse = load; + kind = ft_bytes_shift_zt(&parse); + msg = ft_bytes_shift_zt(&parse); + file = ft_bytes_shift_zt(&parse); + func = ft_bytes_shift_zt(&parse); + ft_strbuf_cat(&rmsg, msg); + + err = fobj__alloc_err(kind.ptr, + (ft_source_position_t){ + .file = file.ptr, + .line = line, + .func = func.ptr, + }, + rmsg.ptr, + kvs, pio_errno ? 1 : 0); + + ft_bytes_free(&load); + ft_strbuf_free(&rmsg); + + return err; } static void -fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg) +fio_iterate_pages_impl(pioDBDrive_i drive, int out, const char *path, + datapagemap_t pagemap, + fio_iterate_pages_request *params) { - int32 decompressed_size; - char decompressed_buf[BLCKSZ]; - - /* If the previous command already have failed, - * then there is no point in bashing a head against the wall - */ - if (async_errormsg) + pioPagesIterator_i pages; + err_i err = $noerr(); + fio_header hdr = {.cop=FIO_ITERATE_DATA}; + BlockNumber finalN; + + pages = $i(pioIteratePages, drive, + .path = path, + .segno = params->segno, + .pagemap = pagemap, + .start_lsn = params->start_lsn, + .calg = params->calg, + .clevel = params->clevel, + .checksum_version = params->checksum_version, + .just_validate = params->just_validate, + .err = &err); + + if ($haserr(err)) + { + fio_send_pio_err(out, err); return; + } + ft_strbuf_t req = ft_strbuf_zero(); + while (true) + { + PageIteratorValue value; - /* decompress chunk */ - decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &async_errormsg); + err_i err = $i(pioNextPage, pages, &value); + if ($haserr(err)) { + fio_send_pio_err(out, err); + return; + } + if (value.page_result == PageIsTruncated) + break; - if (decompressed_size < 0) - return; + //send page + state + size_t value_size = sizeof(PageIteratorValue) - BLCKSZ + value.compressed_size; - if (durable_write(fd, decompressed_buf, decompressed_size) <= 0) - { - async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); - } -} + hdr.size = value_size; -/* check if remote agent encountered any error during execution of async operations */ -int -fio_check_error_file(FILE* f, char **errmsg) -{ - if (fio_is_remote_file(f)) - { - fio_header hdr; + ft_strbuf_reset_for_reuse(&req); + ft_strbuf_catbytes(&req, ft_bytes(&hdr, sizeof(hdr))); + ft_strbuf_catbytes(&req, ft_bytes(&value, value_size)); - hdr.cop = FIO_GET_ASYNC_ERROR; - hdr.size = 0; + IO_CHECK(fio_write_all(out, req.ptr, req.len), req.len); + } - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_strbuf_reset_for_reuse(&req); - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + finalN = $i(pioFinalPageN, pages); + hdr = (fio_header){.cop = FIO_ITERATE_EOF, .size = sizeof(finalN)}; + ft_strbuf_catbytes(&req, ft_bytes(&hdr, sizeof(hdr))); + ft_strbuf_catbytes(&req, ft_bytes(&finalN, sizeof(finalN))); - if (hdr.size > 0) - { - *errmsg = pgut_malloc(ERRMSG_MAX_LEN); - IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); - return 1; - } - } + IO_CHECK(fio_write_all(out, req.ptr, req.len), req.len); - return 0; + ft_strbuf_free(&req); } -/* check if remote agent encountered any error during execution of async operations */ -int -fio_check_error_fd(int fd, char **errmsg) +/* find page border of all-zero tail */ +static size_t +find_zero_tail(char *buf, size_t len) { - if (fio_is_remote_fd(fd)) - { - fio_header hdr; + size_t i, l; + size_t granul = sizeof(zerobuf); - hdr.cop = FIO_GET_ASYNC_ERROR; - hdr.size = 0; + if (len == 0) + return 0; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + /* fast check for last bytes */ + l = Min(len, PAGE_ZEROSEARCH_FINE_GRANULARITY); + i = len - l; + if (memcmp(buf + i, zerobuf, l) != 0) + return len; - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.size > 0) + /* coarse search for zero tail */ + i = (len-1) & ~(granul-1); + l = len - i; + for (;;) + { + if (memcmp(buf+i, zerobuf, l) != 0) { - *errmsg = pgut_malloc(ERRMSG_MAX_LEN); - IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); - return 1; + i += l; + break; } + if (i == 0) + break; + i -= granul; + l = granul; } - return 0; + + len = i; + /* search zero tail with finer granularity */ + for (granul = sizeof(zerobuf)/2; + len > 0 && granul >= PAGE_ZEROSEARCH_FINE_GRANULARITY; + granul /= 2) + { + if (granul > l) + continue; + i = (len-1) & ~(granul-1); + l = len - i; + if (memcmp(buf+i, zerobuf, l) == 0) + len = i; + } + + return len; } -static void -fio_get_async_error_impl(int out) +/* Send open file content + * On error we return FIO_ERROR message with following codes + * FIO_ERROR: + * FILE_MISSING (-1) + * OPEN_FAILED (-2) + * READ_FAILED (-3) + * + * FIO_PAGE + * FIO_SEND_FILE_EOF + * + */ +static bool +fio_send_file_content_impl(int fd, int out, const char* path) { fio_header hdr; - hdr.cop = FIO_GET_ASYNC_ERROR; + int save_errno; + char *buf = pgut_malloc(CHUNK_SIZE); + size_t read_len = 0; + char *errormsg = NULL; + int64_t non_zero_len; - /* send error message */ - if (async_errormsg) + /* copy content */ + for (;;) { - hdr.size = strlen(async_errormsg) + 1; + read_len = fio_read_all(fd, buf, CHUNK_SIZE); - /* send header */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + /* report error */ + if (read_len < 0) + { + save_errno = errno; + hdr.cop = FIO_ERROR; + errormsg = pgut_malloc(ERRMSG_MAX_LEN); + hdr.arg = READ_FAILED; + /* Construct the error message */ + snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot read from file '%s': %s", + path, strerror(save_errno)); + hdr.size = strlen(errormsg) + 1; + /* send header and message */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); - /* send message itself */ - IO_CHECK(fio_write_all(out, async_errormsg, hdr.size), hdr.size); + free(errormsg); + free(buf); - //TODO: should we reset the tainted state ? -// pg_free(async_errormsg); -// async_errormsg = NULL; - } - else - { - hdr.size = 0; - /* send header */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - } -} + return false; + } -/* Read data from stdio file */ -ssize_t -fio_fread(FILE* f, void* buf, size_t size) -{ - size_t rc; - if (fio_is_remote_file(f)) - return fio_read(fio_fileno(f), buf, size); - rc = fread(buf, 1, size, f); - return rc == 0 && !feof(f) ? -1 : rc; -} + if (read_len == 0) + break; -/* Read data from file */ -ssize_t -fio_read(int fd, void* buf, size_t size) -{ - if (fio_is_remote_fd(fd)) - { - fio_header hdr; + /* send chunk */ + non_zero_len = find_zero_tail(buf, read_len); - hdr.cop = FIO_READ; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = size; + if (non_zero_len > 0) + { + hdr.cop = FIO_PAGE; + hdr.size = non_zero_len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, buf, non_zero_len), non_zero_len); + } - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + if (read_len > 0) + { + /* send chunk */ + hdr.cop = FIO_PAGE_ZERO; + hdr.size = 0; + hdr.arg = read_len - non_zero_len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } + } - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + /* we are done, send eof */ + hdr.cop = FIO_SEND_FILE_EOF; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - return hdr.size; - } - else - { - return read(fd, buf, size); - } + free(buf); + return true; } -/* Get information about file */ -int -fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) +#if PG_VERSION_NUM < 120000 +/* + * Read the local file to compute its CRC using traditional algorithm. + * (*_TRADITIONAL_CRC32 macros) + * This was used only in version 2.0.22--2.0.24 + * And never used for PG >= 12 + * To be removed with end of PG-11 support + */ +pg_crc32 +pgFileGetCRC32(const char *file_path, bool missing_ok) { - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - - hdr.cop = FIO_STAT; - hdr.handle = -1; - hdr.arg = follow_symlink; - hdr.size = path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + FILE *fp; + pg_crc32 crc = 0; + char *buf; + size_t len = 0; - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_STAT); - IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); + INIT_TRADITIONAL_CRC32(crc); - if (hdr.arg != 0) + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (missing_ok && errno == ENOENT) { - errno = hdr.arg; - return -1; + FIN_TRADITIONAL_CRC32(crc); + return crc; } - return 0; + + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); } - else + + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + buf = pgut_malloc(STDIO_BUFSIZE); + + /* calc CRC of file */ + do { - return follow_symlink ? stat(path, st) : lstat(path, st); - } -} + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); -/* - * Compare, that filename1 and filename2 is the same file - * in windows compare only filenames - */ -bool -fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location) -{ -#ifndef WIN32 - struct stat stat1, stat2; + len = fread(buf, 1, STDIO_BUFSIZE, fp); - if (fio_stat(filename1, &stat1, follow_symlink, location) < 0) - elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno)); + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); - if (fio_stat(filename2, &stat2, follow_symlink, location) < 0) - elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno)); + COMP_TRADITIONAL_CRC32(crc, buf, len); + } + while (!feof(fp)); - return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev; -#else - char *abs_name1 = make_absolute_path(filename1); - char *abs_name2 = make_absolute_path(filename2); - bool result = strcmp(abs_name1, abs_name2) == 0; - free(abs_name2); - free(abs_name1); - return result; -#endif + FIN_TRADITIONAL_CRC32(crc); + fclose(fp); + pg_free(buf); + + return crc; } +#endif /* PG_VERSION_NUM < 120000 */ /* - * Read value of a symbolic link - * this is a wrapper about readlink() syscall - * side effects: string truncation occur (and it - * can be checked by caller by comparing - * returned value >= valsiz) + * WARNING! this function is not paired with fio_remove_dir + * because there is no such function. Instead, it is paired + * with pioRemoteDrive_pioRemoveDir, see PBCKP-234 for further details */ -ssize_t -fio_readlink(const char *path, char *value, size_t valsiz, fio_location location) -{ - if (!fio_is_remote(location)) - { - /* readlink don't place trailing \0 */ - ssize_t len = readlink(path, value, valsiz); - value[len < valsiz ? len : valsiz] = '\0'; - return len; - } - else - { - fio_header hdr; - size_t path_len = strlen(path) + 1; +static void +fio_remove_dir_impl(int out, char* buf) { + fio_remove_dir_request *frdr = (fio_remove_dir_request *)buf; + pioDrive_i drive = pioDriveForLocation(FIO_LOCAL_HOST); - hdr.cop = FIO_READLINK; - hdr.handle = -1; - Assert(valsiz <= UINT_MAX); /* max value of fio_header.arg */ - hdr.arg = valsiz; - hdr.size = path_len; + // In an essence this all is just a wrapper for a pioRemoveDir call on a local drive + $i(pioRemoveDir, drive, .root = frdr->path, .root_as_well = frdr->root_as_well); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + fio_header hdr; + hdr.cop = FIO_REMOVE_DIR; + hdr.arg = 0; - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_READLINK); - Assert(hdr.size <= valsiz); - IO_CHECK(fio_read_all(fio_stdin, value, hdr.size), hdr.size); - value[hdr.size < valsiz ? hdr.size : valsiz] = '\0'; - return hdr.size; - } + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -/* Check presence of the file */ -int -fio_access(char const* path, int mode, fio_location location) + +PageState * +fio_get_checksum_map(fio_location location, const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) { if (fio_is_remote(location)) { fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_ACCESS; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = mode; + fio_checksum_map_request req_hdr; + PageState *checksum_map = NULL; + size_t path_len = strlen(fullpath) + 1; + + req_hdr.n_blocks = n_blocks; + req_hdr.segmentno = segmentno; + req_hdr.stop_lsn = dest_stop_lsn; + req_hdr.checksumVersion = checksum_version; + + hdr.cop = FIO_GET_CHECKSUM_MAP; + hdr.size = sizeof(req_hdr) + path_len; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); + IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); + /* receive data */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_ACCESS); - if (hdr.arg != 0) + if (hdr.size > 0) { - errno = hdr.arg; - return -1; + checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); + memset(checksum_map, 0, n_blocks * sizeof(PageState)); + IO_CHECK(fio_read_all(fio_stdin, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); } - return 0; - } - else - { - return access(path, mode); - } -} - -/* Create symbolic link */ -int -fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t target_len = strlen(target) + 1; - size_t link_path_len = strlen(link_path) + 1; - hdr.cop = FIO_SYMLINK; - hdr.handle = -1; - hdr.size = target_len + link_path_len; - hdr.arg = overwrite ? 1 : 0; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, target, target_len), target_len); - IO_CHECK(fio_write_all(fio_stdout, link_path, link_path_len), link_path_len); - return 0; + return checksum_map; } else { - if (overwrite) - remove_file_or_dir(link_path); - return symlink(target, link_path); + return get_checksum_map(fullpath, checksum_version, + n_blocks, dest_stop_lsn, segmentno); } } static void -fio_symlink_impl(int out, char *buf, bool overwrite) +fio_get_checksum_map_impl(char *buf, int out) { - char *linked_path = buf; - char *link_path = buf + strlen(buf) + 1; + fio_header hdr; + PageState *checksum_map = NULL; + char *fullpath = (char*) buf + sizeof(fio_checksum_map_request); + fio_checksum_map_request *req = (fio_checksum_map_request*) buf; - if (overwrite) - remove_file_or_dir(link_path); + checksum_map = get_checksum_map(fullpath, req->checksumVersion, + req->n_blocks, req->stop_lsn, req->segmentno); + hdr.size = req->n_blocks; + + /* send array of PageState`s to main process */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size > 0) + IO_CHECK(fio_write_all(out, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); - if (symlink(linked_path, link_path)) - elog(ERROR, "Could not create symbolic link \"%s\": %s", - link_path, strerror(errno)); + pg_free(checksum_map); } -/* Rename file */ -int -fio_rename(char const* old_path, char const* new_path, fio_location location) +datapagemap_t * +fio_get_lsn_map(fio_location location, const char *fullpath, + uint32 checksum_version, int n_blocks, + XLogRecPtr shift_lsn, BlockNumber segmentno) { + datapagemap_t* lsn_map = NULL; + if (fio_is_remote(location)) { fio_header hdr; - size_t old_path_len = strlen(old_path) + 1; - size_t new_path_len = strlen(new_path) + 1; - hdr.cop = FIO_RENAME; - hdr.handle = -1; - hdr.size = old_path_len + new_path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, old_path, old_path_len), old_path_len); - IO_CHECK(fio_write_all(fio_stdout, new_path, new_path_len), new_path_len); - - //TODO: wait for confirmation. + fio_lsn_map_request req_hdr; + size_t path_len = strlen(fullpath) + 1; - return 0; - } - else - { - return rename(old_path, new_path); - } -} + req_hdr.n_blocks = n_blocks; + req_hdr.segmentno = segmentno; + req_hdr.shift_lsn = shift_lsn; + req_hdr.checksumVersion = checksum_version; -/* Sync file to disk */ -int -fio_sync(char const* path, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_SYNC; - hdr.handle = -1; - hdr.size = path_len; + hdr.cop = FIO_GET_LSN_MAP; + hdr.size = sizeof(req_hdr) + path_len; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); + IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); + + /* receive data */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.arg != 0) + if (hdr.size > 0) { - errno = hdr.arg; - return -1; - } + lsn_map = pgut_malloc(sizeof(datapagemap_t)); + memset(lsn_map, 0, sizeof(datapagemap_t)); - return 0; + lsn_map->bitmap = pgut_malloc(hdr.size); + lsn_map->bitmapsize = hdr.size; + + IO_CHECK(fio_read_all(fio_stdin, lsn_map->bitmap, hdr.size), hdr.size); + } } else { - int fd; - - fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); - if (fd < 0) - return -1; - - if (fsync(fd) < 0) - { - close(fd); - return -1; - } - close(fd); - - return 0; + lsn_map = get_lsn_map(fullpath, checksum_version, n_blocks, + shift_lsn, segmentno); } -} -enum { - GET_CRC32_DECOMPRESS = 1, - GET_CRC32_MISSING_OK = 2, - GET_CRC32_TRUNCATED = 4 -}; + return lsn_map; +} -/* Get crc32 of file */ -static pg_crc32 -fio_get_crc32_ex(const char *file_path, fio_location location, - bool decompress, bool missing_ok, bool truncated) +static void +fio_get_lsn_map_impl(char *buf, int out) { - if (decompress && truncated) - elog(ERROR, "Could not calculate CRC for compressed truncated file"); - - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(file_path) + 1; - pg_crc32 crc = 0; - hdr.cop = FIO_GET_CRC32; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = 0; + fio_header hdr; + datapagemap_t *lsn_map = NULL; + char *fullpath = (char*) buf + sizeof(fio_lsn_map_request); + fio_lsn_map_request *req = (fio_lsn_map_request*) buf; - if (decompress) - hdr.arg = GET_CRC32_DECOMPRESS; - if (missing_ok) - hdr.arg |= GET_CRC32_MISSING_OK; - if (truncated) - hdr.arg |= GET_CRC32_TRUNCATED; + lsn_map = get_lsn_map(fullpath, req->checksumVersion, req->n_blocks, + req->shift_lsn, req->segmentno); + if (lsn_map) + hdr.size = lsn_map->bitmapsize; + else + hdr.size = 0; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len); - IO_CHECK(fio_read_all(fio_stdin, &crc, sizeof(crc)), sizeof(crc)); + /* send bitmap to main process */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size > 0) + IO_CHECK(fio_write_all(out, lsn_map->bitmap, hdr.size), hdr.size); - return crc; - } - else + if (lsn_map) { - if (decompress) - return pgFileGetCRCgz(file_path, true, missing_ok); - else if (truncated) - return pgFileGetCRCTruncated(file_path, true, missing_ok); - else - return pgFileGetCRC(file_path, true, missing_ok); + pg_free(lsn_map->bitmap); + pg_free(lsn_map); } } -pg_crc32 -fio_get_crc32(const char *file_path, fio_location location, - bool decompress, bool missing_ok) +/* + * Return pid of postmaster process running in given pgdata on local machine. + * Return 0 if there is none. + * Return 1 if postmaster.pid is mangled. + */ +static pid_t +local_check_postmaster(const char *pgdata) { - return fio_get_crc32_ex(file_path, location, decompress, missing_ok, false); -} + FILE *fp; + pid_t pid; + long long lpid; + char pid_file[MAXPGPATH]; -pg_crc32 -fio_get_crc32_truncated(const char *file_path, fio_location location, - bool missing_ok) -{ - return fio_get_crc32_ex(file_path, location, false, missing_ok, true); -} + join_path_components(pid_file, pgdata, "postmaster.pid"); -/* Remove file */ -int -fio_unlink(char const* path, fio_location location) -{ - if (fio_is_remote(location)) + fp = fopen(pid_file, "r"); + if (fp == NULL) { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_UNLINK; - hdr.handle = -1; - hdr.size = path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + /* No pid file, acceptable*/ + if (errno == ENOENT) + return 0; + else + elog(ERROR, "Cannot open file \"%s\": %s", + pid_file, strerror(errno)); + } - // TODO: error is swallowed ? - return 0; + if (fscanf(fp, "%lli", &lpid) == 1) + { + pid = lpid; } else { - return remove_file_or_dir(path); + /* something is wrong with the file content */ + pid = 1; + } + + if (pid > 1) + { + if (kill(pid, 0) != 0) + { + /* process no longer exists */ + if (errno == ESRCH) + pid = 0; + else + elog(ERROR, "Failed to send signal 0 to a process %lld: %s", + (long long)pid, strerror(errno)); + } } + + fclose(fp); + return pid; } -/* Create directory - * TODO: add strict flag +/* + * Go to the remote host and get postmaster pid from file postmaster.pid + * and check that process is running, if process is running, return its pid number. */ -int -fio_mkdir(char const* path, int mode, fio_location location) +pid_t +fio_check_postmaster(fio_location location, const char *pgdata) { if (fio_is_remote(location)) { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_MKDIR; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = mode; + fio_header hdr = { + .cop = FIO_CHECK_POSTMASTER, + .handle = -1, + .size = strlen(pgdata) + 1, + .arg = 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_write_all(fio_stdout, pgdata, hdr.size), hdr.size); + /* receive result */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_MKDIR); + Assert(hdr.cop == FIO_CHECK_POSTMASTER); return hdr.arg; } else - { - return dir_create_dir(path, mode, false); - } + return local_check_postmaster(pgdata); } -/* Change file mode */ -int -fio_chmod(char const* path, int mode, fio_location location) +static void +fio_check_postmaster_impl(const char *pgdata, int out) { - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_CHMOD; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = mode; + fio_header hdr = { + .cop = FIO_CHECK_POSTMASTER, + .handle = -1, + .size = 0, + .arg = 0, + }; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + hdr.arg = local_check_postmaster(pgdata); - return 0; - } - else - { - return chmod(path, mode); - } + /* send arrays of checksums to main process */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -#ifdef HAVE_LIBZ - -#define ZLIB_BUFFER_SIZE (64*1024) -#define MAX_WBITS 15 /* 32K LZ77 window */ -#define DEF_MEM_LEVEL 8 -/* last bit used to differenciate remote gzFile from local gzFile - * TODO: this is insane, we should create our own scructure for this, - * not flip some bits in someone's else and hope that it will not break - * between zlib versions. - */ -#define FIO_GZ_REMOTE_MARKER 1 - -typedef struct fioGZFile +/* Execute commands at remote host */ +void +fio_communicate(int in, int out) { - z_stream strm; - int fd; - int errnum; - bool compress; - bool eof; - Bytef buf[ZLIB_BUFFER_SIZE]; -} fioGZFile; + /* + * Map of file and directory descriptors. + * The same mapping is used in agent and master process, so we + * can use the same index at both sides. + */ + int fd[FIO_FDMAX]; -/* check if remote agent encountered any error during execution of async operations */ -int -fio_check_error_fd_gz(gzFile f, char **errmsg) -{ - if (f && ((size_t)f & FIO_GZ_REMOTE_MARKER)) - { - fio_header hdr; + fobj_t objs[FIO_FDMAX] = {0}; + err_i async_errs[FIO_FDMAX] = {0}; - hdr.cop = FIO_GET_ASYNC_ERROR; - hdr.size = 0; + size_t buf_size = 128*1024; + char* buf = (char*)pgut_malloc(buf_size); + fio_header hdr; + pioDBDrive_i drive; + pio_stat_t st; + ft_bytes_t bytes; + ft_str_t path; + ft_str_t path2; + int rc; + pg_crc32 crc; + err_i err = $noerr(); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + FOBJ_FUNC_ARP(); - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + drive = pioDBDriveForLocation(FIO_LOCAL_HOST); - if (hdr.size > 0) - { - *errmsg = pgut_malloc(ERRMSG_MAX_LEN); - IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); - return 1; - } - } - return 0; -} +#ifdef WIN32 + SYS_CHECK(setmode(in, _O_BINARY)); + SYS_CHECK(setmode(out, _O_BINARY)); +#endif -/* On error returns NULL and errno should be checked */ -gzFile -fio_gzopen(char const* path, char const* mode, int level, fio_location location) -{ - int rc; - if (fio_is_remote(location)) - { - fioGZFile* gz = (fioGZFile*) pgut_malloc(sizeof(fioGZFile)); - memset(&gz->strm, 0, sizeof(gz->strm)); - gz->eof = 0; - gz->errnum = Z_OK; - /* check if file opened for writing */ - if (strcmp(mode, PG_BINARY_W) == 0) /* compress */ - { - gz->strm.next_out = gz->buf; - gz->strm.avail_out = ZLIB_BUFFER_SIZE; - rc = deflateInit2(&gz->strm, - level, - Z_DEFLATED, - MAX_WBITS + 16, DEF_MEM_LEVEL, - Z_DEFAULT_STRATEGY); - if (rc == Z_OK) - { - gz->compress = 1; - gz->fd = fio_open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, location); - if (gz->fd < 0) - { - free(gz); - return NULL; - } - } - } - else - { - gz->strm.next_in = gz->buf; - gz->strm.avail_in = ZLIB_BUFFER_SIZE; - rc = inflateInit2(&gz->strm, 15 + 16); - gz->strm.avail_in = 0; - if (rc == Z_OK) - { - gz->compress = 0; - gz->fd = fio_open(path, O_RDONLY | PG_BINARY, location); - if (gz->fd < 0) - { - free(gz); - return NULL; - } + /* Main loop until end of processing all master commands */ + while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { + FOBJ_LOOP_ARP(); + if (hdr.size != 0) { + if (hdr.size > buf_size) { + /* Extend buffer on demand */ + buf_size = hdr.size; + buf = (char*)realloc(buf, buf_size); } + IO_CHECK(fio_read_all(in, buf, hdr.size), hdr.size); } - if (rc != Z_OK) - { - elog(ERROR, "zlib internal error when opening file %s: %s", - path, gz->strm.msg); - } - return (gzFile)((size_t)gz + FIO_GZ_REMOTE_MARKER); - } - else - { - gzFile file; - /* check if file opened for writing */ - if (strcmp(mode, PG_BINARY_W) == 0) - { - int fd = open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, FILE_PERMISSIONS); - if (fd < 0) - return NULL; - file = gzdopen(fd, mode); - } - else - file = gzopen(path, mode); - if (file != NULL && level != Z_DEFAULT_COMPRESSION) - { - if (gzsetparams(file, level, Z_DEFAULT_STRATEGY) != Z_OK) - elog(ERROR, "Cannot set compression level %d: %s", - level, strerror(errno)); - } - return file; - } -} - -int -fio_gzread(gzFile f, void *buf, unsigned size) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - int rc; - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - - if (gz->eof) - { - return 0; - } + errno = 0; /* reset errno */ + switch (hdr.cop) { + case FIO_OPEN: /* Open file */ + fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSION); + hdr.arg = fd[hdr.handle] < 0 ? errno : 0; + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + break; + case FIO_CLOSE: /* Close file */ + fio_close_impl(fd[hdr.handle], out); + break; + case FIO_WRITE: /* Write to the current position in file */ +// IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); + fio_write_impl(fd[hdr.handle], buf, hdr.size, out); + break; + case FIO_READ: /* Read from the current position in file */ + if ((size_t)hdr.arg > buf_size) { + buf_size = hdr.arg; + buf = (char*)realloc(buf, buf_size); + } + errno = 0; + rc = read(fd[hdr.handle], buf, hdr.arg); + hdr.cop = FIO_SEND; + hdr.size = rc > 0 ? rc : 0; + hdr.arg = rc >= 0 ? 0 : errno; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size != 0) + IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); + break; + case FIO_AGENT_VERSION: + { + size_t payload_size = prepare_compatibility_str(buf, buf_size); - gz->strm.next_out = (Bytef *)buf; - gz->strm.avail_out = size; + hdr.arg = AGENT_PROTOCOL_VERSION; + hdr.size = payload_size; - while (1) - { - if (gz->strm.avail_in != 0) /* If there is some data in receiver buffer, then decompress it */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); + break; + } + case FIO_STAT: /* Get information about file with specified path */ + hdr.size = sizeof(st); + st = $i(pioStat, drive, buf, .follow_symlink = hdr.arg != 0, + .err = &err); + hdr.arg = $haserr(err) ? getErrno(err) : 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); + break; + case FIO_FILES_ARE_SAME: + bytes = ft_bytes(buf, hdr.size); + path = ft_bytes_shift_zt(&bytes); + path2 = ft_bytes_shift_zt(&bytes); + hdr.arg = (int)$i(pioFilesAreSame, drive, path.ptr, path2.ptr); + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + break; + case FIO_READ_FILE_AT_ONCE: + bytes = $i(pioReadFile, drive, .path = buf, + .binary = hdr.arg != 0, .err = &err); + if ($haserr(err)) { - rc = inflate(&gz->strm, Z_NO_FLUSH); - if (rc == Z_STREAM_END) - { - gz->eof = 1; - } - else if (rc != Z_OK) - { - gz->errnum = rc; - return -1; - } - if (gz->strm.avail_out != size) - { - return size - gz->strm.avail_out; - } - if (gz->strm.avail_in == 0) - { - gz->strm.next_in = gz->buf; - } + fio_send_pio_err(out, err); } else { - gz->strm.next_in = gz->buf; + hdr.arg = 0; + hdr.size = bytes.len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (bytes.len > 0) + IO_CHECK(fio_write_all(out, bytes.ptr, bytes.len), bytes.len); } - rc = fio_read(gz->fd, gz->strm.next_in + gz->strm.avail_in, - gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in); - if (rc > 0) + ft_bytes_free(&bytes); + break; + case FIO_WRITE_FILE_AT_ONCE: + bytes = ft_bytes(buf, hdr.size); + path = ft_bytes_shift_zt(&bytes); + err = $i(pioWriteFile, drive, .path = path.ptr, + .content = bytes, .binary = hdr.arg); + if ($haserr(err)) { - gz->strm.avail_in += rc; + const char *msg = $errmsg(err); + hdr.arg = getErrno(err); + hdr.size = strlen(msg) + 1; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, msg, hdr.size), hdr.size); } else { - if (rc == 0) - { - gz->eof = 1; + hdr.arg = 0; + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } + break; + case FIO_RENAME: /* Rename file */ + /* possible buffer overflow */ + fio_rename_impl(buf, buf + strlen(buf) + 1, out); + break; + case FIO_SYMLINK: /* Create symbolic link */ + fio_symlink_impl(buf, buf + strlen(buf) + 1, hdr.arg == 1, out); + break; + case FIO_REMOVE: /* Remove file or directory (TODO: Win32) */ + fio_remove_impl(buf, hdr.arg == 1, out); + break; + case FIO_MKDIR: /* Create directory */ + fio_mkdir_impl(buf, hdr.arg, hdr.handle == 1, out); + break; + case FIO_SEEK: /* Set current position in file */ + fio_seek_impl(fd[hdr.handle], hdr.arg); + break; + case FIO_REMOVE_DIR: + fio_remove_dir_impl(out, buf); + break; + case FIO_SEND_FILE_CONTENT: + fio_send_file_content_impl(fd[hdr.handle], out, buf); + break; + case PIO_GET_CRC32: + crc = $i(pioGetCRC32, drive, .path = buf, + .compressed = (hdr.arg & GET_CRC32_DECOMPRESS) != 0, + .truncated = (hdr.arg & GET_CRC32_TRUNCATED) != 0, + .err = &err); + if ($haserr(err)) + fio_send_pio_err(out, err); + else + { + hdr.size = 0; + hdr.arg = crc; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } + break; + case FIO_GET_CHECKSUM_MAP: + fio_get_checksum_map_impl(buf, out); + break; + case FIO_GET_LSN_MAP: + fio_get_lsn_map_impl(buf, out); + break; + case FIO_CHECK_POSTMASTER: + fio_check_postmaster_impl(buf, out); + break; + case FIO_DISCONNECT: + hdr.cop = FIO_DISCONNECTED; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + free(buf); + return; + case FIO_READLINK: /* Read content of a symbolic link */ + { + /* + * We need a buf for a arguments and for a result at the same time + * hdr.size = strlen(symlink_name) + 1 + * hdr.arg = bufsize for a answer (symlink content) + */ + size_t filename_size = (size_t)hdr.size; + if (filename_size + hdr.arg > buf_size) { + buf_size = hdr.arg; + buf = (char*)realloc(buf, buf_size); } - return rc; + rc = readlink(buf, buf + filename_size, hdr.arg); + hdr.cop = FIO_READLINK; + hdr.size = rc > 0 ? rc : 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size != 0) + IO_CHECK(fio_write_all(out, buf + filename_size, hdr.size), hdr.size); } - } - } - else - { - return gzread(f, buf, size); - } -} + break; + case FIO_ITERATE_PAGES: + { + ft_bytes_t bytes = {.ptr = buf, .len = hdr.size}; + fio_iterate_pages_request *params; + char *from_fullpath; + datapagemap_t pagemap; -int -fio_gzwrite(gzFile f, void const* buf, unsigned size) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - int rc; - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); + params = (fio_iterate_pages_request*)bytes.ptr; + ft_bytes_consume(&bytes, sizeof(*params)); - gz->strm.next_in = (Bytef *)buf; - gz->strm.avail_in = size; + pagemap.bitmapsize = params->pagemaplen; + pagemap.bitmap = bytes.ptr; + ft_bytes_consume(&bytes, pagemap.bitmapsize); - do - { - if (gz->strm.avail_out == ZLIB_BUFFER_SIZE) /* Compress buffer is empty */ - { - gz->strm.next_out = gz->buf; /* Reset pointer to the beginning of buffer */ + from_fullpath = bytes.ptr; - if (gz->strm.avail_in != 0) /* Has something in input buffer */ - { - rc = deflate(&gz->strm, Z_NO_FLUSH); - Assert(rc == Z_OK); - gz->strm.next_out = gz->buf; /* Reset pointer to the beginning of buffer */ - } - else - { - break; - } - } - rc = fio_write_async(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); - if (rc >= 0) - { - gz->strm.next_out += rc; - gz->strm.avail_out += rc; + fio_iterate_pages_impl(drive, out, from_fullpath, pagemap, params); } + break; + case PIO_OPEN_REWRITE: + { + struct fio_req_open_rewrite *req = (void*)buf; + const char *path = buf + sizeof(*req); + pioWriteCloser_i fl; + err_i err; + + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] == NULL); + + fl = $i(pioOpenRewrite, drive, .path = path, + .permissions = req->permissions, + .binary = req->binary, + .use_temp = req->use_temp, + .sync = req->sync, + .err = &err); + if ($haserr(err)) + fio_send_pio_err(out, err); else { - return rc; + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + objs[hdr.handle] = $ref(fl.self); } - } while (gz->strm.avail_out != ZLIB_BUFFER_SIZE || gz->strm.avail_in != 0); - - return size; - } - else - { - return gzwrite(f, buf, size); - } -} - -int -fio_gzclose(gzFile f) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - int rc; - if (gz->compress) + break; + } + case PIO_OPEN_WRITE: { - gz->strm.next_out = gz->buf; - rc = deflate(&gz->strm, Z_FINISH); - Assert(rc == Z_STREAM_END && gz->strm.avail_out != ZLIB_BUFFER_SIZE); - deflateEnd(&gz->strm); - rc = fio_write(gz->fd, gz->buf, ZLIB_BUFFER_SIZE - gz->strm.avail_out); - if (rc != ZLIB_BUFFER_SIZE - gz->strm.avail_out) + struct fio_req_open_write *req = (void*)buf; + const char *path = buf + sizeof(*req); + pioDBWriter_i fl; + err_i err; + + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] == NULL); + + fl = $i(pioOpenWrite, drive, .path = path, + .permissions = req->permissions, + .exclusive = req->exclusive, + .sync = req->sync, + .err = &err); + if ($haserr(err)) + fio_send_pio_err(out, err); + else { - return -1; + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + objs[hdr.handle] = $ref(fl.self); } + break; } - else + case PIO_WRITE_ASYNC: { - inflateEnd(&gz->strm); - } - rc = fio_close(gz->fd); - free(gz); - return rc; - } - else - { - return gzclose(f); - } -} - -int -fio_gzeof(gzFile f) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - return gz->eof; - } - else - { - return gzeof(f); - } -} - -const char* -fio_gzerror(gzFile f, int *errnum) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - if (errnum) - *errnum = gz->errnum; - return gz->strm.msg; - } - else - { - return gzerror(f, errnum); - } -} - -z_off_t -fio_gzseek(gzFile f, z_off_t offset, int whence) -{ - Assert(!((size_t)f & FIO_GZ_REMOTE_MARKER)); - return gzseek(f, offset, whence); -} - - -#endif - -/* Send file content - * Note: it should not be used for large files. - */ -static void -fio_load_file(int out, char const* path) -{ - int fd = open(path, O_RDONLY); - fio_header hdr; - void* buf = NULL; - - hdr.cop = FIO_SEND; - hdr.size = 0; - - if (fd >= 0) - { - off_t size = lseek(fd, 0, SEEK_END); - buf = pgut_malloc(size); - lseek(fd, 0, SEEK_SET); - IO_CHECK(fio_read_all(fd, buf, size), size); - hdr.size = size; - SYS_CHECK(close(fd)); - } - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (buf) - { - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); - free(buf); - } -} - -/* - * Return number of actually(!) readed blocks, attempts or - * half-readed block are not counted. - * Return values in case of error: - * FILE_MISSING - * OPEN_FAILED - * READ_ERROR - * PAGE_CORRUPTION - * WRITE_FAILED - * - * If none of the above, this function return number of blocks - * readed by remote agent. - * - * In case of DELTA mode horizonLsn must be a valid lsn, - * otherwise it should be set to InvalidXLogRecPtr. - */ -int -fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, - XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber* err_blknum, char **errormsg, - BackupPageHeader2 **headers) -{ - FILE *out = NULL; - char *out_buf = NULL; - struct { - fio_header hdr; - fio_send_request arg; - } req; - BlockNumber n_blocks_read = 0; - BlockNumber blknum = 0; - - /* send message with header - - 16bytes 24bytes var var - -------------------------------------------------------------- - | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | - -------------------------------------------------------------- - */ + err_i err; - req.hdr.cop = FIO_SEND_PAGES; - - if (use_pagemap) - { - req.hdr.size = sizeof(fio_send_request) + (*file).pagemap.bitmapsize + strlen(from_fullpath) + 1; - req.arg.bitmapsize = (*file).pagemap.bitmapsize; - - /* TODO: add optimization for the case of pagemap - * containing small number of blocks with big serial numbers: - * https://github.com/postgrespro/pg_probackup/blob/remote_page_backup/src/utils/file.c#L1211 - */ - } - else - { - req.hdr.size = sizeof(fio_send_request) + strlen(from_fullpath) + 1; - req.arg.bitmapsize = 0; - } + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] != NULL); - req.arg.nblocks = file->size/BLCKSZ; - req.arg.segmentno = file->segno * RELSEG_SIZE; - req.arg.horizonLsn = horizonLsn; - req.arg.checksumVersion = checksum_version; - req.arg.calg = calg; - req.arg.clevel = clevel; - req.arg.path_len = strlen(from_fullpath) + 1; + err = $(pioWrite, objs[hdr.handle], ft_bytes(buf, hdr.size)); + if ($haserr(err)) + $iset(&async_errs[hdr.handle], err); + break; + } + case PIO_WRITE_COMPRESSED_ASYNC: + { + err_i err; - file->compress_alg = calg; /* TODO: wtf? why here? */ + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] != NULL); -//<----- -// datapagemap_iterator_t *iter; -// BlockNumber blkno; -// iter = datapagemap_iterate(pagemap); -// while (datapagemap_next(iter, &blkno)) -// elog(INFO, "block %u", blkno); -// pg_free(iter); -//<----- + err = $(pioWriteCompressed, objs[hdr.handle], ft_bytes(buf, hdr.size), + .compress_alg = hdr.arg); + if ($haserr(err)) + $iset(&async_errs[hdr.handle], err); + break; + } + case PIO_SEEK: + { + err_i err; + uint64_t offs; - /* send header */ - IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] != NULL); + ft_assert(hdr.size == sizeof(uint64_t)); - /* send file path */ - IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); + memcpy(&offs, buf, sizeof(uint64_t)); - /* send pagemap if any */ - if (use_pagemap) - IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); + err = $(pioSeek, objs[hdr.handle], offs); + if ($haserr(err)) + $iset(&async_errs[hdr.handle], err); + break; + } + case PIO_TRUNCATE: + { + err_i err; + uint64_t offs; - while (true) - { - fio_header hdr; - char buf[BLCKSZ + sizeof(BackupPageHeader)]; - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] != NULL); + ft_assert(hdr.size == sizeof(uint64_t)); - if (interrupted) - elog(ERROR, "Interrupted during page reading"); + memcpy(&offs, buf, sizeof(uint64_t)); - if (hdr.cop == FIO_ERROR) + err = $(pioTruncate, objs[hdr.handle], offs); + if ($haserr(err)) + $iset(&async_errs[hdr.handle], err); + break; + } + case PIO_GET_ASYNC_ERROR: { - /* FILE_MISSING, OPEN_FAILED and READ_FAILED */ - if (hdr.size > 0) + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] != NULL); + ft_assert(hdr.size == 0); + + if ($haserr(async_errs[hdr.handle])) { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); + fio_send_pio_err(out, async_errs[hdr.handle]); + $idel(&async_errs[hdr.handle]); } - - return hdr.arg; + else + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + break; } - else if (hdr.cop == FIO_SEND_FILE_CORRUPTION) + case PIO_DIR_OPEN: { - *err_blknum = hdr.arg; + ft_assert(hdr.handle >= 0 && hdr.handle < FIO_FDMAX); + ft_assert(objs[hdr.handle] == NULL); + pioDirIter_i iter; - if (hdr.size > 0) + iter = $i(pioOpenDir, drive, buf, .err = &err); + if ($haserr(err)) + fio_send_pio_err(out, err); + else { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); + objs[hdr.handle] = $ref(iter.self); + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } - return PAGE_CORRUPTION; + break; } - else if (hdr.cop == FIO_SEND_FILE_EOF) + case PIO_DIR_NEXT: { - /* n_blocks_read reported by EOF */ - n_blocks_read = hdr.arg; - - /* receive headers if any */ - if (hdr.size > 0) + ft_assert(hdr.handle >= 0 && hdr.handle < FIO_FDMAX); + ft_assert(objs[hdr.handle] != NULL); + ft_strbuf_t stats = ft_strbuf_zero(); + ft_strbuf_t names = ft_strbuf_zero(); + pio_dirent_t dirent; + int n; + + for (n = 0; n < PIO_DIR_REMOTE_BATCH;) { - *headers = pgut_malloc(hdr.size); - IO_CHECK(fio_read_all(fio_stdin, *headers, hdr.size), hdr.size); - file->n_headers = (hdr.size / sizeof(BackupPageHeader2)) -1; + dirent = $(pioDirNext, objs[hdr.handle], .err = &err); + if ($haserr(err)) + break; + ft_strbuf_catbytes(&stats, FT_BYTES_FOR(dirent.stat)); + ft_strbuf_cat_zt(&names, dirent.name); + n++; + if (dirent.stat.pst_kind == PIO_KIND_UNKNOWN) + break; } + if ($haserr(err)) + fio_send_pio_err(out, err); + else + { + hdr.arg = n; + hdr.size = stats.len + names.len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (n > 0) + { + IO_CHECK(fio_write_all(out, stats.ptr, stats.len), stats.len); + IO_CHECK(fio_write_all(out, names.ptr, names.len), names.len); + } + } + ft_strbuf_free(&stats); + ft_strbuf_free(&names); break; } - else if (hdr.cop == FIO_PAGE) + case PIO_IS_DIR_EMPTY: { - blknum = hdr.arg; + bool is_empty; - Assert(hdr.size <= sizeof(buf)); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + is_empty = $i(pioIsDirEmpty, drive, buf, .err = &err); + if ($haserr(err)) + fio_send_pio_err(out, err); + else + { + hdr.size = 0; + hdr.arg = is_empty; - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } + break; + } + case PIO_SYNC_TREE: + { + err = $i(pioSyncTree, drive, buf); + if ($haserr(err)) + fio_send_pio_err(out, err); + else + { + hdr.size = 0; + hdr.arg = 0; - /* lazily open backup file */ - if (!out) - out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } + break; + } + case PIO_CLOSE: + { + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] != NULL); - if (fio_fwrite(out, buf, hdr.size) != hdr.size) + err = $(pioClose, objs[hdr.handle]); + err = fobj_err_combine(err, async_errs[hdr.handle]); + if ($haserr(err)) { - fio_fclose(out); - *err_blknum = blknum; - return WRITE_FAILED; + fio_send_pio_err(out, err); } - file->write_size += hdr.size; - file->uncompressed_size += BLCKSZ; + else + { + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } + $del(&objs[hdr.handle]); + $idel(&async_errs[hdr.handle]); + break; } - else - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); + case PIO_DISPOSE: + { + ft_assert(hdr.handle >= 0); + ft_assert(objs[hdr.handle] != NULL); + ft_assert(hdr.size == 0); + + $del(&objs[hdr.handle]); + $idel(&async_errs[hdr.handle]); + break; + } + default: + Assert(false); + } + } + free(buf); + if (rc != 0) { /* Not end of stream: normal pipe close */ + perror("read"); + exit(EXIT_FAILURE); } +} - if (out) - fclose(out); - pg_free(out_buf); - return n_blocks_read; +typedef struct pio_recursive_dir { + pioDrive_i drive; + ft_arr_cstr_t recurse; + ft_str_t root; + ft_str_t parent; + pioDirIter_i iter; + pio_dirent_t dirent; + bool dont_recurse_current; + ft_strbuf_t namebuf; +} pioRecursiveDir; +#define kls__pioRecursiveDir mth(fobjDispose) +fobj_klass(pioRecursiveDir); + +pioRecursiveDir* +pioRecursiveDir_alloc(pioDrive_i drive, path_t root, err_i *err) +{ + pioDirIter_i iter; + fobj_reset_err(err); + + iter = $i(pioOpenDir, drive, root, err); + if ($haserr(*err)) + return NULL; + + return $alloc(pioRecursiveDir, .drive = drive, + .root = ft_strdupc(root), + .parent = ft_strdupc(""), + .iter = $iref(iter), + .recurse = ft_arr_init(), + .namebuf = ft_strbuf_zero()); } -/* - * Return number of actually(!) readed blocks, attempts or - * half-readed block are not counted. - * Return values in case of error: - * FILE_MISSING - * OPEN_FAILED - * READ_ERROR - * PAGE_CORRUPTION - * WRITE_FAILED - * - * If none of the above, this function return number of blocks - * readed by remote agent. - * - * In case of DELTA mode horizonLsn must be a valid lsn, - * otherwise it should be set to InvalidXLogRecPtr. - * Взято из fio_send_pages - */ -int -fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, - XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber* err_blknum, char **errormsg) +static pio_dirent_t +pio_recursive_dir_next_impl(pioRecursiveDir* self, err_i* err) { - FILE *out = NULL; - char *out_buf = NULL; - struct { - fio_header hdr; - fio_send_request arg; - } req; - BlockNumber n_blocks_read = 0; - BlockNumber blknum = 0; - - /* send message with header - - 16bytes 24bytes var var - -------------------------------------------------------------- - | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | - -------------------------------------------------------------- - */ - - req.hdr.cop = FIO_SEND_PAGES; - - if (use_pagemap) - { - req.hdr.size = sizeof(fio_send_request) + (*file).pagemap.bitmapsize + strlen(from_fullpath) + 1; - req.arg.bitmapsize = (*file).pagemap.bitmapsize; - - /* TODO: add optimization for the case of pagemap - * containing small number of blocks with big serial numbers: - * https://github.com/postgrespro/pg_probackup/blob/remote_page_backup/src/utils/file.c#L1211 - */ - } - else + if (self->dirent.stat.pst_kind == PIO_KIND_DIRECTORY && + !self->dont_recurse_current) { - req.hdr.size = sizeof(fio_send_request) + strlen(from_fullpath) + 1; - req.arg.bitmapsize = 0; + ft_arr_cstr_push(&self->recurse, ft_strdup(self->dirent.name).ptr); } - req.arg.nblocks = file->size/BLCKSZ; - req.arg.segmentno = file->segno * RELSEG_SIZE; - req.arg.horizonLsn = horizonLsn; - req.arg.checksumVersion = checksum_version; - req.arg.calg = calg; - req.arg.clevel = clevel; - req.arg.path_len = strlen(from_fullpath) + 1; + ft_strbuf_reset_for_reuse(&self->namebuf); + self->dont_recurse_current = false; - file->compress_alg = calg; /* TODO: wtf? why here? */ - -//<----- -// datapagemap_iterator_t *iter; -// BlockNumber blkno; -// iter = datapagemap_iterate(pagemap); -// while (datapagemap_next(iter, &blkno)) -// elog(INFO, "block %u", blkno); -// pg_free(iter); -//<----- - - /* send header */ - IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + self->dirent = $i(pioDirNext, self->iter, .err = err); + if ($haserr(*err)) + return self->dirent; - /* send file path */ - IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); + if (self->dirent.stat.pst_kind != PIO_KIND_UNKNOWN) + { + ft_strbuf_cat(&self->namebuf, self->parent); + ft_strbuf_cat_path(&self->namebuf, self->dirent.name); + self->dirent.name = ft_strbuf_ref(&self->namebuf); + return self->dirent; + } - /* send pagemap if any */ - if (use_pagemap) - IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); + *err = $i(pioClose, self->iter); + $idel(&self->iter); + if ($haserr(*err)) + return self->dirent; - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); - if (out == NULL) - elog(ERROR, "Cannot open restore target file \"%s\": %s", to_fullpath, strerror(errno)); +next_dir: + if (self->recurse.len == 0) + return self->dirent; - /* update file permission */ - if (fio_chmod(to_fullpath, file->mode, FIO_BACKUP_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); + ft_str_free(&self->parent); + self->parent = ft_cstr(ft_arr_cstr_pop(&self->recurse)); - elog(VERBOSE, "ftruncate file \"%s\" to size %lu", - to_fullpath, file->size); - if (fio_ftruncate(out, file->size) == -1) - elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", - to_fullpath, file->size, strerror(errno)); + ft_strbuf_cat(&self->namebuf, self->root); + ft_strbuf_cat_path(&self->namebuf, self->parent); - if (!fio_is_remote_file(out)) + self->iter = $i(pioOpenDir, self->drive, .path = self->namebuf.ptr, + .err = err); + if ($haserr(*err)) { - out_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); + /* someone deleted dir under our feet */ + if (getErrno(*err) == ENOENT) + { + *err = $noerr(); + goto next_dir; + } + + return self->dirent; } - while (true) - { - fio_header hdr; - char buf[BLCKSZ + sizeof(BackupPageHeader)]; - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + $iref(self->iter); - if (interrupted) - elog(ERROR, "Interrupted during page reading"); + return pio_recursive_dir_next_impl(self, err); +} - if (hdr.cop == FIO_ERROR) - { - /* FILE_MISSING, OPEN_FAILED and READ_FAILED */ - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); - } +pio_dirent_t +pioRecursiveDir_next(pioRecursiveDir* dir, err_i* err) +{ + FOBJ_FUNC_ARP(); + pio_dirent_t ent; + fobj_reset_err(err); - return hdr.arg; - } - else if (hdr.cop == FIO_SEND_FILE_CORRUPTION) - { - *err_blknum = hdr.arg; + ent = pio_recursive_dir_next_impl(dir, err); + $iresult(*err); + return ent; +} - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); - } - return PAGE_CORRUPTION; - } - else if (hdr.cop == FIO_SEND_FILE_EOF) - { - /* n_blocks_read reported by EOF */ - n_blocks_read = hdr.arg; +void +pioRecursiveDir_dont_recurse_current(pioRecursiveDir* dir) +{ + ft_assert(dir->dirent.stat.pst_kind == PIO_KIND_DIRECTORY); + dir->dont_recurse_current = true; +} - /* receive headers if any */ - if (hdr.size > 0) - { - char *tmp = pgut_malloc(hdr.size); - IO_CHECK(fio_read_all(fio_stdin, tmp, hdr.size), hdr.size); - pg_free(tmp); - } +static void +pioRecursiveDir_fobjDispose(VSelf) +{ + Self(pioRecursiveDir); + + if ($notNULL(self->iter)) + $i(pioClose, self->iter); + $idel(&self->iter); + ft_str_free(&self->root); + ft_str_free(&self->parent); + ft_arr_cstr_free(&self->recurse); + ft_strbuf_free(&self->namebuf); +} - break; - } - else if (hdr.cop == FIO_PAGE) - { - blknum = hdr.arg; +void +pioRecursiveDir_close(pioRecursiveDir* dir) +{ + /* we are releasing bound resources, + * but self will be dealloced in FOBJ's ARP */ + pioRecursiveDir_fobjDispose(dir); +} +fobj_klass_handle(pioRecursiveDir); - Assert(hdr.size <= sizeof(buf)); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); +// CLASSES - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); +typedef struct pioLocalDrive +{ +} pioLocalDrive; +#define kls__pioLocalDrive iface__pioDBDrive, iface(pioDBDrive) +fobj_klass(pioLocalDrive); - if (fio_fseek(out, blknum * BLCKSZ) < 0) - { - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); - } - // должен прилетать некомпрессированный блок с заголовком - // Вставить assert? - if (fio_fwrite(out, buf + sizeof(BackupPageHeader), hdr.size - sizeof(BackupPageHeader)) != BLCKSZ) - { - fio_fclose(out); - *err_blknum = blknum; - return WRITE_FAILED; - } - file->write_size += BLCKSZ; - file->uncompressed_size += BLCKSZ; - } - else - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); - } +typedef struct pioRemoteDrive +{ +} pioRemoteDrive; +#define kls__pioRemoteDrive iface__pioDBDrive, iface(pioDBDrive) +fobj_klass(pioRemoteDrive); - if (out) - fclose(out); - pg_free(out_buf); +typedef struct pioFile +{ + const char *path; + bool closed; +} pioFile; +#define kls__pioFile mth(fobjDispose) +fobj_klass(pioFile); - return n_blocks_read; -} +/* define it because pioLocalPagesIterator wants stat from local file */ +#define mth__pioFileStat pio_stat_t, (err_i*, err) +fobj_method(pioFileStat); -/* TODO: read file using large buffer - * Return codes: - * FIO_ERROR: - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) +typedef struct pioLocalReadFile +{ + ft_str_t path; + int fd; + uint64_t off; + ft_bytes_t buf; + ft_bytes_t remain; +} pioLocalReadFile; +#define kls__pioLocalReadFile iface__pioReader, iface(pioReader, pioReadStream), \ + mth(pioFileStat) +fobj_klass(pioLocalReadFile); + +typedef struct pioLocalWriteFile +{ + ft_str_t path; + ft_str_t path_tmp; + FILE* fl; + ft_bytes_t buf; + bool use_temp; + bool delete_in_dispose; + bool sync; +} pioLocalWriteFile; +#define kls__pioLocalWriteFile iface__pioDBWriter, mth(fobjDispose), \ + iface(pioWriteCloser, pioDBWriter) +fobj_klass(pioLocalWriteFile); + +typedef struct pioLocalDir +{ + ft_str_t path; + DIR* dir; + ft_strbuf_t name_buf; +} pioLocalDir; +#define kls__pioLocalDir iface__pioDirIter, iface(pioDirIter), mth(fobjDispose) +fobj_klass(pioLocalDir); + +typedef struct pioRemoteFile +{ + pioFile p; + int handle; + bool asyncMode; + bool asyncEof; + bool didAsync; + err_i asyncError; + /* chunks size is CHUNK_SIZE */ + void* asyncChunk; + ft_bytes_t chunkRest; +} pioRemoteFile; +#define kls__pioRemoteFile iface__pioReader, \ + iface(pioReader, pioReadStream), \ + mth(pioSetAsync, pioAsyncRead) +fobj_klass(pioRemoteFile); + +typedef struct pioRemoteWriteFile { + ft_str_t path; + int handle; + bool did_async; +} pioRemoteWriteFile; +#define kls__pioRemoteWriteFile iface__pioDBWriter, mth(fobjDispose), \ + iface(pioWriteCloser, pioDBWriter) +fobj_klass(pioRemoteWriteFile); + +typedef struct pioRemoteDir +{ + ft_str_t path; + int handle; + int pos; + ft_bytes_t names_buf; + ft_arr_dirent_t entries; +} pioRemoteDir; +#define kls__pioRemoteDir iface__pioDirIter, iface(pioDirIter), mth(fobjDispose) +fobj_klass(pioRemoteDir); + +typedef struct pioReadFilter { + pioRead_i wrapped; + pioFilter_i filter; + pioFltInPlace_i inplace; + char* buffer; + size_t len; + size_t capa; + bool eof; + bool finished; +} pioReadFilter; +#define kls__pioReadFilter mth(pioRead, pioClose) +fobj_klass(pioReadFilter); + +typedef struct pioWriteFilter { + pioWriteFlush_i wrapped; + pioFilter_i filter; + pioFltInPlace_i inplace; + char* buffer; + size_t capa; + bool finished; +} pioWriteFilter; +#define kls__pioWriteFilter iface__pioWriteFlush, iface(pioWriteFlush), \ + mth(pioClose) +fobj_klass(pioWriteFilter); - * FIO_SEND_FILE_CORRUPTION - * FIO_SEND_FILE_EOF - */ -static void -fio_send_pages_impl(int out, char* buf) -{ - FILE *in = NULL; - BlockNumber blknum = 0; - int current_pos = 0; - BlockNumber n_blocks_read = 0; - PageState page_st; - char read_buffer[BLCKSZ+1]; - char in_buf[STDIO_BUFSIZE]; - fio_header hdr; - fio_send_request *req = (fio_send_request*) buf; - char *from_fullpath = (char*) buf + sizeof(fio_send_request); - bool with_pagemap = req->bitmapsize > 0 ? true : false; - /* error reporting */ - char *errormsg = NULL; - /* parse buffer */ - datapagemap_t *map = NULL; - datapagemap_iterator_t *iter = NULL; - /* page headers */ - int32 hdr_num = -1; - int32 cur_pos_out = 0; - BackupPageHeader2 *headers = NULL; - - /* open source file */ - in = fopen(from_fullpath, PG_BINARY_R); - if (!in) - { - hdr.cop = FIO_ERROR; - - /* do not send exact wording of ENOENT error message - * because it is a very common error in our case, so - * error code is enough. - */ - if (errno == ENOENT) - { - hdr.arg = FILE_MISSING; - hdr.size = 0; - } - else - { - hdr.arg = OPEN_FAILED; - errormsg = pgut_malloc(ERRMSG_MAX_LEN); - /* Construct the error message */ - snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open file \"%s\": %s", - from_fullpath, strerror(errno)); - hdr.size = strlen(errormsg) + 1; - } +#ifdef HAVE_LIBZ +typedef struct pioGZCompress { + z_stream strm; + bool finished; +} pioGZCompress; + +typedef struct pioGZDecompress { + z_stream strm; + bool eof; + bool finished; + bool ignoreTruncate; +} pioGZDecompress; + +typedef struct pioGZDecompressWrapperObj { + bool ignoreTruncate; +} pioGZDecompressWrapperObj; + +#define kls__pioGZCompress iface__pioFilter, mth(fobjDispose), iface(pioFilter) +fobj_klass(pioGZCompress); +#define kls__pioGZDecompress iface__pioFilter, mth(fobjDispose), iface(pioFilter) +fobj_klass(pioGZDecompress); +#define kls__pioGZDecompressWrapperObj mth(pioWrapRead) +fobj_klass(pioGZDecompressWrapperObj); +#endif - /* send header and message */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (errormsg) - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); +typedef struct pioReSeekableReader { + pioReader_i reader; + pioRead_i wrapped; + pioWrapRead_i wrapper; + int64_t pos; + bool closed; + bool had_err; +} pioReSeekableReader; +#define kls__pioReSeekableReader iface__pioReader, mth(fobjDispose) +fobj_klass(pioReSeekableReader); + +/* zero tail detector */ +typedef struct pioCutZeroTail { + uint64_t read_size; + uint64_t write_size; +} pioCutZeroTail; + +#define kls__pioCutZeroTail iface__pioFilter, iface(pioFilter) +fobj_klass(pioCutZeroTail); + +/* CRC32 counter */ +typedef struct pioDevNull +{ +} pioDevNull; - goto cleanup; - } +#define kls__pioDevNull iface__pioWriteFlush, iface(pioWriteFlush) +fobj_klass(pioDevNull); - if (with_pagemap) - { - map = pgut_malloc(sizeof(datapagemap_t)); - map->bitmapsize = req->bitmapsize; - map->bitmap = (char*) buf + sizeof(fio_send_request) + req->path_len; +typedef struct pioCRC32Counter +{ + pg_crc32 crc; + int64_t size; +} pioCRC32Counter; - /* get first block */ - iter = datapagemap_iterate(map); - datapagemap_next(iter, &blknum); +static pioDBDrive_i localDrive; +static pioDBDrive_i remoteDrive; - setvbuf(in, NULL, _IONBF, BUFSIZ); - } - else - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); +pioDrive_i +pioDriveForLocation(fio_location loc) +{ + if (fio_is_remote(loc)) + return $reduce(pioDrive, remoteDrive); + else + return $reduce(pioDrive, localDrive); +} - /* TODO: what is this barrier for? */ - read_buffer[BLCKSZ] = 1; /* barrier */ +pioDBDrive_i +pioDBDriveForLocation(fio_location loc) +{ + if (fio_is_remote(loc)) + return remoteDrive; + else + return localDrive; +} - while (blknum < req->nblocks) - { - int rc = 0; - size_t read_len = 0; - int retry_attempts = PAGE_READ_ATTEMPTS; +/* Base physical file type */ - /* TODO: handle signals on the agent */ - if (interrupted) - elog(ERROR, "Interrupted during remote page reading"); +static void +pioFile_fobjDispose(VSelf) +{ + Self(pioFile); - /* read page, check header and validate checksumms */ - for (;;) - { - /* - * Optimize stdio buffer usage, fseek only when current position - * does not match the position of requested block. - */ - if (current_pos != blknum*BLCKSZ) - { - current_pos = blknum*BLCKSZ; - if (fseek(in, current_pos, SEEK_SET) != 0) - elog(ERROR, "fseek to position %u is failed on remote file '%s': %s", - current_pos, from_fullpath, strerror(errno)); - } + ft_assert(self->closed, "File \"%s\" is disposing unclosed", self->path); + ft_free((void*)self->path); + self->path = NULL; +} - read_len = fread(read_buffer, 1, BLCKSZ, in); +static bool +common_pioExists(fobj_t self, path_t path, pio_file_kind_e expected_kind, err_i *err) +{ + pio_stat_t buf; + fobj_reset_err(err); - current_pos += read_len; + /* follow symlink ? */ + buf = $(pioStat, self, path, true, err); + if (getErrno(*err) == ENOENT) + { + *err = $noerr(); + return false; + } + if ($noerr(*err) && buf.pst_kind != expected_kind) + *err = $err(SysErr, "File {path:q} is not of an expected kind", path(path)); + if ($haserr(*err)) { + *err = $syserr(getErrno(*err), "Could not check file existance: {cause:$M}", + cause((*err).self)); + } + return $noerr(*err); +} - /* report error */ - if (ferror(in)) - { - hdr.cop = FIO_ERROR; - hdr.arg = READ_FAILED; +/* LOCAL DRIVE */ - errormsg = pgut_malloc(ERRMSG_MAX_LEN); - /* Construct the error message */ - snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot read block %u of '%s': %s", - blknum, from_fullpath, strerror(errno)); - hdr.size = strlen(errormsg) + 1; +static pioReader_i +pioLocalDrive_pioOpenRead(VSelf, path_t path, err_i *err) +{ + int fd; + fobj_reset_err(err); + fobj_t file; - /* send header and message */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); - goto cleanup; - } + fd = open(path, O_RDONLY); + if (fd < 0) + { + *err = $syserr(errno, "Cannot open file {path:q}", path(path)); + return (pioReader_i){NULL}; + } - if (read_len == BLCKSZ) - { - rc = validate_one_page(read_buffer, req->segmentno + blknum, - InvalidXLogRecPtr, &page_st, - req->checksumVersion); + file = $alloc(pioLocalReadFile, + .fd = fd, + .path = ft_strdupc(path), + .buf = ft_bytes_alloc(CHUNK_SIZE)); + return $bind(pioReader, file); +} - /* TODO: optimize copy of zeroed page */ - if (rc == PAGE_IS_ZEROED) - break; - else if (rc == PAGE_IS_VALID) - break; - } +static pioReadStream_i +pioLocalDrive_pioOpenReadStream(VSelf, path_t path, err_i *err) +{ + Self(pioLocalDrive); + return $reduce(pioReadStream, $(pioOpenRead, self, path, .err = err)); +} - if (feof(in)) - goto eof; -// else /* readed less than BLKSZ bytes, retry */ +static pioWriteCloser_i +pioLocalDrive_pioOpenRewrite(VSelf, path_t path, int permissions, + bool binary, bool use_temp, bool sync, err_i *err) +{ + Self(pioLocalDrive); + ft_str_t temppath; + int fd = -1; + FILE* fl; + ft_bytes_t buf; + fobj_t res; - /* File is either has insane header or invalid checksum, - * retry. If retry attempts are exhausted, report corruption. - */ - if (--retry_attempts == 0) - { - hdr.cop = FIO_SEND_FILE_CORRUPTION; - hdr.arg = blknum; + fobj_reset_err(err); - /* Construct the error message */ - if (rc == PAGE_HEADER_IS_INVALID) - get_header_errormsg(read_buffer, &errormsg); - else if (rc == PAGE_CHECKSUM_MISMATCH) - get_checksum_errormsg(read_buffer, &errormsg, - req->segmentno + blknum); + if (use_temp) + { + temppath = ft_asprintf("%s~tmpXXXXXX", path); + fd = mkstemp(temppath.ptr); + } + else + { + temppath = ft_strdupc(path); + fd = open(path, O_CREAT|O_TRUNC|O_WRONLY, permissions); + } - /* if error message is not empty, set payload size to its length */ - hdr.size = errormsg ? strlen(errormsg) + 1 : 0; + if (fd < 0) + { + *err = $syserr(errno, "Create file {path} failed", path(temppath.ptr)); + close(fd); + ft_str_free(&temppath); + return $null(pioWriteCloser); + } - /* send header */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +#ifdef WIN32 + if (binary && _setmode(fd, _O_BINARY) < 0) + { + *err = $syserr(errno, "Changing permissions for {path} failed", + path(temppath.ptr)); + close(fd); + ft_str_free(&temppath); + return $null(pioWriteCloser); + } +#endif - /* send error message if any */ - if (errormsg) - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); + if (chmod(temppath.ptr, permissions)) + { + *err = $syserr(errno, "Changing permissions for {path} failed", + path(temppath.ptr)); + close(fd); + ft_str_free(&temppath); + return $null(pioWriteCloser); + } - goto cleanup; - } - } + fl = fdopen(fd, binary ? "wb" : "w"); + ft_assert(fl != NULL); - n_blocks_read++; + buf = ft_bytes_alloc(CHUNK_SIZE); + setvbuf(fl, buf.ptr, _IOFBF, buf.len); - /* - * horizonLsn is not 0 only in case of delta and ptrack backup. - * As far as unsigned number are always greater or equal than zero, - * there is no sense to add more checks. - */ - if ((req->horizonLsn == InvalidXLogRecPtr) || /* full, page */ - (page_st.lsn == InvalidXLogRecPtr) || /* zeroed page */ - (req->horizonLsn > 0 && page_st.lsn > req->horizonLsn)) /* delta, ptrack */ - { - int compressed_size = 0; - char write_buffer[BLCKSZ*2]; - BackupPageHeader* bph = (BackupPageHeader*)write_buffer; + res = $alloc(pioLocalWriteFile, + .path = ft_strdupc(path), + .path_tmp = temppath, + .use_temp = use_temp, + .delete_in_dispose = true, + .fl = fl, + .sync = sync, + .buf = buf); + return $bind(pioWriteCloser, res); +} - /* compress page */ - hdr.cop = FIO_PAGE; - hdr.arg = blknum; +static pioDBWriter_i +pioLocalDrive_pioOpenWrite(VSelf, path_t path, int permissions, + bool exclusive, bool sync, err_i *err) +{ + Self(pioLocalDrive); + int fd = -1; + FILE* fl; + ft_bytes_t buf; + fobj_t res; + int flags; - compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), - sizeof(write_buffer) - sizeof(BackupPageHeader), - read_buffer, BLCKSZ, req->calg, req->clevel, - NULL); + fobj_reset_err(err); - if (compressed_size <= 0 || compressed_size >= BLCKSZ) - { - /* Do not compress page */ - memcpy(write_buffer + sizeof(BackupPageHeader), read_buffer, BLCKSZ); - compressed_size = BLCKSZ; - } - bph->block = blknum; - bph->compressed_size = compressed_size; + flags = O_CREAT|O_WRONLY|PG_BINARY; + if (exclusive) + flags |= O_EXCL; - hdr.size = compressed_size + sizeof(BackupPageHeader); + fd = open(path, flags, permissions); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); + if (fd < 0) + { + *err = $syserr(errno, "Create file {path} failed", path(path)); + close(fd); + return $null(pioDBWriter); + } - /* set page header for this file */ - hdr_num++; - if (!headers) - headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); - else - headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1) * sizeof(BackupPageHeader2)); + if (!exclusive && chmod(path, permissions)) + { + *err = $syserr(errno, "Changing permissions for {path} failed", + path(path)); + close(fd); + return $null(pioDBWriter); + } - headers[hdr_num].block = blknum; - headers[hdr_num].lsn = page_st.lsn; - headers[hdr_num].checksum = page_st.checksum; - headers[hdr_num].pos = cur_pos_out; + fl = fdopen(fd, "wb"); + ft_assert(fl != NULL); - cur_pos_out += hdr.size; - } + buf = ft_bytes_alloc(CHUNK_SIZE); + setvbuf(fl, buf.ptr, _IOFBF, buf.len); - /* next block */ - if (with_pagemap) - { - /* exit if pagemap is exhausted */ - if (!datapagemap_next(iter, &blknum)) - break; - } - else - blknum++; - } + res = $alloc(pioLocalWriteFile, + .path = ft_strdupc(path), + .path_tmp = ft_strdupc(path), + .use_temp = false, + .delete_in_dispose = exclusive, + .fl = fl, + .sync = sync, + .buf = buf); + return $bind(pioDBWriter, res); +} -eof: - /* We are done, send eof */ - hdr.cop = FIO_SEND_FILE_EOF; - hdr.arg = n_blocks_read; - hdr.size = 0; +static pio_stat_t +pioLocalDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err) +{ + struct stat st = {0}; + pio_stat_t pst = {0}; + int r; + fobj_reset_err(err); - if (headers) + r = follow_symlink ? stat(path, &st) : lstat(path, &st); + if (r < 0) + *err = $syserr(errno, "Cannot stat file {path:q}", path(path)); + else { - hdr.size = (hdr_num+2) * sizeof(BackupPageHeader2); - - /* add dummy header */ - headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+2) * sizeof(BackupPageHeader2)); - headers[hdr_num+1].pos = cur_pos_out; + pst.pst_kind = pio_statmode2file_kind(st.st_mode, path); + pst.pst_mode = pio_limit_mode(st.st_mode); + pst.pst_size = st.st_size; + pst.pst_mtime = st.st_mtime; } - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + return pst; +} - if (headers) - IO_CHECK(fio_write_all(out, headers, hdr.size), hdr.size); +/* + * Compare, that filename1 and filename2 is the same file + * in windows compare only filenames + */ +static bool +pioLocalDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2) +{ +#ifndef WIN32 + struct stat stat1, stat2; -cleanup: - pg_free(map); - pg_free(iter); - pg_free(errormsg); - pg_free(headers); - if (in) - fclose(in); - return; + if (stat(file1, &stat1) < 0) + elog(ERROR, "Can't stat file \"%s\": %s", file1, strerror(errno)); + + if (stat(file2, &stat2) < 0) + elog(ERROR, "Can't stat file \"%s\": %s", file1, strerror(errno)); + + return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev; +#else + char *abs_name1 = make_absolute_path(file1); + char *abs_name2 = make_absolute_path(file2); + bool result = strcmp(abs_name1, abs_name2) == 0; + free(abs_name2); + free(abs_name1); + return result; +#endif } -/* Receive chunks of compressed data, decompress them and write to - * destination file. - * Return codes: - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - * WRITE_FAILED (-4) - * ZLIB_ERROR (-5) - * REMOTE_ERROR (-6) - */ -int -fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) +#define pioLocalDrive_pioExists common_pioExists + +static err_i +pioLocalDrive_pioRemove(VSelf, path_t path, bool missing_ok) { - fio_header hdr; - int exit_code = SEND_OK; - char *in_buf = pgut_malloc(CHUNK_SIZE); /* buffer for compressed data */ - char *out_buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer for decompressed data */ - size_t path_len = strlen(from_fullpath) + 1; - /* decompressor */ - z_stream *strm = NULL; + if (remove_file_or_dir(path) != 0) + { + if (!missing_ok || errno != ENOENT) + return $syserr(errno, "Cannot remove {path:q}", path(path)); + } + return $noerr(); +} - hdr.cop = FIO_SEND_FILE; - hdr.size = path_len; +static err_i +pioLocalDrive_pioRename(VSelf, path_t old_path, path_t new_path) +{ + if (rename(old_path, new_path) != 0) + return $syserr(errno, "Cannot rename file {old_path:q} to {new_path:q}", + old_path(old_path), new_path(new_path)); + return $noerr(); +} -// elog(VERBOSE, "Thread [%d]: Attempting to open remote compressed WAL file '%s'", -// thread_num, from_fullpath); +pg_crc32 +pio_helper_pioGetCRC32(pioOpenReadStream_i self, path_t path, + bool compressed, bool truncated, err_i *err) +{ + FOBJ_FUNC_ARP(); + fobj_reset_err(err); + pioReadStream_i file; + pioRead_i read; + pioCRC32Counter* crc; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); + elog(VERBOSE, "Calculate crc32 for '%s', compressed=%d, truncated=%d", + path, compressed, truncated); - for (;;) + file = $i(pioOpenReadStream, self, .path = path, .err = err); + if ($haserr(*err)) { - fio_header hdr; - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.cop == FIO_SEND_FILE_EOF) - { - break; - } - else if (hdr.cop == FIO_ERROR) - { - /* handle error, reported by the agent */ - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", in_buf); - } - exit_code = hdr.arg; - goto cleanup; - } - else if (hdr.cop == FIO_PAGE) - { - int rc; - Assert(hdr.size <= CHUNK_SIZE); - IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); + $iresult(*err); + return 0; + } - /* We have received a chunk of compressed data, lets decompress it */ - if (strm == NULL) - { - /* Initialize decompressor */ - strm = pgut_malloc(sizeof(z_stream)); - memset(strm, 0, sizeof(z_stream)); + read = $reduce(pioRead, file); + if (compressed) + read = pioWrapReadFilter(read, pioGZDecompressFilter(false), + CHUNK_SIZE); + if (truncated) + read = pioWrapReadFilter(read, $bind(pioFilter, pioCutZeroTail_alloc()), + CHUNK_SIZE); + crc = pioCRC32Counter_alloc(); + read = pioWrapReadFilter(read, $bind(pioFilter, crc), CHUNK_SIZE); + *err = pioCopy(pioDevNull_alloc(), read); + $iresult(*err); + $i(pioClose, file); // ignore error + + return pioCRC32Counter_getCRC32(crc); +} - /* The fields next_in, avail_in initialized before init */ - strm->next_in = (Bytef *)in_buf; - strm->avail_in = hdr.size; +static pg_crc32 +pioLocalDrive_pioGetCRC32(VSelf, path_t path, + bool compressed, bool truncated, + err_i *err) +{ + Self(pioLocalDrive); + return pio_helper_pioGetCRC32($bind(pioOpenReadStream, self), + path, compressed, truncated, err); +} - rc = inflateInit2(strm, 15 + 16); +static bool +pioLocalDrive_pioIsRemote(VSelf) +{ + return false; +} - if (rc != Z_OK) - { - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, ERRMSG_MAX_LEN, - "Failed to initialize decompression stream for file '%s': %i: %s", - from_fullpath, rc, strm->msg); - exit_code = ZLIB_ERROR; - goto cleanup; - } - } - else - { - strm->next_in = (Bytef *)in_buf; - strm->avail_in = hdr.size; - } +static err_i +pioLocalDrive_pioMakeDir(VSelf, path_t path, mode_t mode, bool strict) +{ + int rc = dir_create_dir(path, mode, strict); + if (rc == 0) return $noerr(); + return $syserr(errno, "Cannot make dir {path:q}", path(path)); +} - strm->next_out = (Bytef *)out_buf; /* output buffer */ - strm->avail_out = OUT_BUF_SIZE; /* free space in output buffer */ +static pioDirIter_i +pioLocalDrive_pioOpenDir(VSelf, path_t path, err_i* err) +{ + Self(pioLocalDrive); + DIR* dir; + fobj_reset_err(err); - /* - * From zlib documentation: - * The application must update next_in and avail_in when avail_in - * has dropped to zero. It must update next_out and avail_out when - * avail_out has dropped to zero. - */ - while (strm->avail_in != 0) /* while there is data in input buffer, decompress it */ - { - /* decompress until there is no data to decompress, - * or buffer with uncompressed data is full - */ - rc = inflate(strm, Z_NO_FLUSH); - if (rc == Z_STREAM_END) - /* end of stream */ - break; - else if (rc != Z_OK) - { - /* got an error */ - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, ERRMSG_MAX_LEN, - "Decompression failed for file '%s': %i: %s", - from_fullpath, rc, strm->msg); - exit_code = ZLIB_ERROR; - goto cleanup; - } + dir = opendir(path); + if (dir == NULL) + { + *err = $syserr(errno, "Cannot open dir {path:q}", path(path)); + return $null(pioDirIter); + } - if (strm->avail_out == 0) - { - /* Output buffer is full, write it out */ - if (fwrite(out_buf, 1, OUT_BUF_SIZE, out) != OUT_BUF_SIZE) - { - exit_code = WRITE_FAILED; - goto cleanup; - } - - strm->next_out = (Bytef *)out_buf; /* output buffer */ - strm->avail_out = OUT_BUF_SIZE; - } - } + return $bind(pioDirIter, + $alloc(pioLocalDir, + .path = ft_strdupc(path), + .dir = dir)); +} - /* write out leftovers if any */ - if (strm->avail_out != OUT_BUF_SIZE) - { - int len = OUT_BUF_SIZE - strm->avail_out; +static bool +pioLocalDrive_pioIsDirEmpty(VSelf, path_t path, err_i* err) +{ + Self(pioLocalDrive); + DIR* dir; + struct dirent *dent; + bool is_empty = true; + fobj_reset_err(err); - if (fwrite(out_buf, 1, len, out) != len) - { - exit_code = WRITE_FAILED; - goto cleanup; - } - } - } - else - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); + dir = opendir(path); + if (dir == NULL) + { + if (errno == ENOENT) + return true; + *err = $syserr(errno, "Cannot open dir {path:q}", path(path)); + return false; } -cleanup: - if (exit_code < OPEN_FAILED) - fio_disconnect(); /* discard possible pending data in pipe */ - - if (strm) + for (errno=0;(dent = readdir(dir)) != NULL;errno=0) { - inflateEnd(strm); - pg_free(strm); + if (strcmp(dent->d_name, ".") == 0) + continue; + if (strcmp(dent->d_name, "..") == 0) + continue; + is_empty = false; + break; } - pg_free(in_buf); - pg_free(out_buf); - return exit_code; -} + if (errno) + *err = $syserr(errno, "Couldn't read dir {path:q}", path(path)); -typedef struct send_file_state { - bool calc_crc; - uint32_t crc; - int64_t read_size; - int64_t write_size; -} send_file_state; + closedir(dir); -/* find page border of all-zero tail */ -static size_t -find_zero_tail(char *buf, size_t len) -{ - size_t i, l; - size_t granul = sizeof(zerobuf); + return is_empty; +} - if (len == 0) - return 0; +static void +pioLocalDrive_pioRemoveDir(VSelf, const char *root, bool root_as_well) { + FOBJ_FUNC_ARP(); + Self(pioLocalDrive); + char full_path[MAXPGPATH]; + ft_arr_cstr_t dirs = ft_arr_init(); + ft_arr_cstr_t files = ft_arr_init(); + char *dirname; + char *filename; + DIR* dir; + struct dirent *dirent; + struct stat st; + size_t i; - /* fast check for last bytes */ - l = Min(len, PAGE_ZEROSEARCH_FINE_GRANULARITY); - i = len - l; - if (memcmp(buf + i, zerobuf, l) != 0) - return len; + /* note: we don't dup root, so will not free it */ + ft_arr_cstr_push(&dirs, (char*)root); - /* coarse search for zero tail */ - i = (len-1) & ~(granul-1); - l = len - i; - for (;;) + for (i = 0; i < dirs.len; i++) /* note that dirs.len will grow */ { - if (memcmp(buf+i, zerobuf, l) != 0) + dirname = dirs.ptr[i]; + dir = opendir(dirname); + + if (dir == NULL) { - i += l; - break; + if (errno == ENOENT) + { + elog(WARNING, "Dir \"%s\" disappeared", dirname); + dirs.ptr[i] = NULL; + if (i != 0) + ft_free(dirname); + continue; + } + else + elog(ERROR, "Cannot open dir \"%s\": %m", dirname); + } + + for(errno=0; (dirent = readdir(dir)) != NULL; errno=0) + { + if (strcmp(dirent->d_name, ".") == 0 || + strcmp(dirent->d_name, "..") == 0) + continue; + + join_path_components(full_path, dirname, dirent->d_name); + if (stat(full_path, &st)) + { + if (errno == ENOENT) + { + elog(WARNING, "File \"%s\" disappeared", full_path); + continue; + } + elog(ERROR, "Could not stat \"%s\": %m", full_path); + } + + if (S_ISDIR(st.st_mode)) + ft_arr_cstr_push(&dirs, ft_cstrdup(full_path)); + else + ft_arr_cstr_push(&files, ft_cstrdup(dirent->d_name)); + } + if (errno) + elog(ERROR, "Could not readdir \"%s\": %m", full_path); + closedir(dir); + + while (files.len > 0) + { + filename = ft_arr_cstr_pop(&files); + join_path_components(full_path, dirname, filename); + ft_free(filename); + + if (progress) + elog(INFO, "Progress: delete file \"%s\"", full_path); + if (remove_file_or_dir(full_path) != 0) + { + if (errno == ENOENT) + elog(WARNING, "File \"%s\" disappeared", full_path); + else + elog(ERROR, "Could not remove \"%s\": %m", full_path); + } } - if (i == 0) - break; - i -= granul; - l = granul; } - len = i; - /* search zero tail with finer granularity */ - for (granul = sizeof(zerobuf)/2; - len > 0 && granul >= PAGE_ZEROSEARCH_FINE_GRANULARITY; - granul /= 2) + while (dirs.len > 0) { - if (granul > l) + dirname = ft_arr_cstr_pop(&dirs); + if (dirname == NULL) continue; - i = (len-1) & ~(granul-1); - l = len - i; - if (memcmp(buf+i, zerobuf, l) == 0) - len = i; + + if (dirs.len == 0 && !root_as_well) + break; + + if (progress) + elog(INFO, "Progress: delete dir \"%s\"", full_path); + if (remove_file_or_dir(dirname) != 0) + { + if (errno == ENOENT) + elog(WARNING, "Dir \"%s\" disappeared", full_path); + else + elog(ERROR, "Could not remove \"%s\": %m", full_path); + } + + if (dirs.len != 0) /* we didn't dup root, so don't free it */ + ft_free(dirname); } - return len; + ft_arr_cstr_free(&dirs); + ft_arr_cstr_free(&files); } -static void -fio_send_file_crc(send_file_state* st, char *buf, size_t len) +static ft_bytes_t +pioLocalDrive_pioReadFile(VSelf, path_t path, bool binary, err_i* err) { - int64_t write_size; + Self(pioLocalDrive); + FILE* fl = NULL; + pio_stat_t st; + ft_bytes_t res = ft_bytes(NULL, 0); + size_t amount; - if (!st->calc_crc) - return; + fobj_reset_err(err); - write_size = st->write_size; - while (st->read_size > write_size) + st = $(pioStat, self, .path = path, .follow_symlink = true, .err = err); + if ($haserr(*err)) + { + return res; + } + if (st.pst_kind != PIO_KIND_REGULAR) { - size_t crc_len = Min(st->read_size - write_size, sizeof(zerobuf)); - COMP_FILE_CRC32(true, st->crc, zerobuf, crc_len); - write_size += crc_len; + *err = $err(RT, "File {path:q} is not regular: {kind}", path(path), + kind(pio_file_kind2str(st.pst_kind, path)), + errNo(EACCES)); + return res; } - if (len > 0) - COMP_FILE_CRC32(true, st->crc, buf, len); -} + /* forbid too large file because of remote protocol */ + if (st.pst_size >= PIO_READ_WRITE_FILE_LIMIT) + { + *err = $err(RT, "File {path:q} is too large: {size}", path(path), + size(st.pst_size), errNo(EFBIG)); + return res; + } -static bool -fio_send_file_write(FILE* out, send_file_state* st, char *buf, size_t len) -{ - if (len == 0) - return true; + if (binary) + res = ft_bytes_alloc(st.pst_size); + else + { + res = ft_bytes_alloc(st.pst_size + 1); + res.len -= 1; + } - if (st->read_size > st->write_size && - fseeko(out, st->read_size, SEEK_SET) != 0) + /* + * rely on "local file is read whole at once always". + * Is it true? + */ + fl = fopen(path, binary ? "rb" : "r"); + if (fl == NULL) { - return false; + *err = $syserr(errno, "Opening file {path:q}", path(path)); + ft_bytes_free(&res); + return res; } - if (fwrite(buf, 1, len, out) != len) + amount = fread(res.ptr, 1, res.len, fl); + if (ferror(fl)) { - return false; + *err = $syserr(errno, "Opening file {path:q}", path(path)); + fclose(fl); + ft_bytes_free(&res); + return res; } - st->read_size += len; - st->write_size = st->read_size; + fclose(fl); - return true; + if (amount != st.pst_size) + { + ft_bytes_free(&res); + *err = $err(RT, "File {path:q} is truncated while reading", + path(path), errNo(EBUSY)); + return res; + } + + res.len = amount; + if (!binary) + res.ptr[amount] = 0; + + return res; } -/* Receive chunks of data and write them to destination file. - * Return codes: - * SEND_OK (0) - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - * WRITE_FAILED (-4) - * - * OPEN_FAILED and READ_FAIL should also set errormsg. - * If pgFile is not NULL then we must calculate crc and read_size for it. - */ -int -fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail, - pgFile *file, char **errormsg) +static err_i +pioLocalDrive_pioWriteFile(VSelf, path_t path, ft_bytes_t content, bool binary) { - fio_header hdr; - int exit_code = SEND_OK; - size_t path_len = strlen(from_fullpath) + 1; - char *buf = pgut_malloc(CHUNK_SIZE); /* buffer */ - send_file_state st = {false, 0, 0, 0}; + FOBJ_FUNC_ARP(); + Self(pioLocalDrive); + err_i err; + pioWriteCloser_i fl; - memset(&hdr, 0, sizeof(hdr)); + fobj_reset_err(&err); - if (file) + if (content.len > PIO_READ_WRITE_FILE_LIMIT) { - st.calc_crc = true; - st.crc = file->crc; + err = $err(RT, "File content too large {path:q}: {size}", + path(path), size(content.len), errNo(EOVERFLOW)); + return $iresult(err); } - hdr.cop = FIO_SEND_FILE; - hdr.size = path_len; + fl = $(pioOpenRewrite, self, path, + .binary = binary, .err = &err); + if ($haserr(err)) + return $iresult(err); -// elog(VERBOSE, "Thread [%d]: Attempting to open remote WAL file '%s'", -// thread_num, from_fullpath); + err = $i(pioWrite, fl, content); + if ($haserr(err)) + return $iresult(err); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); + err = $i(pioWriteFinish, fl); + if ($haserr(err)) + return $iresult(err); - for (;;) - { - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + err = $i(pioClose, fl); + return $iresult(err); +} - if (hdr.cop == FIO_SEND_FILE_EOF) +static err_i +pioLocalDrive_pioSyncTree(VSelf, path_t root) +{ + FOBJ_FUNC_ARP(); + Self(pioLocalDrive); + pioRecursiveDir* walker; + err_i err; + pio_dirent_t dirent; + ft_strbuf_t pathbuf = ft_strbuf_zero(); + ft_arr_str_t dirs = ft_arr_init(); + ft_str_t dir; + int fd; + + walker = pioRecursiveDir_alloc($bind(pioDrive, self), root, &err); + if ($haserr(err)) + return $iresult($err(RT, "pioSyncTree: {cause}", cause(err.self))); + + while ((dirent = pioRecursiveDir_next(walker, &err)).stat.pst_kind) + { + if (dirent.stat.pst_kind == PIO_KIND_DIRECTORY) { - if (st.write_size < st.read_size) - { - if (!cut_zero_tail) - { - /* - * We still need to calc crc for zero tail. - */ - fio_send_file_crc(&st, NULL, 0); - - /* - * Let's write single zero byte to the end of file to restore - * logical size. - * Well, it would be better to use ftruncate here actually, - * but then we need to change interface. - */ - st.read_size -= 1; - buf[0] = 0; - if (!fio_send_file_write(out, &st, buf, 1)) - { - exit_code = WRITE_FAILED; - break; - } - } - } - - if (file) - { - file->crc = st.crc; - file->read_size = st.read_size; - file->write_size = st.write_size; - } - break; + ft_arr_str_push(&dirs, ft_strdup(dirent.name)); + continue; } - else if (hdr.cop == FIO_ERROR) + ft_assert(dirent.stat.pst_kind == PIO_KIND_REGULAR); + + ft_strbuf_reset_for_reuse(&pathbuf); + ft_strbuf_catc(&pathbuf, root); + ft_strbuf_cat_path(&pathbuf, dirent.name); + + /* TODO: use fsync_fname_compat when it will return err_i */ + fd = open(pathbuf.ptr, O_RDWR, PG_BINARY); + if (fd < 0) { - /* handle error, reported by the agent */ - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); - } - exit_code = hdr.arg; - break; + if (errno == EACCES) + continue; + err = $syserr(errno, "Couldn't open for sync {path:q}", path(pathbuf.ptr)); + goto cleanup; } - else if (hdr.cop == FIO_PAGE) + if (fsync(fd) < 0) { - Assert(hdr.size <= CHUNK_SIZE); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - - /* We have received a chunk of data data, lets write it out */ - fio_send_file_crc(&st, buf, hdr.size); - if (!fio_send_file_write(out, &st, buf, hdr.size)) - { - exit_code = WRITE_FAILED; - break; - } + err = $syserr(errno, "Couldn't fsync {path:q}", path(pathbuf.ptr)); + close(fd); + goto cleanup; } - else if (hdr.cop == FIO_PAGE_ZERO) - { - Assert(hdr.size == 0); - Assert(hdr.arg <= CHUNK_SIZE); + (void)close(fd); + } + pioRecursiveDir_close(walker); + if ($haserr(err)) + goto cleanup; - /* - * We have received a chunk of zero data, lets just think we - * wrote it. - */ - st.read_size += hdr.arg; + /* in reverse order therefore innermost directories first */ + while (dirs.len > 0) + { + dir = ft_arr_str_pop(&dirs); + + ft_strbuf_reset_for_reuse(&pathbuf); + ft_strbuf_catc(&pathbuf, root); + ft_strbuf_cat_path(&pathbuf, dir); + + ft_str_free(&dir); + + /* TODO: use fsync_fname_compat when it will return err_i */ + fd = open(pathbuf.ptr, O_RDONLY, PG_BINARY); + if (fd < 0) + { + if (errno == EACCES || errno == EISDIR) + continue; + err = $syserr(errno, "Couldn't open for sync {path:q}", path(pathbuf.ptr)); + goto cleanup; } - else + if (fsync(fd) < 0 && !(errno == EBADF || errno == EINVAL)) { - /* TODO: fio_disconnect may get assert fail when running after this */ - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); + err = $syserr(errno, "Couldn't fsync {path:q}", path(pathbuf.ptr)); + close(fd); + goto cleanup; } + (void)close(fd); } - if (exit_code < OPEN_FAILED) - fio_disconnect(); /* discard possible pending data in pipe */ +cleanup: + while (dirs.len > 0) + { + dir = ft_arr_str_pop(&dirs); + ft_str_free(&dir); + } + ft_strbuf_free(&pathbuf); + ft_arr_str_free(&dirs); - pg_free(buf); - return exit_code; + return $iresult(err); } -int -fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, - pgFile *file, char **errormsg) +/* LOCAL FILE */ +static void +pioLocalReadFile_fobjDispose(VSelf) { - FILE* in; - char* buf; - size_t read_len, non_zero_len; - int exit_code = SEND_OK; - send_file_state st = {false, 0, 0, 0}; - - if (file) + Self(pioLocalReadFile); + if (self->fd >= 0) { - st.calc_crc = true; - st.crc = file->crc; + close(self->fd); + self->fd = -1; } + ft_str_free(&self->path); + ft_bytes_free(&self->buf); +} - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* maybe deleted, it's not error in case of backup */ - if (errno == ENOENT) - return FILE_MISSING; +static err_i +pioLocalReadFile_pioClose(VSelf) +{ + Self(pioLocalReadFile); + err_i err = $noerr(); + int r; + ft_assert(self->fd >= 0, "Closed file abused \"%s\"", self->path.ptr); - *errormsg = psprintf("Cannot open file \"%s\": %s", from_fullpath, - strerror(errno)); - return OPEN_FAILED; - } + r = close(self->fd); + if (r < 0) + err = $syserr(errno, "Cannot close file {path:q}", + path(self->path.ptr)); + self->fd = -1; + return err; +} - /* disable stdio buffering for local input/output files to avoid triple buffering */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - setvbuf(out, NULL, _IONBF, BUFSIZ); +static size_t +pioLocalReadFile_pioRead(VSelf, ft_bytes_t buf, err_i *err) +{ + Self(pioLocalReadFile); + size_t buflen = buf.len; + ft_bytes_t to_read; + ssize_t r; + fobj_reset_err(err); - /* allocate 64kB buffer */ - buf = pgut_malloc(CHUNK_SIZE); + ft_assert(self->fd >= 0, "Closed file abused \"%s\"", self->path.ptr); - /* copy content and calc CRC */ - for (;;) - { - read_len = fread(buf, 1, CHUNK_SIZE, in); + ft_bytes_move(&buf, &self->remain); - if (ferror(in)) - { - *errormsg = psprintf("Cannot read from file \"%s\": %s", - from_fullpath, strerror(errno)); - exit_code = READ_FAILED; - goto cleanup; - } + while (buf.len) + { + ft_assert(self->remain.len == 0); - if (read_len > 0) - { - non_zero_len = find_zero_tail(buf, read_len); - /* - * It is dirty trick to silence warnings in CFS GC process: - * backup at least cfs header size bytes. - */ - if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && - st.read_size + read_len > 0) - { - non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, - st.read_size + read_len); - non_zero_len -= st.read_size; - } - if (non_zero_len > 0) - { - fio_send_file_crc(&st, buf, non_zero_len); - if (!fio_send_file_write(out, &st, buf, non_zero_len)) - { - exit_code = WRITE_FAILED; - goto cleanup; - } - } - if (non_zero_len < read_len) - { - /* Just pretend we wrote it. */ - st.read_size += read_len - non_zero_len; - } - } + to_read = buf.len >= self->buf.len/2 ? buf : self->buf; - if (feof(in)) + r = read(self->fd, to_read.ptr, to_read.len); + if (r < 0) + *err = $syserr(errno, "Cannot read from {path:q}", + path(self->path.ptr)); + if (r <= 0) break; - } - if (st.write_size < st.read_size) - { - if (!cut_zero_tail) + if (to_read.ptr == buf.ptr) + ft_bytes_consume(&buf, r); + else { - /* - * We still need to calc crc for zero tail. - */ - fio_send_file_crc(&st, NULL, 0); - - /* - * Let's write single zero byte to the end of file to restore - * logical size. - * Well, it would be better to use ftruncate here actually, - * but then we need to change interface. - */ - st.read_size -= 1; - buf[0] = 0; - if (!fio_send_file_write(out, &st, buf, 1)) - { - exit_code = WRITE_FAILED; - goto cleanup; - } + self->remain = ft_bytes(self->buf.ptr, r); + ft_bytes_move(&buf, &self->remain); } } - if (file) - { - file->crc = st.crc; - file->read_size = st.read_size; - file->write_size = st.write_size; - } - -cleanup: - free(buf); - fclose(in); - return exit_code; + self->off += buflen - buf.len; + return buflen - buf.len; } -/* Send file content - * On error we return FIO_ERROR message with following codes - * FIO_ERROR: - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - * - * FIO_PAGE - * FIO_SEND_FILE_EOF - * - */ -static void -fio_send_file_impl(int out, char const* path) +static err_i +pioLocalReadFile_pioSeek(VSelf, uint64_t offs) { - FILE *fp; - fio_header hdr; - char *buf = pgut_malloc(CHUNK_SIZE); - size_t read_len = 0; - int64_t read_size = 0; - char *errormsg = NULL; - - /* open source file for read */ - /* TODO: check that file is regular file */ - fp = fopen(path, PG_BINARY_R); - if (!fp) - { - hdr.cop = FIO_ERROR; - - /* do not send exact wording of ENOENT error message - * because it is a very common error in our case, so - * error code is enough. - */ - if (errno == ENOENT) - { - hdr.arg = FILE_MISSING; - hdr.size = 0; - } - else - { - hdr.arg = OPEN_FAILED; - errormsg = pgut_malloc(ERRMSG_MAX_LEN); - /* Construct the error message */ - snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open file '%s': %s", path, strerror(errno)); - hdr.size = strlen(errormsg) + 1; - } + Self(pioLocalReadFile); + uint64_t delta; + off_t pos; - /* send header and message */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (errormsg) - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); + ft_assert(self->fd >= 0, "Closed file abused \"%s\"", self->path.ptr); - goto cleanup; + delta = offs - self->off; /* yep, wrapping. but we are check for >= */ + if (offs >= self->off && delta <= self->remain.len) + { + ft_bytes_consume(&self->remain, delta); + self->off = offs; + return $noerr(); } + /* + * Drop buffer if we seek too far or if we seek back. + * Seek back is used to re-read data from disk, so no buffer allowed. + */ + self->remain = ft_bytes(NULL, 0); - /* disable stdio buffering */ - setvbuf(fp, NULL, _IONBF, BUFSIZ); + pos = lseek(self->fd, offs, SEEK_SET); + if (pos == (off_t)-1) + return $syserr(errno, "Can not seek to {offs} in file {path:q}", offs(offs), path(self->path.ptr)); - /* copy content */ - for (;;) + ft_assert(pos == offs); + self->off = offs; + return $noerr(); +} + +static pio_stat_t +pioLocalReadFile_pioFileStat(VSelf, err_i *err) +{ + Self(pioLocalReadFile); + struct stat st = {0}; + pio_stat_t pst = {0}; + int r; + fobj_reset_err(err); + + r = fstat(self->fd, &st); + if (r < 0) + *err = $syserr(errno, "Cannot stat file {path:q}", path(self->path.ptr)); + else { - read_len = fread(buf, 1, CHUNK_SIZE, fp); - memset(&hdr, 0, sizeof(hdr)); + pst.pst_kind = pio_statmode2file_kind(st.st_mode, self->path.ptr); + pst.pst_mode = pio_limit_mode(st.st_mode); + pst.pst_size = st.st_size; + pst.pst_mtime = st.st_mtime; + } + return pst; +} - /* report error */ - if (ferror(fp)) - { - hdr.cop = FIO_ERROR; - errormsg = pgut_malloc(ERRMSG_MAX_LEN); - hdr.arg = READ_FAILED; - /* Construct the error message */ - snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot read from file '%s': %s", path, strerror(errno)); - hdr.size = strlen(errormsg) + 1; - /* send header and message */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); +static fobjStr* +pioLocalReadFile_fobjRepr(VSelf) +{ + Self(pioLocalReadFile); + return $fmt("pioLocalReadFile({path:q}, fd:{fd}", + (path, $S(self->path.ptr)), (fd, $I(self->fd))); +} - goto cleanup; - } +static err_i +pioLocalWriteFile_pioWrite(VSelf, ft_bytes_t buf) +{ + Self(pioLocalWriteFile); + size_t r; - if (read_len > 0) - { - /* send chunk */ - int64_t non_zero_len = find_zero_tail(buf, read_len); - /* - * It is dirty trick to silence warnings in CFS GC process: - * backup at least cfs header size bytes. - */ - if (read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && - read_size + read_len > 0) - { - non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, - read_size + read_len); - non_zero_len -= read_size; - } + if (buf.len == 0) + return $noerr(); - if (non_zero_len > 0) - { - hdr.cop = FIO_PAGE; - hdr.size = non_zero_len; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, buf, non_zero_len), non_zero_len); - } + r = fwrite(buf.ptr, 1, buf.len, self->fl); + if (r < buf.len) + return $syserr(errno, "Writting file {path:q}", + path(self->path_tmp.ptr)); + return $noerr(); +} - if (non_zero_len < read_len) - { - hdr.cop = FIO_PAGE_ZERO; - hdr.size = 0; - hdr.arg = read_len - non_zero_len; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - } +static err_i +pioLocalWriteFile_pioWriteCompressed(VSelf, ft_bytes_t buf, CompressAlg compress_alg) +{ + Self(pioLocalWriteFile); + char decbuf[BLCKSZ]; + const char *errormsg = NULL; + int32 uncompressed_size; - read_size += read_len; - } + ft_assert(buf.len != 0); - if (feof(fp)) - break; + uncompressed_size = do_decompress(decbuf, BLCKSZ, buf.ptr, buf.len, + compress_alg, &errormsg); + if (errormsg != NULL) + { + return $err(RT, "An error occured during decompressing block for {path:q}: {causeStr}", + path(self->path.ptr), causeStr(errormsg)); } - /* we are done, send eof */ - hdr.cop = FIO_SEND_FILE_EOF; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (uncompressed_size != BLCKSZ) + { + return $err(RT, "Page uncompressed to {size} bytes != BLCKSZ (for {path:q})", + path(self->path.ptr), size(uncompressed_size)); + } -cleanup: - if (fp) - fclose(fp); - pg_free(buf); - pg_free(errormsg); - return; + return $(pioWrite, self, ft_bytes(decbuf, BLCKSZ)); } -/* - * Read the local file to compute its CRC. - * We cannot make decision about file decompression because - * user may ask to backup already compressed files and we should be - * obvious about it. - */ -pg_crc32 -pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) +static err_i +pioLocalWriteFile_pioSeek(VSelf, uint64_t offs) { - FILE *fp; - pg_crc32 crc = 0; - char *buf; - size_t len = 0; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = fopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - } + Self(pioLocalWriteFile); - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); - } + ft_assert(self->fl != NULL, "Closed file abused \"%s\"", self->path.ptr); - /* disable stdio buffering */ - setvbuf(fp, NULL, _IONBF, BUFSIZ); - buf = pgut_malloc(STDIO_BUFSIZE); + if (fseeko(self->fl, offs, SEEK_SET)) + return $syserr(errno, "Can not seek to {offs} in file {path:q}", offs(offs), path(self->path.ptr)); - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); + return $noerr(); +} - len = fread(buf, 1, STDIO_BUFSIZE, fp); +static err_i +pioLocalWriteFile_pioWriteFinish(VSelf) +{ + Self(pioLocalWriteFile); + err_i err = $noerr(); - if (ferror(fp)) - elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + if (fflush(self->fl) != 0) + err = $syserr(errno, "Flushing file {path:q}", + path(self->path_tmp.ptr)); + return err; +} - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); +static err_i +pioLocalWriteFile_pioTruncate(VSelf, uint64_t sz) +{ + Self(pioLocalWriteFile); + ft_assert(self->fl != NULL, "Closed file abused \"%s\"", self->path_tmp.ptr); - if (feof(fp)) - break; - } + /* it is better to flush before we will truncate */ + if (fflush(self->fl)) + return $syserr(errno, "Cannot flush file {path:q}", + path(self->path_tmp.ptr)); - FIN_FILE_CRC32(use_crc32c, crc); - fclose(fp); - pg_free(buf); + if (ftruncate(fileno(self->fl), sz) < 0) + return $syserr(errno, "Cannot truncate file {path:q}", + path(self->path_tmp.ptr)); + /* TODO: what to do with file position? */ - return crc; + return $noerr(); } -/* - * Read the local file to compute CRC for it extened to real_size. - */ -pg_crc32 -pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok) +static err_i +pioLocalWriteFile_pioClose(VSelf) { - FILE *fp; - char *buf; - size_t len = 0; - size_t non_zero_len; - send_file_state st = {true, 0, 0, 0}; + Self(pioLocalWriteFile); + int fd; + int r; - INIT_FILE_CRC32(use_crc32c, st.crc); + fd = fileno(self->fl); - /* open file in binary read mode */ - fp = fopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, st.crc); - return st.crc; - } - } + if (fflush(self->fl) != 0) + return $syserr(errno, "Flushing file {path:q}", + path(self->path_tmp.ptr)); - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); + if (ferror(self->fl)) + { + fclose(self->fl); + self->fl = NULL; + return $noerr(); } - /* disable stdio buffering */ - setvbuf(fp, NULL, _IONBF, BUFSIZ); - buf = pgut_malloc(CHUNK_SIZE); - - /* calc CRC of file */ - for (;;) + if (self->sync) { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = fread(buf, 1, STDIO_BUFSIZE, fp); - - if (ferror(fp)) - elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + r = fsync(fd); + if (r < 0) + return $syserr(errno, "Cannot fsync file {path:q}", + path(self->path_tmp.ptr)); + } - non_zero_len = find_zero_tail(buf, len); - /* same trick as in fio_send_file */ - if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && - st.read_size + len > 0) - { - non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, - st.read_size + len); - non_zero_len -= st.read_size; - } - if (non_zero_len) + if (self->use_temp) + { + if (rename(self->path_tmp.ptr, self->path.ptr)) + return $syserr(errno, "Cannot rename file {old_path:q} to {new_path:q}", + old_path(self->path_tmp.ptr), + new_path(self->path.ptr)); + /* mark as renamed so fobjDispose will not delete it */ + self->delete_in_dispose = false; + + if (self->sync) { - fio_send_file_crc(&st, buf, non_zero_len); - st.write_size += st.read_size + non_zero_len; + /* + * To guarantee renaming the file is persistent, fsync the file with its + * new name, and its containing directory. + */ + r = fsync(fd); + if (r < 0) + return $syserr(errno, "Cannot fsync file {path:q}", + path(self->path.ptr)); + + if (fsync_parent_path_compat(self->path.ptr) != 0) + return $syserr(errno, "Cannot fsync file {path:q}", + path(self->path.ptr)); } - st.read_size += len; - - if (feof(fp)) - break; } + else + self->delete_in_dispose = false; - FIN_FILE_CRC32(use_crc32c, st.crc); - fclose(fp); - pg_free(buf); + if (fclose(self->fl)) + return $syserr(errno, "Cannot close file {path:q}", + path(self->path_tmp.ptr)); + self->fl = NULL; - return st.crc; + return $noerr(); } -/* - * Read the local file to compute its CRC. - * We cannot make decision about file decompression because - * user may ask to backup already compressed files and we should be - * obvious about it. - */ -pg_crc32 -pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) +static void +pioLocalWriteFile_fobjDispose(VSelf) { - gzFile fp; - pg_crc32 crc = 0; - int len = 0; - int err; - char *buf; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = gzopen(file_path, PG_BINARY_R); - if (fp == NULL) + Self(pioLocalWriteFile); + if (self->fl != NULL) { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - } - - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); + fclose(self->fl); + self->fl = NULL; + } + if (self->delete_in_dispose) + { + remove(self->path_tmp.ptr); } + ft_str_free(&self->path); + ft_str_free(&self->path_tmp); + ft_bytes_free(&self->buf); +} - buf = pgut_malloc(STDIO_BUFSIZE); +static pio_dirent_t +pioLocalDir_pioDirNext(VSelf, err_i* err) +{ + Self(pioLocalDir); + struct dirent* ent; + pio_dirent_t entry = {.stat={.pst_kind=PIO_KIND_UNKNOWN}}; + char path[MAXPGPATH]; + fobj_reset_err(err); + + ft_assert(self->dir != NULL, "Abuse closed dir"); + + ft_strbuf_reset_for_reuse(&self->name_buf); - /* calc CRC of file */ for (;;) { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); + errno = 0; + ent = readdir(self->dir); + if (ent == NULL && errno != 0) + *err = $syserr(errno, "Could not read dir {path:q}", + path(self->path.ptr)); + if (ent == NULL) + return entry; + + /* Skip '.', '..' and all hidden files as well */ + if (ent->d_name[0] == '.') + continue; - len = gzread(fp, buf, STDIO_BUFSIZE); + join_path_components(path, self->path.ptr, ent->d_name); + entry.stat = $i(pioStat, localDrive, path, true, .err = err); + if (getErrno(*err) == ENOENT) + { /* skip just deleted file */ + fobj_reset_err(err); // will be released within outter ARP. + continue; + } + if ($haserr(*err)) + return entry; - if (len <= 0) + /* + * Add only files, directories and links. Skip sockets and other + * unexpected file formats. + */ + if (entry.stat.pst_kind != PIO_KIND_DIRECTORY && + entry.stat.pst_kind != PIO_KIND_REGULAR) { - /* we either run into eof or error */ - if (gzeof(fp)) - break; - else - { - const char *err_str = NULL; - - err_str = gzerror(fp, &err); - elog(ERROR, "Cannot read from compressed file %s", err_str); - } + elog(WARNING, "Skip '%s': unexpected file kind %s", path, + pio_file_kind2str(entry.stat.pst_kind, path)); + continue; } - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); + ft_strbuf_catc(&self->name_buf, ent->d_name); + entry.name = ft_strbuf_ref(&self->name_buf); + return entry; } +} - FIN_FILE_CRC32(use_crc32c, crc); - gzclose(fp); - pg_free(buf); +static err_i +pioLocalDir_pioClose(VSelf) +{ + Self(pioLocalDir); + int rc; - return crc; + rc = closedir(self->dir); + self->dir = NULL; + if (rc) + return $syserr(errno, "Could not close dir {path:q}", + path(self->path.ptr)); + return $noerr(); } -/* Compile the array of files located on remote machine in directory root */ static void -fio_list_dir_internal(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num) +pioLocalDir_fobjDispose(VSelf) { - fio_header hdr; - fio_list_dir_request req; - char *buf = pgut_malloc(CHUNK_SIZE); + Self(pioLocalDir); - /* Send to the agent message with parameters for directory listing */ - snprintf(req.path, MAXPGPATH, "%s", root); - req.exclude = exclude; - req.follow_symlink = follow_symlink; - req.add_root = add_root; - req.backup_logs = backup_logs; - req.exclusive_backup = exclusive_backup; - req.skip_hidden = skip_hidden; - req.external_dir_num = external_dir_num; - - hdr.cop = FIO_LIST_DIR; - hdr.size = sizeof(req); + if (self->dir) + closedir(self->dir); + self->dir = NULL; + ft_str_free(&self->path); + ft_strbuf_free(&self->name_buf); +} - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, &req, hdr.size), hdr.size); +/* REMOTE DRIVE */ - for (;;) - { - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); +static pioReader_i +pioRemoteDrive_pioOpenRead(VSelf, path_t path, err_i *err) +{ + int handle; + fio_header hdr; + fobj_reset_err(err); + fobj_t file; + fio_ensure_remote(); - if (hdr.cop == FIO_SEND_FILE_EOF) - { - /* the work is done */ - break; - } - else if (hdr.cop == FIO_SEND_FILE) - { - pgFile *file = NULL; - fio_pgFile fio_file; - - /* receive rel_path */ - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - file = pgFileInit(buf); - - /* receive metainformation */ - IO_CHECK(fio_read_all(fio_stdin, &fio_file, sizeof(fio_file)), sizeof(fio_file)); - - file->mode = fio_file.mode; - file->size = fio_file.size; - file->mtime = fio_file.mtime; - file->is_datafile = fio_file.is_datafile; - file->tblspcOid = fio_file.tblspcOid; - file->dbOid = fio_file.dbOid; - file->relOid = fio_file.relOid; - file->forkName = fio_file.forkName; - file->segno = fio_file.segno; - file->external_dir_num = fio_file.external_dir_num; - - if (fio_file.linked_len > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, fio_file.linked_len), fio_file.linked_len); + handle = find_free_handle(); - file->linked = pgut_malloc(fio_file.linked_len); - snprintf(file->linked, fio_file.linked_len, "%s", buf); - } + hdr.cop = FIO_OPEN; + hdr.handle = handle; + hdr.size = strlen(path) + 1; + hdr.arg = O_RDONLY; + set_handle(handle); -// elog(INFO, "Received file: %s, mode: %u, size: %lu, mtime: %lu", -// file->rel_path, file->mode, file->size, file->mtime); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - parray_append(files, file); - } - else - { - /* TODO: fio_disconnect may get assert fail when running after this */ - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); - } - } + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - pg_free(buf); + if (hdr.arg != 0) + { + *err = $syserr((int)hdr.arg, "Cannot open remote file {path:q}", + path(path)); + unset_handle(hdr.handle); + return (pioReader_i){NULL}; + } + file = $alloc(pioRemoteFile, .handle = handle, + .p = { .path = ft_cstrdup(path) }); + return $bind(pioReader, file); } - -/* - * To get the arrays of files we use the same function dir_list_file(), - * that is used for local backup. - * After that we iterate over arrays and for every file send at least - * two messages to main process: - * 1. rel_path - * 2. metainformation (size, mtime, etc) - * 3. link path (optional) - * - * TODO: replace FIO_SEND_FILE and FIO_SEND_FILE_EOF with dedicated messages - */ -static void -fio_list_dir_impl(int out, char* buf) +static pioReadStream_i +pioRemoteDrive_pioOpenReadStream(VSelf, path_t path, err_i *err) { - int i; - fio_header hdr; - fio_list_dir_request *req = (fio_list_dir_request*) buf; - parray *file_files = parray_new(); - - /* - * Disable logging into console any messages with exception of ERROR messages, - * because currently we have no mechanism to notify the main process - * about then message been sent. - * TODO: correctly send elog messages from agent to main process. - */ - instance_config.logger.log_level_console = ERROR; - exclusive_backup = req->exclusive_backup; - - dir_list_file(file_files, req->path, req->exclude, req->follow_symlink, - req->add_root, req->backup_logs, req->skip_hidden, - req->external_dir_num, FIO_LOCAL_HOST); - - /* send information about files to the main process */ - for (i = 0; i < parray_num(file_files); i++) - { - fio_pgFile fio_file; - pgFile *file = (pgFile *) parray_get(file_files, i); - - fio_file.mode = file->mode; - fio_file.size = file->size; - fio_file.mtime = file->mtime; - fio_file.is_datafile = file->is_datafile; - fio_file.tblspcOid = file->tblspcOid; - fio_file.dbOid = file->dbOid; - fio_file.relOid = file->relOid; - fio_file.forkName = file->forkName; - fio_file.segno = file->segno; - fio_file.external_dir_num = file->external_dir_num; - - if (file->linked) - fio_file.linked_len = strlen(file->linked) + 1; - else - fio_file.linked_len = 0; - - hdr.cop = FIO_SEND_FILE; - hdr.size = strlen(file->rel_path) + 1; + Self(pioRemoteDrive); - /* send rel_path first */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, file->rel_path, hdr.size), hdr.size); + pioReader_i fl = $(pioOpenRead, self, path, err); + if ($haserr(*err)) + return $null(pioReadStream); - /* now send file metainformation */ - IO_CHECK(fio_write_all(out, &fio_file, sizeof(fio_file)), sizeof(fio_file)); - - /* If file is a symlink, then send link path */ - if (file->linked) - IO_CHECK(fio_write_all(out, file->linked, fio_file.linked_len), fio_file.linked_len); - - pgFileFree(file); + *err = $(pioSetAsync, fl.self, true); + if ($haserr(*err)) + { + $idel(&fl); + return $null(pioReadStream); } - parray_free(file_files); - hdr.cop = FIO_SEND_FILE_EOF; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + return $reduce(pioReadStream, fl); } -/* Wrapper for directory listing */ -void -fio_list_dir(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num) +static pio_stat_t +pioRemoteDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err) { - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir_internal(files, root, exclude, follow_symlink, add_root, - backup_logs, skip_hidden, external_dir_num); - else - dir_list_file(files, root, exclude, follow_symlink, add_root, - backup_logs, skip_hidden, external_dir_num, FIO_LOCAL_HOST); + pio_stat_t st = {0}; + fio_header hdr = { + .cop = FIO_STAT, + .handle = -1, + .size = strlen(path) + 1, + .arg = follow_symlink, + }; + fobj_reset_err(err); + fio_ensure_remote(); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_dbg_assert(hdr.cop == FIO_STAT); + IO_CHECK(fio_read_all(fio_stdin, &st, sizeof(st)), sizeof(st)); + + if (hdr.arg != 0) + { + *err = $syserr((int)hdr.arg, "Cannot stat remote file {path:q}", + path(path)); + } + return st; } -PageState * -fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, - XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location) +static bool +pioRemoteDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2) { - if (fio_is_remote(location)) - { - fio_header hdr; - fio_checksum_map_request req_hdr; - PageState *checksum_map = NULL; - size_t path_len = strlen(fullpath) + 1; + fio_header hdr = { + .cop = FIO_FILES_ARE_SAME, + .handle = -1, + .arg = 0, + }; + char _buf[512]; + ft_strbuf_t buf = ft_strbuf_init_stack(_buf, sizeof(_buf)); + ft_strbuf_catc_zt(&buf, file1); + ft_strbuf_catc_zt(&buf, file2); + hdr.size = buf.len + 1; + fio_ensure_remote(); - req_hdr.n_blocks = n_blocks; - req_hdr.segmentno = segmentno; - req_hdr.stop_lsn = dest_stop_lsn; - req_hdr.checksumVersion = checksum_version; + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len+1), buf.len+1); - hdr.cop = FIO_GET_CHECKSUM_MAP; - hdr.size = sizeof(req_hdr) + path_len; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_dbg_assert(hdr.cop == FIO_FILES_ARE_SAME); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); - IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); + ft_strbuf_free(&buf); - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + return hdr.arg == 1; +} - if (hdr.size > 0) - { - checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); - memset(checksum_map, 0, n_blocks * sizeof(PageState)); - IO_CHECK(fio_read_all(fio_stdin, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); - } +#define pioRemoteDrive_pioExists common_pioExists - return checksum_map; - } - else - { +static err_i +pioRemoteDrive_pioRemove(VSelf, path_t path, bool missing_ok) +{ + fio_header hdr = { + .cop = FIO_REMOVE, + .handle = -1, + .size = strlen(path) + 1, + .arg = missing_ok ? 1 : 0, + }; + fio_ensure_remote(); - return get_checksum_map(fullpath, checksum_version, - n_blocks, dest_stop_lsn, segmentno); - } + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_dbg_assert(hdr.cop == FIO_REMOVE); + + if (hdr.arg != 0) + { + return $syserr((int)hdr.arg, "Cannot remove remote file {path:q}", + path(path)); + } + return $noerr(); } -static void -fio_get_checksum_map_impl(int out, char *buf) +static err_i +pioRemoteDrive_pioRename(VSelf, path_t old_path, path_t new_path) { - fio_header hdr; - PageState *checksum_map = NULL; - char *fullpath = (char*) buf + sizeof(fio_checksum_map_request); - fio_checksum_map_request *req = (fio_checksum_map_request*) buf; + size_t old_path_len = strlen(old_path) + 1; + size_t new_path_len = strlen(new_path) + 1; + fio_header hdr = { + .cop = FIO_RENAME, + .handle = -1, + .size = old_path_len + new_path_len, + .arg = 0, + }; + fio_ensure_remote(); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, old_path, old_path_len), old_path_len); + IO_CHECK(fio_write_all(fio_stdout, new_path, new_path_len), new_path_len); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_dbg_assert(hdr.cop == FIO_RENAME); + + if (hdr.arg != 0) + { + return $syserr((int)hdr.arg, "Cannot rename remote file {old_path:q} to {new_path:q}", + old_path(old_path), new_path(new_path)); + } + return $noerr(); +} - checksum_map = get_checksum_map(fullpath, req->checksumVersion, - req->n_blocks, req->stop_lsn, req->segmentno); - hdr.size = req->n_blocks; +static pg_crc32 +pioRemoteDrive_pioGetCRC32(VSelf, path_t path, + bool compressed, bool truncated, + err_i *err) +{ + fio_header hdr; + size_t path_len = strlen(path) + 1; + fobj_reset_err(err); + fio_ensure_remote(); + + hdr.cop = PIO_GET_CRC32; + hdr.handle = -1; + hdr.size = path_len; + hdr.arg = 0; + + if (compressed) + hdr.arg = GET_CRC32_DECOMPRESS; + if (truncated) + hdr.arg |= GET_CRC32_TRUNCATED; + elog(VERBOSE, "Remote Drive calculate crc32 for '%s', hdr.arg=%d", + path, compressed); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.cop == FIO_PIO_ERROR) + { + *err = fio_receive_pio_err(&hdr); + return 0; + } - /* send array of PageState`s to main process */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size > 0) - IO_CHECK(fio_write_all(out, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); + return hdr.arg; +} - pg_free(checksum_map); +static bool +pioRemoteDrive_pioIsRemote(VSelf) +{ + return true; } -datapagemap_t * -fio_get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno, - fio_location location) +static err_i +pioRemoteDrive_pioMakeDir(VSelf, path_t path, mode_t mode, bool strict) { - datapagemap_t* lsn_map = NULL; + fio_header hdr = { + .cop = FIO_MKDIR, + .handle = strict ? 1 : 0, /* ugly "hack" to pass more params*/ + .size = strlen(path) + 1, + .arg = mode, + }; + fio_ensure_remote(); - if (fio_is_remote(location)) - { - fio_header hdr; - fio_lsn_map_request req_hdr; - size_t path_len = strlen(fullpath) + 1; + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - req_hdr.n_blocks = n_blocks; - req_hdr.segmentno = segmentno; - req_hdr.shift_lsn = shift_lsn; - req_hdr.checksumVersion = checksum_version; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_MKDIR); - hdr.cop = FIO_GET_LSN_MAP; - hdr.size = sizeof(req_hdr) + path_len; + if (hdr.arg == 0) + { + return $noerr(); + } + return $syserr(hdr.arg, "Cannot make dir {path:q}", path(path)); +} - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); - IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); +static pioDirIter_i +pioRemoteDrive_pioOpenDir(VSelf, path_t path, err_i* err) +{ + Self(pioRemoteDrive); + fio_header hdr = { + .cop = PIO_DIR_OPEN, + .handle = find_free_handle(), + .size = strlen(path)+1, + }; + fobj_reset_err(err); + fio_ensure_remote(); - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - if (hdr.size > 0) - { - lsn_map = pgut_malloc(sizeof(datapagemap_t)); - memset(lsn_map, 0, sizeof(datapagemap_t)); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.cop == FIO_PIO_ERROR) + { + *err = fio_receive_pio_err(&hdr); + return $null(pioDirIter); + } + ft_assert(hdr.cop == PIO_DIR_OPEN); + set_handle(hdr.handle); + return $bind(pioDirIter, + $alloc(pioRemoteDir, + .path = ft_strdupc(path), + .handle = hdr.handle, + .pos = 0)); +} - lsn_map->bitmap = pgut_malloc(hdr.size); - lsn_map->bitmapsize = hdr.size; +static bool +pioRemoteDrive_pioIsDirEmpty(VSelf, path_t path, err_i* err) +{ + Self(pioRemoteDrive); + fio_header hdr = { + .cop = PIO_IS_DIR_EMPTY, + .size = strlen(path)+1, + }; + fobj_reset_err(err); + fio_ensure_remote(); - IO_CHECK(fio_read_all(fio_stdin, lsn_map->bitmap, hdr.size), hdr.size); - } - } - else + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.cop == FIO_PIO_ERROR) { - lsn_map = get_lsn_map(fullpath, checksum_version, n_blocks, - shift_lsn, segmentno); + *err = fio_receive_pio_err(&hdr); + return false; } - - return lsn_map; + ft_assert(hdr.cop == PIO_IS_DIR_EMPTY); + return hdr.arg; } static void -fio_get_lsn_map_impl(int out, char *buf) +pioRemoteDrive_pioRemoveDir(VSelf, const char *root, bool root_as_well) { + FOBJ_FUNC_ARP(); + fio_header hdr; + fio_remove_dir_request req; + fio_ensure_remote(); + + /* Send to the agent message with parameters for directory listing */ + snprintf(req.path, MAXPGPATH, "%s", root); + req.root_as_well = root_as_well; + + hdr.cop = FIO_REMOVE_DIR; + hdr.size = sizeof(req); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &req, hdr.size), hdr.size); + + /* get the response */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_REMOVE_DIR); + + if (hdr.arg != 0) + elog(ERROR, "couldn't remove remote dir"); +} + +static ft_bytes_t +pioRemoteDrive_pioReadFile(VSelf, path_t path, bool binary, err_i* err) { - fio_header hdr; - datapagemap_t *lsn_map = NULL; - char *fullpath = (char*) buf + sizeof(fio_lsn_map_request); - fio_lsn_map_request *req = (fio_lsn_map_request*) buf; + Self(pioRemoteDrive); + ft_bytes_t res; - lsn_map = get_lsn_map(fullpath, req->checksumVersion, req->n_blocks, - req->shift_lsn, req->segmentno); - if (lsn_map) - hdr.size = lsn_map->bitmapsize; - else - hdr.size = 0; + fobj_reset_err(err); - /* send bitmap to main process */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size > 0) - IO_CHECK(fio_write_all(out, lsn_map->bitmap, hdr.size), hdr.size); + fio_ensure_remote(); - if (lsn_map) + fio_header hdr = { + .cop = FIO_READ_FILE_AT_ONCE, + .handle = -1, + .size = strlen(path)+1, + .arg = binary, + }; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + + /* get the response */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.cop == FIO_PIO_ERROR) { - pg_free(lsn_map->bitmap); - pg_free(lsn_map); + *err = fio_receive_pio_err(&hdr); + return ft_bytes(NULL, 0); } + + Assert(hdr.cop == FIO_READ_FILE_AT_ONCE); + + res = ft_bytes_alloc(hdr.size); + IO_CHECK(fio_read_all(fio_stdin, res.ptr, hdr.size), hdr.size); + + return res; } -/* - * Return pid of postmaster process running in given pgdata on local machine. - * Return 0 if there is none. - * Return 1 if postmaster.pid is mangled. - */ -static pid_t -local_check_postmaster(const char *pgdata) +static err_i +pioRemoteDrive_pioWriteFile(VSelf, path_t path, ft_bytes_t content, bool binary) { - FILE *fp; - pid_t pid; - char pid_file[MAXPGPATH]; + FOBJ_FUNC_ARP(); + Self(pioLocalDrive); + fio_header hdr; + ft_bytes_t msg; + err_i err; + ft_strbuf_t buf = ft_strbuf_zero(); - join_path_components(pid_file, pgdata, "postmaster.pid"); + fobj_reset_err(&err); - fp = fopen(pid_file, "r"); - if (fp == NULL) - { - /* No pid file, acceptable*/ - if (errno == ENOENT) - return 0; - else - elog(ERROR, "Cannot open file \"%s\": %s", - pid_file, strerror(errno)); - } + fio_ensure_remote(); - if (fscanf(fp, "%i", &pid) != 1) + if (content.len > PIO_READ_WRITE_FILE_LIMIT) { - /* something is wrong with the file content */ - pid = 1; + err = $err(RT, "File content too large {path:q}: {size}", + path(path), size(content.len), errNo(EOVERFLOW)); + return $iresult(err); } - if (pid > 1) + ft_strbuf_catc_zt(&buf, path); + ft_strbuf_catbytes(&buf, content); + + hdr = (fio_header){ + .cop = FIO_WRITE_FILE_AT_ONCE, + .handle = -1, + .size = buf.len, + .arg = binary, + }; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len), buf.len); + + ft_strbuf_free(&buf); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_dbg_assert(hdr.cop == FIO_WRITE_FILE_AT_ONCE); + + if (hdr.arg != 0) { - if (kill(pid, 0) != 0) - { - /* process no longer exists */ - if (errno == ESRCH) - pid = 0; - else - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - pid, strerror(errno)); - } + msg = ft_bytes_alloc(hdr.size); + IO_CHECK(fio_read_all(fio_stdin, msg.ptr, hdr.size), hdr.size); + err = $syserr((int)hdr.arg, "Could not write remote file {path:q}: {causeStr}", + path(path), causeStr(msg.ptr)); + ft_bytes_free(&msg); + return $iresult(err); } - fclose(fp); - return pid; + return $noerr(); } -/* - * Go to the remote host and get postmaster pid from file postmaster.pid - * and check that process is running, if process is running, return its pid number. - */ -pid_t -fio_check_postmaster(const char *pgdata, fio_location location) +static err_i +pioRemoteDrive_pioSyncTree(VSelf, path_t root) { - if (fio_is_remote(location)) - { - fio_header hdr; + Self(pioRemoteDrive); + fio_header hdr = { + .cop = PIO_SYNC_TREE, + .size = strlen(root)+1, + }; + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, root, hdr.size), hdr.size); - hdr.cop = FIO_CHECK_POSTMASTER; - hdr.size = strlen(pgdata) + 1; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.cop == FIO_PIO_ERROR) + return fio_receive_pio_err(&hdr); + ft_assert(hdr.cop == PIO_SYNC_TREE); + return $noerr(); +} - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, pgdata, hdr.size), hdr.size); +/* REMOTE FILE */ - /* receive result */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - return hdr.arg; +static err_i +pioRemoteFile_doClose(VSelf) +{ + Self(pioRemoteFile); + err_i err = $noerr(); + fio_header hdr; + + hdr = (fio_header){ + .cop = FIO_CLOSE, + .handle = self->handle, + .size = 0, + .arg = 0, + }; + + unset_handle(hdr.handle); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* Wait for response */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_dbg_assert(hdr.cop == FIO_CLOSE); + + if (hdr.arg != 0 && $isNULL(err)) + { + err = $syserr((int)hdr.arg, "Cannot close remote file {path:q}", + path(self->p.path)); } - else - return local_check_postmaster(pgdata); + + self->p.closed = true; + + return err; } -static void -fio_check_postmaster_impl(int out, char *buf) +static err_i +pioRemoteFile_pioClose(VSelf) { - fio_header hdr; - pid_t postmaster_pid; - char *pgdata = (char*) buf; + Self(pioRemoteFile); + err_i err = $noerr(); - postmaster_pid = local_check_postmaster(pgdata); + ft_assert(self->handle >= 0, "Remote closed file abused \"%s\"", self->p.path); - /* send arrays of checksums to main process */ - hdr.arg = postmaster_pid; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + return fobj_err_combine(err, pioRemoteFile_doClose(self)); } -/* - * Delete file pointed by the pgFile. - * If the pgFile points directory, the directory must be empty. - */ -void -fio_delete(mode_t mode, const char *fullpath, fio_location location) +static size_t +pioRemoteFile_pioAsyncRead(VSelf, ft_bytes_t buf, err_i *err) { - if (fio_is_remote(location)) - { - fio_header hdr; + Self(pioRemoteFile); + fio_header hdr = {0}; + size_t buflen = buf.len; + ft_bytes_t bytes; + fobj_reset_err(err); - hdr.cop = FIO_DELETE; - hdr.size = strlen(fullpath) + 1; - hdr.arg = mode; + ft_assert(self->handle >= 0, "Remote closed file abused \"%s\"", self->p.path); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, fullpath, hdr.size), hdr.size); + if (self->asyncEof) + { + return 0; + } + else if (!self->didAsync) + { + /* start reading */ + hdr.cop = FIO_SEND_FILE_CONTENT; + hdr.handle = self->handle; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + if (self->asyncChunk == NULL) + self->asyncChunk = ft_malloc(CHUNK_SIZE); + self->didAsync = true; + } - } - else - pgFileDelete(mode, fullpath); + while (buf.len > 0) + { + if (self->chunkRest.len > 0) + { + ft_bytes_move(&buf, &self->chunkRest); + continue; + } + + if (buf.len >= CHUNK_SIZE) + bytes = ft_bytes(buf.ptr, CHUNK_SIZE); + else + bytes = ft_bytes(self->asyncChunk, CHUNK_SIZE); + + /* receive data */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.cop == FIO_SEND_FILE_EOF) + { + self->asyncEof = true; + break; + } + else if (hdr.cop == FIO_ERROR) + { + int erno = EIO; + switch ((int)hdr.arg) + { + case FILE_MISSING: + erno = ENOENT; + break; + case OPEN_FAILED: + /* We should be already opened. */ + ft_assert((int)hdr.arg != OPEN_FAILED); + break; + case READ_FAILED: + erno = EIO; + break; + } + /* handle error, reported by the agent */ + if (hdr.size > 0) + { + ft_assert(hdr.size < CHUNK_SIZE); + IO_CHECK(fio_read_all(fio_stdin, self->asyncChunk, hdr.size), hdr.size); + ft_assert(((char*)self->asyncChunk)[hdr.size] == 0); + *err = $syserr(erno, "Cannot async read remote file {path:q}: {remotemsg}", + remotemsg(self->asyncChunk), + path(self->p.path)); + break; + } + else + { + *err = $syserr(erno, "Cannot async read remote file {path:q}", + path(self->p.path)); + } + fio_disconnect(); /* discard possible pending data in pipe */ + break; + } + else if (hdr.cop == FIO_PAGE) + { + ft_assert(hdr.size <= CHUNK_SIZE); + IO_CHECK(fio_read_all(fio_stdin, bytes.ptr, hdr.size), hdr.size); + + if (bytes.ptr != buf.ptr) + { + bytes.len = hdr.size; + ft_bytes_move(&buf, &bytes); + self->chunkRest = bytes; + } + else + { + ft_bytes_consume(&buf, hdr.size); + } + } + else if (hdr.cop == FIO_PAGE_ZERO) + { + ft_assert(hdr.arg <= CHUNK_SIZE); + ft_assert(hdr.size == 0); + + memset(bytes.ptr, 0, hdr.arg); + if (bytes.ptr != buf.ptr) + { + bytes.len = hdr.arg; + ft_bytes_move(&buf, &bytes); + self->chunkRest = bytes; + } + else + { + ft_bytes_consume(&buf, hdr.arg); + } + } + else + { + /* TODO: fio_disconnect may get assert fail when running after this */ + elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); + } + } + + return (buflen - buf.len); } -static void -fio_delete_impl(mode_t mode, char *buf) +static size_t +pioRemoteFile_pioRead(VSelf, ft_bytes_t buf, err_i *err) { - char *fullpath = (char*) buf; + Self(pioRemoteFile); + + ft_assert(self->handle >= 0, "Remote closed file abused \"%s\"", self->p.path); + + if (self->asyncMode) + return $(pioAsyncRead, self, buf, err); + + fio_header hdr = { + .cop = FIO_READ, + .handle = self->handle, + .size = 0, + .arg = buf.len, + }; + fobj_reset_err(err); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_dbg_assert(hdr.cop == FIO_SEND); + IO_CHECK(fio_read_all(fio_stdin, buf.ptr, hdr.size), hdr.size); + if (hdr.arg != 0) { + *err = $syserr((int)hdr.arg, "Cannot read remote file {path:q}", + path(self->p.path)); + return 0; + } - pgFileDelete(mode, fullpath); + return hdr.size; } -/* Execute commands at remote host */ -void -fio_communicate(int in, int out) +static err_i +pioRemoteFile_pioSeek(VSelf, uint64_t offs) { - /* - * Map of file and directory descriptors. - * The same mapping is used in agent and master process, so we - * can use the same index at both sides. - */ - int fd[FIO_FDMAX]; - DIR* dir[FIO_FDMAX]; - struct dirent* entry; - size_t buf_size = 128*1024; - char* buf = (char*)pgut_malloc(buf_size); + Self(pioRemoteFile); fio_header hdr; - struct stat st; - int rc; - int tmp_fd; - pg_crc32 crc; -#ifdef WIN32 - SYS_CHECK(setmode(in, _O_BINARY)); - SYS_CHECK(setmode(out, _O_BINARY)); -#endif + ft_assert(self->handle >= 0, "Remote closed file abused \"%s\"", self->p.path); - /* Main loop until end of processing all master commands */ - while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { - if (hdr.size != 0) { - if (hdr.size > buf_size) { - /* Extend buffer on demand */ - buf_size = hdr.size; - buf = (char*)realloc(buf, buf_size); - } - IO_CHECK(fio_read_all(in, buf, hdr.size), hdr.size); + hdr.cop = FIO_SEEK; + hdr.handle = self->handle; + hdr.size = 0; + hdr.arg = offs; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + return $noerr(); +} + +static err_i +pioRemoteFile_pioSetAsync(VSelf, bool async) +{ + Self(pioRemoteFile); + + ft_assert(self->handle >= 0, "Remote closed file abused \"%s\"", self->p.path); + + if (!self->asyncMode && async) + { + self->asyncMode = true; + } + else if (self->asyncMode && !async) + { + self->asyncMode = false; + } + return $noerr(); +} + +static void +pioRemoteFile_fobjDispose(VSelf) +{ + Self(pioRemoteFile); + if (!self->p.closed) + { + err_i err; + + err = pioRemoteFile_doClose(self); + if ($haserr(err)) + elog(WARNING, "%s", $errmsg(err)); + } + $idel(&self->asyncError); + ft_free(self->asyncChunk); +} + +static fobjStr* +pioRemoteFile_fobjRepr(VSelf) +{ + Self(pioRemoteFile); + return $fmt("pioRemoteFile({path:q}, hnd:{hnd}, async:{asyncMode}, err:{asyncError})", + (path, $S(self->p.path)), + (hnd, $I(self->handle)), + (asyncMode, $B(self->asyncMode)), + (err, self->asyncError.self)); +} + +static pioWriteCloser_i +pioRemoteDrive_pioOpenRewrite(VSelf, path_t path, int permissions, + bool binary, bool use_temp, bool sync, err_i *err) +{ + Self(pioRemoteDrive); + ft_strbuf_t buf = ft_strbuf_zero(); + fobj_t fl; + int handle = find_free_handle(); + + fobj_reset_err(err); + + fio_header hdr = { + .cop = PIO_OPEN_REWRITE, + .handle = handle, + }; + + struct fio_req_open_rewrite req = { + .permissions = permissions, + .binary = binary, + .use_temp = use_temp, + .sync = sync, + }; + + fio_ensure_remote(); + + ft_strbuf_catbytes(&buf, ft_bytes(&hdr, sizeof(hdr))); + ft_strbuf_catbytes(&buf, ft_bytes(&req, sizeof(req))); + ft_strbuf_catc_zt(&buf, path); + + ((fio_header*)buf.ptr)->size = buf.len - sizeof(hdr); + + IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len), buf.len); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.cop == FIO_PIO_ERROR) + { + *err = fio_receive_pio_err(&hdr); + return $null(pioWriteCloser); + } + ft_dbg_assert(hdr.cop == PIO_OPEN_REWRITE && + hdr.handle == handle); + + set_handle(handle); + + fl = $alloc(pioRemoteWriteFile, + .path = ft_strdupc(path), + .handle = handle); + return $bind(pioWriteCloser, fl); +} + +static pioDBWriter_i +pioRemoteDrive_pioOpenWrite(VSelf, path_t path, int permissions, + bool exclusive, bool sync, err_i *err) +{ + Self(pioRemoteDrive); + ft_strbuf_t buf = ft_strbuf_zero(); + fobj_t fl; + int handle = find_free_handle(); + + fio_header hdr = { + .cop = PIO_OPEN_WRITE, + .handle = handle, + }; + + struct fio_req_open_write req = { + .permissions = permissions, + .exclusive = exclusive, + .sync = sync, + }; + + fio_ensure_remote(); + + ft_strbuf_catbytes(&buf, ft_bytes(&hdr, sizeof(hdr))); + ft_strbuf_catbytes(&buf, ft_bytes(&req, sizeof(req))); + ft_strbuf_catc_zt(&buf, path); + + ((fio_header*)buf.ptr)->size = buf.len - sizeof(hdr); + + IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len), buf.len); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.cop == FIO_PIO_ERROR) + { + *err = fio_receive_pio_err(&hdr); + return $null(pioDBWriter); + } + ft_dbg_assert(hdr.cop == PIO_OPEN_WRITE && + hdr.handle == handle); + + set_handle(handle); + + fl = $alloc(pioRemoteWriteFile, + .path = ft_strdupc(path), + .handle = handle); + return $bind(pioDBWriter, fl); +} + +static err_i +pioRemoteWriteFile_pioWrite(VSelf, ft_bytes_t buf) +{ + Self(pioRemoteWriteFile); + fio_header hdr; + + ft_assert(self->handle >= 0); + + if (buf.len == 0) + return $noerr(); + + hdr = (fio_header){ + .cop = PIO_WRITE_ASYNC, + .handle = self->handle, + .size = buf.len, + .arg = 0, + }; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len), buf.len); + + self->did_async = true; + + return $noerr(); +} + +static err_i +pioRemoteWriteFile_pioWriteCompressed(VSelf, ft_bytes_t buf, CompressAlg compress_alg) +{ + Self(pioRemoteWriteFile); + fio_header hdr = { + .cop = PIO_WRITE_COMPRESSED_ASYNC, + .handle = self->handle, + .size = buf.len, + .arg = compress_alg, + }; + + ft_assert(self->handle >= 0); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len), buf.len); + + self->did_async = true; + + return $noerr(); +} + +static err_i +pioRemoteWriteFile_pioSeek(VSelf, uint64_t offs) +{ + Self(pioRemoteWriteFile); + struct __attribute__((packed)) { + fio_header hdr; + uint64_t off; + } req = { + .hdr = { + .cop = PIO_SEEK, + .handle = self->handle, + .size = sizeof(uint64_t), + }, + .off = offs, + }; + + ft_assert(self->handle >= 0, "Remote closed file abused \"%s\"", self->path.ptr); + + IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + + self->did_async = true; + + return $noerr(); +} + +static err_i +pioRemoteWriteFile_pioWriteFinish(VSelf) +{ + Self(pioRemoteWriteFile); + fio_header hdr; + + ft_assert(self->handle >= 0); + + hdr = (fio_header){ + .cop = PIO_GET_ASYNC_ERROR, + .handle = self->handle, + }; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + self->did_async = false; + + if (hdr.cop == FIO_PIO_ERROR) + return fio_receive_pio_err(&hdr); + ft_dbg_assert(hdr.cop == PIO_GET_ASYNC_ERROR); + + return $noerr(); +} + +static err_i +pioRemoteWriteFile_pioTruncate(VSelf, uint64_t sz) +{ + Self(pioRemoteWriteFile); + + struct __attribute__((packed)) { + fio_header hdr; + uint64_t off; + } req = { + .hdr = { + .cop = PIO_TRUNCATE, + .handle = self->handle, + .size = sizeof(uint64_t), + }, + .off = sz, + }; + + ft_assert(self->handle >= 0, "Remote closed file abused \"%s\"", self->path.ptr); + + IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + + self->did_async = true; + + return $noerr(); +} + +static err_i +pioRemoteWriteFile_pioClose(VSelf) +{ + Self(pioRemoteWriteFile); + err_i err = $noerr(); + fio_header hdr = {.cop = PIO_CLOSE, .handle = self->handle }; + + ft_assert(self->handle >= 0); + + if (self->did_async) + err = $(pioWriteFinish, self); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + unset_handle(self->handle); + self->handle = -1; + + if (hdr.cop == FIO_PIO_ERROR) + err = fobj_err_combine(err, fio_receive_pio_err(&hdr)); + return err; +} + +static void +pioRemoteWriteFile_fobjDispose(VSelf) +{ + Self(pioRemoteWriteFile); + + if (self->handle >= 0) + { + fio_header hdr = { + .cop = PIO_DISPOSE, + .handle = self->handle, + }; + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + unset_handle(self->handle); + } + ft_str_free(&self->path); +} + +static pio_dirent_t +pioRemoteDir_pioDirNext(VSelf, err_i *err) +{ + Self(pioRemoteDir); + fio_header hdr; + pio_dirent_t entry = {.stat={.pst_kind=PIO_KIND_UNKNOWN}}; + ft_bytes_t tofree = ft_bytes(NULL, 0); + ft_bytes_t buf; + int n; + fobj_reset_err(err); + + ft_assert(self->handle >= 0, "Abuse closed dir"); + + if (self->pos == self->entries.len) + { + ft_bytes_free(&self->names_buf); + ft_arr_dirent_reset_for_reuse(&self->entries); + self->pos = 0; + + hdr = (fio_header){ + .cop = PIO_DIR_NEXT, + .handle = self->handle, + }; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.cop == FIO_PIO_ERROR) + { + *err = fio_receive_pio_err(&hdr); + return entry; } - errno = 0; /* reset errno */ - switch (hdr.cop) { - case FIO_LOAD: /* Send file content */ - fio_load_file(out, buf); - break; - case FIO_OPENDIR: /* Open directory for traversal */ - dir[hdr.handle] = opendir(buf); - hdr.arg = dir[hdr.handle] == NULL ? errno : 0; - hdr.size = 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_READDIR: /* Get next directory entry */ - hdr.cop = FIO_SEND; - entry = readdir(dir[hdr.handle]); - if (entry != NULL) - { - hdr.size = sizeof(*entry); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, entry, hdr.size), hdr.size); - } - else - { - hdr.size = 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - } - break; - case FIO_CLOSEDIR: /* Finish directory traversal */ - SYS_CHECK(closedir(dir[hdr.handle])); - break; - case FIO_OPEN: /* Open file */ - fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS); - hdr.arg = fd[hdr.handle] < 0 ? errno : 0; - hdr.size = 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_CLOSE: /* Close file */ - fio_close_impl(fd[hdr.handle], out); - break; - case FIO_WRITE: /* Write to the current position in file */ -// IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); - fio_write_impl(fd[hdr.handle], buf, hdr.size, out); - break; - case FIO_WRITE_ASYNC: /* Write to the current position in file */ - fio_write_async_impl(fd[hdr.handle], buf, hdr.size, out); - break; - case FIO_WRITE_COMPRESSED_ASYNC: /* Write to the current position in file */ - fio_write_compressed_impl(fd[hdr.handle], buf, hdr.size, hdr.arg); - break; - case FIO_READ: /* Read from the current position in file */ - if ((size_t)hdr.arg > buf_size) { - buf_size = hdr.arg; - buf = (char*)realloc(buf, buf_size); - } - rc = read(fd[hdr.handle], buf, hdr.arg); - hdr.cop = FIO_SEND; - hdr.size = rc > 0 ? rc : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size != 0) - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); + ft_assert(hdr.cop == PIO_DIR_NEXT); + + if (hdr.arg == 0) + { + /* End of iteration */ + return entry; + } + + buf = ft_bytes_alloc(hdr.size); + tofree = buf; + IO_CHECK(fio_read_all(fio_stdin, buf.ptr, buf.len), buf.len); + + for (n = 0; n < hdr.arg; n++) + { + ft_bytes_shift_must(&buf, FT_BYTES_FOR(entry.stat)); + ft_arr_dirent_push(&self->entries, entry); + } + + self->names_buf = ft_bytes_dup(buf); + buf = self->names_buf; + + for (n = 0; n < self->entries.len; n++) + self->entries.ptr[n].name = ft_bytes_shift_zt(&buf); + + ft_bytes_free(&tofree); + } + + entry = self->entries.ptr[self->pos]; + self->pos++; + return entry; +} + +static err_i +pioRemoteDir_pioClose(VSelf) +{ + Self(pioRemoteDir); + err_i err = $noerr(); + fio_header hdr = {.cop = PIO_CLOSE, .handle = self->handle }; + + ft_assert(self->handle >= 0); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + unset_handle(self->handle); + self->handle = -1; + + if (hdr.cop == FIO_PIO_ERROR) + err = fobj_err_combine(err, fio_receive_pio_err(&hdr)); + return err; +} + +static void +pioRemoteDir_fobjDispose(VSelf) +{ + Self(pioRemoteDir); + + if (self->handle >= 0) + { + fio_header hdr = { + .cop = PIO_DISPOSE, + .handle = self->handle, + }; + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + unset_handle(self->handle); + } + ft_str_free(&self->path); + ft_bytes_free(&self->names_buf); + ft_arr_dirent_reset_for_reuse(&self->entries); +} + +pioRead_i +pioWrapReadFilter(pioRead_i fl, pioFilter_i flt, size_t buf_size) +{ + void *buf; + pioReadFilter* wrap; + + buf = ft_malloc(buf_size); + wrap = $alloc(pioReadFilter, + .wrapped = $iref(fl), + .filter = $iref(flt), + .buffer = buf, + .capa = buf_size); + $implements(pioFltInPlace, flt.self, &wrap->inplace); + return bind_pioRead(wrap); +} + +static size_t +pioReadFilter_pioRead(VSelf, ft_bytes_t wbuf, err_i *err) +{ + Self(pioReadFilter); + fobj_reset_err(err); + pioFltTransformResult tr; + size_t wlen = wbuf.len; + ft_bytes_t rbuf; + size_t r; + + if (self->eof && self->finished) + return 0; + + if ($notNULL(self->inplace) && !self->eof) + { + r = $i(pioRead, self->wrapped, wbuf, err); + if (r > 0) + { + err_i flterr; + flterr = $i(pioFltInPlace, self->inplace, ft_bytes(wbuf.ptr, r)); + *err = fobj_err_combine(*err, flterr); + ft_bytes_consume(&wbuf, r); + } + + if ($haserr(*err)) + return wlen - wbuf.len; + if (r == 0) + { + self->eof = true; + goto eof; + } + } + + while (wbuf.len > 0) + { + /* feed filter */ + rbuf = ft_bytes(self->buffer, self->len); + while (rbuf.len > 0) + { + tr = $i(pioFltTransform, self->filter, rbuf, wbuf, err); + if ($haserr(*err)) + return wlen - wbuf.len; + ft_bytes_consume(&rbuf, tr.consumed); + ft_bytes_consume(&wbuf, tr.produced); + + if (tr.produced == 0) /* Probably need more input to produce */ + break; + } + + if (self->eof) + break; + + /* move rest if any */ + if (rbuf.len > 0) + memmove(self->buffer, rbuf.ptr, rbuf.len); + self->len = rbuf.len; + + if (wbuf.len == 0) break; - case FIO_PREAD: /* Read from specified position in file, ignoring pages beyond horizon of delta backup */ - rc = pread(fd[hdr.handle], buf, BLCKSZ, hdr.arg); - hdr.cop = FIO_SEND; - hdr.arg = rc; - hdr.size = rc >= 0 ? rc : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size != 0) - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); + + /* feed buffer */ + rbuf = ft_bytes(self->buffer, self->capa); + ft_bytes_consume(&rbuf, self->len); + ft_assert(rbuf.len > 0); + r = $i(pioRead, self->wrapped, rbuf, err); + if ($haserr(*err)) + return wlen - wbuf.len; + if (r == 0) + self->eof = true; + self->len += r; + } + +eof: + while (wbuf.len > 0 && self->eof) + { + r = $i(pioFltFinish, self->filter, wbuf, err); + if ($haserr(*err)) + return (ssize_t)(wlen - wbuf.len); + ft_bytes_consume(&wbuf, r); + if (r == 0) + { + self->finished = true; + break; + } + } + + return wlen - wbuf.len; +} + +static err_i +pioReadFilter_pioClose(VSelf) +{ + Self(pioReadFilter); + err_i err = $noerr(); + err_i errcl = $noerr(); + size_t r; + + if (!self->finished) + { + r = $i(pioFltFinish, self->filter, ft_bytes(NULL, 0), &err); + ft_assert(r == 0); + } + if ($ifdef(errcl =, pioClose, self->wrapped.self)) + err = fobj_err_combine(err, errcl); + return err; +} + +static void +pioReadFilter_fobjDispose(VSelf) +{ + Self(pioReadFilter); + $idel(&self->wrapped); + $idel(&self->filter); + ft_free(self->buffer); +} + +static fobjStr* +pioReadFilter_fobjRepr(VSelf) +{ + Self(pioReadFilter); + return $fmt("pioReadFilter(wrapped: {wrapped}, filter: {filter})", + (wrapped, self->wrapped.self), + (filter, self->filter.self)); +} + +pioWriteFlush_i +pioWrapWriteFilter(pioWriteFlush_i fl, pioFilter_i flt, size_t buf_size) +{ + void *buf; + pioWriteFilter* wrap; + + buf = ft_malloc(buf_size); + wrap = $alloc(pioWriteFilter, + .wrapped = $iref(fl), + .filter = $iref(flt), + .buffer = buf, + .capa = buf_size); + $implements(pioFltInPlace, flt.self, &wrap->inplace); + return bind_pioWriteFlush(wrap); +} + +static err_i +pioWriteFilter_pioWrite(VSelf, ft_bytes_t rbuf) +{ + Self(pioWriteFilter); + err_i err = $noerr(); + pioFltTransformResult tr; + size_t rlen = rbuf.len; + ft_bytes_t wbuf; + + if ($notNULL(self->inplace)) + { + err = $i(pioFltInPlace, self->inplace, rbuf); + if ($haserr(err)) + return err; + return $i(pioWrite, self->wrapped, rbuf); + } + + while (rbuf.len > 0) + { + wbuf = ft_bytes(self->buffer, self->capa); + while (wbuf.len > 0) + { + tr = $i(pioFltTransform, self->filter, rbuf, wbuf, &err); + if ($haserr(err)) + return err; + ft_bytes_consume(&rbuf, tr.consumed); + ft_bytes_consume(&wbuf, tr.produced); + + if (tr.produced == 0) /* Probably need more input to produce */ + break; + } + + /* feed writer */ + wbuf = ft_bytes(self->buffer, (char*)wbuf.ptr - (char*)self->buffer); + if (wbuf.len == 0) + { + ft_dbg_assert(rbuf.len == 0); + break; + } + err = $i(pioWrite, self->wrapped, wbuf); + if ($haserr(err)) + return err; + } + + if (rbuf.len) + { + return $err(SysErr, "short write: {writtenSz} < {wantedSz}", + writtenSz(rlen - rbuf.len), wantedSz(rbuf.len)); + } + return $noerr(); +} + +static err_i +pioWriteFilter_pioWriteFinish(VSelf) +{ + Self(pioWriteFilter); + err_i err = $noerr(); + ft_bytes_t wbuf; + size_t r; + + while (!self->finished) + { + wbuf = ft_bytes(self->buffer, self->capa); + while (wbuf.len > 0) + { + r = $i(pioFltFinish, self->filter, wbuf, &err); + if ($haserr(err)) + return err; + ft_bytes_consume(&wbuf, r); + if (r == 0) + { + self->finished = true; + break; + } + } + + /* feed writer */ + wbuf = ft_bytes(self->buffer, (char*)wbuf.ptr - (char*)self->buffer); + if (wbuf.len == 0) + break; + + ft_assert(wbuf.len > 0); + err = $i(pioWrite, self->wrapped, wbuf); + if ($haserr(err)) + return err; + } + + return $i(pioWriteFinish, self->wrapped); +} + +static err_i +pioWriteFilter_pioClose(VSelf) +{ + Self(pioWriteFilter); + err_i err = $noerr(); + err_i errcl = $noerr(); + size_t r; + + if (!self->finished) + { + r = $i(pioFltFinish, self->filter, ft_bytes(NULL, 0), &err); + ft_assert(r == 0); + } + if ($ifdef(errcl =, pioClose, self->wrapped.self)) + err = fobj_err_combine(err, errcl); + return err; +} + +static void +pioWriteFilter_fobjDispose(VSelf) +{ + Self(pioWriteFilter); + $idel(&self->wrapped); + $idel(&self->filter); + ft_free(self->buffer); +} + +static fobjStr* +pioWriteFilter_fobjRepr(VSelf) +{ + Self(pioWriteFilter); + return $fmt("pioWriteFilter(wrapped: {wrapped}, filter: {filter})", + (wrapped, self->wrapped.self), + (filter, self->filter.self)); +} + +pioCutZeroTail* +pioCutZeroTail_alloc(void) +{ + return $alloc(pioCutZeroTail, .read_size = 0, .write_size = 0); +} + +static pioFltTransformResult +pioCutZeroTail_pioFltTransform(VSelf, ft_bytes_t rbuf, ft_bytes_t wbuf, err_i *err) +{ + Self(pioCutZeroTail); + pioFltTransformResult tr = {0, 0}; + size_t wbuf_len = wbuf.len; + size_t rbuf_len = rbuf.len; + size_t non_zero_len; + size_t zeroes_to_fill; + size_t to_move; + ft_bytes_t copybuf; + fobj_reset_err(err); + + non_zero_len = find_zero_tail(rbuf.ptr, rbuf.len); + /* + * It is dirty trick to silence warnings in CFS GC process: + * backup at least cfs header size bytes. + */ + if (self->read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && + self->read_size + rbuf.len > 0) + { + non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, + self->read_size + rbuf.len); + non_zero_len -= self->read_size; + } + + if (non_zero_len == 0) + { + /* pretend we read all buffer */ + self->read_size += rbuf.len; + tr.consumed += rbuf.len; + return tr; + } + + if (self->read_size > self->write_size) + { + /* + * Calculating how many zeroes we can actually copy. + * self->read_size - self->write_size always equals the number of zero bytes. + */ + zeroes_to_fill = ft_min(self->read_size - self->write_size, wbuf.len); + + /* Restoring the zeroes gap */ + memset(wbuf.ptr, 0, zeroes_to_fill); + ft_bytes_consume(&wbuf, zeroes_to_fill); + self->write_size += zeroes_to_fill; + + /* + * At this moment, wbuf.len will be deacreased by zeroes_to_fill, so it + * represents the room left in wbuf. + */ + } + + if (self->read_size == self->write_size && wbuf.len > 0) { + /* + * All zeroes are in buffer at this point so self->read_size == self->write_size + * and all we need to copy is non_zero_len bytes. This can be done in multiple + * calls and the data will not be lost since tr.consumed will be increased only + * by to_copy bytes. + */ + copybuf = ft_bytes_split(&rbuf, ft_min(wbuf.len, non_zero_len)); + to_move = ft_bytes_move(&wbuf, ©buf); + + self->write_size += to_move; + self->read_size += to_move; + non_zero_len -= to_move; + } + + /* + * If we've wrote all non_zero_len bytes, we can safely pretend we read + * all buffer. + */ + if (non_zero_len == 0) + { + self->read_size += rbuf.len; + ft_bytes_consume(&rbuf, rbuf.len); + } + + tr.consumed = rbuf_len - rbuf.len; + tr.produced = wbuf_len - wbuf.len; + return tr; +} + +static size_t +pioCutZeroTail_pioFltFinish(VSelf, ft_bytes_t wbuf, err_i *err) +{ + Self(pioCutZeroTail); + fobj_reset_err(err); + + return 0; +} + +int64_t +pioCutZeroTail_getReadSize(pioCutZeroTail* flt) +{ + return (int64_t)flt->read_size; +} + +int64_t +pioCutZeroTail_getWriteSize(pioCutZeroTail* flt) +{ + return (int64_t)flt->write_size; +} + +#ifdef HAVE_LIBZ +#define MAX_WBITS 15 /* 32K LZ77 window */ +#define DEF_MEM_LEVEL 8 + +static err_i +newGZError(const char *gzmsg, int gzerrno) +{ + if (gzerrno == Z_OK && errno == 0) + return $noerr(); + if (gzerrno == Z_ERRNO) { + return $syserr(errno, "System error during GZ"); + } + + return $err(GZ, "GZ error: {gzErrStr}", gzErrStr(gzmsg), gzErrNo(gzerrno)); +} + +pioFilter_i +pioGZCompressFilter(int level) +{ + pioGZCompress *gz; + int rc; + + gz = $alloc(pioGZCompress); + rc = deflateInit2(&gz->strm, + level, + Z_DEFLATED, + MAX_WBITS + 16, DEF_MEM_LEVEL, + Z_DEFAULT_STRATEGY); + ft_assert(rc == Z_OK, "zlib internal error: %s", gz->strm.msg); + return bind_pioFilter(gz); +} + +pioFilter_i +pioGZDecompressFilter(bool ignoreTruncate) +{ + pioGZDecompress *gz; + int rc; + + gz = $alloc(pioGZDecompress, .ignoreTruncate = ignoreTruncate); + + rc = inflateInit2(&gz->strm, 15 + 16); + ft_assert(rc == Z_OK, "zlib internal error: %s", gz->strm.msg); + return bind_pioFilter(gz); +} + +static pioFltTransformResult +pioGZCompress_pioFltTransform(VSelf, ft_bytes_t rbuf, ft_bytes_t wbuf, err_i *err) +{ + Self(pioGZCompress); + pioFltTransformResult tr = {0, 0}; + size_t rlen = rbuf.len; + size_t wlen = wbuf.len; + ssize_t rc; + fobj_reset_err(err); + + if (self->finished) + { + *err = $err(RT, "pioGZCompress already finished"); + return tr; + } + + while (rbuf.len > 0 && wbuf.len > 0) + { + self->strm.next_in = (Bytef *)rbuf.ptr; + self->strm.avail_in = rbuf.len; + self->strm.next_out = (Bytef *)wbuf.ptr; + self->strm.avail_out = wbuf.len; + + rc = deflate(&self->strm, Z_NO_FLUSH); + ft_dbg_assert(rc == Z_OK); + + ft_bytes_consume(&wbuf, wbuf.len - self->strm.avail_out); + ft_bytes_consume(&rbuf, rbuf.len - self->strm.avail_in); + } + + tr.produced = wlen - wbuf.len; + tr.consumed = rlen - rbuf.len; + return tr; +} + +static size_t +pioGZCompress_pioFltFinish(VSelf, ft_bytes_t wbuf, err_i *err) +{ + Self(pioGZCompress); + size_t wlen = wbuf.len; + int rc; + fobj_reset_err(err); + + if (self->finished) + return 0; + + while (wbuf.len > 0) + { + self->strm.avail_in = 0; + self->strm.next_out = (Bytef *)wbuf.ptr; + self->strm.avail_out = wbuf.len; + + rc = deflate(&self->strm, Z_FINISH); + + ft_bytes_consume(&wbuf, wbuf.len - self->strm.avail_out); + + if (rc == Z_STREAM_END) + { + rc = deflateEnd(&self->strm); + ft_dbg_assert(rc == Z_OK); + self->finished = true; + break; + } + ft_dbg_assert(rc == Z_OK); + } + + return wlen - wbuf.len; +} + +static void +pioGZCompress_fobjDispose(VSelf) +{ + Self(pioGZCompress); + int rc; + + if (!self->finished) + { + rc = deflateEnd(&self->strm); + ft_dbg_assert(rc == Z_OK || rc == Z_DATA_ERROR); + } +} + +static fobjStr* +pioGZCompress_fobjRepr(VSelf) +{ + Self(pioGZCompress); + return $S("pioGZCompress"); +} + +static pioFltTransformResult +pioGZDecompress_pioFltTransform(VSelf, ft_bytes_t rbuf, ft_bytes_t wbuf, err_i* err) +{ + Self(pioGZDecompress); + pioFltTransformResult tr = {0, 0}; + size_t rlen = rbuf.len; + size_t wlen = wbuf.len; + int rc; + fobj_reset_err(err); + + if (self->finished) + { + *err = $err(RT, "pioGZDecompress already finished"); + return tr; + } + + if (self->eof) + return tr; + + while (rbuf.len > 0 && wbuf.len > 0) + { + self->strm.next_in = (Bytef *)rbuf.ptr; + self->strm.avail_in = rbuf.len; + self->strm.next_out = (Bytef *)wbuf.ptr; + self->strm.avail_out = wbuf.len; + + rc = inflate(&self->strm, Z_NO_FLUSH); + + ft_bytes_consume(&wbuf, wbuf.len - self->strm.avail_out); + ft_bytes_consume(&rbuf, rbuf.len - self->strm.avail_in); + + if (rc == Z_STREAM_END) + { + self->eof = true; + break; + } + else if (rc != Z_OK) + { + *err = newGZError(self->strm.msg, rc); + break; + } + } + + tr.produced += wlen - wbuf.len; + tr.consumed += rlen - rbuf.len; + return tr; +} + +static size_t +pioGZDecompress_pioFltFinish(VSelf, ft_bytes_t wbuf, err_i *err) +{ + Self(pioGZDecompress); + size_t wlen = wbuf.len; + int rc; + fobj_reset_err(err); + + if (self->finished) + return 0; + + while (wbuf.len > 0 && !self->eof) + { + self->strm.avail_in = 0; + self->strm.next_out = (Bytef *)wbuf.ptr; + self->strm.avail_out = wbuf.len; + + rc = inflate(&self->strm, Z_SYNC_FLUSH); + + ft_bytes_consume(&wbuf, wbuf.len - self->strm.avail_out); + + if (rc == Z_STREAM_END) + { + self->eof = true; + } + else if (rc == Z_BUF_ERROR && self->ignoreTruncate) + { + self->eof = true; + } + else if (rc != Z_OK) + { + *err = newGZError(self->strm.msg, rc); + break; + } + } + + if (self->eof && !self->finished) + { + rc = inflateEnd(&self->strm); + ft_dbg_assert(rc == Z_OK); + self->finished = true; + } + + return wlen - wbuf.len; +} + +static void +pioGZDecompress_fobjDispose(VSelf) +{ + Self(pioGZDecompress); + int rc; + + if (!self->finished) { + rc = inflateEnd(&self->strm); + ft_dbg_assert(rc == Z_OK); + } +} + +static fobjStr* +pioGZDecompress_fobjRepr(VSelf) +{ + Self(pioGZCompress); + return $S("pioGZDecompress"); +} + +pioWrapRead_i +pioGZDecompressWrapper(bool ignoreTruncate) +{ + return $bind(pioWrapRead, $alloc(pioGZDecompressWrapperObj, + .ignoreTruncate = ignoreTruncate)); +} + +static pioRead_i +pioGZDecompressWrapperObj_pioWrapRead(VSelf, pioRead_i rdr, err_i* err) +{ + Self(pioGZDecompressWrapperObj); + pioFilter_i flt; + fobj_reset_err(err); + + flt = pioGZDecompressFilter(self->ignoreTruncate); + return pioWrapReadFilter(rdr, flt, CHUNK_SIZE); +} +#endif + +extern pioReader_i +pioWrapForReSeek(pioReader_i fl, pioWrapRead_i wr) +{ + pioReSeekableReader* reseek; + err_i err; + + reseek = $alloc(pioReSeekableReader, + .reader = $iref(fl), + .wrapper = $iref(wr), + ); + reseek->wrapped = $iref($i(pioWrapRead, wr, + .reader=$reduce(pioRead, fl), + .err=&err)); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "wrap failed"); + + return $bind(pioReader, reseek); +} + +static size_t +pioReSeekableReader_pioRead(VSelf, ft_bytes_t buf, err_i *err) +{ + Self(pioReSeekableReader); + size_t r; + + ft_assert(!self->had_err, "use after error"); + ft_assert(!self->closed, "use after close"); + + r = $i(pioRead, self->wrapped, buf, err); + self->pos += r; + if ($haserr(*err)) + self->had_err = true; + return r; +} + +static err_i +pioReSeekableReader_pioSeek(VSelf, uint64_t pos) +{ + FOBJ_FUNC_ARP(); + Self(pioReSeekableReader); + char buf[4096]; + size_t need, r; + err_i err; + + ft_assert(!self->had_err, "use after error"); + ft_assert(!self->closed, "use after close"); + + if (pos < self->pos) + { + pioRead_i wrapped; + /* had to read from the beginning and reset filter */ + self->had_err = true; + err = $i(pioSeek, self->reader, 0); + if ($haserr(err)) + return $iresult(err); + self->pos = 0; + + wrapped = $i(pioWrapRead, self->wrapper, + .reader = $reduce(pioRead, self->reader), + .err = &err); + if ($haserr(err)) + return $iresult(err); + $iset(&self->wrapped, wrapped); + self->had_err = false; + } + + while (pos > self->pos) + { + need = ft_min(pos - self->pos, sizeof(buf)); + r = $(pioRead, self, ft_bytes(buf, need), .err = &err); + if ($haserr(err)) + { + self->had_err = true; + return $iresult(err); + } + /* lseek/fseek seeks past the file end without error, so we do */ + if (r < need) break; - case FIO_AGENT_VERSION: - { - size_t payload_size = prepare_compatibility_str(buf, buf_size); + } + + return $noerr(); +} + +static err_i +pioReSeekableReader_pioClose(VSelf) +{ + Self(pioReSeekableReader); + err_i err; + + err = $i(pioClose, self->reader); + self->closed = true; + return err; +} + +static void +pioReSeekableReader_fobjDispose(VSelf) +{ + Self(pioReSeekableReader); + if (!self->closed) + $i(pioClose, self->reader); + $idel(&self->reader); + $idel(&self->wrapper); + $idel(&self->wrapped); +} + +/* Transform filter method */ +/* Must count crc32 of new portion of data. No output needed */ +static pioFltTransformResult +pioCRC32Counter_pioFltTransform(VSelf, ft_bytes_t rbuf, ft_bytes_t wbuf, err_i *err) +{ + Self(pioCRC32Counter); + pioFltTransformResult tr = {0, 0}; + fobj_reset_err(err); + size_t copied = ft_min(wbuf.len, rbuf.len); + + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + COMP_CRC32C(self->crc, rbuf.ptr, copied); + + memmove(wbuf.ptr, rbuf.ptr, copied); + + tr.produced = copied; + tr.consumed = copied; + self->size += copied; + + return tr; +} + +static err_i +pioCRC32Counter_pioFltInPlace(VSelf, ft_bytes_t rbuf) +{ + Self(pioCRC32Counter); + + COMP_CRC32C(self->crc, rbuf.ptr, rbuf.len); + self->size += rbuf.len; + + return $noerr(); +} + +static size_t +pioCRC32Counter_pioFltFinish(VSelf, ft_bytes_t wbuf, err_i *err) +{ + Self(pioCRC32Counter); + fobj_reset_err(err); + + FIN_CRC32C(self->crc); + + return 0; +} + +pg_crc32 +pioCRC32Counter_getCRC32(pioCRC32Counter* flt) +{ + return flt->crc; +} + +int64_t +pioCRC32Counter_getSize(pioCRC32Counter* flt) +{ + return flt->size; +} + +pioWriteFlush_i +pioDevNull_alloc(void) +{ + fobj_t wrap; + + wrap = $alloc(pioDevNull); + return bind_pioWriteFlush(wrap); +} + +static err_i +pioDevNull_pioWrite(VSelf, ft_bytes_t buf) +{ + return $noerr(); +} - hdr.arg = AGENT_PROTOCOL_VERSION; - hdr.size = payload_size; +static err_i +pioDevNull_pioWriteFinish(VSelf) +{ + return $noerr(); +} - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); - break; - } - case FIO_STAT: /* Get information about file with specified path */ - hdr.size = sizeof(st); - rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st); - hdr.arg = rc < 0 ? errno : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); - break; - case FIO_ACCESS: /* Check presence of file with specified name */ - hdr.size = 0; - hdr.arg = access(buf, hdr.arg) < 0 ? errno : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_RENAME: /* Rename file */ - SYS_CHECK(rename(buf, buf + strlen(buf) + 1)); - break; - case FIO_SYMLINK: /* Create symbolic link */ - fio_symlink_impl(out, buf, hdr.arg > 0 ? true : false); - break; - case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */ - SYS_CHECK(remove_file_or_dir(buf)); - break; - case FIO_MKDIR: /* Create directory */ - hdr.size = 0; - hdr.arg = dir_create_dir(buf, hdr.arg, false); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_CHMOD: /* Change file mode */ - SYS_CHECK(chmod(buf, hdr.arg)); - break; - case FIO_SEEK: /* Set current position in file */ - fio_seek_impl(fd[hdr.handle], hdr.arg); - break; - case FIO_TRUNCATE: /* Truncate file */ - SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); - break; - case FIO_LIST_DIR: - fio_list_dir_impl(out, buf); - break; - case FIO_SEND_PAGES: - // buf contain fio_send_request header and bitmap. - fio_send_pages_impl(out, buf); - break; - case FIO_SEND_FILE: - fio_send_file_impl(out, buf); - break; - case FIO_SYNC: - /* open file and fsync it */ - tmp_fd = open(buf, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); - if (tmp_fd < 0) - hdr.arg = errno; - else - { - if (fsync(tmp_fd) == 0) - hdr.arg = 0; - else - hdr.arg = errno; - } - close(tmp_fd); +pioCRC32Counter* +pioCRC32Counter_alloc(void) +{ + pioCRC32Counter *res; + pg_crc32 init_crc = 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_GET_CRC32: - Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 || - (hdr.arg & (GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS)) == GET_CRC32_TRUNCATED); - /* calculate crc32 for a file */ - if ((hdr.arg & GET_CRC32_DECOMPRESS)) - crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); - else if ((hdr.arg & GET_CRC32_TRUNCATED)) - crc = pgFileGetCRCTruncated(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); - else - crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); - IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); - break; - case FIO_GET_CHECKSUM_MAP: - /* calculate crc32 for a file */ - fio_get_checksum_map_impl(out, buf); - break; - case FIO_GET_LSN_MAP: - /* calculate crc32 for a file */ - fio_get_lsn_map_impl(out, buf); - break; - case FIO_CHECK_POSTMASTER: - /* calculate crc32 for a file */ - fio_check_postmaster_impl(out, buf); - break; - case FIO_DELETE: - /* delete file */ - fio_delete_impl(hdr.arg, buf); - break; - case FIO_DISCONNECT: - hdr.cop = FIO_DISCONNECTED; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - free(buf); - return; - case FIO_GET_ASYNC_ERROR: - fio_get_async_error_impl(out); - break; - case FIO_READLINK: /* Read content of a symbolic link */ + INIT_CRC32C(init_crc); + + res = $alloc(pioCRC32Counter, .crc = init_crc); + + return res; +} + + +err_i +pioCopyWithFilters(pioWriteFlush_i dest, pioRead_i src, + pioFilter_i *filters, int nfilters, size_t *copied) +{ + FOBJ_FUNC_ARP(); + size_t _fallback_copied = 0; + err_i err = $noerr(); + err_i rerr = $noerr(); + err_i werr = $noerr(); + void* buf; + int i; + + if (copied == NULL) + copied = &_fallback_copied; + + for (i = nfilters - 1; i >= 0; i--) + dest = pioWrapWriteFilter(dest, filters[i], OUT_BUF_SIZE); + + buf = fobj_alloc_temp(OUT_BUF_SIZE); + + while (!$haserr(rerr) && !$haserr(werr)) + { + size_t read_len = 0; + + read_len = $i(pioRead, src, ft_bytes(buf, OUT_BUF_SIZE), &rerr); + + if (read_len == 0) + break; + + werr = $i(pioWrite, dest, ft_bytes(buf, read_len)); + if ($noerr(werr)) + *copied += read_len; + } + + err = fobj_err_combine(rerr, werr); + if ($haserr(err)) + return $iresult(err); + + /* pioWriteFinish will check for async error if destination was remote */ + err = $i(pioWriteFinish, dest); + if ($haserr(err)) + err = $err(SysErr, "Cannot flush file {path}: {cause}", + path($irepr(dest)), cause(err.self)); + return $iresult(err); +} + +void +init_pio_line_reader(pio_line_reader *r, pioRead_i source, size_t max_length) { + r->source = $iref(source); + r->buf = ft_bytes_alloc(max_length); + r->rest = ft_bytes(NULL, 0); +} + +void +deinit_pio_line_reader(pio_line_reader *r) +{ + $idel(&r->source); + ft_bytes_free(&r->buf); + r->rest = ft_bytes(NULL, 0); +} + +ft_bytes_t +pio_line_reader_getline(pio_line_reader *r, err_i *err) +{ + ft_bytes_t res; + ft_bytes_t tmp; + size_t sz; + char last; + + fobj_reset_err(err); + +retry: + res = ft_bytes_shift_line(&r->rest); + /* if we got too long line */ + if (res.len == r->buf.len) + { + *err = $err(RT, "Line doesn't fit buffer of size {size}", + size(r->buf.len)); + /* restore rest to produce error again next time */ + r->rest = r->buf; + return ft_bytes(NULL, 0); + } + + last = res.len != 0 ? res.ptr[res.len-1] : 0; + /* not first time and definitely reached end of line */ + if (res.len != 0 && (last == '\n' || last == '\r')) + return res; + + if (res.ptr != NULL) + memmove(r->buf.ptr, res.ptr, res.len); + + r->rest = ft_bytes(r->buf.ptr, res.len); + tmp = r->buf; + ft_bytes_consume(&tmp, res.len); + sz = $i(pioRead, r->source, tmp, err); + r->rest.len += sz; + if ($haserr(*err)) + return ft_bytes(NULL, 0); + /* reached end of file */ + if (sz == 0) + { + res = r->rest; + r->rest = ft_bytes(NULL, 0); + return res; + } + goto retry; +} + +typedef struct pioRemotePagesIterator +{ + bool valid; + BlockNumber n_blocks; +} pioRemotePagesIterator; + +typedef struct pioLocalPagesIterator +{ + BlockNumber blknum; + BlockNumber lastblkn; + BlockNumber n_blocks; + + bool just_validate; + int segno; + datapagemap_t map; + pioReader_i in; + char* from_fullpath; + /* prev_backup_start_lsn */ + XLogRecPtr start_lsn; + + CompressAlg calg; + int clevel; + uint32 checksum_version; +} pioLocalPagesIterator; + +#define kls__pioLocalPagesIterator iface__pioPagesIterator, iface(pioPagesIterator), \ + mth(fobjDispose) +fobj_klass(pioLocalPagesIterator); + +#define kls__pioRemotePagesIterator iface__pioPagesIterator, iface(pioPagesIterator) +fobj_klass(pioRemotePagesIterator); + +static pioPagesIterator_i +pioRemoteDrive_pioIteratePages(VSelf, path_t from_fullpath, + int segno, datapagemap_t pagemap, + XLogRecPtr start_lsn, + CompressAlg calg, int clevel, + uint32 checksum_version, bool just_validate, err_i *err) +{ + Self(pioRemoteDrive); + fobj_t iter = {0}; + fio_header hdr = {.cop = FIO_ITERATE_PAGES}; + ft_strbuf_t buf = ft_strbuf_zero(); + fio_iterate_pages_request req = { + .segno = segno, + .pagemaplen = pagemap.bitmapsize, + .start_lsn = start_lsn, + .calg = calg, + .clevel = clevel, + .checksum_version = checksum_version, + .just_validate = just_validate, + }; + + ft_strbuf_catbytes(&buf, ft_bytes(&hdr, sizeof(hdr))); + ft_strbuf_catbytes(&buf, ft_bytes(&req, sizeof(req))); + ft_strbuf_catbytes(&buf, ft_bytes(pagemap.bitmap, pagemap.bitmapsize)); + ft_strbuf_catc_zt(&buf, from_fullpath); + + ((fio_header*)buf.ptr)->size = buf.len - sizeof(fio_header); + + IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len), buf.len); + + ft_strbuf_free(&buf); + + iter = $alloc(pioRemotePagesIterator, .valid = true); + + return bind_pioPagesIterator(iter); +} + +static err_i +pioRemotePagesIterator_pioNextPage(VSelf, PageIteratorValue *value) +{ + Self(pioRemotePagesIterator); + + fio_header hdr; + + value->compressed_size = 0; + + if (!self->valid) { + value->page_result = PageIsTruncated; + return $noerr(); + } + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.cop == FIO_PIO_ERROR) + { + self->valid = false; + return fio_receive_pio_err(&hdr); + } + else if (hdr.cop == FIO_ITERATE_EOF) + { + ft_assert(hdr.size == sizeof(BlockNumber)); + self->valid = false; + IO_CHECK(fio_read_all(fio_stdin, &self->n_blocks, sizeof(self->n_blocks)), sizeof(self->n_blocks)); + value->page_result = PageIsTruncated; + return $noerr(); + } + else if (hdr.cop == FIO_ITERATE_DATA) + { + Assert(hdr.size <= sizeof(PageIteratorValue)); + memset(value, 0, sizeof(PageIteratorValue)); + IO_CHECK(fio_read_all(fio_stdin, (void*)value, hdr.size), hdr.size); + + return $noerr(); + } + self->valid = false; + return $err(RT, "Unexpected operation {intCode} in remote pioNextPage", + intCode(hdr.cop)); +} + +static BlockNumber +pioRemotePagesIterator_pioFinalPageN(VSelf) +{ + Self(pioRemotePagesIterator); + return self->n_blocks; +} + +pioPagesIterator_i +doIteratePages_impl(pioIteratePages_i drive, struct doIteratePages_params p) +{ + datapagemap_t pagemap = {0}; + fobj_reset_err(p.err); + + /* + * If page map is empty or file is not present in destination directory, + * then copy backup all pages of the relation. + */ + if (p.file->pagemap.bitmapsize != PageBitmapIsEmpty && + !p.file->pagemap_isabsent && p.file->exists_in_prev && + p.file->pagemap.bitmap) + pagemap = p.file->pagemap; + + /* Skip page if page lsn is less than START_LSN of parent backup. */ + if (p.start_lsn != InvalidXLogRecPtr) + { + if (!p.file->exists_in_prev) + p.start_lsn = InvalidXLogRecPtr; + if (p.backup_mode != BACKUP_MODE_DIFF_DELTA && + p.backup_mode != BACKUP_MODE_DIFF_PTRACK) + p.start_lsn = InvalidXLogRecPtr; + } + + return $i(pioIteratePages, drive, + .path = p.from_fullpath, + .segno = p.file->segno, + .pagemap = pagemap, + .start_lsn = p.start_lsn, + .calg = p.calg, + .clevel = p.clevel, + .checksum_version = p.checksum_version, + .just_validate = p.just_validate, + .err = p.err); +} + +static pioPagesIterator_i +pioLocalDrive_pioIteratePages(VSelf, path_t path, + int segno, datapagemap_t pagemap, + XLogRecPtr start_lsn, + CompressAlg calg, int clevel, + uint32 checksum_version, bool just_validate, + err_i *err) +{ + Self(pioLocalDrive); + fobj_t iter = {0}; + BlockNumber n_blocks; + pioReader_i in; + pio_stat_t st; + + fobj_reset_err(err); + + in = $(pioOpenRead, self, path, .err = err); + if ($haserr(*err)) + return $null(pioPagesIterator); + + /* we know it is pioLocalReadFile which implements pioFileStat */ + st = $(pioFileStat, in.self, .err = err); + if ($haserr(*err)) + return $null(pioPagesIterator); + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + n_blocks = ft_div_i64u32_to_i32(st.pst_size, BLCKSZ); + + iter = $alloc(pioLocalPagesIterator, + .segno = segno, + .n_blocks = n_blocks, + .just_validate = just_validate, + .from_fullpath = ft_cstrdup(path), + .map = pagemap, + .in = $iref(in), + .start_lsn = start_lsn, + .calg = calg, + .clevel = clevel, + .checksum_version = checksum_version); + + return bind_pioPagesIterator(iter); +} + +static void +pioLocalPagesIterator_fobjDispose(VSelf) +{ + Self(pioLocalPagesIterator); + + $idel(&self->in); + ft_free(self->from_fullpath); +} + +static int32 prepare_page(pioLocalPagesIterator *iter, + BlockNumber blknum, + Page page, + PageState *page_st, + err_i *err); + +static err_i +pioLocalPagesIterator_pioNextPage(VSelf, PageIteratorValue *value) +{ + FOBJ_FUNC_ARP(); + Self(pioLocalPagesIterator); + char page_buf[BLCKSZ]; + BlockNumber blknum; + BlockNumber n_blocks; + int rc = PageIsOk; + err_i err; + + blknum = self->blknum; + value->compressed_size = 0; + if (self->blknum >= self->n_blocks) + goto truncated; + + /* next block */ + if (self->map.bitmapsize && + !datapagemap_first(self->map, &blknum)) + { + self->blknum = self->n_blocks; + goto truncated; + } + + value->blknum = blknum; + self->blknum = blknum+1; + + rc = prepare_page(self, blknum, page_buf, &value->state, &err); + if ($haserr(err)) + return $iresult(err); + + value->page_result = rc; + if (rc == PageIsTruncated) + goto re_stat; + self->lastblkn = blknum+1; + if (rc == PageIsOk && !self->just_validate) + { + value->compressed_size = compress_page(value->compressed_page, BLCKSZ, + value->blknum, page_buf, self->calg, + self->clevel, self->from_fullpath); + } + return $noerr(); + +re_stat: + { + /* + * prepare_page found file is shorter than expected. + * Lets re-investigate its length. + */ + pio_stat_t st; + /* abuse we know self->in is pioLocalReadFile */ + st = $(pioFileStat, self->in.self, .err = &err); + if ($haserr(err)) + return $iresult(err); + n_blocks = ft_div_i64u32_to_i32(st.pst_size, BLCKSZ); + /* we should not "forget" already produced pages */ + if (n_blocks < self->lastblkn) + n_blocks = self->lastblkn; + if (n_blocks < self->n_blocks) + self->n_blocks = blknum; + } +truncated: + value->page_result = PageIsTruncated; + return $noerr(); +} + +static BlockNumber +pioLocalPagesIterator_pioFinalPageN(VSelf) +{ + Self(pioLocalPagesIterator); + return self->n_blocks; +} + +/* + * Retrieves a page taking the backup mode into account + * and writes it into argument "page". Argument "page" + * should be a pointer to allocated BLCKSZ of bytes. + * + * Prints appropriate warnings/errors/etc into log. + * Returns: + * PageIsOk(0) if page was successfully retrieved + * PageIsTruncated(-1) if the page was truncated + * SkipCurrentPage(-2) if we need to skip this page, + * only used for DELTA and PTRACK backup + * PageIsCorrupted(-3) if the page checksum mismatch + * or header corruption, + * only used for checkdb + * TODO: probably we should always + * return it to the caller + */ +static int32 +prepare_page(pioLocalPagesIterator *iter, BlockNumber blknum, Page page, + PageState *page_st, err_i *err) +{ + int try_again = PAGE_READ_ATTEMPTS; + bool page_is_valid = false; + const char *from_fullpath = iter->from_fullpath; + BlockNumber absolute_blknum = iter->segno * RELSEG_SIZE + blknum; + int rc = 0; + size_t read_len; + fobj_reset_err(err); + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during page reading"); + + /* + * Read the page and verify its header and checksum. + * Under high write load it's possible that we've read partly + * flushed page, so try several times before throwing an error. + */ + while (!page_is_valid && try_again--) + { + *err = $i(pioSeek, iter->in, (uint64_t)blknum * BLCKSZ); + if ($haserr(*err)) + return PageIsCorrupted; + + read_len = $i(pioRead, iter->in, ft_bytes(page, BLCKSZ), .err = err); + if ($haserr(*err)) + return PageIsCorrupted; + /* The block could have been truncated. It is fine. */ + if (read_len == 0) + { + elog(VERBOSE, "Cannot read block %u of \"%s\": " + "block truncated", blknum, from_fullpath); + return PageIsTruncated; + } + else if (read_len != BLCKSZ) + elog(WARNING, "Cannot read block %u of \"%s\": " + "read %lld of %d, try again", + blknum, from_fullpath, (long long)read_len, BLCKSZ); + else + { + /* We have BLCKSZ of raw data, validate it */ + rc = validate_one_page(page, absolute_blknum, + InvalidXLogRecPtr, page_st, + iter->checksum_version); + switch (rc) { - /* - * We need a buf for a arguments and for a result at the same time - * hdr.size = strlen(symlink_name) + 1 - * hdr.arg = bufsize for a answer (symlink content) - */ - size_t filename_size = (size_t)hdr.size; - if (filename_size + hdr.arg > buf_size) { - buf_size = hdr.arg; - buf = (char*)realloc(buf, buf_size); - } - rc = readlink(buf, buf + filename_size, hdr.arg); - hdr.cop = FIO_READLINK; - hdr.size = rc > 0 ? rc : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size != 0) - IO_CHECK(fio_write_all(out, buf + filename_size, hdr.size), hdr.size); + case PAGE_IS_ZEROED: + elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); + return PageIsOk; + + case PAGE_IS_VALID: + /* in DELTA or PTRACK modes we must compare lsn */ + if (iter->start_lsn != InvalidXLogRecPtr) + page_is_valid = true; + else + return PageIsOk; + break; + + case PAGE_HEADER_IS_INVALID: + elog(VERBOSE, "File: \"%s\" blknum %u have wrong page header, try again", + from_fullpath, blknum); + break; + + case PAGE_CHECKSUM_MISMATCH: + elog(VERBOSE, "File: \"%s\" blknum %u have wrong checksum, try again", + from_fullpath, blknum); + break; + default: + Assert(false); } - break; - default: - Assert(false); } } - free(buf); - if (rc != 0) { /* Not end of stream: normal pipe close */ - perror("read"); - exit(EXIT_FAILURE); + + /* + * If page is not valid after PAGE_READ_ATTEMPTS attempts to read it + * throw an error. + */ + if (!page_is_valid) + { + int elevel = ERROR; + char *errormsg = NULL; + + /* Get the details of corruption */ + if (rc == PAGE_HEADER_IS_INVALID) + get_header_errormsg(page, &errormsg); + else if (rc == PAGE_CHECKSUM_MISMATCH) + get_checksum_errormsg(page, &errormsg, + absolute_blknum); + + /* Error out in case of merge or backup without ptrack support; + * issue warning in case of checkdb or backup with ptrack support + */ + if (iter->just_validate) + elevel = WARNING; + + if (errormsg) + elog(elevel, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, blknum, errormsg); + else + elog(elevel, "Corruption detected in file \"%s\", block %u", + from_fullpath, blknum); + + pg_free(errormsg); + return PageIsCorrupted; + } + + /* Checkdb not going futher */ + if (iter->just_validate) + return PageIsOk; + + /* + * Skip page if page lsn is less than START_LSN of parent backup. + * Nullified pages must be copied by DELTA backup, just to be safe. + */ + if (page_st->lsn > 0 && + page_st->lsn < iter->start_lsn) + { + elog(VERBOSE, "Skipping blknum %u in file: \"%s\", page_st->lsn: %X/%X, prev_backup_start_lsn: %X/%X", + blknum, from_fullpath, + (uint32) (page_st->lsn >> 32), (uint32) page_st->lsn, + (uint32) (iter->start_lsn >> 32), (uint32) iter->start_lsn); + return SkipCurrentPage; + } + + return PageIsOk; +} + +/* + * skip_drive + * + * On Windows, a path may begin with "C:" or "//network/". Advance over + * this and point to the effective start of the path. + * + * (copied from PostgreSQL's src/port/path.c) + */ +#ifdef WIN32 + +static char * +skip_drive(const char *path) +{ + if (IS_DIR_SEP(path[0]) && IS_DIR_SEP(path[1])) + { + path += 2; + while (*path && !IS_DIR_SEP(*path)) + path++; + } + else if (isalpha((unsigned char) path[0]) && path[1] == ':') + { + path += 2; + } + return (char *) path; +} +#else +#define skip_drive(path) (path) +#endif + +bool +ft_strbuf_cat_path(ft_strbuf_t *buf, ft_str_t path) +{ + if (path.len == 0) + return true; + + /* here we repeat join_path_components */ + if (buf->len > 0 && !IS_DIR_SEP(buf->ptr[buf->len-1])) + { + if (*(skip_drive(buf->ptr)) != '\0') + if (!ft_strbuf_cat1(buf, '/')) + return false; } + + return ft_strbuf_cat(buf, path); +} + +fobj_klass_handle(pioLocalPagesIterator); +fobj_klass_handle(pioRemotePagesIterator); + +fobj_klass_handle(pioFile); +fobj_klass_handle(pioLocalDrive); +fobj_klass_handle(pioRemoteDrive); +fobj_klass_handle(pioLocalReadFile, mth(fobjDispose, fobjRepr)); +fobj_klass_handle(pioRemoteFile, inherits(pioFile), mth(fobjDispose, fobjRepr)); +fobj_klass_handle(pioLocalWriteFile); +fobj_klass_handle(pioRemoteWriteFile); +fobj_klass_handle(pioLocalDir); +fobj_klass_handle(pioRemoteDir); +fobj_klass_handle(pioWriteFilter, mth(fobjDispose, fobjRepr)); +fobj_klass_handle(pioReadFilter, mth(fobjDispose, fobjRepr)); +fobj_klass_handle(pioDevNull); +fobj_klass_handle(pioCRC32Counter); +fobj_klass_handle(pioReSeekableReader); +fobj_klass_handle(pioCutZeroTail); + +#ifdef HAVE_LIBZ +fobj_klass_handle(pioGZCompress, mth(fobjRepr)); +fobj_klass_handle(pioGZDecompress, mth(fobjRepr)); +fobj_klass_handle(pioGZDecompressWrapperObj); +#endif + +void +init_pio_objects(void) +{ + FOBJ_FUNC_ARP(); + + localDrive = bindref_pioDBDrive($alloc(pioLocalDrive)); + remoteDrive = bindref_pioDBDrive($alloc(pioRemoteDrive)); } diff --git a/src/utils/file.h b/src/utils/file.h index 01e5a24f4..222ffedd2 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -3,13 +3,23 @@ #include "storage/bufpage.h" #include +#ifndef WIN32 #include +#endif #include #ifdef HAVE_LIBZ #include #endif +#include + +#include "datapagemap.h" + +/* Directory/File permission */ +#define DIR_PERMISSION (0700) +#define FILE_PERMISSION (0600) + typedef enum { /* message for compatibility check */ @@ -17,138 +27,425 @@ typedef enum FIO_OPEN, FIO_CLOSE, FIO_WRITE, - FIO_SYNC, FIO_RENAME, FIO_SYMLINK, - FIO_UNLINK, + FIO_REMOVE, FIO_MKDIR, - FIO_CHMOD, FIO_SEEK, - FIO_TRUNCATE, - FIO_DELETE, - FIO_PREAD, FIO_READ, - FIO_LOAD, FIO_STAT, FIO_SEND, - FIO_ACCESS, - FIO_OPENDIR, - FIO_READDIR, - FIO_CLOSEDIR, FIO_PAGE, - FIO_WRITE_COMPRESSED_ASYNC, - FIO_GET_CRC32, /* used for incremental restore */ FIO_GET_CHECKSUM_MAP, FIO_GET_LSN_MAP, - /* used in fio_send_pages */ - FIO_SEND_PAGES, FIO_ERROR, - FIO_SEND_FILE, -// FIO_CHUNK, FIO_SEND_FILE_EOF, - FIO_SEND_FILE_CORRUPTION, - FIO_SEND_FILE_HEADERS, /* messages for closing connection */ FIO_DISCONNECT, FIO_DISCONNECTED, - FIO_LIST_DIR, + FIO_REMOVE_DIR, FIO_CHECK_POSTMASTER, - FIO_GET_ASYNC_ERROR, - FIO_WRITE_ASYNC, FIO_READLINK, - FIO_PAGE_ZERO + FIO_SEND_FILE_CONTENT, + FIO_PAGE_ZERO, + FIO_FILES_ARE_SAME, + FIO_READ_FILE_AT_ONCE, + FIO_WRITE_FILE_AT_ONCE, + FIO_PIO_ERROR, /* for sending err_i */ + FIO_ITERATE_PAGES, + FIO_ITERATE_DATA, + FIO_ITERATE_EOF, + PIO_GET_CRC32, + PIO_OPEN_REWRITE, + PIO_OPEN_WRITE, + PIO_WRITE_ASYNC, + PIO_WRITE_COMPRESSED_ASYNC, + PIO_SEEK, + PIO_TRUNCATE, + PIO_GET_ASYNC_ERROR, + PIO_DIR_OPEN, + PIO_DIR_NEXT, + PIO_IS_DIR_EMPTY, + PIO_SYNC_TREE, + PIO_CLOSE, + PIO_DISPOSE, } fio_operations; +typedef struct +{ +// fio_operations cop; +// 16 + /* fio operation, see fio_operations enum */ + unsigned cop : 32; + /* */ + unsigned handle : 32; + /* size of additional data sent after this header */ + unsigned size : 32; + /* additional small parameter for requests (varies between operations) or a result code for response */ + unsigned arg; +} fio_header; + typedef enum { FIO_LOCAL_HOST, /* data is locate at local host */ FIO_DB_HOST, /* data is located at Postgres server host */ FIO_BACKUP_HOST, /* data is located at backup host */ - FIO_REMOTE_HOST /* date is located at remote host */ + FIO_REMOTE_HOST, /* date is located at remote host */ } fio_location; -#define FIO_FDMAX 64 -#define FIO_PIPE_MARKER 0x40000000 +typedef enum pio_file_kind { + PIO_KIND_UNKNOWN = 0, + PIO_KIND_REGULAR = 1, + PIO_KIND_DIRECTORY = 2, + PIO_KIND_SYMLINK = 3, + PIO_KIND_FIFO = 4, + PIO_KIND_SOCK = 5, + PIO_KIND_CHARDEV = 6, + PIO_KIND_BLOCKDEV = 7, +} pio_file_kind_e; -#define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0) +typedef struct pio_stat { + pio_file_kind_e pst_kind; + uint32_t pst_mode; + int64_t pst_size; + int64_t pst_mtime; +} pio_stat_t; -typedef struct +typedef struct pio_dirent { + pio_stat_t stat; + ft_str_t name; +} pio_dirent_t; + +#define FT_SLICE dirent +#define FT_SLICE_TYPE pio_dirent_t +#include + +#define FT_SORT dirent +#define FT_SORT_TYPE pio_dirent_t +#include + +ft_inline int +compare_dirent_by_name(pio_dirent_t d1, pio_dirent_t d2) { -// fio_operations cop; -// 16 - unsigned cop : 32; - unsigned handle : 32; - unsigned size : 32; - unsigned arg; -} fio_header; + return ft_strcmp(d1.name, d2.name); +} extern fio_location MyLocation; +extern void setMyLocation(ProbackupSubcmd const subcmd); +/* Check if specified location is local for current node */ +extern bool fio_is_remote(fio_location location); +extern bool fio_is_remote_simple(fio_location location); + +extern void fio_communicate(int in, int out); +extern void fio_disconnect(void); + +#define FIO_FDMAX 64 + /* Check if FILE handle is local or remote (created by FIO) */ #define fio_is_remote_file(file) ((size_t)(file) <= FIO_FDMAX) extern void fio_redirect(int in, int out, int err); -extern void fio_communicate(int in, int out); +extern void fio_error(int rc, int size, const char* file, int line); + +#define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0) extern void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size); -extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); -extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); -extern ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg); -extern size_t fio_fwrite_async(FILE* f, void const* buf, size_t size); -extern int fio_check_error_file(FILE* f, char **errmsg); -extern int fio_check_error_fd(int fd, char **errmsg); -extern int fio_check_error_fd_gz(gzFile f, char **errmsg); -extern ssize_t fio_fread(FILE* f, void* buf, size_t size); -extern int fio_pread(FILE* f, void* buf, off_t offs); -extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); -extern int fio_fflush(FILE* f); -extern int fio_fseek(FILE* f, off_t offs); -extern int fio_ftruncate(FILE* f, off_t size); -extern int fio_fclose(FILE* f); -extern int fio_ffstat(FILE* f, struct stat* st); -extern void fio_error(int rc, int size, char const* file, int line); - -extern int fio_open(char const* name, int mode, fio_location location); -extern ssize_t fio_write(int fd, void const* buf, size_t size); -extern ssize_t fio_write_async(int fd, void const* buf, size_t size); -extern ssize_t fio_read(int fd, void* buf, size_t size); -extern int fio_flush(int fd); -extern int fio_seek(int fd, off_t offs); -extern int fio_fstat(int fd, struct stat* st); -extern int fio_truncate(int fd, off_t size); -extern int fio_close(int fd); -extern void fio_disconnect(void); -extern int fio_sync(char const* path, fio_location location); -extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, - bool decompress, bool missing_ok); -extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location, - bool missing_ok); - -extern int fio_rename(char const* old_path, char const* new_path, fio_location location); -extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); -extern int fio_unlink(char const* path, fio_location location); -extern int fio_mkdir(char const* path, int mode, fio_location location); -extern int fio_chmod(char const* path, int mode, fio_location location); -extern int fio_access(char const* path, int mode, fio_location location); -extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location); -extern bool fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location); -extern ssize_t fio_readlink(const char *path, char *value, size_t valsiz, fio_location location); -extern DIR* fio_opendir(char const* path, fio_location location); -extern struct dirent * fio_readdir(DIR *dirp); -extern int fio_closedir(DIR *dirp); -extern FILE* fio_open_stream(char const* name, fio_location location); -extern int fio_close_stream(FILE* f); + +/* pathname-style functions */ +extern int fio_symlink(fio_location location, const char* target, const char* link_path, bool overwrite); +extern int fio_remove(fio_location location, const char* path, bool missing_ok); +extern ssize_t fio_readlink(fio_location location, const char *path, char *value, size_t valsiz); +extern pid_t fio_check_postmaster(fio_location location, const char *pgdata); + +extern PageState *fio_get_checksum_map(fio_location location, const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); +struct datapagemap; /* defined in datapagemap.h */ +extern struct datapagemap *fio_get_lsn_map(fio_location location, const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno); + +#if PG_VERSION_NUM < 120000 +extern pg_crc32 pgFileGetCRC32(const char *file_path, bool missing_ok); +#endif + +extern pio_file_kind_e pio_statmode2file_kind(mode_t mode, const char* path); +extern pio_file_kind_e pio_str2file_kind(const char* str, const char* path); +extern const char* pio_file_kind2str(pio_file_kind_e kind, const char* path); +extern mode_t pio_limit_mode(mode_t mode); + +// OBJECTS + +extern void init_pio_objects(void); + +typedef const char* path_t; + +fobj_error_cstr_key(remotemsg); +fobj_error_int_key(writtenSz); +fobj_error_int_key(wantedSz); +fobj_error_int_key(size); +fobj_error_cstr_key(kind); +fobj_error_int_key(offs); +fobj_error_int_key(blknum); #ifdef HAVE_LIBZ -extern gzFile fio_gzopen(char const* path, char const* mode, int level, fio_location location); -extern int fio_gzclose(gzFile file); -extern int fio_gzread(gzFile f, void *buf, unsigned size); -extern int fio_gzwrite(gzFile f, void const* buf, unsigned size); -extern int fio_gzeof(gzFile f); -extern z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence); -extern const char* fio_gzerror(gzFile file, int *errnum); +fobj_error_kind(GZ); +fobj_error_int_key(gzErrNo); +fobj_error_cstr_key(gzErrStr); #endif +// File +#define mth__pioClose err_i +#define mth__pioRead size_t, (ft_bytes_t, buf), (err_i *, err) +#define mth__pioWrite err_i, (ft_bytes_t, buf) +#define mth__pioWriteCompressed err_i, (ft_bytes_t, buf), (CompressAlg, compress_alg) +#define mth__pioTruncate err_i, (uint64_t, size) +#define mth__pioWriteFinish err_i +#define mth__pioSeek err_i, (uint64_t, offs) + +fobj_method(pioClose); +fobj_method(pioRead); +fobj_method(pioWrite); +fobj_method(pioWriteCompressed); +fobj_method(pioTruncate); +fobj_method(pioWriteFinish); +fobj_method(pioSeek); + +#define iface__pioReadSeek mth(pioRead, pioSeek) +#define iface__pioReader mth(pioRead, pioClose, pioSeek) +#define iface__pioReadStream mth(pioRead, pioClose) +#define iface__pioWriteFlush mth(pioWrite, pioWriteFinish) +#define iface__pioWriteCloser mth(pioWrite, pioWriteFinish, pioClose) +#define iface__pioDBWriter mth(pioWrite, pioSeek, pioWriteCompressed), \ + mth(pioWriteFinish, pioTruncate, pioClose) +#define iface__pioReadCloser mth(pioRead, pioClose) +fobj_iface(pioReadSeek); +fobj_iface(pioReader); +fobj_iface(pioReadStream); +fobj_iface(pioWriteFlush); +fobj_iface(pioWriteCloser); +fobj_iface(pioDBWriter); +fobj_iface(pioReadCloser); + +// DIR +#define mth__pioDirNext pio_dirent_t, (err_i*, err) +fobj_method(pioDirNext); +#define iface__pioDirIter mth(pioDirNext, pioClose) +fobj_iface(pioDirIter); + +// Pages iterator +typedef struct +{ + PageState state; + BlockNumber blknum; + int page_result; + size_t compressed_size; + char compressed_page[BLCKSZ]; /* MUST be last */ +} PageIteratorValue; + +#define mth__pioNextPage err_i, (PageIteratorValue *, value) +#define mth__pioFinalPageN BlockNumber + +fobj_method(pioNextPage); +fobj_method(pioFinalPageN); + +#define iface__pioPagesIterator mth(pioNextPage, pioFinalPageN) + +fobj_iface(pioPagesIterator); + +// Drive +#define mth__pioOpenRead pioReader_i, (path_t, path), (err_i *, err) +#define mth__pioOpenReadStream pioReadStream_i, (path_t, path), (err_i *, err) +#define mth__pioOpenWrite pioDBWriter_i, (path_t, path), (int, permissions, FILE_PERMISSION), \ + (bool, exclusive, false), (bool, sync, false), \ + (err_i *, err) +#define mth__pioOpenRewrite pioWriteCloser_i, (path_t, path), (int, permissions, FILE_PERMISSION), \ + (bool, binary, true), (bool, use_temp, true), \ + (bool, sync, false), (err_i *, err) +#define mth__pioStat pio_stat_t, (path_t, path), (bool, follow_symlink), \ + (err_i *, err) +#define mth__pioRemove err_i, (path_t, path), (bool, missing_ok) +#define mth__pioRename err_i, (path_t, old_path), (path_t, new_path) +#define mth__pioExists bool, (path_t, path), (pio_file_kind_e, expected_kind, PIO_KIND_REGULAR), \ + (err_i *, err) +#define mth__pioGetCRC32 pg_crc32, (path_t, path), (bool, compressed, false), \ + (bool, truncated, false), (err_i *, err) +/* Compare, that filename1 and filename2 is the same file */ +#define mth__pioFilesAreSame bool, (path_t, file1), (path_t, file2) +#define mth__pioIsRemote bool +#define mth__pioMakeDir err_i, (path_t, path), (mode_t, mode), (bool, strict) +#define mth__pioOpenDir pioDirIter_i, (path_t, path), (err_i*, err) +#define mth__pioIsDirEmpty bool, (path_t, path), (err_i*, err) +#define mth__pioRemoveDir void, (const char *, root), (bool, root_as_well) +/* pioReadFile and pioWriteFile should be used only for small files */ +#define PIO_READ_WRITE_FILE_LIMIT (16*1024*1024) +#define mth__pioReadFile ft_bytes_t, (path_t, path), (bool, binary, true), \ + (err_i *, err) +#define mth__pioWriteFile err_i, (path_t, path), (ft_bytes_t, content), \ + (bool, binary, true) + +/* Sync whole directories tree starting with root */ +#define mth__pioSyncTree err_i, (path_t, root) + +#define mth__pioIteratePages pioPagesIterator_i, (path_t, path), \ + (int, segno), (datapagemap_t, pagemap), (XLogRecPtr, start_lsn), \ + (CompressAlg, calg), (int, clevel), \ + (uint32, checksum_version), (bool, just_validate), (err_i*, err) + +fobj_method(pioOpenRead); +fobj_method(pioOpenReadStream); +fobj_method(pioOpenRewrite); +fobj_method(pioOpenWrite); +fobj_method(pioStat); +fobj_method(pioRemove); +fobj_method(pioRename); +fobj_method(pioExists); +fobj_method(pioIsRemote); +fobj_method(pioGetCRC32); +fobj_method(pioMakeDir); +fobj_method(pioFilesAreSame); +fobj_method(pioOpenDir); +fobj_method(pioIsDirEmpty); +fobj_method(pioRemoveDir); +fobj_method(pioReadFile); +fobj_method(pioWriteFile); +fobj_method(pioIteratePages); +fobj_method(pioSyncTree); + +#define iface__pioDrive mth(pioOpenRead, pioOpenReadStream), \ + mth(pioStat, pioRemove), \ + mth(pioExists, pioGetCRC32, pioIsRemote), \ + mth(pioMakeDir, pioOpenDir, pioIsDirEmpty, pioRemoveDir), \ + mth(pioFilesAreSame, pioReadFile, pioWriteFile), \ + mth(pioOpenRewrite) +fobj_iface(pioDrive); + +#define iface__pioDBDrive iface__pioDrive, mth(pioIteratePages, pioOpenWrite), \ + mth(pioRename, pioSyncTree) +fobj_iface(pioDBDrive); + +extern pioDrive_i pioDriveForLocation(fio_location location); +extern pioDBDrive_i pioDBDriveForLocation(fio_location location); + +extern pg_crc32 +pio_helper_pioGetCRC32(pioOpenReadStream_i self, path_t path, + bool compressed, bool truncated, err_i *err); + + +struct doIteratePages_params { + path_t from_fullpath; + pgFile *file; + XLogRecPtr start_lsn; + CompressAlg calg; + int clevel; + uint32 checksum_version; + BackupMode backup_mode; + bool just_validate; + err_i *err; +}; + +extern pioPagesIterator_i +doIteratePages_impl(pioIteratePages_i drive, struct doIteratePages_params p); +#define doIteratePages(drive, ...) \ + doIteratePages_impl($reduce(pioIteratePages, drive), ((struct doIteratePages_params){ \ + .start_lsn = InvalidXLogRecPtr, \ + .calg = NONE_COMPRESS, .clevel = 0, \ + .just_validate = false, \ + __VA_ARGS__})) + +#define mth__pioSetAsync err_i, (bool, async, true) +#define mth__pioAsyncRead size_t, (ft_bytes_t, buf), (err_i*, err) +fobj_method(pioSetAsync); +fobj_method(pioAsyncRead); + +// Filter +typedef struct pioFltTransformResult { + size_t consumed; + size_t produced; +} pioFltTransformResult; + +#define mth__pioFltTransform pioFltTransformResult, (ft_bytes_t, in), \ + (ft_bytes_t, out), \ + (err_i*, err) +fobj_method(pioFltTransform); +#define mth__pioFltFinish size_t, (ft_bytes_t, out), (err_i*, err) +fobj_method(pioFltFinish); + +#define mth__pioFltInPlace err_i, (ft_bytes_t, buf) +fobj_method(pioFltInPlace); + +#define iface__pioFilter mth(pioFltTransform, pioFltFinish) +fobj_iface(pioFilter); + +extern pioWriteFlush_i pioWrapWriteFilter(pioWriteFlush_i fl, + pioFilter_i flt, + size_t buf_size); +extern pioRead_i pioWrapReadFilter(pioRead_i fl, + pioFilter_i flt, + size_t buf_size); + +#define mth__pioWrapRead pioRead_i, (pioRead_i, reader), (err_i*, err) +fobj_method(pioWrapRead); + +/* + * Usefull for seek-able GZip reader - ie imitate gzseek. + * Has same limitations: will read from start if seek-ed back. + */ +extern pioReader_i pioWrapForReSeek(pioReader_i fl, + pioWrapRead_i wr); + +#ifdef HAVE_LIBZ +extern pioFilter_i pioGZCompressFilter(int level); +extern pioFilter_i pioGZDecompressFilter(bool ignoreTruncate); +extern pioWrapRead_i pioGZDecompressWrapper(bool ignoreTruncate); +#endif +extern pioFilter_i pioCRC32Filter(void); + +typedef struct pioCRC32Counter pioCRC32Counter; +#define kls__pioCRC32Counter iface__pioFilter, mth(pioFltInPlace), iface(pioFilter) +fobj_klass(pioCRC32Counter); +extern pioCRC32Counter* pioCRC32Counter_alloc(void); +extern pg_crc32 pioCRC32Counter_getCRC32(pioCRC32Counter* flt); +extern int64_t pioCRC32Counter_getSize(pioCRC32Counter* flt); + +typedef struct pioCutZeroTail pioCutZeroTail; +extern pioCutZeroTail* pioCutZeroTail_alloc(void); +extern int64_t pioCutZeroTail_getReadSize(pioCutZeroTail* flt); +extern int64_t pioCutZeroTail_getWriteSize(pioCutZeroTail* flt); + +extern pioWriteFlush_i pioDevNull_alloc(void); + +extern err_i pioCopyWithFilters(pioWriteFlush_i dest, pioRead_i src, + pioFilter_i *filters, int nfilters, size_t *copied); +#define pioCopy(dest, src, ...) ({ \ + pioFilter_i _fltrs_[] = {__VA_ARGS__}; \ + pioCopyWithFilters((dest), (src), _fltrs_, ft_arrsz(_fltrs_), NULL); \ +}) + +typedef struct pio_line_reader pio_line_reader; +struct pio_line_reader { + pioRead_i source; + ft_bytes_t buf; + ft_bytes_t rest; +}; + +extern void init_pio_line_reader(pio_line_reader *r, pioRead_i source, size_t max_length); +extern void deinit_pio_line_reader(pio_line_reader *r); +extern ft_bytes_t pio_line_reader_getline(pio_line_reader *r, err_i *err); + +typedef struct pio_recursive_dir pioRecursiveDir; +extern pioRecursiveDir* pioRecursiveDir_alloc(pioDrive_i drive, path_t root, err_i *err); +extern pio_dirent_t pioRecursiveDir_next(pioRecursiveDir* dir, err_i* err); +extern void pioRecursiveDir_dont_recurse_current(pioRecursiveDir* dir); +extern void pioRecursiveDir_close(pioRecursiveDir* dir); + +/* append path component */ +extern bool ft_strbuf_cat_path(ft_strbuf_t *buf, ft_str_t path); +ft_inline bool +ft_strbuf_cat_pathc(ft_strbuf_t *buf, const char *path) +{ + return ft_strbuf_cat_path(buf, ft_cstr(path)); +} + #endif diff --git a/src/utils/logger.c b/src/utils/logger.c index 7ea41f74e..57b96e020 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -53,7 +53,6 @@ void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); static void elog_internal(int elevel, bool file_only, const char *message); static void elog_stderr(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); -static char *get_log_message(const char *fmt, va_list args) pg_attribute_printf(1, 0); /* Functions to work with log files */ static void open_logfile(FILE **file, const char *filename_format); @@ -69,6 +68,7 @@ static FILE *error_log_file = NULL; static bool exit_hook_registered = false; /* Logging of the current thread is in progress */ static bool loggin_in_progress = false; +static __thread bool thread_terminates = false; static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -269,7 +269,7 @@ write_elevel_for_json(PQExpBuffer buf, int elevel) static void exit_if_necessary(int elevel) { - if (elevel > WARNING && !in_cleanup) + if (elevel > WARNING && !in_cleanup && !thread_terminates) { if (loggin_in_progress) { @@ -283,6 +283,8 @@ exit_if_necessary(int elevel) /* If this is not the main thread then don't call exit() */ if (main_tid != pthread_self()) { + /* Notice this thread is quiting */ + thread_terminates = true; /* Interrupt other possible routines */ thread_interrupted = true; #ifdef WIN32 @@ -341,8 +343,8 @@ elog_internal(int elevel, bool file_only, const char *message) if (format_file == JSON || format_console == JSON) { - snprintf(str_pid_json, sizeof(str_pid_json), "%d", my_pid); - snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); + snprintf(str_pid_json, sizeof(str_pid_json), "%lld", (long long)my_pid); + snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num()); initPQExpBuffer(&show_buf); json_add_min(buf_json, JT_BEGIN_OBJECT); @@ -355,7 +357,7 @@ elog_internal(int elevel, bool file_only, const char *message) json_add_min(buf_json, JT_END_OBJECT); } - snprintf(str_pid, sizeof(str_pid), "[%d]:", my_pid); + snprintf(str_pid, sizeof(str_pid), "[%lld]:", (long long)my_pid); /* * Write message to log file. @@ -422,7 +424,7 @@ elog_internal(int elevel, bool file_only, const char *message) { char str_thread[64]; /* [Issue #213] fix pgbadger parsing */ - snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num); + snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num()); fprintf(stderr, "%s ", strfbuf); fprintf(stderr, "%s ", str_pid); @@ -495,8 +497,8 @@ elog_stderr(int elevel, const char *fmt, ...) { strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", localtime(&log_time)); - snprintf(str_pid, sizeof(str_pid), "%d", my_pid); - snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); + snprintf(str_pid, sizeof(str_pid), "%lld", (long long)my_pid); + snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num()); initPQExpBuffer(&show_buf); json_add_min(buf_json, JT_BEGIN_OBJECT); @@ -504,12 +506,12 @@ elog_stderr(int elevel, const char *fmt, ...) json_add_value(buf_json, "pid", str_pid, 0, true); json_add_key(buf_json, "level", 0); write_elevel_for_json(buf_json, elevel); - message = get_log_message(fmt, args); + message = ft_vasprintf(fmt, args).ptr; json_add_value(buf_json, "msg", message, 0, true); json_add_value(buf_json, "my_thread_num", str_thread_json, 0, true); json_add_min(buf_json, JT_END_OBJECT); fputs(buf_json->data, stderr); - pfree(message); + ft_free(message); termPQExpBuffer(buf_json); } else @@ -525,37 +527,6 @@ elog_stderr(int elevel, const char *fmt, ...) exit_if_necessary(elevel); } -/* - * Formats text data under the control of fmt and returns it in an allocated - * buffer. - */ -static char * -get_log_message(const char *fmt, va_list args) -{ - size_t len = 256; /* initial assumption about buffer size */ - - for (;;) - { - char *result; - size_t newlen; - va_list copy_args; - - result = (char *) pgut_malloc(len); - - /* Try to format the data */ - va_copy(copy_args, args); - newlen = pvsnprintf(result, len, fmt, copy_args); - va_end(copy_args); - - if (newlen < len) - return result; /* success */ - - /* Release buffer and loop around to try again with larger len. */ - pfree(result); - len = newlen; - } -} - /* * Logs to stderr or to log file and exit if ERROR. */ @@ -574,11 +545,11 @@ elog(int elevel, const char *fmt, ...) return; va_start(args, fmt); - message = get_log_message(fmt, args); + message = ft_vasprintf(fmt, args).ptr; va_end(args); elog_internal(elevel, false, message); - pfree(message); + ft_free(message); } /* @@ -598,11 +569,65 @@ elog_file(int elevel, const char *fmt, ...) return; va_start(args, fmt); - message = get_log_message(fmt, args); + message = ft_vasprintf(fmt, args).ptr; va_end(args); elog_internal(elevel, true, message); - pfree(message); + free(message); +} + +/* + * Wrapper for ft_log + */ +void +elog_ft_log(enum FT_LOG_LEVEL ft_level, ft_source_position_t srcpos, + const char* error, const char *fmt, va_list args) +{ +#define ERR_MAX 1024 +#define MSG_MAX 4096 + char message[MSG_MAX]; + size_t sz; + int elevel; + + switch (ft_level) + { + case FT_TRACE: + case FT_DEBUG: + elevel = VERBOSE; + break; + case FT_LOG: + elevel = LOG; + break; + case FT_INFO: + elevel = INFO; + break; + case FT_WARNING: + elevel = WARNING; + break; + case FT_FATAL: + case FT_ERROR: + elevel = ERROR; + break; + default: + elevel = WARNING; + } + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < logger_config.log_level_console && + elevel < logger_config.log_level_file && elevel < ERROR) + return; + + /* don't use ft_vasprintf since it could recurse to logging */ + sz = vsnprintf(message, MSG_MAX, fmt, args); + if (error != NULL && sz < MSG_MAX) { + ft_strlcat(message + sz, ": ", MSG_MAX-sz); + ft_strlcat(message + sz, error, MSG_MAX-sz); + } + + elog_internal(elevel, false, message); } /* @@ -644,11 +669,11 @@ pg_log(eLogType type, const char *fmt, ...) return; va_start(args, fmt); - message = get_log_message(fmt, args); + message = ft_vasprintf(fmt, args).ptr; va_end(args); elog_internal(elevel, false, message); - pfree(message); + ft_free(message); } /* @@ -786,11 +811,7 @@ logfile_getname(const char *format, time_t timestamp) len = strlen(filename); /* Treat log_filename as a strftime pattern */ -#ifdef WIN32 - if (pg_strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) -#else if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) -#endif elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); return filename; @@ -946,7 +967,7 @@ open_logfile(FILE **file, const char *filename_format) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); - fprintf(control_file, "%ld", timestamp); + fprintf(control_file, "%lld", (long long)timestamp); fclose(control_file); } diff --git a/src/utils/logger.h b/src/utils/logger.h index adc5061e0..2746d1950 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -57,6 +57,9 @@ extern LoggerConfig logger_config; #undef elog extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); +extern void elog_ft_log(enum FT_LOG_LEVEL, ft_source_position_t srcpos, + const char* error, const char *fmt, va_list args) + pg_attribute_printf(4, 0); extern void init_logger(const char *root_path, LoggerConfig *config); extern void init_console(void); diff --git a/src/utils/parray.c b/src/utils/parray.c index 792e26907..0603c10c4 100644 --- a/src/utils/parray.c +++ b/src/utils/parray.c @@ -217,3 +217,31 @@ bool parray_contains(parray *array, void *elem) } return false; } + +/* effectively remove elements that satisfy certain criterion */ +void +parray_remove_if(parray *array, criterion_fn criterion, void *args, cleanup_fn clean) { + int i = 0; + int j = 0; + + /* removing certain elements */ + while(j < parray_num(array)) { + void *value = array->data[j]; + // if the value satisfies the criterion, clean it up + if(criterion(value, args)) { + clean(value); + j++; + continue; + } + + if(i != j) + array->data[i] = array->data[j]; + + i++; + j++; + } + + /* adjust the number of used elements */ + array->used -= j - i; +} + diff --git a/src/utils/parray.h b/src/utils/parray.h index e92ad728c..08846f252 100644 --- a/src/utils/parray.h +++ b/src/utils/parray.h @@ -16,6 +16,9 @@ */ typedef struct parray parray; +typedef bool (*criterion_fn)(void *value, void *args); +typedef void (*cleanup_fn)(void *ref); + extern parray *parray_new(void); extern void parray_expand(parray *array, size_t newnum); extern void parray_free(parray *array); @@ -32,6 +35,7 @@ extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const extern int parray_bsearch_index(parray *array, const void *key, int(*compare)(const void *, const void *)); extern void parray_walk(parray *array, void (*action)(void *)); extern bool parray_contains(parray *array, void *elem); +extern void parray_remove_if(parray *array, criterion_fn criterion, void *args, cleanup_fn clean); #endif /* PARRAY_H */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 6123c18d8..81f1805ac 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -3,7 +3,7 @@ * pgut.c * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional + * Portions Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -20,17 +20,14 @@ #include "common/string.h" #endif -#if PG_VERSION_NUM >= 100000 #include "common/connect.h" -#else -#include "fe_utils/connect.h" -#endif #include #include "pgut.h" #include "logger.h" #include "file.h" +#include "simple_prompt.h" static char *password = NULL; @@ -85,36 +82,15 @@ prompt_for_password(const char *username) password = NULL; } -#if PG_VERSION_NUM >= 140000 - if (username == NULL) - password = simple_prompt("Password: ", false); - else - { - char message[256]; - snprintf(message, lengthof(message), "Password for user %s: ", username); - password = simple_prompt(message , false); - } -#elif PG_VERSION_NUM >= 100000 password = (char *) pgut_malloc(sizeof(char) * 100 + 1); if (username == NULL) - simple_prompt("Password: ", password, 100, false); + simple_prompt_compat("Password: ", password, 100, false); else { char message[256]; snprintf(message, lengthof(message), "Password for user %s: ", username); - simple_prompt(message, password, 100, false); + simple_prompt_compat(message, password, 100, false); } -#else - if (username == NULL) - password = simple_prompt("Password: ", 100, false); - else - { - char message[256]; - snprintf(message, lengthof(message), "Password for user %s: ", username); - password = simple_prompt(message, 100, false); - } -#endif - in_password = false; } @@ -980,7 +956,7 @@ pgut_strndup(const char *str, size_t n) /* * Allocates new string, that contains part of filepath string minus trailing filename string * If trailing filename string not found, returns copy of filepath. - * Result must be free by caller. + * Result must be freed by caller. */ char * pgut_str_strip_trailing_filename(const char *filepath, const char *filename) @@ -999,23 +975,6 @@ pgut_free(void *p) free(p); } -FILE * -pgut_fopen(const char *path, const char *mode, bool missing_ok) -{ - FILE *fp; - - if ((fp = fio_open_stream(path, FIO_BACKUP_HOST)) == NULL) - { - if (missing_ok && errno == ENOENT) - return NULL; - - elog(ERROR, "could not open file \"%s\": %s", - path, strerror(errno)); - } - - return fp; -} - #ifdef WIN32 static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout); #define select select_win32 @@ -1098,20 +1057,6 @@ init_cancel_handler(void) SetConsoleCtrlHandler(consoleHandler, TRUE); } -int -sleep(unsigned int seconds) -{ - Sleep(seconds * 1000); - return 0; -} - -int -usleep(unsigned int usec) -{ - Sleep((usec + 999) / 1000); /* rounded up */ - return 0; -} - #undef select static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout) diff --git a/src/utils/pgut.h b/src/utils/pgut.h index f8554f9d0..63dbef4b0 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional + * Portions Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -11,6 +11,7 @@ #ifndef PGUT_H #define PGUT_H +#include #include "postgres_fe.h" #include "libpq-fe.h" @@ -70,11 +71,6 @@ extern void pgut_free(void *p); #define pgut_new0(type) ((type *) pgut_malloc0(sizeof(type))) #define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) -/* - * file operations - */ -extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); - /* * Assert */ @@ -92,6 +88,8 @@ extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); #define AssertMacro(x) ((void) 0) #endif +#define SPACES " \t\n\v\f\r" + #define IsSpace(c) (isspace((unsigned char)(c))) #define IsAlpha(c) (isalpha((unsigned char)(c))) #define IsAlnum(c) (isalnum((unsigned char)(c))) @@ -104,15 +102,4 @@ extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); extern int wait_for_socket(int sock, struct timeval *timeout); extern int wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout); -#ifdef WIN32 -extern int sleep(unsigned int seconds); -extern int usleep(unsigned int usec); -#endif - -#ifdef _MSC_VER -#define ARG_SIZE_HINT -#else -#define ARG_SIZE_HINT static -#endif - #endif /* PGUT_H */ diff --git a/src/utils/remote.c b/src/utils/remote.c index 7ef8d3239..e7ccab9db 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -5,12 +5,6 @@ #include #include -#ifdef WIN32 -#define __thread __declspec(thread) -#else -#include -#endif - #include "pg_probackup.h" #include "file.h" @@ -103,7 +97,7 @@ void launch_ssh(char* argv[]) } #endif -static bool needs_quotes(char const* path) +static bool needs_quotes(const char* path) { return strchr(path, ' ') != NULL; } @@ -113,14 +107,14 @@ bool launch_agent(void) char cmd[MAX_CMDLINE_LENGTH]; char* ssh_argv[MAX_CMDLINE_OPTIONS]; int ssh_argc; - int outfd[2]; - int infd[2]; - int errfd[2]; + int outfd[2] = {0, 0}; + int infd[2] = {0, 0}; + int errfd[2] = {0, 0}; int agent_version; ssh_argc = 0; #ifdef WIN32 - ssh_argv[ssh_argc++] = PROGRAM_NAME_FULL; + ssh_argv[ssh_argc++] = (char *) PROGRAM_NAME_FULL; ssh_argv[ssh_argc++] = "ssh"; ssh_argc += 2; /* reserve space for pipe descriptors */ #endif @@ -159,7 +153,7 @@ bool launch_agent(void) if (instance_config.remote.path) { - char const* probackup = PROGRAM_NAME_FULL; + const char* probackup = PROGRAM_NAME_FULL; char* sep = strrchr(probackup, '/'); if (sep != NULL) { probackup = sep + 1; @@ -198,7 +192,9 @@ bool launch_agent(void) ssh_argv[2] = psprintf("%d", outfd[0]); ssh_argv[3] = psprintf("%d", infd[1]); { - intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv); + intptr_t pid = _spawnvp(_P_NOWAIT, + (const char*)ssh_argv[0], + (const char * const *) ssh_argv); if (pid < 0) return false; child_pid = GetProcessId((HANDLE)pid); diff --git a/src/utils/simple_prompt.c b/src/utils/simple_prompt.c new file mode 100644 index 000000000..79f49e1b0 --- /dev/null +++ b/src/utils/simple_prompt.c @@ -0,0 +1,159 @@ +#include "c.h" + +#ifdef HAVE_TERMIOS_H +#include +#endif +#include "simple_prompt.h" + +/* + * simple_prompt + * + * Generalized function especially intended for reading in usernames and + * passwords interactively. Reads from /dev/tty or stdin/stderr. + * + * prompt: The prompt to print, or NULL if none (automatically localized) + * destination: buffer in which to store result + * destlen: allocated length of destination + * echo: Set to false if you want to hide what is entered (for passwords) + * + * The input (without trailing newline) is returned in the destination buffer, + * with a '\0' appended. + */ +void +simple_prompt_compat(const char *prompt, char *destination, size_t destlen, bool echo) +{ + int length; + FILE *termin, + *termout; + +#if defined(HAVE_TERMIOS_H) + struct termios t_orig, + t; +#elif defined(WIN32) + HANDLE t = NULL; + DWORD t_orig = 0; +#endif + +#ifdef WIN32 + + /* + * A Windows console has an "input code page" and an "output code page"; + * these usually match each other, but they rarely match the "Windows ANSI + * code page" defined at system boot and expected of "char *" arguments to + * Windows API functions. The Microsoft CRT write() implementation + * automatically converts text between these code pages when writing to a + * console. To identify such file descriptors, it calls GetConsoleMode() + * on the underlying HANDLE, which in turn requires GENERIC_READ access on + * the HANDLE. Opening termout in mode "w+" allows that detection to + * succeed. Otherwise, write() would not recognize the descriptor as a + * console, and non-ASCII characters would display incorrectly. + * + * XXX fgets() still receives text in the console's input code page. This + * makes non-ASCII credentials unportable. + * + * Unintuitively, we also open termin in mode "w+", even though we only + * read it; that's needed for SetConsoleMode() to succeed. + */ + termin = fopen("CONIN$", "w+"); + termout = fopen("CONOUT$", "w+"); +#else + + /* + * Do not try to collapse these into one "w+" mode file. Doesn't work on + * some platforms (eg, HPUX 10.20). + */ + termin = fopen("/dev/tty", "r"); + termout = fopen("/dev/tty", "w"); +#endif + if (!termin || !termout +#ifdef WIN32 + + /* + * Direct console I/O does not work from the MSYS 1.0.10 console. Writes + * reach nowhere user-visible; reads block indefinitely. XXX This affects + * most Windows terminal environments, including rxvt, mintty, Cygwin + * xterm, Cygwin sshd, and PowerShell ISE. Switch to a more-generic test. + */ + || (getenv("OSTYPE") && strcmp(getenv("OSTYPE"), "msys") == 0) +#endif + ) + { + if (termin) + fclose(termin); + if (termout) + fclose(termout); + termin = stdin; + termout = stderr; + } + + if (!echo) + { +#if defined(HAVE_TERMIOS_H) + /* disable echo via tcgetattr/tcsetattr */ + tcgetattr(fileno(termin), &t); + t_orig = t; + t.c_lflag &= ~ECHO; + tcsetattr(fileno(termin), TCSAFLUSH, &t); +#elif defined(WIN32) + /* need the file's HANDLE to turn echo off */ + t = (HANDLE) _get_osfhandle(_fileno(termin)); + + /* save the old configuration first */ + GetConsoleMode(t, &t_orig); + + /* set to the new mode */ + SetConsoleMode(t, ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT); +#endif + } + + if (prompt) + { + fputs(_(prompt), termout); + fflush(termout); + } + + if (fgets(destination, destlen, termin) == NULL) + destination[0] = '\0'; + + length = strlen(destination); + if (length > 0 && destination[length - 1] != '\n') + { + /* eat rest of the line */ + char buf[128]; + int buflen; + + do + { + if (fgets(buf, sizeof(buf), termin) == NULL) + break; + buflen = strlen(buf); + } while (buflen > 0 && buf[buflen - 1] != '\n'); + } + + /* strip trailing newline, including \r in case we're on Windows */ + while (length > 0 && + (destination[length - 1] == '\n' || + destination[length - 1] == '\r')) + destination[--length] = '\0'; + + if (!echo) + { + /* restore previous echo behavior, then echo \n */ +#if defined(HAVE_TERMIOS_H) + tcsetattr(fileno(termin), TCSAFLUSH, &t_orig); + fputs("\n", termout); + fflush(termout); +#elif defined(WIN32) + SetConsoleMode(t, t_orig); + fputs("\n", termout); + fflush(termout); +#endif + } + + if (termin != stdin) + { + fclose(termin); + fclose(termout); + } +} + diff --git a/src/utils/simple_prompt.h b/src/utils/simple_prompt.h new file mode 100644 index 000000000..0b6fc2608 --- /dev/null +++ b/src/utils/simple_prompt.h @@ -0,0 +1,7 @@ +#ifndef SIMPLE_PROMPT_H +#define SIMPLE_PROMPT_H + +extern void +simple_prompt_compat(const char *prompt, char *destination, size_t destlen, bool echo); + +#endif diff --git a/src/utils/thread.c b/src/utils/thread.c index 1c469bd29..1ad1e772e 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -17,97 +17,17 @@ */ bool thread_interrupted = false; -#ifdef WIN32 -DWORD main_tid = 0; -#else pthread_t main_tid = 0; -#endif -#ifdef WIN32 -#include - -typedef struct win32_pthread -{ - HANDLE handle; - void *(*routine) (void *); - void *arg; - void *result; -} win32_pthread; - -static long mutex_initlock = 0; - -static unsigned __stdcall -win32_pthread_run(void *arg) -{ - win32_pthread *th = (win32_pthread *)arg; - - th->result = th->routine(th->arg); - - return 0; -} - -int -pthread_create(pthread_t *thread, - pthread_attr_t *attr, - void *(*start_routine) (void *), - void *arg) -{ - int save_errno; - win32_pthread *th; - - th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); - th->routine = start_routine; - th->arg = arg; - th->result = NULL; - - th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); - if (th->handle == NULL) - { - save_errno = errno; - free(th); - return save_errno; - } - - *thread = th; - return 0; -} +static __thread int my_thread_num_var = 1; int -pthread_join(pthread_t th, void **thread_return) +my_thread_num(void) { - if (th == NULL || th->handle == NULL) - return errno = EINVAL; - - if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) - { - _dosmaperr(GetLastError()); - return errno; - } - - if (thread_return) - *thread_return = th->result; - - CloseHandle(th->handle); - free(th); - return 0; + return my_thread_num_var; } -#endif /* WIN32 */ - -int -pthread_lock(pthread_mutex_t *mp) +void +set_my_thread_num(int th) { -#ifdef WIN32 - if (*mp == NULL) - { - while (InterlockedExchange(&mutex_initlock, 1) == 1) - /* loop, another thread own the lock */ ; - if (*mp == NULL) - { - if (pthread_mutex_init(mp, NULL)) - return -1; - } - InterlockedExchange(&mutex_initlock, 0); - } -#endif - return pthread_mutex_lock(mp); + my_thread_num_var = th; } diff --git a/src/utils/thread.h b/src/utils/thread.h index 2eaa5fb45..a6c58f70e 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -10,32 +10,18 @@ #ifndef PROBACKUP_THREAD_H #define PROBACKUP_THREAD_H -#ifdef WIN32 -#include "postgres_fe.h" -#include "port/pthread-win32.h" - -/* Use native win32 threads on Windows */ -typedef struct win32_pthread *pthread_t; -typedef int pthread_attr_t; - -#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } -#define PTHREAD_ONCE_INIT false +#if defined(WIN32) && !(defined(__MINGW64__) || defined(__MINGW32__) || defined(HAVE_PTHREAD)) +#error "Windows build supports only 'pthread' threading" +#endif -extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); -extern int pthread_join(pthread_t th, void **thread_return); -#else /* Use platform-dependent pthread capability */ #include -#endif - -#ifdef WIN32 -extern DWORD main_tid; -#else extern pthread_t main_tid; -#endif +#define pthread_lock(mp) pthread_mutex_lock(mp) extern bool thread_interrupted; -extern int pthread_lock(pthread_mutex_t *mp); +int my_thread_num(void); +void set_my_thread_num(int); #endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c index 9372b082c..f6d5a2d70 100644 --- a/src/validate.c +++ b/src/validate.c @@ -3,14 +3,13 @@ * validate.c: validate backup files. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" -#include #include #include "utils/thread.h" @@ -23,6 +22,7 @@ static bool skipped_due_to_lock = false; typedef struct { + pioDrive_i backup_drive; const char *base_path; parray *files; bool corrupted; @@ -56,6 +56,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) pthread_t *threads; validate_files_arg *threads_args; int i; + err_i err; // parray *dbOid_exclude_list = NULL; /* Check backup program version */ @@ -67,9 +68,9 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) /* Check backup server version */ if (strcmp(backup->server_version, PG_MAJORVERSION) != 0) - elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary " + elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary " "compiled with server version %s", - backup_id_of(backup), backup->server_version, PG_MAJORVERSION); + backup_id_of(backup), backup->server_version, PG_MAJORVERSION); if (backup->status == BACKUP_STATUS_RUNNING) { @@ -125,6 +126,8 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) return; } + parray_qsort(files, pgFileCompareByHdrOff); + // if (params && params->partial_db_list) // dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, // params->partial_restore_type); @@ -143,6 +146,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { validate_files_arg *arg = &(threads_args[i]); + arg->backup_drive = backup->backup_location; arg->base_path = backup->database_dir; arg->files = files; arg->corrupted = false; @@ -200,10 +204,16 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) (parse_program_version(backup->program_version) == 20201))) { char path[MAXPGPATH]; + pio_stat_t st; join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); - if (pgFileSize(path) >= (BLCKSZ*500)) + st = $i(pioStat, pioDriveForLocation(FIO_BACKUP_HOST), + .path = path, .follow_symlink = true, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), ""); + + if (st.pst_size >= (BLCKSZ*500)) { elog(WARNING, "Backup %s is a victim of metadata corruption. " "Additional information can be found here: " @@ -224,14 +234,20 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) static void * pgBackupValidateFiles(void *arg) { + FOBJ_FUNC_ARP(); int i; validate_files_arg *arguments = (validate_files_arg *)arg; int num_files = parray_num(arguments->files); pg_crc32 crc; + pioDrive_i backup_drive = arguments->backup_drive; + err_i err; + + header_map_cache_init(); for (i = 0; i < num_files; i++) { - struct stat st; + FOBJ_LOOP_ARP(); + pio_stat_t st; pgFile *file = (pgFile *) parray_get(arguments->files, i); char file_fullpath[MAXPGPATH]; @@ -239,7 +255,7 @@ pgBackupValidateFiles(void *arg) elog(ERROR, "Interrupted during validate"); /* Validate only regular files */ - if (!S_ISREG(file->mode)) + if (file->kind != PIO_KIND_REGULAR) continue; /* @@ -296,9 +312,11 @@ pgBackupValidateFiles(void *arg) join_path_components(file_fullpath, arguments->base_path, file->rel_path); /* TODO: it is redundant to check file existence using stat */ - if (stat(file_fullpath, &st) == -1) + st = $i(pioStat, backup_drive, .path = file_fullpath, .follow_symlink = false, + .err = &err); + if ($haserr(err)) { - if (errno == ENOENT) + if (getErrno(err) == ENOENT) elog(WARNING, "Backup file \"%s\" is not found", file_fullpath); else elog(WARNING, "Cannot stat backup file \"%s\": %s", @@ -307,10 +325,10 @@ pgBackupValidateFiles(void *arg) break; } - if (file->write_size != st.st_size) + if (file->write_size != st.pst_size) { - elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", - file_fullpath, (unsigned long) st.st_size, file->write_size); + elog(WARNING, "Invalid size of backup file \"%s\" : %lld. Expected %lld", + file_fullpath, (long long) st.pst_size, (long long)file->write_size); arguments->corrupted = true; break; } @@ -337,14 +355,32 @@ pgBackupValidateFiles(void *arg) * Starting from 2.0.25 we calculate crc of pg_control differently. */ if (arguments->backup_version >= 20025 && - strcmp(file->name, "pg_control") == 0 && - !file->external_dir_num) - crc = get_pgcontrol_checksum(arguments->base_path); + strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0 && + file->external_dir_num == 0) + crc = get_pgcontrol_checksum(backup_drive, arguments->base_path); else - crc = pgFileGetCRC(file_fullpath, - arguments->backup_version <= 20021 || - arguments->backup_version >= 20025, - false); +#if PG_VERSION_NUM >= 120000 + { + ft_assert(arguments->backup_version >= 20025); + crc = $i(pioGetCRC32, backup_drive, .path = file_fullpath, + .err = &err); + } +#else /* PG_VERSION_NUM < 120000 */ + if (arguments->backup_version <= 20021 || arguments->backup_version >= 20025) + crc = $i(pioGetCRC32, backup_drive, .path = file_fullpath, .err = &err); + else + { + ft_assert(!$i(pioIsRemote, backup_drive)); + crc = pgFileGetCRC32(file_fullpath, false); + } +#endif /* PG_VERSION_NUM < 120000 */ + if ($haserr(err)) + { + ft_logerr(FT_WARNING, $errmsg(err), "Backup file CRC"); + arguments->corrupted = true; + break; + } + if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", @@ -382,64 +418,52 @@ pgBackupValidateFiles(void *arg) int do_validate_all(CatalogState *catalogState, InstanceState *instanceState) { + FOBJ_FUNC_ARP(); corrupted_backup_found = false; skipped_due_to_lock = false; + err_i err; if (instanceState == NULL) { /* Show list of instances */ - DIR *dir; - struct dirent *dent; + pioDirIter_i data_dir; + pio_dirent_t ent; /* open directory and list contents */ - dir = opendir(catalogState->backup_subdir_path); - if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", catalogState->backup_subdir_path, strerror(errno)); + data_dir = $i(pioOpenDir, catalogState->backup_location, + catalogState->backup_subdir_path, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Failed to get backup list"); - errno = 0; - while ((dent = readdir(dir))) + while ((ent = $i(pioDirNext, data_dir, .err=&err)).stat.pst_kind) { - char child[MAXPGPATH]; - struct stat st; - InstanceState *instanceState; - - - /* skip entries point current dir or parent dir */ - if (strcmp(dent->d_name, ".") == 0 || - strcmp(dent->d_name, "..") == 0) - continue; - - join_path_components(child, catalogState->backup_subdir_path, dent->d_name); - - if (lstat(child, &st) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + FOBJ_LOOP_ARP(); + InstanceState *instanceState = NULL; - if (!S_ISDIR(st.st_mode)) + if (ent.stat.pst_kind != PIO_KIND_DIRECTORY) continue; - /* - * Initialize instance configuration. - */ - instanceState = pgut_new(InstanceState); - strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); - - join_path_components(instanceState->instance_backup_subdir_path, - catalogState->backup_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_wal_subdir_path, - catalogState->wal_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_config_path, - instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); - - if (config_read_opt(instanceState->instance_config_path, instance_options, ERROR, false, - true) == 0) + instanceState = makeInstanceState(catalogState, ent.name.ptr); + + if (config_read_opt(catalogState->backup_location, instanceState->instance_config_path, + instance_options, ERROR, false, &err) == 0) { + if ($haserr(err) && getErrno(err) != ENOENT) + ft_logerr(FT_FATAL, $errmsg(err), ""); elog(WARNING, "Configuration file \"%s\" is empty", instanceState->instance_config_path); corrupted_backup_found = true; continue; } do_validate_instance(instanceState); + pgut_free(instanceState); } + + $i(pioClose, data_dir); // ignore error + + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Read backup root directory"); + } else { @@ -708,13 +732,13 @@ do_validate_instance(InstanceState *instanceState) bool validate_tablespace_map(pgBackup *backup, bool no_validate) { + FOBJ_FUNC_ARP(); char map_path[MAXPGPATH]; pgFile *dummy = NULL; pgFile **tablespace_map = NULL; pg_crc32 crc; parray *files = get_backup_filelist(backup, true); - bool use_crc32c = parse_program_version(backup->program_version) <= 20021 || - parse_program_version(backup->program_version) >= 20025; + err_i err; parray_qsort(files, pgFileCompareRelPathWithExternal); join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); @@ -725,6 +749,7 @@ validate_tablespace_map(pgBackup *backup, bool no_validate) if (!tablespace_map) { elog(LOG, "there is no file tablespace_map"); + pgFileFree(dummy); parray_walk(files, pgFileFree); parray_free(files); return false; @@ -739,7 +764,23 @@ validate_tablespace_map(pgBackup *backup, bool no_validate) /* check tablespace map checksumms */ if (!no_validate) { - crc = pgFileGetCRC(map_path, use_crc32c, false); +#if PG_VERSION_NUM >= 120000 + Assert(parse_program_version(backup->program_version) >= 20025); + crc = $i(pioGetCRC32, backup->backup_location, .path = map_path, + .err = &err); +#else /* PG_VERSION_NUM < 120000 */ + if (parse_program_version(backup->program_version) <= 20021 + || parse_program_version(backup->program_version) >= 20025) + crc = $i(pioGetCRC32, backup->backup_location, .path = map_path, + .err = &err); + else + { + ft_assert(!$i(pioIsRemote, backup->backup_location)); + crc = pgFileGetCRC32(map_path, false); + } +#endif /* PG_VERSION_NUM < 120000 */ + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Tablespace map file CRC"); if ((*tablespace_map)->crc != crc) elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, " diff --git a/tests/__init__.py b/tests/__init__.py index c8d2c70c3..891f87447 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,8 +4,8 @@ from . import init_test, merge_test, option_test, show_test, compatibility_test, \ backup_test, delete_test, delta_test, restore_test, validate_test, \ retention_test, pgpro560_test, pgpro589_test, pgpro2068_test, false_positive_test, replica_test, \ - compression_test, page_test, ptrack_test, archive_test, exclude_test, cfs_backup_test, cfs_restore_test, \ - cfs_validate_backup_test, auth_test, time_stamp_test, logging_test, \ + compression_test, page_test, ptrack_test, archive_test, exclude_test, \ + auth_test, time_stamp_test, logging_test, \ locking_test, remote_test, external_test, config_test, checkdb_test, set_backup_test, incr_restore_test, \ catchup_test, CVE_2018_1058_test, time_consuming_test @@ -35,9 +35,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(compatibility_test)) suite.addTests(loader.loadTestsFromModule(checkdb_test)) suite.addTests(loader.loadTestsFromModule(config_test)) - suite.addTests(loader.loadTestsFromModule(cfs_backup_test)) - suite.addTests(loader.loadTestsFromModule(cfs_restore_test)) - suite.addTests(loader.loadTestsFromModule(cfs_validate_backup_test)) suite.addTests(loader.loadTestsFromModule(compression_test)) suite.addTests(loader.loadTestsFromModule(delete_test)) suite.addTests(loader.loadTestsFromModule(delta_test)) diff --git a/tests/archive_test.py b/tests/archive_test.py index b2217a7bf..d2fc4d3d8 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -3,6 +3,7 @@ import gzip import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, GdbException +from .helpers.ptrack_helpers import tail_file, needs_gdb from datetime import datetime, timedelta import subprocess from sys import exit @@ -77,10 +78,6 @@ def test_pgpro434_2(self): 'checkpoint_timeout': '30s'} ) - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -208,12 +205,12 @@ def test_pgpro434_2(self): 'data after restore not equal to original data') # @unittest.skip("skip") + @needs_gdb def test_pgpro434_3(self): """ Check pg_stop_backup_timeout, needed backup_timeout Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -230,7 +227,7 @@ def test_pgpro434_3(self): gdb = self.backup_node( backup_dir, 'node', node, options=[ - "--archive-timeout=60", + "--archive-timeout=10", "--log-level-file=LOG"], gdb=True) @@ -241,6 +238,8 @@ def test_pgpro434_3(self): self.set_auto_conf(node, {'archive_command': 'exit 1'}) node.reload() + sleep(1) + gdb.continue_execution_until_exit() sleep(1) @@ -249,15 +248,9 @@ def test_pgpro434_3(self): with open(log_file, 'r') as f: log_content = f.read() - # in PG =< 9.6 pg_stop_backup always wait - if self.get_version(node) < 100000: - self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", - log_content) - else: - self.assertIn( - "ERROR: WAL segment 000000010000000000000003 could not be archived in 60 seconds", - log_content) + self.assertIn( + "ERROR: WAL segment 000000010000000000000003 could not be archived in 10 seconds", + log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: @@ -269,12 +262,12 @@ def test_pgpro434_3(self): 'PostgreSQL crashed because of a failed assert') # @unittest.skip("skip") + @needs_gdb def test_pgpro434_4(self): """ Check pg_stop_backup_timeout, libpq-timeout requested. Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -291,7 +284,7 @@ def test_pgpro434_4(self): gdb = self.backup_node( backup_dir, 'node', node, options=[ - "--archive-timeout=60", + "--archive-timeout=10", "--log-level-file=info"], gdb=True) @@ -320,7 +313,6 @@ def test_pgpro434_4(self): postgres_gdb.continue_execution_until_running() gdb.continue_execution_until_exit() - # gdb._execute('detach') log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') with open(log_file, 'r') as f: @@ -328,11 +320,11 @@ def test_pgpro434_4(self): if self.get_version(node) < 150000: self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + "ERROR: pg_stop_backup doesn't answer in 10 seconds, cancel it", log_content) else: self.assertIn( - "ERROR: pg_backup_stop doesn't answer in 60 seconds, cancel it", + "ERROR: pg_backup_stop doesn't answer in 10 seconds, cancel it", log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') @@ -397,19 +389,20 @@ def test_archive_push_file_exists(self): 'pg_probackup archive-push WAL file', log_content) - self.assertIn( - 'WAL file already exists in archive with different checksum', - log_content) + if self.archive_compress: + self.assertIn( + 'WAL file already exists and looks like it is damaged', + log_content) + else: + self.assertIn( + 'WAL file already exists in archive with different checksum', + log_content) self.assertNotIn( 'pg_probackup archive-push completed successfully', log_content) - if self.get_version(node) < 100000: - wal_src = os.path.join( - node.data_dir, 'pg_xlog', '000000010000000000000001') - else: - wal_src = os.path.join( - node.data_dir, 'pg_wal', '000000010000000000000001') + wal_src = os.path.join( + node.data_dir, 'pg_wal', '000000010000000000000001') if self.archive_compress: with open(wal_src, 'rb') as f_in, gzip.open( @@ -480,12 +473,15 @@ def test_archive_push_file_exists_overwrite(self): 'DETAIL: The failed archive command was:', log_content) self.assertIn( 'pg_probackup archive-push WAL file', log_content) - self.assertNotIn( - 'WAL file already exists in archive with ' - 'different checksum, overwriting', log_content) - self.assertIn( - 'WAL file already exists in archive with ' - 'different checksum', log_content) + self.assertNotIn('overwriting', log_content) + if self.archive_compress: + self.assertIn( + 'WAL file already exists and looks like ' + 'it is damaged', log_content) + else: + self.assertIn( + 'WAL file already exists in archive with ' + 'different checksum', log_content) self.assertNotIn( 'pg_probackup archive-push completed successfully', log_content) @@ -501,92 +497,20 @@ def test_archive_push_file_exists_overwrite(self): 'pg_probackup archive-push completed successfully' in log_content, 'Expecting messages about successfull execution archive_command') - self.assertIn( - 'WAL file already exists in archive with ' - 'different checksum, overwriting', log_content) - - # @unittest.skip("skip") - def test_archive_push_partial_file_exists(self): - """Archive-push if stale '.part' file exists""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving( - backup_dir, 'node', node, - log_level='verbose', archive_timeout=60) - - node.slow_start() - - # this backup is needed only for validation to xid - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t1(a int)") - - xid = node.safe_psql( - "postgres", - "INSERT INTO t1 VALUES (1) RETURNING (xmin)").decode('utf-8').rstrip() - - if self.get_version(node) < 100000: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() - else: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() - - filename_orig = filename_orig.decode('utf-8') - - # form up path to next .part WAL segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: - filename = filename_orig + '.gz' + '.part' - file = os.path.join(wals_dir, filename) + self.assertIn( + 'WAL file already exists and looks like ' + 'it is damaged, overwriting', log_content) else: - filename = filename_orig + '.part' - file = os.path.join(wals_dir, filename) - - # emulate stale .part file - with open(file, 'a+b') as f: - f.write(b"blahblah") - f.flush() - f.close() - - self.switch_wal_segment(node) - sleep(70) - - # check that segment is archived - if self.archive_compress: - filename_orig = filename_orig + '.gz' - - file = os.path.join(wals_dir, filename_orig) - self.assertTrue(os.path.isfile(file)) - - # successful validate means that archive-push reused stale wal segment - self.validate_pb( - backup_dir, 'node', - options=['--recovery-target-xid={0}'.format(xid)]) - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - self.assertIn( - 'Reusing stale temp WAL file', - log_content) + 'WAL file already exists in archive with ' + 'different checksum, overwriting', log_content) - # @unittest.skip("skip") + @unittest.skip("should be redone with file locking") def test_archive_push_part_file_exists_not_stale(self): """Archive-push if .part file exists and it is not stale""" + # TODO: this test is not completely obsolete, but should be rewritten + # with use of file locking when push_file_internal will use it. backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -608,16 +532,10 @@ def test_archive_push_part_file_exists_not_stale(self): "postgres", "create table t2()") - if self.get_version(node) < 100000: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() - else: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() filename_orig = filename_orig.decode('utf-8') @@ -678,10 +596,6 @@ def test_replica_archive(self): 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) @@ -726,9 +640,6 @@ def test_replica_archive(self): backup_dir, 'replica', replica, options=[ '--archive-timeout=30', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), '--stream']) self.validate_pb(backup_dir, 'replica') @@ -765,9 +676,6 @@ def test_replica_archive(self): replica, backup_type='page', options=[ '--archive-timeout=60', - '--master-db=postgres', - '--master-host=localhost', - '--master-port={0}'.format(master.port), '--stream']) self.validate_pb(backup_dir, 'replica') @@ -804,10 +712,6 @@ def test_master_and_replica_parallel_archiving(self): 'archive_timeout': '10s'} ) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - replica = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() @@ -858,9 +762,6 @@ def test_master_and_replica_parallel_archiving(self): backup_dir, 'replica', replica, options=[ '--archive-timeout=30', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), '--stream']) self.validate_pb(backup_dir, 'replica') @@ -882,9 +783,6 @@ def test_basic_master_and_replica_concurrent_archiving(self): set replica with archiving, make sure that archiving on both node is working. """ - if self.pg_config_version < self.version_to_num('9.6.0'): - self.skipTest('You need PostgreSQL >= 9.6 for this test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'master'), @@ -894,10 +792,6 @@ def test_basic_master_and_replica_concurrent_archiving(self): 'checkpoint_timeout': '30s', 'archive_timeout': '10s'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - replica = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() @@ -1065,10 +959,7 @@ def test_archive_pg_receivexlog(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if self.get_version(node) < 100000: - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') + pg_receivexlog_path = self.get_bin_path('pg_receivewal') pg_receivexlog = self.run_binary( [ @@ -1134,11 +1025,8 @@ def test_archive_pg_receivexlog_compression_pg10(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if self.get_version(node) < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') + pg_receivexlog_path = self.get_bin_path('pg_receivewal') pg_receivexlog = self.run_binary( [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', @@ -1213,10 +1101,6 @@ def test_archive_catalog(self): 'archive_timeout': '30s', 'checkpoint_timeout': '30s'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -1853,10 +1737,6 @@ def test_waldir_outside_pgdata_archiving(self): """ check that archive-push works correct with symlinked waldir """ - if self.pg_config_version < self.version_to_num('10.0'): - self.skipTest( - 'Skipped because waldir outside pgdata is supported since PG 10') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') external_wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'ext_wal_dir') shutil.rmtree(external_wal_dir, ignore_errors=True) @@ -1956,10 +1836,7 @@ def test_archiving_and_slots(self): self.set_archiving(backup_dir, 'node', node, log_level='verbose') node.slow_start() - if self.get_version(node) < 100000: - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') + pg_receivexlog_path = self.get_bin_path('pg_receivewal') # "pg_receivewal --create-slot --slot archive_slot --if-not-exists " # "&& pg_receivewal --synchronous -Z 1 /tmp/wal --slot archive_slot --no-loop" @@ -2021,7 +1898,7 @@ def test_archive_push_sanity(self): self.backup_node(backup_dir, 'node', node) with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() + postgres_log_content = cleanup_ptrack(f.read()) # print(postgres_log_content) # make sure that .backup file is not compressed @@ -2050,7 +1927,7 @@ def test_archive_push_sanity(self): replica.pgbench_init(scale=10) with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - replica_log_content = f.read() + replica_log_content = cleanup_ptrack(f.read()) # make sure that .partial file is not compressed self.assertNotIn('.partial.gz', replica_log_content) @@ -2074,21 +1951,13 @@ def test_archive_pg_receivexlog_partial_handling(self): set_replication=True, initdb_params=['--data-checksums']) - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if self.get_version(node) < 100000: - app_name = 'pg_receivexlog' - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - app_name = 'pg_receivewal' - pg_receivexlog_path = self.get_bin_path('pg_receivewal') + app_name = 'pg_receivewal' + pg_receivexlog_path = self.get_bin_path('pg_receivewal') cmdline = [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', @@ -2265,10 +2134,6 @@ def test_archive_get_batching_sanity(self): set_replication=True, initdb_params=['--data-checksums']) - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2340,7 +2205,7 @@ def test_archive_get_prefetch_corruption(self): self.backup_node(backup_dir, 'node', node, options=['--stream']) - node.pgbench_init(scale=50) + node.pgbench_init(scale=20) replica = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'replica')) @@ -2387,7 +2252,7 @@ def test_archive_get_prefetch_corruption(self): # generate WAL, copy it into prefetch directory, then corrupt # some segment - node.pgbench_init(scale=20) + node.pgbench_init(scale=5) sleep(20) # now copy WAL files into prefetch directory and corrupt some of them @@ -2440,18 +2305,20 @@ def test_archive_get_prefetch_corruption(self): os.remove(os.path.join(replica.logs_dir, 'postgresql.log')) replica.slow_start(replica=True) - sleep(60) - - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() - - self.assertIn( - 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename), - postgres_log_content) - - self.assertIn( - 'LOG: restored log file "{0}" from archive'.format(filename), - postgres_log_content) + prefetch_line = 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename) + has_prefetch_line = False + restored_line = 'LOG: restored log file "{0}" from archive'.format(filename) + has_restored_line = False + for line in tail_file(os.path.join(replica.logs_dir, 'postgresql.log')): + if not has_prefetch_line: + has_prefetch_line = prefetch_line in line + if not has_restored_line: + has_restored_line = restored_line in line + if has_prefetch_line and has_restored_line: + break + + self.assertTrue(has_prefetch_line) + self.assertTrue(has_restored_line) # @unittest.skip("skip") def test_archive_show_partial_files_handling(self): @@ -2480,16 +2347,10 @@ def test_archive_show_partial_files_handling(self): "postgres", "create table t1()") - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() filename = filename.decode('utf-8') @@ -2497,23 +2358,17 @@ def test_archive_show_partial_files_handling(self): os.rename( os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.part'.format(filename))) + os.path.join(wals_dir, '{0}~tmp123451'.format(filename))) # .gz.part file node.safe_psql( "postgres", "create table t2()") - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() filename = filename.decode('utf-8') @@ -2521,23 +2376,17 @@ def test_archive_show_partial_files_handling(self): os.rename( os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.gz.part'.format(filename))) + os.path.join(wals_dir, '{0}.gz~tmp234513'.format(filename))) # .partial file node.safe_psql( "postgres", "create table t3()") - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() filename = filename.decode('utf-8') @@ -2552,16 +2401,10 @@ def test_archive_show_partial_files_handling(self): "postgres", "create table t4()") - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() filename = filename.decode('utf-8') @@ -2669,6 +2512,15 @@ def test_archive_empty_history_file(self): 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), log_content) +def cleanup_ptrack(log_content): + # PBCKP-423 - need to clean ptrack warning + ptrack_is_not = 'Ptrack 1.X is not supported anymore' + if ptrack_is_not in log_content: + lines = [line for line in log_content.splitlines() + if ptrack_is_not not in line] + log_content = "".join(lines) + return log_content + # TODO test with multiple not archived segments. # TODO corrupted file in archive. diff --git a/tests/auth_test.py b/tests/auth_test.py index 52d7e1544..748cdee55 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -80,14 +80,9 @@ def test_backup_via_unprivileged_user(self): "GRANT EXECUTE ON FUNCTION" " pg_backup_start(text, boolean) TO backup;") - if self.get_version(node) < 100000: - node.safe_psql( - 'postgres', - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup") - else: - node.safe_psql( - 'postgres', - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup") + node.safe_psql( + 'postgres', + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup") try: self.backup_node( @@ -128,11 +123,7 @@ def test_backup_via_unprivileged_user(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - if self.get_version(node) < self.version_to_num('10.0'): - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") - elif self.get_version(node) < self.version_to_num('15.0'): + if self.get_version(node) < self.version_to_num('15.0'): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " diff --git a/tests/backup_test.py b/tests/backup_test.py index fc1135cab..eb799b937 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -3,6 +3,7 @@ import re from time import sleep, time from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb import shutil from distutils.dir_util import copy_tree from testgres import ProcessType, QueryException @@ -1090,9 +1091,9 @@ def test_tablespace_handling_2(self): repr(e.message), self.cmd)) # @unittest.skip("skip") + @needs_gdb def test_drop_rel_during_full_backup(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1233,9 +1234,9 @@ def test_drop_db_during_full_backup(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_drop_rel_during_backup_delta(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1300,9 +1301,9 @@ def test_drop_rel_during_backup_delta(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_drop_rel_during_backup_page(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1403,9 +1404,6 @@ def test_basic_temp_slot_for_stream_backup(self): initdb_params=['--data-checksums'], pg_options={'max_wal_size': '40MB'}) - if self.get_version(node) < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1422,9 +1420,9 @@ def test_basic_temp_slot_for_stream_backup(self): options=['--stream', '--slot=slot_1', '--temp-slot']) # @unittest.skip("skip") + @needs_gdb def test_backup_concurrent_drop_table(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1550,9 +1548,9 @@ def test_pg_11_adjusted_wal_segment_size(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_sigint_handling(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1587,9 +1585,9 @@ def test_sigint_handling(self): 'Backup STATUS should be "ERROR"') # @unittest.skip("skip") + @needs_gdb def test_sigterm_handling(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1623,9 +1621,9 @@ def test_sigterm_handling(self): 'Backup STATUS should be "ERROR"') # @unittest.skip("skip") + @needs_gdb def test_sigquit_handling(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1712,22 +1710,11 @@ def test_basic_missing_file_permissions(self): os.chmod(full_path, 000) - try: + with self.assertRaisesRegex(ProbackupException, + r"ERROR: [^\n]*: Permission denied"): # FULL backup self.backup_node( backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Cannot open file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) os.chmod(full_path, 700) @@ -1751,22 +1738,10 @@ def test_basic_missing_dir_permissions(self): os.chmod(full_path, 000) - try: + with self.assertRaisesRegex(ProbackupException, r'ERROR:[^\n]*Cannot open dir'): # FULL backup self.backup_node( backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Cannot open directory', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) os.rmdir(full_path) @@ -1779,7 +1754,7 @@ def test_backup_with_least_privileges_role(self): set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], - pg_options={'archive_timeout': '30s'}) + pg_options={'archive_timeout': '10s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1796,81 +1771,7 @@ def test_backup_with_least_privileges_role(self): "CREATE SCHEMA ptrack; " "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -2180,14 +2081,14 @@ def test_backup_with_less_privileges_role(self): ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], pg_options={ - 'archive_timeout': '30s', + 'archive_timeout': '10s', 'archive_mode': 'always', - 'checkpoint_timeout': '60s', + 'checkpoint_timeout': '30s', 'wal_level': 'logical'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + self.set_config(backup_dir, 'node', options=['--archive-timeout=30s']) self.set_archiving(backup_dir, 'node', node) node.slow_start() @@ -2200,43 +2101,10 @@ def test_backup_with_less_privileges_role(self): 'backupdb', 'CREATE EXTENSION ptrack') - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.get_version(node) < 150000: node.safe_psql( 'backupdb', + "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2309,9 +2177,6 @@ def test_backup_with_less_privileges_role(self): backup_dir, 'node', node, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) - if self.get_version(node) < 90600: - return - # Restore as replica replica = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'replica')) @@ -2322,7 +2187,7 @@ def test_backup_with_less_privileges_role(self): self.add_instance(backup_dir, 'replica', replica) self.set_config( backup_dir, 'replica', - options=['--archive-timeout=120s', '--log-level-console=LOG']) + options=['--archive-timeout=60s', '--log-level-console=LOG']) self.set_archiving(backup_dir, 'replica', replica, replica=True) self.set_auto_conf(replica, {'hot_standby': 'on'}) @@ -2339,9 +2204,10 @@ def test_backup_with_less_privileges_role(self): # self.switch_wal_segment(node) # self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, - datname='backupdb', options=['-U', 'backup']) + with self.switch_wal_after(node, 10): + self.backup_node( + backup_dir, 'replica', replica, + datname='backupdb', options=['-U', 'backup']) # stream full backup from replica self.backup_node( @@ -2351,30 +2217,30 @@ def test_backup_with_less_privileges_role(self): # self.switch_wal_segment(node) # PAGE backup from replica - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='page', - datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) + with self.switch_wal_after(node, 10): + self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + datname='backupdb', options=['-U', 'backup', '--archive-timeout=20s']) self.backup_node( backup_dir, 'replica', replica, backup_type='page', datname='backupdb', options=['--stream', '-U', 'backup']) # DELTA backup from replica - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['-U', 'backup']) + with self.switch_wal_after(node, 10): + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', + datname='backupdb', options=['-U', 'backup']) self.backup_node( backup_dir, 'replica', replica, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK backup from replica if self.ptrack: - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - datname='backupdb', options=['-U', 'backup']) + with self.switch_wal_after(node, 10): + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) @@ -2859,9 +2725,9 @@ def test_incr_backup_filenode_map(self): 'select 1') # @unittest.skip("skip") + @needs_gdb def test_missing_wal_segment(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -2963,50 +2829,7 @@ def test_missing_replication_permission(self): 'postgres', 'CREATE DATABASE backupdb') - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.get_version(node) < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -3113,51 +2936,8 @@ def test_missing_replication_permission_1(self): 'postgres', 'CREATE DATABASE backupdb') - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -3277,9 +3057,9 @@ def test_basic_backup_default_transaction_read_only(self): self.backup_node(backup_dir, 'node', node, backup_type='page') # @unittest.skip("skip") + @needs_gdb def test_backup_atexit(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -3351,15 +3131,7 @@ def test_pg_stop_backup_missing_permissions(self): self.simple_bootstrap(node, 'backup') - if self.get_version(node) < 90600: - node.safe_psql( - 'postgres', - 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() FROM backup') - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'postgres', - 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) FROM backup') - elif self.get_version(node) < 150000: + if self.get_version(node) < 150000: node.safe_psql( 'postgres', 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup') @@ -3368,7 +3140,6 @@ def test_pg_stop_backup_missing_permissions(self): 'postgres', 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) FROM backup') - # Full backup in streaming mode try: self.backup_node( diff --git a/tests/catchup_test.py b/tests/catchup_test.py index 21bcd7973..000420dee 100644 --- a/tests/catchup_test.py +++ b/tests/catchup_test.py @@ -1210,27 +1210,26 @@ def test_catchup_with_replication_slot(self): ).decode('utf-8').rstrip() self.assertEqual(slot_name, 'pg_probackup_perm_slot', 'Slot name mismatch') - # 5. --perm-slot --temp-slot (PG>=10) - if self.get_version(src_pg) >= self.version_to_num('10.0'): - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_5')) - try: - self.catchup_node( - backup_mode = 'FULL', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = [ - '-d', 'postgres', '-p', str(src_pg.port), '--stream', - '--perm-slot', - '--temp-slot' - ] - ) - self.assertEqual(1, 0, "Expecting Error because conflicting options --perm-slot and --temp-slot used together\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: You cannot specify "--perm-slot" option with the "--temp-slot" option', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # 5. --perm-slot --temp-slot + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_5')) + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--perm-slot', + '--temp-slot' + ] + ) + self.assertEqual(1, 0, "Expecting Error because conflicting options --perm-slot and --temp-slot used together\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: You cannot specify "--perm-slot" option with the "--temp-slot" option', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) #self.assertEqual(1, 0, 'Stop test') diff --git a/tests/cfs_backup_test.py b/tests/cfs_backup_test.py deleted file mode 100644 index cd2826d21..000000000 --- a/tests/cfs_backup_test.py +++ /dev/null @@ -1,1231 +0,0 @@ -import os -import unittest -import random -import shutil - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' - - -class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): - # --- Begin --- # - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def setUp(self): - self.backup_dir = os.path.join( - self.tmp_path, self.module_name, self.fname, 'backup') - self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'cfs_encryption': 'off', - 'max_wal_senders': '2', - 'shared_buffers': '200MB' - } - ) - - self.init_pb(self.backup_dir) - self.add_instance(self.backup_dir, 'node', self.node) - self.set_archiving(self.backup_dir, 'node', self.node) - - self.node.slow_start() - - self.node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) - - tblspace = self.node.safe_psql( - "postgres", - "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( - tblspace_name)) - - self.assertIn( - tblspace_name, str(tblspace), - "ERROR: The tablespace not created " - "or it create without compressions") - - self.assertIn( - "compression=true", str(tblspace), - "ERROR: The tablespace not created " - "or it create without compressions") - - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - - # --- Section: Full --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace(self): - """Case: Check fullbackup empty compressed tablespace""" - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_stream(self): - """Case: Check fullbackup empty compressed tablespace with options stream""" - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table(self): - """Case: Make full backup after created table in the tablespace""" - if not self.enterprise: - return - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "\n ERROR: {0}\n CMD: {1}".format( - repr(e.message), - repr(self.cmd) - ) - ) - return False - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in {0}".format( - os.path.join(self.backup_dir, 'node', backup_id)) - ) - - # check cfm size - cfms = find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table_stream(self): - """ - Case: Make full backup after created table in the tablespace with option --stream - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - # --- Section: Incremental from empty tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='ptrack') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_page_doesnt_store_unchanged_cfm(self): - """ - Case: Test page backup doesn't store cfm file if table were not modified - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id_full)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files is found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='page', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # --- Section: Incremental from fill tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='ptrack') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb( - self.backup_dir, 'node', backup_id_ptrack) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_ptrack["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace(--stream). - Make ptrack backup after create table(--stream). - Check: incremental backup size should not be greater than full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,25) i".format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb( - self.backup_dir, 'node', backup_id_ptrack) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_ptrack["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup size should not be greater than full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_multiple_segments(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap', tblspace_name) - ) - - full_result = self.node.table_checksum("t_heap") - - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap') - ) - - page_result = self.node.table_checksum("t_heap") - - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # CHECK FULL BACKUP - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - self.restore_node( - self.backup_dir, 'node', self.node, backup_id=backup_id_full, - options=[ - "-j", "4", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - - self.node.slow_start() - self.assertEqual( - full_result, - self.node.table_checksum("t_heap"), - 'Lost data after restore') - - # CHECK PAGE BACKUP - self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - self.restore_node( - self.backup_dir, 'node', self.node, backup_id=backup_id_page, - options=[ - "-j", "4", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - - self.node.slow_start() - self.assertEqual( - page_result, - self.node.table_checksum("t_heap"), - 'Lost data after restore') - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_multiple_segments_in_multiple_tablespaces(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - tblspace_name_1 = 'tblspace_name_1' - tblspace_name_2 = 'tblspace_name_2' - - self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True) - self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap_1', tblspace_name_1)) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap_2', tblspace_name_2)) - - full_result_1 = self.node.table_checksum("t_heap_1") - full_result_2 = self.node.table_checksum("t_heap_2") - - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap_1') - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap_2') - ) - - page_result_1 = self.node.table_checksum("t_heap_1") - page_result_2 = self.node.table_checksum("t_heap_2") - - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # CHECK FULL BACKUP - self.node.stop() - - self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_full, - options=[ - "-j", "4", "--incremental-mode=checksum", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - self.node.slow_start() - - self.assertEqual( - full_result_1, - self.node.table_checksum("t_heap_1"), - 'Lost data after restore') - self.assertEqual( - full_result_2, - self.node.table_checksum("t_heap_2"), - 'Lost data after restore') - - # CHECK PAGE BACKUP - self.node.stop() - - self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_page, - options=[ - "-j", "4", "--incremental-mode=checksum", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - self.node.slow_start() - - self.assertEqual( - page_result_1, - self.node.table_checksum("t_heap_1"), - 'Lost data after restore') - self.assertEqual( - page_result_2, - self.node.table_checksum("t_heap_2"), - 'Lost data after restore') - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace(--stream). - Make ptrack backup after create table(--stream). - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='page', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # --- Make backup with not valid data(broken .cfm) --- # - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_random_cfm_file_from_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - self.node.safe_psql( - "postgres", - "CHECKPOINT" - ) - - list_cmf = find_by_extensions( - [self.get_tblspace_path(self.node, tblspace_name)], - ['.cfm']) - self.assertTrue( - list_cmf, - "ERROR: .cfm-files not found into tablespace dir" - ) - - os.remove(random.choice(list_cmf)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_file_pg_compression_from_tablespace_dir(self): - os.remove( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression'])[0]) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_random_data_file_from_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - self.node.safe_psql( - "postgres", - "CHECKPOINT" - ) - - list_data_files = find_by_pattern( - [self.get_tblspace_path(self.node, tblspace_name)], - '^.*/\d+$') - self.assertTrue( - list_data_files, - "ERROR: Files of data not found into tablespace dir" - ) - - os.remove(random.choice(list_data_files)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_random_cfm_file_into_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - list_cmf = find_by_extensions( - [self.get_tblspace_path(self.node, tblspace_name)], - ['.cfm']) - self.assertTrue( - list_cmf, - "ERROR: .cfm-files not found into tablespace dir" - ) - - corrupt_file(random.choice(list_cmf)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_random_data_file_into_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - list_data_files = find_by_pattern( - [self.get_tblspace_path(self.node, tblspace_name)], - '^.*/\d+$') - self.assertTrue( - list_data_files, - "ERROR: Files of data not found into tablespace dir" - ) - - corrupt_file(random.choice(list_data_files)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_file_pg_compression_into_tablespace_dir(self): - - corrupted_file = find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression'])[0] - - self.assertTrue( - corrupt_file(corrupted_file), - "ERROR: File is not corrupted or it missing" - ) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - -# # --- End ---# - - -#class CfsBackupEncTest(CfsBackupNoEncTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_catchup_test.py b/tests/cfs_catchup_test.py deleted file mode 100644 index f6760b72c..000000000 --- a/tests/cfs_catchup_test.py +++ /dev/null @@ -1,117 +0,0 @@ -import os -import unittest -import random -import shutil - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase): - - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_full_catchup_with_tablespace(self): - """ - Test tablespace transfers - """ - # preparation - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), - set_replication = True - ) - src_pg.slow_start() - tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old') - self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path, cfs=True) - src_pg.safe_psql( - "postgres", - "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") - src_query_result = src_pg.table_checksum("ultimate_question") - src_pg.safe_psql( - "postgres", - "CHECKPOINT") - - # do full catchup with tablespace mapping - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') - self.catchup_node( - backup_mode = 'FULL', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = [ - '-d', 'postgres', - '-p', str(src_pg.port), - '--stream', - '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) - ] - ) - - # 1st check: compare data directories - self.compare_pgdata( - self.pgdata_content(src_pg.data_dir), - self.pgdata_content(dst_pg.data_dir) - ) - - # check cfm size - cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # make changes in master tablespace - src_pg.safe_psql( - "postgres", - "UPDATE ultimate_question SET answer = -1") - src_pg.safe_psql( - "postgres", - "CHECKPOINT") - - # run&recover catchup'ed instance - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - - # 2nd check: run verification query - dst_query_result = dst_pg.table_checksum("ultimate_question") - self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') - - # and now delta backup - dst_pg.stop() - - self.catchup_node( - backup_mode = 'DELTA', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = [ - '-d', 'postgres', - '-p', str(src_pg.port), - '--stream', - '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) - ] - ) - - # check cfm size again - cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # run&recover catchup'ed instance - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - - - # 3rd check: run verification query - src_query_result = src_pg.table_checksum("ultimate_question") - dst_query_result = dst_pg.table_checksum("ultimate_question") - self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') diff --git a/tests/cfs_restore_test.py b/tests/cfs_restore_test.py deleted file mode 100644 index 2fa35e71a..000000000 --- a/tests/cfs_restore_test.py +++ /dev/null @@ -1,447 +0,0 @@ -""" -restore - Syntax: - - pg_probackup restore -B backupdir --instance instance_name - [-D datadir] - [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] - [-j num_threads] [--progress] [-q] [-v] - -""" -import os -import unittest -import shutil - -from .helpers.cfs_helpers import find_by_name -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' -tblspace_name_new = 'cfs_tblspace_new' - - -class CfsRestoreBase(ProbackupTest, unittest.TestCase): - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def setUp(self): - self.backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ -# 'ptrack_enable': 'on', - 'cfs_encryption': 'off', - } - ) - - self.init_pb(self.backup_dir) - self.add_instance(self.backup_dir, 'node', self.node) - self.set_archiving(self.backup_dir, 'node', self.node) - - self.node.slow_start() - self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) - - self.add_data_in_cluster() - - self.backup_id = None - try: - self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - def add_data_in_cluster(self): - pass - - -class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_empty_tablespace_from_fullbackup(self): - """ - Case: Restore empty tablespace from valid full backup. - """ - self.node.stop(["-m", "immediate"]) - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) - except ProbackupException as e: - self.fail( - "ERROR: Restore failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), - "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." - ) - - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - tblspace = self.node.safe_psql( - "postgres", - "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) - ).decode("UTF-8") - self.assertTrue( - tblspace_name in tblspace and "compression=true" in tblspace, - "ERROR: The tablespace not restored or it restored without compressions" - ) - - -class CfsRestoreNoencTest(CfsRestoreBase): - def add_data_in_cluster(self): - self.node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,1e5) i'.format('t1', tblspace_name) - ) - self.table_t1 = self.node.table_checksum("t1") - - # --- Restore from full backup ---# - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location(self): - """ - Case: Restore instance from valid full backup to old location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in tablespace dir" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - self.node.table_checksum("t1"), - self.table_t1 - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_3_jobs(self): - """ - Case: Restore instance from valid full backup to old location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - self.node.table_checksum("t1"), - self.table_t1 - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_new_location(self): - """ - Case: Restore instance from valid full backup to new location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) - node_new.cleanup() - - try: - self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id) - self.set_auto_conf(node_new, {'port': node_new.port}) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - node_new.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - node_new.table_checksum("t1"), - self.table_t1 - ) - node_new.cleanup() - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_new_location_5_jobs(self): - """ - Case: Restore instance from valid full backup to new location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) - node_new.cleanup() - - try: - self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id, options=['-j', '5']) - self.set_auto_conf(node_new, {'port': node_new.port}) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - node_new.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - node_new.table_checksum("t1"), - self.table_t1 - ) - node_new.cleanup() - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) - - try: - self.restore_node( - self.backup_dir, - 'node', self.node, - backup_id=self.backup_id, - options=["-T", "{0}={1}".format( - self.get_tblspace_path(self.node, tblspace_name), - self.get_tblspace_path(self.node, tblspace_name_new) - ) - ] - ) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), - "ERROR: File pg_compression not found in new tablespace location" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - self.node.table_checksum("t1"), - self.table_t1 - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) - - try: - self.restore_node( - self.backup_dir, - 'node', self.node, - backup_id=self.backup_id, - options=["-j", "3", "-T", "{0}={1}".format( - self.get_tblspace_path(self.node, tblspace_name), - self.get_tblspace_path(self.node, tblspace_name_new) - ) - ] - ) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), - "ERROR: File pg_compression not found in new tablespace location" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - self.node.table_checksum("t1"), - self.table_t1 - ) - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack(self): - """ - Case: Restore from backup to old location - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack_jobs(self): - """ - Case: Restore from backup to old location, four jobs - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack_new_jobs(self): - pass - -# --------------------------------------------------------- # - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page(self): - """ - Case: Restore from backup to old location - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page_jobs(self): - """ - Case: Restore from backup to old location, four jobs - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page_new_jobs(self): - """ - Case: Restore from backup to new location, four jobs - """ - pass - - -#class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() -# -# -#class CfsRestoreEncTest(CfsRestoreNoencTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_validate_backup_test.py b/tests/cfs_validate_backup_test.py deleted file mode 100644 index 343020dfc..000000000 --- a/tests/cfs_validate_backup_test.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import unittest -import random - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' - - -class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): - def setUp(self): - pass - - def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): - pass - - def tearDown(self): - pass - - -#class CfsValidateBackupNoenc(CfsValidateBackupNoenc): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsValidateBackupNoenc).setUp() diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py index 1e6daefdb..c94f15c75 100644 --- a/tests/checkdb_test.py +++ b/tests/checkdb_test.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb from datetime import datetime, timedelta import subprocess from testgres import QueryException @@ -12,9 +13,9 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") + @needs_gdb def test_checkdb_amcheck_only_sanity(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -34,7 +35,7 @@ def test_checkdb_amcheck_only_sanity(self): node.safe_psql( "postgres", "create index on t_heap(id)") - + node.safe_psql( "postgres", "create table idxpart (a int) " @@ -537,16 +538,21 @@ def test_checkdb_checkunique(self): repr(e.message), self.cmd)) self.assertIn( - "Amcheck failed in database 'postgres' for index: 'public.bttest_unique_idx': ERROR: index \"bttest_unique_idx\" is corrupted. There are tuples violating UNIQUE constraint", + "Amcheck failed in database 'postgres' for index: 'public.bttest_unique_idx'", e.message) + self.assertRegex( + e.message, + r"ERROR:[^\n]*(violating UNIQUE constraint|uniqueness is violated)" + ) + # Clean after yourself node.stop() # @unittest.skip("skip") + @needs_gdb def test_checkdb_sigint_handling(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -637,66 +643,8 @@ def test_checkdb_with_least_privileges(self): "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC;") - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') # amcheck-next function - - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' -# 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - # PG 10 - elif self.get_version(node) > 100000 and self.get_version(node) < 110000: + if self.get_version(node) < 110000: node.safe_psql( 'backupdb', 'CREATE ROLE backup WITH LOGIN; ' @@ -722,18 +670,13 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;') - + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;' + ) if ProbackupTest.enterprise: # amcheck-1.1 node.safe_psql( 'backupdb', 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup') - else: - # amcheck-1.0 - node.safe_psql( - 'backupdb', - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup') # >= 11 < 14 elif self.get_version(node) > 110000 and self.get_version(node) < 140000: node.safe_psql( @@ -762,8 +705,8 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + ) # checkunique parameter if ProbackupTest.enterprise: if (self.get_version(node) >= 111300 and self.get_version(node) < 120000 @@ -800,8 +743,8 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + ) # checkunique parameter if ProbackupTest.enterprise: node.safe_psql( @@ -810,9 +753,9 @@ def test_checkdb_with_least_privileges(self): if ProbackupTest.enterprise: node.safe_psql( - 'backupdb', - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # checkdb try: diff --git a/tests/compatibility_test.py b/tests/compatibility_test.py index 591afb069..e94ba6b23 100644 --- a/tests/compatibility_test.py +++ b/tests/compatibility_test.py @@ -1498,3 +1498,46 @@ def test_compatibility_tablespace(self): if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_compatibility_master_options(self): + """ + Test correctness of handling of removed master-db, master-host, master-port, + master-user and replica-timeout options + """ + self.assertTrue( + self.version_to_num(self.old_probackup_version) <= self.version_to_num('2.6.0'), + 'You need pg_probackup old_binary =< 2.6.0 for this test') + + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + # add deprecated options (using probackup< 2.6) into pg_probackup.conf + # don't care about option values, we can use random values here + self.set_config( + backup_dir, 'node', + options=[ + '--master-db=postgres', + '--master-host=localhost', + '--master-port=5432', + '--master-user={0}'.format(self.user), + '--replica-timeout=100500'], + old_binary=True) + + # and try to show config with new binary (those options must be silently skipped) + self.show_config(backup_dir, 'node', old_binary=False) + + # store config with new version (those options must disappear from config) + self.set_config( + backup_dir, 'node', + options=[], + old_binary=False) + + # and check absence + config_options = self.show_config(backup_dir, 'node', old_binary=False) + self.assertFalse( + ['master-db', 'master-host', 'master-port', 'master-user', 'replica-timeout'] & config_options.keys(), + 'Obsolete options found in new config') \ No newline at end of file diff --git a/tests/config_test.py b/tests/config_test.py index b1a0f9295..595b12ef1 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -34,21 +34,9 @@ def test_remove_instance_config(self): os.unlink(os.path.join(backup_dir, 'backups','node', 'pg_probackup.conf')) - try: + with self.assertRaisesRegex(ProbackupException, r'ERROR: Reading instance control.*No such file'): self.backup_node( backup_dir, 'node', node, backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because pg_probackup.conf is missing. " - ".\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: could not open file "{0}": ' - 'No such file or directory'.format(conf_file), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) # @unittest.expectedFailure # @unittest.skip("skip") diff --git a/tests/delta_test.py b/tests/delta_test.py index 8736a079c..5e02c96e8 100644 --- a/tests/delta_test.py +++ b/tests/delta_test.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb from datetime import datetime, timedelta from testgres import QueryException import subprocess @@ -438,12 +439,12 @@ def test_delta_multiple_segments(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_delta_vacuum_full(self): """ make node, make full and delta stream backups, restore them and check data correctness """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index e0d6924b9..f3ebaaa25 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.11 +pg_probackup 2.6.0 diff --git a/tests/external_test.py b/tests/external_test.py index 53f3c5449..73ff0e0fb 100644 --- a/tests/external_test.py +++ b/tests/external_test.py @@ -2,7 +2,7 @@ import os from time import sleep from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from .helpers.cfs_helpers import find_by_name +from .helpers.data_helpers import find_by_name import shutil diff --git a/tests/false_positive_test.py b/tests/false_positive_test.py index fbb785c60..c8b6cd7ff 100644 --- a/tests/false_positive_test.py +++ b/tests/false_positive_test.py @@ -104,9 +104,6 @@ def test_pg_10_waldir(self): """ test group access for PG >= 11 """ - if self.pg_config_version < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - wal_dir = os.path.join( os.path.join(self.tmp_path, self.module_name, self.fname), 'wal_dir') import shutil @@ -203,13 +200,16 @@ def test_recovery_target_time_backup_victim(self): backup_dir, 'node', options=['--recovery-target-time={0}'.format(target_time)]) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") def test_recovery_target_lsn_backup_victim(self): """ Check that for validation to recovery target probackup chooses valid backup https://github.com/postgrespro/pg_probackup/issues/104 + + @y.sokolov: looks like this test should pass. + So I commented 'expectedFailure' """ backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index 4ae3ef8c4..e94e6366b 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,4 +1,4 @@ -__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] +__all__ = ['ptrack_helpers', 'data_helpers', 'expected_errors'] import unittest diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/data_helpers.py similarity index 100% rename from tests/helpers/cfs_helpers.py rename to tests/helpers/data_helpers.py diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 067225d66..f5b1903cc 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1,21 +1,21 @@ # you need os for unittest to work import os -import gc +import sys +import threading import unittest -from sys import exit, argv, version_info +from sys import exit, argv import signal import subprocess import shutil -import six import testgres import hashlib -import re import getpass -import select from time import sleep +from time import time import re import json import random +import contextlib idx_ptrack = { 't_heap': { @@ -132,6 +132,23 @@ def base36enc(number): return sign + base36 +def tail_file(file, linetimeout=10, totaltimeout = 60): + start = time() + with open(file, 'r') as f: + waits = 0 + while waits < linetimeout: + line = f.readline() + if line == '': + waits += 1 + sleep(1) + continue + waits = 0 + yield line + if time() - start > totaltimeout: + raise TimeoutError("total timeout tailing %s"%(file,)) + else: + return # ok + raise TimeoutError("line timeout tailing %s"%(file,)) class ProbackupException(Exception): def __init__(self, message, cmd): @@ -141,6 +158,7 @@ def __init__(self, message, cmd): def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) + class PostgresNodeExtended(testgres.PostgresNode): def __init__(self, base_dir=None, *args, **kwargs): @@ -225,6 +243,7 @@ def table_checksum(self, table, dbname="postgres"): con.close() return sum.hexdigest() + class ProbackupTest(object): # Class attributes enterprise = is_enterprise() @@ -236,8 +255,13 @@ def __init__(self, *args, **kwargs): self.nodes_to_cleanup = [] if isinstance(self, unittest.TestCase): - self.module_name = self.id().split('.')[1] - self.fname = self.id().split('.')[3] + try: + self.module_name = self.id().split('.')[-2] + self.fname = self.id().split('.')[-1] + except IndexError: + print("Couldn't get module name and function name from self.id(): `{}`".format(self.id())) + self.module_name = self.module_name if self.module_name else str(self).split('(')[1].split('.')[1] + self.fname = str(self).split('(')[0] if '-v' in argv or '--verbose' in argv: self.verbose = True @@ -269,8 +293,7 @@ def __init__(self, *args, **kwargs): self.test_env['LC_MESSAGES'] = 'C' self.test_env['LC_TIME'] = 'C' - self.gdb = 'PGPROBACKUP_GDB' in self.test_env and \ - self.test_env['PGPROBACKUP_GDB'] == 'ON' + self._set_gdb() self.paranoia = 'PG_PROBACKUP_PARANOIA' in self.test_env and \ self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON' @@ -399,6 +422,20 @@ def __init__(self, *args, **kwargs): os.environ["PGAPPNAME"] = "pg_probackup" + def _set_gdb(self): + self._gdb_enabled = self.test_env.get('PGPROBACKUP_GDB') == 'ON' + self._gdb_ok = self._gdb_enabled + if not self._gdb_enabled or sys.platform != 'linux': + return + try: + with open('/proc/sys/kernel/yama/ptrace_scope') as f: + ptrace = f.read() + except FileNotFoundError: + self._gdb_ptrace_ok = True + return + self._gdb_ptrace_ok = int(ptrace) == 0 + self._gdb_ok = self._gdb_ok and self._gdb_ptrace_ok + def is_test_result_ok(test_case): # sources of solution: # 1. python versions 2.7 - 3.10, verified on 3.10, 3.7, 2.7, taken from: @@ -520,7 +557,7 @@ def make_simple_node( if node.major_version >= 13: options['wal_keep_size'] = '200MB' else: - options['wal_keep_segments'] = '100' + options['wal_keep_segments'] = '12' # set default values self.set_auto_conf(node, options) @@ -543,38 +580,8 @@ def simple_bootstrap(self, node, role) -> None: 'postgres', 'CREATE ROLE {0} WITH LOGIN REPLICATION'.format(role)) - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'postgres', - 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0};'.format(role)) - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'postgres', - 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.get_version(node) < 150000: node.safe_psql( 'postgres', 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' @@ -720,13 +727,7 @@ def get_md5_per_page_for_fork(self, file, size_in_pages): def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): - if self.get_pgpro_edition(node) == 'enterprise': - if self.get_version(node) < self.version_to_num('10.0'): - header_size = 48 - else: - header_size = 24 - else: - header_size = 24 + header_size = 24 ptrack_bits_for_fork = [] # TODO: use macro instead of hard coded 8KB @@ -1002,9 +1003,10 @@ def run_binary(self, command, asynchronous=False, env=None): except subprocess.CalledProcessError as e: raise ProbackupException(e.output.decode('utf-8'), command) - def init_pb(self, backup_dir, options=[], old_binary=False): + def init_pb(self, backup_dir, options=[], old_binary=False, cleanup=True): - shutil.rmtree(backup_dir, ignore_errors=True) + if cleanup: + shutil.rmtree(backup_dir, ignore_errors=True) # don`t forget to kill old_binary after remote ssh release if self.remote and not old_binary: @@ -1688,29 +1690,33 @@ def version_to_num(self, version): num = num * 100 + int(re.sub(r"[^\d]", "", part)) return num - def switch_wal_segment(self, node): + def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): """ - Execute pg_switch_wal/xlog() in given node + Execute pg_switch_wal() in given node Args: node: an instance of PostgresNode or NodeConnection class """ if isinstance(node, testgres.PostgresNode): - if self.version_to_num( - node.safe_psql('postgres', 'show server_version').decode('utf-8') - ) >= self.version_to_num('10.0'): - node.safe_psql('postgres', 'select pg_switch_wal()') - else: - node.safe_psql('postgres', 'select pg_switch_xlog()') + with node.connect('postgres') as con: + if and_tx: + con.execute('select txid_current()') + con.execute('select pg_switch_wal()') else: - if self.version_to_num( - node.execute('show server_version')[0][0] - ) >= self.version_to_num('10.0'): - node.execute('select pg_switch_wal()') - else: - node.execute('select pg_switch_xlog()') + node.execute('select pg_switch_wal()') - sleep(1) + if sleep_seconds > 0: + sleep(sleep_seconds) + + @contextlib.contextmanager + def switch_wal_after(self, node, seconds, and_tx=True): + tm = threading.Timer(seconds, self.switch_wal_segment, [node, 0, and_tx]) + tm.start() + try: + yield + finally: + tm.cancel() + tm.join() def wait_until_replica_catch_with_master(self, master, replica): @@ -1718,12 +1724,8 @@ def wait_until_replica_catch_with_master(self, master, replica): 'postgres', 'show server_version').decode('utf-8').rstrip() - if self.version_to_num(version) >= self.version_to_num('10.0'): - master_function = 'pg_catalog.pg_current_wal_lsn()' - replica_function = 'pg_catalog.pg_last_wal_replay_lsn()' - else: - master_function = 'pg_catalog.pg_current_xlog_location()' - replica_function = 'pg_catalog.pg_last_xlog_replay_location()' + master_function = 'pg_catalog.pg_current_wal_lsn()' + replica_function = 'pg_catalog.pg_last_wal_replay_lsn()' lsn = master.safe_psql( 'postgres', @@ -1991,12 +1993,26 @@ def gdb_attach(self, pid): return GDBobj([str(pid)], self, attach=True) def _check_gdb_flag_or_skip_test(self): - if not self.gdb: + if not self._gdb_enabled: self.skipTest( "Specify PGPROBACKUP_GDB and build without " "optimizations for run this test" ) + if self._gdb_ok: + return + if not self._gdb_ptrace_ok: + self.fail("set /proc/sys/kernel/yama/ptrace_scope to 0" + " to run GDB tests") + else: + self.fail("use of gdb is not possible") +def needs_gdb(func): + def wrapped(self): + self._gdb_decorated = True + self._check_gdb_flag_or_skip_test() + func(self) + wrapped.__doc__ = func.__doc__ + return wrapped class GdbException(Exception): def __init__(self, message="False"): @@ -2010,12 +2026,16 @@ class GDBobj: def __init__(self, cmd, env, attach=False): self.verbose = env.verbose self.output = '' + self._did_quit = False # Check gdb flag is set up - if not env.gdb: - raise GdbException("No `PGPROBACKUP_GDB=on` is set, " - "test should call ProbackupTest::check_gdb_flag_or_skip_test() on its start " - "and be skipped") + if not getattr(env, "_gdb_decorated", False): + raise GdbException("Test should be marked with @needs_gdb") + if not env._gdb_enabled: + raise GdbException("No `PGPROBACKUP_GDB=on` is set.") + if not env._gdb_ok: + raise GdbException("No gdb usage possible.") + # Check gdb presense try: gdb_version, _ = subprocess.Popen( @@ -2068,14 +2088,24 @@ def __init__(self, cmd, env, attach=False): else: break + def __del__(self): + if not self._did_quit: + try: + self.quit() + except subprocess.TimeoutExpired: + self.kill() + def get_line(self): line = self.proc.stdout.readline() self.output += line return line def kill(self): + self._did_quit = True self.proc.kill() - self.proc.wait() + self.proc.wait(3) + self.proc.stdin.close() + self.proc.stdout.close() def set_breakpoint(self, location): @@ -2203,7 +2233,12 @@ def stopped_in_breakpoint(self): return False def quit(self): - self.proc.terminate() + if not self._did_quit: + self._did_quit = True + self.proc.terminate() + self.proc.wait(3) + self.proc.stdin.close() + self.proc.stdout.close() # use for breakpoint, run, continue def _execute(self, cmd, running=True): diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py index 613e4dd36..49316a77f 100644 --- a/tests/incr_restore_test.py +++ b/tests/incr_restore_test.py @@ -1399,10 +1399,6 @@ def test_make_replica_via_incr_checksum_restore(self): set_replication=True, initdb_params=['--data-checksums']) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', master) self.set_archiving(backup_dir, 'node', master, replica=True) @@ -1467,10 +1463,6 @@ def test_make_replica_via_incr_lsn_restore(self): set_replication=True, initdb_params=['--data-checksums']) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', master) self.set_archiving(backup_dir, 'node', master, replica=True) diff --git a/tests/init_test.py b/tests/init_test.py index 94b076fef..bbbcdf97c 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -1,11 +1,14 @@ import os +import stat import unittest -from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException import shutil +from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException -class InitTest(ProbackupTest, unittest.TestCase): +DIR_PERMISSION = 0o700 if os.name != 'nt' else 0o777 +CATALOG_DIRS = ['backups', 'wal'] +class InitTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_success(self): @@ -15,8 +18,13 @@ def test_success(self): self.init_pb(backup_dir) self.assertEqual( dir_files(backup_dir), - ['backups', 'wal'] + CATALOG_DIRS ) + + for subdir in CATALOG_DIRS: + dirname = os.path.join(backup_dir, subdir) + self.assertEqual(DIR_PERMISSION, stat.S_IMODE(os.stat(dirname).st_mode)) + self.add_instance(backup_dir, 'node', node) self.assertIn( "INFO: Instance 'node' successfully deleted", @@ -107,32 +115,60 @@ def test_add_instance_idempotence(self): dir_backups = os.path.join(backup_dir, 'backups', 'node') dir_wal = os.path.join(backup_dir, 'wal', 'node') - try: + with open(os.path.join(dir_wal, "0000"), 'w'): + pass + + with self.assertRaisesRegex(ProbackupException, r"'node'.*WAL.*already exists"): self.add_instance(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' WAL archive directory already exists: ", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - try: + with self.assertRaisesRegex(ProbackupException, r"'node'.*WAL.*already exists"): self.add_instance(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' WAL archive directory already exists: ", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + + def test_init_backup_catalog_no_access(self): + """ Test pg_probackup init -B backup_dir to a dir with no read access. """ + no_access_dir = os.path.join(self.tmp_path, self.module_name, self.fname, + 'noaccess') + backup_dir = os.path.join(no_access_dir, 'backup') + os.makedirs(no_access_dir) + os.chmod(no_access_dir, stat.S_IREAD) + + expected = 'ERROR: cannot open backup catalog directory.*{0}.*Permission denied'.format(backup_dir) + with self.assertRaisesRegex(ProbackupException, expected): + self.init_pb(backup_dir) + + def test_init_backup_catalog_no_write(self): + """ Test pg_probackup init -B backup_dir to a dir with no write access. """ + no_access_dir = os.path.join(self.tmp_path, self.module_name, self.fname, + 'noaccess') + backup_dir = os.path.join(no_access_dir, 'backup') + os.makedirs(no_access_dir) + os.chmod(no_access_dir, stat.S_IREAD|stat.S_IEXEC) + + expected = 'ERROR: Can not create backup catalog root directory: Cannot make dir "{0}": Permission denied'.format(backup_dir) + with self.assertRaisesRegex(ProbackupException, expected): + self.init_pb(backup_dir) + + def test_init_backup_catalog_no_create(self): + """ Test pg_probackup init -B backup_dir to a dir when backup dir exists but not writeable. """ + parent_dir = os.path.join(self.tmp_path, self.module_name, self.fname, + 'parent') + backup_dir = os.path.join(parent_dir, 'backup') + os.makedirs(backup_dir) + os.chmod(backup_dir, stat.S_IREAD|stat.S_IEXEC) + + backups_dir = os.path.join(backup_dir, 'backups') + expected = 'ERROR: Can not create backup catalog data directory: Cannot make dir "{0}": Permission denied'.format(backups_dir) + with self.assertRaisesRegex(ProbackupException, expected): + self.init_pb(backup_dir, cleanup=False) + + def test_init_backup_catalog_exists_not_empty(self): + """ Test pg_probackup init -B backup_dir which exists and not empty. """ + parent_dir = os.path.join(self.tmp_path, self.module_name, self.fname, + 'parent') + backup_dir = os.path.join(parent_dir, 'backup') + os.makedirs(backup_dir) + with open(os.path.join(backup_dir, 'somefile.txt'), 'wb'): + pass + + with self.assertRaisesRegex(ProbackupException, "ERROR: backup catalog already exist and it's not empty"): + self.init_pb(backup_dir, cleanup=False) diff --git a/tests/locking_test.py b/tests/locking_test.py index 5367c2610..f2740a6e6 100644 --- a/tests/locking_test.py +++ b/tests/locking_test.py @@ -2,19 +2,20 @@ import os from time import sleep from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb class LockingTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure + @needs_gdb def test_locking_running_validate_1(self): """ make node, take full backup, stop it in the middle run validate, expect it to successfully executed, concurrent RUNNING backup with pid file and active process is legal """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -61,6 +62,7 @@ def test_locking_running_validate_1(self): # Clean after yourself gdb.kill() + @needs_gdb def test_locking_running_validate_2(self): """ make node, take full backup, stop it in the middle, @@ -69,7 +71,6 @@ def test_locking_running_validate_2(self): RUNNING backup with pid file AND without active pid is legal, but his status must be changed to ERROR and pid file is deleted """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -130,6 +131,7 @@ def test_locking_running_validate_2(self): # Clean after yourself gdb.kill() + @needs_gdb def test_locking_running_validate_2_specific_id(self): """ make node, take full backup, stop it in the middle, @@ -139,7 +141,6 @@ def test_locking_running_validate_2_specific_id(self): RUNNING backup with pid file AND without active pid is legal, but his status must be changed to ERROR and pid file is deleted """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -229,6 +230,7 @@ def test_locking_running_validate_2_specific_id(self): # Clean after yourself gdb.kill() + @needs_gdb def test_locking_running_3(self): """ make node, take full backup, stop it in the middle, @@ -237,7 +239,6 @@ def test_locking_running_3(self): RUNNING backup without pid file AND without active pid is legal, his status must be changed to ERROR """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -299,6 +300,7 @@ def test_locking_running_3(self): # Clean after yourself gdb.kill() + @needs_gdb def test_locking_restore_locked(self): """ make node, take full backup, take two page backups, @@ -307,7 +309,6 @@ def test_locking_restore_locked(self): Expect restore to sucseed because read-only locks do not conflict """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -341,6 +342,7 @@ def test_locking_restore_locked(self): # Clean after yourself gdb.kill() + @needs_gdb def test_concurrent_delete_and_restore(self): """ make node, take full backup, take page backup, @@ -349,7 +351,6 @@ def test_concurrent_delete_and_restore(self): Expect restore to fail because validation of intermediate backup is impossible """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -398,13 +399,13 @@ def test_concurrent_delete_and_restore(self): # Clean after yourself gdb.kill() + @needs_gdb def test_locking_concurrent_validate_and_backup(self): """ make node, take full backup, launch validate and stop it in the middle, take page backup. Expect PAGE backup to be successfully executed """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -434,13 +435,13 @@ def test_locking_concurrent_validate_and_backup(self): # Clean after yourself gdb.kill() + @needs_gdb def test_locking_concurren_restore_and_delete(self): """ make node, take full backup, launch restore and stop it in the middle, delete full backup. Expect it to fail. """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -572,11 +573,11 @@ def test_empty_lock_file(self): # p1.wait() # p2.wait() + @needs_gdb def test_shared_lock(self): """ Make sure that shared lock leaves no files with pids """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), diff --git a/tests/logging_test.py b/tests/logging_test.py index c5cdfa344..998c92797 100644 --- a/tests/logging_test.py +++ b/tests/logging_test.py @@ -1,6 +1,7 @@ import unittest import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb import datetime class LogTest(ProbackupTest, unittest.TestCase): @@ -8,10 +9,10 @@ class LogTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure # PGPRO-2154 + @needs_gdb def test_log_rotation(self): """ """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), diff --git a/tests/merge_test.py b/tests/merge_test.py index c789298fd..fddaeb6a3 100644 --- a/tests/merge_test.py +++ b/tests/merge_test.py @@ -3,6 +3,7 @@ import unittest import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb from testgres import QueryException import shutil from datetime import datetime, timedelta @@ -914,11 +915,11 @@ def test_merge_delta_delete(self): node_restored.slow_start() # @unittest.skip("skip") + @needs_gdb def test_continue_failed_merge(self): """ Check that failed MERGE can be continued """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -988,11 +989,11 @@ def test_continue_failed_merge(self): self.restore_node(backup_dir, 'node', node) # @unittest.skip("skip") + @needs_gdb def test_continue_failed_merge_with_corrupted_delta_backup(self): """ Fail merge via gdb, corrupt DELTA backup, try to continue merge """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1083,11 +1084,11 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + @needs_gdb def test_continue_failed_merge_2(self): """ Check that failed MERGE on delete can be continued """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1131,11 +1132,15 @@ def test_continue_failed_merge_2(self): gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_remove') + gdb.set_breakpoint('pioRemove__do') gdb.run_until_break() gdb._execute('thread apply all bt') + gdb.remove_all_breakpoints() + + gdb.set_breakpoint('pioRemoveDir__do') gdb.continue_execution_until_break(20) @@ -1152,12 +1157,12 @@ def test_continue_failed_merge_2(self): # Try to continue failed MERGE self.merge_backup(backup_dir, "node", backup_id) + @needs_gdb def test_continue_failed_merge_3(self): """ Check that failed MERGE cannot be continued if intermediate backup is missing. """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1334,12 +1339,12 @@ def test_merge_different_wal_modes(self): self.assertEqual( 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) + @needs_gdb def test_crash_after_opening_backup_control_1(self): """ check that crashing after opening backup.control for writing will not result in losing backup metadata """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1384,13 +1389,13 @@ def test_crash_after_opening_backup_control_1(self): 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) # @unittest.skip("skip") + @needs_gdb def test_crash_after_opening_backup_control_2(self): """ check that crashing after opening backup_content.control for writing will not result in losing metadata about backup files TODO: rewrite """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1474,13 +1479,13 @@ def test_crash_after_opening_backup_control_2(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_losing_file_after_failed_merge(self): """ check that crashing after opening backup_content.control for writing will not result in losing metadata about backup files TODO: rewrite """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1563,10 +1568,10 @@ def test_losing_file_after_failed_merge(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + @needs_gdb def test_failed_merge_after_delete(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1611,7 +1616,8 @@ def test_failed_merge_after_delete(self): gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_remove') + gdb.set_breakpoint('pioRemove__do') gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') @@ -1643,10 +1649,10 @@ def test_failed_merge_after_delete(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + @needs_gdb def test_failed_merge_after_delete_1(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1694,7 +1700,8 @@ def test_failed_merge_after_delete_1(self): # gdb.set_breakpoint('parray_bsearch') # gdb.continue_execution_until_break() - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_remove') + gdb.set_breakpoint('pioRemove__do') gdb.continue_execution_until_break(30) gdb._execute('signal SIGKILL') @@ -1718,10 +1725,10 @@ def test_failed_merge_after_delete_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + @needs_gdb def test_failed_merge_after_delete_2(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1755,7 +1762,10 @@ def test_failed_merge_after_delete_2(self): backup_dir, 'node', page_2, gdb=True, options=['--log-level-console=VERBOSE']) - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + gdb.set_breakpoint('fio_remove') + gdb.set_breakpoint('pioRemove__do') gdb.run_until_break() gdb.continue_execution_until_break(2) gdb._execute('signal SIGKILL') @@ -1779,10 +1789,10 @@ def test_failed_merge_after_delete_2(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + @needs_gdb def test_failed_merge_after_delete_3(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1832,7 +1842,8 @@ def test_failed_merge_after_delete_3(self): gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_remove') + gdb.set_breakpoint('pioRemove__do') gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') @@ -2189,10 +2200,10 @@ def test_smart_merge(self): with open(logfile, 'r') as f: logfile_content = f.read() + @needs_gdb def test_idempotent_merge(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -2473,12 +2484,12 @@ def test_multi_timeline_merge(self): # @unittest.skip("skip") # @unittest.expectedFailure + @needs_gdb def test_merge_page_header_map_retry(self): """ page header map cannot be trusted when running retry """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -2519,10 +2530,10 @@ def test_merge_page_header_map_retry(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_missing_data_file(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -2572,14 +2583,14 @@ def test_missing_data_file(self): logfile_content = f.read() self.assertIn( - 'ERROR: Cannot open backup file "{0}": No such file or directory'.format(file_to_remove), + 'ERROR: Open backup file: Cannot open file "{0}": No such file or directory'.format(file_to_remove), logfile_content) # @unittest.skip("skip") + @needs_gdb def test_missing_non_data_file(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -2632,10 +2643,10 @@ def test_missing_non_data_file(self): 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) # @unittest.skip("skip") + @needs_gdb def test_merge_remote_mode(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( diff --git a/tests/option_test.py b/tests/option_test.py index eec1bab44..1a6baf48a 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -3,7 +3,6 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import locale - class OptionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -136,9 +135,9 @@ def test_options_5(self): self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - 'ERROR: Syntax error in " = INFINITE', + self.assertRegex( e.message, + 'ERROR: Syntax error .* INFINITE', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.clean_pb(backup_dir) @@ -220,12 +219,29 @@ def test_options_5(self): def test_help_6(self): """help options""" if ProbackupTest.enable_nls: - self.test_env['LC_ALL'] = 'ru_RU.utf-8' - with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: - self.assertEqual( - self.run_pb(["--help"]), - help_out.read().decode("utf-8") - ) + if check_locale('ru_RU.utf-8'): + env = self.test_env.copy() + env['LC_MESSAGES'] = 'ru_RU.utf-8' + with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"], env=env), + help_out.read().decode("utf-8") + ) + else: + self.skipTest( + "Locale ru_RU.utf-8 doesn't work. You need install ru_RU.utf-8 locale for this test") else: self.skipTest( 'You need configure PostgreSQL with --enabled-nls option for this test') + + +def check_locale(locale_name): + ret=True + old_locale = locale.setlocale(locale.LC_CTYPE,"") + try: + locale.setlocale(locale.LC_CTYPE, locale_name) + except locale.Error: + ret=False + finally: + locale.setlocale(locale.LC_CTYPE, old_locale) + return ret diff --git a/tests/pgpro2068_test.py b/tests/pgpro2068_test.py index 04f0eb6fa..4582d419b 100644 --- a/tests/pgpro2068_test.py +++ b/tests/pgpro2068_test.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from .helpers.ptrack_helpers import needs_gdb from datetime import datetime, timedelta import subprocess from time import sleep @@ -11,11 +12,11 @@ class BugTest(ProbackupTest, unittest.TestCase): + @needs_gdb def test_minrecpoint_on_replica(self): """ https://jira.postgrespro.ru/browse/PGPRO-2068 """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -126,10 +127,6 @@ def test_minrecpoint_on_replica(self): recovery_config, "recovery_target_action = 'pause'") replica.slow_start(replica=True) - current_xlog_lsn_query = 'SELECT pg_last_wal_replay_lsn() INTO current_xlog_lsn' - if self.get_version(node) < 100000: - current_xlog_lsn_query = 'SELECT min_recovery_end_location INTO current_xlog_lsn FROM pg_control_recovery()' - script = f''' DO $$ @@ -139,7 +136,7 @@ def test_minrecpoint_on_replica(self): pages_from_future RECORD; found_corruption bool := false; BEGIN - {current_xlog_lsn_query}; + SELECT pg_last_wal_replay_lsn() INTO current_xlog_lsn; RAISE NOTICE 'CURRENT LSN: %', current_xlog_lsn; FOR roid IN select oid from pg_class class where relkind IN ('r', 'i', 't', 'm') and relpersistence = 'p' LOOP FOR pages_from_future IN @@ -156,7 +153,7 @@ def test_minrecpoint_on_replica(self): END IF; END; $$ LANGUAGE plpgsql; -'''.format(current_xlog_lsn_query=current_xlog_lsn_query) +''' # Find blocks from future replica.safe_psql( diff --git a/tests/pgpro560_test.py b/tests/pgpro560_test.py index b665fd200..cd2920a51 100644 --- a/tests/pgpro560_test.py +++ b/tests/pgpro560_test.py @@ -31,19 +31,9 @@ def test_pgpro560_control_file_loss(self): # Not delete this file permanently os.rename(file, os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy')) - try: + with self.assertRaisesRegex(ProbackupException, + r'ERROR: Getting system identifier:.*pg_control'): self.backup_node(backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because pg_control was deleted.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Could not open file' in e.message and - 'pg_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) # Return this file to avoid Postger fail os.rename(os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy'), file) @@ -80,20 +70,12 @@ def test_pgpro560_systemid_mismatch(self): "Expecting Error because of SYSTEM ID mismatch.\n " "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) except ProbackupException as e: - if self.get_version(node1) > 90600: - self.assertTrue( - 'ERROR: Backup data directory was ' - 'initialized for system id' in e.message and - 'but connected instance system id is' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - 'ERROR: System identifier mismatch. ' - 'Connected PostgreSQL instance has system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup data directory was ' + 'initialized for system id' in e.message and + 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) sleep(1) @@ -107,17 +89,9 @@ def test_pgpro560_systemid_mismatch(self): "Expecting Error because of of SYSTEM ID mismatch.\n " "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) except ProbackupException as e: - if self.get_version(node1) > 90600: - self.assertTrue( - 'ERROR: Backup data directory was initialized ' - 'for system id' in e.message and - 'but connected instance system id is' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - 'ERROR: System identifier mismatch. ' - 'Connected PostgreSQL instance has system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup data directory was initialized ' + 'for system id' in e.message and + 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py index 7b5bc416b..4597954f1 100644 --- a/tests/ptrack_test.py +++ b/tests/ptrack_test.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from .helpers.ptrack_helpers import needs_gdb from datetime import datetime, timedelta import subprocess from testgres import QueryException, StartNodeException @@ -14,14 +15,13 @@ class PtrackTest(ProbackupTest, unittest.TestCase): def setUp(self): if self.pg_config_version < self.version_to_num('11.0'): self.skipTest('You need PostgreSQL >= 11 for this test') - self.fname = self.id().split('.')[3] # @unittest.skip("skip") + @needs_gdb def test_drop_rel_during_backup_ptrack(self): """ drop relation during ptrack backup """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -484,79 +484,8 @@ def test_ptrack_unprivileged(self): "postgres", "CREATE DATABASE backupdb") - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + # PG < 15 + if self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -817,10 +746,10 @@ def test_ptrack_uncommitted_xact(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") - def test_ptrack_vacuum_full(self): + @needs_gdb + def test_ptrack_vacuum_full_1(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -981,12 +910,12 @@ def test_ptrack_vacuum_truncate(self): node_restored.slow_start() # @unittest.skip("skip") + @needs_gdb def test_ptrack_get_block(self): """ make node, make full and ptrack stream backups, restore them and check data correctness """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -1601,13 +1530,7 @@ def test_create_db_on_replica(self): self.backup_node( backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(node.port), - '--stream' - ] + options=['-j10', '--stream'] ) # CREATE DATABASE DB1 @@ -1625,13 +1548,7 @@ def test_create_db_on_replica(self): backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(node.port) - ] + options=['-j10', '--stream'] ) if self.paranoia: @@ -2316,11 +2233,7 @@ def test_ptrack_clean_replica(self): backup_dir, 'replica', replica, - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['-j10', '--stream']) master.safe_psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -2347,11 +2260,7 @@ def test_ptrack_clean_replica(self): 'replica', replica, backup_type='ptrack', - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['-j10', '--stream']) master.safe_psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -2379,11 +2288,7 @@ def test_ptrack_clean_replica(self): 'replica', replica, backup_type='page', - options=[ - '-j10', '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + options=['-j10', '--stream']) master.safe_psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -2447,8 +2352,7 @@ def test_ptrack_cluster_on_btree(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_btree') @@ -2577,11 +2481,7 @@ def test_ptrack_cluster_on_btree_replica(self): master.safe_psql('postgres', 'vacuum t_heap') master.safe_psql('postgres', 'checkpoint') - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--stream', '--master-host=localhost', - '--master-db=postgres', '--master-port={0}'.format( - master.port)]) + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) for i in idx_ptrack: # get size of heap and indexes. size calculated in pages @@ -2675,9 +2575,7 @@ def test_ptrack_cluster_on_gist_replica(self): self.backup_node( backup_dir, 'replica', replica, options=[ - '-j10', '--stream', '--master-host=localhost', - '--master-db=postgres', '--master-port={0}'.format( - master.port)]) + '-j10', '--stream']) for i in idx_ptrack: # get size of heap and indexes. size calculated in pages @@ -2839,11 +2737,7 @@ def test_ptrack_empty_replica(self): backup_dir, 'replica', replica, - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['-j10', '--stream']) # Create indexes for i in idx_ptrack: @@ -2863,11 +2757,7 @@ def test_ptrack_empty_replica(self): 'replica', replica, backup_type='ptrack', - options=[ - '-j1', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['-j1', '--stream']) if self.paranoia: pgdata = self.pgdata_content(replica.data_dir) @@ -3030,12 +2920,7 @@ def test_basic_ptrack_truncate_replica(self): # Make backup to clean every ptrack self.backup_node( backup_dir, 'replica', replica, - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['-j10', '--stream']) if replica.major_version < 11: for i in idx_ptrack: @@ -3059,12 +2944,7 @@ def test_basic_ptrack_truncate_replica(self): self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['-j10', '--stream']) pgdata = self.pgdata_content(replica.data_dir) @@ -3228,12 +3108,7 @@ def test_ptrack_vacuum_replica(self): replica.safe_psql('postgres', 'checkpoint') # Make FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) if replica.major_version < 11: for i in idx_ptrack: @@ -3407,12 +3282,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): # Take backup to clean every ptrack self.backup_node( backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + options=['-j10', '--stream']) if replica.major_version < 11: for i in idx_ptrack: @@ -3656,12 +3526,7 @@ def test_ptrack_vacuum_full_replica(self): # Take FULL backup to clean every ptrack self.backup_node( backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + options=['-j10', '--stream']) if replica.major_version < 11: for i in idx_ptrack: @@ -3822,13 +3687,7 @@ def test_ptrack_vacuum_truncate_replica(self): # Take FULL backup to clean every ptrack self.backup_node( backup_dir, 'replica', replica, - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port) - ] + options=['-j10', '--stream'] ) if master.major_version < 11: diff --git a/tests/remote_test.py b/tests/remote_test.py index 2d36d7346..0d9894f65 100644 --- a/tests/remote_test.py +++ b/tests/remote_test.py @@ -2,7 +2,6 @@ import os from time import sleep from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from .helpers.cfs_helpers import find_by_name class RemoteTest(ProbackupTest, unittest.TestCase): diff --git a/tests/replica_test.py b/tests/replica_test.py index 17fc5a823..dc70917fa 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -1,6 +1,8 @@ import os +import threading import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from .helpers.ptrack_helpers import needs_gdb from datetime import datetime, timedelta import subprocess import time @@ -25,10 +27,6 @@ def test_replica_switchover(self): set_replication=True, initdb_params=['--data-checksums']) - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) @@ -98,10 +96,6 @@ def test_replica_stream_ptrack_backup(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - if self.pg_config_version > self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'master'), @@ -153,11 +147,7 @@ def test_replica_stream_ptrack_backup(self): backup_id = self.backup_node( backup_dir, 'replica', replica, - options=[ - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['--stream']) self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -189,11 +179,7 @@ def test_replica_stream_ptrack_backup(self): backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', - options=[ - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['--stream']) self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -227,10 +213,6 @@ def test_replica_archive_page_backup(self): 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -278,13 +260,12 @@ def test_replica_archive_page_backup(self): self.wait_until_replica_catch_with_master(master, replica) + tm = threading.Timer(5, call_repeat, [1000, master.execute, 'select txid_current()']) + tm.start() backup_id = self.backup_node( backup_dir, 'replica', replica, - options=[ - '--archive-timeout=60', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['--archive-timeout=60']) + tm.join() self.validate_pb(backup_dir, 'replica') self.assertEqual( @@ -311,16 +292,12 @@ def test_replica_archive_page_backup(self): master.pgbench_init(scale=5) pgbench = master.pgbench( - options=['-T', '30', '-c', '2', '--no-vacuum']) + options=['-T', '10', '-c', '2', '--no-vacuum']) backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='page', - options=[ - '--archive-timeout=60', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['--archive-timeout=10']) pgbench.wait() @@ -364,10 +341,6 @@ def test_basic_make_replica_via_restore(self): pg_options={ 'archive_timeout': '10s'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -417,10 +390,6 @@ def test_take_backup_from_delayed_replica(self): initdb_params=['--data-checksums'], pg_options={'archive_timeout': '10s'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -508,12 +477,12 @@ def test_take_backup_from_delayed_replica(self): pgbench.wait() # @unittest.skip("skip") + @needs_gdb def test_replica_promote(self): """ start backup from replica, during backup promote replica check that backup is failed """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( @@ -525,10 +494,6 @@ def test_replica_promote(self): 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -597,10 +562,10 @@ def test_replica_promote(self): log_content) # @unittest.skip("skip") + @needs_gdb def test_replica_stop_lsn_null_offset(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( @@ -611,10 +576,6 @@ def test_replica_stop_lsn_null_offset(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', master) self.set_archiving(backup_dir, 'node', master) @@ -650,25 +611,9 @@ def test_replica_stop_lsn_null_offset(self): '--stream'], return_id=False) - self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', - output) - - self.assertIn( - 'WARNING: WAL segment 000000010000000000000004 could not be streamed in 30 seconds', - output) - - self.assertIn( - 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', - output) - - self.assertIn( - 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', - output) - self.assertIn( 'has endpoint 0/4000000 which is ' - 'equal or greater than requested LSN 0/4000000', + 'equal or greater than requested LSN', output) self.assertIn( @@ -679,10 +624,10 @@ def test_replica_stop_lsn_null_offset(self): gdb_checkpointer.kill() # @unittest.skip("skip") + @needs_gdb def test_replica_stop_lsn_null_offset_next_record(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( @@ -693,10 +638,6 @@ def test_replica_stop_lsn_null_offset_next_record(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -759,28 +700,25 @@ def test_replica_stop_lsn_null_offset_next_record(self): log_content = f.read() self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', - log_content) - - self.assertIn( - 'LOG: Looking for segment: 000000010000000000000004', + 'has endpoint 0/4000000 which is ' + 'equal or greater than requested LSN', log_content) self.assertIn( - 'LOG: First record in WAL segment "000000010000000000000004": 0/4000028', + 'LOG: Found prior LSN:', log_content) self.assertIn( - 'INFO: stop_lsn: 0/4000000', + 'INFO: backup->stop_lsn 0/4000000', log_content) self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') # @unittest.skip("skip") + @needs_gdb def test_archive_replica_null_offset(self): """ """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( @@ -791,10 +729,6 @@ def test_archive_replica_null_offset(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', master) self.set_archiving(backup_dir, 'node', master) @@ -819,28 +753,16 @@ def test_archive_replica_null_offset(self): replica.slow_start(replica=True) self.switch_wal_segment(master) - self.switch_wal_segment(master) - - # take backup from replica - output = self.backup_node( - backup_dir, 'node', replica, replica.data_dir, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate'], - return_id=False) - - self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', - output) - - self.assertIn( - 'WARNING: WAL segment 000000010000000000000004 could not be archived in 30 seconds', - output) - self.assertIn( - 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', - output) + with self.switch_wal_after(master, 10): + # take backup from replica + output = self.backup_node( + backup_dir, 'node', replica, replica.data_dir, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate'], + return_id=False) self.assertIn( 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', @@ -855,8 +777,6 @@ def test_archive_replica_null_offset(self): 'LOG: Found prior LSN:', output) - print(output) - # @unittest.skip("skip") def test_archive_replica_not_null_offset(self): """ @@ -867,13 +787,10 @@ def test_archive_replica_not_null_offset(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ + 'archive_timeout' : '10s', 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', master) self.set_archiving(backup_dir, 'node', master) @@ -898,51 +815,34 @@ def test_archive_replica_not_null_offset(self): backup_dir, 'node', replica, replica.data_dir, options=[ '--archive-timeout=30', - '--log-level-console=LOG', '--no-validate'], return_id=False) - try: - self.backup_node( - backup_dir, 'node', replica, replica.data_dir, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of archive timeout. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - # vanilla -- 0/4000060 - # pgproee -- 0/4000078 - self.assertRegex( - e.message, - r'LOG: Looking for LSN (0/4000060|0/4000078) in segment: 000000010000000000000004', - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + output = self.backup_node( + backup_dir, 'node', replica, replica.data_dir, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate'], + return_id=False) - self.assertRegex( - e.message, - r'INFO: Wait for LSN (0/4000060|0/4000078) in archived WAL segment', - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.assertRegex( + output, + r'LOG: Record \S+ has endpoint 0/4000000 which is equal.*0/4000000', + "\n CMD: {0}".format(self.cmd)) - self.assertIn( - 'ERROR: WAL segment 000000010000000000000004 could not be archived in 30 seconds', - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.assertRegex( + output, + r'INFO: Backup \w+ completed\s*\Z', + "\n CMD: {0}".format(self.cmd)) # @unittest.skip("skip") + @needs_gdb def test_replica_toast(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( @@ -954,10 +854,6 @@ def test_replica_toast(self): 'wal_level': 'replica', 'shared_buffers': '128MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -1008,10 +904,6 @@ def test_replica_toast(self): pgdata = self.pgdata_content(replica.data_dir) - self.assertIn( - 'WARNING: Could not read WAL record at', - output) - self.assertIn( 'LOG: Found prior LSN:', output) @@ -1053,10 +945,6 @@ def test_start_stop_lsn_in_the_same_segno(self): 'wal_level': 'replica', 'shared_buffers': '128MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -1126,10 +1014,6 @@ def test_replica_promote_1(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) # set replica True, so archive_mode 'always' is used. @@ -1244,10 +1128,6 @@ def test_replica_promote_archive_delta(self): 'checkpoint_timeout': '30s', 'archive_timeout': '30s'}) - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) self.set_config( @@ -1364,10 +1244,6 @@ def test_replica_promote_archive_page(self): 'checkpoint_timeout': '30s', 'archive_timeout': '30s'}) - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) self.set_archiving(backup_dir, 'node', node1) @@ -1481,10 +1357,6 @@ def test_parent_choosing(self): set_replication=True, initdb_params=['--data-checksums']) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) @@ -1623,11 +1495,7 @@ def test_replica_via_basebackup(self): # restore stream backup self.restore_node(backup_dir, 'node', node) - xlog_dir = 'pg_wal' - if self.get_version(node) < 100000: - xlog_dir = 'pg_xlog' - - filepath = os.path.join(node.data_dir, xlog_dir, "00000002.history") + filepath = os.path.join(node.data_dir, 'pg_wal', "00000002.history") self.assertTrue( os.path.exists(filepath), "History file do not exists: {0}".format(filepath)) @@ -1649,6 +1517,9 @@ def test_replica_via_basebackup(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start(replica=True) +def call_repeat(times, func, *args): + for i in range(times): + func(*args) # TODO: # null offset STOP LSN and latest record in previous segment is conrecord (manual only) # archiving from promoted delayed replica diff --git a/tests/restore_test.py b/tests/restore_test.py index da3ebffb4..8f9a00eeb 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb import subprocess import sys from time import sleep @@ -333,9 +334,6 @@ def test_restore_to_lsn_inclusive(self): base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - if self.get_version(node) < self.version_to_num('10.0'): - return - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -399,9 +397,6 @@ def test_restore_to_lsn_not_inclusive(self): base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - if self.get_version(node) < self.version_to_num('10.0'): - return - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2032,10 +2027,7 @@ def test_restore_target_new_options(self): with node.connect("postgres") as con: con.execute("INSERT INTO tbl0005 VALUES (1)") con.commit() - if self.get_version(node) > self.version_to_num('10.0'): - res = con.execute("SELECT pg_current_wal_lsn()") - else: - res = con.execute("SELECT pg_current_xlog_location()") + res = con.execute("SELECT pg_current_wal_lsn()") con.commit() con.execute("INSERT INTO tbl0005 VALUES (2)") @@ -2126,33 +2118,32 @@ def test_restore_target_new_options(self): node.slow_start() # Restore with recovery target lsn - if self.get_version(node) >= 100000: - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-lsn={0}'.format(target_lsn), - "--recovery-target-action=promote", - '--recovery-target-timeline=1', - ]) + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-lsn={0}'.format(target_lsn), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) - with open(recovery_conf, 'r') as f: - recovery_conf_content = f.read() + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() - self.assertIn( - "recovery_target_lsn = '{0}'".format(target_lsn), - recovery_conf_content) + self.assertIn( + "recovery_target_lsn = '{0}'".format(target_lsn), + recovery_conf_content) - self.assertIn( - "recovery_target_action = 'promote'", - recovery_conf_content) + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) - self.assertIn( - "recovery_target_timeline = '1'", - recovery_conf_content) + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) - node.slow_start() + node.slow_start() # @unittest.skip("skip") def test_smart_restore(self): @@ -2252,9 +2243,9 @@ def test_pg_11_group_access(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_restore_concurrent_drop_table(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -2995,8 +2986,8 @@ def test_empty_and_mangled_database_map(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: field "dbOid" is not found in the line 42 of ' - 'the file backup_content.control', e.message, + 'ERROR: backup_content.control file has invalid format in line 42', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -3011,8 +3002,8 @@ def test_empty_and_mangled_database_map(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: field "dbOid" is not found in the line 42 of ' - 'the file backup_content.control', e.message, + 'ERROR: backup_content.control file has invalid format in line 42', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -3048,81 +3039,8 @@ def test_missing_database_map(self): "postgres", "CREATE DATABASE backupdb") - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + # PG < 15 + if self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -3651,9 +3569,9 @@ def test_truncate_postgresql_auto_conf(self): self.assertTrue(os.path.exists(auto_path)) # @unittest.skip("skip") + @needs_gdb def test_concurrent_restore(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( diff --git a/tests/retention_test.py b/tests/retention_test.py index 88432a00f..b9b94ed59 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -2,6 +2,7 @@ import unittest from datetime import datetime, timedelta from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb from time import sleep from distutils.dir_util import copy_tree @@ -1437,6 +1438,7 @@ def test_window_error_backups(self): # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # @unittest.skip("skip") + @needs_gdb def test_window_error_backups_1(self): """ DELTA @@ -1444,7 +1446,6 @@ def test_window_error_backups_1(self): FULL -------window """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -1483,6 +1484,7 @@ def test_window_error_backups_1(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) # @unittest.skip("skip") + @needs_gdb def test_window_error_backups_2(self): """ DELTA @@ -1490,7 +1492,6 @@ def test_window_error_backups_2(self): FULL -------window """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -1517,11 +1518,6 @@ def test_window_error_backups_2(self): self.show_pb(backup_dir, 'node')[1]['id'] - if self.get_version(node) < 90600: - node.safe_psql( - 'postgres', - 'SELECT pg_catalog.pg_stop_backup()') - # Take DELTA backup self.backup_node( backup_dir, 'node', node, backup_type='delta', @@ -1529,17 +1525,14 @@ def test_window_error_backups_2(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + @needs_gdb def test_retention_redundancy_overlapping_chains(self): """""" - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - if self.get_version(node) < 90600: - self.skipTest('Skipped because ptrack support is disabled') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1574,17 +1567,14 @@ def test_retention_redundancy_overlapping_chains(self): self.validate_pb(backup_dir, 'node') + @needs_gdb def test_retention_redundancy_overlapping_chains_1(self): """""" - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - if self.get_version(node) < 90600: - self.skipTest('Skipped because ptrack support is disabled') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1673,11 +1663,11 @@ def test_wal_purge_victim(self): e.message) # @unittest.skip("skip") + @needs_gdb def test_failed_merge_redundancy_retention(self): """ Check that retention purge works correctly with MERGING backups """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( @@ -2451,11 +2441,11 @@ def test_basic_wal_depth(self): self.validate_pb(backup_dir, 'node') + @needs_gdb def test_concurrent_running_full_backup(self): """ https://github.com/postgrespro/pg_probackup/issues/328 """ - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( diff --git a/tests/validate_test.py b/tests/validate_test.py index 4ff44941f..7c5a34bf2 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import needs_gdb from datetime import datetime, timedelta from pathlib import Path import subprocess @@ -1046,11 +1047,11 @@ def test_validate_instance_with_several_corrupt_backups(self): 'Backup STATUS should be "OK"') # @unittest.skip("skip") + @needs_gdb def test_validate_instance_with_several_corrupt_backups_interrupt(self): """ check that interrupt during validation is handled correctly """ - self._check_gdb_flag_or_skip_test() node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), @@ -1689,14 +1690,9 @@ def test_validate_corrupt_wal_between_backups(self): con.commit() target_xid = res[0][0] - if self.get_version(node) < self.version_to_num('10.0'): - walfile = node.safe_psql( - 'postgres', - 'select pg_xlogfile_name(pg_current_xlog_location())').decode('utf-8').rstrip() - else: - walfile = node.safe_psql( - 'postgres', - 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() if self.archive_compress: walfile = walfile + '.gz' @@ -3386,12 +3382,8 @@ def test_corrupt_pg_control_via_resetxlog(self): backup_id = self.backup_node(backup_dir, 'node', node) - if self.get_version(node) < 100000: - pg_resetxlog_path = self.get_bin_path('pg_resetxlog') - wal_dir = 'pg_xlog' - else: - pg_resetxlog_path = self.get_bin_path('pg_resetwal') - wal_dir = 'pg_wal' + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' os.mkdir( os.path.join( @@ -3437,9 +3429,9 @@ def test_corrupt_pg_control_via_resetxlog(self): repr(e.message), self.cmd)) # @unittest.skip("skip") + @needs_gdb def test_validation_after_backup(self): """""" - self._check_gdb_flag_or_skip_test() backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( diff --git a/unit/CUnit-Run.dtd b/unit/CUnit-Run.dtd new file mode 100644 index 000000000..e0f56b8a3 --- /dev/null +++ b/unit/CUnit-Run.dtd @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/unit/CUnit-Run.xsl b/unit/CUnit-Run.xsl new file mode 100644 index 000000000..b46b729fc --- /dev/null +++ b/unit/CUnit-Run.xsl @@ -0,0 +1,173 @@ + + + + + + + CUnit - Automated Test Run Summary Report + + + + + + + + + +
+

+ CUnit - A Unit testing framework for C.
+ http://cunit.sourceforge.net/ +

+
+ + + +

+

+

Automated Test Run Results

+
+ + + + + + + + +
+ + + + + + + + + + + + + Running Suite + + + + + + + + + + + + + Running Group + + + + + + + + + + + + + Running test ... + + Passed + + + + + + + + Running test ... + + Failed + + + + + + + + + + + + + + + +
File Name + + Line Number + +
Condition + +
+ + + + + + + + Running Suite ... + + + + + + + + + + + Running Group ... + + + + + + + + +

+ + + + + + + + + + + + + + + + + + + + + + +
Cumulative Summary for Run
Type Total Run Succeeded Failed Inactive
+ + + +

+

+
+ + + diff --git a/unit/Makefile b/unit/Makefile new file mode 100644 index 000000000..b4a2b5015 --- /dev/null +++ b/unit/Makefile @@ -0,0 +1,26 @@ +APPS=unit/test_unit_pio unit/test_unit_probackup + +test_unit: $(APPS) + + +TEST_FILES=unit/test_probackup.o unit/test_pio.o + +unit/test_unit_pio: LIBS += -lcunit +unit/test_unit_pio: $(PGPOBJS) unit/pgunit.o unit/test_pio.o + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LIBS) $(PG_LIBS_INTERNAL) -o $@ + +unit/test_unit_probackup: LIBS += -lcunit +unit/test_unit_probackup: $(PGPOBJS) unit/pgunit.o unit/test_probackup.o + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LIBS) $(PG_LIBS_INTERNAL) -o $@ + +test_unit_run: test_unit + (cd unit; ./test_unit_pio) + (cd unit; ./test_unit_probackup) + +test_unit_pub: test_unit_run + (cd unit ; /bin/sh unit_pub.sh) + +clean: test_clean + +test_clean: + rm -rf $(APPS) unit/*.o *~ *.gcda *.gcno *.html *.xml pb.info gmon.out report/ diff --git a/unit/pg_control.TEST b/unit/pg_control.TEST new file mode 100644 index 000000000..824236718 Binary files /dev/null and b/unit/pg_control.TEST differ diff --git a/unit/pgpbkp.c b/unit/pgpbkp.c new file mode 100644 index 000000000..43be09c4a --- /dev/null +++ b/unit/pgpbkp.c @@ -0,0 +1,37 @@ +#include +#include +#include + +#include + +#include +#include + +#include + +#include "pgunit.h" + +/* Emulate pgprobackup */ +bool show_color = true; +ShowFormat show_format = SHOW_PLAIN; +const char *PROGRAM_NAME = NULL; +const char *PROGRAM_NAME_FULL = NULL; +const char *PROGRAM_FULL_PATH = NULL; +pid_t my_pid = 0; +bool is_archive_cmd = false; +bool remote_agent = false; +time_t current_time = 0; +char *replication_slot = NULL; +pgBackup current; +bool perm_slot = false; +bool temp_slot = false; +bool progress = false; +int num_threads = 1; +bool delete_wal = false; +bool merge_expired = false; +bool smooth_checkpoint; +bool skip_block_validation = false; +bool dry_run = false; +bool delete_expired = false; + +/***********************/ diff --git a/unit/pgunit.c b/unit/pgunit.c new file mode 100644 index 000000000..f326945bf --- /dev/null +++ b/unit/pgunit.c @@ -0,0 +1,183 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "pgunit.h" + +pioDrive_i drive; +pioDBDrive_i dbdrive; +pioDrive_i cloud_drive; +pioDBDrive_i local_drive; + +int should_be_remote; + +void +init_test_drives() +{ + local_drive = pioDBDriveForLocation(FIO_LOCAL_HOST); +} + +int +USE_LOCAL() +{ + drive = $reduce(pioDrive, local_drive); + dbdrive = local_drive; + should_be_remote = 0; + printf("USE_LOCAL\n"); + return 0; +} + +static int +clean_basic_suite() +{ + return 0; +} + +#define FNAMES "abcdefghijklmnopqrstuvwxyz0123456789" + +static int rand_init=0; +ft_str_t +random_path(void) +{ + char name[MAXPGPATH]; + + if(!rand_init) { + srand(time(NULL)); + rand_init = 1; + } + + int len = 3 + rand() % 20; + int fnlen = strlen(FNAMES); + name[0]=0; + snprintf(name, MAXPGPATH, "/tmp/%d_", getpid()); + int i; + int l=strlen(name); + for(i=l; i < len+l; ++i) + { + name[i] = FNAMES[rand()%fnlen]; + } + name[i] = 0; + + return ft_strdupc(name); +} + +char * +random_name(void) +{ + char name[MAXPGPATH]; + + if(!rand_init) { + srand(time(NULL)); + rand_init = 1; + } + + int len = 3 + rand() % 10; + int fnlen = strlen(FNAMES); + int i; + for(i=0;i=0); + int fdout = open(to, O_CREAT|O_RDWR|O_TRUNC, FILE_PERMISSION); + CU_ASSERT_FATAL(fdout>=0); + while(1) + { + char buf[BUFSZ]; + int rc = read(fdin, buf, BUFSZ); + CU_ASSERT_FATAL(rc>=0); + if(rc==0) break; + int written = write(fdout, buf, rc); + CU_ASSERT_FATAL(written == rc); + } + close(fdin); + fsync(fdout); + close(fdout); +} + +void +init_fake_server(const char *path) +{ + char global[8192]; + snprintf(global, 8192, "%s/global", path); + int rc = mkdir(path, DIR_PERMISSION); + CU_ASSERT_FATAL(rc == 0); + rc = mkdir(global, DIR_PERMISSION); + CU_ASSERT_FATAL(rc == 0); + char global2[MAXPGPATH]; + snprintf(global2, MAXPGPATH, "%s/pg_control", global); + copy_file("pg_control.TEST", global2); +} + +void +pbk_add_tests(int (*init)(void), const char *suite_name, PBK_test_description *tests) +{ + CU_pSuite pSuite; + int i; + + pSuite = CU_add_suite(suite_name, init, clean_basic_suite); + if(pSuite==NULL) + { + fprintf(stderr, "Can't add a suite %s\n", suite_name); + CU_cleanup_registry(); + abort(); + } + + for(i = 0; tests[i].name; ++i) + { + if(CU_add_test(pSuite, tests[i].name, tests[i].foo) == NULL) + { + fprintf(stderr, "Can't add test %s.%s\n", suite_name, tests[i].name); + CU_cleanup_registry(); + abort(); + } + } +} + +void +pio_write(pioDrive_i drive, path_t path, const char *data) +{ + FOBJ_FUNC_ARP(); + err_i err=$noerr(); + err = $i(pioWriteFile, drive, .path = path, .content = ft_bytes((char *)data, strlen(data)), .binary = true); + CU_ASSERT(!$haserr(err)); +} + +bool +pio_exists(pioDrive_i drive, path_t path) +{ + FOBJ_FUNC_ARP(); + err_i err=$noerr(); + + bool exists = $i(pioExists, drive, .path = path, .expected_kind = PIO_KIND_REGULAR, &err); + if ($haserr(err)) + fprintf(stderr, "pio_exists: %s\n", $errmsg(err)); + CU_ASSERT(!$haserr(err)); + return exists; +} +bool +pio_exists_d(pioDrive_i drive, path_t path) +{ + FOBJ_FUNC_ARP(); + err_i err=$noerr(); + + bool exists = $i(pioExists, drive, .path = path, .expected_kind = PIO_KIND_DIRECTORY, &err); + CU_ASSERT(!$haserr(err)); + return exists; +} diff --git a/unit/pgunit.h b/unit/pgunit.h new file mode 100644 index 000000000..2628900f8 --- /dev/null +++ b/unit/pgunit.h @@ -0,0 +1,27 @@ +#ifdef __pgunit_h__ +#error "Double #include of pgunit.h" +#endif + +#define __pgunit_h__ 1 + +#define BUFSZ 8192 + +typedef struct { + const char *name; + void (*foo)(void); +} PBK_test_description; + +extern pioDrive_i drive; +extern pioDBDrive_i dbdrive; +extern int should_be_remote; + +void init_test_drives(void); +int USE_LOCAL(void); +ft_str_t random_path(void); +char *random_name(void); +void pbk_add_tests(int (*init)(void), const char *sub_name, PBK_test_description *tests); +void pio_write(pioDrive_i drive, path_t name, const char *data); +bool pio_exists(pioDrive_i drive, path_t path); +bool pio_exists_d(pioDrive_i drive, path_t path); +void copy_file(const char *from, const char *to); +void init_fake_server(const char *path); diff --git a/unit/test_pio.c b/unit/test_pio.c new file mode 100644 index 000000000..d4cb40841 --- /dev/null +++ b/unit/test_pio.c @@ -0,0 +1,626 @@ +#include +#include +#include + +#include +#include + +#include +#include + +#include "pgunit.h" + +#define TEST_STR "test\n" +#define BUFSZ 8192 + +#define XXX_STR "XXX" + +static void +test_pioStat() +{ + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + ft_str_t path = random_path(); + + err = $i(pioWriteFile, drive, .path = path.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + time_t now = time(NULL); + + pio_stat_t pst = $i(pioStat, drive, .path = path.ptr, .follow_symlink = false, .err = &err); + + CU_ASSERT(!$haserr(err)); + + CU_ASSERT(pst.pst_kind == PIO_KIND_REGULAR); + CU_ASSERT(pst.pst_mode == FILE_PERMISSION); + CU_ASSERT(abs(now-pst.pst_mtime) < 2); + CU_ASSERT(pst.pst_size == 5); + + ft_str_free(&path); +} + +static void +test_pioRemove() +{ + FOBJ_FUNC_ARP(); + + ft_str_t path = random_path(); + pio_write(drive, path.ptr, TEST_STR); + CU_ASSERT(pio_exists(drive, path.ptr)); + + err_i err = $i(pioRemove, drive, .path = path.ptr, .missing_ok = false); + + CU_ASSERT(!$haserr(err)); + + CU_ASSERT(!pio_exists(drive, path.ptr)); + + ft_str_free(&path); +} + +static void +test_pioExists() +{ + FOBJ_FUNC_ARP(); + + err_i err = $noerr(); + bool exists = $i(pioExists, drive, .path = "/", .expected_kind = PIO_KIND_DIRECTORY, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(exists); + + ft_str_t path = random_path(); + err = $noerr(); + exists = $i(pioExists, drive, .path = path.ptr, .expected_kind = PIO_KIND_REGULAR, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(!exists); + + ft_str_t name = random_path(); + pio_write(drive, name.ptr, TEST_STR); + exists = $i(pioExists, drive, .path = name.ptr, .expected_kind = PIO_KIND_REGULAR, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(exists); + + ft_str_free(&path); + ft_str_free(&name); +} + +static void +test_pioIsRemote() +{ + FOBJ_FUNC_ARP(); + + if(should_be_remote) { + CU_ASSERT( $i(pioIsRemote, drive) ); + } else { + CU_ASSERT( !$i(pioIsRemote, drive) ); + } +} + +static void +test_pioWriteFile() +{ + FOBJ_FUNC_ARP(); + + err_i err = $noerr(); + ft_str_t path = random_path(); + + CU_ASSERT(!pio_exists(drive, path.ptr)); + + err = $i(pioWriteFile, drive, .path = path.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + + CU_ASSERT(pio_exists(drive, path.ptr)); + + ft_bytes_t result = $i(pioReadFile, drive, .path = path.ptr, .binary = true, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(result.len==strlen(TEST_STR)); + CU_ASSERT(!strncmp(result.ptr, TEST_STR, strlen(TEST_STR))); + + ft_bytes_free(&result); + ft_str_free(&path); +} + +static void +test_pioOpenRead() +{ + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + ft_str_t path = random_path(); + pio_write(drive, path.ptr, TEST_STR); + + CU_ASSERT(pio_exists(drive, path.ptr)); + + pioReader_i reader = $i(pioOpenRead, drive, .path = path.ptr, &err); + CU_ASSERT(!$haserr(err)); + char B0[8192]; + ft_bytes_t buf = ft_bytes(B0, 8192); + size_t ret = $i(pioRead, reader, .buf = buf, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(ret==strlen(TEST_STR)); + CU_ASSERT(!strncmp(buf.ptr, TEST_STR, strlen(TEST_STR))); + err = $i(pioSeek, reader, 0); + CU_ASSERT(!$haserr(err)); + ft_bytes_t buf2 = ft_bytes(B0+100, 8192); + ret = $i(pioRead, reader, .buf = buf2, &err); + CU_ASSERT(ret==strlen(TEST_STR)); + CU_ASSERT(!strncmp(buf2.ptr, TEST_STR, strlen(TEST_STR))); + + $i(pioClose, reader); + + //ft_bytes_free(&result); + + ft_str_free(&path); +} + +static void +test_pioOpenReadStream() +{ + // return enoent for non existent file. same for pioStat + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + ft_str_t path = random_path(); + + pioReadStream_i stream; + /* Crash in pioCloudDrive */ + stream = $i(pioOpenReadStream, drive, .path = path.ptr, &err); + CU_ASSERT($haserr(err)); + + pio_write(drive, path.ptr, TEST_STR); + + stream = $i(pioOpenReadStream, drive, .path = path.ptr, &err); + CU_ASSERT(!$haserr(err)); + + char B0[8192]; + ft_bytes_t buf = ft_bytes(B0, 8192); + size_t ret = $i(pioRead, stream, .buf= buf, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(ret==strlen(TEST_STR)); + CU_ASSERT(!strncmp(buf.ptr, TEST_STR, strlen(TEST_STR))); + $i(pioClose, stream); + ft_str_free(&path); +} + +static void +test_pioGetCRC32() +{ + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + ft_str_t path = random_path(); + pg_crc32 crc; + +#if 0 + //crashes. should return errno in err + crc = $i(pioGetCRC32, drive, .path = path.ptr, .compressed = false, .err = &err); + CU_ASSERT($haserr(err)); +#endif + pio_write(drive, path.ptr, TEST_STR); + crc = $i(pioGetCRC32, drive, .path = path.ptr, .compressed = false, .err = &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(crc==0xFA94FDDF) +} + + +static void +test_pioMakeDir() +{ + FOBJ_FUNC_ARP(); + + ft_str_t path = random_path(); + + CU_ASSERT(!pio_exists(drive, path.ptr)); + err_i err = $i(pioMakeDir, drive, .path = path.ptr, .mode = DIR_PERMISSION, .strict = true); + CU_ASSERT(!$haserr(err)); + + CU_ASSERT(pio_exists_d(drive, path.ptr)); +} + +static void +test_pioMakeDirWithParent() +{ + FOBJ_FUNC_ARP(); + char child[MAXPGPATH]; + ft_str_t parent = random_path(); + CU_ASSERT(!pio_exists(drive, parent.ptr)); + snprintf(child, MAXPGPATH, "%s/TEST", parent.ptr); + + err_i err = $i(pioMakeDir, drive, .path = child, .mode = DIR_PERMISSION, .strict = true); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(pio_exists_d(drive, parent.ptr)); + CU_ASSERT(pio_exists_d(drive, child)); + + ft_str_free(&parent); +} + +static void +test_pioListDirCanWithSlash() +{ + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + ft_str_t root = random_path(); + ft_str_t slash = ft_asprintf("%s/", root.ptr); + ft_str_t child = ft_asprintf("%s/sample.txt", root.ptr); + + CU_ASSERT(!pio_exists(drive, root.ptr)); + err = $i(pioMakeDir, drive, .path = root.ptr, .mode = DIR_PERMISSION, .strict = true); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(pio_exists_d(drive, root.ptr)); + + err = $i(pioWriteFile, drive, .path = child.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + + pioDirIter_i dir = $i(pioOpenDir, drive, .path = slash.ptr, .err = &err); + CU_ASSERT(!$haserr(err)); + + int count = 0; + while (true) + { + pio_dirent_t entry = $i(pioDirNext, dir, &err); + CU_ASSERT(!$haserr(err)); + if (entry.stat.pst_kind == PIO_KIND_UNKNOWN) break; + CU_ASSERT(ft_strcmp(entry.name, ft_cstr("sample.txt")) == FT_CMP_EQ); + count++; + } + CU_ASSERT(count == 1); + err = $i(pioClose, dir); + CU_ASSERT(!$haserr(err)); + + ft_str_free(&root); + ft_str_free(&slash); + ft_str_free(&child); +} + +static void +test_pioListDir() +{ + FOBJ_FUNC_ARP(); + ft_str_t root = random_path(); + ft_str_t child = ft_asprintf("%s/sample.txt", root.ptr); + ft_str_t sub_dir = ft_asprintf("%s/subdir", root.ptr); + ft_str_t sub_child = ft_asprintf("%s/subdir/xxx.txt", root.ptr); + err_i err = $noerr(); + int i; + + CU_ASSERT(!pio_exists(drive, root.ptr)); + err = $i(pioMakeDir, drive, .path = root.ptr, .mode = DIR_PERMISSION, .strict = true); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(pio_exists_d(drive, root.ptr)); + + err = $i(pioWriteFile, drive, .path = child.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + + err = $i(pioMakeDir, drive, .path = sub_dir.ptr, .mode = DIR_PERMISSION, .strict = true); + CU_ASSERT(!$haserr(err)); + + err = $i(pioWriteFile, drive, .path = sub_child.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + + pioDirIter_i dir = $i(pioOpenDir, drive, .path = root.ptr, .err = &err); + CU_ASSERT(!$haserr(err)); + +#define NUM_EXPECTED 2 + const char *expected[NUM_EXPECTED] = {"sample.txt", "subdir"}; + int count = 0; + for (count = 0; true; count++) + { + pio_dirent_t entry = $i(pioDirNext, dir, &err); + CU_ASSERT(!$haserr(err)); + + if (entry.stat.pst_kind == PIO_KIND_UNKNOWN) break; + + for(i = 0; i < NUM_EXPECTED; ++i) + { + if(ft_strcmp(entry.name, ft_cstr(expected[i])) != FT_CMP_EQ) + continue; + expected[i] = NULL; + } + } + + for(i = 0; i < NUM_EXPECTED; ++i) + { + CU_ASSERT(expected[i] == NULL); + } + + CU_ASSERT(count == NUM_EXPECTED); + + err = $i(pioClose, dir); + CU_ASSERT(!$haserr(err)); + + dir = $i(pioOpenDir, drive, .path = sub_dir.ptr, .err = &err); + CU_ASSERT(!$haserr(err)); + + count = 0; + for (count = 0; true; count++) + { + pio_dirent_t entry = $i(pioDirNext, dir, &err); + CU_ASSERT(!$haserr(err)); + + if (entry.stat.pst_kind == PIO_KIND_UNKNOWN) break; + + CU_ASSERT(ft_strcmp(entry.name, ft_cstr("xxx.txt")) == FT_CMP_EQ); + } + + for(i = 0; i < NUM_EXPECTED; ++i) + { + CU_ASSERT(expected[i] == NULL); + } + + CU_ASSERT(count == 1); + + err = $i(pioClose, dir); + CU_ASSERT(!$haserr(err)); + +#undef NUM_EXPECTED +} + +static void +test_pioListDirMTimeAndSize() +{ + FOBJ_FUNC_ARP(); + ft_str_t root = random_path(); + ft_str_t child = ft_asprintf("%s/sample.txt", root.ptr); + err_i err = $noerr(); + int i; + + CU_ASSERT(!pio_exists(drive, root.ptr)); + err = $i(pioMakeDir, drive, .path = root.ptr, .mode = DIR_PERMISSION, .strict = true); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(pio_exists_d(drive, root.ptr)); + + err = $i(pioWriteFile, drive, .path = child.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + time_t created = time(NULL); + + pioDirIter_i dir = $i(pioOpenDir, drive, .path = root.ptr, .err = &err); + CU_ASSERT(!$haserr(err)); + +#define NUM_EXPECTED 1 + const char *expected[NUM_EXPECTED] = {"sample.txt"}; + int count = 0; + for (count = 0; true; count++) + { + pio_dirent_t entry = $i(pioDirNext, dir, &err); + CU_ASSERT(!$haserr(err)); + + if (entry.stat.pst_kind == PIO_KIND_UNKNOWN) break; + + CU_ASSERT(entry.stat.pst_mtime == created); + CU_ASSERT(entry.stat.pst_size == strlen(TEST_STR)); + + for(i = 0; i < NUM_EXPECTED; ++i) + { + if(ft_strcmp(entry.name, ft_cstr(expected[i])) != FT_CMP_EQ) + continue; + expected[i] = NULL; + } + } + + for(i = 0; i < NUM_EXPECTED; ++i) + { + CU_ASSERT(expected[i] == NULL); + } + + CU_ASSERT(count == NUM_EXPECTED); + + err = $i(pioClose, dir); + CU_ASSERT(!$haserr(err)); +} + +static void +test_pioRemoveDir() +{ + FOBJ_FUNC_ARP(); + ft_str_t path = random_path(); + err_i err = $noerr(); + char path2[8192]; + snprintf(path2, 8192, "%s/%s", path.ptr, "sample.txt"); + + CU_ASSERT(!pio_exists(drive, path.ptr)); + err = $i(pioMakeDir, drive, .path = path.ptr, .mode = DIR_PERMISSION, .strict = true); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(pio_exists_d(drive, path.ptr)); + + err = $i(pioWriteFile, drive, .path = path2, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(pio_exists(drive, path2)); + $i(pioRemoveDir, drive, path.ptr, .root_as_well=false); + CU_ASSERT(!pio_exists(drive, path2)); + CU_ASSERT(pio_exists_d(drive, path.ptr)); +} + +static void +test_pioFilesAreSame() +{ + FOBJ_FUNC_ARP(); + + err_i err = $noerr(); + ft_str_t path1 = random_path(); + ft_str_t path2 = random_path(); + + CU_ASSERT(!pio_exists(drive, path1.ptr)); + CU_ASSERT(!pio_exists(drive, path2.ptr)); + + err = $i(pioWriteFile, drive, .path = path1.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(pio_exists(drive, path1.ptr)); + + err = $i(pioWriteFile, drive, .path = path2.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(pio_exists(drive, path2.ptr)); + + ft_bytes_t result1 = $i(pioReadFile, drive, .path = path1.ptr, .binary = true, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(result1.len==strlen(TEST_STR)); + CU_ASSERT(!strncmp(result1.ptr, TEST_STR, strlen(TEST_STR))); + + ft_bytes_t result2 = $i(pioReadFile, drive, .path = path2.ptr, .binary = true, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(result2.len==strlen(TEST_STR)); + CU_ASSERT(!strncmp(result2.ptr, TEST_STR, strlen(TEST_STR))); + + CU_ASSERT(result1.len == result2.len); + CU_ASSERT(!memcmp(result1.ptr, result2.ptr, result1.len)); + + ft_bytes_free(&result1); + ft_bytes_free(&result2); + + ft_str_free(&path1); + ft_str_free(&path2); +} + +static void +test_pioReadFile() +{ + FOBJ_FUNC_ARP(); + + err_i err = $noerr(); + ft_str_t path = random_path(); + + CU_ASSERT(!pio_exists(drive, path.ptr)); + + err = $i(pioWriteFile, drive, .path = path.ptr, .content = ft_bytes(TEST_STR, strlen(TEST_STR)), .binary = true); + CU_ASSERT(!$haserr(err)); + + CU_ASSERT(pio_exists(drive, path.ptr)); + + ft_bytes_t result = $i(pioReadFile, drive, .path = path.ptr, .binary = true, &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(result.len==strlen(TEST_STR)); + CU_ASSERT(!strncmp(result.ptr, TEST_STR, strlen(TEST_STR))); + + ft_bytes_free(&result); + + ft_str_free(&path); +} + +static void +test_pioOpenRewrite() +{ + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + ft_str_t path = random_path(); + pio_write(drive, path.ptr, TEST_STR); + + CU_ASSERT(pio_exists(drive, path.ptr)); + + pioWriteCloser_i writer = $i(pioOpenRewrite, drive, .path = path.ptr, + .permissions = FILE_PERMISSION, .binary = true, + .use_temp=true, .sync = true, .err = &err); + CU_ASSERT(!$haserr(err)); + char B0[8192]; + snprintf(B0, 8192, XXX_STR); + ft_bytes_t buf = ft_bytes(B0, strlen(B0)); + err = $i(pioWrite, writer, .buf = buf); + CU_ASSERT(!$haserr(err)); + $i(pioClose, writer); + + ft_bytes_t result = $i(pioReadFile, drive, .path = path.ptr, .binary = true, &err); + CU_ASSERT(strlen(XXX_STR) == result.len); + CU_ASSERT(!memcmp(XXX_STR, result.ptr, result.len)); + ft_bytes_free(&result); + + ft_str_free(&path); +} + +static void +test_pioSeek() +{ + FOBJ_FUNC_ARP(); + err_i err = $noerr(); + ft_str_t path = random_path(); + pioWriteCloser_i writer = $i(pioOpenRewrite, drive, .path = path.ptr, + .permissions = FILE_PERMISSION, .binary = true, + .use_temp=true, .sync = true, .err = &err); + CU_ASSERT(!$haserr(err)); + char B0[8192]; + snprintf(B0, 8192, "012345678901234567890123012345678901234567890123"); + ft_bytes_t buf = ft_bytes(B0, strlen(B0)); + err = $i(pioWrite, writer, .buf = buf); + CU_ASSERT(!$haserr(err)); + $i(pioClose, writer); + + pioReader_i reader = $i(pioOpenRead, drive, .path = path.ptr, &err); + CU_ASSERT (!$haserr(err)); + +#define TRY_OFFT 1 +#define TRY_LEN 24 + err = $i(pioSeek, reader, TRY_OFFT); + CU_ASSERT(!$haserr(err)); + + ft_bytes_t read_buf = ft_bytes_alloc(TRY_LEN); + size_t rc = $i(pioRead, reader, .buf = read_buf, .err = &err); + CU_ASSERT(!$haserr(err)); + CU_ASSERT(rc == TRY_LEN); + CU_ASSERT(!memcmp(B0+TRY_OFFT, read_buf.ptr, TRY_LEN)); +} + +/* pioDBDrive */ +static void +test_pioRename() +{ + FOBJ_FUNC_ARP(); + pioDBDrive_i db_drive = pioDBDriveForLocation(FIO_LOCAL_HOST); + ft_str_t name = random_path(); + ft_str_t another_name = random_path(); + + pio_write(drive, name.ptr, TEST_STR); + CU_ASSERT(pio_exists(drive, name.ptr)); + + err_i err = $i(pioRename, db_drive, .old_path = name.ptr, .new_path = another_name.ptr); + CU_ASSERT(!$haserr(err)); + + CU_ASSERT(!pio_exists(drive, name.ptr)); + CU_ASSERT(pio_exists(drive, another_name.ptr)); +} + +PBK_test_description PIO_DRIVE_TESTS[] = { + {"Test pioOpenRead", test_pioOpenRead}, + {"Test pioOpenReadStream", test_pioOpenReadStream}, + {"Test pioStat", test_pioStat}, + {"Test pioRemove", test_pioRemove}, + {"Test pioExists", test_pioExists}, + {"Test pioGetCRC32", test_pioGetCRC32}, + {"Test pioIsRemote", test_pioIsRemote}, + {"Test pioMakeDir", test_pioMakeDir}, + {"Test pioMakeDirWithParent", test_pioMakeDirWithParent}, + {"Test pioListDir", test_pioListDir}, + {"Test pioListDirCanWithSlash", test_pioListDirCanWithSlash}, + {"Test pioListDirMTimeAndSize", test_pioListDirMTimeAndSize}, + {"Test pioRemoveDir", test_pioRemoveDir}, + {"Test pioFilesAreSame", test_pioFilesAreSame}, + {"Test pioReadFile", test_pioReadFile}, + {"Test pioWriteFile", test_pioWriteFile}, + {"Test pioOpenRewrite", test_pioOpenRewrite}, + {"Test pioSeek", test_pioSeek}, + {NULL, NULL} +}; + +PBK_test_description PIO_DB_DRIVE_TESTS[] = { + {"Test pioRename", test_pioRename}, + {NULL, NULL} +}; + +int +main(int argc, char *argv[]) +{ + ft_init_log(elog_ft_log); + fobj_init(); + FOBJ_FUNC_ARP(); + init_pio_objects(); + + init_test_drives(); + if(CUE_SUCCESS != CU_initialize_registry()) + return CU_get_error(); + + pbk_add_tests(USE_LOCAL, "Local pioDrive", PIO_DRIVE_TESTS); + pbk_add_tests(USE_LOCAL, "LOcal pioDBDrive", PIO_DB_DRIVE_TESTS); + + CU_list_tests_to_file(); + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_set_output_filename("test_pio"); + + CU_basic_run_tests(); + CU_automated_run_tests(); + + CU_cleanup_registry(); + + return CU_get_error(); +} diff --git a/unit/test_probackup.c b/unit/test_probackup.c new file mode 100644 index 000000000..f3e2503e2 --- /dev/null +++ b/unit/test_probackup.c @@ -0,0 +1,87 @@ +#include +#include +#include + +#include +#include + +#include +#include + +#include "pgunit.h" + +static void +test_do_init() +{ + FOBJ_FUNC_ARP(); + ft_str_t backup_path = random_path(); + CatalogState *catalogState = catalog_new(backup_path.ptr); + int rc; + + rc = do_init(catalogState); + + CU_ASSERT(rc == 0); +} + +static void +test_do_add_instance() +{ + FOBJ_FUNC_ARP(); + int rc; + ft_str_t backup_path = random_path(); + char *instance_name = random_name(); + ft_str_t server_path = random_path(); + init_fake_server(server_path.ptr); + + CatalogState *catalogState = catalog_new(backup_path.ptr); + catalogState->backup_location = drive; + rc = do_init(catalogState); + CU_ASSERT(rc == 0); + + //CU_ASSERT(pio_exists_d(drive, backup_path.ptr)); + + init_config(&instance_config, instance_name); + instance_config.pgdata = server_path.ptr; + InstanceState *instanceState = makeInstanceState(catalogState, instance_name); + instanceState->database_location = $reduce(pioDrive, dbdrive); + rc = do_add_instance(instanceState, &instance_config); + CU_ASSERT(rc == 0); + + char buf[MAXPGPATH]; + snprintf(buf, MAXPGPATH, "%s/%s/%s", catalogState->backup_subdir_path, instance_name, BACKUP_CATALOG_CONF_FILE); + CU_ASSERT(pio_exists(drive, buf)); +} + + +PBK_test_description PIO_INIT_TESTS[] = { + {"Test do_init", test_do_init}, + {"Test do_add_instance", test_do_add_instance}, + {NULL,NULL}, +}; + +int +main(int argc, char *argv[]) +{ + ft_init_log(elog_ft_log); + fobj_init(); + FOBJ_FUNC_ARP(); + init_pio_objects(); + + init_test_drives(); + + if(CUE_SUCCESS != CU_initialize_registry()) + return CU_get_error(); + + pbk_add_tests(USE_LOCAL, "Local init", PIO_INIT_TESTS); + + CU_basic_set_mode(CU_BRM_VERBOSE); + + CU_basic_run_tests(); + CU_set_output_filename("test_probackup"); + //CU_list_tests_to_file(); + CU_automated_run_tests(); + + CU_cleanup_registry(); + + return CU_get_error(); +} diff --git a/unit/unit_pub.sh b/unit/unit_pub.sh new file mode 100644 index 000000000..6ba09e298 --- /dev/null +++ b/unit/unit_pub.sh @@ -0,0 +1,9 @@ +xsltproc test_pio-Results.xml > results-pio.html 2> /dev/null +xsltproc test_probackup-Results.xml > results-init.html 2> /dev/null +lcov -t "pb" --output pb.info --capture --directory . --directory ../s3 --directory ../src --rc lcov_branch_coverage=1 > /dev/null 2>&1 +genhtml --output report --branch-coverage pb.info > /dev/null 2>&1 +xdg-open report/index.html > /dev/null 2>&1 +xdg-open results-pio.html > /dev/null 2>&1 +if test -s results-init.html ; then + xdg-open results-init.html > /dev/null 2>&1 +fi pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy