From b8a3eed8e17380ce15b1889cc7719c5cbaa955df Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Mon, 23 Jan 2023 16:09:29 +0700 Subject: [PATCH 01/11] Switch from Travis to GitHub Actions --- .github/workflows/test.yml | 41 ++++++++++++++++++++++++++++++++ .travis.yml | 48 -------------------------------------- README.md | 2 +- 3 files changed, 42 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..2387ac9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,41 @@ +name: Test + +on: + push: + branches: + - "**" + pull_request: + branches: + - main + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + Test: + strategy: + matrix: + pg_branch: [master, REL_11_STABLE, REL_12_STABLE, REL_13_STABLE, REL_14_STABLE] + test_case: [tap, all] + mode: [none, legacy, paranoia] + exclude: + - test_case: tap + mode: paranoia + - test_case: all + mode: none + - test_case: all + mode: legacy + continue-on-error: true + runs-on: + - ubuntu-latest + env: + PG_BRANCH: ${{ matrix.pg_branch }} + TEST_CASE: ${{ matrix.test_case }} + steps: + - uses: actions/checkout@v2 + - name: Build the Docker image + run: | + ./make_dockerfile.sh + docker-compose build + - name: Test + run: docker-compose run $(bash <(curl -s https://codecov.io/env)) tests + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b6bd63a..0000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -os: linux - -dist: bionic - -language: c - -services: - - docker - -install: - - ./make_dockerfile.sh - - docker-compose build - -script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - -notifications: - email: - on_success: change - on_failure: always - -# keep in sync with codecov.yml number of builds -env: - - PG_BRANCH=master TEST_CASE=tap - - PG_BRANCH=master TEST_CASE=tap MODE=legacy -# - PG_BRANCH=master TEST_CASE=all - - PG_BRANCH=master TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_14_STABLE TEST_CASE=tap - - PG_BRANCH=REL_14_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_14_STABLE TEST_CASE=all - - PG_BRANCH=REL_14_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_13_STABLE TEST_CASE=tap - - PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_13_STABLE TEST_CASE=all - - PG_BRANCH=REL_13_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_12_STABLE TEST_CASE=tap - - PG_BRANCH=REL_12_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_12_STABLE TEST_CASE=all - - PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_11_STABLE TEST_CASE=tap - - PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_11_STABLE TEST_CASE=all - - PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia - -jobs: - allow_failures: - - if: env(PG_BRANCH) = master - diff --git a/README.md b/README.md index ece43af..02fd2f6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.com/postgrespro/ptrack.svg?branch=master)](https://travis-ci.com/postgrespro/ptrack) +![example workflow](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg) [![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) From 7bb1c9dc5ee6c15f601db88aff401cfc2a061d23 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 24 Jan 2023 11:14:30 +0700 Subject: [PATCH 02/11] Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02fd2f6..79af797 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![example workflow](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg) +![test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg) [![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) From 3a416565debc58e096b24de54cbb4d3871108213 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 24 Jan 2023 11:23:33 +0700 Subject: [PATCH 03/11] Continue on error only for experimental jobs --- .github/workflows/test.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2387ac9..1bf8fbe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,23 @@ jobs: Test: strategy: matrix: - pg_branch: [master, REL_11_STABLE, REL_12_STABLE, REL_13_STABLE, REL_14_STABLE] + pg_branch: [REL_11_STABLE, REL_12_STABLE, REL_13_STABLE, REL_14_STABLE] test_case: [tap, all] mode: [none, legacy, paranoia] + experimental: [false] + include: + - pg_branch: master + test_case: tap + mode: none + experimental: true + - pg_branch: master + test_case: tap + mode: legacy + experimental: true + - pg_branch: master + test_case: all + mode: paranoia + experimental: true exclude: - test_case: tap mode: paranoia @@ -24,7 +38,7 @@ jobs: mode: none - test_case: all mode: legacy - continue-on-error: true + continue-on-error: ${{ matrix.experimental }} runs-on: - ubuntu-latest env: From 1577a6d7e3833d3a175d701d18a16de3dc927966 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 24 Jan 2023 15:28:14 +0700 Subject: [PATCH 04/11] Don't terminate other parallel jobs if some fail --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1bf8fbe..e4aeedb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,6 +38,7 @@ jobs: mode: none - test_case: all mode: legacy + fail-fast: false continue-on-error: ${{ matrix.experimental }} runs-on: - ubuntu-latest From e854bf1f36748e4a24dd746c7b043d4d281309d6 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Thu, 26 Jan 2023 11:21:23 +0700 Subject: [PATCH 05/11] Link the status badge to the workflow results --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 79af797..f9eff7d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg) +[![Test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg)](https://github.com/postgrespro/ptrack/actions/workflows/test.yml) [![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) From ad21a7129b264416f05a5ffb726bf6655a021049 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 18 Aug 2023 09:27:25 +0700 Subject: [PATCH 06/11] Refactor the test workflow --- .github/workflows/test.yml | 116 +++++++++++++++++++++++---------- Dockerfile.in | 25 -------- Makefile | 43 ++++++++++++- README.md | 66 ++++++++++++++++--- codecov.yml | 5 +- docker-compose.yml | 17 ----- make_dockerfile.sh | 33 ---------- run_tests.sh | 127 ------------------------------------- 8 files changed, 183 insertions(+), 249 deletions(-) delete mode 100644 Dockerfile.in delete mode 100644 docker-compose.yml delete mode 100755 make_dockerfile.sh delete mode 100755 run_tests.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4aeedb..14756f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,43 +14,89 @@ jobs: Test: strategy: matrix: - pg_branch: [REL_11_STABLE, REL_12_STABLE, REL_13_STABLE, REL_14_STABLE] - test_case: [tap, all] - mode: [none, legacy, paranoia] - experimental: [false] - include: - - pg_branch: master - test_case: tap - mode: none - experimental: true - - pg_branch: master - test_case: tap - mode: legacy - experimental: true - - pg_branch: master - test_case: all - mode: paranoia - experimental: true + # pg_version: [15] + pg_version: [11, 12, 13, 14, 15] + os: [ubuntu-22.04] + # tests: [tap] + tests: [tap, python] + # test_mode: [normal, legacy, paranoia] + test_mode: [normal, paranoia] exclude: - - test_case: tap - mode: paranoia - - test_case: all - mode: none - - test_case: all - mode: legacy + - tests: tap + test_mode: paranoia + - tests: python + test_mode: normal + - tests: python + test_mode: legacy fail-fast: false - continue-on-error: ${{ matrix.experimental }} - runs-on: - - ubuntu-latest - env: - PG_BRANCH: ${{ matrix.pg_branch }} - TEST_CASE: ${{ matrix.test_case }} + name: ${{ format('Ptrack ({0}, PostgreSQL {1}, {2} tests, {3} mode)', matrix.os, matrix.pg_version, matrix.tests, matrix.test_mode) }} + container: + image: ${{ format('ghcr.io/postgres-dev/{0}:1.0', matrix.os) }} + env: + PG_BRANCH: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} + PREFIX: /home/dev/pgsql + LD_LIBRARY_PATH: /home/dev/pgsql/lib + PGDATA: /home/dev/data + TEST_MODE: ${{ matrix.test_mode }} + options: --privileged steps: - - uses: actions/checkout@v2 - - name: Build the Docker image + - name: Get Postgres sources + uses: actions/checkout@v3 + with: + repository: postgres/postgres + ref: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} + path: postgres + - name: Get Ptrack sources + uses: actions/checkout@v3 + with: + path: ptrack + - name: Get Pg_probackup sources + uses: actions/checkout@v3 + with: + repository: postgrespro/pg_probackup + path: pg_probackup + - name: Apply ptrack patches + run: make patch top_builddir=../postgres + working-directory: ptrack + - name: Install Postgres run: | - ./make_dockerfile.sh - docker-compose build + if [ $TEST_MODE = legacy ]; then + ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet + else + ./configure --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests + fi && + make -sj $(nproc) install && + make -C contrib -sj $(nproc) install && + echo $PREFIX/bin >> $GITHUB_PATH + working-directory: postgres + - name: Install Ptrack + run: make install USE_PGXS=1 PG_CPPFLAGS=-coverage SHLIB_LINK=-coverage + working-directory: ptrack + - name: Install Pg_probackup + run: make install-pg-probackup USE_PGXS=1 top_srcdir=../postgres + working-directory: ptrack + shell: bash {0} + - name: Install additional packages + run: | + apt update && + apt install -y python3-pip python3-six python3-pytest python3-pytest-xdist curl && + pip3 install --no-input testgres + # All steps have been so far executed by root but ptrack tests run from an + # unprivileged user so change some permissions + - name: Adjust the permissions of ptrack test folders + run: | + mkdir pg_probackup/tests/tmp_dirs + chown -R "dev:" pg_probackup ptrack - name: Test - run: docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - + run: make test-${{ matrix.tests }} USE_PGXS=1 + working-directory: ptrack + shell: runuser dev {0} + - name: Collect coverage results + run: make coverage + working-directory: ptrack + shell: runuser dev {0} + - name: Upload coverage results to Codecov + uses: codecov/codecov-action@v3 + with: + working-directory: ptrack + runs-on: ubuntu-latest diff --git a/Dockerfile.in b/Dockerfile.in deleted file mode 100644 index c2b0ffd..0000000 --- a/Dockerfile.in +++ /dev/null @@ -1,25 +0,0 @@ -FROM ololobus/postgres-dev:stretch - -USER root -RUN apt-get update -RUN apt-get -yq install python python-pip python-virtualenv - -# Environment -ENV PG_BRANCH=${PG_BRANCH} -ENV LANG=C.UTF-8 PGHOME=/testdir/pgbin -ENV MODE=${MODE} TEST_CASE=${TEST_CASE} TEST_REPEATS=${TEST_REPEATS} - -# Make directories -RUN mkdir -p /testdir - -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - -COPY . /testdir/ptrack -WORKDIR /testdir - -# Grant privileges -RUN chown -R postgres:postgres /testdir - -USER postgres -ENTRYPOINT /run.sh diff --git a/Makefile b/Makefile index e3d25a4..ce68344 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ + # contrib/ptrack/Makefile MODULE_big = ptrack @@ -20,8 +21,48 @@ PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else -subdir = contrib/ptrack top_builddir = ../.. +# Makefile.global is a build artifact and initially may not be available +ifneq ($(wildcard $(top_builddir)/src/Makefile.global), ) include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif +endif + +# Assuming make is started in the ptrack directory +patch: + @cd $(top_builddir) && \ + echo Applying the ptrack patch... && \ + git apply --verbose --3way $(CURDIR)/patches/${PG_BRANCH}-ptrack-core.diff +ifeq ($(MODE), paranoia) + @echo Applying turn-off-hint-bits.diff for the paranoia mode... && \ + git apply --verbose --3way $(CURDIR)/patches/turn-off-hint-bits.diff +endif + +TEST_MODE ?= normal +test-tap: +ifeq ($(TEST_MODE), legacy) + setarch x86_64 --addr-no-randomize $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) +else + $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) +endif + +pg_probackup_dir ?= $(CURDIR)/../pg_probackup +# Pg_probackup's Makefile uses top_srcdir when building via PGXS so set it when calling this target +# At the moment building pg_probackup with multiple threads may run some jobs too early and end with an error so do not set the -j option +install-pg-probackup: + $(MAKE) -C $(pg_probackup_dir) install USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) top_srcdir=$(top_srcdir) + +PYTEST_PROCESSES ?= $(shell nproc) +test-python: + cd $(pg_probackup_dir); \ + env="PG_PROBACKUP_PTRACK=ON PG_CONFIG=$(PG_CONFIG)"; \ + if [ "$(TEST_MODE)" = normal ]; then \ + env="$$env PG_PROBACKUP_TEST_BASIC=ON"; \ + elif [ "$(TEST_MODE)" = paranoia ]; then \ + env="$$env PG_PROBACKUP_PARANOIA=ON"; \ + fi; \ + env $$env python3 -m pytest -svv -n $(PYTEST_PROCESSES)$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py + +coverage: + gcov *.c *.h diff --git a/README.md b/README.md index f9eff7d..f1deb82 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg)](https://github.com/postgrespro/ptrack/actions/workflows/test.yml) -[![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) +[![Codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) # ptrack @@ -12,7 +12,7 @@ It is designed to allow false positives (i.e. block/page is marked in the `ptrac Currently, `ptrack` codebase is split between small PostgreSQL core patch and extension. All public SQL API methods and main engine are placed in the `ptrack` extension, while the core patch contains only certain hooks and modifies binary utilities to ignore `ptrack.map.*` files. -This extension is compatible with PostgreSQL [11](https://github.com/postgrespro/ptrack/blob/master/patches/REL_11_STABLE-ptrack-core.diff), [12](https://github.com/postgrespro/ptrack/blob/master/patches/REL_12_STABLE-ptrack-core.diff), [13](https://github.com/postgrespro/ptrack/blob/master/patches/REL_13_STABLE-ptrack-core.diff), [14](https://github.com/postgrespro/ptrack/blob/master/patches/REL_14_STABLE-ptrack-core.diff). +This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff), [16](patches/REL_16_STABLE-ptrack-core.diff). ## Installation @@ -162,20 +162,66 @@ Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare) ### Tests -Everything is tested automatically with [travis-ci.com](https://travis-ci.com/postgrespro/ptrack) and [codecov.io](https://codecov.io/gh/postgrespro/ptrack), but you can also run tests locally via `Docker`: +All changes of the source code in this repository are checked by CI - see commit statuses and the project status badge. You can also run tests locally by executing a few Makefile targets. + +#### Prerequisites + +To run Python tests inbstall the following packages: + +OS packages: + - python3-pip + - python3-six + - python3-pytest + - python3-pytest-xdist + +PIP packages: + - testgres + +For example, for Ubuntu: + +```sh +apt update +apt install python3-pip python3-six python3-pytest python3-pytest-xdist +pip3 install testgres +``` + +#### Testing ```sh -export PG_BRANCH=REL_14_STABLE -export TEST_CASE=all -export MODE=paranoia +export PG_BRANCH=REL_15_STABLE +export PREFIX=/path/to/pgsql +export PATH=$PREFIX/bin:$PATH + +cd /path/to/ptrack +make patch top_builddir=/path/to/postgres + +cd /path/to/postgres +./configure --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests +make -sj `nproc` install +make -C contrib -sj `nproc` install + +export USE_PGXS=1 -./make_dockerfile.sh +cd /path/to/ptrack +make install +make install-pg-probackup top_srcdir=/path/to/postgres +make test-tap +make test-python +``` + +You can use a public Docker image which already has the necessary build environment (but not the testing prerequisites): -docker-compose build -docker-compose run tests +```sh +docker run -e USER_ID=`id -u` -it -v $PWD:/work --name=ptrack ghcr.io/postgres-dev/ubuntu-22.04:1.0 +dev@a033797d2f73:~$ ``` -Available test modes (`MODE`) are `basic` (default) and `paranoia` (per-block checksum comparison of `PGDATA` content before and after backup-restore process). Available test cases (`TEST_CASE`) are `tap` (minimalistic PostgreSQL [tap test](https://github.com/postgrespro/ptrack/blob/master/t/001_basic.pl)), `all` or any specific [pg_probackup test](https://github.com/postgrespro/pg_probackup/blob/master/tests/ptrack.py), e.g. `test_ptrack_simple`. +You can control how tests are executed by using the following environment (or `make`) variables: + +| Variable | Possible values | Required | Default value | Description | +| - | - | - | - | - | +| TESTS | A Pytest filter expression | No | Not set (run all Python tests) | A filter to include only selected tests into the run. See the Pytest `-k` option for more information. This variable is only applicable to `test-python` for the tests located in [tests](https://github.com/postgrespro/pg_probackup/tree/master/tests). | +| TEST_MODE | normal, legacy, paranoia | No | normal | The "legacy" mode runs tests in an environment similar to a 32-bit Windows system. This mode is only applicable to `test-tap`. The "paranoia" mode compares the checksums of each block of the database catalog (PGDATA) contents before making a backuo and after the restoration. This mode is only applicable to `test-python`.| ### TODO diff --git a/codecov.yml b/codecov.yml index fe3b308..00b744e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,9 @@ codecov: notify: - after_n_builds: 12 # keep in sync with .travis.yml number of builds + # must be equal to the total number of parallel jobs in a CI pipeline + # (Postgres versions x test types x test modes x OSes minus excluded + # combinations) + after_n_builds: 10 # datapagemap.c/.h are copied from Postgres, so let's remove it # from report. Otherwise, we would have to remove some currently diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index fc65455..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.7" -services: - tests: - build: - context: . - - cap_add: - - SYS_PTRACE - - security_opt: - - seccomp=unconfined - - # don't work - #sysctls: - # kernel.yama.ptrace_scope: 0 - privileged: true - diff --git a/make_dockerfile.sh b/make_dockerfile.sh deleted file mode 100755 index 409a5b9..0000000 --- a/make_dockerfile.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env sh - -if [ -z ${PG_BRANCH+x} ]; then - echo PG_BRANCH is not set! - exit 1 -fi - -if [ -z ${MODE+x} ]; then - MODE=basic -else - echo MODE=${MODE} -fi - -if [ -z ${TEST_CASE+x} ]; then - TEST_CASE=all -else - echo TEST_CASE=${TEST_CASE} -fi - -if [ -z ${TEST_REPEATS+x} ]; then - TEST_REPEATS=1 -else - echo TEST_REPEATS=${TEST_REPEATS} -fi - -echo PG_BRANCH=${PG_BRANCH} - -sed \ - -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ - -e 's/${MODE}/'${MODE}/g \ - -e 's/${TEST_CASE}/'${TEST_CASE}/g \ - -e 's/${TEST_REPEATS}/'${TEST_REPEATS}/g \ -Dockerfile.in > Dockerfile diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index 1b4a693..0000000 --- a/run_tests.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright (c) 2019-2021, Postgres Professional -# - -PTRACK_SRC=${PWD}/ptrack -PG_SRC=${PWD}/postgres -PBK_SRC=${PWD}/pg_probackup -status=0 - -######################################################### -# Clone Postgres -echo "############### Getting Postgres sources" -git clone https://github.com/postgres/postgres.git --depth=1 --branch=${PG_BRANCH} ${PG_SRC} - -# Clone pg_probackup -echo "############### Getting pg_probackup sources" -git clone https://github.com/postgrespro/pg_probackup.git --depth=1 --branch=master ${PBK_SRC} - -######################################################### -# Compile and install Postgres -cd ${PG_SRC} # Go to postgres dir - -echo "############### Applying ptrack patch" -git apply --verbose --3way ${PTRACK_SRC}/patches/${PG_BRANCH}-ptrack-core.diff - -if [ "${MODE}" = "paranoia" ]; then - echo "############### Paranoia mode: applying turn-off-hint-bits.diff" - git apply --verbose --3way ${PTRACK_SRC}/patches/turn-off-hint-bits.diff -fi - -echo "############### Compiling Postgres" -if [ "${TEST_CASE}" = "tap" ] && [ "${MODE}" = "legacy" ]; then - ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet -else - ./configure --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet -fi -make --quiet --jobs=$(nproc) install -make --quiet --jobs=$(nproc) --directory=contrib/ install - -# Override default Postgres instance -export PATH=${PGHOME}/bin:${PATH} -export LD_LIBRARY_PATH=${PGHOME}/lib -export PG_CONFIG=$(which pg_config) - -# Show pg_config path (just in case) -echo "############### pg_config path" -which pg_config - -# Show pg_config just in case -echo "############### pg_config" -pg_config - -######################################################### -# Build and install ptrack extension -echo "############### Compiling and installing ptrack extension" -cp --recursive ${PTRACK_SRC} ${PG_SRC}/contrib/ptrack -make USE_PGXS=1 --directory=${PG_SRC}/contrib/ptrack/ clean -make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" --directory=${PG_SRC}/contrib/ptrack/ install - -if [ "${TEST_CASE}" = "tap" ]; then - - # Run tap tests - echo "############### Running tap tests" - if [ "${MODE}" = "legacy" ]; then - # There is a known issue with attaching shared memory segment using the same - # address each time, when EXEC_BACKEND mechanism is turned on. It happens due - # to the ASLR address space randomization, so we are trying to attach a segment - # to the already occupied location. That way we simply turning off ASLR here. - # - # Postgres comment: https://github.com/postgres/postgres/blob/5cbfce562f7cd2aab0cdc4694ce298ec3567930e/src/backend/postmaster/postmaster.c#L4929 - setarch x86_64 --addr-no-randomize make --directory=${PG_SRC}/contrib/ptrack check || status=$? - else - make --directory=${PG_SRC}/contrib/ptrack check || status=$? - fi - -else - # Set kernel params (used for debugging -- probackup tests) - echo "############### setting kernel params" - sudo sh -c 'echo 0 > /proc/sys/kernel/yama/ptrace_scope' - - # Build and install pg_probackup - echo "############### Compiling and installing pg_probackup" - cd ${PBK_SRC} # Go to pg_probackup dir - make USE_PGXS=1 top_srcdir=${PG_SRC} install - - # Setup python environment - echo "############### Setting up python env" - virtualenv --python=/usr/bin/python3 pyenv - source pyenv/bin/activate - pip install testgres==1.8.2 - - echo "############### Testing" - export PG_PROBACKUP_PTRACK=ON - if [ "${MODE}" = "basic" ]; then - export PG_PROBACKUP_TEST_BASIC=ON - elif [ "${MODE}" = "paranoia" ]; then - export PG_PROBACKUP_PARANOIA=ON - fi - - if [ "${TEST_CASE}" = "all" ]; then - # Run all pg_probackup ptrack tests - PBK_TEST_CASE=tests.ptrack - else - PBK_TEST_CASE=tests.ptrack.PtrackTest.${TEST_CASE} - fi - for i in `seq ${TEST_REPEATS}`; do - python3 -m unittest -v ${PBK_TEST_CASE} || status=$? - done - - # Exit virtualenv - deactivate -fi - -######################################################### -# codecov -echo "############### Codecov" -cd ${PTRACK_SRC} -# Generate *.gcov files -gcov ${PG_SRC}/contrib/ptrack/*.c ${PG_SRC}/contrib/ptrack/*.h - -# Send coverage stats to Codecov -bash <(curl -s https://codecov.io/bash) - -# Something went wrong, exit with code 1 -if [ ${status} -ne 0 ]; then exit 1; fi From 182e96a5093bb7e06f6181e445b27911f0bacd33 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 18 Aug 2023 09:52:23 +0700 Subject: [PATCH 07/11] [no ci] Remove the mentioning of the 16th version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1deb82..b4854d3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It is designed to allow false positives (i.e. block/page is marked in the `ptrac Currently, `ptrack` codebase is split between small PostgreSQL core patch and extension. All public SQL API methods and main engine are placed in the `ptrack` extension, while the core patch contains only certain hooks and modifies binary utilities to ignore `ptrack.map.*` files. -This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff), [16](patches/REL_16_STABLE-ptrack-core.diff). +This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff). ## Installation From a7f17a0befd60057924f74ecb1b4803748d75001 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 18 Aug 2023 14:50:29 +0700 Subject: [PATCH 08/11] Install Postgres from the ptrack Makefile --- .github/workflows/test.yml | 16 ++++------------ Makefile | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14756f0..f05773a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,9 +34,7 @@ jobs: image: ${{ format('ghcr.io/postgres-dev/{0}:1.0', matrix.os) }} env: PG_BRANCH: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} - PREFIX: /home/dev/pgsql - LD_LIBRARY_PATH: /home/dev/pgsql/lib - PGDATA: /home/dev/data + PGDATA: $HOME/data TEST_MODE: ${{ matrix.test_mode }} options: --privileged steps: @@ -60,15 +58,9 @@ jobs: working-directory: ptrack - name: Install Postgres run: | - if [ $TEST_MODE = legacy ]; then - ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet - else - ./configure --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests - fi && - make -sj $(nproc) install && - make -C contrib -sj $(nproc) install && - echo $PREFIX/bin >> $GITHUB_PATH - working-directory: postgres + make install-postgres top_builddir=$GITHUB_WORKSPACE/postgres prefix=$HOME/pgsql && + echo $HOME/pgsql/bin >> $GITHUB_PATH + working-directory: ptrack - name: Install Ptrack run: make install USE_PGXS=1 PG_CPPFLAGS=-coverage SHLIB_LINK=-coverage working-directory: ptrack diff --git a/Makefile b/Makefile index ce68344..33b06b0 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,24 @@ ifeq ($(MODE), paranoia) git apply --verbose --3way $(CURDIR)/patches/turn-off-hint-bits.diff endif +nproc := $(shell nproc) +prefix ?= $(abspath $(top_builddir)/pgsql) TEST_MODE ?= normal +# Postgres Makefile skips some targets depending on the MAKELEVEL variable so +# reset it when calling install targets as if they are started directly from the +# command line +install-postgres: + @cd $(top_builddir) && \ + if [ "$(TEST_MODE)" = legacy ]; then \ + ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet; \ + else \ + ./configure --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests; \ + fi && \ + $(MAKE) -sj $(nproc) install MAKELEVEL=0 && \ + $(MAKE) -sj $(nproc) -C contrib/ install MAKELEVEL=0 + +# Now when Postgres is built call all remainig targets with USE_PGXS=1 + test-tap: ifeq ($(TEST_MODE), legacy) setarch x86_64 --addr-no-randomize $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) From daae1656e0c4c3bca0ca9b3092d2ba9dc8811948 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Tue, 22 Aug 2023 15:34:06 +0700 Subject: [PATCH 09/11] Run Python tests in parallel only if pytest-xdist is available --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 33b06b0..9f3d62f 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test-python: elif [ "$(TEST_MODE)" = paranoia ]; then \ env="$$env PG_PROBACKUP_PARANOIA=ON"; \ fi; \ - env $$env python3 -m pytest -svv -n $(PYTEST_PROCESSES)$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py + env $$env python3 -m pytest -svv$(if $(shell python3 -m pytest --help | grep '\-n '), -n $(PYTEST_PROCESSES))$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py coverage: gcov *.c *.h From 2eaf70d1a5188ea219a52fcd92eddeac02a13f91 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Wed, 23 Aug 2023 09:03:12 +0700 Subject: [PATCH 10/11] Use the default pg_config location --- Makefile | 20 +++++------ README.md | 106 +++++++++++++++++++++++++++++++----------------------- 2 files changed, 71 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 9f3d62f..499067a 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,9 @@ TAP_TESTS = 1 # with Mkvcbuild.pm on PGv15+ PG_LIBS_INTERNAL += $(libpq_pgport) -ifdef USE_PGXS PG_CONFIG ?= pg_config + +ifdef USE_PGXS PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else @@ -33,14 +34,14 @@ endif patch: @cd $(top_builddir) && \ echo Applying the ptrack patch... && \ - git apply --verbose --3way $(CURDIR)/patches/${PG_BRANCH}-ptrack-core.diff + git apply --3way -v $(CURDIR)/patches/${PG_BRANCH}-ptrack-core.diff ifeq ($(MODE), paranoia) @echo Applying turn-off-hint-bits.diff for the paranoia mode... && \ - git apply --verbose --3way $(CURDIR)/patches/turn-off-hint-bits.diff + git apply --3way -v $(CURDIR)/patches/turn-off-hint-bits.diff endif -nproc := $(shell nproc) -prefix ?= $(abspath $(top_builddir)/pgsql) +NPROC ?= $(shell nproc) +prefix := $(abspath $(top_builddir)/pgsql) TEST_MODE ?= normal # Postgres Makefile skips some targets depending on the MAKELEVEL variable so # reset it when calling install targets as if they are started directly from the @@ -52,8 +53,8 @@ install-postgres: else \ ./configure --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests; \ fi && \ - $(MAKE) -sj $(nproc) install MAKELEVEL=0 && \ - $(MAKE) -sj $(nproc) -C contrib/ install MAKELEVEL=0 + $(MAKE) -sj $(NPROC) install MAKELEVEL=0 && \ + $(MAKE) -sj $(NPROC) -C contrib/ install MAKELEVEL=0 # Now when Postgres is built call all remainig targets with USE_PGXS=1 @@ -64,13 +65,12 @@ else $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) endif -pg_probackup_dir ?= $(CURDIR)/../pg_probackup +pg_probackup_dir = ../pg_probackup # Pg_probackup's Makefile uses top_srcdir when building via PGXS so set it when calling this target # At the moment building pg_probackup with multiple threads may run some jobs too early and end with an error so do not set the -j option install-pg-probackup: $(MAKE) -C $(pg_probackup_dir) install USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) top_srcdir=$(top_srcdir) -PYTEST_PROCESSES ?= $(shell nproc) test-python: cd $(pg_probackup_dir); \ env="PG_PROBACKUP_PTRACK=ON PG_CONFIG=$(PG_CONFIG)"; \ @@ -79,7 +79,7 @@ test-python: elif [ "$(TEST_MODE)" = paranoia ]; then \ env="$$env PG_PROBACKUP_PARANOIA=ON"; \ fi; \ - env $$env python3 -m pytest -svv$(if $(shell python3 -m pytest --help | grep '\-n '), -n $(PYTEST_PROCESSES))$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py + env $$env python3 -m pytest -svv$(if $(shell python3 -m pytest --help | grep '\-n '), -n $(NPROC))$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py coverage: gcov *.c *.h diff --git a/README.md b/README.md index b4854d3..92d922b 100644 --- a/README.md +++ b/README.md @@ -16,40 +16,62 @@ This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-c ## Installation -1) Get latest `ptrack` sources: +1) Specify the PostgreSQL branch to work with: ```shell -git clone https://github.com/postgrespro/ptrack.git +export PG_BRANCH=REL_15_STABLE +``` + +2) Get the latest PostgreSQL sources: + +```shell +git clone https://github.com/postgres/postgres.git -b $PG_BRANCH +``` + +3) Get the latest `ptrack` sources: + +```shell +git clone https://github.com/postgrespro/ptrack.git postgres/contrib/ptrack ``` -2) Get latest PostgreSQL sources: +4) Change to the `ptrack` directory: ```shell -git clone https://github.com/postgres/postgres.git -b REL_14_STABLE && cd postgres +cd postgres/contrib/ptrack ``` -3) Apply PostgreSQL core patch: +5) Apply the PostgreSQL core patch: ```shell -git apply -3 ../ptrack/patches/REL_14_STABLE-ptrack-core.diff +make patch ``` -4) Compile and install PostgreSQL +6) Compile and install PostgreSQL: -5) Set `ptrack.map_size` (in MB) +```shell +make install-postgres prefix=$PWD/pgsql # or some other prefix of your choice +``` + +7) Add the newly created binaries to the PATH: ```shell -echo "shared_preload_libraries = 'ptrack'" >> postgres_data/postgresql.conf -echo "ptrack.map_size = 64" >> postgres_data/postgresql.conf +export PATH=$PWD/pgsql/bin:$PATH ``` -6) Compile and install `ptrack` extension +8) Compile and install `ptrack`: ```shell -USE_PGXS=1 make -C /path/to/ptrack/ install +make install USE_PGXS=1 ``` -7) Run PostgreSQL and create `ptrack` extension +9) Set `ptrack.map_size` (in MB): + +```shell +echo "shared_preload_libraries = 'ptrack'" >> /postgresql.conf +echo "ptrack.map_size = 64" >> /postgresql.conf +``` + +10) Run PostgreSQL and create the `ptrack` extension: ```sql postgres=# CREATE EXTENSION ptrack; @@ -158,13 +180,13 @@ To gather the whole changeset of modified blocks in `ptrack_get_pagemapset()` we ## Contribution -Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare), [fill up issues](https://github.com/postgrespro/ptrack/issues/new), or just reach one of us directly (e.g. <[Alexey Kondratov](mailto:a.kondratov@postgrespro.ru?subject=[GitHub]%20Ptrack), [@ololobus](https://github.com/ololobus)>) if you are interested in `ptrack`. +Feel free to [send a pull request](https://github.com/postgrespro/ptrack/compare), [create an issue](https://github.com/postgrespro/ptrack/issues/new) or [reach us by e-mail](mailto:team-wd40@lists.postgrespro.ru??subject=[GitHub]%20Ptrack) if you are interested in `ptrack`. -### Tests +## Tests All changes of the source code in this repository are checked by CI - see commit statuses and the project status badge. You can also run tests locally by executing a few Makefile targets. -#### Prerequisites +### Prerequisites To run Python tests inbstall the following packages: @@ -179,49 +201,43 @@ PIP packages: For example, for Ubuntu: -```sh -apt update -apt install python3-pip python3-six python3-pytest python3-pytest-xdist -pip3 install testgres +```shell +sudo apt update +sudo apt install python3-pip python3-six python3-pytest python3-pytest-xdist +sudo pip3 install testgres ``` -#### Testing - -```sh -export PG_BRANCH=REL_15_STABLE -export PREFIX=/path/to/pgsql -export PATH=$PREFIX/bin:$PATH - -cd /path/to/ptrack -make patch top_builddir=/path/to/postgres - -cd /path/to/postgres -./configure --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests -make -sj `nproc` install -make -C contrib -sj `nproc` install +### Testing -export USE_PGXS=1 - -cd /path/to/ptrack -make install -make install-pg-probackup top_srcdir=/path/to/postgres -make test-tap +Install PostgreSQL and ptrack as described in [Installation](#installation), install the testing prerequisites, then do (assuming the current directory is `ptrack`): +```shell +git clone https://github.com/postgrespro/pg_probackup.git ../pg_probackup # clone the repository into postgres/contrib/pg_probackup +# remember to export PATH=/path/to/pgsql/bin:$PATH +make install-pg-probackup USE_PGXS=1 top_srcdir=../.. +make test-tap USE_PGXS=1 make test-python ``` +If `pg_probackup` is not located in `postgres/contrib` then additionally specify the path to the `pg_probackup` directory when building `pg_probackup`: +```shell +make install-pg-probackup USE_PGXS=1 top_srcdir=/path/to/postgres pg_probackup_dir=/path/to/pg_probackup +``` + You can use a public Docker image which already has the necessary build environment (but not the testing prerequisites): -```sh +```shell docker run -e USER_ID=`id -u` -it -v $PWD:/work --name=ptrack ghcr.io/postgres-dev/ubuntu-22.04:1.0 dev@a033797d2f73:~$ ``` -You can control how tests are executed by using the following environment (or `make`) variables: +## Environment variables -| Variable | Possible values | Required | Default value | Description | -| - | - | - | - | - | +| Variable | Possible values | Required | Default value | Description | +| - | - | - | - | - | +| NPROC | An integer greater than 0 | No | Output of `nproc` | The number of threads used for building and running tests | +| PG_CONFIG | File path | No | pg_config (from the PATH) | The path to the `pg_config` binary | | TESTS | A Pytest filter expression | No | Not set (run all Python tests) | A filter to include only selected tests into the run. See the Pytest `-k` option for more information. This variable is only applicable to `test-python` for the tests located in [tests](https://github.com/postgrespro/pg_probackup/tree/master/tests). | -| TEST_MODE | normal, legacy, paranoia | No | normal | The "legacy" mode runs tests in an environment similar to a 32-bit Windows system. This mode is only applicable to `test-tap`. The "paranoia" mode compares the checksums of each block of the database catalog (PGDATA) contents before making a backuo and after the restoration. This mode is only applicable to `test-python`.| +| TEST_MODE | normal, legacy, paranoia | No | normal | The "legacy" mode runs tests in an environment similar to a 32-bit Windows system. This mode is only applicable to `test-tap`. The "paranoia" mode compares the checksums of each block of the database catalog (PGDATA) contents before making a backup and after the restoration. This mode is only applicable to `test-python`.| ### TODO From 44a437a0cae8b15bc02f147f6e21e6c0ea3b698a Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 28 Aug 2023 18:34:44 +0700 Subject: [PATCH 11/11] Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 92d922b..8d257ef 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ All changes of the source code in this repository are checked by CI - see commit ### Prerequisites -To run Python tests inbstall the following packages: +To run Python tests install the following packages: OS packages: - python3-pip pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy