/etc/letsencrypt/renewal-hooks/deploy/envoy.deploy
+ #!/bin/bash
+ umask 0177
+ export DOMAIN=${CertificateSubdomain}.${CertificateHostedZone}
+ export DATA_DIR=/etc/envoy/certs/
+ cp /etc/letsencrypt/live/$DOMAIN/fullchain.pem $DATA_DIR/fullchain1.pem
+ cp /etc/letsencrypt/live/$DOMAIN/privkey.pem $DATA_DIR/privkey1.pem
+ EOF
+ sudo chmod +x /etc/letsencrypt/renewal-hooks/deploy/envoy.deploy
+
+ sudo systemctl enable envoy
+ sudo systemctl start envoy
+ fi
+
+ MountPoint:
+ Type: AWS::EC2::VolumeAttachment
+ Properties:
+ InstanceId: !Ref DLEInstance
+ VolumeId: !Ref ZFSVolume
+ Device: /dev/xvdh
+
+ DLEElasticIP:
+ Type: AWS::EC2::EIP
+ Properties:
+ Domain: vpc
+ InstanceId: !Ref DLEInstance
+
+ SubDomain:
+ Type: AWS::Route53::RecordSet
+ Condition: CreateSubDomain
+ Properties:
+ HostedZoneName: !Sub '${CertificateHostedZone}.'
+ Comment: DNS name for DLE instance.
+ Name: !Sub '${CertificateSubdomain}.${CertificateHostedZone}'
+ Type: CNAME
+ TTL: 60
+ ResourceRecords:
+ - !GetAtt DLEInstance.PublicDnsName
+ DependsOn:
+ - DLEInstance
+ - DLEElasticIP
+
+ DLESecurityGroup:
+ Type: 'AWS::EC2::SecurityGroup'
+ Properties:
+ GroupDescription: Enable ssh access via port 22
+ SecurityGroupIngress:
+ - IpProtocol: tcp
+ FromPort: 22
+ ToPort: 22
+ CidrIp: !Ref SSHLocation
+ SecurityGroupEgress:
+ - IpProtocol: -1
+ CidrIp: '0.0.0.0/0'
+ VpcId: !Ref VPC
+
+ DLEUISecurityGroup:
+ Type: 'AWS::EC2::SecurityGroup'
+ Condition: CreateSubDomain
+ Properties:
+ GroupDescription: Enable ports to access DLE UI
+ SecurityGroupIngress:
+ - IpProtocol: tcp
+ FromPort: 80
+ ToPort: 80
+ CidrIp: !Ref SSHLocation
+
+ - IpProtocol: tcp
+ FromPort: 443
+ ToPort: 443
+ CidrIp: !Ref SSHLocation
+
+ - IpProtocol: tcp
+ FromPort: 446
+ ToPort: 446
+ CidrIp: !Ref SSHLocation
+ SecurityGroupEgress:
+ - IpProtocol: -1
+ CidrIp: '0.0.0.0/0'
+ VpcId: !Ref VPC
+
+Outputs:
+ VerificationToken:
+ Description: 'DLE verification token'
+ Value: !Ref DLEVerificationToken
+
+ DLE:
+ Description: URL for newly created DLE instance
+ Value: !Sub 'https://${CertificateSubdomain}.${CertificateHostedZone}'
+ Condition: CreateSubDomain
+
+ UI:
+ Description: UI URL with a domain for newly created DLE instance
+ Value: !Sub 'https://${CertificateSubdomain}.${CertificateHostedZone}:446'
+ Condition: CreateSubDomain
+
+ EBSVolume:
+ Description: Size of provisioned EBS volume
+ Value: !GetAtt SizeCalculate.Value
+
+ DNSName:
+ Description: Public DNS name
+ Value: !GetAtt DLEInstance.PublicDnsName
+
+ EC2SSH:
+ Description: SSH connection to the EC2 instance with Database Lab Engine
+ Value: !Sub
+ - 'ssh -i YOUR_PRIVATE_KEY ubuntu@${DNSName}'
+ - DNSName: !GetAtt DLEInstance.PublicDnsName
+
+ DLETunnel:
+ Description: Create an SSH-tunnel to Database Lab Engine
+ Value: !Sub
+ - 'ssh -N -L 2345:${DNSName}:2345 -i YOUR_PRIVATE_KEY ubuntu@${DNSName}'
+ - DNSName: !GetAtt DLEInstance.PublicDnsName
+
+ UITunnel:
+ Description: Create an SSH-tunnel to Database Lab UI
+ Value: !Sub
+ - 'ssh -N -L 2346:${DNSName}:2346 -i YOUR_PRIVATE_KEY ubuntu@${DNSName}'
+ - DNSName: !GetAtt DLEInstance.PublicDnsName
+
+ CloneTunnel:
+ Description: Create an SSH-tunnel to Database Lab clones
+ Value: !Sub
+ - 'ssh -N -L CLONE_PORT:${DNSName}:CLONE_PORT -i YOUR_PRIVATE_KEY ubuntu@${DNSName}'
+ - DNSName: !GetAtt DLEInstance.PublicDnsName
+
diff --git a/cloudformation/getAMIs.sh b/cloudformation/getAMIs.sh
new file mode 100755
index 00000000..27a71304
--- /dev/null
+++ b/cloudformation/getAMIs.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# This script takes a a parameter which needs to be a name of an AWS AMI
+# The string will have to identify the AMI uniquely in all regions.
+# The script will generate an output which can be copied into json files of AWS CloudFormation
+#
+# The script uses the AWS command line tools.
+# The AWS command line tools have to have a default profile with the permission to
+# describe a region and to describe an image
+
+# The script can be run with normal OS user privileges.
+# The script is not supposed to modify anything.
+# There is no warranty. Please check the script upfront. You will use it on your own risk
+# String to be used when no AMI is available in region
+NOAMI="NOT_SUPPORTED"
+# Change your aws prfile if needed here:
+PROFILE=" --profile default"
+# Check whether AWS CLI is installed and in search path
+if ! aws_loc="$(type -p "aws")" || [ -z "$aws_loc" ]; then
+echo "Error: Script requeres AWS CLI . Install it and retry"
+exit 1
+fi
+# Check whether parameter has been provided
+if [ -z "$1" ]
+then
+NAME=DBLABserver*
+echo "No parameter provided."
+else
+NAME=$1
+fi
+echo "Will search for AMIs with name: ${NAME}"
+echo "---------------------------------------"
+##NAME=DBLABserver*
+R=$(aws ec2 describe-regions --query "Regions[].{Name:RegionName}" --output text ${PROFILE})
+for i in $R; do
+AMI=`aws ec2 describe-images --owners 005923036815 --region $i --filters "Name=name,Values=${NAME}" --output json | jq -r '.Images | sort_by(.CreationDate) | last(.[]).ImageId'`
+if [ -z "$AMI" ]
+then
+AMI=$NOAMI
+fi
+echo " "${i}: $'\n' " "HVM64: ${AMI}
+done
+
diff --git a/configs/config.example.physical_pgbackrest.yml b/configs/config.example.physical_pgbackrest.yml
deleted file mode 100644
index e69de29b..00000000
diff --git a/engine/.gitlab-ci.yml b/engine/.gitlab-ci.yml
index eb52e7fa..0d47c139 100644
--- a/engine/.gitlab-ci.yml
+++ b/engine/.gitlab-ci.yml
@@ -159,6 +159,17 @@ build-image-feature-server:
DOCKER_NAME: "${CI_REGISTRY}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/dblab-server"
TAGS: "${DOCKER_NAME}:${CI_COMMIT_REF_SLUG}"
+build-image-feature-server-zfs08:
+ <<: *build_image_definition
+ <<: *only_feature
+ variables:
+ REGISTRY_USER: "${CI_REGISTRY_USER}"
+ REGISTRY_PASSWORD: "${CI_REGISTRY_PASSWORD}"
+ REGISTRY: "${CI_REGISTRY}"
+ DOCKER_FILE: "Dockerfile.dblab-server-zfs08"
+ DOCKER_NAME: "${CI_REGISTRY}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/dblab-server"
+ TAGS: "${DOCKER_NAME}:${CI_COMMIT_REF_SLUG}-zfs0.8"
+
build-image-feature-ci-checker:
<<: *build_image_definition
<<: *only_feature
@@ -189,6 +200,14 @@ build-image-master-server:
DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-server"
TAGS: "${DOCKER_NAME}:master,${DOCKER_NAME}:master-${CI_COMMIT_SHORT_SHA}"
+build-image-master-server-zfs08:
+ <<: *build_image_definition
+ <<: *only_master
+ variables:
+ DOCKER_FILE: "Dockerfile.dblab-server-zfs08"
+ DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-server"
+ TAGS: "${DOCKER_NAME}:master-zfs0.8,${DOCKER_NAME}:master-${CI_COMMIT_SHORT_SHA}-zfs0.8"
+
build-image-master-ci-checker:
<<: *build_image_definition
<<: *only_master
@@ -219,6 +238,20 @@ build-image-latest-server:
- export LATEST_TAG=$(echo ${CLEAN_TAG%.*}-latest)
- export TAGS="${DOCKER_NAME}:${LATEST_TAG},${DOCKER_NAME}:${CLEAN_TAG}"
+build-image-latest-server-zfs08:
+ <<: *build_image_definition
+ <<: *only_tag_release
+ variables:
+ REGISTRY_USER: "${DH_CI_REGISTRY_USER}"
+ REGISTRY_PASSWORD: "${DH_CI_REGISTRY_PASSWORD}"
+ REGISTRY: "${DH_CI_REGISTRY}"
+ DOCKER_FILE: "Dockerfile.dblab-server-zfs08"
+ DOCKER_NAME: "postgresai/dblab-server"
+ before_script:
+ - export CLEAN_TAG=$(echo ${CI_COMMIT_TAG#"v"})
+ - export LATEST_TAG=$(echo ${CLEAN_TAG%.*}-latest)
+ - export TAGS="${DOCKER_NAME}:${LATEST_TAG}-zfs0.8,${DOCKER_NAME}:${CLEAN_TAG}-zfs0.8"
+
build-image-latest-server-dev:
<<: *build_image_definition
<<: *only_tag_release
@@ -286,6 +319,19 @@ build-image-rc-server:
DOCKER_FILE: "Dockerfile.dblab-server"
DOCKER_NAME: "postgresai/dblab-server"
+build-image-rc-server-zfs08:
+ <<: *build_image_definition
+ <<: *only_tag_rc
+ before_script:
+ - export CLEAN_TAG=$(echo ${CI_COMMIT_TAG#"v"})
+ - export TAGS="${DOCKER_NAME}:${CLEAN_TAG}-zfs0.8"
+ variables:
+ REGISTRY_USER: "${DH_CI_REGISTRY_USER}"
+ REGISTRY_PASSWORD: "${DH_CI_REGISTRY_PASSWORD}"
+ REGISTRY: "${DH_CI_REGISTRY}"
+ DOCKER_FILE: "Dockerfile.dblab-server-zfs08"
+ DOCKER_NAME: "postgresai/dblab-server"
+
build-image-rc-server-dev:
<<: *build_image_definition
<<: *only_tag_rc
@@ -299,6 +345,19 @@ build-image-rc-server-dev:
DOCKER_FILE: "Dockerfile.dblab-server"
DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-server"
+build-image-rc-server-dev-zfs08:
+ <<: *build_image_definition
+ <<: *only_tag_rc
+ before_script:
+ - export CLEAN_TAG=$(echo ${CI_COMMIT_TAG#"v"})
+ - export TAGS="${DOCKER_NAME}:${CLEAN_TAG}-zfs0.8"
+ variables:
+ REGISTRY_USER: "${CI_REGISTRY_USER}"
+ REGISTRY_PASSWORD: "${CI_REGISTRY_PASSWORD}"
+ REGISTRY: "${CI_REGISTRY}"
+ DOCKER_FILE: "Dockerfile.dblab-server-zfs08"
+ DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-server"
+
build-image-rc-ci-checker:
<<: *build_image_definition
<<: *only_tag_rc
diff --git a/engine/Dockerfile.dblab-server b/engine/Dockerfile.dblab-server
index c3f1de89..ba5a22bb 100644
--- a/engine/Dockerfile.dblab-server
+++ b/engine/Dockerfile.dblab-server
@@ -2,8 +2,10 @@
FROM docker:20.10.12
-# Install dependencies.
-RUN apk update && apk add --no-cache zfs lvm2 bash util-linux
+# Install dependencies
+RUN apk update \
+ && apk add zfs=2.1.4-r0 --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main \
+ && apk add --no-cache lvm2 bash util-linux
RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/main' >> /etc/apk/repositories \
&& echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/community' >> /etc/apk/repositories \
&& apk add bcc-tools=0.18.0-r0 bcc-doc=0.18.0-r0 && ln -s $(which python3) /usr/bin/python \
diff --git a/engine/Dockerfile.dblab-server-zfs2 b/engine/Dockerfile.dblab-server-zfs08
similarity index 86%
rename from engine/Dockerfile.dblab-server-zfs2
rename to engine/Dockerfile.dblab-server-zfs08
index 48dd566e..02d55262 100644
--- a/engine/Dockerfile.dblab-server-zfs2
+++ b/engine/Dockerfile.dblab-server-zfs08
@@ -2,9 +2,8 @@
FROM docker:20.10.12
-# Install dependencies
-RUN apk update \
- && apk add zfs=2.1.2-r0 --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main \
+# Install dependencies.
+RUN apk update && apk add zfs=0.8.4-r0 --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/v3.12/main \
&& apk add --no-cache lvm2 bash util-linux
RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/main' >> /etc/apk/repositories \
&& echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/community' >> /etc/apk/repositories \
diff --git a/engine/configs/config.example.logical_generic.yml b/engine/configs/config.example.logical_generic.yml
index 0033fad3..a5bcc160 100644
--- a/engine/configs/config.example.logical_generic.yml
+++ b/engine/configs/config.example.logical_generic.yml
@@ -122,7 +122,7 @@ databaseConfigs: &db_configs
# Adding shared preload libraries, make sure that there are "pg_stat_statements, auto_explain, logerrors" in the list.
# They are needed for query analysis and DB migration testing.
# Note, if you are using PostgreSQL 9.6 and older, remove the logerrors extension from the list since it is not supported.
- shared_preload_libraries: "pg_stat_statements, auto_explain, logerrors"
+ shared_preload_libraries: "pg_stat_statements, pg_stat_kcache, auto_explain, logerrors"
# work_mem and all the Query Planning parameters – copy the values from the source.
# Detailed guide: https://postgres.ai/docs/how-to-guides/administration/postgresql-configuration#postgresql-configuration-in-clones
work_mem: "100MB"
@@ -262,7 +262,7 @@ retrieval:
# Option to adjust PostgreSQL configuration for a logical restore job
# It's useful if a restored database contains non-standard extensions.
- # <<: *db_configs
+ <<: *db_configs
# Option for specifying the database list that must be restored.
# By default, DLE restores all available databases.
diff --git a/engine/configs/config.example.logical_rds_iam.yml b/engine/configs/config.example.logical_rds_iam.yml
index a7a27d55..02d5655e 100644
--- a/engine/configs/config.example.logical_rds_iam.yml
+++ b/engine/configs/config.example.logical_rds_iam.yml
@@ -122,7 +122,7 @@ databaseConfigs: &db_configs
# Adding shared preload libraries, make sure that there are "pg_stat_statements, auto_explain, logerrors" in the list.
# It is necessary to perform query and db migration analysis.
# Note, if you are using PostgreSQL 9.6 and older, remove the logerrors extension from the list since it is not supported.
- shared_preload_libraries: "pg_stat_statements, auto_explain, logerrors"
+ shared_preload_libraries: "pg_stat_statements, pg_stat_kcache, auto_explain, logerrors"
# work_mem and all the Query Planning parameters – copy the values from the source.
# To do it, use this query:
# select format($$%s = '%s'$$, name, setting)
@@ -270,7 +270,7 @@ retrieval:
# Option to adjust PostgreSQL configuration for a logical restore job
# It's useful if a restored database contains non-standard extensions.
- # <<: *db_configs
+ <<: *db_configs
# Option for specifying the database list that must be restored.
# By default, DLE restores all available databases.
diff --git a/engine/configs/config.example.physical_generic.yml b/engine/configs/config.example.physical_generic.yml
index 30f31ece..df15ee63 100644
--- a/engine/configs/config.example.physical_generic.yml
+++ b/engine/configs/config.example.physical_generic.yml
@@ -122,7 +122,7 @@ databaseConfigs: &db_configs
# Adding shared preload libraries, make sure that there are "pg_stat_statements, auto_explain, logerrors" in the list.
# It is necessary to perform query and db migration analysis.
# Note, if you are using PostgreSQL 9.6 and older, remove the logerrors extension from the list since it is not supported.
- shared_preload_libraries: "pg_stat_statements, auto_explain, logerrors"
+ shared_preload_libraries: "pg_stat_statements, pg_stat_kcache, auto_explain, logerrors"
# work_mem and all the Query Planning parameters – copy the values from the source.
# To do it, use this query:
# select format($$%s = '%s'$$, name, setting)
diff --git a/engine/configs/config.example.physical_pgbackrest.yml b/engine/configs/config.example.physical_pgbackrest.yml
index 5aef0675..0a1642b9 100644
--- a/engine/configs/config.example.physical_pgbackrest.yml
+++ b/engine/configs/config.example.physical_pgbackrest.yml
@@ -122,7 +122,7 @@ databaseConfigs: &db_configs
# Adding shared preload libraries, make sure that there are "pg_stat_statements, auto_explain, logerrors" in the list.
# It is necessary to perform query and db migration analysis.
# Note, if you are using PostgreSQL 9.6 and older, remove the logerrors extension from the list since it is not supported.
- shared_preload_libraries: "pg_stat_statements, auto_explain, logerrors"
+ shared_preload_libraries: "pg_stat_statements, pg_stat_kcache, auto_explain, logerrors"
# work_mem and all the Query Planning parameters – copy the values from the source.
# To do it, use this query:
# select format($$%s = '%s'$$, name, setting)
diff --git a/engine/configs/config.example.physical_walg.yml b/engine/configs/config.example.physical_walg.yml
index 4b7090fb..1dc860bf 100644
--- a/engine/configs/config.example.physical_walg.yml
+++ b/engine/configs/config.example.physical_walg.yml
@@ -122,7 +122,7 @@ databaseConfigs: &db_configs
# Adding shared preload libraries, make sure that there are "pg_stat_statements, auto_explain, logerrors" in the list.
# It is necessary to perform query and db migration analysis.
# Note, if you are using PostgreSQL 9.6 and older, remove the logerrors extension from the list since it is not supported.
- shared_preload_libraries: "pg_stat_statements, auto_explain, logerrors"
+ shared_preload_libraries: "pg_stat_statements, pg_stat_kcache, auto_explain, logerrors"
# work_mem and all the Query Planning parameters – copy the values from the source.
# To do it, use this query:
# select format($$%s = '%s'$$, name, setting)
diff --git a/engine/internal/observer/stats.go b/engine/internal/observer/stats.go
index be450e92..87470cd9 100644
--- a/engine/internal/observer/stats.go
+++ b/engine/internal/observer/stats.go
@@ -105,7 +105,22 @@ func (c *ObservingClone) getDBSize(ctx context.Context, dbSize *int64) error {
// getMaxQueryTime gets maximum query duration.
func (c *ObservingClone) getMaxQueryTime(ctx context.Context, maxTime *float64) error {
- row := c.superUserDB.QueryRow(ctx, "select max(max_time) from pg_stat_statements")
+ var pgVersion int
+
+ row := c.superUserDB.QueryRow(ctx, "select current_setting('server_version_num')::int")
+
+ if err := row.Scan(&pgVersion); err != nil {
+ return fmt.Errorf("failed to define Postgres Version: %w", err)
+ }
+
+ const pgVersion13 = 130000
+
+ maxTimeQuery := "select max(max_time) from pg_stat_statements"
+ if pgVersion >= pgVersion13 {
+ maxTimeQuery = "select max(max_exec_time + max_plan_time) from pg_stat_statements"
+ }
+
+ row = c.superUserDB.QueryRow(ctx, maxTimeQuery)
return row.Scan(&maxTime)
}
diff --git a/engine/internal/provision/databases/postgres/postgres_mgmt.go b/engine/internal/provision/databases/postgres/postgres_mgmt.go
index c396bae6..6326dd95 100644
--- a/engine/internal/provision/databases/postgres/postgres_mgmt.go
+++ b/engine/internal/provision/databases/postgres/postgres_mgmt.go
@@ -114,6 +114,7 @@ declare
begin
new_owner := @usernameStr;
+ -- Schemas
-- allow working with all schemas
for r in select * from pg_namespace loop
raise debug 'Changing ownership of schema % to %',
@@ -125,6 +126,31 @@ begin
);
end loop;
+ -- Types and Domains
+ -- d: domain (assuming that ALTER TYPE will be equivalent to ALTER DOMAIN)
+ -- e: enum
+ -- r: range
+ -- m: multirange
+ for r in
+ select n.nspname, t.typname
+ from pg_type t
+ join pg_namespace n on
+ n.oid = t.typnamespace
+ and not n.nspname in ('pg_catalog', 'information_schema')
+ and t.typtype in ('d', 'e', 'r', 'm')
+ order by t.typname
+ loop
+ raise debug 'Changing ownership of type %.% to %',
+ r.nspname, r.typname, new_owner;
+ execute format(
+ 'alter type %I.%I owner to %I;',
+ r.nspname,
+ r.typname,
+ new_owner
+ );
+ end loop;
+
+ -- Relations
-- c: composite type
-- p: partitioned table
-- i: index
@@ -163,9 +189,10 @@ begin
end loop;
end loop;
- -- Functions,
+ -- Functions and Procedures,
for r in
select
+ p.prokind,
p.proname,
n.nspname,
pg_catalog.pg_get_function_identity_arguments(p.oid) as args
@@ -173,11 +200,19 @@ begin
join pg_catalog.pg_proc as p on p.pronamespace = n.oid
where not n.nspname in ('pg_catalog', 'information_schema')
and p.proname not ilike 'dblink%' -- We do not want dblink to be involved (exclusion)
+ and p.prokind in ('f', 'p', 'a', 'w')
loop
raise debug 'Changing ownership of function %.%(%) to %',
r.nspname, r.proname, r.args, new_owner;
execute format(
- 'alter function %I.%I(%s) owner to %I', -- todo: check support CamelStyle r.args
+ 'alter %s %I.%I(%s) owner to %I', -- todo: check support CamelStyle r.args,
+ case r.prokind
+ when 'f' then 'function'
+ when 'w' then 'function'
+ when 'p' then 'procedure'
+ when 'a' then 'aggregate'
+ else 'unknown'
+ end,
r.nspname,
r.proname,
r.args,
diff --git a/engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go b/engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go
index 31337247..b3b63c0a 100644
--- a/engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go
+++ b/engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go
@@ -17,7 +17,7 @@ const (
hcInterval = 5 * time.Second
hcTimeout = 2 * time.Second
hcStartPeriod = 3 * time.Second
- hcRetries = 5
+ hcRetries = 15
// DefaultRestoreInterval defines a default health check interval for database restoration.
DefaultRestoreInterval = 5 * time.Second
diff --git a/engine/internal/retrieval/engine/postgres/tools/tools.go b/engine/internal/retrieval/engine/postgres/tools/tools.go
index 12281ba0..a02bc43e 100644
--- a/engine/internal/retrieval/engine/postgres/tools/tools.go
+++ b/engine/internal/retrieval/engine/postgres/tools/tools.go
@@ -302,7 +302,8 @@ func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, c
return nil
case types.Unhealthy:
- return errors.New("container health check failed")
+ return fmt.Errorf("container health check failed. The maximum number of attempts has reached: %d",
+ resp.Config.Healthcheck.Retries)
}
if healthCheckLength := len(resp.State.Health.Log); healthCheckLength > 0 {
diff --git a/packer/install-dblabcli.sh b/packer/install-dblabcli.sh
index 5865a80d..0065fe25 100644
--- a/packer/install-dblabcli.sh
+++ b/packer/install-dblabcli.sh
@@ -1,12 +1,28 @@
#/!bin/bash
set -x
+sudo su - ubuntu
mkdir ~/.dblab
-curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/scripts/cli_install.sh | bash
+curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/engine/scripts/cli_install.sh | bash
sudo mv ~/.dblab/dblab /usr/local/bin/dblab
echo $dle_version > ~/.dblab/dle_version
-sudo curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/configs/config.example.logical_generic.yml --output ~/.dblab/config.example.logical_generic.yml
-sudo curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/configs/config.example.logical_rds_iam.yml --output ~/.dblab/config.example.logical_rds_iam.yml
-sudo curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/configs/config.example.physical_generic.yml --output ~/.dblab/config.example.physical_generic.yml
-sudo curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/configs/config.example.physical_walg.yml --output ~/.dblab/config.example.physical_walg.yml
+# Copy base templates
+curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/engine/configs/config.example.logical_generic.yml --output ~/.dblab/config.example.logical_generic.yml
+curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/engine/configs/config.example.logical_rds_iam.yml --output ~/.dblab/config.example.logical_rds_iam.yml
+curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/engine/configs/config.example.physical_generic.yml --output ~/.dblab/config.example.physical_generic.yml
+curl https://gitlab.com/postgres-ai/database-lab/-/raw/$dle_version/engine/configs/config.example.physical_walg.yml --output ~/.dblab/config.example.physical_walg.yml
+
+# Adjust DLE config
+dle_config_path="/home/ubuntu/.dblab/engine/configs"
+dle_meta_path="/home/ubuntu/.dblab/engine/meta"
+postgres_conf_path="/home/ubuntu/.dblab/postgres_conf"
+
+mkdir -p $dle_config_path
+mkdir -p $dle_meta_path
+mkdir -p $postgres_conf_path
+
+curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/engine/configs/config.example.logical_generic.yml --output $dle_config_path/server.yml
+curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/engine/configs/standard/postgres/control/pg_hba.conf \
+ --output $postgres_conf_path/pg_hba.conf
+curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/engine/configs/standard/postgres/control/postgresql.conf --output $postgres_conf_path/postgresql.conf
diff --git a/packer/install-prereqs.sh b/packer/install-prereqs.sh
index 9f277589..7fadff84 100644
--- a/packer/install-prereqs.sh
+++ b/packer/install-prereqs.sh
@@ -2,11 +2,28 @@
set -euxo pipefail
+### Upgrade the existing software
sudo apt update -y
sudo apt upgrade -y
sudo apt full-upgrade
-sudo apt-get update && sudo apt-get install -y \
+### Extend the list of apt repositories
+# yq repo
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+sudo add-apt-repository ppa:rmescandon/yq
+
+# Docker repo
+wget --quiet -O - https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+
+# Postgres PGDG repo
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+sudo add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main"
+
+sudo apt-get update -y
+
+### Installation steps – first, using apt, then non-apt steps, and, finally, get Docker container for DLE
+sudo apt-get install -y \
apt-transport-https \
ca-certificates \
gnupg-agent \
@@ -15,36 +32,22 @@ sudo apt-get update && sudo apt-get install -y \
curl \
gnupg2 \
zfsutils-linux \
-
-# Install Docker
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-
-sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-sudo apt-get update && sudo apt-get install -y \
docker-ce \
docker-ce-cli \
- containerd.io
-
-#sudo docker pull postgresai/dblab-server:$dle_version
-
-#install postgres client
-wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
-echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee /etc/apt/sources.list.d/pgdg.list
-sudo apt-get update && sudo apt-get install -y postgresql-client-13
+ containerd.io \
+ postgresql-client-14 \
+ s3fs \
+ yq \
+ jq
-#install certbot
+# Install certbot
sudo snap install certbot --classic
sudo ln -s /snap/bin/certbot /usr/bin/certbot
-#install envoy
+# Install Envoy
curl https://func-e.io/install.sh | sudo bash -s -- -b /usr/local/bin
sudo /usr/local/bin/func-e use 1.19.1 # https://www.envoyproxy.io/docs/envoy/latest/version_history/v1.20.0#incompatible-behavior-changes
-
-#install s3fs
-sudo apt install s3fs
-
-#install yq
-sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
-sudo add-apt-repository ppa:rmescandon/yq
-sudo apt update && sudo apt install yq -y
+# Pull DLE image
+image_version=$(echo ${dle_version} | sed 's/v*//')
+sudo docker pull registry.gitlab.com/postgres-ai/database-lab/dblab-server:$image_version
diff --git a/packer/template.json.pkr.hcl b/packer/template.json.pkr.hcl
index e23203e2..ab4c579c 100644
--- a/packer/template.json.pkr.hcl
+++ b/packer/template.json.pkr.hcl
@@ -12,7 +12,7 @@ data "amazon-ami" "base" {
filters = {
architecture = "x86_64"
"block-device-mapping.volume-type" = "gp2"
- name = "*ubuntu-focal-20.04-amd64-server-*"
+ name = "*ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"
root-device-type = "ebs"
virtualization-type = "hvm"
}
@@ -21,7 +21,7 @@ data "amazon-ami" "base" {
}
source "amazon-ebs" "base" {
- ami_description = "Installed AMI with Ubuntu 20.04, ZFS, Docker, Envoy proxy and Database Lab Engine 2.0 with client CLI."
+ ami_description = "Ubuntu 20.04 with ZFS, Docker, Envoy proxy and Database Lab Engine 3.1 with client CLI."
ami_name = "${var.ami_name_prefix}-${var.dle_version}-${formatdate("YYYY-MM-DD", timestamp())}-${uuidv4()}"
instance_type = "t2.large"
source_ami = "${data.amazon-ami.base.id}"
@@ -50,7 +50,10 @@ build {
provisioner "shell" {
environment_vars = ["dle_version=${var.dle_version}"]
- scripts = ["${path.root}/install-prereqs.sh", "${path.root}/install-envoy.sh"]
+ scripts = ["${path.root}/install-prereqs.sh", "${path.root}/install-envoy.sh","${path.root}/install-dblabcli.sh"]
}
+ provisioner "shell" {
+ inline = ["echo 'remove authorized keys'", "sudo rm /home/ubuntu/.ssh/authorized_keys", "sudo rm /root/.ssh/authorized_keys"]
+ }
}
diff --git a/translations/README.german.md b/translations/README.german.md
index c9c9844e..f0b45ccb 100644
--- a/translations/README.german.md
+++ b/translations/README.german.md
@@ -1,4 +1,9 @@
-
+
+
+
Database Lab Engine (DLE)
@@ -167,8 +172,6 @@ Der DLE-Quellcode ist unter der vom OSI genehmigten Open-Source-Lizenz GNU Affer
Wenden Sie sich an das Postgres.ai-Team, wenn Sie eine Test- oder kommerzielle Lizenz wünschen, die die GPL-Klauseln nicht enthält: [Kontaktseite](https://postgres.ai/contact).
-[](https://app .fossa.io/projects/git%2Bgithub.com%2Fpostgres-ai%2Fdatabase-lab-engine?ref=badge_large)
-
## Gemeinschaftliche Unterstützung
- ["Verhaltenskodex der Database Lab Engine Community Covenant"](../CODE_OF_CONDUCT.md)
- Wo Sie Hilfe bekommen: [Kontaktseite](https://postgres.ai/contact)
diff --git a/translations/README.portuguese-br.md b/translations/README.portuguese-br.md
new file mode 100644
index 00000000..f11cfe11
--- /dev/null
+++ b/translations/README.portuguese-br.md
@@ -0,0 +1,183 @@
+
+
+
+
+Database Lab Engine (DLE)
+
+
+
+
+ :zap: Clonagem ultrarrápida de bancos de dados PostgreSQL :elephant:
+ Thin clones de bancos de dados PostgreSQL para viabilizar desenvolvimento, testes, QA e ambientes de staging poderosos.
+ Disponível para qualquer PostgreSQL, incluindo AWS RDS, GCP CloudSQL, Heroku, Digital Ocean e instâncias autoadministradas.
+
+
+
+
+
+
+
+
+## Por que DLE?
+- Construa ambientes de dev/QA/staging baseados nos bancos de dados de produção completos.
+- Disponibilize clones temporários completos dos bancos de dados de produção para análises de queries e otimizações (veja também: [SQL optimization chatbot Joe](https://gitlab.com/postgres-ai/joe)).
+- Faça testes automatizados em pipelines de integração contínua para evitar incidentes em produção.
+
+Por examplo, clonar um banco de dados PostgreSQL de 1 TiB dura ~10 segundos. Dezenas de clones independentes são iniciados numa mesma máquina, suportando vários atividades de desenvolvimento e teste, sem aumento de custo de hardware.
+
+
+
+Teste agora mesmo:
+- entre no [Database Lab Platform](https://console.postgres.ai/), associe-se a uma organização "Demo", e teste clonar um banco de dados demo de ~1TiB, ou
+- faça o check out de um outro setup demo, DLE CE: https://nik-tf-test.aws.postgres.ai:446/instance, use o token `demo` para acessar (este setup tem certificados autoassinado, então ignore os alertas do navegador)
+
+## Como funciona
+Thin cloning é rápido pois utiliza [Copy-on-Write (CoW)](https://en.wikipedia.org/wiki/Copy-on-write#In_computer_storage). DLE suporta duas tecnologias para abilitar CoW e thin cloning: [ZFS](https://en.wikipedia.org/wiki/ZFS) (default) e [LVM](https://en.wikipedia.org/wiki/Logical_Volume_Manager_(Linux)).
+
+Com ZFS, o Database Lab Engine periodicamente cria um novo snapshot do diretório de dados e mantém um conjunto de snapshots, limpando os antigos e não utilizados. Quando solicitam um novo clone, usuários podem escolher qual snapshot utilizar.
+
+Leia mais:
+- [Como funciona](https://postgres.ai/products/how-it-works)
+- [Testando Database Migrations](https://postgres.ai/products/database-migration-testing)
+- [Otimização SQL com Joe Bot](https://postgres.ai/products/joe)
+- [Perguntas e Respostas](https://postgres.ai/docs/questions-and-answers)
+
+## Onde começar
+- [Tutorial do Database Lab para qualquer banco de dados PostgreSQL](https://postgres.ai/docs/tutorials/database-lab-tutorial)
+- [Tutorial do Database Lab para Amazon RDS](https://postgres.ai/docs/tutorials/database-lab-tutorial-amazon-rds)
+- [Template com Terraform module (AWS)](https://postgres.ai/docs/how-to-guides/administration/install-database-lab-with-terraform)
+
+## Estudos de Caso
+- Qiwi: [Como o Qiwi controla os dados para acelerar o desenvolvimento](https://postgres.ai/resources/case-studies/qiwi)
+- GitLab: [Como o GitLab itera na otimização de performances SQL para reduzir os riscos de downtime](https://postgres.ai/resources/case-studies/gitlab)
+
+## Funcionalidades
+- Clonagem the bancos de dados Postgres ultrarrápidos - apenas alguns segundos para criar um novo clone pronto para aceitar conexões e queries, independentemente do tamanho do banco de dados.
+- O número máximo teórico de snapshots e clones é 264 ([ZFS](https://en.wikipedia.org/wiki/ZFS), default).
+- O número máximo teórico de do diretório de dados do PostgreSQL: 256 quatrilhões zebibytes, ou 2128 bytes ([ZFS](https://en.wikipedia.org/wiki/ZFS), default).
+- Versões _major_ do PostgreSQL suportadas: 9.6–14.
+- Duas tecnologias são suportadas para viabilizar o thin cloning ([CoW](https://en.wikipedia.org/wiki/Copy-on-write)): [ZFS](https://en.wikipedia.org/wiki/ZFS) e [LVM](https://en.wikipedia.org/wiki/Logical_Volume_Manager_(Linux)).
+- Todos os componentes estão empacotados em docker containers.
+- UI para tornar o trabalho manual mais conveniente.
+- API e CLI para automatizar o trabalho com DLE snapshots e clones.
+- Por default, os PostgreSQL containers incluem várias extensões populares ([docs](https://postgres.ai/docs/database-lab/supported-databases#extensions-included-by-default)).
+- PostgreSQL containers podem ser customizados ([docs](https://postgres.ai/docs/database-lab/supported-databases#how-to-add-more-extensions)).
+- O banco de dados original pode estar localizado em qualquer lugar (Postgres autoadministrado, AWS RDS, GCP CloudSQL, Azure, Timescale Cloud, etc) e NÃO requer nenhum ajuste. Não há NENHUM requerimento para instalar o ZFS ou Docker nos bancos de dados originais (production).
+- Um provisionamento de dados inicial pode ser feito tanto no nível físico (pg_basebackup, backup / ferramentes de arquivamento como WAL-G ou pgBackRest) ou lógico (dump/restore direto da origem ou de arquivos armazenados na AWS S3).
+- Para o modo lógico, suporta a retenção parcial de dados (bancos específicos, tabelas específicas).
+- Para o modo físico, um estado de atualização contínua é suportado ("sync container"), tornando o DLE uma versão especializada de um standby Postgres.
+- Para o modo lógico, suporta atualização completa periódica, automatizada, e controlada pelo DLE. É possível utilizar multiplos discos contendo diferentes versões do banco de dados, para que a atualização completa não precise de _downtime_.
+- Fast Point in Time Recovery (PITR) para os pontos disponíveis em DLE _snapshots_.
+- Clones não utilizados são automaticamente removidos.
+- "Deletion protection" _flag_ pode ser utilizada para bloquear remoções automáticas ou manuais dos clones.
+- _Snapshot retention policies_ são suportadas na configuração do DLE.
+- Clones Persistentes: clones sobrevivem a DLE _restarts_ (incluindo _reboot_ total da VM).
+- O comand "reset" pode ser utilizado para trocar para uma versão diferente dos dados.
+- O componente DB Migration Checker coleta vários artefatos úteis para testar o banco de dados em CI ([docs](https://postgres.ai/docs/db-migration-checker)).
+- SSH port forwarding para conexões com a API e o Postgres.
+- É possível especificar parâmetros para configurar Docker _containers_ na configuração do DLE.
+- Cotas de utilização de recursos para clones: CPU, RAM (cotas de container, suportadas pelo Docker)
+- Parâmetros de configuração do Postgres podem ser especificados na configuração do DLE (separadamente para clones, o "sync" _container_, e o "promote" _container_).
+- Monitoramento: `/healthz` API _endpoint_ livre de autenticação, `/status` extendido (requer autenticação), [Netdata module](https://gitlab.com/postgres-ai/netdata_for_dle).
+
+## Contribuindo
+### Adicione um estrela ao projeto
+A forma mais simples the contribuir é adicionar uma estrela ao projeto no GitHub/GitLab:
+
+
+
+### Compartilhe o projeto
+Poste um tweet mencionando [@Database_Lab](https://twitter.com/Database_Lab) ou compartilhe o linke para este repositório na sua rede social favorita.
+
+Se você usa o DLE ativamente, conte aos outros sobre a sua experiência. Você pode usar o logo que está referenciado abaixo salvo na pasta `./assets`. Fique à vontade para por nos seus documentos, apresentações, aplicações, e interfaces web para mostrar que você utiliza o DLE.
+
+_Snippet_ HTML para _backgrounds_ mais claros:
+
+ 
+
+
+```html
+
+
+
+```
+
+Para _backgrounds_ mais escuros:
+
+ 
+
+
+```html
+
+
+
+```
+
+### Proponha uma idéia ou reporte um _bug_
+Veja o nosso [guia de contribuição](../CONTRIBUTING.md) para mais detalhes.
+
+### Participate in development
+Veja o nosso [guia de contribuição](../CONTRIBUTING.md) para mais detalhes.
+
+### Traduza o README
+Tornar o Database Lab Engine mais acessível aos engenheiros no mundo todo é uma ótima ajuda para o projeto. Veja detalhes na [seção de tradução do guia de contribuição](../CONTRIBUTING.md#Translation).
+
+### Referências
+- [Componentes DLE](https://postgres.ai/docs/reference-guides/database-lab-engine-components)
+- [Configuração do DLE](https://postgres.ai/docs/database-lab/config-reference)
+- [Documentação da API](https://postgres.ai/swagger-ui/dblab/)
+- [Documentação do Client CLI](https://postgres.ai/docs/database-lab/cli-reference)
+
+### How-to guides
+- [Como instalar o Database Lab com Terraform na AWS](https://postgres.ai/docs/how-to-guides/administration/install-database-lab-with-terraform)
+- [Como instalar e inicializar o Database Lab CLI](https://postgres.ai/docs/guides/cli/cli-install-init)
+- [Como administrar o DLE](https://postgres.ai/docs/how-to-guides/administration)
+- [Como trabalhar com clones](https://postgres.ai/docs/how-to-guides/cloning)
+
+Você pode encontrar mais [no seção "How-to guides"](https://postgres.ai/docs/how-to-guides) dos documentos.
+
+### Diversos
+- [Imagens Docker do DLE](https://hub.docker.com/r/postgresai/dblab-server)
+- [Imagens Docker extendidas para PostgreSQL (com uma penca de extensões)](https://hub.docker.com/r/postgresai/extended-postgres)
+- [SQL Optimization chatbot (Joe Bot)](https://postgres.ai/docs/joe-bot)
+- [DB Migration Checker](https://postgres.ai/docs/db-migration-checker)
+
+## Licença
+O código fonte do DLE está licensiado pela licença de código aberto GNU Affero General Public License version 3 (AGPLv3), aprovada pela OSI.
+
+Contacte o time do Postgres.ai se você desejar uma licença _trial_ ou comercial que não contenha as cláusulas da GPL: [Página de contato](https://postgres.ai/contact).
+
+[](https://app.fossa.io/projects/git%2Bgithub.com%2Fpostgres-ai%2Fdatabase-lab-engine?ref=badge_large)
+
+## Comunidade e Suporte
+- ["Acordo da Comunidade do Database Lab Engine de Código de Conduta"](../CODE_OF_CONDUCT.md)
+- Onde conseguir ajuda: [Página de contato](https://postgres.ai/contact)
+- [Comunidade no Slack](https://slack.postgres.ai)
+- Se você precisa reportar um problema de segurança, siga as instruções em ["Database Lab Engine guia de segurança"](../SECURITY.md).
+
+[](../CODE_OF_CONDUCT.md)
diff --git a/translations/README.russian.md b/translations/README.russian.md
index 02cc9266..d754820a 100644
--- a/translations/README.russian.md
+++ b/translations/README.russian.md
@@ -1,4 +1,9 @@
-
+
+
+
Database Lab Engine (DLE)
@@ -165,8 +170,6 @@ HTML-код для светлых фонов:
Свяжитесь с командой Postgres.ai, если вам нужна коммерческая лицензия, которая не содержит предложений GPL, а также, если вам нужна поддержка: [Контактная страница](https://postgres.ai/contact).
-[](https://app.fossa.io/projects/git%2Bgithub.com%2Fpostgres-ai%2Fdatabase-lab-engine?ref=badge_large)
-
## Сообщество и Поддержка
- ["Кодекс поведения сообщества Database Lab Engine"](../CODE_OF_CONDUCT.md)
- Где получить помощь: [Контактная страница](https://postgres.ai/contact)
diff --git a/translations/README.spanish.md b/translations/README.spanish.md
index 722fac42..421d2269 100644
--- a/translations/README.spanish.md
+++ b/translations/README.spanish.md
@@ -1,4 +1,9 @@
-
+
+
+
Database Lab Engine (DLE)
@@ -164,8 +169,6 @@ El código fuente de DLE tiene la licencia de código abierto aprobada por OSI G
Comuníquese con el equipo de Postgres.ai si desea una licencia comercial o de prueba que no contenga las cláusulas GPL: [Página de contacto](https://postgres.ai/contact).
-[](https://app.fossa.io/projects/git%2Bgithub.com%2Fpostgres-ai%2Fdatabase-lab-engine?ref=badge_large)
-
## Comunidad & Apoyo
- ["Código de conducta del Pacto de la comunidad de motor de laboratorio de base de datos"](../CODE_OF_CONDUCT.md)
- Dónde obtener ayuda: [Página de contacto](https://postgres.ai/contact)
diff --git a/translations/README.ukrainian.md b/translations/README.ukrainian.md
index cf96bc83..b57af638 100644
--- a/translations/README.ukrainian.md
+++ b/translations/README.ukrainian.md
@@ -1,4 +1,9 @@
-
+
+
+
Database Lab Engine (DLE)
@@ -165,8 +170,6 @@ HTML-код для світлих фонів:
Зв'яжіться з командою Postgres.ai, якщо вам потрібна комерційна ліцензія, яка не містить пунктів GPL, а також якщо вам потрібна підтримка: [Контактна сторінка](https://postgres.ai/contact).
-[](https://app .fossa.io/projects/git%2Bgithub.com%2Fpostgres-ai%2Fdatabase-lab-engine?ref=badge_large)
-
## Спільнота та підтримка
- ["Кодекс поведінки спільноти Database Lab Engine"](../CODE_OF_CONDUCT.md)
- Де отримати допомогу: [Контактна сторінка](https://postgres.ai/contact)
pFad - Phonifier reborn
Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy
- 
+
+
+
+ 
+