Skip to content

Commit 2d10346

Browse files
vbwagnervbwagner
committed
gitlab CI: split into many stages, added helper scripts for collecting logs and setup temporary installation for make installcheck
1 parent 2fb43b9 commit 2d10346

File tree

5 files changed

+227
-15
lines changed

5 files changed

+227
-15
lines changed

‎.ci/collect_logs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env python
2+
"""
3+
Prints out all the files with .log extension
4+
"""
5+
import os,re,subprocess
6+
cores=[]
7+
for root,dirs,files in os.walk("."):
8+
for name in files:
9+
path= root+"/"+name
10+
if name.endswith(".log") or name=="regression.diffs":
11+
print "*" * 76
12+
print "* %-72s *"%path
13+
print "*" * 76
14+
with open(path,"r") as f:
15+
print f.read()
16+
if name == "core" or re.match("core\\.[0-9]+",name) or re.match(".*\\.core",name):
17+
cores.append(path)
18+
print ", ".join(cores)
19+
for f in cores:
20+
print "*" * 76
21+
print "* %-72s *"%f
22+
print "*" * 76
23+
gdb=subprocess.Popen(["gdb","src/backend/postgres",f],
24+
stderr=subprocess.PIPE,stdout=subprocess.PIPE,stdin=subprocess.PIPE)
25+
out,err=gdb.communicate("bt\nq\n")
26+
if err.index("core file may not match specified")>=0:
27+
m=re.search("Core was generated by `(.*)'",out)
28+
if m:
29+
gdb=subprocess.Popen(["gdb",m.group(1),f],
30+
stdout=subprocess.PIPE,stdin=subprocess.PIPE)
31+
out,err=gdb.communicate("bt\nq\n")
32+
33+
34+
print out
35+
36+
37+
38+

‎.ci/find_free_port

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/python
2+
import sys,socket,errno
3+
if len(sys.argv)>1:
4+
port=int(sys.argv[1])
5+
else:
6+
port=5900
7+
8+
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
9+
while True:
10+
try:
11+
s.bind(("",port))
12+
except socket.error as e:
13+
if e.errno== errno.EADDRINUSE:
14+
port+=1
15+
continue
16+
else:
17+
raise e
18+
break
19+
20+
s.close()
21+
print port
22+

‎.ci/make_test_base

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env python
2+
"""
3+
Receives one parameter - PGDATA directory
4+
Creates test base using first initdb found in the PATH
5+
Searches contrib directory for additional directives to
6+
put into postgresql conf.
7+
8+
On success, starts database
9+
"""
10+
import os,sys,subprocess,glob,re,os.path,time
11+
12+
datadir=sys.argv[1]
13+
14+
if not datadir:
15+
print >>sys.stderr,"Usage %s directory"%sys.argv[0]
16+
sys.exit(1)
17+
18+
if os.access(datadir,os.R_OK):
19+
import shutil
20+
shutil.rmtree(datadir)
21+
os.mkdir(datadir)
22+
with open("initdb.log","w") as f:
23+
exitcode=subprocess.call(["initdb","-E","UTF8",datadir],stdout=f,stderr=subprocess.STDOUT)
24+
if exitcode:
25+
sys.exit(exitcode)
26+
# Collect extra config option
27+
addopts={}
28+
for module in glob.glob("contrib/*"):
29+
if not os.path.isdir(module):
30+
continue
31+
if not os.access(module+"/Makefile",os.R_OK):
32+
continue
33+
with open(module+"/Makefile","r") as makefile:
34+
var={"top_srcdir":os.getcwd()}
35+
for line in makefile:
36+
m=re.match("\s*(\w+)\s*=\s*(.*)",line)
37+
if m:
38+
var[m.group(1)]=m.group(2)
39+
if "EXTRA_REGRESS_OPTS" in var:
40+
m=re.search("--temp-config=(\S+)",var["EXTRA_REGRESS_OPTS"])
41+
if m:
42+
filename=re.sub("\\$[{(](\w+)[})]",lambda m: var[m.group(1)],m.group(1))
43+
with open(filename,"r") as config:
44+
for line in config:
45+
m=re.match("(\w\S*\w)\s*=\s*(\S.*)\s*",line)
46+
if m:
47+
opt=m.group(1)
48+
value=m.group(2)
49+
if opt in addopts:
50+
if value[0]=="'":
51+
addopts[opt]=addopts[opt][:-1]+", "+value[1:]
52+
else:
53+
addopts[opt]+=", ".value
54+
else:
55+
addopts[opt]=value
56+
57+
if addopts:
58+
with open(datadir+"/postgresql.conf","a") as f:
59+
for opt,value in addopts.items():
60+
print >> f,"%s=%s"%(opt,value)
61+
with open("initdb.log","a") as f:
62+
exitcode=subprocess.call(["pg_ctl","start","-D",datadir,"-l",datadir+"/postmaster.log"],stdout=f,stderr=subprocess.STDOUT)
63+
if exitcode:
64+
sys.exit(exitcode)
65+
66+
failtime=time.time()+60
67+
print "Waiting for database to start"
68+
while time.time() < failtime:
69+
exitcode=subprocess.call(["psql","postgres","-c","select version()"],stderr=subprocess.STDOUT)
70+
if exitcode == 0:
71+
sys.exit(0)
72+
print >>sys.stderr,"Database havent't started in 60 seconds"
73+
sys.exit(1)
74+
75+

‎.ci/run_install_check

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/sh
2+
# Makes temporary installation and runs installcheck-world on it
3+
# Should be run as non-privileged user.
4+
# Exit on all errors
5+
set -e
6+
# Install
7+
make install prefix=`pwd`/tmp_install
8+
make install prefix=`pwd`/tmp_install -C contrib
9+
10+
# Setup an environment
11+
LD_LIBRARY_PATH=$(pwd)/tmp_install/lib
12+
PATH=$(pwd)/tmp_install/bin:${PATH}
13+
PGDATA=$(pwd)/tmp_base
14+
export LD_LIBRARY_PATH PATH PG_DATA
15+
16+
# create installation
17+
PGPORT=`./.ci/find_free_port 5432`
18+
export PGPORT
19+
./.ci/make_test_base $PGDATA
20+
#run checks
21+
set +e
22+
make installcheck-world prefix=`pwd`/tmp_install
23+
code=$?
24+
pg_ctl stop -D $PGDATA
25+
exit $code

‎.gitlab-ci.yml

Lines changed: 67 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,79 @@
1-
image: debian
2-
31
stages:
2+
- prepare
3+
- build
44
- test
5+
- installtest
6+
- collect_logs
57

6-
before_script:
8+
prepare-doc:
9+
stage: prepare
10+
image: ubuntu:16.04
11+
tags:
12+
- deb
13+
- docker
14+
only:
15+
- /^PGPRO.*9_[56]$/
16+
before_script:
717
- export DEBIAN_FRONTEND=noninteractive
818
- export CORES=$(grep -c ^processor /proc/cpuinfo)
919
- uname -a
1020
- df -h
11-
12-
test:ubuntu-16.04:
13-
stage: test
14-
image: ubuntu:16.04
15-
only:
16-
- PGPROEE9_6
17-
before_script:
21+
script:
1822
- echo "Current user id is `id`"
1923
- apt-get update && apt-get install -y sudo gcc make flex bison libreadline-dev zlib1g-dev openjade libzstd-dev opensp docbook docbook-xml docbook-xsl libxml2-utils xsltproc python-dev libicu-dev
2024
- id -u postgres || adduser --disabled-login --gecos "Postgres" postgres
25+
build_job:
26+
stage: build
27+
tags:
28+
- unix
29+
only:
30+
- /^PGPRO.*9_[56]$/
2131
script:
22-
- ./configure --prefix=/opt/pgproee --with-zstd --with-icu --with-python
23-
- make -j $CORES world
24-
- sudo -u postgres make check-world
25-
after_script:
26-
- cat src/test/regress/log/initdb.log
32+
- if [ -z "$PGPORT" ]; then PGPORT=5789; fi
33+
- ./configure --prefix=$(pwd)/tmp_install --port=--with-zstd --with-icu --with-python
34+
- make
35+
- make -C contrib
36+
when: always
37+
38+
build_docs_job:
39+
stage: build
2740
when: always
41+
only:
42+
- /^PGPRO.*9_[56]$/
43+
tags:
44+
- docs
45+
script:
46+
- make -C doc
47+
48+
make_check_job:
49+
stage: test
50+
when: on_success
51+
only:
52+
- /^PGPRO.*9_[56]$/
53+
tags:
54+
- unix
55+
before_script:
56+
- unset SUCMD
57+
- if [ $(id -u) -eq 0 ]; then SUCMD="sudo -u postgres"; fi
58+
- export SUCMD
59+
script:
60+
- $SUCMD make check-world
61+
62+
make_installcheck_job:
63+
stage: installtest
64+
when: on_success
65+
only:
66+
- /^PGPRO.*9_[56]$/
67+
tags:
68+
- unix
69+
before_script:
70+
- unset SUCMD
71+
- if [ $(id -u) -eq 0 ]; then SUCMD="sudo -u postgres"; fi
72+
- export SUCMD
73+
script:
74+
- $SUCMD ./.ci/run_install_check
75+
collect_logs_job:
76+
stage: collect_logs
77+
when: on_failure
78+
script:
79+
- ./.ci/collect_logs

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy