Skip to content

Commit 50865d1

Browse files
rnpridgeonrnpridgeon
committed
move cluster scripts to docker dir, remove urlib3 retries, consolidate test runner scrip
1 parent 64267b6 commit 50865d1

File tree

11 files changed

+103
-94
lines changed

11 files changed

+103
-94
lines changed

‎confluent_kafka/avro/cached_schema_registry_client.py

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@
2525
from collections import defaultdict
2626

2727
from requests import Session, utils
28-
from requests.adapters import HTTPAdapter
29-
from requests.packages.urllib3.util.retry import Retry
3028

3129
from .error import ClientError
3230
from . import loads
@@ -96,10 +94,6 @@ def __init__(self, url, max_schemas_per_subject=1000, ca_location=None, cert_loc
9694
s.cert = self._configure_client_tls(conf)
9795
s.auth = self._configure_basic_auth(conf)
9896

99-
retries = Retry(connect=10, read=10, backoff_factor=.5)
100-
s.mount('http://', HTTPAdapter(max_retries=retries))
101-
s.mount('https://', HTTPAdapter(max_retries=retries))
102-
10397
self.url = conf.pop('url')
10498
self._session = s
10599

‎docker/bin/cluster_down.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/usr/bin/env bash
2+
3+
DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
4+
source ${DOCKER_BIN}/../.env
5+
6+
echo "Destroying cluster.."
7+
docker-compose -f ${DOCKER_CONTEXT} down -v --remove-orphans

‎docker/bin/cluster_up.sh

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/usr/bin/env bash
2+
3+
DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
4+
source ${DOCKER_BIN}/../.env
5+
6+
await_http() {
7+
local exit_code
8+
local attempt=0
9+
curl -s "$2" > /dev/null; exit_code=$?
10+
11+
while [ ${attempt} -lt 5 ]; do
12+
echo "awaiting $1 ..."
13+
curl -s "$2" > /dev/null; exit_code=$?
14+
15+
if [ ${exit_code} -eq "0" ]; then
16+
return
17+
fi
18+
let "attempt+=1"
19+
sleep 5
20+
done
21+
echo "$1 readiness test failed aborting..."
22+
exit 1
23+
}
24+
25+
echo "Configure Environment..."
26+
source ${DOCKER_SOURCE}/.env
27+
28+
echo "Generate SSL certs..."
29+
${DOCKER_BIN}/certify.sh
30+
31+
echo "Deploying cluster..."
32+
docker-compose -f ${DOCKER_CONTEXT} up -d
33+
34+
echo "Setting throttle for throttle test..."
35+
docker-compose -f ${DOCKER_CONTEXT} exec kafka sh -c "
36+
/usr/bin/kafka-configs --zookeeper zookeeper:2181 \
37+
--alter --add-config 'producer_byte_rate=1,consumer_byte_rate=1,request_percentage=001' \
38+
--entity-name throttled_client --entity-type clients"
39+
40+
await_http "schema-registry" "http://localhost:8081"
41+
await_http "schema-registry-basic-auth" "http://localhost:8083"

‎examples/integration_test.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1187,7 +1187,9 @@ def resolve_envs(_conf):
11871187
_conf[k] = os.getenv(v[1:])
11881188

11891189

1190-
test_modes = ['consumer', 'producer', 'avro', 'performance', 'admin', 'avro-https', 'avro-basic-auth', 'throttle']
1190+
test_modes = ['consumer', 'producer', 'performance', 'admin',
1191+
'avro', 'avro-https', 'avro-basic-auth',
1192+
'throttle']
11911193

11921194

11931195
def print_usage(exitcode, reason=None):
@@ -1216,6 +1218,9 @@ def print_usage(exitcode, reason=None):
12161218
while len(sys.argv) > 1 and sys.argv[1].startswith('--'):
12171219
opt = sys.argv.pop(1)[2:]
12181220

1221+
if opt == "help":
1222+
print_usage(0, None)
1223+
12191224
if opt not in test_modes:
12201225
print_usage(1, 'unknown option --' + opt)
12211226
modes.append(opt)

‎tests/README.md

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,12 @@ Integration tests
2525
### Cluster setup
2626
**Note** Manual cluster set up is not required when using ./tests/run_all.sh
2727

28-
./tests/cluster_up.sh
28+
./docker/bin/cluster_up.sh
2929

3030
### Cluster teardown
3131
**Note** Manual cluster teardown is not required when using ./tests/run_all.sh
3232

33-
./tests/cluster_down.sh
33+
./docker/bin/cluster_down.sh
3434

3535
### Configuration
3636
Tests are configured with a JSON encoded file referred to as a `testconf` to be provided as the last argument upon test execution.
@@ -39,29 +39,25 @@ Advanced users can reference the provided configuration file, [testconf.json](..
3939
Most developers however should just use the defaults.
4040

4141
### Running tests
42-
To run all test `modes` uncomment the following line from `tox.ini`
42+
To run the entire test suite:
4343

4444
- With tox installed (will run against all supported interpreters)
4545
1. Uncomment the following line from [tox.ini](../tox.ini)
4646
- ```#python examples/integration_test.py ./docker/conf/testconf.json```
4747
2. Execute the following script
48-
- ```./tests/run_all_tox.sh```
48+
- ```./tests/run.sh tox [options]```
4949

5050
- Without tox (will run against current interpreter)
5151
- ```./tests/run_all.sh```
5252

53-
To run the full test-suite manually execute the following
53+
To run a specific `mode` or set of `modes` use the following syntax
5454

55-
python examples/integration_test.py ./docker/conf/testconf.json
56-
57-
To run a specific test `mode` or set of test `modes` use the following syntax
58-
59-
python examples/integration_test.py [--test mode 1] [--test mode n...] <testconf>
55+
./tests/run.sh [test mode 1] [test mode n...]
6056

6157
For example:
6258

63-
python examples/integration_test.py --producer --consumer ./docker/conf/testconf.json
59+
./tests/run.sh producer consumer
6460

65-
To get a list of `modes` you can run the integration test manually with the `--help` flag
61+
To get a list of `modes` simply supply the `help` option
6662

67-
python examples/integration_tests.py --help
63+
./tests/run.sh help

‎tests/cluster_down.sh

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎tests/cluster_up.sh

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎tests/run.sh

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
3+
TEST_SOURCE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
4+
source ${TEST_SOURCE}/../docker/.env
5+
6+
if [ "$1" == "help" ]; then
7+
python ${TEST_SOURCE}/../examples/integration_test.py --help
8+
exit 1
9+
fi
10+
11+
#start cluster
12+
${DOCKER_BIN}/cluster_up.sh
13+
14+
if [ "$?" -ne "0" ]; then
15+
exit 1
16+
fi
17+
18+
run_tox() {
19+
echo "Executing tox $@"
20+
cd ${TEST_SOURCE}/..
21+
tox -r "$@"
22+
}
23+
24+
run_native() {
25+
for mode in "$@"; do
26+
modes="${modes} --${mode}"
27+
done
28+
echo "Executing test modes $@"
29+
python ${TEST_SOURCE}/../examples/integration_test.py ${modes} ${DOCKER_CONF}/testconf.json
30+
}
31+
32+
if [ "$1" == "tox" ]; then
33+
shift
34+
run_tox $@
35+
else
36+
run_native $@
37+
fi
38+
39+
#teardown cluster
40+
${DOCKER_BIN}/cluster_down.sh

‎tests/run_all.sh

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎tests/run_all_tox.sh

Lines changed: 0 additions & 14 deletions
This file was deleted.

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy