Skip to content

Commit 64267b6

Browse files
rnpridgeonrnpridgeon
committed
complete dockerization add run all scripts
1 parent a485245 commit 64267b6

17 files changed

+476
-668
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,4 @@ wheelhouse
2020
dl-*
2121
*.whl
2222
.pytest_cache
23+
docker/conf/tls/*

‎confluent_kafka/avro/cached_schema_registry_client.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@
2424
import warnings
2525
from collections import defaultdict
2626

27-
import requests
27+
from requests import Session, utils
28+
from requests.adapters import HTTPAdapter
29+
from requests.packages.urllib3.util.retry import Retry
2830

2931
from .error import ClientError
3032
from . import loads
@@ -89,11 +91,15 @@ def __init__(self, url, max_schemas_per_subject=1000, ca_location=None, cert_loc
8991
# subj => { schema => version }
9092
self.subject_to_schema_versions = defaultdict(dict)
9193

92-
s = requests.Session()
94+
s = Session()
9395
s.verify = conf.pop('ssl.ca.location', None)
9496
s.cert = self._configure_client_tls(conf)
9597
s.auth = self._configure_basic_auth(conf)
9698

99+
retries = Retry(connect=10, read=10, backoff_factor=.5)
100+
s.mount('http://', HTTPAdapter(max_retries=retries))
101+
s.mount('https://', HTTPAdapter(max_retries=retries))
102+
97103
self.url = conf.pop('url')
98104
self._session = s
99105

@@ -127,9 +133,9 @@ def _configure_basic_auth(conf):
127133
elif auth_provider == 'USER_INFO':
128134
auth = tuple(conf.pop('basic.auth.user.info', '').split(':'))
129135
else:
130-
auth = requests.utils.get_auth_from_url(url)
136+
auth = utils.get_auth_from_url(url)
131137

132-
conf['url'] = requests.utils.urldefragauth(url)
138+
conf['url'] = utils.urldefragauth(url)
133139
return auth
134140

135141
@staticmethod

‎docker/.env

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
#!/usr/bin/env bash
22

3-
SOURCE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
4-
TLS=$SOURCE/conf/tls
3+
export DOCKER_SOURCE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
4+
export DOCKER_CONTEXT=$DOCKER_SOURCE/docker-compose.yaml
5+
export DOCKER_BIN=$DOCKER_SOURCE/bin
6+
export DOCKER_CONF=$DOCKER_SOURCE/conf
7+
export TLS=$DOCKER_CONF/tls
58

69
export MY_BOOTSTRAP_SERVER_ENV=localhost:29092
7-
export MY_SCHEMA_REGISTRY_URL_ENV="http://ckp_tester:test_secret@localhost:8081"
8-
export MY_SCHEMA_REGISTRY_SSL_URL_ENV="https://ckp_tester:test_secret@$(hostname -f):8082"
10+
export MY_SCHEMA_REGISTRY_URL_ENV=http://$(hostname):8081
11+
export MY_SCHEMA_REGISTRY_SSL_URL_ENV=https://$(hostname -f):8082
912
export MY_SCHEMA_REGISTRY_SSL_CA_LOCATION_ENV=$TLS/ca-cert
1013
export MY_SCHEMA_REGISTRY_SSL_CERTIFICATE_LOCATION_ENV=$TLS/client.pem
1114
export MY_SCHEMA_REGISTRY_SSL_KEY_LOCATION_ENV=$TLS/client.key

‎docker/bin/certify

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎docker/bin/certify.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
3+
DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
4+
export PASS="abcdefgh"
5+
source ${DOCKER_BIN}/../.env
6+
7+
if [ -f ${TLS}/ca-cert ]; then
8+
echo "${TLS}/ca-cert found; skipping certificate generation.."
9+
exit 0
10+
fi
11+
12+
# Clean up old certs
13+
#for file in $(ls ${TLS});do
14+
# rm ${TLS}/${file}
15+
#done
16+
17+
echo "Creating ca-cert..."
18+
${DOCKER_BIN}/gen-ssl-certs.sh ca ${TLS}/ca-cert $(hostname -f)
19+
echo "Creating server cert..."
20+
${DOCKER_BIN}/gen-ssl-certs.sh -k server ${TLS}/ca-cert ${TLS}/ $(hostname -f) $(hostname -f)
21+
echo "Creating client cert..."
22+
${DOCKER_BIN}/gen-ssl-certs.sh client ${TLS}/ca-cert ${TLS}/ $(hostname -f) $(hostname -f)
23+
24+
echo "Creating key ..."
25+
openssl rsa -in ${TLS}/client.key -out ${TLS}/client.key -passin pass:${PASS}
26+

‎docker/bin/gen-ssl-certs.sh

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/usr/bin/env bash
22
#
33
#
44
# This scripts generates:
@@ -9,6 +9,8 @@
99
# https://cwiki.apache.org/confluence/display/KAFKA/Deploying+SSL+for+Kafka
1010
#
1111

12+
DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
13+
1214

1315
if [[ "$1" == "-k" ]]; then
1416
USE_KEYTOOL=1
@@ -28,10 +30,6 @@ L=NN
2830
O=NN
2931
OU=NN
3032
CN="$HOST"
31-
32-
33-
# Password
34-
PASS="abcdefgh"
3533

3634
# Cert validity, in days
3735
VALIDITY=10000
@@ -83,7 +81,6 @@ EOF
8381

8482
echo "############ Sign certificate"
8583
openssl x509 -req -CA $CA_CERT -CAkey ${CA_CERT}.key -in ${PFX}cert-file -out ${PFX}cert-signed -days $VALIDITY -CAcreateserial -passin "pass:$PASS"
86-
mv $SOURCE/.srl $TLS/ca-cert.srl
8784

8885
echo "############ Import CA"
8986
keytool -storepass "$PASS" -keypass "$PASS" -keystore ${PFX}server.keystore.jks -alias CARoot -import -file $CA_CERT <<EOF
@@ -119,7 +116,6 @@ EOF
119116

120117
echo "########### Sign certificate"
121118
openssl x509 -req -CA ${CA_CERT} -CAkey ${CA_CERT}.key -in ${PFX}cert-file -out ${PFX}cert-signed -days $VALIDITY -CAcreateserial -passin pass:$PASS
122-
mv $SOURCE/.srl $TLS/ca-cert.srl
123119

124120
echo "########### Import CA"
125121
keytool -storepass "$PASS" -keypass "$PASS" -keystore ${PFX}client.keystore.jks -alias CARoot -import -file ${CA_CERT} <<EOF

‎docker/conf/testconf.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"schema.registry.ssl.key.location": "$MY_SCHEMA_REGISTRY_SSL_KEY_LOCATION_ENV"
1010
},
1111
"avro-basic-auth": {
12+
"schema.registry.url": "http://localhost:8083",
1213
"schema.registry.basic.auth.user.info": "ckp_tester:test_secret",
1314
"sasl.username": "ckp_tester",
1415
"sasl.password": "test_secret"

‎docker/docker-compose.yaml

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,27 +19,50 @@ services:
1919
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
2020
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
2121
schema-registry:
22+
image: confluentinc/cp-schema-registry
23+
depends_on:
24+
- zookeeper
25+
- kafka
26+
ports:
27+
- 8081:8081
28+
- 8082:8082
29+
volumes:
30+
- ./conf:/conf
31+
environment:
32+
SCHEMA_REGISTRY_HOST_NAME: schema-registry
33+
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
34+
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081, https://0.0.0.0:8082
35+
SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: https
36+
SCHEMA_REGISTRY_SSL_KEYSTORE_LOCATION: /conf/tls/server.keystore.jks
37+
SCHEMA_REGISTRY_SSL_KEYSTORE_PASSWORD: ${PASS:-abcdefgh}
38+
SCHEMA_REGISTRY_SSL_KEY_PASSWORD: ${PASS:-abcdefgh}
39+
SCHEMA_REGISTRY_SSL_TRUSTSTORE_LOCATION: /conf/tls/server.truststore.jks
40+
SCHEMA_REGISTRY_SSL_TRUSTSTORE_PASSWORD: abcdefgh
41+
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
42+
schema-registry-basic-auth:
2243
image: confluentinc/cp-schema-registry
2344
depends_on:
2445
- zookeeper
2546
- kafka
2647
ports:
27-
- 8081:8081
28-
- 8082:8082
48+
- 8083:8083
49+
- 8084:8084
2950
volumes:
3051
- ./conf:/conf
3152
environment:
32-
SCHEMA_REGISTRY_HOST_NAME: schema-registry
53+
SCHEMA_REGISTRY_HOST_NAME: schema-registry2
54+
SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas2
55+
SCHEMA_REGISTRY_SCHEMA_REGISTRY_ZK_NAMESPACE: schema_registry2
3356
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
34-
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081, https://0.0.0.0:8082
57+
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8083, https://0.0.0.0:8084
3558
SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: https
3659
SCHEMA_REGISTRY_SSL_KEYSTORE_LOCATION: /conf/tls/server.keystore.jks
37-
SCHEMA_REGISTRY_SSL_KEYSTORE_PASSWORD: abcdefgh
38-
SCHEMA_REGISTRY_SSL_KEY_PASSWORD: abcdefgh
60+
SCHEMA_REGISTRY_SSL_KEYSTORE_PASSWORD: ${PASS:-abcdefgh}
61+
SCHEMA_REGISTRY_SSL_KEY_PASSWORD: ${PASS:-abcdefgh}
3962
SCHEMA_REGISTRY_SSL_TRUSTSTORE_LOCATION: /conf/tls/server.truststore.jks
4063
SCHEMA_REGISTRY_SSL_TRUSTSTORE_PASSWORD: abcdefgh
4164
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
42-
SCHEMA_REGISTRY_AUTHENTICATION_METHOD: ${REST_AUTHENTICATION_METHOD:-NONE}
65+
SCHEMA_REGISTRY_AUTHENTICATION_METHOD: BASIC
4366
SCHEMA_REGISTRY_AUTHENTICATION_REALM: SchemaRegistry
4467
SCHEMA_REGISTRY_AUTHENTICATION_ROLES: Testers
45-
SCHEMA_REGISTRY_OPTS: -Djava.security.auth.login.config=/conf/schema-registry/schema-registry.jaas
68+
SCHEMA_REGISTRY_OPTS: -Djava.security.auth.login.config=/conf/schema-registry/schema-registry.jaas

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy