From 5c0d5499b6c30290098cef505ceadcfbe9e00636 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Thu, 23 Jan 2025 20:41:51 +0100 Subject: [PATCH 1/9] add ldappool module --- Lib/ldappool/README.md | 65 ++++++ Lib/ldappool/__init__.py | 422 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 Lib/ldappool/README.md create mode 100644 Lib/ldappool/__init__.py diff --git a/Lib/ldappool/README.md b/Lib/ldappool/README.md new file mode 100644 index 00000000..c926097a --- /dev/null +++ b/Lib/ldappool/README.md @@ -0,0 +1,65 @@ +# LDAP Pooling example + +## entries as dict + +```python +from ldappool import ConnectionPool +pool = ConnectionPool( + params={"keep": True, "autoBind": True, "retries": 2}, + max=5) +pool.set_uri("ldaps://ldap.example.com:636/dc=example,dc=com?uid,mail?sub?(|(uid=test)(mail=test@example.com))") +pool.set_credentials("binddn", "bindpw") +with pool.get() as conn: + for entry in conn.search_s(pool.basedn, + pool.scope, + pool.filter, + pool.attributes): + print(f"{entry[0]}: {entry[1].get('uid')} {entry[1].get('mail')}") + for member in entry[1].get("memberOf", []): + print(member) +``` + +## entry to dataclass example +```python +from ldappool import ConnectionPool +from ldappool import e2c +pool = ConnectionPool( + params={"keep": True, "autoBind": True, "retries": 2}, + max=5) +pool.set_uri("ldaps://ldap.example.com:636/dc=example,dc=com?uid,mail?sub?(|(uid=test)(mail=test@example.com))") +pool.set_credentials("binddn", "bindpw") +with pool.get() as conn: + for entry in map(e2c, conn.search_s(pool.basedn, + pool.scope, + pool.filter, + pool.attributes)): + print(f"{entry.dn}: {entry.uid} {entry.mail}") + for member in entry.memberOf: + print(member) +``` + +## changing the connection or credentials for the pool + +```python +from ldappool import ConnectionPool +from ldappool import e2c +pool = ConnectionPool( + params={"keep": True, "autoBind": True, "retries": 2}, + max=5) +pool.set_uri("ldaps://ldap.example.com:636/dc=example,dc=com?uid,mail?sub?(|(uid=test)(mail=test@example.com))") +pool.set_credentials("binddn", "bindpw") +with pool.get() as conn: + for entry in map(e2c, conn.search_s(pool.basedn, + pool.scope, + pool.filter, + pool.attributes)): + print(f"{entry.dn}: {entry.uid} {entry.mail}") + +pool.set_credentials(entry.dn, "changeme") +with pool.get() as conn: + for entry in map(e2c, conn.search_s(pool.basedn, + pool.scope, + pool.filter, + pool.attributes)): + print(f"{entry.dn}: {entry.uid} {entry.mail}") +``` diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py new file mode 100644 index 00000000..60aad335 --- /dev/null +++ b/Lib/ldappool/__init__.py @@ -0,0 +1,422 @@ +import dataclasses +import logging +import sys +import threading +import time +from urllib.parse import urlparse + +import ldap +from ldapurl import LDAPUrl + +# nano seconds to ensure we know the locked time +ns = 1_000_000_000 +ns_locktimeout = 15.0 + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + + +class LDAPPoolExhausted(Exception): + pass + + +class LDAPPoolDown(Exception): + pass + + +class LDAPLockTimeout(Exception): + pass + + +def e2c(entry): + cls = dataclasses.make_dataclass("", ["dn"] + list(entry[1].keys()), frozen=True) + return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) + + +class Connection(object): + def __init__( + self, + uri: LDAPUrl, + binddn: str, + bindpw: str, + params: dict = {}, + ): + self.uri = uri + self.binddn = binddn + self.bindpw = bindpw + self.params = params + self.established = False + self.inUse = False + self._whoami = None + self._conn = False + self._lock = threading.Lock() + self._pool = None + self._health = 0.0 + (f"ConnectionPool new Connection {self}") + if self.params.get("prewarm", False): + self.__enter__() + + def __locktime(self): + if self._health == 0.0: + self._health = time.perf_counter_ns() + return True + if (time.perf_counter_ns() - self._health) / ns < ns_locktimeout: + return False + return True + + @property + def whoami(self): + return self._whoami + + def __whoami(self): + # do not stress the connection too often + if not self.__locktime(): + return + for r in range(self.params.get("retries", 3)): + try: + self._whoami = self._conn.whoami_s() + return + except ldap.SERVER_DOWN as ldaperr: + logging.error(f"__whoami ConnectionPool {ldaperr}") + self.established = False + # just catch that error until we finished iterating + try: + self.__enter__() + except: + pass + raise ldap.SERVER_DOWN( + f"max retries {self.params.get('retries', 3)} reached" + ) + + @property + def conn(self): + if self._conn == False: + self.__enter__() + try: + if self.established: + self.__whoami() + except ldap.SERVER_DOWN as ldaperr: + self.established = False + raise LDAPPoolDown( + f"could not establish connection with {self.uri.initializeUrl()}" + + f" with max retries of {self.params.get('retries', 3)}" + ) + return self._conn + + def __lock_acquire(self): + try: + if self._lock.acquire(blocking=True, timeout=1): + return True + else: + raise LDAPLockTimeout() + except Exception as lockerr: + return False + + def __lock_release(self): + try: + self._lock.release() + return True + except Exception as lockerr: + return False + + def authenticate( + self, + binddn: str, + bindpw: str, + ): + + if not self.__lock_acquire(): + raise LDAPLockTimeout() + + try: + self.conn.simple_bind_s( + binddn, + bindpw, + ) + if not self.__lock_release(): + raise LDAPLockTimeout() + except ldap.INVALID_CREDENTIALS as ldaperr: + # rollback auth anyway + self.__lock_release() + self.__authenticate__() + raise ldap.INVALID_CREDENTIALS + # rollback auth anyway + self.__authenticate__() + return True + + def __authenticate__(self): + if not self.__lock_acquire(): + raise LDAPLockTimeout() + try: + self.conn.simple_bind_s( + self.binddn, + self.bindpw, + ) + logging.debug("__whoami from __authenticate__") + self.__whoami() + self.__lock_release() + except ldap.INVALID_CREDENTIALS as ldaperr: + self.__lock_release() + logging.info(ldaperr) + raise ldap.INVALID_CREDENTIALS + + def __set_connection_parameters__(self): + self._conn.set_option(ldap.OPT_REFERRALS, self.params.get("referrals", False)) + self._conn.set_option( + ldap.OPT_NETWORK_TIMEOUT, self.params.get("network_timeout", 10.0) + ) + self._conn.set_option(ldap.OPT_TIMEOUT, self.params.get("timeout", 10.0)) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_IDLE, self.params.get("keepalive_idle", 10) + ) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_INTERVAL, self.params.get("keepalive_interval", 5) + ) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_PROBES, self.params.get("keepalive_probes", 3) + ) + self._conn.set_option(ldap.OPT_RESTART, ldap.OPT_ON) + if self.params.get("allow_tls_fallback", False): + logging.debug("TLS Fallback enabled in LDAP") + self._conn.set_option(ldap.OPT_X_TLS_TRY, 1) + self._conn.set_option(ldap.OPT_X_TLS_NEWCTX, ldap.OPT_OFF) + + def __enter__(self): + self.inUse = True + if not self.established: + logging.debug( + f"ConnectionPool {self} initializin LDAP {self.uri.initializeUrl()}" + ) + try: + self._conn = ldap.initialize(self.uri.initializeUrl()) + if self.params.get("autoBind", False): + ( + f"ConnectionPool {self} autoBind with {self.binddn} password {'x'*len(self.bindpw)}" + ) + self.__authenticate__() + except Exception as ldaperr: + (ldaperr) + raise ldaperr + self.established = True + return self.conn + + def giveback(self): + try: + if self.params.get("autoBind", False): + if not self.params.get("keep", False): + logging.debug(f"ConnectionPool unbind connection {self}") + try: + self._conn.unbind_s() + except Exception as ldaperr: + logging.error( + "ConnectionPool unbind connection" + + f"{self} exception {ldaperr}" + ) + self.inUse = False + except AttributeError: + self.inUse = False + + def __del__(self): + self.giveback() + if all([self._pool is not None, not self.params.get("keep", False)]): + logging.debug(f"ConnectionPool deleteing connection {self} from Pool") + self._pool.delete(self) + + def __exit__(self, type, value, traceback): + self.giveback() + if all([self._pool is not None, not self.params.get("keep", False)]): + self._pool.delete(self) + + def __cmp__(self, other): + if isinstance(other, LDAPUrl): + return self.uri.initializeUrl() == other.uri.initializeUrl() + return False + + def set_uri(self, uri: LDAPUrl): + self.uri = uri + return True + + def set_binddn(self, binddn: str): + self.binddn = binddn + return True + + def set_bindpw(self, bindpw: str): + self.bindpw = bindpw + return True + + def set_credentials(self, binddn: str, bindpw: str): + self.set_binddn(binddn) + self.set_bindpw(bindpw) + return True + + +class ConnectionPool(object): + def __init__( + self, + uri: LDAPUrl = LDAPUrl("ldap:///"), + binddn: str = "", + bindpw: str = "", + params: dict = {}, + max: int = 10, + ): + self.uri = uri + self.binddn = binddn + self.bindpw = bindpw + self.params = params + self.max = int(max) + self._lock = threading.Lock() + self._pool = [] + logging.debug(f"ConnectionPool {self} starting with {self.max} connections") + if self.params.get("prewarm", False): + self.scale + + @property + def basedn(self): + return self.uri.dn + + @property + def scope(self): + return self.uri.scope + + @property + def filter(self): + return self.uri.filterstr + + @property + def attributes(self): + return self.uri.attrs + + @property + def extensions(self): + return self.uri.extensions + + def set_uri(self, uri: LDAPUrl): + if not isinstance(uri, LDAPUrl): + uri = LDAPUrl(uri) + if len(self._pool) > 0: + map( + lambda c: (c.set_uri(uri), c.giveback(force=True)), + filter(lambda cp: cp.uri != uri, self._pool), + ) + self.uri = uri + return True + + def set_binddn(self, binddn: str): + if len(self._pool) > 0: + map( + lambda c: (c.set_binddn(binddn), c.giveback(force=True)), + filter(lambda cp: cp.binddn != binddn, self._pool), + ) + self.binddn = binddn + return True + + def set_bindpw(self, bindpw: str): + if len(self._pool) > 0: + map( + lambda c: (c.set_bindpw(bindpw), c.giveback(force=True)), + filter(lambda cp: cp.bindpw != bindpw, self._pool), + ) + self.bindpw = bindpw + return True + + def set_credentials(self, binddn: str, bindpw: str): + self.set_binddn(binddn) + self.set_bindpw(bindpw) + return True + + @property + def scale(self): + for _ in range(self.max - len(self._pool)): + self.put( + Connection( + uri=self.uri, + binddn=self.binddn, + bindpw=self.bindpw, + params=self.params, + ) + ) + + def __enter__(self): + if len(self._pool) == 0: + self.scale + with self.get() as conn: + yield conn + self.put(conn) + + @property + def ping(self): + with self.get() as conn: + try: + return True + except Exception as ldaperr: + try: + if conn.search_s("cn=config", ldap.SCOPE_ONELEVEL) != []: + return True + else: + # we might have ACI's in place + return True + except Exception as ldaperr: # collect with parent exception + pass + logging.error( + f"LDAP exception pinging server {self.uri.initializeUrl()} {ldaperr}" + ) + raise ldaperr + return True + + def get(self, binddn: str = "", bindpw: str = ""): + if len(self._pool) == 0: + self.scale + self._lock.acquire() + if len(self._pool) == 0: + self._lock.release() + logging.warning( + f"max connections {self.max} reached, consider increasing pool size" + ) + raise LDAPPoolExhausted( + f"max connections {self.max} reached, consider increasing pool size" + ) + try: + con = list(filter(lambda x: not x.inUse, self._pool))[0] + except IndexError: + self._lock.release() + logging.warning( + f"all connections {self.max} in use, consider increasing pool size" + ) + raise LDAPPoolExhausted( + f"all connections {self.max} in use, consider increasing pool size" + ) + con.inUse = True + self._lock.release() + if all([binddn != "", bindpw != ""]): + try: + con.authenticate(binddn, bindpw) + except ldap.INVALID_CREDENTIALS: + self.put(con) + raise ldap.INVALID_CREDENTIALS + return con + + def put(self, connection): + self._lock.acquire() + if connection.inUse: + connection.giveback() + if not connection in self._pool: + self._pool.append(connection) + connection._pool = self + self._lock.release() + return True + + def status(self): + self._lock.acquire() + for p in self._pool: + if p.inUse: + if sys.getrefcount(p) < 4: + p.giveback() + logging.info(f"Id {p} inUse {p.inUse} {p.established} {p.whoami}") + self._lock.release() + + def delete(self, connection, force=True): + return + self._lock.acquire() + if connection in self._pool: + if any([not self.params.get("keep", False), force]): + self._pool.remove(connection) + self._lock.release() From 665dc810fe2206fd40680de4159599309269aced Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Mon, 27 Jan 2025 13:19:06 +0100 Subject: [PATCH 2/9] rebased for easier integration --- Lib/ldappool/__init__.py | 59 +++-- Tests/__init__.py | 21 +- Tests/t_ldappool.py | 513 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 556 insertions(+), 37 deletions(-) create mode 100644 Tests/t_ldappool.py diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index 60aad335..0dffa137 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -82,10 +82,8 @@ def __whoami(self): try: self.__enter__() except: - pass - raise ldap.SERVER_DOWN( - f"max retries {self.params.get('retries', 3)} reached" - ) + continue + raise ldap.SERVER_DOWN(f"max retries {self.params.get('retries', 3)} reached") @property def conn(self): @@ -188,6 +186,7 @@ def __enter__(self): ) try: self._conn = ldap.initialize(self.uri.initializeUrl()) + self.__set_connection_parameters__() if self.params.get("autoBind", False): ( f"ConnectionPool {self} autoBind with {self.binddn} password {'x'*len(self.bindpw)}" @@ -199,8 +198,19 @@ def __enter__(self): self.established = True return self.conn - def giveback(self): + def giveback(self, force=False): try: + if force: + try: + self._conn.unbind_s() + except Exception as ldaperr: + logging.error( + "ConnectionPool unbind connection" + + f"{self} exception {ldaperr}" + ) + self.inUse = False + return + if self.params.get("autoBind", False): if not self.params.get("keep", False): logging.debug(f"ConnectionPool unbind connection {self}") @@ -227,7 +237,7 @@ def __exit__(self, type, value, traceback): self._pool.delete(self) def __cmp__(self, other): - if isinstance(other, LDAPUrl): + if isinstance(other, Connection): return self.uri.initializeUrl() == other.uri.initializeUrl() return False @@ -293,27 +303,33 @@ def set_uri(self, uri: LDAPUrl): if not isinstance(uri, LDAPUrl): uri = LDAPUrl(uri) if len(self._pool) > 0: - map( - lambda c: (c.set_uri(uri), c.giveback(force=True)), - filter(lambda cp: cp.uri != uri, self._pool), + list( + map( + lambda c: (c.set_uri(uri), c.giveback(force=True)), + filter(lambda cp: cp.uri != uri, self._pool), + ) ) self.uri = uri return True def set_binddn(self, binddn: str): if len(self._pool) > 0: - map( - lambda c: (c.set_binddn(binddn), c.giveback(force=True)), - filter(lambda cp: cp.binddn != binddn, self._pool), + list( + map( + lambda c: (c.set_binddn(binddn), c.giveback(force=True)), + filter(lambda cp: cp.binddn != binddn, self._pool), + ) ) self.binddn = binddn return True def set_bindpw(self, bindpw: str): if len(self._pool) > 0: - map( - lambda c: (c.set_bindpw(bindpw), c.giveback(force=True)), - filter(lambda cp: cp.bindpw != bindpw, self._pool), + list( + map( + lambda c: (c.set_bindpw(bindpw), c.giveback(force=True)), + filter(lambda cp: cp.bindpw != bindpw, self._pool), + ) ) self.bindpw = bindpw return True @@ -365,7 +381,7 @@ def ping(self): def get(self, binddn: str = "", bindpw: str = ""): if len(self._pool) == 0: self.scale - self._lock.acquire() + self._lock.acquire(timeout=1) if len(self._pool) == 0: self._lock.release() logging.warning( @@ -395,7 +411,7 @@ def get(self, binddn: str = "", bindpw: str = ""): return con def put(self, connection): - self._lock.acquire() + self._lock.acquire(timeout=1) if connection.inUse: connection.giveback() if not connection in self._pool: @@ -405,7 +421,7 @@ def put(self, connection): return True def status(self): - self._lock.acquire() + self._lock.acquire(timeout=1) for p in self._pool: if p.inUse: if sys.getrefcount(p) < 4: @@ -414,9 +430,12 @@ def status(self): self._lock.release() def delete(self, connection, force=True): - return - self._lock.acquire() + self._lock.acquire(timeout=1) if connection in self._pool: if any([not self.params.get("keep", False), force]): self._pool.remove(connection) + del connection self._lock.release() + + def __len__(self): + return len(self._pool) diff --git a/Tests/__init__.py b/Tests/__init__.py index ea28d0ce..b1391a87 100644 --- a/Tests/__init__.py +++ b/Tests/__init__.py @@ -4,20 +4,7 @@ See https://www.python-ldap.org/ for details. """ - -from . import t_bind -from . import t_cext -from . import t_cidict -from . import t_ldap_dn -from . import t_ldap_filter -from . import t_ldap_functions -from . import t_ldap_modlist -from . import t_ldap_schema_tokenizer -from . import t_ldapurl -from . import t_ldif -from . import t_ldapobject -from . import t_edit -from . import t_ldap_schema_subentry -from . import t_untested_mods -from . import t_ldap_controls_libldap -from . import t_ldap_options +from . import (t_bind, t_cext, t_cidict, t_edit, t_ldap_controls_libldap, t_ldap_dn, + t_ldap_filter, t_ldap_functions, t_ldap_modlist, t_ldap_options, + t_ldap_schema_subentry, t_ldap_schema_tokenizer, t_ldapobject, + t_ldappool, t_ldapurl, t_ldif, t_untested_mods) diff --git a/Tests/t_ldappool.py b/Tests/t_ldappool.py new file mode 100644 index 00000000..dbc57333 --- /dev/null +++ b/Tests/t_ldappool.py @@ -0,0 +1,513 @@ +""" +Automatic tests for python-ldap's module ldappool + +See https://www.python-ldap.org/ for details. +""" + +import os +import sys +import unittest +import time + +# Switch off processing .ldaprc or ldap.conf before importing _ldap +os.environ["LDAPNOINIT"] = "1" + +sys.path.append("../Lib") +import ldappool +from ldappool import Connection, ConnectionPool + +import ldap as _ldap +from ldapurl import LDAPUrl +import ldapurl + + +class ldapmock: + """Mocking some LDAP methods to avoid having a + full LDAP Setup for unittestst""" + + def __init__(self, fail=0, down=0): + self.fail = int(fail) + self.down = int(down) + + @property + def __whoami_s(self): + """if down was set when initializing + we fail for the count until we return success + """ + for _ in range(self.down): + self.down -= 1 + if self.down < 0: + self.down = 0 + raise _ldap.SERVER_DOWN() + return "cn=tester,dc=example,dc=com" + + def whoami_s(self): + """if down was set when initializing + we fail for the count until we return success + """ + for _ in range(self.down): + self.down -= 1 + if self.down < 0: + self.down = 0 + raise _ldap.SERVER_DOWN() + return "cn=tester,dc=example,dc=com" + + def initialize(self, uri): + return self + + def simple_bind_s(self, binddn, bindpw): + """if fail was set when initializing + we fail for the count until we return success + """ + for _ in range(self.fail): + self.fail -= 1 + if self.fail < 0: + self.fail = 0 + raise _ldap.INVALID_CREDENTIALS() + return (97, []) + + def set_option(self, *args, **kwargs): + return True + + def search_s(self, *args, **kwargs): + return True + + def authenticate(self, binddn, bindpw): + """if fail was set when initializing + we fail for the count until we return success + """ + for _ in range(self.fail): + self.fail -= 1 + if self.fail < 0: + self.fail = 0 + raise _ldap.INVALID_CREDENTIALS() + return (97, []) + + def unbind_s(self, *args, **kwargs): + return True + + def __enter__(self): + return + +class TestConnection(unittest.TestCase): + + def test_Connectionparams(self): + """test if a Connection handles parameters correctly""" + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + assert connection.params.get("retries") == 3 + assert connection.inUse == False + assert connection.established == False + assert connection.binddn == "cn=tester,dc=example,dc=com" + assert connection.bindpw == "changeme" + + def test_Connectionhandling(self): + """test if a Connection changes state when in use""" + ldap = ldapmock(fail=0) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + with connection as ctx: + assert connection.inUse == True + assert connection.established == True + assert connection.whoami == "cn=tester,dc=example,dc=com" + + def test_Connectionhandlingautherror(self): + """test if a connection raises exception if credentials are wrong""" + ldap = ldapmock(fail=2) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "allow_tls_fallback": True}, + ) + with self.assertRaises(_ldap.INVALID_CREDENTIALS) as ctx: + connection.conn() + + def test_Connectionhandlingauthentication(self): + """test if a connection can authenticate for someone""" + ldap = ldapmock(fail=0, down=0) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 1, "allow_tls_fallback": True}, + ) + connection._conn = ldap + with connection as ctx: + assert ctx.authenticate("test", "test") == (97, []) + + def test_Connectionhandlingserverdown(self): + """test if aconnection retries until params.retries reached""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + with connection as ctx: + assert connection.whoami == "cn=tester,dc=example,dc=com" + + def test_ConnectionhandlingserverdownExceed(self): + """test to ensure we raise ldap.SERVER_DOWN after max retries + has been reached and we have not succeeded""" + ldap = ldapmock(down=10) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + with self.assertRaises(_ldap.SERVER_DOWN) as ctx: + with connection as ctx: + connection.whoami + + """test to ensure without context ldap.SERVER_DOWN after max retries + has been reached and we have not succeeded""" + ldap = ldapmock(down=10) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + with self.assertRaises(_ldap.SERVER_DOWN) as ctx: + connection.conn.search_s() + + def test_Connectionconfigchange(self): + """test if a connection updates configuration changes accordingly""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + connection.set_credentials("cn=another,dc=example,dc=com", "changetoo") + assert connection.binddn == "cn=another,dc=example,dc=com" + assert connection.bindpw == "changetoo" + + def test_Connectionlocktime(self): + """test locktime which ensures we do not stress the connections too often""" + conn = Connection(LDAPUrl("ldap:///"), "", "") + assert conn._Connection__locktime() == True + assert conn._Connection__locktime() == False + time.sleep(15) + assert conn._Connection__locktime() == True + + + def test_Connectionmethods(self): + """test Connection methods which are there for + simplifying handling with the class""" + + """check set_uri""" + conn = Connection(LDAPUrl("ldap://127.0.0.1/"), "", "") + conn.set_uri(LDAPUrl("ldap://localhost/dc=example,dc=com")) + assert conn.uri == Connection(LDAPUrl("ldap://localhost/dc=example,dc=com"), "", "").uri + + """check set_binddn""" + conn = Connection(LDAPUrl("ldap://127.0.0.1/"), "", "") + conn.set_binddn("cn=Directory Manager") + assert conn.binddn == "cn=Directory Manager" + + """check set_bindpw""" + conn = Connection(LDAPUrl("ldap://127.0.0.1/"), "", "") + conn.set_bindpw("changeme") + assert conn.bindpw == "changeme" + + """check set_credentials""" + conn = Connection(LDAPUrl("ldap://127.0.0.1/"), "", "") + conn.set_credentials("cn=Directory Manager", "changeme") + assert conn.binddn == "cn=Directory Manager" + assert conn.bindpw == "changeme" + + +class TestConnectionPool(unittest.TestCase): + + def test_ConnectionPoolParams(self): + """test if ConnectionPool handles parameters correctly""" + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "autoBind": True}, + max=3, + ) + assert pool.params.get("retries") == 3 + assert pool.params.get("autoBind") == True + assert pool.binddn == "cn=tester,dc=example,dc=com" + assert pool.bindpw == "changeme" + assert pool.basedn == "dc=example,dc=com" + assert pool.scope == _ldap.SCOPE_SUBTREE + assert pool.filter == "(uid=tester)" + assert pool.attributes == ["uid", "mail"] + + def test_ConnectionPoolhandling(self): + """test if ConnectionPool context for Connection + is handled correctly""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + assert len(pool._pool) == 3 + assert pool.ping + with pool.get() as conn: + assert conn.search_s("something") == True + assert conn.authenticate("test", "test") == (97, []) + + def test_ConnectionPoolCleanup(self): + """test if ConnectionPool removing Connection + without deleting it""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True, "autoBind": True}, + max=3, + ) + assert len(pool._pool) == 3 + assert pool.ping + conn = pool.get() + pool.delete(conn) + + def test_ConnectionPoolconfigchange(self): + """test if ConnectionPool delegates configuration changes + to Connections in pool during runtime change""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.scale + assert len(pool) == 3 + pool.set_credentials("cn=another,dc=example,dc=com", "changetoo") + assert pool.binddn == "cn=another,dc=example,dc=com" + assert pool.bindpw == "changetoo" + for conn in pool._pool: + assert conn.binddn == "cn=another,dc=example,dc=com" + assert conn.bindpw == "changetoo" + + def test_ConnectionPoolLDAPPoolExhausted(self): + """test if ConnectionPool raises Exception when + all connections are in use""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.scale + assert len(pool) == 3 + with self.assertRaises(ldappool.LDAPPoolExhausted) as ctx: + for _ in range(pool.max + 1): + c = pool.get() + + def test_ConnectionPoolLDAPPoolExhausted(self): + """test if all connections are free'ed when returned + to the Pool but connection kept established""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.scale + assert len(pool) == 3 + + conn = [] + for _ in range(pool.max - 1): + conn.append(pool.get()) + for c in conn: + pool.put(c) + assert c.inUse == False + assert c.established == True + + def test_ConnectionPoolLDAPPoolGiveback(self): + """test if ConnectionPool giveback returns + connections to the pool""" + ldap = ldapmock() + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.scale + assert len(pool) == 3 + conn = pool.get() + conn.giveback() + assert conn.inUse == False + + """test if all connections are free'ed when returned + to the Pool but connection kept established""" + conn = [] + for _ in range(pool.max - 1): + conn.append(pool.get()) + for c in conn: + pool.put(c) + assert c.inUse == False + assert c.established == True + + def test_ConnectionPoolLDAPLockTimeout(self): + """test if ConnectionPool raises Locktimeout accordingly + when connection has been locked""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + pool.scale + assert len(pool) == 1 + with self.assertRaises(ldappool.LDAPLockTimeout) as ctx: + c = pool.get() + if not c._lock.acquire(blocking=True, timeout=1): + raise ldappool.LDAPLockTimeout() + if not c._lock.acquire(blocking=True, timeout=1): + raise ldappool.LDAPLockTimeout() + c.authenticate("test", "test") == (97, []) + + """test if ConnectionPool releases lock when Connection + is used and returned""" + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + pool.scale + assert len(pool) == 1 + with pool.get() as ctx: + pass + with pool.get() as ctx: + pass + + def test_ConnectionPoolmethods(self): + """test ConnectionPool methods which are there for + simplifying handling with the class""" + + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + assert pool.scope == _ldap.SCOPE_SUBTREE + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?base?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + assert pool.scope == _ldap.SCOPE_BASE + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?one?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + assert pool.scope == _ldap.SCOPE_ONELEVEL + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?one?(uid=tester)?extensiontest" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + assert isinstance(pool.extensions, ldapurl.LDAPUrlExtensions) + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?one?(uid=tester)?extensiontest" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.status + +if __name__ == "__main__": + unittest.main() From 306c2e41adab0630bc0fa6933f9dae6bfdf31f83 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Tue, 28 Jan 2025 09:32:08 +0100 Subject: [PATCH 3/9] added ldappool to setup and deploy as otherwise the unittest will fail --- setup.py | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 8e7963a1..92a2d4dc 100644 --- a/setup.py +++ b/setup.py @@ -142,6 +142,7 @@ class OpenLDAP2: py_modules = [ 'ldapurl', 'ldif', + 'ldappool', ], packages = [ diff --git a/tox.ini b/tox.ini index 22752067..c6def5c4 100644 --- a/tox.ini +++ b/tox.ini @@ -85,6 +85,7 @@ commands = Tests/t_ldap_modlist.py \ Tests/t_ldap_schema_tokenizer.py \ Tests/t_ldapurl.py \ + Tests/t_ldappool.py \ Tests/t_ldif.py \ Tests/t_untested_mods.py From 7576c47eef4510680cd73ade8793b9dd5dda4a4b Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Tue, 28 Jan 2025 19:26:31 +0100 Subject: [PATCH 4/9] fixed includes in ldappool unittest, fixed packages section --- Tests/t_ldappool.py | 1 - pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Tests/t_ldappool.py b/Tests/t_ldappool.py index dbc57333..6c3ddc60 100644 --- a/Tests/t_ldappool.py +++ b/Tests/t_ldappool.py @@ -12,7 +12,6 @@ # Switch off processing .ldaprc or ldap.conf before importing _ldap os.environ["LDAPNOINIT"] = "1" -sys.path.append("../Lib") import ldappool from ldappool import Connection, ConnectionPool diff --git a/pyproject.toml b/pyproject.toml index dda8dbc1..89596235 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,5 +4,5 @@ target-version = ['py36', 'py37', 'py38'] [tool.isort] line_length=88 -known_first_party=['ldap', '_ldap', 'ldapurl', 'ldif', 'slapdtest'] +known_first_party=['ldap', '_ldap', 'ldapurl', 'ldif', 'slapdtest', 'ldappool'] sections=['FUTURE', 'STDLIB', 'THIRDPARTY', 'FIRSTPARTY', 'LOCALFOLDER'] diff --git a/setup.py b/setup.py index 92a2d4dc..46974b73 100644 --- a/setup.py +++ b/setup.py @@ -142,7 +142,6 @@ class OpenLDAP2: py_modules = [ 'ldapurl', 'ldif', - 'ldappool', ], packages = [ @@ -152,6 +151,7 @@ class OpenLDAP2: 'ldap.schema', 'slapdtest', 'slapdtest.certs', + 'ldappool', ], package_dir = {'': 'Lib',}, data_files = LDAP_CLASS.extra_files, From 1140996bf35c982607d122d573da738f93544188 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Tue, 28 Jan 2025 19:58:46 +0100 Subject: [PATCH 5/9] added import error for python3.6 on dataclasses, since its not mandatory we just ignore it --- Lib/ldappool/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index 0dffa137..b0afb15c 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -1,4 +1,8 @@ -import dataclasses +try: + import dataclasses +except ImportError: + # we are on python < 3.7 so ignore + pass import logging import sys import threading @@ -28,8 +32,12 @@ class LDAPLockTimeout(Exception): def e2c(entry): - cls = dataclasses.make_dataclass("", ["dn"] + list(entry[1].keys()), frozen=True) - return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) + try: + cls = dataclasses.make_dataclass("", ["dn"] + list(entry[1].keys()), frozen=True) + return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) + except NameError as dcerror: + print(f"dataclasses not supported") + return entry class Connection(object): From e5e335b9d7d5a419e05e6a8bcf9eda630e559554 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Tue, 28 Jan 2025 21:27:32 +0100 Subject: [PATCH 6/9] some ldap options are not available < 3.9 --- Lib/ldappool/__init__.py | 46 +++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index b0afb15c..f243bfaf 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -33,7 +33,9 @@ class LDAPLockTimeout(Exception): def e2c(entry): try: - cls = dataclasses.make_dataclass("", ["dn"] + list(entry[1].keys()), frozen=True) + cls = dataclasses.make_dataclass( + "", ["dn"] + list(entry[1].keys()), frozen=True + ) return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) except NameError as dcerror: print(f"dataclasses not supported") @@ -166,25 +168,29 @@ def __authenticate__(self): raise ldap.INVALID_CREDENTIALS def __set_connection_parameters__(self): - self._conn.set_option(ldap.OPT_REFERRALS, self.params.get("referrals", False)) - self._conn.set_option( - ldap.OPT_NETWORK_TIMEOUT, self.params.get("network_timeout", 10.0) - ) - self._conn.set_option(ldap.OPT_TIMEOUT, self.params.get("timeout", 10.0)) - self._conn.set_option( - ldap.OPT_X_KEEPALIVE_IDLE, self.params.get("keepalive_idle", 10) - ) - self._conn.set_option( - ldap.OPT_X_KEEPALIVE_INTERVAL, self.params.get("keepalive_interval", 5) - ) - self._conn.set_option( - ldap.OPT_X_KEEPALIVE_PROBES, self.params.get("keepalive_probes", 3) - ) - self._conn.set_option(ldap.OPT_RESTART, ldap.OPT_ON) - if self.params.get("allow_tls_fallback", False): - logging.debug("TLS Fallback enabled in LDAP") - self._conn.set_option(ldap.OPT_X_TLS_TRY, 1) - self._conn.set_option(ldap.OPT_X_TLS_NEWCTX, ldap.OPT_OFF) + try: + self._conn.set_option( + ldap.OPT_REFERRALS, self.params.get("referrals", False) + ) + self._conn.set_option( + ldap.OPT_NETWORK_TIMEOUT, self.params.get("network_timeout", 10.0) + ) + self._conn.set_option(ldap.OPT_TIMEOUT, self.params.get("timeout", 10.0)) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_IDLE, self.params.get("keepalive_idle", 10) + ) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_INTERVAL, self.params.get("keepalive_interval", 5) + ) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_PROBES, self.params.get("keepalive_probes", 3) + ) + self._conn.set_option(ldap.OPT_RESTART, ldap.OPT_ON) + if self.params.get("allow_tls_fallback", False): + self._conn.set_option(ldap.OPT_X_TLS_TRY, 1) + self._conn.set_option(ldap.OPT_X_TLS_NEWCTX, ldap.OPT_OFF) + except Exception as connerr: + logging.error(f"cannot set LDAP option {connerr}") def __enter__(self): self.inUse = True From 85eae2464efa5c5a1fdbe6b6e8126f7ccf892ae8 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Wed, 29 Jan 2025 07:40:24 +0100 Subject: [PATCH 7/9] added python 3.6-8 checks and exceptions --- Lib/ldappool/__init__.py | 44 +++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index f243bfaf..b4902cb0 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -1,14 +1,19 @@ -try: +import sys + +if sys.version_info.minor >= 3.8: import dataclasses -except ImportError: - # we are on python < 3.7 so ignore - pass import logging import sys import threading -import time from urllib.parse import urlparse +import sys + +if sys.version_info.minor > 6: + from time import perf_counter_ns as perf_counter +elif sys.version_info.minor == 6: + from time import perf_counter + import ldap from ldapurl import LDAPUrl @@ -31,15 +36,22 @@ class LDAPLockTimeout(Exception): pass -def e2c(entry): - try: - cls = dataclasses.make_dataclass( - "", ["dn"] + list(entry[1].keys()), frozen=True - ) - return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) - except NameError as dcerror: - print(f"dataclasses not supported") - return entry +if sys.version_info >= 3.8: + + def e2c(entry): + try: + cls = dataclasses.make_dataclass( + "", ["dn"] + list(entry[1].keys()), frozen=True + ) + return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) + except NameError as dcerror: + print(f"dataclasses not supported") + return entry + +else: + + def e2c(entry): + return f"dataclasses not support on python < {sys.version_info.minor}" class Connection(object): @@ -67,9 +79,9 @@ def __init__( def __locktime(self): if self._health == 0.0: - self._health = time.perf_counter_ns() + self._health = perf_counter() return True - if (time.perf_counter_ns() - self._health) / ns < ns_locktimeout: + if (perf_counter() - self._health) / ns < ns_locktimeout: return False return True From e80b2f4cd5555d0a6ddbf8cdffe7fc71c812e447 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Wed, 29 Jan 2025 07:59:45 +0100 Subject: [PATCH 8/9] fixed one misses version_info compare --- Lib/ldappool/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index b4902cb0..6c8e1c74 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -36,7 +36,7 @@ class LDAPLockTimeout(Exception): pass -if sys.version_info >= 3.8: +if sys.version_info.minor >= 3.8: def e2c(entry): try: From 5e9bd910f5bd33d328097fc2cc7b742dba50a68c Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Thu, 30 Jan 2025 06:56:11 +0100 Subject: [PATCH 9/9] minor compare is single int not float --- Lib/ldappool/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index 6c8e1c74..8530e005 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -1,6 +1,6 @@ import sys -if sys.version_info.minor >= 3.8: +if sys.version_info.minor >= 8: import dataclasses import logging import sys @@ -36,7 +36,7 @@ class LDAPLockTimeout(Exception): pass -if sys.version_info.minor >= 3.8: +if sys.version_info.minor >= 8: def e2c(entry): try: pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy