From 6421640a1515894d8beeb5016420536189b1d267 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 12:54:10 -0800 Subject: [PATCH 01/15] Add admin API related exceptions --- include/cppkafka/exceptions.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/cppkafka/exceptions.h b/include/cppkafka/exceptions.h index 8bfd8016..7cde7ad4 100644 --- a/include/cppkafka/exceptions.h +++ b/include/cppkafka/exceptions.h @@ -134,6 +134,22 @@ class CPPKAFKA_API QueueException : public Exception { Error error_; }; +/** + * Admin operation options exception + */ +class CPPKAFKA_API AdminOperationOptionsException : public Exception { +public: + using Exception::Exception; +}; + +/** + * Admin operation exception + */ +class CPPKAFKA_API AdminOperationException : public Exception { +public: + using Exception::Exception; +}; + } // cppkafka #endif // CPPKAFKA_EXCEPTIONS_H From 3065e4f126342a674ca8452d24a05150290021c9 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 12:54:35 -0800 Subject: [PATCH 02/15] Add admin API operation options --- include/cppkafka/admin/operation_options.h | 116 +++++++++++++++++++++ src/admin/operation_options.cpp | 103 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 include/cppkafka/admin/operation_options.h create mode 100644 src/admin/operation_options.cpp diff --git a/include/cppkafka/admin/operation_options.h b/include/cppkafka/admin/operation_options.h new file mode 100644 index 00000000..a85c6dd2 --- /dev/null +++ b/include/cppkafka/admin/operation_options.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef CPPKAFKA_ADMIN_OPERATION_OPTIONS_H +#define CPPKAFKA_ADMIN_OPERATION_OPTIONS_H + +#include +#include +#include +#include +#include "../error.h" +#include "../kafka_handle_base.h" +#include "../macros.h" + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +class CPPKAFKA_API OperationOptions { +public: + /** + * \brief Constructs an instance of OperationOptions + * + * Note that this instance is tied to the kafka handle and will fail if + * it's used on an operation that's applied on a different handle. + * + * \param kafka_handle The kafka handle to create this operation options in + * \param type The type of operation this applies to + */ + OperationOptions(KafkaHandleBase& kafka_handle, rd_kafka_admin_op_t type); + + /** + * \brief Sets the overall request timeout + * + * This calls rd_kafka_AdminOptions_set_request_timeout under the hood + * using the provided timeout. + * + * \param timeout The timeout to be set for the request + */ + void set_request_timeout(std::chrono::milliseconds timeout); + + /** + * \brief Sets the overall operation timeout + * + * This calls rd_kafka_AdminOptions_set_operation_timeout under the hood + * using the provided timeout. + * + * \param timeout The timeout to be set for the request + */ + void set_operation_timeout(std::chrono::milliseconds timeout); + + /** + * \brief Indicates whether the operation should only be validated + * + * This calls rd_kafka_AdminOptions_set_validate_only under the hood + * using the provided value. + * + * \param validate Whether to only validate the request + */ + void set_validate_only(bool validate); + + /** + * \brief Override what broker the Admin request will be sent to. + * + * This calls rd_kafka_AdminOptions_set_broker under the hood + * using the provided broker id. + * + * \param broker_id The broker id to be set for the request + */ + void set_operation_timeout(uint32_t broker_id); + + /** + * \brief Gets the internal rd_kafka_AdminOptions_t pointer + */ + rd_kafka_AdminOptions_t* get_handle() const; +private: + using HandlePtr = std::unique_ptr; + + HandlePtr handle_; + // Kept only for validation when the OperationOptions is used + const KafkaHandleBase* kafka_handle_; +}; + +} // admin +} // cppkafka + +#endif // Admin API +#endif // CPPKAFKA_ADMIN_OPERATION_OPTIONS_H diff --git a/src/admin/operation_options.cpp b/src/admin/operation_options.cpp new file mode 100644 index 00000000..9ace7cd5 --- /dev/null +++ b/src/admin/operation_options.cpp @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "admin/operation_options.h" + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +OperationOptions::OperationOptions(KafkaHandleBase& kafka_handle, rd_kafka_admin_op_t type) +: handle_(rd_kafka_AdminOptions_new(kafka_handle.get_handle(), type), + rd_kafka_AdminOptions_destroy), + kafka_handle_(&kafka_handle) { + +} + +void OperationOptions::set_request_timeout(std::chrono::milliseconds timeout) { + char error_bufer[512] = { 0 }; + const Error result = rd_kafka_AdminOptions_set_request_timeout( + handle_.get(), + timeout.count(), + error_bufer, + sizeof(error_bufer) + ); + if (result) { + throw AdminOperationOptionsException(error_bufer); + } +} + +void OperationOptions::set_operation_timeout(std::chrono::milliseconds timeout) { + char error_bufer[512] = { 0 }; + const Error result = rd_kafka_AdminOptions_set_operation_timeout( + handle_.get(), + timeout.count(), + error_bufer, + sizeof(error_bufer) + ); + if (result) { + throw AdminOperationOptionsException(error_bufer); + } +} + +void OperationOptions::set_validate_only(bool validate) { + char error_bufer[512] = { 0 }; + const Error result = rd_kafka_AdminOptions_set_validate_only( + handle_.get(), + validate, + error_bufer, + sizeof(error_bufer) + ); + if (result) { + throw AdminOperationOptionsException(error_bufer); + } +} + +void OperationOptions::set_operation_timeout(uint32_t broker_id) { + char error_bufer[512] = { 0 }; + const Error result = rd_kafka_AdminOptions_set_broker( + handle_.get(), + broker_id, + error_bufer, + sizeof(error_bufer) + ); + if (result) { + throw AdminOperationOptionsException(error_bufer); + } +} + +rd_kafka_AdminOptions_t* OperationOptions::get_handle() const { + return handle_.get(); +} + +} // admin +} // cppkafka + +#endif From a93a9ab72dfc88771937c7d2918b42ea687837e4 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 14:00:56 -0800 Subject: [PATCH 03/15] Allow checking for operation options -> kafka handle association --- include/cppkafka/admin/operation_options.h | 7 +++++++ src/admin/operation_options.cpp | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/include/cppkafka/admin/operation_options.h b/include/cppkafka/admin/operation_options.h index a85c6dd2..778f721e 100644 --- a/include/cppkafka/admin/operation_options.h +++ b/include/cppkafka/admin/operation_options.h @@ -100,6 +100,13 @@ class CPPKAFKA_API OperationOptions { * \brief Gets the internal rd_kafka_AdminOptions_t pointer */ rd_kafka_AdminOptions_t* get_handle() const; + + /** + * \brief Indicates whether this handle was constructed with the provided kafka handle + * + * \param kafka_handle The kafka handle to check association with + */ + bool is_associated_with(const KafkaHandleBase& kafka_handle) const; private: using HandlePtr = std::unique_ptr; diff --git a/src/admin/operation_options.cpp b/src/admin/operation_options.cpp index 9ace7cd5..0e2dbbc9 100644 --- a/src/admin/operation_options.cpp +++ b/src/admin/operation_options.cpp @@ -97,6 +97,10 @@ rd_kafka_AdminOptions_t* OperationOptions::get_handle() const { return handle_.get(); } +bool OperationOptions::is_associated_with(const KafkaHandleBase& kafka_handle) const { + return kafka_handle_ == &kafka_handle; +} + } // admin } // cppkafka From b194a7b7c67a12962d6bcbc6eb696b21d1ee527d Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 14:03:32 -0800 Subject: [PATCH 04/15] Add admin API operation base class --- include/cppkafka/admin/operation.h | 84 ++++++++++++++++++++++++++++++ src/admin/operation.cpp | 50 ++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 include/cppkafka/admin/operation.h create mode 100644 src/admin/operation.cpp diff --git a/include/cppkafka/admin/operation.h b/include/cppkafka/admin/operation.h new file mode 100644 index 00000000..efc895db --- /dev/null +++ b/include/cppkafka/admin/operation.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef CPPKAFKA_ADMIN_OPERATION_H +#define CPPKAFKA_ADMIN_OPERATION_H + +#include "operation_options.h" +#include "../kafka_handle_base.h" +#include "../macros.h" + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + + +/** + * \brief Represents an admin operation to be applied on a kafka handle + */ +class CPPKAFKA_API Operation { +public: + virtual ~Operation() = default; + + /** + * \brief Execute this operation on the given handle + * + * The operation will be asynchronously executed on the given KafkaHandleBase and + * the result of it will be written into the provided Queue. + * + * \param kafka_handle The kafka handle to run this operation on + * \param queue The queue in which to post the result of this operation + * \param options The options to be used for this operation + */ + void execute(KafkaHandleBase& kafka_handle, Queue& queue, const OperationOptions& options); + + /** + * \brief Execute this operation on the given handle + * + * The operation will be asynchronously executed on the given KafkaHandleBase and + * the result of it will be written into the provided Queue. + * + * This will use the default options for this operation. + * + * \param kafka_handle The kafka handle to run this operation on + * \param queue The queue in which to post the result of this operation + */ + void execute(KafkaHandleBase& kafka_handle, Queue& queue); +protected: + virtual void do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) = 0; +}; + +} // admin +} // cppkafka + +#endif // Admin API +#endif // CPPKAFKA_ADMIN_OPERATION_H diff --git a/src/admin/operation.cpp b/src/admin/operation.cpp new file mode 100644 index 00000000..dcc3a3f4 --- /dev/null +++ b/src/admin/operation.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "admin/operation.h" +#include "exceptions.h" + +namespace cppkafka { +namespace admin { + +void Operation::execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions& options) { + if (!options.is_associated_with(kafka_handle)) { + throw AdminOperationException("Invalid kafka handle"); + } + do_execute(kafka_handle, queue, &options); +} + +void Operation::execute(KafkaHandleBase& kafka_handle, Queue& queue) { + do_execute(kafka_handle, queue, nullptr); +} + +} // admin +} // cppkafka From cb34974b10522fe3927b9c4d88a0d170602b753e Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 14:03:46 -0800 Subject: [PATCH 05/15] Add create topic operation --- .../cppkafka/admin/create_topic_operation.h | 115 ++++++++++++++++++ src/admin/create_topic_operation.cpp | 113 +++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 include/cppkafka/admin/create_topic_operation.h create mode 100644 src/admin/create_topic_operation.cpp diff --git a/include/cppkafka/admin/create_topic_operation.h b/include/cppkafka/admin/create_topic_operation.h new file mode 100644 index 00000000..2d567d37 --- /dev/null +++ b/include/cppkafka/admin/create_topic_operation.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef CPPKAFKA_CREATE_TOPIC_OPERATION_H +#define CPPKAFKA_CREATE_TOPIC_OPERATION_H + +#include +#include +#include +#include +#include "operation.h" +#include "../configuration_option.h" + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +class CreateTopicOperation : public Operation { +public: + /** + * \brief Constructs an instance of a CreateTopicOperation + * + * Note that calling set_replica_assignment is invalid when using this constructor. + * Use the other one which doesn't take a replication factor if you want to manually + * assign the replica assignment to each partition + * + * \param name The name of the topic to be created + * \param partitions The number of partitions to be created for this topic + * \param replication_factor The topic's replication factor + */ + CreateTopicOperation(const std::string& name, + unsigned partitions, + unsigned replication_factor); + + /** + * \brief Constructs an instance of a CreateTopicOperation + * + * When calling this constructor, the user *must* call set_replica_assignment for each + * of the partitions in ascending order. This is an API restriction imposed by librdkafka. + * + * \param name The name of the topic to be created + * \param partitions The number of partitions to be created for this topic + */ + CreateTopicOperation(const std::string& name, + unsigned partitions); + + /** + * \brief Sets the replica assignment for a particular partition + * + * This calls rd_kafka_NewTopic_set_replica_assignment under the hood + * + * This method can only be called if the constructor called was the one that doesn't take + * a replication factor. + * + * This method must be called for every each partition in this topic (starting from 0) + * in ascending order. This is an API restriction imposed by librdkafka. + * + * + * \param partition The partition for which to set the replica assignment + * \param broker_ids The list of broker ids that will replicate this partition + */ + void set_replica_assignment(int partition, const std::vector& broker_ids); + + /** + * \brief Sets a server side configuration option + * + * \param config_option The configuration option to be set + */ + void set_config(const ConfigurationOption& config_option); + +private: + void do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) override; + + using HandlePtr = std::unique_ptr; + + void init(const std::string& topic, unsigned partitions, int replication_factor); + + HandlePtr handle_; +}; + +} // admin +} // cppkafka + +#endif // Admin API +#endif // CPPKAFKA_CREATE_TOPIC_OPERATION_H diff --git a/src/admin/create_topic_operation.cpp b/src/admin/create_topic_operation.cpp new file mode 100644 index 00000000..67f24e92 --- /dev/null +++ b/src/admin/create_topic_operation.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "admin/create_topic_operation.h" + +using std::string; +using std::vector; + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +CreateTopicOperation::CreateTopicOperation(const string& name, + unsigned partitions, + unsigned replication_factor) +: handle_(nullptr, nullptr) { + init(name, partitions, static_cast(replication_factor)); +} + +CreateTopicOperation::CreateTopicOperation(const string& name, + unsigned partitions) +: handle_(nullptr, nullptr) { + init(name, partitions, -1); +} + +void CreateTopicOperation::set_replica_assignment(int partition, + const vector& broker_ids) { + char error_buffer[512] = { 0 }; + const Error result = rd_kafka_NewTopic_set_replica_assignment( + handle_.get(), + partition, + const_cast(broker_ids.data()), + broker_ids.size(), + error_buffer, + sizeof(error_buffer) + ); + if (result) { + throw AdminOperationException(error_buffer); + } +} + +void CreateTopicOperation::set_config(const ConfigurationOption& config_option) { + const Error result = rd_kafka_NewTopic_set_config( + handle_.get(), + config_option.get_key().data(), + config_option.get_value().data() + ); + if (result) { + throw AdminOperationException(result.to_string()); + } +} + +void CreateTopicOperation::do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) { + rd_kafka_NewTopic_t* new_topic_handle = handle_.get(); + rd_kafka_CreateTopics( + kafka_handle.get_handle(), + &new_topic_handle, + 1 /*number of topics*/, + options ? options->get_handle() : nullptr, + queue.get_handle() + ); +} + +void CreateTopicOperation::init(const string& topic, + unsigned partitions, + int replication_factor) { + char error_buffer[512] = { 0 }; + auto rdkafka_handle = rd_kafka_NewTopic_new( + topic.data(), + static_cast(partitions), + replication_factor, + error_buffer, + sizeof(error_buffer) + ); + if (!rdkafka_handle) { + throw AdminOperationException(error_buffer); + } + handle_ = HandlePtr(rdkafka_handle, rd_kafka_NewTopic_destroy); +} + +} // admin +} // cppkafka + +#endif // Admin API From 28cb46f6001f4e52703138e62fc027b30b3a5842 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 14:05:43 -0800 Subject: [PATCH 06/15] Allow getting the internal handle out of a create topic op --- include/cppkafka/admin/create_topic_operation.h | 4 ++++ src/admin/create_topic_operation.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/include/cppkafka/admin/create_topic_operation.h b/include/cppkafka/admin/create_topic_operation.h index 2d567d37..549deab6 100644 --- a/include/cppkafka/admin/create_topic_operation.h +++ b/include/cppkafka/admin/create_topic_operation.h @@ -95,6 +95,10 @@ class CreateTopicOperation : public Operation { */ void set_config(const ConfigurationOption& config_option); + /** + * \brief Gets a pointer to the internal handle + */ + rd_kafka_NewTopic_t* get_handle() const; private: void do_execute(KafkaHandleBase& kafka_handle, Queue& queue, diff --git a/src/admin/create_topic_operation.cpp b/src/admin/create_topic_operation.cpp index 67f24e92..9eaedf60 100644 --- a/src/admin/create_topic_operation.cpp +++ b/src/admin/create_topic_operation.cpp @@ -77,6 +77,10 @@ void CreateTopicOperation::set_config(const ConfigurationOption& config_option) } } +rd_kafka_NewTopic_t* CreateTopicOperation::get_handle() const { + return handle_.get(); +} + void CreateTopicOperation::do_execute(KafkaHandleBase& kafka_handle, Queue& queue, const OperationOptions* options) { From 6fd98bb4edd95b596518694178933e73e62de782 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 14:43:19 -0800 Subject: [PATCH 07/15] Add CompoundOperation template class --- include/cppkafka/admin/compound_operation.h | 121 ++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 include/cppkafka/admin/compound_operation.h diff --git a/include/cppkafka/admin/compound_operation.h b/include/cppkafka/admin/compound_operation.h new file mode 100644 index 00000000..e6ed70b9 --- /dev/null +++ b/include/cppkafka/admin/compound_operation.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include "operation.h" + +#ifndef CPPKAFKA_ADMIN_COMPOUND_OPERATION_H +#define CPPKAFKA_ADMIN_COMPOUND_OPERATION_H + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { +namespace detail { + +template +using CompoundOperationFunction = void(*)(rd_kafka_t*, + typename Operation::HandleType**, + size_t, + const rd_kafka_AdminOptions_t*, + rd_kafka_queue_t*); + +} // details + +/** + * \brief Bundles several operations of the same type and executes them in a single call + * + * This can bundle operations (say CreateTopicOperation) and execute them on a single call + * rather than having to execute each of them individually. + * + * This class should typically be used via the type alias defined in other files, + * e.g. CreateTopicsOperation + */ +template Function> +class CompoundOperation : public Operation { +public: + /** + * \brief Default constructs a compound operation + */ + CompoundOperation() = default; + + /** + * \brief Constructs a compound operation using a list of sub operations + * + * \param operations The list of sub operations to use + */ + CompoundOperation(std::vector operations); + + /** + * \brief Adds an sub operation to this compound operation + * + * \param operation The sub operation to be added + */ + void add_operation(OperationType operation); +private: + void do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) override; + + std::vector operations_; +}; + +template E> +CompoundOperation::CompoundOperation(std::vector operations) +: operations_(std::move(operations)) { +} + +template E> +void CompoundOperation::add_operation(T operation) { + operations_.emplace_back(std::move(operation)); +} + +template E> +void CompoundOperation::do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) { + std::vector handles; + for (auto& operation : operations_) { + handles.emplace_back(operation.get_handle()); + } + E( + kafka_handle.get_handle(), + handles.data(), + handles.size(), + options ? options->get_handle() : nullptr, + queue.get_handle() + ); +} + + +} // admin +} // cppkafka + +#endif // Admin API +#endif // CPPKAFKA_ADMIN_COMPOUND_OPERATION_H From 0ffb70fb9c039a976f9d3343aa2c680c1b32ff5d Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 14:43:31 -0800 Subject: [PATCH 08/15] Add type alias to create multiple topics --- include/cppkafka/admin/create_topic_operation.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/include/cppkafka/admin/create_topic_operation.h b/include/cppkafka/admin/create_topic_operation.h index 549deab6..57761fe8 100644 --- a/include/cppkafka/admin/create_topic_operation.h +++ b/include/cppkafka/admin/create_topic_operation.h @@ -35,6 +35,7 @@ #include #include #include "operation.h" +#include "compound_operation.h" #include "../configuration_option.h" #if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION @@ -44,6 +45,11 @@ namespace admin { class CreateTopicOperation : public Operation { public: + /** + * The rdkafka type used for this operation's handle + */ + using HandleType = rd_kafka_NewTopic_t; + /** * \brief Constructs an instance of a CreateTopicOperation * @@ -112,6 +118,12 @@ class CreateTopicOperation : public Operation { HandlePtr handle_; }; +/** + * \brief Creates several topics + */ +using CreateTopicsOperation = CompoundOperation; + } // admin } // cppkafka From b8a695bdfedc4be48478451cb132e3b4a4c5082e Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 14:55:20 -0800 Subject: [PATCH 09/15] Add delete topic operation --- .../cppkafka/admin/delete_topic_operation.h | 84 +++++++++++++++++++ src/admin/delete_topic_operation.cpp | 64 ++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 include/cppkafka/admin/delete_topic_operation.h create mode 100644 src/admin/delete_topic_operation.cpp diff --git a/include/cppkafka/admin/delete_topic_operation.h b/include/cppkafka/admin/delete_topic_operation.h new file mode 100644 index 00000000..d925f6ce --- /dev/null +++ b/include/cppkafka/admin/delete_topic_operation.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef CPPKAFKA_ADMIN_DELETE_TOPIC_OPERATION_H +#define CPPKAFKA_ADMIN_DELETE_TOPIC_OPERATION_H + +#include +#include "compound_operation.h" +#include "operation.h" + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +/** + * \brief Deletes a topic + */ +class DeleteTopicOperation : public Operation { +public: + /** + * The rdkafka type used for this operation's handle + */ + using HandleType = rd_kafka_DeleteTopic_t; + + /** + * \brief Constructs an instance given a topic name + * + * \param name The name of the topic to be deleted + */ + DeleteTopicOperation(const std::string& name); + + /** + * \brief Gets the underlying rdkafka handle + */ + HandleType* get_handle() const; +private: + using HandlePtr = std::unique_ptr; + + void do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) override; + + HandlePtr handle_; +}; + +/** + * \brief Deletes several topics + */ +using DeleteTopicsOperation = CompoundOperation; + +} // admin +} // cppkafka + +#endif // Admin API +#endif // CPPKAFKA_ADMIN_DELETE_TOPIC_OPERATION_H diff --git a/src/admin/delete_topic_operation.cpp b/src/admin/delete_topic_operation.cpp new file mode 100644 index 00000000..5009ebcf --- /dev/null +++ b/src/admin/delete_topic_operation.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "admin/delete_topic_operation.h" + +using std::string; + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +DeleteTopicOperation::DeleteTopicOperation(const string& name) +: handle_(rd_kafka_DeleteTopic_new(name.data()), &rd_kafka_DeleteTopic_destroy) { + +} + +void DeleteTopicOperation::do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) { + rd_kafka_DeleteTopic_t* delete_topic_handle = handle_.get(); + rd_kafka_DeleteTopics( + kafka_handle.get_handle(), + &delete_topic_handle, + 1 /*number of operations*/, + options ? options->get_handle() : nullptr, + queue.get_handle() + ); +} + +DeleteTopicOperation::HandleType* DeleteTopicOperation::get_handle() const { + return handle_.get(); +} + +} // admin +} // cppkafka + +#endif // Admin API From fde8fda059676a4c4c5d804789969bf5f5be2d63 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 10 Nov 2018 14:56:09 -0800 Subject: [PATCH 10/15] Use HandleType in create topic operation --- include/cppkafka/admin/create_topic_operation.h | 4 ++-- src/admin/create_topic_operation.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/cppkafka/admin/create_topic_operation.h b/include/cppkafka/admin/create_topic_operation.h index 57761fe8..2975e1ad 100644 --- a/include/cppkafka/admin/create_topic_operation.h +++ b/include/cppkafka/admin/create_topic_operation.h @@ -102,9 +102,9 @@ class CreateTopicOperation : public Operation { void set_config(const ConfigurationOption& config_option); /** - * \brief Gets a pointer to the internal handle + * \brief Gets the underlying rdkafka handle */ - rd_kafka_NewTopic_t* get_handle() const; + HandleType* get_handle() const; private: void do_execute(KafkaHandleBase& kafka_handle, Queue& queue, diff --git a/src/admin/create_topic_operation.cpp b/src/admin/create_topic_operation.cpp index 9eaedf60..66ab1136 100644 --- a/src/admin/create_topic_operation.cpp +++ b/src/admin/create_topic_operation.cpp @@ -77,7 +77,7 @@ void CreateTopicOperation::set_config(const ConfigurationOption& config_option) } } -rd_kafka_NewTopic_t* CreateTopicOperation::get_handle() const { +CreateTopicOperation::HandleType* CreateTopicOperation::get_handle() const { return handle_.get(); } From 00ca59ccdd1a1b7469c6885901cee5d5b5eb5747 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 17 Nov 2018 17:11:55 -0800 Subject: [PATCH 11/15] Add helper function to make running admin operations easier --- include/cppkafka/admin/operation.h | 23 ++++++++++++++++++++++- src/admin/create_topic_operation.cpp | 9 +-------- src/admin/delete_topic_operation.cpp | 9 +-------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/include/cppkafka/admin/operation.h b/include/cppkafka/admin/operation.h index efc895db..a71e89ac 100644 --- a/include/cppkafka/admin/operation.h +++ b/include/cppkafka/admin/operation.h @@ -39,7 +39,6 @@ namespace cppkafka { namespace admin { - /** * \brief Represents an admin operation to be applied on a kafka handle */ @@ -72,9 +71,31 @@ class CPPKAFKA_API Operation { */ void execute(KafkaHandleBase& kafka_handle, Queue& queue); protected: + template + using RunOperationType = void(*)(rd_kafka_t*, + T**, + size_t, + const rd_kafka_AdminOptions_t*, + rd_kafka_queue_t*); + virtual void do_execute(KafkaHandleBase& kafka_handle, Queue& queue, const OperationOptions* options) = 0; + + template + void run_operation(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options, + T* operation_handle, + RunOperationType functor) { + functor( + kafka_handle.get_handle(), + &operation_handle, + 1 /*number of operations*/, + options ? options->get_handle() : nullptr, + queue.get_handle() + ); + } }; } // admin diff --git a/src/admin/create_topic_operation.cpp b/src/admin/create_topic_operation.cpp index 66ab1136..5547dce5 100644 --- a/src/admin/create_topic_operation.cpp +++ b/src/admin/create_topic_operation.cpp @@ -84,14 +84,7 @@ CreateTopicOperation::HandleType* CreateTopicOperation::get_handle() const { void CreateTopicOperation::do_execute(KafkaHandleBase& kafka_handle, Queue& queue, const OperationOptions* options) { - rd_kafka_NewTopic_t* new_topic_handle = handle_.get(); - rd_kafka_CreateTopics( - kafka_handle.get_handle(), - &new_topic_handle, - 1 /*number of topics*/, - options ? options->get_handle() : nullptr, - queue.get_handle() - ); + run_operation(kafka_handle, queue, options, handle_.get(), &rd_kafka_CreateTopics); } void CreateTopicOperation::init(const string& topic, diff --git a/src/admin/delete_topic_operation.cpp b/src/admin/delete_topic_operation.cpp index 5009ebcf..aba6cf01 100644 --- a/src/admin/delete_topic_operation.cpp +++ b/src/admin/delete_topic_operation.cpp @@ -44,14 +44,7 @@ DeleteTopicOperation::DeleteTopicOperation(const string& name) void DeleteTopicOperation::do_execute(KafkaHandleBase& kafka_handle, Queue& queue, const OperationOptions* options) { - rd_kafka_DeleteTopic_t* delete_topic_handle = handle_.get(); - rd_kafka_DeleteTopics( - kafka_handle.get_handle(), - &delete_topic_handle, - 1 /*number of operations*/, - options ? options->get_handle() : nullptr, - queue.get_handle() - ); + run_operation(kafka_handle, queue, options, handle_.get(), &rd_kafka_DeleteTopics); } DeleteTopicOperation::HandleType* DeleteTopicOperation::get_handle() const { From 94443b1a097d1b04be37d96534ddf692e37e98e7 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 17 Nov 2018 17:23:35 -0800 Subject: [PATCH 12/15] Add create partitions admin API class --- .../admin/create_partitions_operation.h | 98 +++++++++++++++++++ src/admin/create_partitions_operation.cpp | 81 +++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 include/cppkafka/admin/create_partitions_operation.h create mode 100644 src/admin/create_partitions_operation.cpp diff --git a/include/cppkafka/admin/create_partitions_operation.h b/include/cppkafka/admin/create_partitions_operation.h new file mode 100644 index 00000000..041c745f --- /dev/null +++ b/include/cppkafka/admin/create_partitions_operation.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef CPPKAFKA_CREATE_PARTITIONS_OPERATION_H +#define CPPKAFKA_CREATE_PARTITIONS_OPERATION_H + +#include +#include +#include +#include +#include "operation.h" +#include "compound_operation.h" +#include "../configuration_option.h" + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +class CreatePartitionsOperation : public Operation { +public: + /** + * The rdkafka type used for this operation's handle + */ + using HandleType = rd_kafka_NewPartitions_t; + + /** + * \brief Constructs a create partitions operation + * + * \param topic The name of the topic to modify + * \param partitions The desired number of partitions to set on the topic + */ + CreatePartitionsOperation(std::string topic, size_t partitions); + + /** + * \brief Sets the replica assignment for a partition + * + * This calls rd_kafka_NewPartitions_set_replica_assignment under the hood + * + * The replica assignment has to be set for either all partitions or none of them. + * This method has to be called consecutively for all indexes from 0 to the partitions + * constructor parameter - 1. + * + * Note that the provided parameter is the new partition index rather than the + * partition itself. + * + * \param new_partition_index The index of the new partition to be configured + * \param broker_ids The list of broker ids that will replicate this partition + */ + void set_replica_assignment(int new_partition_index, std::vector brokers); +private: + void do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) override; + + using HandlePtr = std::unique_ptr; + + HandlePtr handle_; +}; + +/** + * \brief Sets the partition count for multiple topics + */ +using CompoundCreatePartitionsOperation = CompoundOperation; + +} // admin +} // cppkafka + +#endif // Admin API +#endif // CPPKAFKA_CREATE_PARTITIONS_OPERATION_H diff --git a/src/admin/create_partitions_operation.cpp b/src/admin/create_partitions_operation.cpp new file mode 100644 index 00000000..6884f936 --- /dev/null +++ b/src/admin/create_partitions_operation.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "admin/create_partitions_operation.h" +#include "exceptions.h" + +using std::string; +using std::vector; + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +CreatePartitionsOperation::CreatePartitionsOperation(string topic, size_t partitions) +: handle_(nullptr, nullptr) { + char error_buffer[512] = { 0 }; + auto rdkafka_handle = rd_kafka_NewPartitions_new( + topic.data(), + partitions, + error_buffer, + sizeof(error_buffer) + ); + if (!rdkafka_handle) { + throw AdminOperationException(error_buffer); + } + handle_ = HandlePtr(rdkafka_handle, rd_kafka_NewPartitions_destroy); +} + +void CreatePartitionsOperation::set_replica_assignment(int new_partition_index, + vector brokers) { + char error_buffer[512] = { 0 }; + const Error result = rd_kafka_NewPartitions_set_replica_assignment( + handle_.get(), + new_partition_index, + brokers.data(), + brokers.size(), + error_buffer, + sizeof(error_buffer) + ); + if (result) { + throw AdminOperationException(error_buffer); + } +} + +void CreatePartitionsOperation::do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) { + run_operation(kafka_handle, queue, options, handle_.get(), &rd_kafka_CreatePartitions); +} + +} // admin +} // cppkafka + +#endif // Admin API From 31a5c8b60a74ce26c8c5105950a839d65514ced9 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 17 Nov 2018 17:24:15 -0800 Subject: [PATCH 13/15] Rename compound operations as Compound* --- include/cppkafka/admin/create_topic_operation.h | 4 ++-- include/cppkafka/admin/delete_topic_operation.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/cppkafka/admin/create_topic_operation.h b/include/cppkafka/admin/create_topic_operation.h index 2975e1ad..0b3e8718 100644 --- a/include/cppkafka/admin/create_topic_operation.h +++ b/include/cppkafka/admin/create_topic_operation.h @@ -121,8 +121,8 @@ class CreateTopicOperation : public Operation { /** * \brief Creates several topics */ -using CreateTopicsOperation = CompoundOperation; +using CompoundCreateTopicOperation = CompoundOperation; } // admin } // cppkafka diff --git a/include/cppkafka/admin/delete_topic_operation.h b/include/cppkafka/admin/delete_topic_operation.h index d925f6ce..81cb0ba7 100644 --- a/include/cppkafka/admin/delete_topic_operation.h +++ b/include/cppkafka/admin/delete_topic_operation.h @@ -74,8 +74,8 @@ class DeleteTopicOperation : public Operation { /** * \brief Deletes several topics */ -using DeleteTopicsOperation = CompoundOperation; +using CompoundDeleteTopicOperation = CompoundOperation; } // admin } // cppkafka From 0d3db05bd72e4a52bc34f399d7e35df90503614d Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sun, 18 Nov 2018 10:42:08 -0800 Subject: [PATCH 14/15] Add describe configs operation --- .../admin/describe_configs_operation.h | 103 ++++++++++++++++++ src/admin/describe_configs_operation.cpp | 66 +++++++++++ 2 files changed, 169 insertions(+) create mode 100644 include/cppkafka/admin/describe_configs_operation.h create mode 100644 src/admin/describe_configs_operation.cpp diff --git a/include/cppkafka/admin/describe_configs_operation.h b/include/cppkafka/admin/describe_configs_operation.h new file mode 100644 index 00000000..c2be7a8a --- /dev/null +++ b/include/cppkafka/admin/describe_configs_operation.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef CPPKAFKA_ADMIN_DESCRIBE_CONFIGS_OPERATION_H +#define CPPKAFKA_ADMIN_DESCRIBE_CONFIGS_OPERATION_H + +#include +#include "compound_operation.h" +#include "operation.h" + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +/** + * \brief Describes a set of cluster, broker or topic configurations + */ +class DescribeConfigsOperation : public Operation { +public: + /** + * The rdkafka type used for this operation's handle + */ + using HandleType = rd_kafka_ConfigResource_t; + + /** + * \brief Represents the resource type being queried + */ + enum class ResourceType { + Unknown = RD_KAFKA_RESOURCE_UNKNOWN, + Any = RD_KAFKA_RESOURCE_ANY, + Topic = RD_KAFKA_RESOURCE_TOPIC, + Group = RD_KAFKA_RESOURCE_GROUP, + Broker = RD_KAFKA_RESOURCE_BROKER + }; + + /** + * \brief Constructs a describe configs operation + * + * The name depends on the type being used. For example, when querying for a topic + * resource type, this will be the name of the topic to be queried. + * + * See KIP-133 for more information. + * + * \param type The resource type to be queried + * \param name The name of the resource to be queried + */ + DescribeConfigsOperation(ResourceType type, const std::string& name); + + /** + * \brief Adds a config key to be queried + * + * \param key The config key to be queried + */ + void add_config(const std::string& key); +private: + using HandlePtr = std::unique_ptr; + + void do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) override; + + HandlePtr handle_; +}; + +/** + * \brief Describes the config for multiple resources + */ +using CompoundDescribeConfigsOperation = CompoundOperation; + +} // admin +} // cppkafka + +#endif // Admin API +#endif // CPPKAFKA_ADMIN_DESCRIBE_CONFIGS_OPERATION_H diff --git a/src/admin/describe_configs_operation.cpp b/src/admin/describe_configs_operation.cpp new file mode 100644 index 00000000..753da726 --- /dev/null +++ b/src/admin/describe_configs_operation.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "admin/describe_configs_operation.h" +#include "exceptions.h" + +using std::string; + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +DescribeConfigsOperation::DescribeConfigsOperation(ResourceType type, const string& name) +: handle_(nullptr, nullptr) { + auto ptr = rd_kafka_ConfigResource_new(static_cast(type), + name.data()); + if (!ptr) { + throw AdminOperationException("Failed to create describe config handle"); + } + handle_ = HandlePtr(ptr, rd_kafka_ConfigResource_destroy); +} + +void DescribeConfigsOperation::add_config(const string& key) { + const Error result = rd_kafka_ConfigResource_set_config(handle_.get(), key.data(), nullptr); + if (!result) { + throw AdminOperationException(result.to_string()); + } +} + +void DescribeConfigsOperation::do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) { + run_operation(kafka_handle, queue, options, handle_.get(), &rd_kafka_DescribeConfigs); +} + +} // admin +} // cppkafka + +#endif // Admin API From bd9816135ca6ee9808ab86c7126dfcac812416c7 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sun, 18 Nov 2018 10:57:31 -0800 Subject: [PATCH 15/15] Add alter configs operation --- .../cppkafka/admin/alter_configs_operation.h | 115 ++++++++++++++++++ src/admin/alter_configs_operation.cpp | 77 ++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 include/cppkafka/admin/alter_configs_operation.h create mode 100644 src/admin/alter_configs_operation.cpp diff --git a/include/cppkafka/admin/alter_configs_operation.h b/include/cppkafka/admin/alter_configs_operation.h new file mode 100644 index 00000000..ca76820f --- /dev/null +++ b/include/cppkafka/admin/alter_configs_operation.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef CPPKAFKA_ADMIN_ALTER_CONFIGS_OPERATION_H +#define CPPKAFKA_ADMIN_ALTER_CONFIGS_OPERATION_H + +#include +#include "compound_operation.h" +#include "operation.h" +#include "../configuration_option.h" + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +/** + * \brief Modifies the configuration for a cluster, broker or topic + */ +class AlterConfigsOperation : public Operation { +public: + /** + * The rdkafka type used for this operation's handle + */ + using HandleType = rd_kafka_ConfigResource_t; + + /** + * \brief Represents the resource type being modified + */ + enum class ResourceType { + Unknown = RD_KAFKA_RESOURCE_UNKNOWN, + Any = RD_KAFKA_RESOURCE_ANY, + Topic = RD_KAFKA_RESOURCE_TOPIC, + Group = RD_KAFKA_RESOURCE_GROUP, + Broker = RD_KAFKA_RESOURCE_BROKER + }; + + /** + * \brief Constructs an alter configs operation + * + * The name depends on the type being used. For example, when modifying a topic + * type, this will be the name of the topic to be modifies. + * + * See KIP-133 for more information. + * + * \param type The type of the resource to be modified + * \param name The name of the resource to be modified + */ + AlterConfigsOperation(ResourceType type, const std::string& name); + + /** + * \brief Sets a config option + * + * \param config_option The configuration option to be set + */ + void set_config(const ConfigurationOption& config_option); + + /** + * \brief Removes a configuration option. + * + * Note that this implies removing it from the server side, not on list + * of configs added to this alter configs operation via AlterConfigsOperation::set_config + * + * \param key The config key to be removed + */ + void remove_config(const std::string& key); +private: + using HandlePtr = std::unique_ptr; + + void do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) override; + + HandlePtr handle_; +}; + +/** + * \brief Alters the config for multiple resources + */ +using CompoundAlterConfigsOperation = CompoundOperation; + +} // admin +} // cppkafka + +#endif // Admin API + +#endif // CPPKAFKA_ADMIN_ALTER_CONFIGS_OPERATION_H diff --git a/src/admin/alter_configs_operation.cpp b/src/admin/alter_configs_operation.cpp new file mode 100644 index 00000000..af48690e --- /dev/null +++ b/src/admin/alter_configs_operation.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018, Matias Fontanini + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "admin/alter_configs_operation.h" +#include "exceptions.h" + +using std::string; + +#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION + +namespace cppkafka { +namespace admin { + +AlterConfigsOperation::AlterConfigsOperation(ResourceType type, const string& name) +: handle_(nullptr, nullptr) { + auto ptr = rd_kafka_ConfigResource_new(static_cast(type), + name.data()); + if (!ptr) { + throw AdminOperationException("Failed to create alter config handle"); + } + handle_ = HandlePtr(ptr, rd_kafka_ConfigResource_destroy); +} + +void AlterConfigsOperation::set_config(const ConfigurationOption& config_option) { + const string& key = config_option.get_key(); + const string& value = config_option.get_value(); + const Error result = rd_kafka_ConfigResource_set_config(handle_.get(), + key.data(), + value.data()); + if (!result) { + throw AdminOperationException(result.to_string()); + } +} + +void AlterConfigsOperation::remove_config(const string& key) { + const Error result = rd_kafka_ConfigResource_set_config(handle_.get(), key.data(), nullptr); + if (!result) { + throw AdminOperationException(result.to_string()); + } +} + +void AlterConfigsOperation::do_execute(KafkaHandleBase& kafka_handle, + Queue& queue, + const OperationOptions* options) { + run_operation(kafka_handle, queue, options, handle_.get(), &rd_kafka_AlterConfigs); +} + +} // admin +} // cppkafka + +#endif // Admin API pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy