From 7aedcdc30fa2ae9dd383a001c00a9a229be1ac28 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 7 Oct 2017 14:02:40 -0700 Subject: [PATCH 01/20] Add configuration mocking code --- CMakeLists.txt | 3 +- include/cppkafka/clonable_ptr.h | 7 + mocking/CMakeLists.txt | 1 + mocking/include/cppkafka/mocking/api.h | 74 ++++++++ .../cppkafka/mocking/configuration_mock.h | 100 +++++++++++ .../include/cppkafka/mocking/handle_wrapper.h | 32 ++++ mocking/src/CMakeLists.txt | 13 ++ mocking/src/api.cpp | 160 ++++++++++++++++++ mocking/src/configuration_mock.cpp | 134 +++++++++++++++ 9 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 mocking/CMakeLists.txt create mode 100644 mocking/include/cppkafka/mocking/api.h create mode 100644 mocking/include/cppkafka/mocking/configuration_mock.h create mode 100644 mocking/include/cppkafka/mocking/handle_wrapper.h create mode 100644 mocking/src/CMakeLists.txt create mode 100644 mocking/src/api.cpp create mode 100644 mocking/src/configuration_mock.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 249ac4fd..b47a4aaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ find_package(Boost REQUIRED) find_package(RdKafka REQUIRED) add_subdirectory(src) +add_subdirectory(mocking) add_subdirectory(include) add_subdirectory(examples) @@ -95,4 +96,4 @@ configure_file( # Add uninstall target add_custom_target(uninstall - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) \ No newline at end of file + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) diff --git a/include/cppkafka/clonable_ptr.h b/include/cppkafka/clonable_ptr.h index 842e3088..505f0985 100644 --- a/include/cppkafka/clonable_ptr.h +++ b/include/cppkafka/clonable_ptr.h @@ -84,6 +84,13 @@ class ClonablePtr { T* get() const { return handle_.get(); } + + /** + * Resets the internal pointer + */ + void reset(T* ptr) { + handle_.reset(ptr); + } private: std::unique_ptr handle_; Cloner cloner_; diff --git a/mocking/CMakeLists.txt b/mocking/CMakeLists.txt new file mode 100644 index 00000000..febd4f0a --- /dev/null +++ b/mocking/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(src) diff --git a/mocking/include/cppkafka/mocking/api.h b/mocking/include/cppkafka/mocking/api.h new file mode 100644 index 00000000..8c18d73c --- /dev/null +++ b/mocking/include/cppkafka/mocking/api.h @@ -0,0 +1,74 @@ +#ifndef CPPKAFKA_MOCKING_API_H +#define CPPKAFKA_MOCKING_API_H + +#include +#include +#include +#include + +struct rd_kafka_conf_s : cppkafka::mocking::HandleWrapper { + using cppkafka::mocking::HandleWrapper::HandleWrapper; +}; + +struct rd_kafka_topic_conf_s : rd_kafka_conf_s { + using rd_kafka_conf_s::rd_kafka_conf_s; +}; + +// rd_kafka_conf_t +CPPKAFKA_API rd_kafka_conf_t* rd_kafka_conf_new(); +CPPKAFKA_API void rd_kafka_conf_destroy(rd_kafka_conf_t* conf); +CPPKAFKA_API rd_kafka_conf_t* rd_kafka_conf_dup(const rd_kafka_conf_t* conf); +CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_conf_set(rd_kafka_conf_t* conf, + const char* name, const char* value, + char* errstr, size_t errstr_size); +CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_conf_get(const rd_kafka_conf_t* conf, + const char* name, char* dest, + size_t* dest_size); +CPPKAFKA_API +void rd_kafka_conf_set_dr_msg_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::DeliveryReportCallback* cb); +CPPKAFKA_API +void rd_kafka_conf_set_rebalance_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::RebalanceCallback* cb); +CPPKAFKA_API +void rd_kafka_conf_set_offset_commit_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::OffsetCommitCallback* cb); +CPPKAFKA_API +void rd_kafka_conf_set_error_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::ErrorCallback* cb); +CPPKAFKA_API +void rd_kafka_conf_set_throttle_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::ThrottleCallback* cb); +CPPKAFKA_API +void rd_kafka_conf_set_log_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::LogCallback* cb); +CPPKAFKA_API +void rd_kafka_conf_set_stats_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::StatsCallback* cb); +CPPKAFKA_API +void rd_kafka_conf_set_socket_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::SocketCallback* cb); +CPPKAFKA_API +void rd_kafka_topic_conf_set_partitioner_cb(rd_kafka_conf_t* conf, + cppkafka::mocking::ConfigurationMock::PartitionerCallback* cb); +CPPKAFKA_API +void rd_kafka_conf_set_default_topic_conf(rd_kafka_conf_t* conf, + rd_kafka_topic_conf_t* tconf); +CPPKAFKA_API void rd_kafka_conf_set_opaque(rd_kafka_conf_t* conf, void* opaque); +CPPKAFKA_API const char** rd_kafka_conf_dump(rd_kafka_conf_t* conf, size_t* cntp); +CPPKAFKA_API void rd_kafka_conf_dump_free(const char** arr, size_t cnt); + +// rd_kafka_topic_conf_t +CPPKAFKA_API rd_kafka_topic_conf_t* rd_kafka_topic_conf_new(); +CPPKAFKA_API void rd_kafka_topic_conf_destroy(rd_kafka_topic_conf_t* conf); +CPPKAFKA_API rd_kafka_topic_conf_t* rd_kafka_topic_conf_dup(const rd_kafka_topic_conf_t* conf); +CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_topic_conf_set(rd_kafka_topic_conf_t* conf, + const char* name, const char* value, + char* errstr, size_t errstr_size); +CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_topic_conf_get(const rd_kafka_topic_conf_t *conf, + const char *name, char *dest, + size_t *dest_size); +CPPKAFKA_API +const char** rd_kafka_topic_conf_dump(rd_kafka_topic_conf_t* conf, size_t* cntp); + +#endif // CPPKAFKA_MOCKING_API_H diff --git a/mocking/include/cppkafka/mocking/configuration_mock.h b/mocking/include/cppkafka/mocking/configuration_mock.h new file mode 100644 index 00000000..d661541b --- /dev/null +++ b/mocking/include/cppkafka/mocking/configuration_mock.h @@ -0,0 +1,100 @@ +#ifndef CPPKAFKA_MOCKING_CONFIGURATION_MOCK_H +#define CPPKAFKA_MOCKING_CONFIGURATION_MOCK_H + +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class ConfigurationMock { +public: + using DeliveryReportCallback = void(rd_kafka_t*, const rd_kafka_message_t*, void *); + using RebalanceCallback = void(rd_kafka_t*, rd_kafka_resp_err_t, + rd_kafka_topic_partition_list_t*, void*); + using OffsetCommitCallback = void(rd_kafka_t*, rd_kafka_resp_err_t, + rd_kafka_topic_partition_list_t*, void*); + using ErrorCallback = void(rd_kafka_t*, int, const char*, void*); + using ThrottleCallback = void(rd_kafka_t*, const char*, int32_t, int, void*); + using LogCallback = void(const rd_kafka_t*, int, const char*, const char*); + using StatsCallback = int(rd_kafka_t*, char*, size_t, void*); + using SocketCallback = int(int, int, int, void*); + using PartitionerCallback = int32_t(const rd_kafka_topic_t*, const void*, + size_t, int32_t, void*, void*); + + ConfigurationMock(); + + void set(std::string key, std::string value); + size_t has_key(const std::string& key) const; + std::string get(const std::string& key) const; + + void set_delivery_report_callback(DeliveryReportCallback* callback); + void set_rebalance_callback(RebalanceCallback* callback); + void set_offset_commit_callback(OffsetCommitCallback* callback); + void set_error_callback(ErrorCallback* callback); + void set_throttle_callback(ThrottleCallback* callback); + void set_log_callback(LogCallback* callback); + void set_stats_callback(StatsCallback* callback); + void set_socket_callback(SocketCallback* callback); + void set_partitioner_callback(PartitionerCallback* callback); + void set_default_topic_configuration(const ConfigurationMock& conf); + void set_opaque(void* ptr); + + DeliveryReportCallback* get_delivery_report_callback() const; + RebalanceCallback* get_rebalance_callback() const; + OffsetCommitCallback* get_offset_commit_callback() const; + ErrorCallback* get_error_callback() const; + ThrottleCallback* get_throttle_callback() const; + LogCallback* get_log_callback() const; + StatsCallback* get_stats_callback() const; + SocketCallback* get_socket_callback() const; + PartitionerCallback* get_partitioner_callback() const; + const ConfigurationMock* get_default_topic_configuration() const; + void* get_opaque() const; + std::unordered_map get_options() const; +private: + enum class Callback { + DeliveryReport, + Rebalance, + OffsetCommit, + Error, + Throttle, + Log, + Stats, + Socket, + Partitioner + }; + + struct Hasher { + size_t operator()(Callback c) const { + return static_cast(c); + } + }; + + struct Cloner { + ConfigurationMock* operator()(const ConfigurationMock* ptr) const { + return new ConfigurationMock(*ptr); + } + }; + + using TopicConfigirationPtr = ClonablePtr, + Cloner>; + + template + void set_callback(Callback type, Functor* ptr); + template + Functor* get_callback(Callback type) const; + + std::unordered_map options_; + std::unordered_map callbacks_; + TopicConfigirationPtr default_topic_configuration_; + void* opaque_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_CONFIGURATION_MOCK_H diff --git a/mocking/include/cppkafka/mocking/handle_wrapper.h b/mocking/include/cppkafka/mocking/handle_wrapper.h new file mode 100644 index 00000000..09dae327 --- /dev/null +++ b/mocking/include/cppkafka/mocking/handle_wrapper.h @@ -0,0 +1,32 @@ +#ifndef CPPKAFKA_MOCKING_HANDLE_WRAPPER_H +#define CPPKAFKA_MOCKING_HANDLE_WRAPPER_H + +#include + +namespace cppkafka { +namespace mocking { + +template +class HandleWrapper { +public: + template + HandleWrapper(Args&&... args) + : handle_(std::forward(args)...) { + + } + + T& get_handle() { + return handle_; + } + + const T& get_handle() const { + return handle_; + } +private: + T handle_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_HANDLE_WRAPPER_H diff --git a/mocking/src/CMakeLists.txt b/mocking/src/CMakeLists.txt new file mode 100644 index 00000000..8ef17c52 --- /dev/null +++ b/mocking/src/CMakeLists.txt @@ -0,0 +1,13 @@ +set(SOURCES + configuration_mock.cpp + api.cpp +) + +include_directories(${PROJECT_SOURCE_DIR}/mocking/include/ + ${PROJECT_SOURCE_DIR}/include/) +include_directories(SYSTEM ${Boost_INCLUDE_DIRS} ${RDKAFKA_INCLUDE_DIR}) + +add_library(cppkafka_mock EXCLUDE_FROM_ALL ${CPPKAFKA_LIBRARY_TYPE} ${SOURCES}) +set_target_properties(cppkafka_mock PROPERTIES VERSION ${CPPKAFKA_VERSION} + SOVERSION ${CPPKAFKA_VERSION}) +target_link_libraries(cppkafka_mock ${RDKAFKA_LIBRARY}) diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp new file mode 100644 index 00000000..1d414746 --- /dev/null +++ b/mocking/src/api.cpp @@ -0,0 +1,160 @@ +#include +#include +#include + +using std::string; +using std::copy; + +using namespace cppkafka::mocking; + +// rd_kafka_conf_t + +rd_kafka_conf_t* rd_kafka_conf_new() { + return new rd_kafka_conf_t(); +} + +void rd_kafka_conf_destroy(rd_kafka_conf_t* conf) { + delete conf; +} + +rd_kafka_conf_t* rd_kafka_conf_dup(const rd_kafka_conf_t* conf) { + return new rd_kafka_conf_t(conf->get_handle()); +} + +rd_kafka_conf_res_t rd_kafka_conf_set(rd_kafka_conf_t* conf, + const char* name, + const char* value, + char*, size_t) { + conf->get_handle().set(name, value); + return RD_KAFKA_CONF_OK; +} + +rd_kafka_conf_res_t rd_kafka_conf_get(const rd_kafka_conf_t* conf, + const char* name_raw, + char* dest, size_t* dest_size) { + const string name = name_raw; + if (!conf->get_handle().has_key(name)) { + return RD_KAFKA_CONF_UNKNOWN; + } + if (dest == nullptr) { + *dest_size = conf->get_handle().get(name).size(); + } + else { + const string value = conf->get_handle().get(name); + if (value.size() > *dest_size + 1) { + return RD_KAFKA_CONF_INVALID; + } + else { + copy(value.begin(), value.end(), dest); + } + } + return RD_KAFKA_CONF_OK; +} + +void rd_kafka_conf_set_dr_msg_cb(rd_kafka_conf_t* conf, + ConfigurationMock::DeliveryReportCallback* cb) { + conf->get_handle().set_delivery_report_callback(cb); +} + +void rd_kafka_conf_set_rebalance_cb(rd_kafka_conf_t* conf, + ConfigurationMock::RebalanceCallback* cb) { + conf->get_handle().set_rebalance_callback(cb); +} + +void rd_kafka_conf_set_offset_commit_cb(rd_kafka_conf_t* conf, + ConfigurationMock::OffsetCommitCallback* cb) { + conf->get_handle().set_offset_commit_callback(cb); +} + +void rd_kafka_conf_set_error_cb(rd_kafka_conf_t* conf, + ConfigurationMock::ErrorCallback* cb) { + conf->get_handle().set_error_callback(cb); +} + +void rd_kafka_conf_set_throttle_cb(rd_kafka_conf_t* conf, + ConfigurationMock::ThrottleCallback* cb) { + conf->get_handle().set_throttle_callback(cb); +} + +void rd_kafka_conf_set_log_cb(rd_kafka_conf_t* conf, + ConfigurationMock::LogCallback* cb) { + conf->get_handle().set_log_callback(cb); +} + +void rd_kafka_conf_set_stats_cb(rd_kafka_conf_t* conf, + ConfigurationMock::StatsCallback* cb) { + conf->get_handle().set_stats_callback(cb); +} + +void rd_kafka_conf_set_socket_cb(rd_kafka_conf_t* conf, + ConfigurationMock::SocketCallback* cb) { + conf->get_handle().set_socket_callback(cb); +} + +void rd_kafka_topic_conf_set_partitioner_cb(rd_kafka_conf_t* conf, + ConfigurationMock::PartitionerCallback* cb) { + conf->get_handle().set_partitioner_callback(cb); +} + +void rd_kafka_conf_set_default_topic_conf(rd_kafka_conf_t* conf, + rd_kafka_topic_conf_t* tconf) { + conf->get_handle().set_default_topic_configuration(tconf->get_handle()); +} + +void rd_kafka_conf_set_opaque(rd_kafka_conf_t* conf, void* opaque) { + conf->get_handle().set_opaque(opaque); +} + +const char** rd_kafka_conf_dump(rd_kafka_conf_t* conf, size_t* cntp) { + const auto options = conf->get_handle().get_options(); + *cntp = options.size() * 2; + // Allocate enough for all (key, value) pairs + char** output = new char*[*cntp]; + size_t i = 0; + const auto set_value = [&](const string& value) { + output[i] = new char[value.size() + 1]; + copy(value.begin(), value.end(), output[i]); + ++i; + }; + for (const auto& option : options) { + set_value(option.first); + set_value(option.second); + } + return const_cast(output); +} + +void rd_kafka_conf_dump_free(const char** arr, size_t cnt) { + for (size_t i = 0; i < cnt; ++i) { + delete[] arr[i]; + } + delete[] arr; +} + +// rd_kafka_topic_conf_t +rd_kafka_topic_conf_t* rd_kafka_topic_conf_new() { + return new rd_kafka_topic_conf_t(); +} + +void rd_kafka_topic_conf_destroy(rd_kafka_topic_conf_t* conf) { + delete conf; +} + +rd_kafka_topic_conf_t* rd_kafka_topic_conf_dup(const rd_kafka_topic_conf_t* conf) { + return new rd_kafka_topic_conf_t(conf->get_handle()); +} + +rd_kafka_conf_res_t rd_kafka_topic_conf_set(rd_kafka_topic_conf_t* conf, + const char* name, + const char* value, + char* errstr, size_t errstr_size) { + return rd_kafka_conf_set(conf, name, value, errstr, errstr_size); +} + +rd_kafka_conf_res_t rd_kafka_topic_conf_get(const rd_kafka_topic_conf_t *conf, + const char *name, char *dest, size_t *dest_size) { + return rd_kafka_conf_get(conf, name, dest, dest_size); +} + +const char** rd_kafka_topic_conf_dump(rd_kafka_topic_conf_t* conf, size_t* cntp) { + return rd_kafka_conf_dump(conf, cntp); +} diff --git a/mocking/src/configuration_mock.cpp b/mocking/src/configuration_mock.cpp new file mode 100644 index 00000000..8f0ab7dd --- /dev/null +++ b/mocking/src/configuration_mock.cpp @@ -0,0 +1,134 @@ +#include + +using std::string; +using std::default_delete; +using std::unordered_map; + +namespace cppkafka { +namespace mocking { + +ConfigurationMock::ConfigurationMock() +: default_topic_configuration_(nullptr, default_delete{}, Cloner{}) { + +} + +void ConfigurationMock::set(string key, string value) { + options_[move(key)] = move(value); +} + +size_t ConfigurationMock::has_key(const string& key) const { + return options_.count(key); +} + +string ConfigurationMock::get(const string& key) const { + return options_.at(key); +} + +void ConfigurationMock::set_delivery_report_callback(DeliveryReportCallback* callback) { + set_callback(Callback::DeliveryReport, callback); +} + +void ConfigurationMock::set_rebalance_callback(RebalanceCallback* callback) { + set_callback(Callback::Rebalance, callback); +} + +void ConfigurationMock::set_offset_commit_callback(OffsetCommitCallback* callback) { + set_callback(Callback::OffsetCommit, callback); +} + +void ConfigurationMock::set_error_callback(ErrorCallback* callback) { + set_callback(Callback::Error, callback); +} + +void ConfigurationMock::set_throttle_callback(ThrottleCallback* callback) { + set_callback(Callback::Throttle, callback); +} + +void ConfigurationMock::set_log_callback(LogCallback* callback) { + set_callback(Callback::Log, callback); +} + +void ConfigurationMock::set_stats_callback(StatsCallback* callback) { + set_callback(Callback::Stats, callback); +} + +void ConfigurationMock::set_socket_callback(SocketCallback* callback) { + set_callback(Callback::Socket, callback); +} + +void ConfigurationMock::set_partitioner_callback(PartitionerCallback* callback) { + set_callback(Callback::Partitioner, callback); +} + +void ConfigurationMock::set_default_topic_configuration(const ConfigurationMock& conf) { + default_topic_configuration_.reset(Cloner{}(&conf)); +} + +void ConfigurationMock::set_opaque(void* opaque) { + opaque_ = opaque; +} + +ConfigurationMock::DeliveryReportCallback* +ConfigurationMock::get_delivery_report_callback() const { + return get_callback(Callback::DeliveryReport); +} + +ConfigurationMock::RebalanceCallback* +ConfigurationMock::get_rebalance_callback() const { + return get_callback(Callback::Rebalance); +} + +ConfigurationMock::OffsetCommitCallback* +ConfigurationMock::get_offset_commit_callback() const { + return get_callback(Callback::OffsetCommit); +} + +ConfigurationMock::ErrorCallback* ConfigurationMock::get_error_callback() const { + return get_callback(Callback::Error); +} + +ConfigurationMock::ThrottleCallback* ConfigurationMock::get_throttle_callback() const { + return get_callback(Callback::Throttle); +} + +ConfigurationMock::LogCallback* ConfigurationMock::get_log_callback() const { + return get_callback(Callback::Log); +} + +ConfigurationMock::StatsCallback* ConfigurationMock::get_stats_callback() const { + return get_callback(Callback::Stats); +} + +ConfigurationMock::SocketCallback* ConfigurationMock::get_socket_callback() const { + return get_callback(Callback::Socket); +} + +ConfigurationMock::PartitionerCallback* ConfigurationMock::get_partitioner_callback() const { + return get_callback(Callback::Partitioner); +} + +const ConfigurationMock* ConfigurationMock::get_default_topic_configuration() const { + return default_topic_configuration_.get(); +} + +void* ConfigurationMock::get_opaque() const { + return opaque_; +} + +unordered_map ConfigurationMock::get_options() const { + return options_; +} + +template +void ConfigurationMock::set_callback(Callback type, Functor* ptr) { + callbacks_[type] = reinterpret_cast(ptr); +} + +template +Functor* ConfigurationMock::get_callback(Callback type) const { + auto iter = callbacks_.find(type); + return iter == callbacks_.end() ? nullptr : reinterpret_cast(iter->second); +} + +} // mocking +} // cppkafka From c14bdf508ed3d822ab7713b397529ad287e6faba Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 7 Oct 2017 14:24:01 -0700 Subject: [PATCH 02/20] Add mocking code for topic partition list --- mocking/include/cppkafka/mocking/api.h | 15 ++++++++-- mocking/src/api.cpp | 41 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/mocking/include/cppkafka/mocking/api.h b/mocking/include/cppkafka/mocking/api.h index 8c18d73c..8f52fa6c 100644 --- a/mocking/include/cppkafka/mocking/api.h +++ b/mocking/include/cppkafka/mocking/api.h @@ -66,9 +66,20 @@ CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_topic_conf_set(rd_kafka_topic_conf_t* const char* name, const char* value, char* errstr, size_t errstr_size); CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_topic_conf_get(const rd_kafka_topic_conf_t *conf, - const char *name, char *dest, - size_t *dest_size); + const char* name, char* dest, + size_t* dest_size); CPPKAFKA_API const char** rd_kafka_topic_conf_dump(rd_kafka_topic_conf_t* conf, size_t* cntp); +// rd_kafka_topic_* + +CPPKAFKA_API void rd_kafka_topic_partition_destroy(rd_kafka_topic_partition_t* toppar); +CPPKAFKA_API rd_kafka_topic_partition_list_t* rd_kafka_topic_partition_list_new(int size); +CPPKAFKA_API +void rd_kafka_topic_partition_list_destroy(rd_kafka_topic_partition_list_t* toppar_list); +CPPKAFKA_API +rd_kafka_topic_partition_t* +rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list, + const char* topic, int32_t partition); + #endif // CPPKAFKA_MOCKING_API_H diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp index 1d414746..34e0d81a 100644 --- a/mocking/src/api.cpp +++ b/mocking/src/api.cpp @@ -1,9 +1,11 @@ #include #include +#include #include using std::string; using std::copy; +using std::strlen; using namespace cppkafka::mocking; @@ -158,3 +160,42 @@ rd_kafka_conf_res_t rd_kafka_topic_conf_get(const rd_kafka_topic_conf_t *conf, const char** rd_kafka_topic_conf_dump(rd_kafka_topic_conf_t* conf, size_t* cntp) { return rd_kafka_conf_dump(conf, cntp); } + +// rd_kafka_topic_* + +void rd_kafka_topic_partition_destroy (rd_kafka_topic_partition_t* toppar) { + delete toppar->topic; + delete toppar; +} + +rd_kafka_topic_partition_list_t* rd_kafka_topic_partition_list_new(int size) { + rd_kafka_topic_partition_list_t* output = new rd_kafka_topic_partition_list_t{}; + output->size = size; + output->elems = new rd_kafka_topic_partition_t[size]; + for (int i = 0; i < size; ++i) { + output->elems[i] = {}; + } + return output; +} + +void rd_kafka_topic_partition_list_destroy(rd_kafka_topic_partition_list_t* toppar_list) { + for (int i = 0; i < toppar_list->cnt; ++i) { + delete toppar_list->elems[i].topic; + } + delete[] toppar_list->elems; + delete toppar_list; +} + +rd_kafka_topic_partition_t* +rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list, + const char* topic, int32_t partition) { + if (toppar_list->cnt >= toppar_list->size) { + return nullptr; + } + rd_kafka_topic_partition_t* output = &toppar_list->elems[toppar_list->cnt++]; + const size_t length = strlen(topic); + output->topic = new char[length + 1]; + copy(topic, topic + length, output->topic); + output->partition = partition; + return output; +} From d64ce6a721f3f61a23d1cf3eef949d2e919e9954 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sun, 8 Oct 2017 12:49:44 -0700 Subject: [PATCH 03/20] Add initial kafka cluster mocking code --- .../include/cppkafka/mocking/kafka_cluster.h | 30 ++++++++++++++ .../cppkafka/mocking/kafka_cluster_registry.h | 26 ++++++++++++ .../include/cppkafka/mocking/kafka_message.h | 32 +++++++++++++++ .../cppkafka/mocking/kafka_partition.h | 25 +++++++++++ .../include/cppkafka/mocking/kafka_topic.h | 24 +++++++++++ mocking/src/CMakeLists.txt | 5 +++ mocking/src/kafka_cluster.cpp | 41 +++++++++++++++++++ mocking/src/kafka_cluster_registry.cpp | 33 +++++++++++++++ mocking/src/kafka_message.cpp | 30 ++++++++++++++ mocking/src/kafka_partition.cpp | 32 +++++++++++++++ mocking/src/kafka_topic.cpp | 24 +++++++++++ 11 files changed, 302 insertions(+) create mode 100644 mocking/include/cppkafka/mocking/kafka_cluster.h create mode 100644 mocking/include/cppkafka/mocking/kafka_cluster_registry.h create mode 100644 mocking/include/cppkafka/mocking/kafka_message.h create mode 100644 mocking/include/cppkafka/mocking/kafka_partition.h create mode 100644 mocking/include/cppkafka/mocking/kafka_topic.h create mode 100644 mocking/src/kafka_cluster.cpp create mode 100644 mocking/src/kafka_cluster_registry.cpp create mode 100644 mocking/src/kafka_message.cpp create mode 100644 mocking/src/kafka_partition.cpp create mode 100644 mocking/src/kafka_topic.cpp diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h new file mode 100644 index 00000000..982ee928 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -0,0 +1,30 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_CLUSTER_H +#define CPPKAFKA_MOCKING_KAFKA_CLUSTER_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaCluster { +public: + KafkaCluster(std::string url); + KafkaCluster(const KafkaCluster&) = delete; + KafkaCluster& operator=(const KafkaCluster&) = delete; + ~KafkaCluster(); + + const std::string& get_url() const; + + void add_topic(const std::string& name, unsigned partitions); + void produce(const std::string& topic, unsigned partition, KafkaMessage message); +private: + const std::string url_; + std::unordered_map topics_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_CLUSTER_H diff --git a/mocking/include/cppkafka/mocking/kafka_cluster_registry.h b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h new file mode 100644 index 00000000..255e41a0 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h @@ -0,0 +1,26 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_CLUSTER_REGISTRY_H +#define CPPKAFKA_MOCKING_KAFKA_CLUSTER_REGISTRY_H + +#include +#include + +namespace cppkafka { +namespace mocking { +namespace detail { + +class KafkaClusterRegistry { +public: + static KafkaClusterRegistry& instance(); + + void add_cluster(KafkaCluster* cluster); + void remove_cluster(KafkaCluster* cluster); +private: + std::unordered_map clusters_; + mutable std::mutex clusters_mutex_; +}; + +} // detail +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_CLUSTER_REGISTRY_H diff --git a/mocking/include/cppkafka/mocking/kafka_message.h b/mocking/include/cppkafka/mocking/kafka_message.h new file mode 100644 index 00000000..d90e4b96 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_message.h @@ -0,0 +1,32 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_MESSAGE_H +#define CPPKAFKA_MOCKING_KAFKA_MESSAGE_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaMessage { +public: + using Buffer = std::vector; + + KafkaMessage(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, + int64_t timestamp); + + const Buffer& get_key() const; + const Buffer& get_payload() const; + rd_kafka_timestamp_type_t get_timestamp_type() const; + int64_t get_timestamp() const; +private: + const Buffer key_; + const Buffer payload_; + rd_kafka_timestamp_type_t timestamp_type_; + int64_t timestamp_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_MESSAGE_H diff --git a/mocking/include/cppkafka/mocking/kafka_partition.h b/mocking/include/cppkafka/mocking/kafka_partition.h new file mode 100644 index 00000000..7fd90dea --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_partition.h @@ -0,0 +1,25 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_PARTITION_H +#define CPPKAFKA_MOCKING_KAFKA_PARTITION_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaPartition { +public: + void add_message(KafkaMessage message); + const KafkaMessage& get_message(uint64_t offset) const; + size_t get_message_count() const; +private: + uint64_t base_offset_{0}; + std::deque messages_; + mutable std::mutex messages_mutex_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_PARTITION_H diff --git a/mocking/include/cppkafka/mocking/kafka_topic.h b/mocking/include/cppkafka/mocking/kafka_topic.h new file mode 100644 index 00000000..98849303 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_topic.h @@ -0,0 +1,24 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_TOPIC_H +#define CPPKAFKA_MOCKING_KAFKA_TOPIC_H + +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaTopic { +public: + KafkaTopic(std::string name, unsigned partition_count); + void add_message(unsigned partition, KafkaMessage message); +private: + const std::string name_; + std::vector partitions_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_TOPIC_H diff --git a/mocking/src/CMakeLists.txt b/mocking/src/CMakeLists.txt index 8ef17c52..909a8edb 100644 --- a/mocking/src/CMakeLists.txt +++ b/mocking/src/CMakeLists.txt @@ -1,5 +1,10 @@ set(SOURCES configuration_mock.cpp + kafka_topic.cpp + kafka_partition.cpp + kafka_message.cpp + kafka_cluster.cpp + kafka_cluster_registry.cpp api.cpp ) diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp new file mode 100644 index 00000000..621265c6 --- /dev/null +++ b/mocking/src/kafka_cluster.cpp @@ -0,0 +1,41 @@ +#include +#include +#include + +using std::string; +using std::invalid_argument; +using std::piecewise_construct; +using std::forward_as_tuple; +using std::move; + +namespace cppkafka { +namespace mocking { + +KafkaCluster::KafkaCluster(string url) +: url_(move(url)) { + detail::KafkaClusterRegistry::instance().add_cluster(this); +} + +KafkaCluster::~KafkaCluster() { + detail::KafkaClusterRegistry::instance().remove_cluster(this); +} + +const string& KafkaCluster::get_url() const { + return url_; +} + +void KafkaCluster::add_topic(const string& name, unsigned partitions) { + topics_.emplace(piecewise_construct, forward_as_tuple(name), + forward_as_tuple(name, partitions)); +} + +void KafkaCluster::produce(const string& topic, unsigned partition, KafkaMessage message) { + auto iter = topics_.find(topic); + if (iter == topics_.end()) { + throw invalid_argument("topic does not exist"); + } + iter->second.add_message(partition, move(message)); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_cluster_registry.cpp b/mocking/src/kafka_cluster_registry.cpp new file mode 100644 index 00000000..315112cd --- /dev/null +++ b/mocking/src/kafka_cluster_registry.cpp @@ -0,0 +1,33 @@ +#include +#include + +using std::lock_guard; +using std::mutex; +using std::runtime_error; + +namespace cppkafka { +namespace mocking { +namespace detail { + +KafkaClusterRegistry& KafkaClusterRegistry::instance() { + static KafkaClusterRegistry registry; + return registry; +} + +void KafkaClusterRegistry::add_cluster(KafkaCluster* cluster) { + lock_guard _(clusters_mutex_); + auto iter = clusters_.find(cluster->get_url()); + if (iter != clusters_.end()) { + throw runtime_error("cluster already registered"); + } + clusters_.emplace(cluster->get_url(), cluster); +} + +void KafkaClusterRegistry::remove_cluster(KafkaCluster* cluster) { + lock_guard _(clusters_mutex_); + clusters_.erase(cluster->get_url()); +} + +} // detail +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_message.cpp b/mocking/src/kafka_message.cpp new file mode 100644 index 00000000..5b9172e9 --- /dev/null +++ b/mocking/src/kafka_message.cpp @@ -0,0 +1,30 @@ +#include + +namespace cppkafka { +namespace mocking { + +KafkaMessage::KafkaMessage(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, + int64_t timestamp) +: key_(move(key)), payload_(move(payload)), timestamp_type_(timestamp_type), + timestamp_(timestamp) { + +} + +const KafkaMessage::Buffer& KafkaMessage::get_key() const { + return key_; +} + +const KafkaMessage::Buffer& KafkaMessage::get_payload() const { + return payload_; +} + +rd_kafka_timestamp_type_t KafkaMessage::get_timestamp_type() const { + return timestamp_type_; +} + +int64_t KafkaMessage::get_timestamp() const { + return timestamp_; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_partition.cpp b/mocking/src/kafka_partition.cpp new file mode 100644 index 00000000..740b1622 --- /dev/null +++ b/mocking/src/kafka_partition.cpp @@ -0,0 +1,32 @@ +#include +#include + +using std::lock_guard; +using std::mutex; +using std::out_of_range; +using std::move; + +namespace cppkafka { +namespace mocking { + +void KafkaPartition::add_message(KafkaMessage message) { + lock_guard _(messages_mutex_); + messages_.emplace_back(move(message)); +} + +const KafkaMessage& KafkaPartition::get_message(uint64_t offset) const { + const uint64_t index = offset - base_offset_; + lock_guard _(messages_mutex_); + if (messages_.size() >= index) { + throw out_of_range("invalid message index"); + } + return messages_[index]; +} + +size_t KafkaPartition::get_message_count() const { + lock_guard _(messages_mutex_); + return messages_.size(); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_topic.cpp b/mocking/src/kafka_topic.cpp new file mode 100644 index 00000000..c7fd6132 --- /dev/null +++ b/mocking/src/kafka_topic.cpp @@ -0,0 +1,24 @@ +#include +#include + +using std::string; +using std::out_of_range; +using std::move; + +namespace cppkafka { +namespace mocking { + +KafkaTopic::KafkaTopic(string name, unsigned partition_count) +: name_(move(name)), partitions_(partition_count) { + +} + +void KafkaTopic::add_message(unsigned partition, KafkaMessage message) { + if (partitions_.size() >= partition) { + throw out_of_range("invalid partition index"); + } + partitions_[partition].add_message(move(message)); +} + +} // mocking +} // cppkafka From 9c4d8b2178b9bb10da22bf4627b6f3ec734eb5b4 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Mon, 9 Oct 2017 20:42:19 -0700 Subject: [PATCH 04/20] Rename classes from Kafka* to *Mock --- .../include/cppkafka/mocking/kafka_cluster.h | 8 ++--- .../cppkafka/mocking/kafka_partition.h | 25 ---------------- .../include/cppkafka/mocking/kafka_topic.h | 24 --------------- .../{kafka_message.h => message_mock.h} | 12 ++++---- .../include/cppkafka/mocking/partition_mock.h | 25 ++++++++++++++++ mocking/include/cppkafka/mocking/topic_mock.h | 25 ++++++++++++++++ mocking/src/CMakeLists.txt | 6 ++-- mocking/src/kafka_cluster.cpp | 2 +- mocking/src/kafka_message.cpp | 30 ------------------- mocking/src/message_mock.cpp | 30 +++++++++++++++++++ ...kafka_partition.cpp => partition_mock.cpp} | 8 ++--- .../src/{kafka_topic.cpp => topic_mock.cpp} | 7 +++-- 12 files changed, 102 insertions(+), 100 deletions(-) delete mode 100644 mocking/include/cppkafka/mocking/kafka_partition.h delete mode 100644 mocking/include/cppkafka/mocking/kafka_topic.h rename mocking/include/cppkafka/mocking/{kafka_message.h => message_mock.h} (64%) create mode 100644 mocking/include/cppkafka/mocking/partition_mock.h create mode 100644 mocking/include/cppkafka/mocking/topic_mock.h delete mode 100644 mocking/src/kafka_message.cpp create mode 100644 mocking/src/message_mock.cpp rename mocking/src/{kafka_partition.cpp => partition_mock.cpp} (71%) rename mocking/src/{kafka_topic.cpp => topic_mock.cpp} (63%) diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index 982ee928..3d52e035 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -2,8 +2,8 @@ #define CPPKAFKA_MOCKING_KAFKA_CLUSTER_H #include -#include -#include +#include +#include namespace cppkafka { namespace mocking { @@ -18,10 +18,10 @@ class KafkaCluster { const std::string& get_url() const; void add_topic(const std::string& name, unsigned partitions); - void produce(const std::string& topic, unsigned partition, KafkaMessage message); + void produce(const std::string& topic, unsigned partition, MessageMock message); private: const std::string url_; - std::unordered_map topics_; + std::unordered_map topics_; }; } // mocking diff --git a/mocking/include/cppkafka/mocking/kafka_partition.h b/mocking/include/cppkafka/mocking/kafka_partition.h deleted file mode 100644 index 7fd90dea..00000000 --- a/mocking/include/cppkafka/mocking/kafka_partition.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef CPPKAFKA_MOCKING_KAFKA_PARTITION_H -#define CPPKAFKA_MOCKING_KAFKA_PARTITION_H - -#include -#include -#include - -namespace cppkafka { -namespace mocking { - -class KafkaPartition { -public: - void add_message(KafkaMessage message); - const KafkaMessage& get_message(uint64_t offset) const; - size_t get_message_count() const; -private: - uint64_t base_offset_{0}; - std::deque messages_; - mutable std::mutex messages_mutex_; -}; - -} // mocking -} // cppkafka - -#endif // CPPKAFKA_MOCKING_KAFKA_PARTITION_H diff --git a/mocking/include/cppkafka/mocking/kafka_topic.h b/mocking/include/cppkafka/mocking/kafka_topic.h deleted file mode 100644 index 98849303..00000000 --- a/mocking/include/cppkafka/mocking/kafka_topic.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef CPPKAFKA_MOCKING_KAFKA_TOPIC_H -#define CPPKAFKA_MOCKING_KAFKA_TOPIC_H - -#include -#include -#include -#include - -namespace cppkafka { -namespace mocking { - -class KafkaTopic { -public: - KafkaTopic(std::string name, unsigned partition_count); - void add_message(unsigned partition, KafkaMessage message); -private: - const std::string name_; - std::vector partitions_; -}; - -} // mocking -} // cppkafka - -#endif // CPPKAFKA_MOCKING_KAFKA_TOPIC_H diff --git a/mocking/include/cppkafka/mocking/kafka_message.h b/mocking/include/cppkafka/mocking/message_mock.h similarity index 64% rename from mocking/include/cppkafka/mocking/kafka_message.h rename to mocking/include/cppkafka/mocking/message_mock.h index d90e4b96..49fdd505 100644 --- a/mocking/include/cppkafka/mocking/kafka_message.h +++ b/mocking/include/cppkafka/mocking/message_mock.h @@ -1,5 +1,5 @@ -#ifndef CPPKAFKA_MOCKING_KAFKA_MESSAGE_H -#define CPPKAFKA_MOCKING_KAFKA_MESSAGE_H +#ifndef CPPKAFKA_MOCKING_MESSAGE_MOCK_H +#define CPPKAFKA_MOCKING_MESSAGE_MOCK_H #include #include @@ -8,12 +8,12 @@ namespace cppkafka { namespace mocking { -class KafkaMessage { +class MessageMock { public: using Buffer = std::vector; - KafkaMessage(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, - int64_t timestamp); + MessageMock(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, + int64_t timestamp); const Buffer& get_key() const; const Buffer& get_payload() const; @@ -29,4 +29,4 @@ class KafkaMessage { } // mocking } // cppkafka -#endif // CPPKAFKA_MOCKING_KAFKA_MESSAGE_H +#endif // CPPKAFKA_MOCKING_MESSAGE_MOCK_H diff --git a/mocking/include/cppkafka/mocking/partition_mock.h b/mocking/include/cppkafka/mocking/partition_mock.h new file mode 100644 index 00000000..6dc29ed0 --- /dev/null +++ b/mocking/include/cppkafka/mocking/partition_mock.h @@ -0,0 +1,25 @@ +#ifndef CPPKAFKA_MOCKING_PARTITION_MOCK_H +#define CPPKAFKA_MOCKING_PARTITION_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class PartitionMock { +public: + void add_message(MessageMock message); + const MessageMock& get_message(uint64_t offset) const; + size_t get_message_count() const; +private: + uint64_t base_offset_{0}; + std::deque messages_; + mutable std::mutex messages_mutex_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_PARTITION_MOCK_H diff --git a/mocking/include/cppkafka/mocking/topic_mock.h b/mocking/include/cppkafka/mocking/topic_mock.h new file mode 100644 index 00000000..bc260274 --- /dev/null +++ b/mocking/include/cppkafka/mocking/topic_mock.h @@ -0,0 +1,25 @@ +#ifndef CPPKAFKA_MOCKING_TOPIC_MOCK_H +#define CPPKAFKA_MOCKING_TOPIC_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class MessageMock; + +class TopicMock { +public: + TopicMock(std::string name, unsigned partition_count); + void add_message(unsigned partition, MessageMock message); +private: + const std::string name_; + std::vector partitions_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_TOPIC_MOCK_H diff --git a/mocking/src/CMakeLists.txt b/mocking/src/CMakeLists.txt index 909a8edb..7b94de2d 100644 --- a/mocking/src/CMakeLists.txt +++ b/mocking/src/CMakeLists.txt @@ -1,8 +1,8 @@ set(SOURCES configuration_mock.cpp - kafka_topic.cpp - kafka_partition.cpp - kafka_message.cpp + topic_mock.cpp + partition_mock.cpp + message_mock.cpp kafka_cluster.cpp kafka_cluster_registry.cpp api.cpp diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index 621265c6..49910deb 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -29,7 +29,7 @@ void KafkaCluster::add_topic(const string& name, unsigned partitions) { forward_as_tuple(name, partitions)); } -void KafkaCluster::produce(const string& topic, unsigned partition, KafkaMessage message) { +void KafkaCluster::produce(const string& topic, unsigned partition, MessageMock message) { auto iter = topics_.find(topic); if (iter == topics_.end()) { throw invalid_argument("topic does not exist"); diff --git a/mocking/src/kafka_message.cpp b/mocking/src/kafka_message.cpp deleted file mode 100644 index 5b9172e9..00000000 --- a/mocking/src/kafka_message.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include - -namespace cppkafka { -namespace mocking { - -KafkaMessage::KafkaMessage(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, - int64_t timestamp) -: key_(move(key)), payload_(move(payload)), timestamp_type_(timestamp_type), - timestamp_(timestamp) { - -} - -const KafkaMessage::Buffer& KafkaMessage::get_key() const { - return key_; -} - -const KafkaMessage::Buffer& KafkaMessage::get_payload() const { - return payload_; -} - -rd_kafka_timestamp_type_t KafkaMessage::get_timestamp_type() const { - return timestamp_type_; -} - -int64_t KafkaMessage::get_timestamp() const { - return timestamp_; -} - -} // mocking -} // cppkafka diff --git a/mocking/src/message_mock.cpp b/mocking/src/message_mock.cpp new file mode 100644 index 00000000..d64a65cc --- /dev/null +++ b/mocking/src/message_mock.cpp @@ -0,0 +1,30 @@ +#include + +namespace cppkafka { +namespace mocking { + +MessageMock::MessageMock(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, + int64_t timestamp) +: key_(move(key)), payload_(move(payload)), timestamp_type_(timestamp_type), + timestamp_(timestamp) { + +} + +const MessageMock::Buffer& MessageMock::get_key() const { + return key_; +} + +const MessageMock::Buffer& MessageMock::get_payload() const { + return payload_; +} + +rd_kafka_timestamp_type_t MessageMock::get_timestamp_type() const { + return timestamp_type_; +} + +int64_t MessageMock::get_timestamp() const { + return timestamp_; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_partition.cpp b/mocking/src/partition_mock.cpp similarity index 71% rename from mocking/src/kafka_partition.cpp rename to mocking/src/partition_mock.cpp index 740b1622..68e6ad6f 100644 --- a/mocking/src/kafka_partition.cpp +++ b/mocking/src/partition_mock.cpp @@ -1,5 +1,5 @@ #include -#include +#include using std::lock_guard; using std::mutex; @@ -9,12 +9,12 @@ using std::move; namespace cppkafka { namespace mocking { -void KafkaPartition::add_message(KafkaMessage message) { +void PartitionMock::add_message(MessageMock message) { lock_guard _(messages_mutex_); messages_.emplace_back(move(message)); } -const KafkaMessage& KafkaPartition::get_message(uint64_t offset) const { +const MessageMock& PartitionMock::get_message(uint64_t offset) const { const uint64_t index = offset - base_offset_; lock_guard _(messages_mutex_); if (messages_.size() >= index) { @@ -23,7 +23,7 @@ const KafkaMessage& KafkaPartition::get_message(uint64_t offset) const { return messages_[index]; } -size_t KafkaPartition::get_message_count() const { +size_t PartitionMock::get_message_count() const { lock_guard _(messages_mutex_); return messages_.size(); } diff --git a/mocking/src/kafka_topic.cpp b/mocking/src/topic_mock.cpp similarity index 63% rename from mocking/src/kafka_topic.cpp rename to mocking/src/topic_mock.cpp index c7fd6132..2531465a 100644 --- a/mocking/src/kafka_topic.cpp +++ b/mocking/src/topic_mock.cpp @@ -1,5 +1,6 @@ #include -#include +#include +#include using std::string; using std::out_of_range; @@ -8,12 +9,12 @@ using std::move; namespace cppkafka { namespace mocking { -KafkaTopic::KafkaTopic(string name, unsigned partition_count) +TopicMock::TopicMock(string name, unsigned partition_count) : name_(move(name)), partitions_(partition_count) { } -void KafkaTopic::add_message(unsigned partition, KafkaMessage message) { +void TopicMock::add_message(unsigned partition, MessageMock message) { if (partitions_.size() >= partition) { throw out_of_range("invalid partition index"); } From 5ec8dfd8dd36338971e231daee46626cde1146d9 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Tue, 10 Oct 2017 19:24:21 -0700 Subject: [PATCH 05/20] Make sure KafkaClusters live in a shared_ptr --- mocking/include/cppkafka/mocking/kafka_cluster.h | 6 +++++- .../cppkafka/mocking/kafka_cluster_registry.h | 9 ++++++--- mocking/src/kafka_cluster.cpp | 12 ++++++++++-- mocking/src/kafka_cluster_registry.cpp | 12 +++++++----- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index 3d52e035..8e61922f 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -2,6 +2,7 @@ #define CPPKAFKA_MOCKING_KAFKA_CLUSTER_H #include +#include #include #include @@ -10,7 +11,8 @@ namespace mocking { class KafkaCluster { public: - KafkaCluster(std::string url); + static std::shared_ptr make_cluster(std::string url); + KafkaCluster(const KafkaCluster&) = delete; KafkaCluster& operator=(const KafkaCluster&) = delete; ~KafkaCluster(); @@ -20,6 +22,8 @@ class KafkaCluster { void add_topic(const std::string& name, unsigned partitions); void produce(const std::string& topic, unsigned partition, MessageMock message); private: + KafkaCluster(std::string url); + const std::string url_; std::unordered_map topics_; }; diff --git a/mocking/include/cppkafka/mocking/kafka_cluster_registry.h b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h index 255e41a0..822e2aaf 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster_registry.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h @@ -2,6 +2,7 @@ #define CPPKAFKA_MOCKING_KAFKA_CLUSTER_REGISTRY_H #include +#include #include namespace cppkafka { @@ -10,12 +11,14 @@ namespace detail { class KafkaClusterRegistry { public: + using ClusterPtr = std::shared_ptr; + static KafkaClusterRegistry& instance(); - void add_cluster(KafkaCluster* cluster); - void remove_cluster(KafkaCluster* cluster); + void add_cluster(ClusterPtr cluster); + void remove_cluster(const KafkaCluster& cluster); private: - std::unordered_map clusters_; + std::unordered_map clusters_; mutable std::mutex clusters_mutex_; }; diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index 49910deb..0d96c421 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -2,6 +2,8 @@ #include #include +using std::shared_ptr; +using std::make_shared; using std::string; using std::invalid_argument; using std::piecewise_construct; @@ -11,13 +13,19 @@ using std::move; namespace cppkafka { namespace mocking { +shared_ptr KafkaCluster::make_cluster(string url) { + shared_ptr output{ new KafkaCluster(move(url)) }; + detail::KafkaClusterRegistry::instance().add_cluster(output); + return output; +} + KafkaCluster::KafkaCluster(string url) : url_(move(url)) { - detail::KafkaClusterRegistry::instance().add_cluster(this); + } KafkaCluster::~KafkaCluster() { - detail::KafkaClusterRegistry::instance().remove_cluster(this); + detail::KafkaClusterRegistry::instance().remove_cluster(*this); } const string& KafkaCluster::get_url() const { diff --git a/mocking/src/kafka_cluster_registry.cpp b/mocking/src/kafka_cluster_registry.cpp index 315112cd..b16e2f95 100644 --- a/mocking/src/kafka_cluster_registry.cpp +++ b/mocking/src/kafka_cluster_registry.cpp @@ -1,6 +1,7 @@ #include #include +using std::string; using std::lock_guard; using std::mutex; using std::runtime_error; @@ -14,18 +15,19 @@ KafkaClusterRegistry& KafkaClusterRegistry::instance() { return registry; } -void KafkaClusterRegistry::add_cluster(KafkaCluster* cluster) { +void KafkaClusterRegistry::add_cluster(ClusterPtr cluster) { + const string& url = cluster->get_url(); lock_guard _(clusters_mutex_); - auto iter = clusters_.find(cluster->get_url()); + auto iter = clusters_.find(url); if (iter != clusters_.end()) { throw runtime_error("cluster already registered"); } - clusters_.emplace(cluster->get_url(), cluster); + clusters_.emplace(url, move(cluster)); } -void KafkaClusterRegistry::remove_cluster(KafkaCluster* cluster) { +void KafkaClusterRegistry::remove_cluster(const KafkaCluster& cluster) { lock_guard _(clusters_mutex_); - clusters_.erase(cluster->get_url()); + clusters_.erase(cluster.get_url()); } } // detail From fe053f4fa219fd5b2a9da8d00d8dab3fe8293f5d Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 14 Oct 2017 19:28:57 -0700 Subject: [PATCH 06/20] Add initial producer mock code --- .../cppkafka/mocking/event_processor.h | 37 ++++++++ .../cppkafka/mocking/events/event_base.h | 30 +++++++ .../mocking/events/produce_message_event.h | 28 ++++++ .../include/cppkafka/mocking/handle_mock.h | 41 +++++++++ .../include/cppkafka/mocking/kafka_cluster.h | 8 +- .../{message_mock.h => kafka_message_mock.h} | 12 +-- .../cppkafka/mocking/kafka_partition_mock.h | 25 ++++++ .../cppkafka/mocking/kafka_topic_mock.h | 25 ++++++ .../include/cppkafka/mocking/message_handle.h | 55 ++++++++++++ .../include/cppkafka/mocking/partition_mock.h | 25 ------ .../include/cppkafka/mocking/producer_mock.h | 24 ++++++ .../include/cppkafka/mocking/topic_handle.h | 31 +++++++ mocking/include/cppkafka/mocking/topic_mock.h | 25 ------ mocking/src/CMakeLists.txt | 14 ++- mocking/src/event_processor.cpp | 50 +++++++++++ mocking/src/events/event_base.cpp | 17 ++++ mocking/src/events/produce_message_event.cpp | 26 ++++++ mocking/src/handle_mock.cpp | 47 ++++++++++ mocking/src/kafka_cluster.cpp | 2 +- mocking/src/kafka_message_mock.cpp | 30 +++++++ ...tion_mock.cpp => kafka_partition_mock.cpp} | 8 +- .../{topic_mock.cpp => kafka_topic_mock.cpp} | 8 +- mocking/src/message_handle.cpp | 86 +++++++++++++++++++ mocking/src/message_mock.cpp | 30 ------- mocking/src/producer_mock.cpp | 14 +++ 25 files changed, 596 insertions(+), 102 deletions(-) create mode 100644 mocking/include/cppkafka/mocking/event_processor.h create mode 100644 mocking/include/cppkafka/mocking/events/event_base.h create mode 100644 mocking/include/cppkafka/mocking/events/produce_message_event.h create mode 100644 mocking/include/cppkafka/mocking/handle_mock.h rename mocking/include/cppkafka/mocking/{message_mock.h => kafka_message_mock.h} (62%) create mode 100644 mocking/include/cppkafka/mocking/kafka_partition_mock.h create mode 100644 mocking/include/cppkafka/mocking/kafka_topic_mock.h create mode 100644 mocking/include/cppkafka/mocking/message_handle.h delete mode 100644 mocking/include/cppkafka/mocking/partition_mock.h create mode 100644 mocking/include/cppkafka/mocking/producer_mock.h create mode 100644 mocking/include/cppkafka/mocking/topic_handle.h delete mode 100644 mocking/include/cppkafka/mocking/topic_mock.h create mode 100644 mocking/src/event_processor.cpp create mode 100644 mocking/src/events/event_base.cpp create mode 100644 mocking/src/events/produce_message_event.cpp create mode 100644 mocking/src/handle_mock.cpp create mode 100644 mocking/src/kafka_message_mock.cpp rename mocking/src/{partition_mock.cpp => kafka_partition_mock.cpp} (68%) rename mocking/src/{topic_mock.cpp => kafka_topic_mock.cpp} (60%) create mode 100644 mocking/src/message_handle.cpp delete mode 100644 mocking/src/message_mock.cpp create mode 100644 mocking/src/producer_mock.cpp diff --git a/mocking/include/cppkafka/mocking/event_processor.h b/mocking/include/cppkafka/mocking/event_processor.h new file mode 100644 index 00000000..db07d0df --- /dev/null +++ b/mocking/include/cppkafka/mocking/event_processor.h @@ -0,0 +1,37 @@ +#ifndef CPPKAFKA_MOCKING_EVENT_PROCESSOR_ +#define CPPKAFKA_MOCKING_EVENT_PROCESSOR_ + +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class EventProcessor { +public: + using EventPtr = std::unique_ptr; + + EventProcessor(); + EventProcessor(const EventProcessor&) = delete; + EventProcessor& operator=(const EventProcessor&) = delete; + ~EventProcessor(); + + void add_event(EventPtr event); +private: + void process_events(); + + std::thread processing_thread_; + std::mutex events_mutex_; + std::condition_variable events_condition_; + std::queue events_; + bool running_{true}; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_EVENT_PROCESSOR_ diff --git a/mocking/include/cppkafka/mocking/events/event_base.h b/mocking/include/cppkafka/mocking/events/event_base.h new file mode 100644 index 00000000..1d601a83 --- /dev/null +++ b/mocking/include/cppkafka/mocking/events/event_base.h @@ -0,0 +1,30 @@ +#ifndef CPPKAFKA_MOCKING_EVENT_BASE_H +#define CPPKAFKA_MOCKING_EVENT_BASE_H + +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaCluster; + +class EventBase { +public: + using ClusterPtr = std::shared_ptr; + + EventBase(ClusterPtr cluster); + virtual ~EventBase() = default; + + void execute(); + virtual std::string get_type() const = 0; +private: + virtual void execute_event(KafkaCluster& cluster) = 0; + + ClusterPtr cluster_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_EVENT_BASE_H diff --git a/mocking/include/cppkafka/mocking/events/produce_message_event.h b/mocking/include/cppkafka/mocking/events/produce_message_event.h new file mode 100644 index 00000000..207a4687 --- /dev/null +++ b/mocking/include/cppkafka/mocking/events/produce_message_event.h @@ -0,0 +1,28 @@ +#ifndef CPPKAFKA_MOCKING_PRODUCE_MESSAGE_EVENT_H +#define CPPKAFKA_MOCKING_PRODUCE_MESSAGE_EVENT_H + +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class ProduceMessageEvent : public EventBase { +public: + ProduceMessageEvent(ClusterPtr cluster, MessageHandle message_handle); + + std::string get_type() const; +private: + void execute_event(KafkaCluster& cluster); + + MessageHandle message_handle_; + std::string topic_; + unsigned partition_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_PRODUCE_MESSAGE_EVENT_H diff --git a/mocking/include/cppkafka/mocking/handle_mock.h b/mocking/include/cppkafka/mocking/handle_mock.h new file mode 100644 index 00000000..1fa232b0 --- /dev/null +++ b/mocking/include/cppkafka/mocking/handle_mock.h @@ -0,0 +1,41 @@ +#ifndef CPPKAFKA_MOCKING_HANDLE_MOCK_H +#define CPPKAFKA_MOCKING_HANDLE_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaCluster; + +class HandleMock { +public: + using ClusterPtr = std::shared_ptr; + using EventProcessorPtr = std::shared_ptr; + + HandleMock(EventProcessorPtr processor); + HandleMock(EventProcessorPtr processor, ClusterPtr cluster); + virtual ~HandleMock() = default; + + void set_cluster(ClusterPtr cluster); +protected: + using EventPtr = EventProcessor::EventPtr; + + KafkaCluster& get_cluster(); + const KafkaCluster& get_cluster() const; + void generate_event(EventPtr event); + template + void generate_event(Args&&... args) { + generate_event(EventPtr(new T(cluster_, std::forward(args)...))); + } +private: + EventProcessorPtr processor_; + ClusterPtr cluster_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_HANDLE_MOCK_H diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index 8e61922f..74bb702c 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -3,8 +3,8 @@ #include #include -#include -#include +#include +#include namespace cppkafka { namespace mocking { @@ -20,12 +20,12 @@ class KafkaCluster { const std::string& get_url() const; void add_topic(const std::string& name, unsigned partitions); - void produce(const std::string& topic, unsigned partition, MessageMock message); + void produce(const std::string& topic, unsigned partition, KafkaMessageMock message); private: KafkaCluster(std::string url); const std::string url_; - std::unordered_map topics_; + std::unordered_map topics_; }; } // mocking diff --git a/mocking/include/cppkafka/mocking/message_mock.h b/mocking/include/cppkafka/mocking/kafka_message_mock.h similarity index 62% rename from mocking/include/cppkafka/mocking/message_mock.h rename to mocking/include/cppkafka/mocking/kafka_message_mock.h index 49fdd505..5ed568f6 100644 --- a/mocking/include/cppkafka/mocking/message_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_message_mock.h @@ -1,5 +1,5 @@ -#ifndef CPPKAFKA_MOCKING_MESSAGE_MOCK_H -#define CPPKAFKA_MOCKING_MESSAGE_MOCK_H +#ifndef CPPKAFKA_MOCKING_KAFKA_MESSAGE_MOCK_H +#define CPPKAFKA_MOCKING_KAFKA_MESSAGE_MOCK_H #include #include @@ -8,12 +8,12 @@ namespace cppkafka { namespace mocking { -class MessageMock { +class KafkaMessageMock { public: using Buffer = std::vector; - MessageMock(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, - int64_t timestamp); + KafkaMessageMock(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, + int64_t timestamp); const Buffer& get_key() const; const Buffer& get_payload() const; @@ -29,4 +29,4 @@ class MessageMock { } // mocking } // cppkafka -#endif // CPPKAFKA_MOCKING_MESSAGE_MOCK_H +#endif // CPPKAFKA_MOCKING_KAFKA_MESSAGE_MOCK_H diff --git a/mocking/include/cppkafka/mocking/kafka_partition_mock.h b/mocking/include/cppkafka/mocking/kafka_partition_mock.h new file mode 100644 index 00000000..98cfc845 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_partition_mock.h @@ -0,0 +1,25 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_PARTITION_MOCK_H +#define CPPKAFKA_MOCKING_KAFKA_PARTITION_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaPartitionMock { +public: + void add_message(KafkaMessageMock message); + const KafkaMessageMock& get_message(uint64_t offset) const; + size_t get_message_count() const; +private: + uint64_t base_offset_{0}; + std::deque messages_; + mutable std::mutex messages_mutex_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_PARTITION_MOCK_H diff --git a/mocking/include/cppkafka/mocking/kafka_topic_mock.h b/mocking/include/cppkafka/mocking/kafka_topic_mock.h new file mode 100644 index 00000000..1ebd23e8 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_topic_mock.h @@ -0,0 +1,25 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_TOPIC_MOCK_H +#define CPPKAFKA_MOCKING_KAFKA_TOPIC_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaMessageMock; + +class KafkaTopicMock { +public: + KafkaTopicMock(std::string name, unsigned partition_count); + void add_message(unsigned partition, KafkaMessageMock message); +private: + const std::string name_; + std::vector partitions_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_TOPIC_MOCK_H diff --git a/mocking/include/cppkafka/mocking/message_handle.h b/mocking/include/cppkafka/mocking/message_handle.h new file mode 100644 index 00000000..b770742f --- /dev/null +++ b/mocking/include/cppkafka/mocking/message_handle.h @@ -0,0 +1,55 @@ +#ifndef CPPKAFKA_MOCKING_MESSAGE_HANDLE_H +#define CPPKAFKA_MOCKING_MESSAGE_HANDLE_H + +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaMessageMock; + +class MessageHandlePrivateData { +public: + MessageHandlePrivateData() = default; + MessageHandlePrivateData(rd_kafka_timestamp_type_t timestamp_type, int64_t timestamp); + + rd_kafka_timestamp_type_t get_timestamp_type() const; + int64_t get_timestamp() const; +private: + rd_kafka_timestamp_type_t timestamp_type_; + int64_t timestamp_; +}; + +class MessageHandle { +public: + enum class PointerOwnership { + Owned, + Unowned + }; + + MessageHandle(std::unique_ptr topic, int partition, int64_t offset, void* key, + size_t key_size, void* payload, size_t payload_size, int error_code, + MessageHandlePrivateData private_data, PointerOwnership ownership); + MessageHandle(MessageHandle&& other); + MessageHandle& operator=(MessageHandle&& other); + ~MessageHandle(); + + const TopicHandle& get_topic() const; + const rd_kafka_message_t& get_message() const; + KafkaMessageMock make_message_mock() const; +private: + void set_private_data_pointer(); + + std::unique_ptr topic_; + rd_kafka_message_t message_{}; + MessageHandlePrivateData private_data_; + PointerOwnership ownership_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_MESSAGE_HANDLE_H diff --git a/mocking/include/cppkafka/mocking/partition_mock.h b/mocking/include/cppkafka/mocking/partition_mock.h deleted file mode 100644 index 6dc29ed0..00000000 --- a/mocking/include/cppkafka/mocking/partition_mock.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef CPPKAFKA_MOCKING_PARTITION_MOCK_H -#define CPPKAFKA_MOCKING_PARTITION_MOCK_H - -#include -#include -#include - -namespace cppkafka { -namespace mocking { - -class PartitionMock { -public: - void add_message(MessageMock message); - const MessageMock& get_message(uint64_t offset) const; - size_t get_message_count() const; -private: - uint64_t base_offset_{0}; - std::deque messages_; - mutable std::mutex messages_mutex_; -}; - -} // mocking -} // cppkafka - -#endif // CPPKAFKA_MOCKING_PARTITION_MOCK_H diff --git a/mocking/include/cppkafka/mocking/producer_mock.h b/mocking/include/cppkafka/mocking/producer_mock.h new file mode 100644 index 00000000..c5fe91e4 --- /dev/null +++ b/mocking/include/cppkafka/mocking/producer_mock.h @@ -0,0 +1,24 @@ +#ifndef CPPKAFKA_MOCKING_PRODUCER_MOCK_H +#define CPPKAFKA_MOCKING_PRODUCER_MOCK_H + +#include +#include + +namespace cppkafka { +namespace mocking { + +class ProducerMock : public HandleMock { +public: + using ClusterPtr = std::shared_ptr; + + using HandleMock::HandleMock; + + void produce_message(MessageHandle message_handle); +private: + +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_PRODUCER_MOCK_H diff --git a/mocking/include/cppkafka/mocking/topic_handle.h b/mocking/include/cppkafka/mocking/topic_handle.h new file mode 100644 index 00000000..a4a6e163 --- /dev/null +++ b/mocking/include/cppkafka/mocking/topic_handle.h @@ -0,0 +1,31 @@ +#ifndef CPPKAFKA_MOCKING_TOPIC_HANDLE_H +#define CPPKAFKA_MOCKING_TOPIC_HANDLE_H + +#include + +namespace cppkafka { +namespace mocking { + +class TopicHandle { +public: + TopicHandle(std::string topic, void* opaque) + : topic_(move(topic)), opaque_(opaque) { + + } + + const std::string& get_topic() const { + return topic_; + } + + void* get_opaque() const { + return opaque_; + } +private: + const std::string topic_; + void* opaque_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_TOPIC_HANDLE_H diff --git a/mocking/include/cppkafka/mocking/topic_mock.h b/mocking/include/cppkafka/mocking/topic_mock.h deleted file mode 100644 index bc260274..00000000 --- a/mocking/include/cppkafka/mocking/topic_mock.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef CPPKAFKA_MOCKING_TOPIC_MOCK_H -#define CPPKAFKA_MOCKING_TOPIC_MOCK_H - -#include -#include -#include - -namespace cppkafka { -namespace mocking { - -class MessageMock; - -class TopicMock { -public: - TopicMock(std::string name, unsigned partition_count); - void add_message(unsigned partition, MessageMock message); -private: - const std::string name_; - std::vector partitions_; -}; - -} // mocking -} // cppkafka - -#endif // CPPKAFKA_MOCKING_TOPIC_MOCK_H diff --git a/mocking/src/CMakeLists.txt b/mocking/src/CMakeLists.txt index 7b94de2d..30f5c3cd 100644 --- a/mocking/src/CMakeLists.txt +++ b/mocking/src/CMakeLists.txt @@ -1,10 +1,18 @@ set(SOURCES configuration_mock.cpp - topic_mock.cpp - partition_mock.cpp - message_mock.cpp + kafka_topic_mock.cpp + kafka_partition_mock.cpp + kafka_message_mock.cpp kafka_cluster.cpp kafka_cluster_registry.cpp + handle_mock.cpp + message_handle.cpp + producer_mock.cpp + + events/event_base.cpp + events/produce_message_event.cpp + event_processor.cpp + api.cpp ) diff --git a/mocking/src/event_processor.cpp b/mocking/src/event_processor.cpp new file mode 100644 index 00000000..4c6f2c48 --- /dev/null +++ b/mocking/src/event_processor.cpp @@ -0,0 +1,50 @@ +#include + +using std::lock_guard; +using std::unique_lock; +using std::mutex; +using std::move; + +namespace cppkafka { +namespace mocking { + +EventProcessor::EventProcessor() +: processing_thread_(&EventProcessor::process_events, this) { + +} + +EventProcessor::~EventProcessor() { + { + lock_guard _(events_mutex_); + running_ = false; + events_condition_.notify_one(); + } + processing_thread_.join(); +} + +void EventProcessor::add_event(EventPtr event) { + lock_guard _(events_mutex_); + events_.push(move(event)); + events_condition_.notify_one(); +} + +void EventProcessor::process_events() { + while (true) { + unique_lock lock(events_mutex_); + while (running_ && events_.empty()) { + events_condition_.wait(lock); + } + + if (!running_) { + break; + } + EventPtr event = move(events_.front()); + events_.pop(); + + lock.unlock(); + event->execute(); + } +} + +} // mocking +} // cppkafka diff --git a/mocking/src/events/event_base.cpp b/mocking/src/events/event_base.cpp new file mode 100644 index 00000000..bfc570fd --- /dev/null +++ b/mocking/src/events/event_base.cpp @@ -0,0 +1,17 @@ +#include +#include + +namespace cppkafka { +namespace mocking { + +EventBase::EventBase(ClusterPtr cluster) +: cluster_(move(cluster)) { + +} + +void EventBase::execute() { + execute_event(*cluster_); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/events/produce_message_event.cpp b/mocking/src/events/produce_message_event.cpp new file mode 100644 index 00000000..8977cfeb --- /dev/null +++ b/mocking/src/events/produce_message_event.cpp @@ -0,0 +1,26 @@ +#include +#include + +using std::string; +using std::move; + +namespace cppkafka { +namespace mocking { + +ProduceMessageEvent::ProduceMessageEvent(ClusterPtr cluster, MessageHandle message_handle) +: EventBase(move(cluster)), message_handle_(move(message_handle)) { + +} + +string ProduceMessageEvent::get_type() const { + return "produce message"; +} + +void ProduceMessageEvent::execute_event(KafkaCluster& cluster) { + const string& topic = message_handle_.get_topic().get_topic(); + const int partition = message_handle_.get_message().partition; + cluster.produce(topic, partition, message_handle_.make_message_mock()); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/handle_mock.cpp b/mocking/src/handle_mock.cpp new file mode 100644 index 00000000..a25e0d3c --- /dev/null +++ b/mocking/src/handle_mock.cpp @@ -0,0 +1,47 @@ +#include +#include + +using std::runtime_error; +using std::move; + +namespace cppkafka { +namespace mocking { + +HandleMock::HandleMock(EventProcessorPtr processor) +: processor_(move(processor)) { + +} + +HandleMock::HandleMock(EventProcessorPtr processor, ClusterPtr cluster) +: processor_(move(processor)), cluster_(move(cluster)) { + +} + +void HandleMock::set_cluster(ClusterPtr cluster) { + // Don't allow changing the cluster + if (cluster_) { + throw runtime_error("can't change the cluster"); + } + cluster_ = move(cluster); +} + +KafkaCluster& HandleMock::get_cluster() { + if (!cluster_) { + throw runtime_error("cluster not set"); + } + return *cluster_; +} + +const KafkaCluster& HandleMock::get_cluster() const { + if (!cluster_) { + throw runtime_error("cluster not set"); + } + return *cluster_; +} + +void HandleMock::generate_event(EventPtr event) { + processor_->add_event(move(event)); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index 0d96c421..3b93d78f 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -37,7 +37,7 @@ void KafkaCluster::add_topic(const string& name, unsigned partitions) { forward_as_tuple(name, partitions)); } -void KafkaCluster::produce(const string& topic, unsigned partition, MessageMock message) { +void KafkaCluster::produce(const string& topic, unsigned partition, KafkaMessageMock message) { auto iter = topics_.find(topic); if (iter == topics_.end()) { throw invalid_argument("topic does not exist"); diff --git a/mocking/src/kafka_message_mock.cpp b/mocking/src/kafka_message_mock.cpp new file mode 100644 index 00000000..5fd100a8 --- /dev/null +++ b/mocking/src/kafka_message_mock.cpp @@ -0,0 +1,30 @@ +#include + +namespace cppkafka { +namespace mocking { + +KafkaMessageMock::KafkaMessageMock(Buffer key, Buffer payload, + rd_kafka_timestamp_type_t timestamp_type, int64_t timestamp) +: key_(move(key)), payload_(move(payload)), timestamp_type_(timestamp_type), + timestamp_(timestamp) { + +} + +const KafkaMessageMock::Buffer& KafkaMessageMock::get_key() const { + return key_; +} + +const KafkaMessageMock::Buffer& KafkaMessageMock::get_payload() const { + return payload_; +} + +rd_kafka_timestamp_type_t KafkaMessageMock::get_timestamp_type() const { + return timestamp_type_; +} + +int64_t KafkaMessageMock::get_timestamp() const { + return timestamp_; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/partition_mock.cpp b/mocking/src/kafka_partition_mock.cpp similarity index 68% rename from mocking/src/partition_mock.cpp rename to mocking/src/kafka_partition_mock.cpp index 68e6ad6f..dbb5adff 100644 --- a/mocking/src/partition_mock.cpp +++ b/mocking/src/kafka_partition_mock.cpp @@ -1,5 +1,5 @@ #include -#include +#include using std::lock_guard; using std::mutex; @@ -9,12 +9,12 @@ using std::move; namespace cppkafka { namespace mocking { -void PartitionMock::add_message(MessageMock message) { +void KafkaPartitionMock::add_message(KafkaMessageMock message) { lock_guard _(messages_mutex_); messages_.emplace_back(move(message)); } -const MessageMock& PartitionMock::get_message(uint64_t offset) const { +const KafkaMessageMock& KafkaPartitionMock::get_message(uint64_t offset) const { const uint64_t index = offset - base_offset_; lock_guard _(messages_mutex_); if (messages_.size() >= index) { @@ -23,7 +23,7 @@ const MessageMock& PartitionMock::get_message(uint64_t offset) const { return messages_[index]; } -size_t PartitionMock::get_message_count() const { +size_t KafkaPartitionMock::get_message_count() const { lock_guard _(messages_mutex_); return messages_.size(); } diff --git a/mocking/src/topic_mock.cpp b/mocking/src/kafka_topic_mock.cpp similarity index 60% rename from mocking/src/topic_mock.cpp rename to mocking/src/kafka_topic_mock.cpp index 2531465a..169c137c 100644 --- a/mocking/src/topic_mock.cpp +++ b/mocking/src/kafka_topic_mock.cpp @@ -1,6 +1,6 @@ #include -#include -#include +#include +#include using std::string; using std::out_of_range; @@ -9,12 +9,12 @@ using std::move; namespace cppkafka { namespace mocking { -TopicMock::TopicMock(string name, unsigned partition_count) +KafkaTopicMock::KafkaTopicMock(string name, unsigned partition_count) : name_(move(name)), partitions_(partition_count) { } -void TopicMock::add_message(unsigned partition, MessageMock message) { +void KafkaTopicMock::add_message(unsigned partition, KafkaMessageMock message) { if (partitions_.size() >= partition) { throw out_of_range("invalid partition index"); } diff --git a/mocking/src/message_handle.cpp b/mocking/src/message_handle.cpp new file mode 100644 index 00000000..3f93cc2e --- /dev/null +++ b/mocking/src/message_handle.cpp @@ -0,0 +1,86 @@ +#include +#include +#include + +using std::unique_ptr; +using std::move; +using std::swap; + +namespace cppkafka { +namespace mocking { + +MessageHandlePrivateData::MessageHandlePrivateData(rd_kafka_timestamp_type_t timestamp_type, + int64_t timestamp) +: timestamp_type_(timestamp_type), timestamp_(timestamp) { + +} + +rd_kafka_timestamp_type_t MessageHandlePrivateData::get_timestamp_type() const { + return timestamp_type_; +} + +int64_t MessageHandlePrivateData::get_timestamp() const { + return timestamp_; +} + +MessageHandle::MessageHandle(unique_ptr topic, int partition, int64_t offset, + void* key, size_t key_size, void* payload, size_t payload_size, + int error_code, MessageHandlePrivateData private_data, + PointerOwnership ownership) +: topic_(move(topic)), private_data_(private_data), ownership_(ownership) { + message_.rkt = reinterpret_cast(topic_.get()); + message_.partition = partition; + message_.payload = payload; + message_.len = payload_size; + message_.key = key; + message_.key_len = key_size; + message_.offset = offset; + set_private_data_pointer(); +} + +MessageHandle::MessageHandle(MessageHandle&& other) +: ownership_(PointerOwnership::Unowned) { + *this = move(other); +} + +MessageHandle& MessageHandle::operator=(MessageHandle&& other) { + swap(topic_, other.topic_); + swap(message_, other.message_); + swap(ownership_, other.ownership_); + set_private_data_pointer(); + other.set_private_data_pointer(); + return *this; +} + +MessageHandle::~MessageHandle() { + if (ownership_ == PointerOwnership::Owned) { + free(message_.payload); + free(message_.key); + } +} + +const TopicHandle& MessageHandle::get_topic() const { + return *topic_; +} + +const rd_kafka_message_t& MessageHandle::get_message() const { + return message_; +} + +KafkaMessageMock MessageHandle::make_message_mock() const { + auto key_ptr = reinterpret_cast(message_.key); + auto payload_ptr = reinterpret_cast(message_.payload); + return { + KafkaMessageMock::Buffer(key_ptr, key_ptr + message_.key_len), + KafkaMessageMock::Buffer(payload_ptr, payload_ptr + message_.len), + private_data_.get_timestamp_type(), + private_data_.get_timestamp() + }; +} + +void MessageHandle::set_private_data_pointer() { + message_._private = reinterpret_cast(&private_data_); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/message_mock.cpp b/mocking/src/message_mock.cpp deleted file mode 100644 index d64a65cc..00000000 --- a/mocking/src/message_mock.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include - -namespace cppkafka { -namespace mocking { - -MessageMock::MessageMock(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, - int64_t timestamp) -: key_(move(key)), payload_(move(payload)), timestamp_type_(timestamp_type), - timestamp_(timestamp) { - -} - -const MessageMock::Buffer& MessageMock::get_key() const { - return key_; -} - -const MessageMock::Buffer& MessageMock::get_payload() const { - return payload_; -} - -rd_kafka_timestamp_type_t MessageMock::get_timestamp_type() const { - return timestamp_type_; -} - -int64_t MessageMock::get_timestamp() const { - return timestamp_; -} - -} // mocking -} // cppkafka diff --git a/mocking/src/producer_mock.cpp b/mocking/src/producer_mock.cpp new file mode 100644 index 00000000..abb65b71 --- /dev/null +++ b/mocking/src/producer_mock.cpp @@ -0,0 +1,14 @@ +#include +#include + +using std::move; + +namespace cppkafka { +namespace mocking { + +void ProducerMock::produce_message(MessageHandle message_handle) { + generate_event(move(message_handle)); +} + +} // mocking +} // cppkafka From 7ef55a974740d6c5d8211af504bd3a9992d82a7b Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Mon, 16 Oct 2017 21:10:08 -0700 Subject: [PATCH 07/20] Allow constructing mock producers --- mocking/include/cppkafka/mocking/api.h | 72 ++----------------- .../include/cppkafka/mocking/handle_wrapper.h | 24 +++++-- .../cppkafka/mocking/kafka_cluster_registry.h | 1 + .../include/cppkafka/mocking/producer_mock.h | 5 +- mocking/src/api.cpp | 21 ++++++ mocking/src/kafka_cluster_registry.cpp | 6 ++ mocking/src/producer_mock.cpp | 6 ++ 7 files changed, 60 insertions(+), 75 deletions(-) diff --git a/mocking/include/cppkafka/mocking/api.h b/mocking/include/cppkafka/mocking/api.h index 8f52fa6c..0c4c6c28 100644 --- a/mocking/include/cppkafka/mocking/api.h +++ b/mocking/include/cppkafka/mocking/api.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include struct rd_kafka_conf_s : cppkafka::mocking::HandleWrapper { using cppkafka::mocking::HandleWrapper::HandleWrapper; @@ -14,72 +14,8 @@ struct rd_kafka_topic_conf_s : rd_kafka_conf_s { using rd_kafka_conf_s::rd_kafka_conf_s; }; -// rd_kafka_conf_t -CPPKAFKA_API rd_kafka_conf_t* rd_kafka_conf_new(); -CPPKAFKA_API void rd_kafka_conf_destroy(rd_kafka_conf_t* conf); -CPPKAFKA_API rd_kafka_conf_t* rd_kafka_conf_dup(const rd_kafka_conf_t* conf); -CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_conf_set(rd_kafka_conf_t* conf, - const char* name, const char* value, - char* errstr, size_t errstr_size); -CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_conf_get(const rd_kafka_conf_t* conf, - const char* name, char* dest, - size_t* dest_size); -CPPKAFKA_API -void rd_kafka_conf_set_dr_msg_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::DeliveryReportCallback* cb); -CPPKAFKA_API -void rd_kafka_conf_set_rebalance_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::RebalanceCallback* cb); -CPPKAFKA_API -void rd_kafka_conf_set_offset_commit_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::OffsetCommitCallback* cb); -CPPKAFKA_API -void rd_kafka_conf_set_error_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::ErrorCallback* cb); -CPPKAFKA_API -void rd_kafka_conf_set_throttle_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::ThrottleCallback* cb); -CPPKAFKA_API -void rd_kafka_conf_set_log_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::LogCallback* cb); -CPPKAFKA_API -void rd_kafka_conf_set_stats_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::StatsCallback* cb); -CPPKAFKA_API -void rd_kafka_conf_set_socket_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::SocketCallback* cb); -CPPKAFKA_API -void rd_kafka_topic_conf_set_partitioner_cb(rd_kafka_conf_t* conf, - cppkafka::mocking::ConfigurationMock::PartitionerCallback* cb); -CPPKAFKA_API -void rd_kafka_conf_set_default_topic_conf(rd_kafka_conf_t* conf, - rd_kafka_topic_conf_t* tconf); -CPPKAFKA_API void rd_kafka_conf_set_opaque(rd_kafka_conf_t* conf, void* opaque); -CPPKAFKA_API const char** rd_kafka_conf_dump(rd_kafka_conf_t* conf, size_t* cntp); -CPPKAFKA_API void rd_kafka_conf_dump_free(const char** arr, size_t cnt); - -// rd_kafka_topic_conf_t -CPPKAFKA_API rd_kafka_topic_conf_t* rd_kafka_topic_conf_new(); -CPPKAFKA_API void rd_kafka_topic_conf_destroy(rd_kafka_topic_conf_t* conf); -CPPKAFKA_API rd_kafka_topic_conf_t* rd_kafka_topic_conf_dup(const rd_kafka_topic_conf_t* conf); -CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_topic_conf_set(rd_kafka_topic_conf_t* conf, - const char* name, const char* value, - char* errstr, size_t errstr_size); -CPPKAFKA_API rd_kafka_conf_res_t rd_kafka_topic_conf_get(const rd_kafka_topic_conf_t *conf, - const char* name, char* dest, - size_t* dest_size); -CPPKAFKA_API -const char** rd_kafka_topic_conf_dump(rd_kafka_topic_conf_t* conf, size_t* cntp); - -// rd_kafka_topic_* - -CPPKAFKA_API void rd_kafka_topic_partition_destroy(rd_kafka_topic_partition_t* toppar); -CPPKAFKA_API rd_kafka_topic_partition_list_t* rd_kafka_topic_partition_list_new(int size); -CPPKAFKA_API -void rd_kafka_topic_partition_list_destroy(rd_kafka_topic_partition_list_t* toppar_list); -CPPKAFKA_API -rd_kafka_topic_partition_t* -rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list, - const char* topic, int32_t partition); +struct rd_kafka_s : cppkafka::mocking::HandleWrapper { + using cppkafka::mocking::HandleWrapper::HandleWrapper; +}; #endif // CPPKAFKA_MOCKING_API_H diff --git a/mocking/include/cppkafka/mocking/handle_wrapper.h b/mocking/include/cppkafka/mocking/handle_wrapper.h index 09dae327..3a564216 100644 --- a/mocking/include/cppkafka/mocking/handle_wrapper.h +++ b/mocking/include/cppkafka/mocking/handle_wrapper.h @@ -2,6 +2,7 @@ #define CPPKAFKA_MOCKING_HANDLE_WRAPPER_H #include +#include namespace cppkafka { namespace mocking { @@ -9,21 +10,34 @@ namespace mocking { template class HandleWrapper { public: + HandleWrapper() + : handle_(new T()) { + + } + template - HandleWrapper(Args&&... args) - : handle_(std::forward(args)...) { + HandleWrapper(const Args&... args) + : handle_(new T(args...)) { + + } + + template + explicit HandleWrapper(U* ptr) + : handle_(ptr) { } + + T& get_handle() { - return handle_; + return *handle_; } const T& get_handle() const { - return handle_; + return *handle_; } private: - T handle_; + std::unique_ptr handle_; }; } // mocking diff --git a/mocking/include/cppkafka/mocking/kafka_cluster_registry.h b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h index 822e2aaf..1fbae645 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster_registry.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h @@ -17,6 +17,7 @@ class KafkaClusterRegistry { void add_cluster(ClusterPtr cluster); void remove_cluster(const KafkaCluster& cluster); + ClusterPtr get_cluster(const std::string& name) const; private: std::unordered_map clusters_; mutable std::mutex clusters_mutex_; diff --git a/mocking/include/cppkafka/mocking/producer_mock.h b/mocking/include/cppkafka/mocking/producer_mock.h index c5fe91e4..b90543eb 100644 --- a/mocking/include/cppkafka/mocking/producer_mock.h +++ b/mocking/include/cppkafka/mocking/producer_mock.h @@ -2,6 +2,7 @@ #define CPPKAFKA_MOCKING_PRODUCER_MOCK_H #include +#include #include namespace cppkafka { @@ -11,11 +12,11 @@ class ProducerMock : public HandleMock { public: using ClusterPtr = std::shared_ptr; - using HandleMock::HandleMock; + ProducerMock(ConfigurationMock config, EventProcessorPtr processor, ClusterPtr cluster); void produce_message(MessageHandle message_handle); private: - + ConfigurationMock config_; }; } // mocking diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp index 34e0d81a..1e59111b 100644 --- a/mocking/src/api.cpp +++ b/mocking/src/api.cpp @@ -2,12 +2,16 @@ #include #include #include +#include +#include using std::string; using std::copy; using std::strlen; +using std::make_shared; using namespace cppkafka::mocking; +using namespace cppkafka::mocking::detail; // rd_kafka_conf_t @@ -199,3 +203,20 @@ rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list, output->partition = partition; return output; } + +// rd_kafka_t + +rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf_ptr, + char *errstr, size_t errstr_size) { + static const string BROKERS_OPTION = "metadata.broker.list"; + const auto& conf = conf_ptr->get_handle(); + HandleMock::ClusterPtr cluster; + if (conf.has_key(BROKERS_OPTION)) { + cluster = KafkaClusterRegistry::instance().get_cluster(conf.get(BROKERS_OPTION)); + } + if (type == RD_KAFKA_PRODUCER) { + return new rd_kafka_t(new ProducerMock(conf, make_shared(), + move(cluster))); + } + return nullptr; +} diff --git a/mocking/src/kafka_cluster_registry.cpp b/mocking/src/kafka_cluster_registry.cpp index b16e2f95..dc28cac4 100644 --- a/mocking/src/kafka_cluster_registry.cpp +++ b/mocking/src/kafka_cluster_registry.cpp @@ -30,6 +30,12 @@ void KafkaClusterRegistry::remove_cluster(const KafkaCluster& cluster) { clusters_.erase(cluster.get_url()); } +KafkaClusterRegistry::ClusterPtr KafkaClusterRegistry::get_cluster(const string& name) const { + lock_guard _(clusters_mutex_); + auto iter = clusters_.find(name); + return iter != clusters_.end() ? iter->second : ClusterPtr{}; +} + } // detail } // mocking } // cppkafka diff --git a/mocking/src/producer_mock.cpp b/mocking/src/producer_mock.cpp index abb65b71..6a9621ff 100644 --- a/mocking/src/producer_mock.cpp +++ b/mocking/src/producer_mock.cpp @@ -6,6 +6,12 @@ using std::move; namespace cppkafka { namespace mocking { +ProducerMock::ProducerMock(ConfigurationMock config, EventProcessorPtr processor, + ClusterPtr cluster) +: HandleMock(move(processor), move(cluster)), config_(move(config)) { + +} + void ProducerMock::produce_message(MessageHandle message_handle) { generate_event(move(message_handle)); } From 8217a01b14e482899e15da69d80c1838d6f2476a Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Thu, 19 Oct 2017 20:41:48 -0700 Subject: [PATCH 08/20] Allow subscribing to topics --- .../include/cppkafka/mocking/kafka_cluster.h | 2 + .../cppkafka/mocking/kafka_partition_mock.h | 13 ++++ .../cppkafka/mocking/kafka_topic_mock.h | 30 ++++++++- .../include/cppkafka/mocking/offset_manager.h | 34 +++++++++++ .../cppkafka/mocking/topic_partition_mock.h | 29 +++++++++ mocking/src/CMakeLists.txt | 2 + mocking/src/kafka_cluster.cpp | 4 +- mocking/src/kafka_partition_mock.cpp | 34 ++++++++++- mocking/src/kafka_topic_mock.cpp | 61 ++++++++++++++++++- mocking/src/offset_manager.cpp | 46 ++++++++++++++ mocking/src/topic_partition_mock.cpp | 30 +++++++++ 11 files changed, 278 insertions(+), 7 deletions(-) create mode 100644 mocking/include/cppkafka/mocking/offset_manager.h create mode 100644 mocking/include/cppkafka/mocking/topic_partition_mock.h create mode 100644 mocking/src/offset_manager.cpp create mode 100644 mocking/src/topic_partition_mock.cpp diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index 74bb702c..24dceb8b 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace cppkafka { namespace mocking { @@ -25,6 +26,7 @@ class KafkaCluster { KafkaCluster(std::string url); const std::string url_; + std::shared_ptr offset_manager_; std::unordered_map topics_; }; diff --git a/mocking/include/cppkafka/mocking/kafka_partition_mock.h b/mocking/include/cppkafka/mocking/kafka_partition_mock.h index 98cfc845..5e287a06 100644 --- a/mocking/include/cppkafka/mocking/kafka_partition_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_partition_mock.h @@ -3,6 +3,9 @@ #include #include +#include +#include +#include #include namespace cppkafka { @@ -10,13 +13,23 @@ namespace mocking { class KafkaPartitionMock { public: + using MessageCallback = std::function; + using SubscriberId = uint64_t; + void add_message(KafkaMessageMock message); const KafkaMessageMock& get_message(uint64_t offset) const; size_t get_message_count() const; + SubscriberId subscribe(MessageCallback callback); + void unsubscribe(SubscriberId id); private: + std::vector get_subscriber_callbacks() const; + uint64_t base_offset_{0}; + SubscriberId current_subscriber_id_{0}; std::deque messages_; + std::unordered_map subscribers_; mutable std::mutex messages_mutex_; + mutable std::mutex subscribers_mutex_; }; } // mocking diff --git a/mocking/include/cppkafka/mocking/kafka_topic_mock.h b/mocking/include/cppkafka/mocking/kafka_topic_mock.h index 1ebd23e8..513d7ab7 100644 --- a/mocking/include/cppkafka/mocking/kafka_topic_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_topic_mock.h @@ -1,9 +1,17 @@ #ifndef CPPKAFKA_MOCKING_KAFKA_TOPIC_MOCK_H #define CPPKAFKA_MOCKING_KAFKA_TOPIC_MOCK_H +#include +#include #include +#include #include +#include +#include +#include #include +#include +#include namespace cppkafka { namespace mocking { @@ -12,11 +20,31 @@ class KafkaMessageMock; class KafkaTopicMock { public: - KafkaTopicMock(std::string name, unsigned partition_count); + using AssignmentCallback = std::function&)>; + using RevocationCallback = std::function&)>; + using OffsetManagerPtr = std::shared_ptr; + + KafkaTopicMock(std::string name, unsigned partition_count, OffsetManagerPtr offset_manager); void add_message(unsigned partition, KafkaMessageMock message); + void subscribe(const std::string& group_id, uint64_t consumer_id, + AssignmentCallback assignment_callback, + RevocationCallback revocation_callback); + void unsubscribe(const std::string& group_id, uint64_t consumer_id); private: + struct MemberMetadata { + const AssignmentCallback assignment_callback; + const RevocationCallback revocation_callback; + std::vector partitions_assigned; + }; + using MembersMetadataMap = std::unordered_map; + + void generate_assignments(const std::string& group_id, MembersMetadataMap& members_metadata); + const std::string name_; std::vector partitions_; + OffsetManagerPtr offset_manager_; + std::unordered_map subscribers_; + mutable std::mutex subscribers_mutex_; }; } // mocking diff --git a/mocking/include/cppkafka/mocking/offset_manager.h b/mocking/include/cppkafka/mocking/offset_manager.h new file mode 100644 index 00000000..775259cb --- /dev/null +++ b/mocking/include/cppkafka/mocking/offset_manager.h @@ -0,0 +1,34 @@ +#ifndef CPPKAFKA_MOCKING_OFFSET_MANAGER_H +#define CPPKAFKA_MOCKING_OFFSET_MANAGER_H + +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class TopicPartitionMock; + +class OffsetManager { +public: + void commit_offsets(const std::string& group_id, const std::vector&); + std::vector get_offsets(const std::string& group_id, + std::vector) const; +private: + // (consumer, topic, partition) + using OffsetIdentifier = std::tuple; + using OffsetMap = std::map; + + OffsetMap offsets_; + mutable std::mutex offsets_mutex_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_OFFSET_MANAGER_H + diff --git a/mocking/include/cppkafka/mocking/topic_partition_mock.h b/mocking/include/cppkafka/mocking/topic_partition_mock.h new file mode 100644 index 00000000..f5f8676c --- /dev/null +++ b/mocking/include/cppkafka/mocking/topic_partition_mock.h @@ -0,0 +1,29 @@ +#ifndef CPPKAFKA_MOCKING_TOPIC_PARTITION_MOCK_H +#define CPPKAFKA_MOCKING_TOPIC_PARTITION_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class TopicPartitionMock { +public: + TopicPartitionMock(std::string topic, int partition, int64_t offset = RD_KAFKA_OFFSET_INVALID); + + const std::string& get_topic() const; + int get_partition() const; + int64_t get_offset() const; + + void set_offset(int64_t offset); +private: + std::string topic_; + int partition_; + int64_t offset_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_TOPIC_PARTITION_MOCK_H diff --git a/mocking/src/CMakeLists.txt b/mocking/src/CMakeLists.txt index 30f5c3cd..4e855fe4 100644 --- a/mocking/src/CMakeLists.txt +++ b/mocking/src/CMakeLists.txt @@ -8,6 +8,8 @@ set(SOURCES handle_mock.cpp message_handle.cpp producer_mock.cpp + topic_partition_mock.cpp + offset_manager.cpp events/event_base.cpp events/produce_message_event.cpp diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index 3b93d78f..dcd6772e 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -20,7 +20,7 @@ shared_ptr KafkaCluster::make_cluster(string url) { } KafkaCluster::KafkaCluster(string url) -: url_(move(url)) { +: url_(move(url)), offset_manager_(make_shared()) { } @@ -34,7 +34,7 @@ const string& KafkaCluster::get_url() const { void KafkaCluster::add_topic(const string& name, unsigned partitions) { topics_.emplace(piecewise_construct, forward_as_tuple(name), - forward_as_tuple(name, partitions)); + forward_as_tuple(name, partitions, offset_manager_)); } void KafkaCluster::produce(const string& topic, unsigned partition, KafkaMessageMock message) { diff --git a/mocking/src/kafka_partition_mock.cpp b/mocking/src/kafka_partition_mock.cpp index dbb5adff..5a6229e7 100644 --- a/mocking/src/kafka_partition_mock.cpp +++ b/mocking/src/kafka_partition_mock.cpp @@ -1,6 +1,7 @@ #include #include +using std::vector; using std::lock_guard; using std::mutex; using std::out_of_range; @@ -10,8 +11,16 @@ namespace cppkafka { namespace mocking { void KafkaPartitionMock::add_message(KafkaMessageMock message) { - lock_guard _(messages_mutex_); - messages_.emplace_back(move(message)); + const uint64_t offset = [&] { + lock_guard _(messages_mutex_); + messages_.emplace_back(move(message)); + return messages_.size() - 1; + }(); + + const vector callbacks = get_subscriber_callbacks(); + for (const MessageCallback& callback : callbacks) { + callback(offset); + } } const KafkaMessageMock& KafkaPartitionMock::get_message(uint64_t offset) const { @@ -28,5 +37,26 @@ size_t KafkaPartitionMock::get_message_count() const { return messages_.size(); } +KafkaPartitionMock::SubscriberId KafkaPartitionMock::subscribe(MessageCallback callback) { + lock_guard _(subscribers_mutex_); + auto id = current_subscriber_id_++; + subscribers_.emplace(id, move(callback)); + return id; +} + +void KafkaPartitionMock::unsubscribe(SubscriberId id) { + lock_guard _(subscribers_mutex_); + subscribers_.erase(id); +} + +vector KafkaPartitionMock::get_subscriber_callbacks() const { + lock_guard _(subscribers_mutex_); + vector output; + for (const auto& subcriber_pair : subscribers_) { + output.emplace_back(subcriber_pair.second); + } + return output; +} + } // mocking } // cppkafka diff --git a/mocking/src/kafka_topic_mock.cpp b/mocking/src/kafka_topic_mock.cpp index 169c137c..4d168ebf 100644 --- a/mocking/src/kafka_topic_mock.cpp +++ b/mocking/src/kafka_topic_mock.cpp @@ -1,16 +1,22 @@ #include +#include #include #include using std::string; using std::out_of_range; using std::move; +using std::lock_guard; +using std::mutex; +using std::vector; +using std::iota; namespace cppkafka { namespace mocking { -KafkaTopicMock::KafkaTopicMock(string name, unsigned partition_count) -: name_(move(name)), partitions_(partition_count) { +KafkaTopicMock::KafkaTopicMock(string name, unsigned partition_count, + OffsetManagerPtr offset_manager) +: name_(move(name)), partitions_(partition_count), offset_manager_(move(offset_manager)) { } @@ -21,5 +27,56 @@ void KafkaTopicMock::add_message(unsigned partition, KafkaMessageMock message) { partitions_[partition].add_message(move(message)); } +void KafkaTopicMock::subscribe(const string& group_id, uint64_t consumer_id, + AssignmentCallback assignment_callback, + RevocationCallback revocation_callback) { + lock_guard _(subscribers_mutex_); + auto& members_data = subscribers_[group_id]; + // If we're already subscribed, there's nothing to do + if (members_data.count(consumer_id)) { + return; + } + members_data.emplace(consumer_id, + MemberMetadata{move(assignment_callback), move(revocation_callback)}); + generate_assignments(group_id, members_data); +} + +void KafkaTopicMock::unsubscribe(const std::string& group_id, uint64_t consumer_id) { + lock_guard _(subscribers_mutex_); + auto iter = subscribers_.find(group_id); + if (iter != subscribers_.end()) { + iter->second.erase(consumer_id); + generate_assignments(group_id, iter->second); + } +} + +void KafkaTopicMock::generate_assignments(const string& group_id, + MembersMetadataMap& members_metadata) { + vector all_partitions(partitions_.size()); + iota(all_partitions.begin(), all_partitions.end(), 0); + + size_t chunk_size = all_partitions.size() / members_metadata.size(); + size_t member_index = 0; + for (auto& member_data_pair : members_metadata) { + MemberMetadata& member = member_data_pair.second; + if (!member.partitions_assigned.empty()) { + member.revocation_callback(member.partitions_assigned); + } + const size_t chunk_start = chunk_size * member_index; + // For the last one, add any remainders + if (member_index == members_metadata.size() - 1) { + chunk_size += all_partitions.size() % members_metadata.size(); + } + vector topic_partitions; + for (size_t i = 0; i < chunk_size; ++i) { + topic_partitions.emplace_back(name_, chunk_start + i); + } + topic_partitions = offset_manager_->get_offsets(group_id, move(topic_partitions)); + member.assignment_callback(topic_partitions); + member.partitions_assigned = move(topic_partitions); + member_index++; + } +} + } // mocking } // cppkafka diff --git a/mocking/src/offset_manager.cpp b/mocking/src/offset_manager.cpp new file mode 100644 index 00000000..d3a3acd1 --- /dev/null +++ b/mocking/src/offset_manager.cpp @@ -0,0 +1,46 @@ +#include +#include + +using std::string; +using std::vector; +using std::mutex; +using std::lock_guard; +using std::make_tuple; + +namespace cppkafka { +namespace mocking { + +void OffsetManager::commit_offsets(const string& group_id, + const vector& topic_partitions) { + lock_guard _(offsets_mutex_); + for (const TopicPartitionMock& topic_partition : topic_partitions) { + auto key = make_tuple(group_id, topic_partition.get_topic(), + topic_partition.get_partition()); + auto iter = offsets_.find(key); + if (iter == offsets_.end()) { + offsets_.emplace(key, topic_partition.get_offset()).first; + } + else { + iter->second = topic_partition.get_offset(); + } + } +} + +vector +OffsetManager::get_offsets(const string& group_id, + vector topic_partitions) const { + lock_guard _(offsets_mutex_); + for (TopicPartitionMock& topic_partition : topic_partitions) { + if (topic_partition.get_offset() != RD_KAFKA_OFFSET_INVALID) { + auto iter = offsets_.find(make_tuple(group_id, topic_partition.get_topic(), + topic_partition.get_partition())); + if (iter != offsets_.end()) { + topic_partition.set_offset(iter->second); + } + } + } + return topic_partitions; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/topic_partition_mock.cpp b/mocking/src/topic_partition_mock.cpp new file mode 100644 index 00000000..6397744c --- /dev/null +++ b/mocking/src/topic_partition_mock.cpp @@ -0,0 +1,30 @@ +#include + +using std::string; + +namespace cppkafka { +namespace mocking { + +TopicPartitionMock::TopicPartitionMock(string topic, int partition, int64_t offset) +: topic_(move(topic)), partition_(partition), offset_(offset) { + +} + +const string& TopicPartitionMock::get_topic() const { + return topic_; +} + +int TopicPartitionMock::get_partition() const { + return partition_; +} + +int64_t TopicPartitionMock::get_offset() const { + return offset_; +} + +void TopicPartitionMock::set_offset(int64_t offset) { + offset_ = offset; +} + +} // mocking +} // cppkafka From 45f494e1930e097e49c4aea02028984e6a195f8e Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Thu, 19 Oct 2017 21:01:06 -0700 Subject: [PATCH 09/20] Subscribe/unsubscribe from assigned partitions --- .../cppkafka/mocking/kafka_topic_mock.h | 6 ++++- mocking/src/kafka_topic_mock.cpp | 24 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/mocking/include/cppkafka/mocking/kafka_topic_mock.h b/mocking/include/cppkafka/mocking/kafka_topic_mock.h index 513d7ab7..cdab6f53 100644 --- a/mocking/include/cppkafka/mocking/kafka_topic_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_topic_mock.h @@ -22,19 +22,23 @@ class KafkaTopicMock { public: using AssignmentCallback = std::function&)>; using RevocationCallback = std::function&)>; + using MessageCallback = KafkaPartitionMock::MessageCallback; using OffsetManagerPtr = std::shared_ptr; KafkaTopicMock(std::string name, unsigned partition_count, OffsetManagerPtr offset_manager); void add_message(unsigned partition, KafkaMessageMock message); void subscribe(const std::string& group_id, uint64_t consumer_id, AssignmentCallback assignment_callback, - RevocationCallback revocation_callback); + RevocationCallback revocation_callback, + MessageCallback message_callback); void unsubscribe(const std::string& group_id, uint64_t consumer_id); private: struct MemberMetadata { const AssignmentCallback assignment_callback; const RevocationCallback revocation_callback; + const MessageCallback message_callback; std::vector partitions_assigned; + std::unordered_map partition_subscriptions; }; using MembersMetadataMap = std::unordered_map; diff --git a/mocking/src/kafka_topic_mock.cpp b/mocking/src/kafka_topic_mock.cpp index 4d168ebf..5d1a35fd 100644 --- a/mocking/src/kafka_topic_mock.cpp +++ b/mocking/src/kafka_topic_mock.cpp @@ -29,7 +29,8 @@ void KafkaTopicMock::add_message(unsigned partition, KafkaMessageMock message) { void KafkaTopicMock::subscribe(const string& group_id, uint64_t consumer_id, AssignmentCallback assignment_callback, - RevocationCallback revocation_callback) { + RevocationCallback revocation_callback, + MessageCallback message_callback) { lock_guard _(subscribers_mutex_); auto& members_data = subscribers_[group_id]; // If we're already subscribed, there's nothing to do @@ -37,7 +38,9 @@ void KafkaTopicMock::subscribe(const string& group_id, uint64_t consumer_id, return; } members_data.emplace(consumer_id, - MemberMetadata{move(assignment_callback), move(revocation_callback)}); + MemberMetadata{move(assignment_callback), + move(revocation_callback), + move(message_callback)}); generate_assignments(group_id, members_data); } @@ -59,9 +62,17 @@ void KafkaTopicMock::generate_assignments(const string& group_id, size_t member_index = 0; for (auto& member_data_pair : members_metadata) { MemberMetadata& member = member_data_pair.second; + // Execute revocation callback and unsubscribe from the partition object if (!member.partitions_assigned.empty()) { member.revocation_callback(member.partitions_assigned); + for (const auto& partition_subscription : member.partition_subscriptions) { + auto& partition_mock = partitions_[partition_subscription.first]; + partition_mock.unsubscribe(partition_subscription.second); + } } + member.partition_subscriptions.clear(); + + // Generate partition assignment const size_t chunk_start = chunk_size * member_index; // For the last one, add any remainders if (member_index == members_metadata.size() - 1) { @@ -71,9 +82,18 @@ void KafkaTopicMock::generate_assignments(const string& group_id, for (size_t i = 0; i < chunk_size; ++i) { topic_partitions.emplace_back(name_, chunk_start + i); } + // Try to load the offsets and execute assignment callback topic_partitions = offset_manager_->get_offsets(group_id, move(topic_partitions)); member.assignment_callback(topic_partitions); member.partitions_assigned = move(topic_partitions); + + // Subscribe to every assigned partition + for (const TopicPartitionMock& topic_partition : member.partitions_assigned) { + auto& partition_mock = partitions_[topic_partition.get_partition()]; + const auto subscriber_id = partition_mock.subscribe(member.message_callback); + member.partition_subscriptions.emplace(topic_partition.get_partition(), + subscriber_id); + } member_index++; } } From 7484c74c59f4839e6dbbf42511bdfcaa7f5118a8 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 21 Oct 2017 12:22:14 -0700 Subject: [PATCH 10/20] Add initial consumer mock code --- .../include/cppkafka/mocking/consumer_mock.h | 39 ++++++++++ .../include/cppkafka/mocking/kafka_cluster.h | 19 +++++ mocking/src/CMakeLists.txt | 1 + mocking/src/consumer_mock.cpp | 73 +++++++++++++++++++ mocking/src/kafka_cluster.cpp | 8 ++ 5 files changed, 140 insertions(+) create mode 100644 mocking/include/cppkafka/mocking/consumer_mock.h create mode 100644 mocking/src/consumer_mock.cpp diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h new file mode 100644 index 00000000..45342916 --- /dev/null +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -0,0 +1,39 @@ +#ifndef CPPKAFKA_MOCKING_CONSUMER_MOCK_H +#define CPPKAFKA_MOCKING_CONSUMER_MOCK_H + +#include +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class ConsumerMock : public HandleMock { +public: + ConsumerMock(ConfigurationMock config, EventProcessorPtr processor, ClusterPtr cluster); + ConsumerMock(const ConsumerMock&) = delete; + ConsumerMock& operator=(const ConsumerMock&) = delete; + ~ConsumerMock(); + + void subscribe(const std::vector& topics); +private: + static uint64_t make_consumer_id(); + + void on_assignment(std::vector& topic_partitions); + void on_revocation(const std::vector& topic_partitions); + void on_message(uint64_t offset); + + ConfigurationMock config_; + const std::string group_id_; + std::unordered_set subscribed_topics_; + uint64_t consumer_id_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_CONSUMER_MOCK_H diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index 24dceb8b..a3023902 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include #include @@ -21,15 +23,32 @@ class KafkaCluster { const std::string& get_url() const; void add_topic(const std::string& name, unsigned partitions); + bool topic_exists(const std::string& name) const; void produce(const std::string& topic, unsigned partition, KafkaMessageMock message); + template + void acquire_topic(const std::string& topic, const Functor& functor); private: KafkaCluster(std::string url); const std::string url_; std::shared_ptr offset_manager_; std::unordered_map topics_; + mutable std::mutex topics_mutex_; }; +template +void KafkaCluster::acquire_topic(const std::string& topic, const Functor& functor) { + std::unique_lock lock(topics_mutex_); + auto iter = topics_.find(topic); + if (iter == topics_.end()) { + throw std::runtime_error("Topic " + topic + " doesn't exist"); + } + // Unlock and execute callback. We won't remove topics so this is thread safe on a + // cluster level + lock.unlock(); + functor(iter->second); +} + } // mocking } // cppkafka diff --git a/mocking/src/CMakeLists.txt b/mocking/src/CMakeLists.txt index 4e855fe4..cb445652 100644 --- a/mocking/src/CMakeLists.txt +++ b/mocking/src/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES handle_mock.cpp message_handle.cpp producer_mock.cpp + consumer_mock.cpp topic_partition_mock.cpp offset_manager.cpp diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp new file mode 100644 index 00000000..6c9d679d --- /dev/null +++ b/mocking/src/consumer_mock.cpp @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +using std::atomic; +using std::vector; +using std::string; +using std::move; +using std::bind; +using std::runtime_error; + +namespace cppkafka { +namespace mocking { + +static const string CONFIG_GROUP_ID = "group.id"; + +uint64_t ConsumerMock::make_consumer_id() { + static atomic current_id{0}; + return current_id++; +} + +ConsumerMock::ConsumerMock(ConfigurationMock config, EventProcessorPtr processor, + ClusterPtr cluster) +: HandleMock(move(processor), move(cluster)), config_(move(config)), + consumer_id_(make_consumer_id()) { + if (!config_.has_key(CONFIG_GROUP_ID)) { + throw runtime_error("Failed to find " + CONFIG_GROUP_ID + " in config"); + } +} + +ConsumerMock::~ConsumerMock() { + auto& cluster = get_cluster(); + for (const string& topic_name : subscribed_topics_) { + cluster.acquire_topic(topic_name, [&](KafkaTopicMock& topic) { + topic.unsubscribe(group_id_, consumer_id_); + }); + } +} + +void ConsumerMock::subscribe(const vector& topics) { + using std::placeholders::_1; + auto& cluster = get_cluster(); + for (const string& topic_name : topics) { + if (subscribed_topics_.count(topic_name) > 0) { + continue; + } + cluster.acquire_topic(topic_name, [&](KafkaTopicMock& topic) { + topic.subscribe( + group_id_, + consumer_id_, + bind(&ConsumerMock::on_assignment, this, _1), + bind(&ConsumerMock::on_revocation, this, _1), + bind(&ConsumerMock::on_message, this, _1) + ); + }); + } +} + +void ConsumerMock::on_assignment(vector& topic_partitions) { + +} + +void ConsumerMock::on_revocation(const vector& topic_partitions) { + +} + +void ConsumerMock::on_message(uint64_t offset) { + +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index dcd6772e..051d6f9f 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -9,6 +9,8 @@ using std::invalid_argument; using std::piecewise_construct; using std::forward_as_tuple; using std::move; +using std::lock_guard; +using std::mutex; namespace cppkafka { namespace mocking { @@ -33,10 +35,16 @@ const string& KafkaCluster::get_url() const { } void KafkaCluster::add_topic(const string& name, unsigned partitions) { + lock_guard _(topics_mutex_); topics_.emplace(piecewise_construct, forward_as_tuple(name), forward_as_tuple(name, partitions, offset_manager_)); } +bool KafkaCluster::topic_exists(const string& name) const { + lock_guard _(topics_mutex_); + return topics_.count(name) > 0; +} + void KafkaCluster::produce(const string& topic, unsigned partition, KafkaMessageMock message) { auto iter = topics_.find(topic); if (iter == topics_.end()) { From 798254d2c58feef54a950ca10c9e5fbfb53a1813 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 21 Oct 2017 17:17:34 -0700 Subject: [PATCH 11/20] Handle assignment/revocation in consumer mock --- .../include/cppkafka/mocking/consumer_mock.h | 17 ++++++++ .../cppkafka/mocking/topic_partition_mock.h | 13 +++++- mocking/src/api.cpp | 1 + mocking/src/consumer_mock.cpp | 42 ++++++++++++++++++- mocking/src/topic_partition_mock.cpp | 28 +++++++++++++ 5 files changed, 97 insertions(+), 4 deletions(-) diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index 45342916..e5f50a3b 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -20,16 +22,31 @@ class ConsumerMock : public HandleMock { ~ConsumerMock(); void subscribe(const std::vector& topics); + void set_opaque(void* opaque); private: static uint64_t make_consumer_id(); + struct TopicPartitionInfo { + int64_t offset; + bool paused; + }; + + using TopicPartitionId = std::tuple; + + static TopicPartitionId make_id(const TopicPartitionMock& topic_partition); void on_assignment(std::vector& topic_partitions); void on_revocation(const std::vector& topic_partitions); void on_message(uint64_t offset); + template + void handle_rebalance(rd_kafka_resp_err_t type, List& topic_partitions); + void handle_assign(const TopicPartitionMock& topic_partition); + void handle_unassign(const TopicPartitionMock& topic_partition); ConfigurationMock config_; const std::string group_id_; std::unordered_set subscribed_topics_; + std::map assigned_partitions_; + void* opaque_; uint64_t consumer_id_; }; diff --git a/mocking/include/cppkafka/mocking/topic_partition_mock.h b/mocking/include/cppkafka/mocking/topic_partition_mock.h index f5f8676c..90cbf9de 100644 --- a/mocking/include/cppkafka/mocking/topic_partition_mock.h +++ b/mocking/include/cppkafka/mocking/topic_partition_mock.h @@ -3,6 +3,8 @@ #include #include +#include +#include #include namespace cppkafka { @@ -18,11 +20,18 @@ class TopicPartitionMock { void set_offset(int64_t offset); private: - std::string topic_; - int partition_; + const std::string topic_; + const int partition_; int64_t offset_; }; +using TopicPartitionMockListPtr = std::unique_ptr; +TopicPartitionMockListPtr +to_rdkafka_handle(const std::vector& topic_partitions); +std::vector +from_rdkafka_handle(const rd_kafka_topic_partition_list_t& topic_partitions); + } // mocking } // cppkafka diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp index 1e59111b..c6a3726b 100644 --- a/mocking/src/api.cpp +++ b/mocking/src/api.cpp @@ -201,6 +201,7 @@ rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list, output->topic = new char[length + 1]; copy(topic, topic + length, output->topic); output->partition = partition; + output->offset = RD_KAFKA_OFFSET_INVALID; return output; } diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index 6c9d679d..c9f22a6f 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -9,6 +9,7 @@ using std::string; using std::move; using std::bind; using std::runtime_error; +using std::make_tuple; namespace cppkafka { namespace mocking { @@ -57,16 +58,53 @@ void ConsumerMock::subscribe(const vector& topics) { } } -void ConsumerMock::on_assignment(vector& topic_partitions) { +void ConsumerMock::set_opaque(void* opaque) { + opaque_ = opaque; +} +ConsumerMock::TopicPartitionId ConsumerMock::make_id(const TopicPartitionMock& topic_partition) { + return make_tuple(topic_partition.get_topic(), topic_partition.get_partition()); } -void ConsumerMock::on_revocation(const vector& topic_partitions) { +void ConsumerMock::on_assignment(vector& topic_partitions) { + handle_rebalance(RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, topic_partitions); + for (const TopicPartitionMock& topic_partition : topic_partitions) { + handle_assign(topic_partition); + } +} +void ConsumerMock::on_revocation(const vector& topic_partitions) { + handle_rebalance(RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS, topic_partitions); + for (const TopicPartitionMock& topic_partition : topic_partitions) { + handle_unassign(topic_partition); + } } void ConsumerMock::on_message(uint64_t offset) { + +} + +template +void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type, List& topic_partitions) { + auto rebalance_callback = config_.get_rebalance_callback(); + if (rebalance_callback) { + auto handle = to_rdkafka_handle(topic_partitions); + rebalance_callback(nullptr, type, handle.get(), opaque_); + } +} + +void ConsumerMock::handle_assign(const TopicPartitionMock& topic_partition) { + const auto id = make_id(topic_partition); + if (assigned_partitions_.count(id)) { + return; + } + assigned_partitions_[id] = { + topic_partition.get_offset() + }; +} +void ConsumerMock::handle_unassign(const TopicPartitionMock& topic_partition) { + assigned_partitions_.erase(make_id(topic_partition)); } } // mocking diff --git a/mocking/src/topic_partition_mock.cpp b/mocking/src/topic_partition_mock.cpp index 6397744c..51f28746 100644 --- a/mocking/src/topic_partition_mock.cpp +++ b/mocking/src/topic_partition_mock.cpp @@ -1,6 +1,10 @@ +#include #include using std::string; +using std::vector; +using std::tie; +using std::unique_ptr; namespace cppkafka { namespace mocking { @@ -26,5 +30,29 @@ void TopicPartitionMock::set_offset(int64_t offset) { offset_ = offset; } +TopicPartitionMockListPtr to_rdkafka_handle(const vector& topic_partitions){ + const size_t count = topic_partitions.size(); + TopicPartitionMockListPtr output{rd_kafka_topic_partition_list_new(count), + &rd_kafka_topic_partition_list_destroy}; + for (const TopicPartitionMock& topic_partition : topic_partitions) { + auto* ptr = rd_kafka_topic_partition_list_add(output.get(), + topic_partition.get_topic().data(), + topic_partition.get_partition()); + ptr->offset = topic_partition.get_offset(); + } + return output; +} + +vector +from_rdkafka_handle(const rd_kafka_topic_partition_list_t& topic_partitions){ + vector output; + for (int i = 0; i < topic_partitions.cnt; ++i) { + const auto& topic_partition = topic_partitions.elems[i]; + output.emplace_back(topic_partition.topic, topic_partition.partition, + topic_partition.offset); + } + return output; +} + } // mocking } // cppkafka From 1c5e21a324fa817da7c64b73a3c98e052dab301d Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 21 Oct 2017 19:54:43 -0700 Subject: [PATCH 12/20] Handle message fetching --- .../include/cppkafka/mocking/consumer_mock.h | 29 ++++++- .../cppkafka/mocking/kafka_partition_mock.h | 5 +- .../cppkafka/mocking/kafka_topic_mock.h | 7 +- mocking/src/consumer_mock.cpp | 76 ++++++++++++++++--- mocking/src/kafka_partition_mock.cpp | 17 ++++- mocking/src/kafka_topic_mock.cpp | 29 ++++++- 6 files changed, 142 insertions(+), 21 deletions(-) diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index e5f50a3b..bfc5a66c 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -14,6 +17,9 @@ namespace cppkafka { namespace mocking { +class KafkaMessageMock; +class KafkaTopicMock; + class ConsumerMock : public HandleMock { public: ConsumerMock(ConfigurationMock config, EventProcessorPtr processor, ClusterPtr cluster); @@ -27,8 +33,17 @@ class ConsumerMock : public HandleMock { static uint64_t make_consumer_id(); struct TopicPartitionInfo { - int64_t offset; - bool paused; + TopicPartitionInfo(uint64_t next_offset); + + uint64_t next_offset; + bool paused{false}; + }; + + struct MessageAggregate { + std::string topic; + unsigned partition; + uint64_t offset; + const KafkaMessageMock* message; }; using TopicPartitionId = std::tuple; @@ -36,16 +51,24 @@ class ConsumerMock : public HandleMock { static TopicPartitionId make_id(const TopicPartitionMock& topic_partition); void on_assignment(std::vector& topic_partitions); void on_revocation(const std::vector& topic_partitions); - void on_message(uint64_t offset); + void on_message(const std::string& topic_name, unsigned partition, uint64_t offset, + const KafkaMessageMock* message); template void handle_rebalance(rd_kafka_resp_err_t type, List& topic_partitions); void handle_assign(const TopicPartitionMock& topic_partition); void handle_unassign(const TopicPartitionMock& topic_partition); + void fetch_existing_messages(unsigned partition, uint64_t next_offset, + KafkaTopicMock& topic); ConfigurationMock config_; const std::string group_id_; std::unordered_set subscribed_topics_; std::map assigned_partitions_; + mutable std::mutex assigned_partitions_mutex_; + std::queue new_message_queue_; + std::queue available_message_queue_; + mutable std::mutex message_queue_mutex_; + std::condition_variable message_queue_condition_; void* opaque_; uint64_t consumer_id_; }; diff --git a/mocking/include/cppkafka/mocking/kafka_partition_mock.h b/mocking/include/cppkafka/mocking/kafka_partition_mock.h index 5e287a06..3ebaf5fa 100644 --- a/mocking/include/cppkafka/mocking/kafka_partition_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_partition_mock.h @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace cppkafka { @@ -13,7 +14,7 @@ namespace mocking { class KafkaPartitionMock { public: - using MessageCallback = std::function; + using MessageCallback = std::function; using SubscriberId = uint64_t; void add_message(KafkaMessageMock message); @@ -21,6 +22,8 @@ class KafkaPartitionMock { size_t get_message_count() const; SubscriberId subscribe(MessageCallback callback); void unsubscribe(SubscriberId id); + // Returns interval [lowest offset, largest offset) + std::tuple get_offset_bounds() const; private: std::vector get_subscriber_callbacks() const; diff --git a/mocking/include/cppkafka/mocking/kafka_topic_mock.h b/mocking/include/cppkafka/mocking/kafka_topic_mock.h index cdab6f53..99a49d6b 100644 --- a/mocking/include/cppkafka/mocking/kafka_topic_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_topic_mock.h @@ -22,16 +22,21 @@ class KafkaTopicMock { public: using AssignmentCallback = std::function&)>; using RevocationCallback = std::function&)>; - using MessageCallback = KafkaPartitionMock::MessageCallback; + using MessageCallback = std::function; using OffsetManagerPtr = std::shared_ptr; KafkaTopicMock(std::string name, unsigned partition_count, OffsetManagerPtr offset_manager); + const std::string& get_name() const; + void add_message(unsigned partition, KafkaMessageMock message); void subscribe(const std::string& group_id, uint64_t consumer_id, AssignmentCallback assignment_callback, RevocationCallback revocation_callback, MessageCallback message_callback); void unsubscribe(const std::string& group_id, uint64_t consumer_id); + KafkaPartitionMock& get_partition(unsigned partition); + const KafkaPartitionMock& get_partition(unsigned partition) const; private: struct MemberMetadata { const AssignmentCallback assignment_callback; diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index c9f22a6f..682dacc7 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include using std::atomic; using std::vector; @@ -10,6 +12,10 @@ using std::move; using std::bind; using std::runtime_error; using std::make_tuple; +using std::tie; +using std::lock_guard; +using std::unique_lock; +using std::mutex; namespace cppkafka { namespace mocking { @@ -40,7 +46,7 @@ ConsumerMock::~ConsumerMock() { } void ConsumerMock::subscribe(const vector& topics) { - using std::placeholders::_1; + using namespace std::placeholders; auto& cluster = get_cluster(); for (const string& topic_name : topics) { if (subscribed_topics_.count(topic_name) > 0) { @@ -52,7 +58,7 @@ void ConsumerMock::subscribe(const vector& topics) { consumer_id_, bind(&ConsumerMock::on_assignment, this, _1), bind(&ConsumerMock::on_revocation, this, _1), - bind(&ConsumerMock::on_message, this, _1) + bind(&ConsumerMock::on_message, this, _1, _2, _3, _4) ); }); } @@ -80,8 +86,21 @@ void ConsumerMock::on_revocation(const vector& topic_partiti } } -void ConsumerMock::on_message(uint64_t offset) { - +void ConsumerMock::on_message(const string& topic_name, unsigned partition, uint64_t offset, + const KafkaMessageMock* message) { + // We should only process this if we don't have this topic/partition assigned (assignment + // pending?) or the message offset comes after the next offset we have stored + const bool valid_offset = [&]() { + lock_guard _(assigned_partitions_mutex_); + auto iter = assigned_partitions_.find(make_tuple(topic_name, partition)); + return iter == assigned_partitions_.end() || iter->second.next_offset >= offset; + }(); + if (!valid_offset) { + return; + } + unique_lock lock(message_queue_mutex_); + new_message_queue_.push({ topic_name, partition, offset, message }); + message_queue_condition_.notify_one(); } template @@ -95,17 +114,56 @@ void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type, List& topic_partit void ConsumerMock::handle_assign(const TopicPartitionMock& topic_partition) { const auto id = make_id(topic_partition); - if (assigned_partitions_.count(id)) { - return; + // We'll store the next offset from the one we've seen so far + const uint64_t next_offset = topic_partition.get_offset() + 1; + + { + lock_guard _(assigned_partitions_mutex_); + if (assigned_partitions_.count(id)) { + return; + } + assigned_partitions_.emplace(id, next_offset); } - assigned_partitions_[id] = { - topic_partition.get_offset() - }; + + // Fetch any existing messages and push them to the available message queue + auto& cluster = get_cluster(); + cluster.acquire_topic(topic_partition.get_topic(), [&](KafkaTopicMock& topic) { + fetch_existing_messages(topic_partition.get_partition(), next_offset, topic); + }); } void ConsumerMock::handle_unassign(const TopicPartitionMock& topic_partition) { + lock_guard _(assigned_partitions_mutex_); assigned_partitions_.erase(make_id(topic_partition)); } +void ConsumerMock::fetch_existing_messages(unsigned partition_index, uint64_t next_offset, + KafkaTopicMock& topic) { + KafkaPartitionMock& partition = topic.get_partition(partition_index); + uint64_t start_offset; + uint64_t end_offset; + tie(start_offset, end_offset) = partition.get_offset_bounds(); + if (start_offset < next_offset) { + throw runtime_error("Stored offset is too high"); + } + // Nothing to fetch + if (next_offset == end_offset) { + return; + } + unique_lock lock(message_queue_mutex_); + for (uint64_t i = next_offset; i != end_offset; ++i) { + const KafkaMessageMock& message = partition.get_message(i); + available_message_queue_.push({ topic.get_name(), partition_index, i, &message }); + } + message_queue_condition_.notify_all(); +} + +// TopicPartitionInfo + +ConsumerMock::TopicPartitionInfo::TopicPartitionInfo(uint64_t next_offset) +: next_offset(next_offset) { + +} + } // mocking } // cppkafka diff --git a/mocking/src/kafka_partition_mock.cpp b/mocking/src/kafka_partition_mock.cpp index 5a6229e7..06abfe18 100644 --- a/mocking/src/kafka_partition_mock.cpp +++ b/mocking/src/kafka_partition_mock.cpp @@ -1,4 +1,5 @@ #include +#include #include using std::vector; @@ -6,20 +7,25 @@ using std::lock_guard; using std::mutex; using std::out_of_range; using std::move; +using std::tuple; +using std::tie; +using std::make_tuple; namespace cppkafka { namespace mocking { void KafkaPartitionMock::add_message(KafkaMessageMock message) { - const uint64_t offset = [&] { + KafkaMessageMock* message_ptr; + uint64_t offset; + tie(message_ptr, offset) = [&] { lock_guard _(messages_mutex_); messages_.emplace_back(move(message)); - return messages_.size() - 1; + return make_tuple(&messages_.back(), messages_.size() - 1); }(); const vector callbacks = get_subscriber_callbacks(); for (const MessageCallback& callback : callbacks) { - callback(offset); + callback(offset, message_ptr); } } @@ -49,6 +55,11 @@ void KafkaPartitionMock::unsubscribe(SubscriberId id) { subscribers_.erase(id); } +tuple KafkaPartitionMock::get_offset_bounds() const { + lock_guard _(messages_mutex_); + return make_tuple(base_offset_, base_offset_ + messages_.size()); +} + vector KafkaPartitionMock::get_subscriber_callbacks() const { lock_guard _(subscribers_mutex_); vector output; diff --git a/mocking/src/kafka_topic_mock.cpp b/mocking/src/kafka_topic_mock.cpp index 5d1a35fd..d587305c 100644 --- a/mocking/src/kafka_topic_mock.cpp +++ b/mocking/src/kafka_topic_mock.cpp @@ -4,12 +4,14 @@ #include using std::string; -using std::out_of_range; using std::move; using std::lock_guard; using std::mutex; using std::vector; using std::iota; +using std::bind; +using std::out_of_range; +using std::runtime_error; namespace cppkafka { namespace mocking { @@ -20,6 +22,10 @@ KafkaTopicMock::KafkaTopicMock(string name, unsigned partition_count, } +const string& KafkaTopicMock::get_name() const { + return name_; +} + void KafkaTopicMock::add_message(unsigned partition, KafkaMessageMock message) { if (partitions_.size() >= partition) { throw out_of_range("invalid partition index"); @@ -53,6 +59,17 @@ void KafkaTopicMock::unsubscribe(const std::string& group_id, uint64_t consumer_ } } +KafkaPartitionMock& KafkaTopicMock::get_partition(unsigned partition) { + if (partition >= partitions_.size()) { + throw runtime_error("Partition doesn't exist"); + } + return partitions_[partition]; +} + +const KafkaPartitionMock& KafkaTopicMock::get_partition(unsigned partition) const { + return const_cast(*this).get_partition(partition); +} + void KafkaTopicMock::generate_assignments(const string& group_id, MembersMetadataMap& members_metadata) { vector all_partitions(partitions_.size()); @@ -82,18 +99,22 @@ void KafkaTopicMock::generate_assignments(const string& group_id, for (size_t i = 0; i < chunk_size; ++i) { topic_partitions.emplace_back(name_, chunk_start + i); } - // Try to load the offsets and execute assignment callback + // Try to load the offsets and store the topic partitions topic_partitions = offset_manager_->get_offsets(group_id, move(topic_partitions)); - member.assignment_callback(topic_partitions); member.partitions_assigned = move(topic_partitions); // Subscribe to every assigned partition + using namespace std::placeholders; for (const TopicPartitionMock& topic_partition : member.partitions_assigned) { auto& partition_mock = partitions_[topic_partition.get_partition()]; - const auto subscriber_id = partition_mock.subscribe(member.message_callback); + auto callback = bind(member.message_callback, topic_partition.get_topic(), + topic_partition.get_partition(), _1, _2); + const auto subscriber_id = partition_mock.subscribe(move(callback)); member.partition_subscriptions.emplace(topic_partition.get_partition(), subscriber_id); } + // Now trigger the assignment callback + member.assignment_callback(member.partitions_assigned); member_index++; } } From 2034ce505345356bb444ddef904e69916c3ed61b Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sun, 22 Oct 2017 12:01:01 -0700 Subject: [PATCH 13/20] Handle (un)subscribe at the cluster level --- .../include/cppkafka/mocking/consumer_mock.h | 1 - .../include/cppkafka/mocking/kafka_cluster.h | 38 ++++- .../cppkafka/mocking/kafka_topic_mock.h | 28 +--- mocking/src/consumer_mock.cpp | 30 +--- mocking/src/kafka_cluster.cpp | 156 +++++++++++++++++- mocking/src/kafka_topic_mock.cpp | 85 +--------- 6 files changed, 207 insertions(+), 131 deletions(-) diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index bfc5a66c..10cfb3ac 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -62,7 +62,6 @@ class ConsumerMock : public HandleMock { ConfigurationMock config_; const std::string group_id_; - std::unordered_set subscribed_topics_; std::map assigned_partitions_; mutable std::mutex assigned_partitions_mutex_; std::queue new_message_queue_; diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index a3023902..83de6853 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -2,8 +2,10 @@ #define CPPKAFKA_MOCKING_KAFKA_CLUSTER_H #include +#include #include #include +#include #include #include #include @@ -14,6 +16,11 @@ namespace mocking { class KafkaCluster { public: + using AssignmentCallback = std::function&)>; + using RevocationCallback = std::function&)>; + using MessageCallback = std::function; + static std::shared_ptr make_cluster(std::string url); KafkaCluster(const KafkaCluster&) = delete; @@ -22,18 +29,47 @@ class KafkaCluster { const std::string& get_url() const; - void add_topic(const std::string& name, unsigned partitions); + void create_topic(const std::string& name, unsigned partitions); bool topic_exists(const std::string& name) const; void produce(const std::string& topic, unsigned partition, KafkaMessageMock message); template void acquire_topic(const std::string& topic, const Functor& functor); + KafkaTopicMock& get_topic(const std::string& name); + const KafkaTopicMock& get_topic(const std::string& name) const; + void subscribe(const std::string& group_id, uint64_t consumer_id, + const std::vector& topics, + AssignmentCallback assignment_callback, + RevocationCallback revocation_callback, + MessageCallback message_callback); + void unsubscribe(const std::string& group_id, uint64_t consumer_id); private: + struct ConsumerMetadata { + using PartitionSubscriptionMap = std::unordered_map; + + const AssignmentCallback assignment_callback; + const RevocationCallback revocation_callback; + const MessageCallback message_callback; + std::vector partitions_assigned; + std::unordered_map subscriptions; + }; + + using ConsumerSet = std::unordered_set; + using TopicConsumersMap = std::unordered_map; + KafkaCluster(std::string url); + void generate_assignments(const std::string& group_id, + const TopicConsumersMap& topic_consumers); + void generate_revocations(const TopicConsumersMap& topic_consumers); + void do_unsubscribe(const std::string& group_id, uint64_t consumer_id); + const std::string url_; std::shared_ptr offset_manager_; std::unordered_map topics_; mutable std::mutex topics_mutex_; + std::unordered_map consumer_data_; + std::unordered_map group_topics_data_; + mutable std::mutex consumer_data_mutex_; }; template diff --git a/mocking/include/cppkafka/mocking/kafka_topic_mock.h b/mocking/include/cppkafka/mocking/kafka_topic_mock.h index 99a49d6b..2f7dd3c9 100644 --- a/mocking/include/cppkafka/mocking/kafka_topic_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_topic_mock.h @@ -10,8 +10,6 @@ #include #include #include -#include -#include namespace cppkafka { namespace mocking { @@ -20,39 +18,17 @@ class KafkaMessageMock; class KafkaTopicMock { public: - using AssignmentCallback = std::function&)>; - using RevocationCallback = std::function&)>; - using MessageCallback = std::function; - using OffsetManagerPtr = std::shared_ptr; - - KafkaTopicMock(std::string name, unsigned partition_count, OffsetManagerPtr offset_manager); + KafkaTopicMock(std::string name, unsigned partition_count); const std::string& get_name() const; void add_message(unsigned partition, KafkaMessageMock message); - void subscribe(const std::string& group_id, uint64_t consumer_id, - AssignmentCallback assignment_callback, - RevocationCallback revocation_callback, - MessageCallback message_callback); - void unsubscribe(const std::string& group_id, uint64_t consumer_id); KafkaPartitionMock& get_partition(unsigned partition); const KafkaPartitionMock& get_partition(unsigned partition) const; + size_t get_partition_count() const; private: - struct MemberMetadata { - const AssignmentCallback assignment_callback; - const RevocationCallback revocation_callback; - const MessageCallback message_callback; - std::vector partitions_assigned; - std::unordered_map partition_subscriptions; - }; - using MembersMetadataMap = std::unordered_map; - - void generate_assignments(const std::string& group_id, MembersMetadataMap& members_metadata); const std::string name_; std::vector partitions_; - OffsetManagerPtr offset_manager_; - std::unordered_map subscribers_; mutable std::mutex subscribers_mutex_; }; diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index 682dacc7..cde473a5 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -37,31 +37,19 @@ ConsumerMock::ConsumerMock(ConfigurationMock config, EventProcessorPtr processor } ConsumerMock::~ConsumerMock() { - auto& cluster = get_cluster(); - for (const string& topic_name : subscribed_topics_) { - cluster.acquire_topic(topic_name, [&](KafkaTopicMock& topic) { - topic.unsubscribe(group_id_, consumer_id_); - }); - } + get_cluster().unsubscribe(group_id_, consumer_id_); } void ConsumerMock::subscribe(const vector& topics) { using namespace std::placeholders; - auto& cluster = get_cluster(); - for (const string& topic_name : topics) { - if (subscribed_topics_.count(topic_name) > 0) { - continue; - } - cluster.acquire_topic(topic_name, [&](KafkaTopicMock& topic) { - topic.subscribe( - group_id_, - consumer_id_, - bind(&ConsumerMock::on_assignment, this, _1), - bind(&ConsumerMock::on_revocation, this, _1), - bind(&ConsumerMock::on_message, this, _1, _2, _3, _4) - ); - }); - } + get_cluster().subscribe( + group_id_, + consumer_id_, + topics, + bind(&ConsumerMock::on_assignment, this, _1), + bind(&ConsumerMock::on_revocation, this, _1), + bind(&ConsumerMock::on_message, this, _1, _2, _3, _4) + ); } void ConsumerMock::set_opaque(void* opaque) { diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index 051d6f9f..ed0c5d3f 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -1,16 +1,21 @@ #include +#include #include #include +#include using std::shared_ptr; using std::make_shared; using std::string; +using std::vector; using std::invalid_argument; +using std::runtime_error; using std::piecewise_construct; using std::forward_as_tuple; using std::move; using std::lock_guard; using std::mutex; +using std::iota; namespace cppkafka { namespace mocking { @@ -34,10 +39,10 @@ const string& KafkaCluster::get_url() const { return url_; } -void KafkaCluster::add_topic(const string& name, unsigned partitions) { +void KafkaCluster::create_topic(const string& name, unsigned partitions) { lock_guard _(topics_mutex_); topics_.emplace(piecewise_construct, forward_as_tuple(name), - forward_as_tuple(name, partitions, offset_manager_)); + forward_as_tuple(name, partitions)); } bool KafkaCluster::topic_exists(const string& name) const { @@ -53,5 +58,152 @@ void KafkaCluster::produce(const string& topic, unsigned partition, KafkaMessage iter->second.add_message(partition, move(message)); } +KafkaTopicMock& KafkaCluster::get_topic(const string& name) { + lock_guard _(topics_mutex_); + auto iter = topics_.find(name); + if (iter == topics_.end()) { + throw runtime_error("Topic " + name + " doesn't exist"); + } + return iter->second; +} + +const KafkaTopicMock& KafkaCluster::get_topic(const string& name) const { + return const_cast(*this).get_topic(name); +} + +void KafkaCluster::subscribe(const string& group_id, uint64_t consumer_id, + const vector& topics, + AssignmentCallback assignment_callback, + RevocationCallback revocation_callback, + MessageCallback message_callback) { + lock_guard _(consumer_data_mutex_); + auto iter = consumer_data_.find(consumer_id); + // If it's already subscribed to something, unsubscribe from it + if (iter != consumer_data_.end()) { + do_unsubscribe(group_id, consumer_id); + } + ConsumerMetadata data = { + move(assignment_callback), + move(revocation_callback), + move(message_callback), + }; + iter = consumer_data_.emplace(consumer_id, move(data)).first; + + auto& group_data = group_topics_data_[group_id]; + for (const string& topic : topics) { + group_data[topic].emplace(consumer_id); + } + // First revoke any assignment that involve consumers in this group + generate_revocations(group_data); + // Now generate the assignments + generate_assignments(group_id, group_data); +} + +void KafkaCluster::unsubscribe(const string& group_id, uint64_t consumer_id) { + lock_guard _(consumer_data_mutex_); + do_unsubscribe(group_id, consumer_id); +} + +void KafkaCluster::generate_assignments(const string& group_id, + const TopicConsumersMap& topic_consumers) { + for (const auto& topic_consumers_pair : topic_consumers) { + const string& topic_name = topic_consumers_pair.first; + const ConsumerSet& consumers = topic_consumers_pair.second; + KafkaTopicMock& topic = get_topic(topic_name); + vector all_partitions(topic.get_partition_count()); + iota(all_partitions.begin(), all_partitions.end(), 0); + + size_t chunk_size = all_partitions.size() / consumers.size(); + size_t consumer_index = 0; + for (const uint64_t consumer_id : consumers) { + ConsumerMetadata& consumer = consumer_data_[consumer_id]; + + // Generate partition assignment + const size_t chunk_start = chunk_size * consumer_index; + // For the last one, add any remainders + if (consumer_index == consumers.size() - 1) { + chunk_size += all_partitions.size() % consumers.size(); + } + for (size_t i = 0; i < chunk_size; ++i) { + consumer.partitions_assigned.emplace_back(topic_name, chunk_start + i); + } + + // Subscribe to every assigned partition + using namespace std::placeholders; + for (const TopicPartitionMock& topic_partition : consumer.partitions_assigned) { + auto& partition_mock = topic.get_partition(topic_partition.get_partition()); + auto callback = bind(consumer.message_callback, topic_partition.get_topic(), + topic_partition.get_partition(), _1, _2); + const auto subscriber_id = partition_mock.subscribe(move(callback)); + consumer.subscriptions[&topic].emplace(topic_partition.get_partition(), + subscriber_id); + } + // Now trigger the assignment callback + consumer_index++; + } + } + // Now do another pass and trigger the assignment callbacks + for (const auto& topic_consumers_pair : topic_consumers) { + for (const uint64_t consumer_id : topic_consumers_pair.second) { + ConsumerMetadata& consumer = consumer_data_[consumer_id]; + // Try to load the offsets for this consumer, and store the updated offsets and + // trigger the assignment callback + auto partitions_assigned = move(consumer.partitions_assigned); + consumer.partitions_assigned = offset_manager_->get_offsets(group_id, + move(partitions_assigned)); + consumer.assignment_callback(consumer.partitions_assigned); + } + } +} + +void KafkaCluster::generate_revocations(const TopicConsumersMap& topic_consumers) { + for (const auto& topic_consumers_pair : topic_consumers) { + const ConsumerSet& consumers = topic_consumers_pair.second; + for (const uint64_t consumer_id : consumers) { + ConsumerMetadata& consumer = consumer_data_[consumer_id]; + // Execute revocation callback and unsubscribe from the partition object + if (!consumer.partitions_assigned.empty()) { + consumer.revocation_callback(consumer.partitions_assigned); + for (const auto& topic_subscription : consumer.subscriptions) { + KafkaTopicMock& topic = *topic_subscription.first; + for (const auto& partition_subscription : topic_subscription.second) { + auto& partition_mock = topic.get_partition(partition_subscription.first); + partition_mock.unsubscribe(partition_subscription.second); + } + } + consumer.partitions_assigned.clear(); + } + consumer.subscriptions.clear(); + } + } +} + +void KafkaCluster::do_unsubscribe(const string& group_id, uint64_t consumer_id) { + auto iter = consumer_data_.find(consumer_id); + if (iter == consumer_data_.end()) { + return; + } + auto& group_data = group_topics_data_[group_id]; + // Revoke for all consumers + generate_revocations(group_data); + + for (const auto& topic_subscription : iter->second.subscriptions) { + const string& topic_name = topic_subscription.first->get_name(); + auto& topic_data = group_data[topic_name]; + topic_data.erase(consumer_id); + if (topic_data.empty()) { + group_data.erase(topic_name); + } + } + if (group_data.empty()) { + // If we ran out of consumers for this group, erase it + group_topics_data_.erase(group_id); + } + else { + // Otherwise re-generate the assignments + generate_assignments(group_id, group_data); + } +} + } // mocking } // cppkafka diff --git a/mocking/src/kafka_topic_mock.cpp b/mocking/src/kafka_topic_mock.cpp index d587305c..3d2231d9 100644 --- a/mocking/src/kafka_topic_mock.cpp +++ b/mocking/src/kafka_topic_mock.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -8,17 +7,14 @@ using std::move; using std::lock_guard; using std::mutex; using std::vector; -using std::iota; -using std::bind; using std::out_of_range; using std::runtime_error; namespace cppkafka { namespace mocking { -KafkaTopicMock::KafkaTopicMock(string name, unsigned partition_count, - OffsetManagerPtr offset_manager) -: name_(move(name)), partitions_(partition_count), offset_manager_(move(offset_manager)) { +KafkaTopicMock::KafkaTopicMock(string name, unsigned partition_count) +: name_(move(name)), partitions_(partition_count) { } @@ -33,35 +29,9 @@ void KafkaTopicMock::add_message(unsigned partition, KafkaMessageMock message) { partitions_[partition].add_message(move(message)); } -void KafkaTopicMock::subscribe(const string& group_id, uint64_t consumer_id, - AssignmentCallback assignment_callback, - RevocationCallback revocation_callback, - MessageCallback message_callback) { - lock_guard _(subscribers_mutex_); - auto& members_data = subscribers_[group_id]; - // If we're already subscribed, there's nothing to do - if (members_data.count(consumer_id)) { - return; - } - members_data.emplace(consumer_id, - MemberMetadata{move(assignment_callback), - move(revocation_callback), - move(message_callback)}); - generate_assignments(group_id, members_data); -} - -void KafkaTopicMock::unsubscribe(const std::string& group_id, uint64_t consumer_id) { - lock_guard _(subscribers_mutex_); - auto iter = subscribers_.find(group_id); - if (iter != subscribers_.end()) { - iter->second.erase(consumer_id); - generate_assignments(group_id, iter->second); - } -} - KafkaPartitionMock& KafkaTopicMock::get_partition(unsigned partition) { if (partition >= partitions_.size()) { - throw runtime_error("Partition doesn't exist"); + throw runtime_error("partition doesn't exist"); } return partitions_[partition]; } @@ -70,53 +40,8 @@ const KafkaPartitionMock& KafkaTopicMock::get_partition(unsigned partition) cons return const_cast(*this).get_partition(partition); } -void KafkaTopicMock::generate_assignments(const string& group_id, - MembersMetadataMap& members_metadata) { - vector all_partitions(partitions_.size()); - iota(all_partitions.begin(), all_partitions.end(), 0); - - size_t chunk_size = all_partitions.size() / members_metadata.size(); - size_t member_index = 0; - for (auto& member_data_pair : members_metadata) { - MemberMetadata& member = member_data_pair.second; - // Execute revocation callback and unsubscribe from the partition object - if (!member.partitions_assigned.empty()) { - member.revocation_callback(member.partitions_assigned); - for (const auto& partition_subscription : member.partition_subscriptions) { - auto& partition_mock = partitions_[partition_subscription.first]; - partition_mock.unsubscribe(partition_subscription.second); - } - } - member.partition_subscriptions.clear(); - - // Generate partition assignment - const size_t chunk_start = chunk_size * member_index; - // For the last one, add any remainders - if (member_index == members_metadata.size() - 1) { - chunk_size += all_partitions.size() % members_metadata.size(); - } - vector topic_partitions; - for (size_t i = 0; i < chunk_size; ++i) { - topic_partitions.emplace_back(name_, chunk_start + i); - } - // Try to load the offsets and store the topic partitions - topic_partitions = offset_manager_->get_offsets(group_id, move(topic_partitions)); - member.partitions_assigned = move(topic_partitions); - - // Subscribe to every assigned partition - using namespace std::placeholders; - for (const TopicPartitionMock& topic_partition : member.partitions_assigned) { - auto& partition_mock = partitions_[topic_partition.get_partition()]; - auto callback = bind(member.message_callback, topic_partition.get_topic(), - topic_partition.get_partition(), _1, _2); - const auto subscriber_id = partition_mock.subscribe(move(callback)); - member.partition_subscriptions.emplace(topic_partition.get_partition(), - subscriber_id); - } - // Now trigger the assignment callback - member.assignment_callback(member.partitions_assigned); - member_index++; - } +size_t KafkaTopicMock::get_partition_count() const { + return partitions_.size(); } } // mocking From 8480cf3b1e74bfabed29bfe5da95fa06e251fa32 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sun, 22 Oct 2017 15:31:36 -0700 Subject: [PATCH 14/20] Handle (un)assign on consumer mock --- .../include/cppkafka/mocking/consumer_mock.h | 12 +++-- .../include/cppkafka/mocking/kafka_cluster.h | 19 +------ mocking/src/consumer_mock.cpp | 49 ++++++++++++------- mocking/src/kafka_cluster.cpp | 2 +- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index 10cfb3ac..2c45b5bb 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -28,6 +28,8 @@ class ConsumerMock : public HandleMock { ~ConsumerMock(); void subscribe(const std::vector& topics); + void assign(const std::vector& topic_partitions); + void unassign(); void set_opaque(void* opaque); private: static uint64_t make_consumer_id(); @@ -49,14 +51,14 @@ class ConsumerMock : public HandleMock { using TopicPartitionId = std::tuple; static TopicPartitionId make_id(const TopicPartitionMock& topic_partition); - void on_assignment(std::vector& topic_partitions); - void on_revocation(const std::vector& topic_partitions); + void on_assignment(const std::vector& topic_partitions); + void on_revocation(); void on_message(const std::string& topic_name, unsigned partition, uint64_t offset, const KafkaMessageMock* message); - template - void handle_rebalance(rd_kafka_resp_err_t type, List& topic_partitions); + void handle_rebalance(rd_kafka_resp_err_t type, + const std::vector& topic_partitions); void handle_assign(const TopicPartitionMock& topic_partition); - void handle_unassign(const TopicPartitionMock& topic_partition); + void handle_unassign(); void fetch_existing_messages(unsigned partition, uint64_t next_offset, KafkaTopicMock& topic); diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index 83de6853..2fb0548a 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -16,8 +16,8 @@ namespace mocking { class KafkaCluster { public: - using AssignmentCallback = std::function&)>; - using RevocationCallback = std::function&)>; + using AssignmentCallback = std::function&)>; + using RevocationCallback = std::function; using MessageCallback = std::function; @@ -32,8 +32,6 @@ class KafkaCluster { void create_topic(const std::string& name, unsigned partitions); bool topic_exists(const std::string& name) const; void produce(const std::string& topic, unsigned partition, KafkaMessageMock message); - template - void acquire_topic(const std::string& topic, const Functor& functor); KafkaTopicMock& get_topic(const std::string& name); const KafkaTopicMock& get_topic(const std::string& name) const; void subscribe(const std::string& group_id, uint64_t consumer_id, @@ -72,19 +70,6 @@ class KafkaCluster { mutable std::mutex consumer_data_mutex_; }; -template -void KafkaCluster::acquire_topic(const std::string& topic, const Functor& functor) { - std::unique_lock lock(topics_mutex_); - auto iter = topics_.find(topic); - if (iter == topics_.end()) { - throw std::runtime_error("Topic " + topic + " doesn't exist"); - } - // Unlock and execute callback. We won't remove topics so this is thread safe on a - // cluster level - lock.unlock(); - functor(iter->second); -} - } // mocking } // cppkafka diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index cde473a5..66d98308 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -13,6 +13,7 @@ using std::bind; using std::runtime_error; using std::make_tuple; using std::tie; +using std::get; using std::lock_guard; using std::unique_lock; using std::mutex; @@ -47,11 +48,22 @@ void ConsumerMock::subscribe(const vector& topics) { consumer_id_, topics, bind(&ConsumerMock::on_assignment, this, _1), - bind(&ConsumerMock::on_revocation, this, _1), + bind(&ConsumerMock::on_revocation, this), bind(&ConsumerMock::on_message, this, _1, _2, _3, _4) ); } +void ConsumerMock::assign(const vector& topic_partitions) { + for (const TopicPartitionMock& topic_partition : topic_partitions) { + handle_assign(topic_partition); + } +} + +void ConsumerMock::unassign() { + lock_guard _(assigned_partitions_mutex_); + assigned_partitions_.clear(); +} + void ConsumerMock::set_opaque(void* opaque) { opaque_ = opaque; } @@ -60,18 +72,23 @@ ConsumerMock::TopicPartitionId ConsumerMock::make_id(const TopicPartitionMock& t return make_tuple(topic_partition.get_topic(), topic_partition.get_partition()); } -void ConsumerMock::on_assignment(vector& topic_partitions) { +void ConsumerMock::on_assignment(const vector& topic_partitions) { handle_rebalance(RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, topic_partitions); - for (const TopicPartitionMock& topic_partition : topic_partitions) { - handle_assign(topic_partition); - } } -void ConsumerMock::on_revocation(const vector& topic_partitions) { +void ConsumerMock::on_revocation() { + // Fetch and reset all assigned topic partitions + vector topic_partitions = [&]() { + lock_guard _(assigned_partitions_mutex_); + vector output; + for (const auto& topic_partition_pair : assigned_partitions_) { + const TopicPartitionId& id = topic_partition_pair.first; + output.emplace_back(get<0>(id), get<1>(id)); + } + assigned_partitions_.clear(); + return output; + }(); handle_rebalance(RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS, topic_partitions); - for (const TopicPartitionMock& topic_partition : topic_partitions) { - handle_unassign(topic_partition); - } } void ConsumerMock::on_message(const string& topic_name, unsigned partition, uint64_t offset, @@ -91,8 +108,8 @@ void ConsumerMock::on_message(const string& topic_name, unsigned partition, uint message_queue_condition_.notify_one(); } -template -void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type, List& topic_partitions) { +void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type, + const vector& topic_partitions) { auto rebalance_callback = config_.get_rebalance_callback(); if (rebalance_callback) { auto handle = to_rdkafka_handle(topic_partitions); @@ -115,14 +132,8 @@ void ConsumerMock::handle_assign(const TopicPartitionMock& topic_partition) { // Fetch any existing messages and push them to the available message queue auto& cluster = get_cluster(); - cluster.acquire_topic(topic_partition.get_topic(), [&](KafkaTopicMock& topic) { - fetch_existing_messages(topic_partition.get_partition(), next_offset, topic); - }); -} - -void ConsumerMock::handle_unassign(const TopicPartitionMock& topic_partition) { - lock_guard _(assigned_partitions_mutex_); - assigned_partitions_.erase(make_id(topic_partition)); + KafkaTopicMock& topic = cluster.get_topic(topic_partition.get_topic()); + fetch_existing_messages(topic_partition.get_partition(), next_offset, topic); } void ConsumerMock::fetch_existing_messages(unsigned partition_index, uint64_t next_offset, diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index ed0c5d3f..271756aa 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -163,7 +163,7 @@ void KafkaCluster::generate_revocations(const TopicConsumersMap& topic_consumers ConsumerMetadata& consumer = consumer_data_[consumer_id]; // Execute revocation callback and unsubscribe from the partition object if (!consumer.partitions_assigned.empty()) { - consumer.revocation_callback(consumer.partitions_assigned); + consumer.revocation_callback(); for (const auto& topic_subscription : consumer.subscriptions) { KafkaTopicMock& topic = *topic_subscription.first; for (const auto& partition_subscription : topic_subscription.second) { From 41ce47f6a10e05b7e02738c70b8df3a6b2c6261e Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Wed, 25 Oct 2017 19:37:31 -0700 Subject: [PATCH 15/20] Handle assignment, pause and resume on consumer mock --- .../include/cppkafka/mocking/consumer_mock.h | 36 ++-- .../include/cppkafka/mocking/kafka_cluster.h | 9 +- .../cppkafka/mocking/kafka_partition_mock.h | 15 +- mocking/src/consumer_mock.cpp | 198 ++++++++++++------ mocking/src/kafka_cluster.cpp | 81 +++++-- mocking/src/kafka_partition_mock.cpp | 19 +- 6 files changed, 240 insertions(+), 118 deletions(-) diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index 2c45b5bb..a51dfdaa 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -5,14 +5,18 @@ #include #include #include +#include #include #include #include #include #include +#include +#include #include #include #include +#include namespace cppkafka { namespace mocking { @@ -31,16 +35,12 @@ class ConsumerMock : public HandleMock { void assign(const std::vector& topic_partitions); void unassign(); void set_opaque(void* opaque); + void pause_partitions(const std::vector& topic_partitions); + void resume_partitions(const std::vector& topic_partitions); + boost::optional poll(std::chrono::milliseconds timeout); private: static uint64_t make_consumer_id(); - struct TopicPartitionInfo { - TopicPartitionInfo(uint64_t next_offset); - - uint64_t next_offset; - bool paused{false}; - }; - struct MessageAggregate { std::string topic; unsigned partition; @@ -48,28 +48,30 @@ class ConsumerMock : public HandleMock { const KafkaMessageMock* message; }; + struct TopicPartitionInfo { + uint64_t next_offset; + std::queue messages; + }; + using TopicPartitionId = std::tuple; static TopicPartitionId make_id(const TopicPartitionMock& topic_partition); void on_assignment(const std::vector& topic_partitions); void on_revocation(); void on_message(const std::string& topic_name, unsigned partition, uint64_t offset, - const KafkaMessageMock* message); + const KafkaMessageMock& message); void handle_rebalance(rd_kafka_resp_err_t type, const std::vector& topic_partitions); - void handle_assign(const TopicPartitionMock& topic_partition); - void handle_unassign(); - void fetch_existing_messages(unsigned partition, uint64_t next_offset, - KafkaTopicMock& topic); ConfigurationMock config_; + // TODO: initialize this and make it const + bool emit_eofs_; const std::string group_id_; std::map assigned_partitions_; - mutable std::mutex assigned_partitions_mutex_; - std::queue new_message_queue_; - std::queue available_message_queue_; - mutable std::mutex message_queue_mutex_; - std::condition_variable message_queue_condition_; + std::set consumable_topic_partitions_; + std::set paused_topic_partitions_; + mutable std::mutex mutex_; + std::condition_variable messages_condition_; void* opaque_; uint64_t consumer_id_; }; diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index 2fb0548a..e80bd4f5 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -19,7 +19,7 @@ class KafkaCluster { using AssignmentCallback = std::function&)>; using RevocationCallback = std::function; using MessageCallback = std::function; + uint64_t offset, const KafkaMessageMock&)>; static std::shared_ptr make_cluster(std::string url); @@ -37,16 +37,17 @@ class KafkaCluster { void subscribe(const std::string& group_id, uint64_t consumer_id, const std::vector& topics, AssignmentCallback assignment_callback, - RevocationCallback revocation_callback, - MessageCallback message_callback); + RevocationCallback revocation_callback); void unsubscribe(const std::string& group_id, uint64_t consumer_id); + void assign(uint64_t consumer_id, const std::vector& topic_partitions, + const MessageCallback& message_callback); + void unassign(uint64_t consumer_id); private: struct ConsumerMetadata { using PartitionSubscriptionMap = std::unordered_map; const AssignmentCallback assignment_callback; const RevocationCallback revocation_callback; - const MessageCallback message_callback; std::vector partitions_assigned; std::unordered_map subscriptions; }; diff --git a/mocking/include/cppkafka/mocking/kafka_partition_mock.h b/mocking/include/cppkafka/mocking/kafka_partition_mock.h index 3ebaf5fa..5827237c 100644 --- a/mocking/include/cppkafka/mocking/kafka_partition_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_partition_mock.h @@ -14,7 +14,7 @@ namespace mocking { class KafkaPartitionMock { public: - using MessageCallback = std::function; + using MessageCallback = std::function; using SubscriberId = uint64_t; void add_message(KafkaMessageMock message); @@ -24,6 +24,10 @@ class KafkaPartitionMock { void unsubscribe(SubscriberId id); // Returns interval [lowest offset, largest offset) std::tuple get_offset_bounds() const; + + // Acquire this partition so that no messages can be produced while the callback is executed. + template + void acquire(const Functor& functor); private: std::vector get_subscriber_callbacks() const; @@ -31,10 +35,15 @@ class KafkaPartitionMock { SubscriberId current_subscriber_id_{0}; std::deque messages_; std::unordered_map subscribers_; - mutable std::mutex messages_mutex_; - mutable std::mutex subscribers_mutex_; + mutable std::recursive_mutex mutex_; }; +template +void KafkaPartitionMock::acquire(const Functor& functor) { + std::lock_guard _(mutex_); + functor(); +} + } // mocking } // cppkafka diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index 66d98308..230d722b 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -8,16 +9,23 @@ using std::atomic; using std::vector; using std::string; +using std::to_string; using std::move; using std::bind; using std::runtime_error; using std::make_tuple; +using std::unique_ptr; using std::tie; using std::get; using std::lock_guard; using std::unique_lock; using std::mutex; +using std::chrono::milliseconds; +using std::chrono::steady_clock; + +using boost::optional; + namespace cppkafka { namespace mocking { @@ -48,19 +56,44 @@ void ConsumerMock::subscribe(const vector& topics) { consumer_id_, topics, bind(&ConsumerMock::on_assignment, this, _1), - bind(&ConsumerMock::on_revocation, this), - bind(&ConsumerMock::on_message, this, _1, _2, _3, _4) + bind(&ConsumerMock::on_revocation, this) ); } void ConsumerMock::assign(const vector& topic_partitions) { - for (const TopicPartitionMock& topic_partition : topic_partitions) { - handle_assign(topic_partition); + { + lock_guard _(mutex_); + // Create entries for all topic partitions in our assigned partitions map + for (const TopicPartitionMock& topic_partition : topic_partitions) { + const auto id = make_id(topic_partition); + // We'll store the next offset from the one we've seen so far + const uint64_t next_offset = topic_partition.get_offset() + 1; + + auto iter = assigned_partitions_.find(id); + if (iter == assigned_partitions_.end()) { + iter = assigned_partitions_.emplace(id, TopicPartitionInfo{}).first; + } + else { + // The offset changed, clean up any messages with a lower offset than the + // next one + auto& queue = iter->second.messages; + while (!queue.empty() && queue.front().offset < next_offset) { + queue.pop(); + } + } + iter->second.next_offset = next_offset; + } } + using namespace std::placeholders; + // Now assign these partitions. This will atomically fetch all message we should fetch and + // then subscribe us to the topic/partitions + get_cluster().assign(consumer_id_, topic_partitions, + bind(&ConsumerMock::on_message, this, _1, _2, _3, _4)); } void ConsumerMock::unassign() { - lock_guard _(assigned_partitions_mutex_); + lock_guard _(mutex_); + get_cluster().unassign(consumer_id_); assigned_partitions_.clear(); } @@ -68,6 +101,82 @@ void ConsumerMock::set_opaque(void* opaque) { opaque_ = opaque; } +void ConsumerMock::pause_partitions(const vector& topic_partitions) { + lock_guard _(mutex_); + for (const TopicPartitionMock& topic_partition : topic_partitions) { + auto id = make_id(topic_partition); + consumable_topic_partitions_.erase(id); + paused_topic_partitions_.emplace(move(id)); + } +} + +void ConsumerMock::resume_partitions(const vector& topic_partitions) { + lock_guard _(mutex_); + for (const TopicPartitionMock& topic_partition : topic_partitions) { + auto id = make_id(topic_partition); + paused_topic_partitions_.erase(id); + auto iter = assigned_partitions_.find(id); + if (iter != assigned_partitions_.end() && !iter->second.messages.empty()) { + consumable_topic_partitions_.emplace(move(id)); + } + } +} + +optional ConsumerMock::poll(std::chrono::milliseconds timeout) { + auto wait_until = steady_clock::now() + timeout; + unique_lock lock(mutex_); + while(consumable_topic_partitions_.empty() && steady_clock::now() > wait_until) { + messages_condition_.wait_until(lock, wait_until); + } + /*MessageHandle(std::unique_ptr topic, int partition, int64_t offset, void* key, + size_t key_size, void* payload, size_t payload_size, int error_code, + MessageHandlePrivateData private_data, PointerOwnership ownership);*/ + if (consumable_topic_partitions_.empty()) { + return boost::none; + } + const auto id = *consumable_topic_partitions_.begin(); + auto iter = assigned_partitions_.find(id); + assert(iter != assigned_partitions_.end()); + + auto& queue = iter->second.messages; + if (emit_eofs_ && queue.empty()) { + // We emit the EOF so it's no longer consumable + consumable_topic_partitions_.erase(id); + return MessageHandle( + unique_ptr(new TopicHandle(get<0>(id), nullptr)), + get<1>(id), + iter->second.next_offset, + nullptr, 0, // key + nullptr, 0, // payload + RD_KAFKA_RESP_ERR_NO_ERROR, + MessageHandlePrivateData{}, + MessageHandle::PointerOwnership::Unowned + ); + } + else { + assert(!queue.empty()); + } + MessageAggregate aggregate = move(queue.front()); + queue.pop(); + + // If we have no more mesages we can't consume from it anymore + if (queue.empty()) { + consumable_topic_partitions_.erase(id); + } + + const auto& message = *aggregate.message; + return MessageHandle( + unique_ptr(new TopicHandle(get<0>(id), nullptr)), + get<1>(id), + iter->second.next_offset, + (void*)message.get_key().data(), message.get_key().size(), + (void*)message.get_payload().data(), message.get_payload().size(), + RD_KAFKA_RESP_ERR__PARTITION_EOF, + MessageHandlePrivateData{message.get_timestamp_type(), message.get_timestamp()}, + MessageHandle::PointerOwnership::Unowned + ); +} + ConsumerMock::TopicPartitionId ConsumerMock::make_id(const TopicPartitionMock& topic_partition) { return make_tuple(topic_partition.get_topic(), topic_partition.get_partition()); } @@ -79,7 +188,7 @@ void ConsumerMock::on_assignment(const vector& topic_partiti void ConsumerMock::on_revocation() { // Fetch and reset all assigned topic partitions vector topic_partitions = [&]() { - lock_guard _(assigned_partitions_mutex_); + lock_guard _(mutex_); vector output; for (const auto& topic_partition_pair : assigned_partitions_) { const TopicPartitionId& id = topic_partition_pair.first; @@ -92,20 +201,30 @@ void ConsumerMock::on_revocation() { } void ConsumerMock::on_message(const string& topic_name, unsigned partition, uint64_t offset, - const KafkaMessageMock* message) { + const KafkaMessageMock& message) { + auto id = make_tuple(topic_name, partition); + // We should only process this if we don't have this topic/partition assigned (assignment // pending?) or the message offset comes after the next offset we have stored - const bool valid_offset = [&]() { - lock_guard _(assigned_partitions_mutex_); - auto iter = assigned_partitions_.find(make_tuple(topic_name, partition)); - return iter == assigned_partitions_.end() || iter->second.next_offset >= offset; - }(); - if (!valid_offset) { + lock_guard _(mutex_); + auto iter = assigned_partitions_.find(id); + MessageAggregate aggregate = { topic_name, partition, offset, &message }; + if (iter != assigned_partitions_.end()) { + throw runtime_error("got message for unexpected partition " + to_string(partition)); + } + if (offset > iter->second.next_offset) { + throw runtime_error("got message with unexpected offset " + to_string(offset)); + } + else if (offset < iter->second.next_offset) { return; } - unique_lock lock(message_queue_mutex_); - new_message_queue_.push({ topic_name, partition, offset, message }); - message_queue_condition_.notify_one(); + // This is the message we were waiting for + iter->second.next_offset++; + iter->second.messages.push(move(aggregate)); + if (!paused_topic_partitions_.count(id)) { + consumable_topic_partitions_.emplace(move(id)); + messages_condition_.notify_one(); + } } void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type, @@ -117,52 +236,5 @@ void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type, } } -void ConsumerMock::handle_assign(const TopicPartitionMock& topic_partition) { - const auto id = make_id(topic_partition); - // We'll store the next offset from the one we've seen so far - const uint64_t next_offset = topic_partition.get_offset() + 1; - - { - lock_guard _(assigned_partitions_mutex_); - if (assigned_partitions_.count(id)) { - return; - } - assigned_partitions_.emplace(id, next_offset); - } - - // Fetch any existing messages and push them to the available message queue - auto& cluster = get_cluster(); - KafkaTopicMock& topic = cluster.get_topic(topic_partition.get_topic()); - fetch_existing_messages(topic_partition.get_partition(), next_offset, topic); -} - -void ConsumerMock::fetch_existing_messages(unsigned partition_index, uint64_t next_offset, - KafkaTopicMock& topic) { - KafkaPartitionMock& partition = topic.get_partition(partition_index); - uint64_t start_offset; - uint64_t end_offset; - tie(start_offset, end_offset) = partition.get_offset_bounds(); - if (start_offset < next_offset) { - throw runtime_error("Stored offset is too high"); - } - // Nothing to fetch - if (next_offset == end_offset) { - return; - } - unique_lock lock(message_queue_mutex_); - for (uint64_t i = next_offset; i != end_offset; ++i) { - const KafkaMessageMock& message = partition.get_message(i); - available_message_queue_.push({ topic.get_name(), partition_index, i, &message }); - } - message_queue_condition_.notify_all(); -} - -// TopicPartitionInfo - -ConsumerMock::TopicPartitionInfo::TopicPartitionInfo(uint64_t next_offset) -: next_offset(next_offset) { - -} - } // mocking } // cppkafka diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index 271756aa..7263b475 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -7,12 +8,14 @@ using std::shared_ptr; using std::make_shared; using std::string; +using std::to_string; using std::vector; using std::invalid_argument; using std::runtime_error; using std::piecewise_construct; using std::forward_as_tuple; using std::move; +using std::tie; using std::lock_guard; using std::mutex; using std::iota; @@ -51,6 +54,7 @@ bool KafkaCluster::topic_exists(const string& name) const { } void KafkaCluster::produce(const string& topic, unsigned partition, KafkaMessageMock message) { + lock_guard _(topics_mutex_); auto iter = topics_.find(topic); if (iter == topics_.end()) { throw invalid_argument("topic does not exist"); @@ -74,8 +78,7 @@ const KafkaTopicMock& KafkaCluster::get_topic(const string& name) const { void KafkaCluster::subscribe(const string& group_id, uint64_t consumer_id, const vector& topics, AssignmentCallback assignment_callback, - RevocationCallback revocation_callback, - MessageCallback message_callback) { + RevocationCallback revocation_callback) { lock_guard _(consumer_data_mutex_); auto iter = consumer_data_.find(consumer_id); // If it's already subscribed to something, unsubscribe from it @@ -85,7 +88,6 @@ void KafkaCluster::subscribe(const string& group_id, uint64_t consumer_id, ConsumerMetadata data = { move(assignment_callback), move(revocation_callback), - move(message_callback), }; iter = consumer_data_.emplace(consumer_id, move(data)).first; @@ -104,6 +106,60 @@ void KafkaCluster::unsubscribe(const string& group_id, uint64_t consumer_id) { do_unsubscribe(group_id, consumer_id); } +void KafkaCluster::assign(uint64_t consumer_id, const vector& topic_partitions, + const MessageCallback& message_callback) { + lock_guard _(consumer_data_mutex_); + auto iter = consumer_data_.find(consumer_id); + if (iter == consumer_data_.end()) { + iter = consumer_data_.emplace(consumer_id, ConsumerMetadata{}).first; + } + ConsumerMetadata& consumer = iter->second; + using namespace std::placeholders; + for (const TopicPartitionMock& topic_partition : topic_partitions) { + KafkaTopicMock& topic = get_topic(topic_partition.get_topic()); + auto& partition = topic.get_partition(topic_partition.get_partition()); + auto callback = bind(message_callback, topic_partition.get_topic(), + topic_partition.get_partition(), _1, _2); + partition.acquire([&]() { + uint64_t start_offset; + uint64_t end_offset; + tie(start_offset, end_offset) = partition.get_offset_bounds(); + const uint64_t next_offset = topic_partition.get_offset(); + if (start_offset < next_offset) { + throw runtime_error("stored offset is too high"); + } + // Nothing to fetch + if (next_offset == end_offset) { + return; + } + for (uint64_t i = next_offset; i != end_offset; ++i) { + const KafkaMessageMock& message = partition.get_message(i); + callback(i, message); + } + const auto subscriber_id = partition.subscribe(move(callback)); + consumer.subscriptions[&topic].emplace(topic_partition.get_partition(), + subscriber_id); + }); + } +} + +void KafkaCluster::unassign(uint64_t consumer_id) { + lock_guard _(consumer_data_mutex_); + auto iter = consumer_data_.find(consumer_id); + if (iter == consumer_data_.end()) { + throw runtime_error("called unassign with unknown consumer id " + to_string(consumer_id)); + } + ConsumerMetadata& consumer = iter->second; + for (const auto& topic_subscription : consumer.subscriptions) { + KafkaTopicMock& topic = *topic_subscription.first; + for (const auto& partition_subscription : topic_subscription.second) { + auto& partition_mock = topic.get_partition(partition_subscription.first); + partition_mock.unsubscribe(partition_subscription.second); + } + } + consumer.subscriptions.clear(); +} + void KafkaCluster::generate_assignments(const string& group_id, const TopicConsumersMap& topic_consumers) { for (const auto& topic_consumers_pair : topic_consumers) { @@ -127,18 +183,6 @@ void KafkaCluster::generate_assignments(const string& group_id, for (size_t i = 0; i < chunk_size; ++i) { consumer.partitions_assigned.emplace_back(topic_name, chunk_start + i); } - - // Subscribe to every assigned partition - using namespace std::placeholders; - for (const TopicPartitionMock& topic_partition : consumer.partitions_assigned) { - auto& partition_mock = topic.get_partition(topic_partition.get_partition()); - auto callback = bind(consumer.message_callback, topic_partition.get_topic(), - topic_partition.get_partition(), _1, _2); - const auto subscriber_id = partition_mock.subscribe(move(callback)); - consumer.subscriptions[&topic].emplace(topic_partition.get_partition(), - subscriber_id); - } - // Now trigger the assignment callback consumer_index++; } } @@ -164,13 +208,6 @@ void KafkaCluster::generate_revocations(const TopicConsumersMap& topic_consumers // Execute revocation callback and unsubscribe from the partition object if (!consumer.partitions_assigned.empty()) { consumer.revocation_callback(); - for (const auto& topic_subscription : consumer.subscriptions) { - KafkaTopicMock& topic = *topic_subscription.first; - for (const auto& partition_subscription : topic_subscription.second) { - auto& partition_mock = topic.get_partition(partition_subscription.first); - partition_mock.unsubscribe(partition_subscription.second); - } - } consumer.partitions_assigned.clear(); } consumer.subscriptions.clear(); diff --git a/mocking/src/kafka_partition_mock.cpp b/mocking/src/kafka_partition_mock.cpp index 06abfe18..b603afb6 100644 --- a/mocking/src/kafka_partition_mock.cpp +++ b/mocking/src/kafka_partition_mock.cpp @@ -5,6 +5,7 @@ using std::vector; using std::lock_guard; using std::mutex; +using std::recursive_mutex; using std::out_of_range; using std::move; using std::tuple; @@ -15,23 +16,23 @@ namespace cppkafka { namespace mocking { void KafkaPartitionMock::add_message(KafkaMessageMock message) { - KafkaMessageMock* message_ptr; + const KafkaMessageMock* message_ptr; uint64_t offset; tie(message_ptr, offset) = [&] { - lock_guard _(messages_mutex_); + lock_guard _(mutex_); messages_.emplace_back(move(message)); return make_tuple(&messages_.back(), messages_.size() - 1); }(); const vector callbacks = get_subscriber_callbacks(); for (const MessageCallback& callback : callbacks) { - callback(offset, message_ptr); + callback(offset, *message_ptr); } } const KafkaMessageMock& KafkaPartitionMock::get_message(uint64_t offset) const { const uint64_t index = offset - base_offset_; - lock_guard _(messages_mutex_); + lock_guard _(mutex_); if (messages_.size() >= index) { throw out_of_range("invalid message index"); } @@ -39,29 +40,29 @@ const KafkaMessageMock& KafkaPartitionMock::get_message(uint64_t offset) const { } size_t KafkaPartitionMock::get_message_count() const { - lock_guard _(messages_mutex_); + lock_guard _(mutex_); return messages_.size(); } KafkaPartitionMock::SubscriberId KafkaPartitionMock::subscribe(MessageCallback callback) { - lock_guard _(subscribers_mutex_); + lock_guard _(mutex_); auto id = current_subscriber_id_++; subscribers_.emplace(id, move(callback)); return id; } void KafkaPartitionMock::unsubscribe(SubscriberId id) { - lock_guard _(subscribers_mutex_); + lock_guard _(mutex_); subscribers_.erase(id); } tuple KafkaPartitionMock::get_offset_bounds() const { - lock_guard _(messages_mutex_); + lock_guard _(mutex_); return make_tuple(base_offset_, base_offset_ + messages_.size()); } vector KafkaPartitionMock::get_subscriber_callbacks() const { - lock_guard _(subscribers_mutex_); + lock_guard _(mutex_); vector output; for (const auto& subcriber_pair : subscribers_) { output.emplace_back(subcriber_pair.second); From ff4011896c65ac4b7cee146afcf22403aa0a6beb Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Thu, 26 Oct 2017 21:21:32 -0700 Subject: [PATCH 16/20] Add support for most API calls --- .../include/cppkafka/mocking/consumer_mock.h | 6 +- .../cppkafka/mocking/event_processor.h | 8 +- .../include/cppkafka/mocking/handle_mock.h | 5 + .../include/cppkafka/mocking/handle_wrapper.h | 12 +- .../include/cppkafka/mocking/message_handle.h | 7 + .../include/cppkafka/mocking/producer_mock.h | 4 +- mocking/src/api.cpp | 216 ++++++++++++++++++ mocking/src/consumer_mock.cpp | 34 +-- mocking/src/event_processor.cpp | 22 +- mocking/src/handle_mock.cpp | 16 ++ mocking/src/message_handle.cpp | 18 ++ mocking/src/producer_mock.cpp | 14 +- 12 files changed, 335 insertions(+), 27 deletions(-) diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index a51dfdaa..40349058 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -32,12 +31,13 @@ class ConsumerMock : public HandleMock { ~ConsumerMock(); void subscribe(const std::vector& topics); + void unsubscribe(); void assign(const std::vector& topic_partitions); void unassign(); - void set_opaque(void* opaque); void pause_partitions(const std::vector& topic_partitions); void resume_partitions(const std::vector& topic_partitions); - boost::optional poll(std::chrono::milliseconds timeout); + std::unique_ptr poll(std::chrono::milliseconds timeout); + std::vector get_assignment() const; private: static uint64_t make_consumer_id(); diff --git a/mocking/include/cppkafka/mocking/event_processor.h b/mocking/include/cppkafka/mocking/event_processor.h index db07d0df..1b0609ac 100644 --- a/mocking/include/cppkafka/mocking/event_processor.h +++ b/mocking/include/cppkafka/mocking/event_processor.h @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace cppkafka { @@ -21,12 +22,15 @@ class EventProcessor { ~EventProcessor(); void add_event(EventPtr event); + size_t get_event_count() const; + bool wait_until_empty(std::chrono::milliseconds timeout); private: void process_events(); std::thread processing_thread_; - std::mutex events_mutex_; - std::condition_variable events_condition_; + mutable std::mutex events_mutex_; + std::condition_variable new_events_condition_; + std::condition_variable no_events_condition_; std::queue events_; bool running_{true}; }; diff --git a/mocking/include/cppkafka/mocking/handle_mock.h b/mocking/include/cppkafka/mocking/handle_mock.h index 1fa232b0..445dad02 100644 --- a/mocking/include/cppkafka/mocking/handle_mock.h +++ b/mocking/include/cppkafka/mocking/handle_mock.h @@ -19,7 +19,10 @@ class HandleMock { HandleMock(EventProcessorPtr processor, ClusterPtr cluster); virtual ~HandleMock() = default; + void* get_opaque() const; + size_t get_event_count() const; void set_cluster(ClusterPtr cluster); + void set_opaque(void* opaque); protected: using EventPtr = EventProcessor::EventPtr; @@ -30,9 +33,11 @@ class HandleMock { void generate_event(Args&&... args) { generate_event(EventPtr(new T(cluster_, std::forward(args)...))); } + EventProcessor& get_event_processor(); private: EventProcessorPtr processor_; ClusterPtr cluster_; + void* opaque_; }; } // mocking diff --git a/mocking/include/cppkafka/mocking/handle_wrapper.h b/mocking/include/cppkafka/mocking/handle_wrapper.h index 3a564216..de03fab9 100644 --- a/mocking/include/cppkafka/mocking/handle_wrapper.h +++ b/mocking/include/cppkafka/mocking/handle_wrapper.h @@ -27,8 +27,6 @@ class HandleWrapper { } - - T& get_handle() { return *handle_; } @@ -36,6 +34,16 @@ class HandleWrapper { const T& get_handle() const { return *handle_; } + + template + U& get_handle() { + return static_cast(get_handle()); + } + + template + const U& get_handle() const { + return static_cast(get_handle()); + } private: std::unique_ptr handle_; }; diff --git a/mocking/include/cppkafka/mocking/message_handle.h b/mocking/include/cppkafka/mocking/message_handle.h index b770742f..904109cf 100644 --- a/mocking/include/cppkafka/mocking/message_handle.h +++ b/mocking/include/cppkafka/mocking/message_handle.h @@ -10,6 +10,7 @@ namespace cppkafka { namespace mocking { class KafkaMessageMock; +class MessageHandle; class MessageHandlePrivateData { public: @@ -18,9 +19,14 @@ class MessageHandlePrivateData { rd_kafka_timestamp_type_t get_timestamp_type() const; int64_t get_timestamp() const; + MessageHandle* get_owner() const; + void set_owner(MessageHandle* handle); + void set_opaque(void* opaque); private: rd_kafka_timestamp_type_t timestamp_type_; int64_t timestamp_; + MessageHandle* owner_{nullptr}; + void* opaque_; }; class MessageHandle { @@ -38,6 +44,7 @@ class MessageHandle { ~MessageHandle(); const TopicHandle& get_topic() const; + rd_kafka_message_t& get_message(); const rd_kafka_message_t& get_message() const; KafkaMessageMock make_message_mock() const; private: diff --git a/mocking/include/cppkafka/mocking/producer_mock.h b/mocking/include/cppkafka/mocking/producer_mock.h index b90543eb..ffccce9d 100644 --- a/mocking/include/cppkafka/mocking/producer_mock.h +++ b/mocking/include/cppkafka/mocking/producer_mock.h @@ -14,7 +14,9 @@ class ProducerMock : public HandleMock { ProducerMock(ConfigurationMock config, EventProcessorPtr processor, ClusterPtr cluster); - void produce_message(MessageHandle message_handle); + void produce(MessageHandle message_handle); + bool flush(std::chrono::milliseconds timeout); + size_t poll(std::chrono::milliseconds timeout); private: ConfigurationMock config_; }; diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp index c6a3726b..c0046268 100644 --- a/mocking/src/api.cpp +++ b/mocking/src/api.cpp @@ -1,14 +1,21 @@ #include #include #include +#include #include #include +#include #include using std::string; using std::copy; +using std::vector; using std::strlen; +using std::move; using std::make_shared; +using std::unique_ptr; + +using std::chrono::milliseconds; using namespace cppkafka::mocking; using namespace cppkafka::mocking::detail; @@ -205,6 +212,25 @@ rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list, return output; } +// rd_kafka_topic_t + +rd_kafka_topic_t* rd_kafka_topic_new(rd_kafka_t* rk, const char* topic, + rd_kafka_topic_conf_t* conf) { + return reinterpret_cast(new TopicHandle(topic, nullptr)); +} + +const char* rd_kafka_topic_name(const rd_kafka_topic_t* rkt) { + return reinterpret_cast(rkt)->get_topic().c_str(); +} + +void rd_kafka_topic_destroy(rd_kafka_topic_t* rkt) { + delete reinterpret_cast(rkt); +} + +int rd_kafka_topic_partition_available(const rd_kafka_topic_t* rkt, int32_t partition) { + return 1; +} + // rd_kafka_t rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf_ptr, @@ -219,5 +245,195 @@ rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf_ptr, return new rd_kafka_t(new ProducerMock(conf, make_shared(), move(cluster))); } + else if (type == RD_KAFKA_CONSUMER) { + if (!conf.has_key("group.id")) { + const string error = "Local: Unknown topic"; + if (error.size() < errstr_size) { + copy(error.begin(), error.end(), errstr); + errstr[error.size()] = 0; + } + return nullptr; + } + return new rd_kafka_t(new ConsumerMock(conf, make_shared(), + move(cluster))); + } return nullptr; } + +void rd_kafka_destroy(rd_kafka_t* rk) { + delete &rk->get_handle(); +} + +int rd_kafka_brokers_add(rd_kafka_t* rk, const char* brokerlist) { + auto cluster = KafkaClusterRegistry::instance().get_cluster(brokerlist); + if (cluster) { + rk->get_handle().set_cluster(move(cluster)); + } + return 1; +} + +const char* rd_kafka_name(const rd_kafka_t* rk) { + return "cppkafka mock handle"; +} + +rd_kafka_message_t* rd_kafka_consumer_poll(rd_kafka_t* rk, int timeout_ms) { + auto& consumer = rk->get_handle(); + auto message_ptr = consumer.poll(milliseconds(timeout_ms)); + if (!message_ptr) { + return nullptr; + } + else { + return &message_ptr.release()->get_message(); + } +} + +void rd_kafka_message_destroy(rd_kafka_message_t* rkmessage) { + delete static_cast(rkmessage->_private)->get_owner(); +} + +rd_kafka_resp_err_t rd_kafka_pause_partitions(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t* partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + auto& consumer = rk->get_handle(); + consumer.pause_partitions(topic_partitions); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_resume_partitions(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t* partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + auto& consumer = rk->get_handle(); + consumer.resume_partitions(topic_partitions); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_subscribe(rd_kafka_t* rk, + const rd_kafka_topic_partition_list_t* partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + vector topics; + for (const TopicPartitionMock& topic_partition : topic_partitions) { + topics.emplace_back(topic_partition.get_topic()); + } + auto& consumer = rk->get_handle(); + consumer.subscribe(topics); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_unsubscribe(rd_kafka_t* rk) { + auto& consumer = rk->get_handle(); + consumer.unsubscribe(); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_assign(rd_kafka_t* rk, + const rd_kafka_topic_partition_list_t* partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + auto& consumer = rk->get_handle(); + consumer.assign(topic_partitions); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_assignment(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t** partitions) { + auto& consumer = rk->get_handle(); + const vector assignment = consumer.get_assignment(); + *partitions = to_rdkafka_handle(assignment).release(); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_flush(rd_kafka_t* rk, int timeout_ms) { + if (rk->get_handle().flush(milliseconds(timeout_ms))) { + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + else { + return RD_KAFKA_RESP_ERR__TIMED_OUT; + } +} + +int rd_kafka_poll(rd_kafka_t* rk, int timeout_ms) { + return rk->get_handle().poll(milliseconds(timeout_ms)); +} + +rd_kafka_resp_err_t rd_kafka_producev(rd_kafka_t* rk, ...) { + va_list args; + int vtype; + unique_ptr topic; + unsigned partition = RD_KAFKA_PARTITION_UA; + void* key_ptr = nullptr; + size_t key_size = 0; + void* payload_ptr = nullptr; + size_t payload_size = 0; + void* opaque = nullptr; + MessageHandle::PointerOwnership ownership = MessageHandle::PointerOwnership::Unowned; + int64_t timestamp = 0; + + va_start(args, rk); + while ((vtype = va_arg(args, int)) != RD_KAFKA_VTYPE_END) { + switch (vtype) { + case RD_KAFKA_VTYPE_TOPIC: + topic.reset(new TopicHandle(va_arg(args, const char *), nullptr)); + break; + case RD_KAFKA_VTYPE_PARTITION: + partition = va_arg(args, int32_t); + break; + case RD_KAFKA_VTYPE_VALUE: + payload_ptr = va_arg(args, void *); + payload_size = va_arg(args, size_t); + break; + case RD_KAFKA_VTYPE_KEY: + key_ptr = va_arg(args, void *); + key_size = va_arg(args, size_t); + break; + case RD_KAFKA_VTYPE_OPAQUE: + opaque = va_arg(args, void *); + break; + case RD_KAFKA_VTYPE_MSGFLAGS: + if (va_arg(args, int) == static_cast(MessageHandle::PointerOwnership::Owned)) { + ownership = MessageHandle::PointerOwnership::Owned; + } + break; + case RD_KAFKA_VTYPE_TIMESTAMP: + timestamp = va_arg(args, int64_t); + break; + default: + return RD_KAFKA_RESP_ERR__INVALID_ARG; + } + } + va_end(args); + + MessageHandlePrivateData private_data(RD_KAFKA_TIMESTAMP_CREATE_TIME, timestamp); + private_data.set_opaque(opaque); + rk->get_handle().produce(MessageHandle( + move(topic), + partition, + -1, // offset + key_ptr, key_size, + payload_ptr, payload_size, + RD_KAFKA_RESP_ERR_NO_ERROR, + private_data, + ownership + )); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +int rd_kafka_outq_len(rd_kafka_t* rk) { + return rk->get_handle().get_event_count(); +} + +void* rd_kafka_opaque(const rd_kafka_t* rk) { + return rk->get_handle().get_opaque(); +} + +void rd_kafka_set_log_level(rd_kafka_t* /*rk*/, int /*level*/) { + +} + +// misc + +const char* rd_kafka_err2str(rd_kafka_resp_err_t err) { + return "cppkafka mock: error"; +} + +rd_kafka_resp_err_t rd_kafka_errno2err(int errnox) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index 230d722b..cb2e69e1 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -24,8 +24,6 @@ using std::mutex; using std::chrono::milliseconds; using std::chrono::steady_clock; -using boost::optional; - namespace cppkafka { namespace mocking { @@ -60,6 +58,10 @@ void ConsumerMock::subscribe(const vector& topics) { ); } +void ConsumerMock::unsubscribe() { + get_cluster().unsubscribe(group_id_, consumer_id_); +} + void ConsumerMock::assign(const vector& topic_partitions) { { lock_guard _(mutex_); @@ -97,10 +99,6 @@ void ConsumerMock::unassign() { assigned_partitions_.clear(); } -void ConsumerMock::set_opaque(void* opaque) { - opaque_ = opaque; -} - void ConsumerMock::pause_partitions(const vector& topic_partitions) { lock_guard _(mutex_); for (const TopicPartitionMock& topic_partition : topic_partitions) { @@ -122,17 +120,14 @@ void ConsumerMock::resume_partitions(const vector& topic_par } } -optional ConsumerMock::poll(std::chrono::milliseconds timeout) { +unique_ptr ConsumerMock::poll(std::chrono::milliseconds timeout) { auto wait_until = steady_clock::now() + timeout; unique_lock lock(mutex_); while(consumable_topic_partitions_.empty() && steady_clock::now() > wait_until) { messages_condition_.wait_until(lock, wait_until); } - /*MessageHandle(std::unique_ptr topic, int partition, int64_t offset, void* key, - size_t key_size, void* payload, size_t payload_size, int error_code, - MessageHandlePrivateData private_data, PointerOwnership ownership);*/ if (consumable_topic_partitions_.empty()) { - return boost::none; + return nullptr; } const auto id = *consumable_topic_partitions_.begin(); auto iter = assigned_partitions_.find(id); @@ -142,7 +137,7 @@ optional ConsumerMock::poll(std::chrono::milliseconds timeout) { if (emit_eofs_ && queue.empty()) { // We emit the EOF so it's no longer consumable consumable_topic_partitions_.erase(id); - return MessageHandle( + return unique_ptr(new MessageHandle( unique_ptr(new TopicHandle(get<0>(id), nullptr)), get<1>(id), iter->second.next_offset, @@ -151,7 +146,7 @@ optional ConsumerMock::poll(std::chrono::milliseconds timeout) { RD_KAFKA_RESP_ERR_NO_ERROR, MessageHandlePrivateData{}, MessageHandle::PointerOwnership::Unowned - ); + )); } else { assert(!queue.empty()); @@ -165,7 +160,7 @@ optional ConsumerMock::poll(std::chrono::milliseconds timeout) { } const auto& message = *aggregate.message; - return MessageHandle( + return unique_ptr(new MessageHandle( unique_ptr(new TopicHandle(get<0>(id), nullptr)), get<1>(id), iter->second.next_offset, @@ -174,7 +169,16 @@ optional ConsumerMock::poll(std::chrono::milliseconds timeout) { RD_KAFKA_RESP_ERR__PARTITION_EOF, MessageHandlePrivateData{message.get_timestamp_type(), message.get_timestamp()}, MessageHandle::PointerOwnership::Unowned - ); + )); +} + +vector ConsumerMock::get_assignment() const { + vector output; + lock_guard _(mutex_); + for (const auto& partition_pair : assigned_partitions_) { + output.emplace_back(get<0>(partition_pair.first), get<1>(partition_pair.first)); + } + return output; } ConsumerMock::TopicPartitionId ConsumerMock::make_id(const TopicPartitionMock& topic_partition) { diff --git a/mocking/src/event_processor.cpp b/mocking/src/event_processor.cpp index 4c6f2c48..407bac3c 100644 --- a/mocking/src/event_processor.cpp +++ b/mocking/src/event_processor.cpp @@ -5,6 +5,8 @@ using std::unique_lock; using std::mutex; using std::move; +using std::chrono::milliseconds; + namespace cppkafka { namespace mocking { @@ -17,7 +19,8 @@ EventProcessor::~EventProcessor() { { lock_guard _(events_mutex_); running_ = false; - events_condition_.notify_one(); + new_events_condition_.notify_all(); + no_events_condition_.notify_all(); } processing_thread_.join(); } @@ -25,14 +28,27 @@ EventProcessor::~EventProcessor() { void EventProcessor::add_event(EventPtr event) { lock_guard _(events_mutex_); events_.push(move(event)); - events_condition_.notify_one(); + new_events_condition_.notify_one(); +} + +size_t EventProcessor::get_event_count() const { + lock_guard _(events_mutex_); + return events_.size(); +} + +bool EventProcessor::wait_until_empty(milliseconds timeout) { + unique_lock lock(events_mutex_); + if (running_ && !events_.empty()) { + no_events_condition_.wait_for(lock, timeout); + } + return events_.empty(); } void EventProcessor::process_events() { while (true) { unique_lock lock(events_mutex_); while (running_ && events_.empty()) { - events_condition_.wait(lock); + new_events_condition_.wait(lock); } if (!running_) { diff --git a/mocking/src/handle_mock.cpp b/mocking/src/handle_mock.cpp index a25e0d3c..ad876b15 100644 --- a/mocking/src/handle_mock.cpp +++ b/mocking/src/handle_mock.cpp @@ -17,6 +17,14 @@ HandleMock::HandleMock(EventProcessorPtr processor, ClusterPtr cluster) } +void* HandleMock::get_opaque() const { + return opaque_; +} + +size_t HandleMock::get_event_count() const { + return processor_->get_event_count(); +} + void HandleMock::set_cluster(ClusterPtr cluster) { // Don't allow changing the cluster if (cluster_) { @@ -25,6 +33,10 @@ void HandleMock::set_cluster(ClusterPtr cluster) { cluster_ = move(cluster); } +void HandleMock::set_opaque(void* opaque) { + opaque_ = opaque; +} + KafkaCluster& HandleMock::get_cluster() { if (!cluster_) { throw runtime_error("cluster not set"); @@ -43,5 +55,9 @@ void HandleMock::generate_event(EventPtr event) { processor_->add_event(move(event)); } +EventProcessor& HandleMock::get_event_processor() { + return *processor_; +} + } // mocking } // cppkafka diff --git a/mocking/src/message_handle.cpp b/mocking/src/message_handle.cpp index 3f93cc2e..8b393c97 100644 --- a/mocking/src/message_handle.cpp +++ b/mocking/src/message_handle.cpp @@ -23,6 +23,18 @@ int64_t MessageHandlePrivateData::get_timestamp() const { return timestamp_; } +MessageHandle* MessageHandlePrivateData::get_owner() const { + return owner_; +} + +void MessageHandlePrivateData::set_owner(MessageHandle* handle) { + owner_ = handle; +} + +void MessageHandlePrivateData::set_opaque(void* opaque) { + opaque_ = opaque; +} + MessageHandle::MessageHandle(unique_ptr topic, int partition, int64_t offset, void* key, size_t key_size, void* payload, size_t payload_size, int error_code, MessageHandlePrivateData private_data, @@ -47,6 +59,7 @@ MessageHandle& MessageHandle::operator=(MessageHandle&& other) { swap(topic_, other.topic_); swap(message_, other.message_); swap(ownership_, other.ownership_); + swap(private_data_, other.private_data_); set_private_data_pointer(); other.set_private_data_pointer(); return *this; @@ -63,6 +76,10 @@ const TopicHandle& MessageHandle::get_topic() const { return *topic_; } +rd_kafka_message_t& MessageHandle::get_message() { + return message_; +} + const rd_kafka_message_t& MessageHandle::get_message() const { return message_; } @@ -79,6 +96,7 @@ KafkaMessageMock MessageHandle::make_message_mock() const { } void MessageHandle::set_private_data_pointer() { + private_data_.set_owner(this); message_._private = reinterpret_cast(&private_data_); } diff --git a/mocking/src/producer_mock.cpp b/mocking/src/producer_mock.cpp index 6a9621ff..fad0982f 100644 --- a/mocking/src/producer_mock.cpp +++ b/mocking/src/producer_mock.cpp @@ -3,6 +3,8 @@ using std::move; +using std::chrono::milliseconds; + namespace cppkafka { namespace mocking { @@ -12,9 +14,19 @@ ProducerMock::ProducerMock(ConfigurationMock config, EventProcessorPtr processor } -void ProducerMock::produce_message(MessageHandle message_handle) { +void ProducerMock::produce(MessageHandle message_handle) { generate_event(move(message_handle)); } +bool ProducerMock::flush(milliseconds timeout) { + // TODO: produce buffered events + return get_event_processor().wait_until_empty(timeout); +} + +size_t ProducerMock::poll(milliseconds timeout) { + // TODO: do something + return 0; +} + } // mocking } // cppkafka From 4665bf624d671e0f8f4ca9a324906f3725c2e357 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 28 Oct 2017 11:28:00 -0700 Subject: [PATCH 17/20] Implement missing API functions --- .../include/cppkafka/mocking/consumer_mock.h | 1 + .../include/cppkafka/mocking/handle_mock.h | 4 +- mocking/src/CMakeLists.txt | 1 + mocking/src/api.cpp | 143 +++++++++++++++++- mocking/src/consumer_mock.cpp | 4 + src/consumer.cpp | 5 +- 6 files changed, 154 insertions(+), 4 deletions(-) diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index 40349058..a09b310f 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -30,6 +30,7 @@ class ConsumerMock : public HandleMock { ConsumerMock& operator=(const ConsumerMock&) = delete; ~ConsumerMock(); + void close(); void subscribe(const std::vector& topics); void unsubscribe(); void assign(const std::vector& topic_partitions); diff --git a/mocking/include/cppkafka/mocking/handle_mock.h b/mocking/include/cppkafka/mocking/handle_mock.h index 445dad02..61bf492f 100644 --- a/mocking/include/cppkafka/mocking/handle_mock.h +++ b/mocking/include/cppkafka/mocking/handle_mock.h @@ -23,11 +23,11 @@ class HandleMock { size_t get_event_count() const; void set_cluster(ClusterPtr cluster); void set_opaque(void* opaque); + KafkaCluster& get_cluster(); + const KafkaCluster& get_cluster() const; protected: using EventPtr = EventProcessor::EventPtr; - KafkaCluster& get_cluster(); - const KafkaCluster& get_cluster() const; void generate_event(EventPtr event); template void generate_event(Args&&... args) { diff --git a/mocking/src/CMakeLists.txt b/mocking/src/CMakeLists.txt index cb445652..0ca6c79c 100644 --- a/mocking/src/CMakeLists.txt +++ b/mocking/src/CMakeLists.txt @@ -26,4 +26,5 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIRS} ${RDKAFKA_INCLUDE_DIR}) add_library(cppkafka_mock EXCLUDE_FROM_ALL ${CPPKAFKA_LIBRARY_TYPE} ${SOURCES}) set_target_properties(cppkafka_mock PROPERTIES VERSION ${CPPKAFKA_VERSION} SOVERSION ${CPPKAFKA_VERSION}) +add_dependencies(cppkafka_mock cppkafka) target_link_libraries(cppkafka_mock ${RDKAFKA_LIBRARY}) diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp index c0046268..51534368 100644 --- a/mocking/src/api.cpp +++ b/mocking/src/api.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -14,6 +15,8 @@ using std::strlen; using std::move; using std::make_shared; using std::unique_ptr; +using std::tie; +using std::exception; using std::chrono::milliseconds; @@ -104,7 +107,7 @@ void rd_kafka_conf_set_socket_cb(rd_kafka_conf_t* conf, conf->get_handle().set_socket_callback(cb); } -void rd_kafka_topic_conf_set_partitioner_cb(rd_kafka_conf_t* conf, +void rd_kafka_topic_conf_set_partitioner_cb(rd_kafka_topic_conf_t* conf, ConfigurationMock::PartitionerCallback* cb) { conf->get_handle().set_partitioner_callback(cb); } @@ -118,6 +121,10 @@ void rd_kafka_conf_set_opaque(rd_kafka_conf_t* conf, void* opaque) { conf->get_handle().set_opaque(opaque); } +void rd_kafka_topic_conf_set_opaque(rd_kafka_topic_conf_t* conf, void* opaque) { + conf->get_handle().set_opaque(opaque); +} + const char** rd_kafka_conf_dump(rd_kafka_conf_t* conf, size_t* cntp) { const auto options = conf->get_handle().get_options(); *cntp = options.size() * 2; @@ -325,6 +332,12 @@ rd_kafka_resp_err_t rd_kafka_unsubscribe(rd_kafka_t* rk) { return RD_KAFKA_RESP_ERR_NO_ERROR; } +rd_kafka_resp_err_t rd_kafka_subscription(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t** topics) { + // TODO: implement + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + rd_kafka_resp_err_t rd_kafka_assign(rd_kafka_t* rk, const rd_kafka_topic_partition_list_t* partitions) { const vector topic_partitions = from_rdkafka_handle(*partitions); @@ -354,6 +367,17 @@ int rd_kafka_poll(rd_kafka_t* rk, int timeout_ms) { return rk->get_handle().poll(milliseconds(timeout_ms)); } +rd_kafka_queue_t* rd_kafka_queue_get_consumer(rd_kafka_t* rk) { + // TODO: implement + return nullptr; +} + +ssize_t rd_kafka_consume_batch_queue(rd_kafka_queue_t* rkqu, int timeout_ms, + rd_kafka_message_t** rkmessages, size_t rkmessages_size) { + // TODO: implement + return 0; +} + rd_kafka_resp_err_t rd_kafka_producev(rd_kafka_t* rk, ...) { va_list args; int vtype; @@ -428,6 +452,107 @@ void rd_kafka_set_log_level(rd_kafka_t* /*rk*/, int /*level*/) { } +rd_kafka_resp_err_t rd_kafka_query_watermark_offsets(rd_kafka_t* rk, const char* topic, + int32_t partition, int64_t* low, + int64_t* high, int /*timeout_ms*/) { + return rd_kafka_get_watermark_offsets(rk, topic, partition, low, high); +} + +rd_kafka_resp_err_t rd_kafka_get_watermark_offsets(rd_kafka_t* rk, const char *topic, + int32_t partition, int64_t *low, + int64_t *high) { + const auto& cluster = rk->get_handle().get_cluster(); + if (!cluster.topic_exists(topic)) { + return RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC; + } + try { + const auto& topic_object = cluster.get_topic(topic); + if (static_cast(partition) >= topic_object.get_partition_count()) { + return RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION; + } + const auto& partition_object = topic_object.get_partition(partition); + uint64_t lowest; + uint64_t largest; + tie(lowest, largest) = partition_object.get_offset_bounds(); + *low = lowest; + *high = largest; + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + catch (const exception&) { + return RD_KAFKA_RESP_ERR_UNKNOWN; + } +} + +rd_kafka_resp_err_t rd_kafka_offsets_for_times(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t* offsets, + int timeout_ms) { + // TODO: implement this one + return RD_KAFKA_RESP_ERR_UNKNOWN; +} + +rd_kafka_resp_err_t rd_kafka_metadata(rd_kafka_t* rk, int all_topics, + rd_kafka_topic_t* only_rkt, + const struct rd_kafka_metadata** metadatap, + int timeout_ms) { + // TODO: implement this one + return RD_KAFKA_RESP_ERR_UNKNOWN; +} + +void rd_kafka_metadata_destroy(const struct rd_kafka_metadata* /*metadata*/) { + // TODO: implement this one +} + +rd_kafka_resp_err_t rd_kafka_list_groups(rd_kafka_t* rk, const char* group, + const struct rd_kafka_group_list** grplistp, + int timeout_ms) { + // TODO: implement this one + return RD_KAFKA_RESP_ERR_UNKNOWN; +} + +void rd_kafka_group_list_destroy(const struct rd_kafka_group_list* /*grplist*/) { + // TODO: implement this one +} + +rd_kafka_resp_err_t rd_kafka_poll_set_consumer(rd_kafka_t*) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_consumer_close(rd_kafka_t* rk) { + rk->get_handle().close(); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_committed(rd_kafka_t* rk, rd_kafka_topic_partition_list_t *partitions, + int timeout_ms) { + // TODO: implement + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_commit(rd_kafka_t* rk, const rd_kafka_topic_partition_list_t* offsets, + int async) { + // TODO: implement + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_commit_message(rd_kafka_t* rk, const rd_kafka_message_t* rkmessage, + int async) { + // TODO: implement + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_position(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t* partitions) { + // TODO: implement + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +char* rd_kafka_memberid(const rd_kafka_t* rk) { + // TODO: make this better + char* output = (char*)malloc(strlen("cppkafka_mock") + 1); + strcpy(output, "cppkafka_mock"); + return output; +} + // misc const char* rd_kafka_err2str(rd_kafka_resp_err_t err) { @@ -437,3 +562,19 @@ const char* rd_kafka_err2str(rd_kafka_resp_err_t err) { rd_kafka_resp_err_t rd_kafka_errno2err(int errnox) { return RD_KAFKA_RESP_ERR_NO_ERROR; } + +int32_t rd_kafka_msg_partitioner_consistent_random(const rd_kafka_topic_t* rkt, const void *key, + size_t keylen, int32_t partition_cnt, + void *opaque, void *msg_opaque) { + unsigned hash = 0; + const char* key_ptr = reinterpret_cast(key); + for (size_t i = 0; i < keylen; ++i) { + hash += key_ptr[i]; + } + return hash % partition_cnt; +} + +rd_kafka_resp_err_t rd_kafka_last_error (void) { + // TODO: fix this + return RD_KAFKA_RESP_ERR_UNKNOWN; +} diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index cb2e69e1..7e575683 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -47,6 +47,10 @@ ConsumerMock::~ConsumerMock() { get_cluster().unsubscribe(group_id_, consumer_id_); } +void ConsumerMock::close() { + unsubscribe(); +} + void ConsumerMock::subscribe(const vector& topics) { using namespace std::placeholders; get_cluster().subscribe( diff --git a/src/consumer.cpp b/src/consumer.cpp index c9efe05a..88251d20 100644 --- a/src/consumer.cpp +++ b/src/consumer.cpp @@ -178,7 +178,10 @@ TopicPartitionList Consumer::get_assignment() const { } string Consumer::get_member_id() const { - return rd_kafka_memberid(get_handle()); + char* id = rd_kafka_memberid(get_handle()); + string output = id; + free(id); + return output; } const Consumer::AssignmentCallback& Consumer::get_assignment_callback() const { From e0289f035a1857a3c04aaab77f2544e3894c234c Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Sat, 4 Nov 2017 12:17:16 -0700 Subject: [PATCH 18/20] Fix several issues and make consumption work --- .../cppkafka/mocking/configuration_mock.h | 2 +- .../include/cppkafka/mocking/consumer_mock.h | 8 ++- .../include/cppkafka/mocking/kafka_cluster.h | 9 ++- .../cppkafka/mocking/kafka_cluster_registry.h | 10 ++- .../cppkafka/mocking/kafka_partition_mock.h | 2 +- mocking/src/api.cpp | 29 ++++++--- mocking/src/consumer_mock.cpp | 61 ++++++++++++++---- mocking/src/event_processor.cpp | 10 +-- mocking/src/kafka_cluster.cpp | 62 +++++++++++-------- mocking/src/kafka_cluster_registry.cpp | 14 +++-- mocking/src/kafka_partition_mock.cpp | 4 +- mocking/src/kafka_topic_mock.cpp | 2 +- 12 files changed, 138 insertions(+), 75 deletions(-) diff --git a/mocking/include/cppkafka/mocking/configuration_mock.h b/mocking/include/cppkafka/mocking/configuration_mock.h index d661541b..fc8aee82 100644 --- a/mocking/include/cppkafka/mocking/configuration_mock.h +++ b/mocking/include/cppkafka/mocking/configuration_mock.h @@ -75,7 +75,7 @@ class ConfigurationMock { struct Cloner { ConfigurationMock* operator()(const ConfigurationMock* ptr) const { - return new ConfigurationMock(*ptr); + return ptr ? new ConfigurationMock(*ptr) : nullptr; } }; diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index a09b310f..0266d7fb 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -16,6 +16,7 @@ #include #include #include +#include namespace cppkafka { namespace mocking { @@ -57,6 +58,8 @@ class ConsumerMock : public HandleMock { using TopicPartitionId = std::tuple; static TopicPartitionId make_id(const TopicPartitionMock& topic_partition); + KafkaCluster::ResetOffsetPolicy get_offset_policy() const; + bool get_partition_eof_enabled() const; void on_assignment(const std::vector& topic_partitions); void on_revocation(); void on_message(const std::string& topic_name, unsigned partition, uint64_t offset, @@ -65,15 +68,14 @@ class ConsumerMock : public HandleMock { const std::vector& topic_partitions); ConfigurationMock config_; - // TODO: initialize this and make it const + std::string group_id_; + const KafkaCluster::ResetOffsetPolicy offset_reset_policy_; bool emit_eofs_; - const std::string group_id_; std::map assigned_partitions_; std::set consumable_topic_partitions_; std::set paused_topic_partitions_; mutable std::mutex mutex_; std::condition_variable messages_condition_; - void* opaque_; uint64_t consumer_id_; }; diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index e80bd4f5..7f5a9f04 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -21,6 +21,11 @@ class KafkaCluster { using MessageCallback = std::function; + enum class ResetOffsetPolicy { + Earliest = 1, + Latest = 2 + }; + static std::shared_ptr make_cluster(std::string url); KafkaCluster(const KafkaCluster&) = delete; @@ -40,7 +45,7 @@ class KafkaCluster { RevocationCallback revocation_callback); void unsubscribe(const std::string& group_id, uint64_t consumer_id); void assign(uint64_t consumer_id, const std::vector& topic_partitions, - const MessageCallback& message_callback); + ResetOffsetPolicy policy, const MessageCallback& message_callback); void unassign(uint64_t consumer_id); private: struct ConsumerMetadata { @@ -68,7 +73,7 @@ class KafkaCluster { mutable std::mutex topics_mutex_; std::unordered_map consumer_data_; std::unordered_map group_topics_data_; - mutable std::mutex consumer_data_mutex_; + mutable std::recursive_mutex consumer_data_mutex_; }; } // mocking diff --git a/mocking/include/cppkafka/mocking/kafka_cluster_registry.h b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h index 1fbae645..f6d33ecd 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster_registry.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h @@ -11,15 +11,13 @@ namespace detail { class KafkaClusterRegistry { public: - using ClusterPtr = std::shared_ptr; - static KafkaClusterRegistry& instance(); - void add_cluster(ClusterPtr cluster); - void remove_cluster(const KafkaCluster& cluster); - ClusterPtr get_cluster(const std::string& name) const; + void add_cluster(std::shared_ptr cluster); + void remove_cluster(const std::string& url); + std::shared_ptr get_cluster(const std::string& name) const; private: - std::unordered_map clusters_; + std::unordered_map> clusters_; mutable std::mutex clusters_mutex_; }; diff --git a/mocking/include/cppkafka/mocking/kafka_partition_mock.h b/mocking/include/cppkafka/mocking/kafka_partition_mock.h index 5827237c..a041ce63 100644 --- a/mocking/include/cppkafka/mocking/kafka_partition_mock.h +++ b/mocking/include/cppkafka/mocking/kafka_partition_mock.h @@ -23,7 +23,7 @@ class KafkaPartitionMock { SubscriberId subscribe(MessageCallback callback); void unsubscribe(SubscriberId id); // Returns interval [lowest offset, largest offset) - std::tuple get_offset_bounds() const; + std::tuple get_offset_bounds() const; // Acquire this partition so that no messages can be produced while the callback is executed. template diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp index 51534368..c6d2c6c9 100644 --- a/mocking/src/api.cpp +++ b/mocking/src/api.cpp @@ -115,6 +115,7 @@ void rd_kafka_topic_conf_set_partitioner_cb(rd_kafka_topic_conf_t* conf, void rd_kafka_conf_set_default_topic_conf(rd_kafka_conf_t* conf, rd_kafka_topic_conf_t* tconf) { conf->get_handle().set_default_topic_configuration(tconf->get_handle()); + rd_kafka_topic_conf_destroy(tconf); } void rd_kafka_conf_set_opaque(rd_kafka_conf_t* conf, void* opaque) { @@ -198,7 +199,7 @@ rd_kafka_topic_partition_list_t* rd_kafka_topic_partition_list_new(int size) { void rd_kafka_topic_partition_list_destroy(rd_kafka_topic_partition_list_t* toppar_list) { for (int i = 0; i < toppar_list->cnt; ++i) { - delete toppar_list->elems[i].topic; + delete[] toppar_list->elems[i].topic; } delete[] toppar_list->elems; delete toppar_list; @@ -214,6 +215,7 @@ rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list, const size_t length = strlen(topic); output->topic = new char[length + 1]; copy(topic, topic + length, output->topic); + output->topic[length] = 0; output->partition = partition; output->offset = RD_KAFKA_OFFSET_INVALID; return output; @@ -240,16 +242,17 @@ int rd_kafka_topic_partition_available(const rd_kafka_topic_t* rkt, int32_t part // rd_kafka_t -rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf_ptr, +rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t* conf_ptr, char *errstr, size_t errstr_size) { static const string BROKERS_OPTION = "metadata.broker.list"; - const auto& conf = conf_ptr->get_handle(); + auto& conf = conf_ptr->get_handle(); HandleMock::ClusterPtr cluster; if (conf.has_key(BROKERS_OPTION)) { cluster = KafkaClusterRegistry::instance().get_cluster(conf.get(BROKERS_OPTION)); } if (type == RD_KAFKA_PRODUCER) { - return new rd_kafka_t(new ProducerMock(conf, make_shared(), + unique_ptr _(conf_ptr); + return new rd_kafka_t(new ProducerMock(move(conf), make_shared(), move(cluster))); } else if (type == RD_KAFKA_CONSUMER) { @@ -261,14 +264,15 @@ rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf_ptr, } return nullptr; } - return new rd_kafka_t(new ConsumerMock(conf, make_shared(), + unique_ptr _(conf_ptr); + return new rd_kafka_t(new ConsumerMock(move(conf), make_shared(), move(cluster))); } return nullptr; } void rd_kafka_destroy(rd_kafka_t* rk) { - delete &rk->get_handle(); + delete rk; } int rd_kafka_brokers_add(rd_kafka_t* rk, const char* brokerlist) { @@ -340,9 +344,14 @@ rd_kafka_resp_err_t rd_kafka_subscription(rd_kafka_t* rk, rd_kafka_resp_err_t rd_kafka_assign(rd_kafka_t* rk, const rd_kafka_topic_partition_list_t* partitions) { - const vector topic_partitions = from_rdkafka_handle(*partitions); auto& consumer = rk->get_handle(); - consumer.assign(topic_partitions); + if (partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + consumer.assign(topic_partitions); + } + else { + consumer.unassign(); + } return RD_KAFKA_RESP_ERR_NO_ERROR; } @@ -471,8 +480,8 @@ rd_kafka_resp_err_t rd_kafka_get_watermark_offsets(rd_kafka_t* rk, const char *t return RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION; } const auto& partition_object = topic_object.get_partition(partition); - uint64_t lowest; - uint64_t largest; + int64_t lowest; + int64_t largest; tie(lowest, largest) = partition_object.get_offset_bounds(); *low = lowest; *high = largest; diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index 7e575683..dbbf303c 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -9,6 +9,7 @@ using std::atomic; using std::vector; using std::string; +using std::unordered_map; using std::to_string; using std::move; using std::bind; @@ -37,10 +38,12 @@ uint64_t ConsumerMock::make_consumer_id() { ConsumerMock::ConsumerMock(ConfigurationMock config, EventProcessorPtr processor, ClusterPtr cluster) : HandleMock(move(processor), move(cluster)), config_(move(config)), + offset_reset_policy_(get_offset_policy()), emit_eofs_(get_partition_eof_enabled()), consumer_id_(make_consumer_id()) { if (!config_.has_key(CONFIG_GROUP_ID)) { throw runtime_error("Failed to find " + CONFIG_GROUP_ID + " in config"); } + group_id_ = config_.get(CONFIG_GROUP_ID); } ConsumerMock::~ConsumerMock() { @@ -73,7 +76,13 @@ void ConsumerMock::assign(const vector& topic_partitions) { for (const TopicPartitionMock& topic_partition : topic_partitions) { const auto id = make_id(topic_partition); // We'll store the next offset from the one we've seen so far - const uint64_t next_offset = topic_partition.get_offset() + 1; + uint64_t next_offset; + if (topic_partition.get_offset() == RD_KAFKA_OFFSET_INVALID) { + next_offset = 0; + } + else { + next_offset = topic_partition.get_offset() + 1; + } auto iter = assigned_partitions_.find(id); if (iter == assigned_partitions_.end()) { @@ -93,14 +102,15 @@ void ConsumerMock::assign(const vector& topic_partitions) { using namespace std::placeholders; // Now assign these partitions. This will atomically fetch all message we should fetch and // then subscribe us to the topic/partitions - get_cluster().assign(consumer_id_, topic_partitions, + get_cluster().assign(consumer_id_, topic_partitions, offset_reset_policy_, bind(&ConsumerMock::on_message, this, _1, _2, _3, _4)); } void ConsumerMock::unassign() { lock_guard _(mutex_); - get_cluster().unassign(consumer_id_); assigned_partitions_.clear(); + consumable_topic_partitions_.clear(); + get_cluster().unassign(consumer_id_); } void ConsumerMock::pause_partitions(const vector& topic_partitions) { @@ -124,11 +134,10 @@ void ConsumerMock::resume_partitions(const vector& topic_par } } -unique_ptr ConsumerMock::poll(std::chrono::milliseconds timeout) { - auto wait_until = steady_clock::now() + timeout; +unique_ptr ConsumerMock::poll(milliseconds timeout) { unique_lock lock(mutex_); - while(consumable_topic_partitions_.empty() && steady_clock::now() > wait_until) { - messages_condition_.wait_until(lock, wait_until); + if (consumable_topic_partitions_.empty()) { + messages_condition_.wait_for(lock, timeout); } if (consumable_topic_partitions_.empty()) { return nullptr; @@ -189,12 +198,41 @@ ConsumerMock::TopicPartitionId ConsumerMock::make_id(const TopicPartitionMock& t return make_tuple(topic_partition.get_topic(), topic_partition.get_partition()); } +KafkaCluster::ResetOffsetPolicy ConsumerMock::get_offset_policy() const { + static const string KEY_NAME = "auto.offset.reset"; + static unordered_map MAPPINGS = { + { "smallest", KafkaCluster::ResetOffsetPolicy::Earliest }, + { "earliest", KafkaCluster::ResetOffsetPolicy::Earliest }, + { "beginning", KafkaCluster::ResetOffsetPolicy::Earliest }, + { "latest", KafkaCluster::ResetOffsetPolicy::Latest }, + { "largest", KafkaCluster::ResetOffsetPolicy::Latest }, + { "end", KafkaCluster::ResetOffsetPolicy::Latest }, + }; + + const ConfigurationMock* topic_config = config_.get_default_topic_configuration(); + if (!topic_config || !topic_config->has_key(KEY_NAME)) { + return KafkaCluster::ResetOffsetPolicy::Earliest; + } + else { + auto iter = MAPPINGS.find(topic_config->get(KEY_NAME)); + if (iter == MAPPINGS.end()) { + throw runtime_error("invalid auto.offset.reset value"); + } + return iter->second; + } +} + +bool ConsumerMock::get_partition_eof_enabled() const { + static const string KEY_NAME = "enable.partition.eof"; + return !config_.has_key(KEY_NAME) || config_.get(KEY_NAME) == "true"; +} + void ConsumerMock::on_assignment(const vector& topic_partitions) { handle_rebalance(RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, topic_partitions); } void ConsumerMock::on_revocation() { - // Fetch and reset all assigned topic partitions + // Fetch and all assigned topic partitions vector topic_partitions = [&]() { lock_guard _(mutex_); vector output; @@ -202,7 +240,6 @@ void ConsumerMock::on_revocation() { const TopicPartitionId& id = topic_partition_pair.first; output.emplace_back(get<0>(id), get<1>(id)); } - assigned_partitions_.clear(); return output; }(); handle_rebalance(RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS, topic_partitions); @@ -211,13 +248,13 @@ void ConsumerMock::on_revocation() { void ConsumerMock::on_message(const string& topic_name, unsigned partition, uint64_t offset, const KafkaMessageMock& message) { auto id = make_tuple(topic_name, partition); + MessageAggregate aggregate = { topic_name, partition, offset, &message }; // We should only process this if we don't have this topic/partition assigned (assignment // pending?) or the message offset comes after the next offset we have stored lock_guard _(mutex_); auto iter = assigned_partitions_.find(id); - MessageAggregate aggregate = { topic_name, partition, offset, &message }; - if (iter != assigned_partitions_.end()) { + if (iter == assigned_partitions_.end()) { throw runtime_error("got message for unexpected partition " + to_string(partition)); } if (offset > iter->second.next_offset) { @@ -240,7 +277,7 @@ void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type, auto rebalance_callback = config_.get_rebalance_callback(); if (rebalance_callback) { auto handle = to_rdkafka_handle(topic_partitions); - rebalance_callback(nullptr, type, handle.get(), opaque_); + rebalance_callback(nullptr, type, handle.get(), config_.get_opaque()); } } diff --git a/mocking/src/event_processor.cpp b/mocking/src/event_processor.cpp index 407bac3c..12d80e61 100644 --- a/mocking/src/event_processor.cpp +++ b/mocking/src/event_processor.cpp @@ -1,5 +1,6 @@ #include +using std::thread; using std::lock_guard; using std::unique_lock; using std::mutex; @@ -10,9 +11,8 @@ using std::chrono::milliseconds; namespace cppkafka { namespace mocking { -EventProcessor::EventProcessor() -: processing_thread_(&EventProcessor::process_events, this) { - +EventProcessor::EventProcessor() { + processing_thread_ = thread(&EventProcessor::process_events, this); } EventProcessor::~EventProcessor() { @@ -50,10 +50,12 @@ void EventProcessor::process_events() { while (running_ && events_.empty()) { new_events_condition_.wait(lock); } - if (!running_) { break; } + if (events_.empty()) { + continue; + } EventPtr event = move(events_.front()); events_.pop(); diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index 7263b475..6d02dc86 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -18,6 +18,7 @@ using std::move; using std::tie; using std::lock_guard; using std::mutex; +using std::recursive_mutex; using std::iota; namespace cppkafka { @@ -35,7 +36,7 @@ KafkaCluster::KafkaCluster(string url) } KafkaCluster::~KafkaCluster() { - detail::KafkaClusterRegistry::instance().remove_cluster(*this); + detail::KafkaClusterRegistry::instance().remove_cluster(url_); } const string& KafkaCluster::get_url() const { @@ -76,10 +77,9 @@ const KafkaTopicMock& KafkaCluster::get_topic(const string& name) const { } void KafkaCluster::subscribe(const string& group_id, uint64_t consumer_id, - const vector& topics, - AssignmentCallback assignment_callback, + const vector& topics, AssignmentCallback assignment_callback, RevocationCallback revocation_callback) { - lock_guard _(consumer_data_mutex_); + lock_guard _(consumer_data_mutex_); auto iter = consumer_data_.find(consumer_id); // If it's already subscribed to something, unsubscribe from it if (iter != consumer_data_.end()) { @@ -87,7 +87,7 @@ void KafkaCluster::subscribe(const string& group_id, uint64_t consumer_id, } ConsumerMetadata data = { move(assignment_callback), - move(revocation_callback), + move(revocation_callback) }; iter = consumer_data_.emplace(consumer_id, move(data)).first; @@ -102,13 +102,13 @@ void KafkaCluster::subscribe(const string& group_id, uint64_t consumer_id, } void KafkaCluster::unsubscribe(const string& group_id, uint64_t consumer_id) { - lock_guard _(consumer_data_mutex_); + lock_guard _(consumer_data_mutex_); do_unsubscribe(group_id, consumer_id); } void KafkaCluster::assign(uint64_t consumer_id, const vector& topic_partitions, - const MessageCallback& message_callback) { - lock_guard _(consumer_data_mutex_); + ResetOffsetPolicy policy, const MessageCallback& message_callback) { + lock_guard _(consumer_data_mutex_); auto iter = consumer_data_.find(consumer_id); if (iter == consumer_data_.end()) { iter = consumer_data_.emplace(consumer_id, ConsumerMetadata{}).first; @@ -121,20 +121,25 @@ void KafkaCluster::assign(uint64_t consumer_id, const vector auto callback = bind(message_callback, topic_partition.get_topic(), topic_partition.get_partition(), _1, _2); partition.acquire([&]() { - uint64_t start_offset; - uint64_t end_offset; + int64_t start_offset; + int64_t end_offset; tie(start_offset, end_offset) = partition.get_offset_bounds(); - const uint64_t next_offset = topic_partition.get_offset(); - if (start_offset < next_offset) { - throw runtime_error("stored offset is too high"); + int64_t next_offset = topic_partition.get_offset(); + if (next_offset == RD_KAFKA_OFFSET_INVALID) { + switch (policy) { + case ResetOffsetPolicy::Earliest: + next_offset = start_offset; + break; + case ResetOffsetPolicy::Latest: + next_offset = end_offset; + break; + } } - // Nothing to fetch - if (next_offset == end_offset) { - return; - } - for (uint64_t i = next_offset; i != end_offset; ++i) { - const KafkaMessageMock& message = partition.get_message(i); - callback(i, message); + if (start_offset >= next_offset && next_offset != end_offset && end_offset > 0) { + for (auto i = next_offset; i != end_offset; ++i) { + const KafkaMessageMock& message = partition.get_message(i); + callback(i, message); + } } const auto subscriber_id = partition.subscribe(move(callback)); consumer.subscriptions[&topic].emplace(topic_partition.get_partition(), @@ -144,7 +149,7 @@ void KafkaCluster::assign(uint64_t consumer_id, const vector } void KafkaCluster::unassign(uint64_t consumer_id) { - lock_guard _(consumer_data_mutex_); + lock_guard _(consumer_data_mutex_); auto iter = consumer_data_.find(consumer_id); if (iter == consumer_data_.end()) { throw runtime_error("called unassign with unknown consumer id " + to_string(consumer_id)); @@ -224,14 +229,17 @@ void KafkaCluster::do_unsubscribe(const string& group_id, uint64_t consumer_id) // Revoke for all consumers generate_revocations(group_data); - for (const auto& topic_subscription : iter->second.subscriptions) { - const string& topic_name = topic_subscription.first->get_name(); - auto& topic_data = group_data[topic_name]; - topic_data.erase(consumer_id); - if (topic_data.empty()) { - group_data.erase(topic_name); + auto topic_iter = group_data.begin(); + while (topic_iter != group_data.end()) { + topic_iter->second.erase(consumer_id); + if (topic_iter->second.empty()) { + topic_iter = group_data.erase(topic_iter); + } + else { + ++topic_iter; } } + consumer_data_.erase(consumer_id); if (group_data.empty()) { // If we ran out of consumers for this group, erase it group_topics_data_.erase(group_id); diff --git a/mocking/src/kafka_cluster_registry.cpp b/mocking/src/kafka_cluster_registry.cpp index dc28cac4..22f4eea0 100644 --- a/mocking/src/kafka_cluster_registry.cpp +++ b/mocking/src/kafka_cluster_registry.cpp @@ -5,6 +5,8 @@ using std::string; using std::lock_guard; using std::mutex; using std::runtime_error; +using std::shared_ptr; +using std::weak_ptr; namespace cppkafka { namespace mocking { @@ -15,25 +17,25 @@ KafkaClusterRegistry& KafkaClusterRegistry::instance() { return registry; } -void KafkaClusterRegistry::add_cluster(ClusterPtr cluster) { +void KafkaClusterRegistry::add_cluster(shared_ptr cluster) { const string& url = cluster->get_url(); lock_guard _(clusters_mutex_); auto iter = clusters_.find(url); if (iter != clusters_.end()) { throw runtime_error("cluster already registered"); } - clusters_.emplace(url, move(cluster)); + clusters_.emplace(url, weak_ptr(cluster)); } -void KafkaClusterRegistry::remove_cluster(const KafkaCluster& cluster) { +void KafkaClusterRegistry::remove_cluster(const string& url) { lock_guard _(clusters_mutex_); - clusters_.erase(cluster.get_url()); + clusters_.erase(url); } -KafkaClusterRegistry::ClusterPtr KafkaClusterRegistry::get_cluster(const string& name) const { +shared_ptr KafkaClusterRegistry::get_cluster(const string& name) const { lock_guard _(clusters_mutex_); auto iter = clusters_.find(name); - return iter != clusters_.end() ? iter->second : ClusterPtr{}; + return iter != clusters_.end() ? iter->second.lock() : nullptr; } } // detail diff --git a/mocking/src/kafka_partition_mock.cpp b/mocking/src/kafka_partition_mock.cpp index b603afb6..3079fe59 100644 --- a/mocking/src/kafka_partition_mock.cpp +++ b/mocking/src/kafka_partition_mock.cpp @@ -33,7 +33,7 @@ void KafkaPartitionMock::add_message(KafkaMessageMock message) { const KafkaMessageMock& KafkaPartitionMock::get_message(uint64_t offset) const { const uint64_t index = offset - base_offset_; lock_guard _(mutex_); - if (messages_.size() >= index) { + if (index >= messages_.size()) { throw out_of_range("invalid message index"); } return messages_[index]; @@ -56,7 +56,7 @@ void KafkaPartitionMock::unsubscribe(SubscriberId id) { subscribers_.erase(id); } -tuple KafkaPartitionMock::get_offset_bounds() const { +tuple KafkaPartitionMock::get_offset_bounds() const { lock_guard _(mutex_); return make_tuple(base_offset_, base_offset_ + messages_.size()); } diff --git a/mocking/src/kafka_topic_mock.cpp b/mocking/src/kafka_topic_mock.cpp index 3d2231d9..86bd0031 100644 --- a/mocking/src/kafka_topic_mock.cpp +++ b/mocking/src/kafka_topic_mock.cpp @@ -23,7 +23,7 @@ const string& KafkaTopicMock::get_name() const { } void KafkaTopicMock::add_message(unsigned partition, KafkaMessageMock message) { - if (partitions_.size() >= partition) { + if (partition >= partitions_.size()) { throw out_of_range("invalid partition index"); } partitions_[partition].add_message(move(message)); From 1ea4e07432a60843f417dbecb14651dc26134606 Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Tue, 7 Nov 2017 19:55:09 -0800 Subject: [PATCH 19/20] Make mocked offset commits work --- examples/kafka_consumer.cpp | 1 + .../include/cppkafka/mocking/consumer_mock.h | 5 ++- .../include/cppkafka/mocking/kafka_cluster.h | 2 ++ mocking/src/api.cpp | 4 +-- mocking/src/consumer_mock.cpp | 35 ++++++++++++++++--- mocking/src/kafka_cluster.cpp | 5 +++ mocking/src/offset_manager.cpp | 4 +-- 7 files changed, 46 insertions(+), 10 deletions(-) diff --git a/examples/kafka_consumer.cpp b/examples/kafka_consumer.cpp index 69f06699..289d9804 100644 --- a/examples/kafka_consumer.cpp +++ b/examples/kafka_consumer.cpp @@ -65,6 +65,7 @@ int main(int argc, char* argv[]) { // Print the assigned partitions on assignment consumer.set_assignment_callback([](const TopicPartitionList& partitions) { cout << "Got assigned: " << partitions << endl; + std::cout << "Offset: " << partitions[0].get_offset() << std::endl; }); // Print the revoked partitions on revocation diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index 0266d7fb..f66feb42 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -32,6 +32,7 @@ class ConsumerMock : public HandleMock { ~ConsumerMock(); void close(); + void commit(const rd_kafka_message_t& message); void subscribe(const std::vector& topics); void unsubscribe(); void assign(const std::vector& topic_partitions); @@ -60,6 +61,7 @@ class ConsumerMock : public HandleMock { static TopicPartitionId make_id(const TopicPartitionMock& topic_partition); KafkaCluster::ResetOffsetPolicy get_offset_policy() const; bool get_partition_eof_enabled() const; + bool get_auto_commit() const; void on_assignment(const std::vector& topic_partitions); void on_revocation(); void on_message(const std::string& topic_name, unsigned partition, uint64_t offset, @@ -70,7 +72,8 @@ class ConsumerMock : public HandleMock { ConfigurationMock config_; std::string group_id_; const KafkaCluster::ResetOffsetPolicy offset_reset_policy_; - bool emit_eofs_; + const bool emit_eofs_; + const bool auto_commit_; std::map assigned_partitions_; std::set consumable_topic_partitions_; std::set paused_topic_partitions_; diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h index 7f5a9f04..6a4377b1 100644 --- a/mocking/include/cppkafka/mocking/kafka_cluster.h +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -47,6 +47,8 @@ class KafkaCluster { void assign(uint64_t consumer_id, const std::vector& topic_partitions, ResetOffsetPolicy policy, const MessageCallback& message_callback); void unassign(uint64_t consumer_id); + void commit(const std::string& group_id, uint64_t consumer_id, + const std::vector& topic_partitions); private: struct ConsumerMetadata { using PartitionSubscriptionMap = std::unordered_map; diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp index c6d2c6c9..4f2aa1d6 100644 --- a/mocking/src/api.cpp +++ b/mocking/src/api.cpp @@ -543,9 +543,9 @@ rd_kafka_resp_err_t rd_kafka_commit(rd_kafka_t* rk, const rd_kafka_topic_partiti return RD_KAFKA_RESP_ERR_NO_ERROR; } -rd_kafka_resp_err_t rd_kafka_commit_message(rd_kafka_t* rk, const rd_kafka_message_t* rkmessage, +rd_kafka_resp_err_t rd_kafka_commit_message(rd_kafka_t* rk, const rd_kafka_message_t* message, int async) { - // TODO: implement + rk->get_handle().commit(*message); return RD_KAFKA_RESP_ERR_NO_ERROR; } diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index dbbf303c..7171127e 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -39,7 +39,7 @@ ConsumerMock::ConsumerMock(ConfigurationMock config, EventProcessorPtr processor ClusterPtr cluster) : HandleMock(move(processor), move(cluster)), config_(move(config)), offset_reset_policy_(get_offset_policy()), emit_eofs_(get_partition_eof_enabled()), - consumer_id_(make_consumer_id()) { + auto_commit_(get_auto_commit()), consumer_id_(make_consumer_id()) { if (!config_.has_key(CONFIG_GROUP_ID)) { throw runtime_error("Failed to find " + CONFIG_GROUP_ID + " in config"); } @@ -54,6 +54,14 @@ void ConsumerMock::close() { unsubscribe(); } +void ConsumerMock::commit(const rd_kafka_message_t& message) { + get_cluster().commit( + group_id_, + consumer_id_, + { { rd_kafka_topic_name(message.rkt), message.partition, message.offset + 1 } } + ); +} + void ConsumerMock::subscribe(const vector& topics) { using namespace std::placeholders; get_cluster().subscribe( @@ -75,13 +83,12 @@ void ConsumerMock::assign(const vector& topic_partitions) { // Create entries for all topic partitions in our assigned partitions map for (const TopicPartitionMock& topic_partition : topic_partitions) { const auto id = make_id(topic_partition); - // We'll store the next offset from the one we've seen so far uint64_t next_offset; if (topic_partition.get_offset() == RD_KAFKA_OFFSET_INVALID) { next_offset = 0; } else { - next_offset = topic_partition.get_offset() + 1; + next_offset = topic_partition.get_offset(); } auto iter = assigned_partitions_.find(id); @@ -173,16 +180,20 @@ unique_ptr ConsumerMock::poll(milliseconds timeout) { } const auto& message = *aggregate.message; - return unique_ptr(new MessageHandle( + unique_ptr output(new MessageHandle( unique_ptr(new TopicHandle(get<0>(id), nullptr)), get<1>(id), - iter->second.next_offset, + aggregate.offset, (void*)message.get_key().data(), message.get_key().size(), (void*)message.get_payload().data(), message.get_payload().size(), RD_KAFKA_RESP_ERR__PARTITION_EOF, MessageHandlePrivateData{message.get_timestamp_type(), message.get_timestamp()}, MessageHandle::PointerOwnership::Unowned )); + if (auto_commit_) { + commit(output->get_message()); + } + return output; } vector ConsumerMock::get_assignment() const { @@ -227,6 +238,20 @@ bool ConsumerMock::get_partition_eof_enabled() const { return !config_.has_key(KEY_NAME) || config_.get(KEY_NAME) == "true"; } +bool ConsumerMock::get_auto_commit() const { + static const vector KEY_NAMES = { + "enable.auto.commit", + "auto.commit.enable" + }; + for (const string& key : KEY_NAMES) { + if (config_.has_key(key)) { + return config_.get(key) == "true"; + } + } + // By default, auto commit + return true; +} + void ConsumerMock::on_assignment(const vector& topic_partitions) { handle_rebalance(RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, topic_partitions); } diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp index 6d02dc86..6764b372 100644 --- a/mocking/src/kafka_cluster.cpp +++ b/mocking/src/kafka_cluster.cpp @@ -165,6 +165,11 @@ void KafkaCluster::unassign(uint64_t consumer_id) { consumer.subscriptions.clear(); } +void KafkaCluster::commit(const string& group_id, uint64_t consumer_id, + const vector& topic_partitions) { + offset_manager_->commit_offsets(group_id, topic_partitions); +} + void KafkaCluster::generate_assignments(const string& group_id, const TopicConsumersMap& topic_consumers) { for (const auto& topic_consumers_pair : topic_consumers) { diff --git a/mocking/src/offset_manager.cpp b/mocking/src/offset_manager.cpp index d3a3acd1..e845abd3 100644 --- a/mocking/src/offset_manager.cpp +++ b/mocking/src/offset_manager.cpp @@ -18,7 +18,7 @@ void OffsetManager::commit_offsets(const string& group_id, topic_partition.get_partition()); auto iter = offsets_.find(key); if (iter == offsets_.end()) { - offsets_.emplace(key, topic_partition.get_offset()).first; + offsets_.emplace(key, topic_partition.get_offset()); } else { iter->second = topic_partition.get_offset(); @@ -31,7 +31,7 @@ OffsetManager::get_offsets(const string& group_id, vector topic_partitions) const { lock_guard _(offsets_mutex_); for (TopicPartitionMock& topic_partition : topic_partitions) { - if (topic_partition.get_offset() != RD_KAFKA_OFFSET_INVALID) { + if (topic_partition.get_offset() == RD_KAFKA_OFFSET_INVALID) { auto iter = offsets_.find(make_tuple(group_id, topic_partition.get_topic(), topic_partition.get_partition())); if (iter != offsets_.end()) { From 9d45aa080aa346ddea29abd3a62cb8b14b15d1eb Mon Sep 17 00:00:00 2001 From: Matias Fontanini Date: Tue, 7 Nov 2017 20:07:12 -0800 Subject: [PATCH 20/20] Allow committing topic/partitions --- mocking/include/cppkafka/mocking/consumer_mock.h | 1 + mocking/src/api.cpp | 2 +- mocking/src/consumer_mock.cpp | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h index f66feb42..da5535df 100644 --- a/mocking/include/cppkafka/mocking/consumer_mock.h +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -33,6 +33,7 @@ class ConsumerMock : public HandleMock { void close(); void commit(const rd_kafka_message_t& message); + void commit(const std::vector& topic_partitions); void subscribe(const std::vector& topics); void unsubscribe(); void assign(const std::vector& topic_partitions); diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp index 4f2aa1d6..328f0419 100644 --- a/mocking/src/api.cpp +++ b/mocking/src/api.cpp @@ -539,7 +539,7 @@ rd_kafka_resp_err_t rd_kafka_committed(rd_kafka_t* rk, rd_kafka_topic_partition_ rd_kafka_resp_err_t rd_kafka_commit(rd_kafka_t* rk, const rd_kafka_topic_partition_list_t* offsets, int async) { - // TODO: implement + rk->get_handle().commit(from_rdkafka_handle(*offsets)); return RD_KAFKA_RESP_ERR_NO_ERROR; } diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp index 7171127e..5d4dd8b4 100644 --- a/mocking/src/consumer_mock.cpp +++ b/mocking/src/consumer_mock.cpp @@ -62,6 +62,10 @@ void ConsumerMock::commit(const rd_kafka_message_t& message) { ); } +void ConsumerMock::commit(const vector& topic_partitions) { + get_cluster().commit(group_id_,consumer_id_, topic_partitions); +} + void ConsumerMock::subscribe(const vector& topics) { using namespace std::placeholders; get_cluster().subscribe( pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy