diff --git a/CMakeLists.txt b/CMakeLists.txt index 249ac4fd..b47a4aaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ find_package(Boost REQUIRED) find_package(RdKafka REQUIRED) add_subdirectory(src) +add_subdirectory(mocking) add_subdirectory(include) add_subdirectory(examples) @@ -95,4 +96,4 @@ configure_file( # Add uninstall target add_custom_target(uninstall - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) \ No newline at end of file + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) diff --git a/examples/kafka_consumer.cpp b/examples/kafka_consumer.cpp index 69f06699..289d9804 100644 --- a/examples/kafka_consumer.cpp +++ b/examples/kafka_consumer.cpp @@ -65,6 +65,7 @@ int main(int argc, char* argv[]) { // Print the assigned partitions on assignment consumer.set_assignment_callback([](const TopicPartitionList& partitions) { cout << "Got assigned: " << partitions << endl; + std::cout << "Offset: " << partitions[0].get_offset() << std::endl; }); // Print the revoked partitions on revocation diff --git a/include/cppkafka/clonable_ptr.h b/include/cppkafka/clonable_ptr.h index 842e3088..505f0985 100644 --- a/include/cppkafka/clonable_ptr.h +++ b/include/cppkafka/clonable_ptr.h @@ -84,6 +84,13 @@ class ClonablePtr { T* get() const { return handle_.get(); } + + /** + * Resets the internal pointer + */ + void reset(T* ptr) { + handle_.reset(ptr); + } private: std::unique_ptr handle_; Cloner cloner_; diff --git a/mocking/CMakeLists.txt b/mocking/CMakeLists.txt new file mode 100644 index 00000000..febd4f0a --- /dev/null +++ b/mocking/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(src) diff --git a/mocking/include/cppkafka/mocking/api.h b/mocking/include/cppkafka/mocking/api.h new file mode 100644 index 00000000..0c4c6c28 --- /dev/null +++ b/mocking/include/cppkafka/mocking/api.h @@ -0,0 +1,21 @@ +#ifndef CPPKAFKA_MOCKING_API_H +#define CPPKAFKA_MOCKING_API_H + +#include +#include +#include +#include + +struct rd_kafka_conf_s : cppkafka::mocking::HandleWrapper { + using cppkafka::mocking::HandleWrapper::HandleWrapper; +}; + +struct rd_kafka_topic_conf_s : rd_kafka_conf_s { + using rd_kafka_conf_s::rd_kafka_conf_s; +}; + +struct rd_kafka_s : cppkafka::mocking::HandleWrapper { + using cppkafka::mocking::HandleWrapper::HandleWrapper; +}; + +#endif // CPPKAFKA_MOCKING_API_H diff --git a/mocking/include/cppkafka/mocking/configuration_mock.h b/mocking/include/cppkafka/mocking/configuration_mock.h new file mode 100644 index 00000000..fc8aee82 --- /dev/null +++ b/mocking/include/cppkafka/mocking/configuration_mock.h @@ -0,0 +1,100 @@ +#ifndef CPPKAFKA_MOCKING_CONFIGURATION_MOCK_H +#define CPPKAFKA_MOCKING_CONFIGURATION_MOCK_H + +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class ConfigurationMock { +public: + using DeliveryReportCallback = void(rd_kafka_t*, const rd_kafka_message_t*, void *); + using RebalanceCallback = void(rd_kafka_t*, rd_kafka_resp_err_t, + rd_kafka_topic_partition_list_t*, void*); + using OffsetCommitCallback = void(rd_kafka_t*, rd_kafka_resp_err_t, + rd_kafka_topic_partition_list_t*, void*); + using ErrorCallback = void(rd_kafka_t*, int, const char*, void*); + using ThrottleCallback = void(rd_kafka_t*, const char*, int32_t, int, void*); + using LogCallback = void(const rd_kafka_t*, int, const char*, const char*); + using StatsCallback = int(rd_kafka_t*, char*, size_t, void*); + using SocketCallback = int(int, int, int, void*); + using PartitionerCallback = int32_t(const rd_kafka_topic_t*, const void*, + size_t, int32_t, void*, void*); + + ConfigurationMock(); + + void set(std::string key, std::string value); + size_t has_key(const std::string& key) const; + std::string get(const std::string& key) const; + + void set_delivery_report_callback(DeliveryReportCallback* callback); + void set_rebalance_callback(RebalanceCallback* callback); + void set_offset_commit_callback(OffsetCommitCallback* callback); + void set_error_callback(ErrorCallback* callback); + void set_throttle_callback(ThrottleCallback* callback); + void set_log_callback(LogCallback* callback); + void set_stats_callback(StatsCallback* callback); + void set_socket_callback(SocketCallback* callback); + void set_partitioner_callback(PartitionerCallback* callback); + void set_default_topic_configuration(const ConfigurationMock& conf); + void set_opaque(void* ptr); + + DeliveryReportCallback* get_delivery_report_callback() const; + RebalanceCallback* get_rebalance_callback() const; + OffsetCommitCallback* get_offset_commit_callback() const; + ErrorCallback* get_error_callback() const; + ThrottleCallback* get_throttle_callback() const; + LogCallback* get_log_callback() const; + StatsCallback* get_stats_callback() const; + SocketCallback* get_socket_callback() const; + PartitionerCallback* get_partitioner_callback() const; + const ConfigurationMock* get_default_topic_configuration() const; + void* get_opaque() const; + std::unordered_map get_options() const; +private: + enum class Callback { + DeliveryReport, + Rebalance, + OffsetCommit, + Error, + Throttle, + Log, + Stats, + Socket, + Partitioner + }; + + struct Hasher { + size_t operator()(Callback c) const { + return static_cast(c); + } + }; + + struct Cloner { + ConfigurationMock* operator()(const ConfigurationMock* ptr) const { + return ptr ? new ConfigurationMock(*ptr) : nullptr; + } + }; + + using TopicConfigirationPtr = ClonablePtr, + Cloner>; + + template + void set_callback(Callback type, Functor* ptr); + template + Functor* get_callback(Callback type) const; + + std::unordered_map options_; + std::unordered_map callbacks_; + TopicConfigirationPtr default_topic_configuration_; + void* opaque_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_CONFIGURATION_MOCK_H diff --git a/mocking/include/cppkafka/mocking/consumer_mock.h b/mocking/include/cppkafka/mocking/consumer_mock.h new file mode 100644 index 00000000..da5535df --- /dev/null +++ b/mocking/include/cppkafka/mocking/consumer_mock.h @@ -0,0 +1,89 @@ +#ifndef CPPKAFKA_MOCKING_CONSUMER_MOCK_H +#define CPPKAFKA_MOCKING_CONSUMER_MOCK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaMessageMock; +class KafkaTopicMock; + +class ConsumerMock : public HandleMock { +public: + ConsumerMock(ConfigurationMock config, EventProcessorPtr processor, ClusterPtr cluster); + ConsumerMock(const ConsumerMock&) = delete; + ConsumerMock& operator=(const ConsumerMock&) = delete; + ~ConsumerMock(); + + void close(); + void commit(const rd_kafka_message_t& message); + void commit(const std::vector& topic_partitions); + void subscribe(const std::vector& topics); + void unsubscribe(); + void assign(const std::vector& topic_partitions); + void unassign(); + void pause_partitions(const std::vector& topic_partitions); + void resume_partitions(const std::vector& topic_partitions); + std::unique_ptr poll(std::chrono::milliseconds timeout); + std::vector get_assignment() const; +private: + static uint64_t make_consumer_id(); + + struct MessageAggregate { + std::string topic; + unsigned partition; + uint64_t offset; + const KafkaMessageMock* message; + }; + + struct TopicPartitionInfo { + uint64_t next_offset; + std::queue messages; + }; + + using TopicPartitionId = std::tuple; + + static TopicPartitionId make_id(const TopicPartitionMock& topic_partition); + KafkaCluster::ResetOffsetPolicy get_offset_policy() const; + bool get_partition_eof_enabled() const; + bool get_auto_commit() const; + void on_assignment(const std::vector& topic_partitions); + void on_revocation(); + void on_message(const std::string& topic_name, unsigned partition, uint64_t offset, + const KafkaMessageMock& message); + void handle_rebalance(rd_kafka_resp_err_t type, + const std::vector& topic_partitions); + + ConfigurationMock config_; + std::string group_id_; + const KafkaCluster::ResetOffsetPolicy offset_reset_policy_; + const bool emit_eofs_; + const bool auto_commit_; + std::map assigned_partitions_; + std::set consumable_topic_partitions_; + std::set paused_topic_partitions_; + mutable std::mutex mutex_; + std::condition_variable messages_condition_; + uint64_t consumer_id_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_CONSUMER_MOCK_H diff --git a/mocking/include/cppkafka/mocking/event_processor.h b/mocking/include/cppkafka/mocking/event_processor.h new file mode 100644 index 00000000..1b0609ac --- /dev/null +++ b/mocking/include/cppkafka/mocking/event_processor.h @@ -0,0 +1,41 @@ +#ifndef CPPKAFKA_MOCKING_EVENT_PROCESSOR_ +#define CPPKAFKA_MOCKING_EVENT_PROCESSOR_ + +#include +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class EventProcessor { +public: + using EventPtr = std::unique_ptr; + + EventProcessor(); + EventProcessor(const EventProcessor&) = delete; + EventProcessor& operator=(const EventProcessor&) = delete; + ~EventProcessor(); + + void add_event(EventPtr event); + size_t get_event_count() const; + bool wait_until_empty(std::chrono::milliseconds timeout); +private: + void process_events(); + + std::thread processing_thread_; + mutable std::mutex events_mutex_; + std::condition_variable new_events_condition_; + std::condition_variable no_events_condition_; + std::queue events_; + bool running_{true}; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_EVENT_PROCESSOR_ diff --git a/mocking/include/cppkafka/mocking/events/event_base.h b/mocking/include/cppkafka/mocking/events/event_base.h new file mode 100644 index 00000000..1d601a83 --- /dev/null +++ b/mocking/include/cppkafka/mocking/events/event_base.h @@ -0,0 +1,30 @@ +#ifndef CPPKAFKA_MOCKING_EVENT_BASE_H +#define CPPKAFKA_MOCKING_EVENT_BASE_H + +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaCluster; + +class EventBase { +public: + using ClusterPtr = std::shared_ptr; + + EventBase(ClusterPtr cluster); + virtual ~EventBase() = default; + + void execute(); + virtual std::string get_type() const = 0; +private: + virtual void execute_event(KafkaCluster& cluster) = 0; + + ClusterPtr cluster_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_EVENT_BASE_H diff --git a/mocking/include/cppkafka/mocking/events/produce_message_event.h b/mocking/include/cppkafka/mocking/events/produce_message_event.h new file mode 100644 index 00000000..207a4687 --- /dev/null +++ b/mocking/include/cppkafka/mocking/events/produce_message_event.h @@ -0,0 +1,28 @@ +#ifndef CPPKAFKA_MOCKING_PRODUCE_MESSAGE_EVENT_H +#define CPPKAFKA_MOCKING_PRODUCE_MESSAGE_EVENT_H + +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class ProduceMessageEvent : public EventBase { +public: + ProduceMessageEvent(ClusterPtr cluster, MessageHandle message_handle); + + std::string get_type() const; +private: + void execute_event(KafkaCluster& cluster); + + MessageHandle message_handle_; + std::string topic_; + unsigned partition_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_PRODUCE_MESSAGE_EVENT_H diff --git a/mocking/include/cppkafka/mocking/handle_mock.h b/mocking/include/cppkafka/mocking/handle_mock.h new file mode 100644 index 00000000..61bf492f --- /dev/null +++ b/mocking/include/cppkafka/mocking/handle_mock.h @@ -0,0 +1,46 @@ +#ifndef CPPKAFKA_MOCKING_HANDLE_MOCK_H +#define CPPKAFKA_MOCKING_HANDLE_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaCluster; + +class HandleMock { +public: + using ClusterPtr = std::shared_ptr; + using EventProcessorPtr = std::shared_ptr; + + HandleMock(EventProcessorPtr processor); + HandleMock(EventProcessorPtr processor, ClusterPtr cluster); + virtual ~HandleMock() = default; + + void* get_opaque() const; + size_t get_event_count() const; + void set_cluster(ClusterPtr cluster); + void set_opaque(void* opaque); + KafkaCluster& get_cluster(); + const KafkaCluster& get_cluster() const; +protected: + using EventPtr = EventProcessor::EventPtr; + + void generate_event(EventPtr event); + template + void generate_event(Args&&... args) { + generate_event(EventPtr(new T(cluster_, std::forward(args)...))); + } + EventProcessor& get_event_processor(); +private: + EventProcessorPtr processor_; + ClusterPtr cluster_; + void* opaque_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_HANDLE_MOCK_H diff --git a/mocking/include/cppkafka/mocking/handle_wrapper.h b/mocking/include/cppkafka/mocking/handle_wrapper.h new file mode 100644 index 00000000..de03fab9 --- /dev/null +++ b/mocking/include/cppkafka/mocking/handle_wrapper.h @@ -0,0 +1,54 @@ +#ifndef CPPKAFKA_MOCKING_HANDLE_WRAPPER_H +#define CPPKAFKA_MOCKING_HANDLE_WRAPPER_H + +#include +#include + +namespace cppkafka { +namespace mocking { + +template +class HandleWrapper { +public: + HandleWrapper() + : handle_(new T()) { + + } + + template + HandleWrapper(const Args&... args) + : handle_(new T(args...)) { + + } + + template + explicit HandleWrapper(U* ptr) + : handle_(ptr) { + + } + + T& get_handle() { + return *handle_; + } + + const T& get_handle() const { + return *handle_; + } + + template + U& get_handle() { + return static_cast(get_handle()); + } + + template + const U& get_handle() const { + return static_cast(get_handle()); + } +private: + std::unique_ptr handle_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_HANDLE_WRAPPER_H diff --git a/mocking/include/cppkafka/mocking/kafka_cluster.h b/mocking/include/cppkafka/mocking/kafka_cluster.h new file mode 100644 index 00000000..6a4377b1 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_cluster.h @@ -0,0 +1,84 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_CLUSTER_H +#define CPPKAFKA_MOCKING_KAFKA_CLUSTER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaCluster { +public: + using AssignmentCallback = std::function&)>; + using RevocationCallback = std::function; + using MessageCallback = std::function; + + enum class ResetOffsetPolicy { + Earliest = 1, + Latest = 2 + }; + + static std::shared_ptr make_cluster(std::string url); + + KafkaCluster(const KafkaCluster&) = delete; + KafkaCluster& operator=(const KafkaCluster&) = delete; + ~KafkaCluster(); + + const std::string& get_url() const; + + void create_topic(const std::string& name, unsigned partitions); + bool topic_exists(const std::string& name) const; + void produce(const std::string& topic, unsigned partition, KafkaMessageMock message); + KafkaTopicMock& get_topic(const std::string& name); + const KafkaTopicMock& get_topic(const std::string& name) const; + void subscribe(const std::string& group_id, uint64_t consumer_id, + const std::vector& topics, + AssignmentCallback assignment_callback, + RevocationCallback revocation_callback); + void unsubscribe(const std::string& group_id, uint64_t consumer_id); + void assign(uint64_t consumer_id, const std::vector& topic_partitions, + ResetOffsetPolicy policy, const MessageCallback& message_callback); + void unassign(uint64_t consumer_id); + void commit(const std::string& group_id, uint64_t consumer_id, + const std::vector& topic_partitions); +private: + struct ConsumerMetadata { + using PartitionSubscriptionMap = std::unordered_map; + + const AssignmentCallback assignment_callback; + const RevocationCallback revocation_callback; + std::vector partitions_assigned; + std::unordered_map subscriptions; + }; + + using ConsumerSet = std::unordered_set; + using TopicConsumersMap = std::unordered_map; + + KafkaCluster(std::string url); + + void generate_assignments(const std::string& group_id, + const TopicConsumersMap& topic_consumers); + void generate_revocations(const TopicConsumersMap& topic_consumers); + void do_unsubscribe(const std::string& group_id, uint64_t consumer_id); + + const std::string url_; + std::shared_ptr offset_manager_; + std::unordered_map topics_; + mutable std::mutex topics_mutex_; + std::unordered_map consumer_data_; + std::unordered_map group_topics_data_; + mutable std::recursive_mutex consumer_data_mutex_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_CLUSTER_H diff --git a/mocking/include/cppkafka/mocking/kafka_cluster_registry.h b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h new file mode 100644 index 00000000..f6d33ecd --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_cluster_registry.h @@ -0,0 +1,28 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_CLUSTER_REGISTRY_H +#define CPPKAFKA_MOCKING_KAFKA_CLUSTER_REGISTRY_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { +namespace detail { + +class KafkaClusterRegistry { +public: + static KafkaClusterRegistry& instance(); + + void add_cluster(std::shared_ptr cluster); + void remove_cluster(const std::string& url); + std::shared_ptr get_cluster(const std::string& name) const; +private: + std::unordered_map> clusters_; + mutable std::mutex clusters_mutex_; +}; + +} // detail +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_CLUSTER_REGISTRY_H diff --git a/mocking/include/cppkafka/mocking/kafka_message_mock.h b/mocking/include/cppkafka/mocking/kafka_message_mock.h new file mode 100644 index 00000000..5ed568f6 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_message_mock.h @@ -0,0 +1,32 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_MESSAGE_MOCK_H +#define CPPKAFKA_MOCKING_KAFKA_MESSAGE_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaMessageMock { +public: + using Buffer = std::vector; + + KafkaMessageMock(Buffer key, Buffer payload, rd_kafka_timestamp_type_t timestamp_type, + int64_t timestamp); + + const Buffer& get_key() const; + const Buffer& get_payload() const; + rd_kafka_timestamp_type_t get_timestamp_type() const; + int64_t get_timestamp() const; +private: + const Buffer key_; + const Buffer payload_; + rd_kafka_timestamp_type_t timestamp_type_; + int64_t timestamp_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_MESSAGE_MOCK_H diff --git a/mocking/include/cppkafka/mocking/kafka_partition_mock.h b/mocking/include/cppkafka/mocking/kafka_partition_mock.h new file mode 100644 index 00000000..a041ce63 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_partition_mock.h @@ -0,0 +1,50 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_PARTITION_MOCK_H +#define CPPKAFKA_MOCKING_KAFKA_PARTITION_MOCK_H + +#include +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaPartitionMock { +public: + using MessageCallback = std::function; + using SubscriberId = uint64_t; + + void add_message(KafkaMessageMock message); + const KafkaMessageMock& get_message(uint64_t offset) const; + size_t get_message_count() const; + SubscriberId subscribe(MessageCallback callback); + void unsubscribe(SubscriberId id); + // Returns interval [lowest offset, largest offset) + std::tuple get_offset_bounds() const; + + // Acquire this partition so that no messages can be produced while the callback is executed. + template + void acquire(const Functor& functor); +private: + std::vector get_subscriber_callbacks() const; + + uint64_t base_offset_{0}; + SubscriberId current_subscriber_id_{0}; + std::deque messages_; + std::unordered_map subscribers_; + mutable std::recursive_mutex mutex_; +}; + +template +void KafkaPartitionMock::acquire(const Functor& functor) { + std::lock_guard _(mutex_); + functor(); +} + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_PARTITION_MOCK_H diff --git a/mocking/include/cppkafka/mocking/kafka_topic_mock.h b/mocking/include/cppkafka/mocking/kafka_topic_mock.h new file mode 100644 index 00000000..2f7dd3c9 --- /dev/null +++ b/mocking/include/cppkafka/mocking/kafka_topic_mock.h @@ -0,0 +1,38 @@ +#ifndef CPPKAFKA_MOCKING_KAFKA_TOPIC_MOCK_H +#define CPPKAFKA_MOCKING_KAFKA_TOPIC_MOCK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaMessageMock; + +class KafkaTopicMock { +public: + KafkaTopicMock(std::string name, unsigned partition_count); + const std::string& get_name() const; + + void add_message(unsigned partition, KafkaMessageMock message); + KafkaPartitionMock& get_partition(unsigned partition); + const KafkaPartitionMock& get_partition(unsigned partition) const; + size_t get_partition_count() const; +private: + + const std::string name_; + std::vector partitions_; + mutable std::mutex subscribers_mutex_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_KAFKA_TOPIC_MOCK_H diff --git a/mocking/include/cppkafka/mocking/message_handle.h b/mocking/include/cppkafka/mocking/message_handle.h new file mode 100644 index 00000000..904109cf --- /dev/null +++ b/mocking/include/cppkafka/mocking/message_handle.h @@ -0,0 +1,62 @@ +#ifndef CPPKAFKA_MOCKING_MESSAGE_HANDLE_H +#define CPPKAFKA_MOCKING_MESSAGE_HANDLE_H + +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class KafkaMessageMock; +class MessageHandle; + +class MessageHandlePrivateData { +public: + MessageHandlePrivateData() = default; + MessageHandlePrivateData(rd_kafka_timestamp_type_t timestamp_type, int64_t timestamp); + + rd_kafka_timestamp_type_t get_timestamp_type() const; + int64_t get_timestamp() const; + MessageHandle* get_owner() const; + void set_owner(MessageHandle* handle); + void set_opaque(void* opaque); +private: + rd_kafka_timestamp_type_t timestamp_type_; + int64_t timestamp_; + MessageHandle* owner_{nullptr}; + void* opaque_; +}; + +class MessageHandle { +public: + enum class PointerOwnership { + Owned, + Unowned + }; + + MessageHandle(std::unique_ptr topic, int partition, int64_t offset, void* key, + size_t key_size, void* payload, size_t payload_size, int error_code, + MessageHandlePrivateData private_data, PointerOwnership ownership); + MessageHandle(MessageHandle&& other); + MessageHandle& operator=(MessageHandle&& other); + ~MessageHandle(); + + const TopicHandle& get_topic() const; + rd_kafka_message_t& get_message(); + const rd_kafka_message_t& get_message() const; + KafkaMessageMock make_message_mock() const; +private: + void set_private_data_pointer(); + + std::unique_ptr topic_; + rd_kafka_message_t message_{}; + MessageHandlePrivateData private_data_; + PointerOwnership ownership_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_MESSAGE_HANDLE_H diff --git a/mocking/include/cppkafka/mocking/offset_manager.h b/mocking/include/cppkafka/mocking/offset_manager.h new file mode 100644 index 00000000..775259cb --- /dev/null +++ b/mocking/include/cppkafka/mocking/offset_manager.h @@ -0,0 +1,34 @@ +#ifndef CPPKAFKA_MOCKING_OFFSET_MANAGER_H +#define CPPKAFKA_MOCKING_OFFSET_MANAGER_H + +#include +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class TopicPartitionMock; + +class OffsetManager { +public: + void commit_offsets(const std::string& group_id, const std::vector&); + std::vector get_offsets(const std::string& group_id, + std::vector) const; +private: + // (consumer, topic, partition) + using OffsetIdentifier = std::tuple; + using OffsetMap = std::map; + + OffsetMap offsets_; + mutable std::mutex offsets_mutex_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_OFFSET_MANAGER_H + diff --git a/mocking/include/cppkafka/mocking/producer_mock.h b/mocking/include/cppkafka/mocking/producer_mock.h new file mode 100644 index 00000000..ffccce9d --- /dev/null +++ b/mocking/include/cppkafka/mocking/producer_mock.h @@ -0,0 +1,27 @@ +#ifndef CPPKAFKA_MOCKING_PRODUCER_MOCK_H +#define CPPKAFKA_MOCKING_PRODUCER_MOCK_H + +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class ProducerMock : public HandleMock { +public: + using ClusterPtr = std::shared_ptr; + + ProducerMock(ConfigurationMock config, EventProcessorPtr processor, ClusterPtr cluster); + + void produce(MessageHandle message_handle); + bool flush(std::chrono::milliseconds timeout); + size_t poll(std::chrono::milliseconds timeout); +private: + ConfigurationMock config_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_PRODUCER_MOCK_H diff --git a/mocking/include/cppkafka/mocking/topic_handle.h b/mocking/include/cppkafka/mocking/topic_handle.h new file mode 100644 index 00000000..a4a6e163 --- /dev/null +++ b/mocking/include/cppkafka/mocking/topic_handle.h @@ -0,0 +1,31 @@ +#ifndef CPPKAFKA_MOCKING_TOPIC_HANDLE_H +#define CPPKAFKA_MOCKING_TOPIC_HANDLE_H + +#include + +namespace cppkafka { +namespace mocking { + +class TopicHandle { +public: + TopicHandle(std::string topic, void* opaque) + : topic_(move(topic)), opaque_(opaque) { + + } + + const std::string& get_topic() const { + return topic_; + } + + void* get_opaque() const { + return opaque_; + } +private: + const std::string topic_; + void* opaque_; +}; + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_TOPIC_HANDLE_H diff --git a/mocking/include/cppkafka/mocking/topic_partition_mock.h b/mocking/include/cppkafka/mocking/topic_partition_mock.h new file mode 100644 index 00000000..90cbf9de --- /dev/null +++ b/mocking/include/cppkafka/mocking/topic_partition_mock.h @@ -0,0 +1,38 @@ +#ifndef CPPKAFKA_MOCKING_TOPIC_PARTITION_MOCK_H +#define CPPKAFKA_MOCKING_TOPIC_PARTITION_MOCK_H + +#include +#include +#include +#include +#include + +namespace cppkafka { +namespace mocking { + +class TopicPartitionMock { +public: + TopicPartitionMock(std::string topic, int partition, int64_t offset = RD_KAFKA_OFFSET_INVALID); + + const std::string& get_topic() const; + int get_partition() const; + int64_t get_offset() const; + + void set_offset(int64_t offset); +private: + const std::string topic_; + const int partition_; + int64_t offset_; +}; + +using TopicPartitionMockListPtr = std::unique_ptr; +TopicPartitionMockListPtr +to_rdkafka_handle(const std::vector& topic_partitions); +std::vector +from_rdkafka_handle(const rd_kafka_topic_partition_list_t& topic_partitions); + +} // mocking +} // cppkafka + +#endif // CPPKAFKA_MOCKING_TOPIC_PARTITION_MOCK_H diff --git a/mocking/src/CMakeLists.txt b/mocking/src/CMakeLists.txt new file mode 100644 index 00000000..0ca6c79c --- /dev/null +++ b/mocking/src/CMakeLists.txt @@ -0,0 +1,30 @@ +set(SOURCES + configuration_mock.cpp + kafka_topic_mock.cpp + kafka_partition_mock.cpp + kafka_message_mock.cpp + kafka_cluster.cpp + kafka_cluster_registry.cpp + handle_mock.cpp + message_handle.cpp + producer_mock.cpp + consumer_mock.cpp + topic_partition_mock.cpp + offset_manager.cpp + + events/event_base.cpp + events/produce_message_event.cpp + event_processor.cpp + + api.cpp +) + +include_directories(${PROJECT_SOURCE_DIR}/mocking/include/ + ${PROJECT_SOURCE_DIR}/include/) +include_directories(SYSTEM ${Boost_INCLUDE_DIRS} ${RDKAFKA_INCLUDE_DIR}) + +add_library(cppkafka_mock EXCLUDE_FROM_ALL ${CPPKAFKA_LIBRARY_TYPE} ${SOURCES}) +set_target_properties(cppkafka_mock PROPERTIES VERSION ${CPPKAFKA_VERSION} + SOVERSION ${CPPKAFKA_VERSION}) +add_dependencies(cppkafka_mock cppkafka) +target_link_libraries(cppkafka_mock ${RDKAFKA_LIBRARY}) diff --git a/mocking/src/api.cpp b/mocking/src/api.cpp new file mode 100644 index 00000000..328f0419 --- /dev/null +++ b/mocking/src/api.cpp @@ -0,0 +1,589 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using std::string; +using std::copy; +using std::vector; +using std::strlen; +using std::move; +using std::make_shared; +using std::unique_ptr; +using std::tie; +using std::exception; + +using std::chrono::milliseconds; + +using namespace cppkafka::mocking; +using namespace cppkafka::mocking::detail; + +// rd_kafka_conf_t + +rd_kafka_conf_t* rd_kafka_conf_new() { + return new rd_kafka_conf_t(); +} + +void rd_kafka_conf_destroy(rd_kafka_conf_t* conf) { + delete conf; +} + +rd_kafka_conf_t* rd_kafka_conf_dup(const rd_kafka_conf_t* conf) { + return new rd_kafka_conf_t(conf->get_handle()); +} + +rd_kafka_conf_res_t rd_kafka_conf_set(rd_kafka_conf_t* conf, + const char* name, + const char* value, + char*, size_t) { + conf->get_handle().set(name, value); + return RD_KAFKA_CONF_OK; +} + +rd_kafka_conf_res_t rd_kafka_conf_get(const rd_kafka_conf_t* conf, + const char* name_raw, + char* dest, size_t* dest_size) { + const string name = name_raw; + if (!conf->get_handle().has_key(name)) { + return RD_KAFKA_CONF_UNKNOWN; + } + if (dest == nullptr) { + *dest_size = conf->get_handle().get(name).size(); + } + else { + const string value = conf->get_handle().get(name); + if (value.size() > *dest_size + 1) { + return RD_KAFKA_CONF_INVALID; + } + else { + copy(value.begin(), value.end(), dest); + } + } + return RD_KAFKA_CONF_OK; +} + +void rd_kafka_conf_set_dr_msg_cb(rd_kafka_conf_t* conf, + ConfigurationMock::DeliveryReportCallback* cb) { + conf->get_handle().set_delivery_report_callback(cb); +} + +void rd_kafka_conf_set_rebalance_cb(rd_kafka_conf_t* conf, + ConfigurationMock::RebalanceCallback* cb) { + conf->get_handle().set_rebalance_callback(cb); +} + +void rd_kafka_conf_set_offset_commit_cb(rd_kafka_conf_t* conf, + ConfigurationMock::OffsetCommitCallback* cb) { + conf->get_handle().set_offset_commit_callback(cb); +} + +void rd_kafka_conf_set_error_cb(rd_kafka_conf_t* conf, + ConfigurationMock::ErrorCallback* cb) { + conf->get_handle().set_error_callback(cb); +} + +void rd_kafka_conf_set_throttle_cb(rd_kafka_conf_t* conf, + ConfigurationMock::ThrottleCallback* cb) { + conf->get_handle().set_throttle_callback(cb); +} + +void rd_kafka_conf_set_log_cb(rd_kafka_conf_t* conf, + ConfigurationMock::LogCallback* cb) { + conf->get_handle().set_log_callback(cb); +} + +void rd_kafka_conf_set_stats_cb(rd_kafka_conf_t* conf, + ConfigurationMock::StatsCallback* cb) { + conf->get_handle().set_stats_callback(cb); +} + +void rd_kafka_conf_set_socket_cb(rd_kafka_conf_t* conf, + ConfigurationMock::SocketCallback* cb) { + conf->get_handle().set_socket_callback(cb); +} + +void rd_kafka_topic_conf_set_partitioner_cb(rd_kafka_topic_conf_t* conf, + ConfigurationMock::PartitionerCallback* cb) { + conf->get_handle().set_partitioner_callback(cb); +} + +void rd_kafka_conf_set_default_topic_conf(rd_kafka_conf_t* conf, + rd_kafka_topic_conf_t* tconf) { + conf->get_handle().set_default_topic_configuration(tconf->get_handle()); + rd_kafka_topic_conf_destroy(tconf); +} + +void rd_kafka_conf_set_opaque(rd_kafka_conf_t* conf, void* opaque) { + conf->get_handle().set_opaque(opaque); +} + +void rd_kafka_topic_conf_set_opaque(rd_kafka_topic_conf_t* conf, void* opaque) { + conf->get_handle().set_opaque(opaque); +} + +const char** rd_kafka_conf_dump(rd_kafka_conf_t* conf, size_t* cntp) { + const auto options = conf->get_handle().get_options(); + *cntp = options.size() * 2; + // Allocate enough for all (key, value) pairs + char** output = new char*[*cntp]; + size_t i = 0; + const auto set_value = [&](const string& value) { + output[i] = new char[value.size() + 1]; + copy(value.begin(), value.end(), output[i]); + ++i; + }; + for (const auto& option : options) { + set_value(option.first); + set_value(option.second); + } + return const_cast(output); +} + +void rd_kafka_conf_dump_free(const char** arr, size_t cnt) { + for (size_t i = 0; i < cnt; ++i) { + delete[] arr[i]; + } + delete[] arr; +} + +// rd_kafka_topic_conf_t +rd_kafka_topic_conf_t* rd_kafka_topic_conf_new() { + return new rd_kafka_topic_conf_t(); +} + +void rd_kafka_topic_conf_destroy(rd_kafka_topic_conf_t* conf) { + delete conf; +} + +rd_kafka_topic_conf_t* rd_kafka_topic_conf_dup(const rd_kafka_topic_conf_t* conf) { + return new rd_kafka_topic_conf_t(conf->get_handle()); +} + +rd_kafka_conf_res_t rd_kafka_topic_conf_set(rd_kafka_topic_conf_t* conf, + const char* name, + const char* value, + char* errstr, size_t errstr_size) { + return rd_kafka_conf_set(conf, name, value, errstr, errstr_size); +} + +rd_kafka_conf_res_t rd_kafka_topic_conf_get(const rd_kafka_topic_conf_t *conf, + const char *name, char *dest, size_t *dest_size) { + return rd_kafka_conf_get(conf, name, dest, dest_size); +} + +const char** rd_kafka_topic_conf_dump(rd_kafka_topic_conf_t* conf, size_t* cntp) { + return rd_kafka_conf_dump(conf, cntp); +} + +// rd_kafka_topic_* + +void rd_kafka_topic_partition_destroy (rd_kafka_topic_partition_t* toppar) { + delete toppar->topic; + delete toppar; +} + +rd_kafka_topic_partition_list_t* rd_kafka_topic_partition_list_new(int size) { + rd_kafka_topic_partition_list_t* output = new rd_kafka_topic_partition_list_t{}; + output->size = size; + output->elems = new rd_kafka_topic_partition_t[size]; + for (int i = 0; i < size; ++i) { + output->elems[i] = {}; + } + return output; +} + +void rd_kafka_topic_partition_list_destroy(rd_kafka_topic_partition_list_t* toppar_list) { + for (int i = 0; i < toppar_list->cnt; ++i) { + delete[] toppar_list->elems[i].topic; + } + delete[] toppar_list->elems; + delete toppar_list; +} + +rd_kafka_topic_partition_t* +rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list, + const char* topic, int32_t partition) { + if (toppar_list->cnt >= toppar_list->size) { + return nullptr; + } + rd_kafka_topic_partition_t* output = &toppar_list->elems[toppar_list->cnt++]; + const size_t length = strlen(topic); + output->topic = new char[length + 1]; + copy(topic, topic + length, output->topic); + output->topic[length] = 0; + output->partition = partition; + output->offset = RD_KAFKA_OFFSET_INVALID; + return output; +} + +// rd_kafka_topic_t + +rd_kafka_topic_t* rd_kafka_topic_new(rd_kafka_t* rk, const char* topic, + rd_kafka_topic_conf_t* conf) { + return reinterpret_cast(new TopicHandle(topic, nullptr)); +} + +const char* rd_kafka_topic_name(const rd_kafka_topic_t* rkt) { + return reinterpret_cast(rkt)->get_topic().c_str(); +} + +void rd_kafka_topic_destroy(rd_kafka_topic_t* rkt) { + delete reinterpret_cast(rkt); +} + +int rd_kafka_topic_partition_available(const rd_kafka_topic_t* rkt, int32_t partition) { + return 1; +} + +// rd_kafka_t + +rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t* conf_ptr, + char *errstr, size_t errstr_size) { + static const string BROKERS_OPTION = "metadata.broker.list"; + auto& conf = conf_ptr->get_handle(); + HandleMock::ClusterPtr cluster; + if (conf.has_key(BROKERS_OPTION)) { + cluster = KafkaClusterRegistry::instance().get_cluster(conf.get(BROKERS_OPTION)); + } + if (type == RD_KAFKA_PRODUCER) { + unique_ptr _(conf_ptr); + return new rd_kafka_t(new ProducerMock(move(conf), make_shared(), + move(cluster))); + } + else if (type == RD_KAFKA_CONSUMER) { + if (!conf.has_key("group.id")) { + const string error = "Local: Unknown topic"; + if (error.size() < errstr_size) { + copy(error.begin(), error.end(), errstr); + errstr[error.size()] = 0; + } + return nullptr; + } + unique_ptr _(conf_ptr); + return new rd_kafka_t(new ConsumerMock(move(conf), make_shared(), + move(cluster))); + } + return nullptr; +} + +void rd_kafka_destroy(rd_kafka_t* rk) { + delete rk; +} + +int rd_kafka_brokers_add(rd_kafka_t* rk, const char* brokerlist) { + auto cluster = KafkaClusterRegistry::instance().get_cluster(brokerlist); + if (cluster) { + rk->get_handle().set_cluster(move(cluster)); + } + return 1; +} + +const char* rd_kafka_name(const rd_kafka_t* rk) { + return "cppkafka mock handle"; +} + +rd_kafka_message_t* rd_kafka_consumer_poll(rd_kafka_t* rk, int timeout_ms) { + auto& consumer = rk->get_handle(); + auto message_ptr = consumer.poll(milliseconds(timeout_ms)); + if (!message_ptr) { + return nullptr; + } + else { + return &message_ptr.release()->get_message(); + } +} + +void rd_kafka_message_destroy(rd_kafka_message_t* rkmessage) { + delete static_cast(rkmessage->_private)->get_owner(); +} + +rd_kafka_resp_err_t rd_kafka_pause_partitions(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t* partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + auto& consumer = rk->get_handle(); + consumer.pause_partitions(topic_partitions); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_resume_partitions(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t* partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + auto& consumer = rk->get_handle(); + consumer.resume_partitions(topic_partitions); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_subscribe(rd_kafka_t* rk, + const rd_kafka_topic_partition_list_t* partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + vector topics; + for (const TopicPartitionMock& topic_partition : topic_partitions) { + topics.emplace_back(topic_partition.get_topic()); + } + auto& consumer = rk->get_handle(); + consumer.subscribe(topics); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_unsubscribe(rd_kafka_t* rk) { + auto& consumer = rk->get_handle(); + consumer.unsubscribe(); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_subscription(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t** topics) { + // TODO: implement + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_assign(rd_kafka_t* rk, + const rd_kafka_topic_partition_list_t* partitions) { + auto& consumer = rk->get_handle(); + if (partitions) { + const vector topic_partitions = from_rdkafka_handle(*partitions); + consumer.assign(topic_partitions); + } + else { + consumer.unassign(); + } + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_assignment(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t** partitions) { + auto& consumer = rk->get_handle(); + const vector assignment = consumer.get_assignment(); + *partitions = to_rdkafka_handle(assignment).release(); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_flush(rd_kafka_t* rk, int timeout_ms) { + if (rk->get_handle().flush(milliseconds(timeout_ms))) { + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + else { + return RD_KAFKA_RESP_ERR__TIMED_OUT; + } +} + +int rd_kafka_poll(rd_kafka_t* rk, int timeout_ms) { + return rk->get_handle().poll(milliseconds(timeout_ms)); +} + +rd_kafka_queue_t* rd_kafka_queue_get_consumer(rd_kafka_t* rk) { + // TODO: implement + return nullptr; +} + +ssize_t rd_kafka_consume_batch_queue(rd_kafka_queue_t* rkqu, int timeout_ms, + rd_kafka_message_t** rkmessages, size_t rkmessages_size) { + // TODO: implement + return 0; +} + +rd_kafka_resp_err_t rd_kafka_producev(rd_kafka_t* rk, ...) { + va_list args; + int vtype; + unique_ptr topic; + unsigned partition = RD_KAFKA_PARTITION_UA; + void* key_ptr = nullptr; + size_t key_size = 0; + void* payload_ptr = nullptr; + size_t payload_size = 0; + void* opaque = nullptr; + MessageHandle::PointerOwnership ownership = MessageHandle::PointerOwnership::Unowned; + int64_t timestamp = 0; + + va_start(args, rk); + while ((vtype = va_arg(args, int)) != RD_KAFKA_VTYPE_END) { + switch (vtype) { + case RD_KAFKA_VTYPE_TOPIC: + topic.reset(new TopicHandle(va_arg(args, const char *), nullptr)); + break; + case RD_KAFKA_VTYPE_PARTITION: + partition = va_arg(args, int32_t); + break; + case RD_KAFKA_VTYPE_VALUE: + payload_ptr = va_arg(args, void *); + payload_size = va_arg(args, size_t); + break; + case RD_KAFKA_VTYPE_KEY: + key_ptr = va_arg(args, void *); + key_size = va_arg(args, size_t); + break; + case RD_KAFKA_VTYPE_OPAQUE: + opaque = va_arg(args, void *); + break; + case RD_KAFKA_VTYPE_MSGFLAGS: + if (va_arg(args, int) == static_cast(MessageHandle::PointerOwnership::Owned)) { + ownership = MessageHandle::PointerOwnership::Owned; + } + break; + case RD_KAFKA_VTYPE_TIMESTAMP: + timestamp = va_arg(args, int64_t); + break; + default: + return RD_KAFKA_RESP_ERR__INVALID_ARG; + } + } + va_end(args); + + MessageHandlePrivateData private_data(RD_KAFKA_TIMESTAMP_CREATE_TIME, timestamp); + private_data.set_opaque(opaque); + rk->get_handle().produce(MessageHandle( + move(topic), + partition, + -1, // offset + key_ptr, key_size, + payload_ptr, payload_size, + RD_KAFKA_RESP_ERR_NO_ERROR, + private_data, + ownership + )); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +int rd_kafka_outq_len(rd_kafka_t* rk) { + return rk->get_handle().get_event_count(); +} + +void* rd_kafka_opaque(const rd_kafka_t* rk) { + return rk->get_handle().get_opaque(); +} + +void rd_kafka_set_log_level(rd_kafka_t* /*rk*/, int /*level*/) { + +} + +rd_kafka_resp_err_t rd_kafka_query_watermark_offsets(rd_kafka_t* rk, const char* topic, + int32_t partition, int64_t* low, + int64_t* high, int /*timeout_ms*/) { + return rd_kafka_get_watermark_offsets(rk, topic, partition, low, high); +} + +rd_kafka_resp_err_t rd_kafka_get_watermark_offsets(rd_kafka_t* rk, const char *topic, + int32_t partition, int64_t *low, + int64_t *high) { + const auto& cluster = rk->get_handle().get_cluster(); + if (!cluster.topic_exists(topic)) { + return RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC; + } + try { + const auto& topic_object = cluster.get_topic(topic); + if (static_cast(partition) >= topic_object.get_partition_count()) { + return RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION; + } + const auto& partition_object = topic_object.get_partition(partition); + int64_t lowest; + int64_t largest; + tie(lowest, largest) = partition_object.get_offset_bounds(); + *low = lowest; + *high = largest; + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + catch (const exception&) { + return RD_KAFKA_RESP_ERR_UNKNOWN; + } +} + +rd_kafka_resp_err_t rd_kafka_offsets_for_times(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t* offsets, + int timeout_ms) { + // TODO: implement this one + return RD_KAFKA_RESP_ERR_UNKNOWN; +} + +rd_kafka_resp_err_t rd_kafka_metadata(rd_kafka_t* rk, int all_topics, + rd_kafka_topic_t* only_rkt, + const struct rd_kafka_metadata** metadatap, + int timeout_ms) { + // TODO: implement this one + return RD_KAFKA_RESP_ERR_UNKNOWN; +} + +void rd_kafka_metadata_destroy(const struct rd_kafka_metadata* /*metadata*/) { + // TODO: implement this one +} + +rd_kafka_resp_err_t rd_kafka_list_groups(rd_kafka_t* rk, const char* group, + const struct rd_kafka_group_list** grplistp, + int timeout_ms) { + // TODO: implement this one + return RD_KAFKA_RESP_ERR_UNKNOWN; +} + +void rd_kafka_group_list_destroy(const struct rd_kafka_group_list* /*grplist*/) { + // TODO: implement this one +} + +rd_kafka_resp_err_t rd_kafka_poll_set_consumer(rd_kafka_t*) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_consumer_close(rd_kafka_t* rk) { + rk->get_handle().close(); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_committed(rd_kafka_t* rk, rd_kafka_topic_partition_list_t *partitions, + int timeout_ms) { + // TODO: implement + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_commit(rd_kafka_t* rk, const rd_kafka_topic_partition_list_t* offsets, + int async) { + rk->get_handle().commit(from_rdkafka_handle(*offsets)); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_commit_message(rd_kafka_t* rk, const rd_kafka_message_t* message, + int async) { + rk->get_handle().commit(*message); + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t rd_kafka_position(rd_kafka_t* rk, + rd_kafka_topic_partition_list_t* partitions) { + // TODO: implement + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +char* rd_kafka_memberid(const rd_kafka_t* rk) { + // TODO: make this better + char* output = (char*)malloc(strlen("cppkafka_mock") + 1); + strcpy(output, "cppkafka_mock"); + return output; +} + +// misc + +const char* rd_kafka_err2str(rd_kafka_resp_err_t err) { + return "cppkafka mock: error"; +} + +rd_kafka_resp_err_t rd_kafka_errno2err(int errnox) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +int32_t rd_kafka_msg_partitioner_consistent_random(const rd_kafka_topic_t* rkt, const void *key, + size_t keylen, int32_t partition_cnt, + void *opaque, void *msg_opaque) { + unsigned hash = 0; + const char* key_ptr = reinterpret_cast(key); + for (size_t i = 0; i < keylen; ++i) { + hash += key_ptr[i]; + } + return hash % partition_cnt; +} + +rd_kafka_resp_err_t rd_kafka_last_error (void) { + // TODO: fix this + return RD_KAFKA_RESP_ERR_UNKNOWN; +} diff --git a/mocking/src/configuration_mock.cpp b/mocking/src/configuration_mock.cpp new file mode 100644 index 00000000..8f0ab7dd --- /dev/null +++ b/mocking/src/configuration_mock.cpp @@ -0,0 +1,134 @@ +#include + +using std::string; +using std::default_delete; +using std::unordered_map; + +namespace cppkafka { +namespace mocking { + +ConfigurationMock::ConfigurationMock() +: default_topic_configuration_(nullptr, default_delete{}, Cloner{}) { + +} + +void ConfigurationMock::set(string key, string value) { + options_[move(key)] = move(value); +} + +size_t ConfigurationMock::has_key(const string& key) const { + return options_.count(key); +} + +string ConfigurationMock::get(const string& key) const { + return options_.at(key); +} + +void ConfigurationMock::set_delivery_report_callback(DeliveryReportCallback* callback) { + set_callback(Callback::DeliveryReport, callback); +} + +void ConfigurationMock::set_rebalance_callback(RebalanceCallback* callback) { + set_callback(Callback::Rebalance, callback); +} + +void ConfigurationMock::set_offset_commit_callback(OffsetCommitCallback* callback) { + set_callback(Callback::OffsetCommit, callback); +} + +void ConfigurationMock::set_error_callback(ErrorCallback* callback) { + set_callback(Callback::Error, callback); +} + +void ConfigurationMock::set_throttle_callback(ThrottleCallback* callback) { + set_callback(Callback::Throttle, callback); +} + +void ConfigurationMock::set_log_callback(LogCallback* callback) { + set_callback(Callback::Log, callback); +} + +void ConfigurationMock::set_stats_callback(StatsCallback* callback) { + set_callback(Callback::Stats, callback); +} + +void ConfigurationMock::set_socket_callback(SocketCallback* callback) { + set_callback(Callback::Socket, callback); +} + +void ConfigurationMock::set_partitioner_callback(PartitionerCallback* callback) { + set_callback(Callback::Partitioner, callback); +} + +void ConfigurationMock::set_default_topic_configuration(const ConfigurationMock& conf) { + default_topic_configuration_.reset(Cloner{}(&conf)); +} + +void ConfigurationMock::set_opaque(void* opaque) { + opaque_ = opaque; +} + +ConfigurationMock::DeliveryReportCallback* +ConfigurationMock::get_delivery_report_callback() const { + return get_callback(Callback::DeliveryReport); +} + +ConfigurationMock::RebalanceCallback* +ConfigurationMock::get_rebalance_callback() const { + return get_callback(Callback::Rebalance); +} + +ConfigurationMock::OffsetCommitCallback* +ConfigurationMock::get_offset_commit_callback() const { + return get_callback(Callback::OffsetCommit); +} + +ConfigurationMock::ErrorCallback* ConfigurationMock::get_error_callback() const { + return get_callback(Callback::Error); +} + +ConfigurationMock::ThrottleCallback* ConfigurationMock::get_throttle_callback() const { + return get_callback(Callback::Throttle); +} + +ConfigurationMock::LogCallback* ConfigurationMock::get_log_callback() const { + return get_callback(Callback::Log); +} + +ConfigurationMock::StatsCallback* ConfigurationMock::get_stats_callback() const { + return get_callback(Callback::Stats); +} + +ConfigurationMock::SocketCallback* ConfigurationMock::get_socket_callback() const { + return get_callback(Callback::Socket); +} + +ConfigurationMock::PartitionerCallback* ConfigurationMock::get_partitioner_callback() const { + return get_callback(Callback::Partitioner); +} + +const ConfigurationMock* ConfigurationMock::get_default_topic_configuration() const { + return default_topic_configuration_.get(); +} + +void* ConfigurationMock::get_opaque() const { + return opaque_; +} + +unordered_map ConfigurationMock::get_options() const { + return options_; +} + +template +void ConfigurationMock::set_callback(Callback type, Functor* ptr) { + callbacks_[type] = reinterpret_cast(ptr); +} + +template +Functor* ConfigurationMock::get_callback(Callback type) const { + auto iter = callbacks_.find(type); + return iter == callbacks_.end() ? nullptr : reinterpret_cast(iter->second); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/consumer_mock.cpp b/mocking/src/consumer_mock.cpp new file mode 100644 index 00000000..5d4dd8b4 --- /dev/null +++ b/mocking/src/consumer_mock.cpp @@ -0,0 +1,314 @@ +#include +#include +#include +#include +#include +#include +#include + +using std::atomic; +using std::vector; +using std::string; +using std::unordered_map; +using std::to_string; +using std::move; +using std::bind; +using std::runtime_error; +using std::make_tuple; +using std::unique_ptr; +using std::tie; +using std::get; +using std::lock_guard; +using std::unique_lock; +using std::mutex; + +using std::chrono::milliseconds; +using std::chrono::steady_clock; + +namespace cppkafka { +namespace mocking { + +static const string CONFIG_GROUP_ID = "group.id"; + +uint64_t ConsumerMock::make_consumer_id() { + static atomic current_id{0}; + return current_id++; +} + +ConsumerMock::ConsumerMock(ConfigurationMock config, EventProcessorPtr processor, + ClusterPtr cluster) +: HandleMock(move(processor), move(cluster)), config_(move(config)), + offset_reset_policy_(get_offset_policy()), emit_eofs_(get_partition_eof_enabled()), + auto_commit_(get_auto_commit()), consumer_id_(make_consumer_id()) { + if (!config_.has_key(CONFIG_GROUP_ID)) { + throw runtime_error("Failed to find " + CONFIG_GROUP_ID + " in config"); + } + group_id_ = config_.get(CONFIG_GROUP_ID); +} + +ConsumerMock::~ConsumerMock() { + get_cluster().unsubscribe(group_id_, consumer_id_); +} + +void ConsumerMock::close() { + unsubscribe(); +} + +void ConsumerMock::commit(const rd_kafka_message_t& message) { + get_cluster().commit( + group_id_, + consumer_id_, + { { rd_kafka_topic_name(message.rkt), message.partition, message.offset + 1 } } + ); +} + +void ConsumerMock::commit(const vector& topic_partitions) { + get_cluster().commit(group_id_,consumer_id_, topic_partitions); +} + +void ConsumerMock::subscribe(const vector& topics) { + using namespace std::placeholders; + get_cluster().subscribe( + group_id_, + consumer_id_, + topics, + bind(&ConsumerMock::on_assignment, this, _1), + bind(&ConsumerMock::on_revocation, this) + ); +} + +void ConsumerMock::unsubscribe() { + get_cluster().unsubscribe(group_id_, consumer_id_); +} + +void ConsumerMock::assign(const vector& topic_partitions) { + { + lock_guard _(mutex_); + // Create entries for all topic partitions in our assigned partitions map + for (const TopicPartitionMock& topic_partition : topic_partitions) { + const auto id = make_id(topic_partition); + uint64_t next_offset; + if (topic_partition.get_offset() == RD_KAFKA_OFFSET_INVALID) { + next_offset = 0; + } + else { + next_offset = topic_partition.get_offset(); + } + + auto iter = assigned_partitions_.find(id); + if (iter == assigned_partitions_.end()) { + iter = assigned_partitions_.emplace(id, TopicPartitionInfo{}).first; + } + else { + // The offset changed, clean up any messages with a lower offset than the + // next one + auto& queue = iter->second.messages; + while (!queue.empty() && queue.front().offset < next_offset) { + queue.pop(); + } + } + iter->second.next_offset = next_offset; + } + } + using namespace std::placeholders; + // Now assign these partitions. This will atomically fetch all message we should fetch and + // then subscribe us to the topic/partitions + get_cluster().assign(consumer_id_, topic_partitions, offset_reset_policy_, + bind(&ConsumerMock::on_message, this, _1, _2, _3, _4)); +} + +void ConsumerMock::unassign() { + lock_guard _(mutex_); + assigned_partitions_.clear(); + consumable_topic_partitions_.clear(); + get_cluster().unassign(consumer_id_); +} + +void ConsumerMock::pause_partitions(const vector& topic_partitions) { + lock_guard _(mutex_); + for (const TopicPartitionMock& topic_partition : topic_partitions) { + auto id = make_id(topic_partition); + consumable_topic_partitions_.erase(id); + paused_topic_partitions_.emplace(move(id)); + } +} + +void ConsumerMock::resume_partitions(const vector& topic_partitions) { + lock_guard _(mutex_); + for (const TopicPartitionMock& topic_partition : topic_partitions) { + auto id = make_id(topic_partition); + paused_topic_partitions_.erase(id); + auto iter = assigned_partitions_.find(id); + if (iter != assigned_partitions_.end() && !iter->second.messages.empty()) { + consumable_topic_partitions_.emplace(move(id)); + } + } +} + +unique_ptr ConsumerMock::poll(milliseconds timeout) { + unique_lock lock(mutex_); + if (consumable_topic_partitions_.empty()) { + messages_condition_.wait_for(lock, timeout); + } + if (consumable_topic_partitions_.empty()) { + return nullptr; + } + const auto id = *consumable_topic_partitions_.begin(); + auto iter = assigned_partitions_.find(id); + assert(iter != assigned_partitions_.end()); + + auto& queue = iter->second.messages; + if (emit_eofs_ && queue.empty()) { + // We emit the EOF so it's no longer consumable + consumable_topic_partitions_.erase(id); + return unique_ptr(new MessageHandle( + unique_ptr(new TopicHandle(get<0>(id), nullptr)), + get<1>(id), + iter->second.next_offset, + nullptr, 0, // key + nullptr, 0, // payload + RD_KAFKA_RESP_ERR_NO_ERROR, + MessageHandlePrivateData{}, + MessageHandle::PointerOwnership::Unowned + )); + } + else { + assert(!queue.empty()); + } + MessageAggregate aggregate = move(queue.front()); + queue.pop(); + + // If we have no more mesages we can't consume from it anymore + if (queue.empty()) { + consumable_topic_partitions_.erase(id); + } + + const auto& message = *aggregate.message; + unique_ptr output(new MessageHandle( + unique_ptr(new TopicHandle(get<0>(id), nullptr)), + get<1>(id), + aggregate.offset, + (void*)message.get_key().data(), message.get_key().size(), + (void*)message.get_payload().data(), message.get_payload().size(), + RD_KAFKA_RESP_ERR__PARTITION_EOF, + MessageHandlePrivateData{message.get_timestamp_type(), message.get_timestamp()}, + MessageHandle::PointerOwnership::Unowned + )); + if (auto_commit_) { + commit(output->get_message()); + } + return output; +} + +vector ConsumerMock::get_assignment() const { + vector output; + lock_guard _(mutex_); + for (const auto& partition_pair : assigned_partitions_) { + output.emplace_back(get<0>(partition_pair.first), get<1>(partition_pair.first)); + } + return output; +} + +ConsumerMock::TopicPartitionId ConsumerMock::make_id(const TopicPartitionMock& topic_partition) { + return make_tuple(topic_partition.get_topic(), topic_partition.get_partition()); +} + +KafkaCluster::ResetOffsetPolicy ConsumerMock::get_offset_policy() const { + static const string KEY_NAME = "auto.offset.reset"; + static unordered_map MAPPINGS = { + { "smallest", KafkaCluster::ResetOffsetPolicy::Earliest }, + { "earliest", KafkaCluster::ResetOffsetPolicy::Earliest }, + { "beginning", KafkaCluster::ResetOffsetPolicy::Earliest }, + { "latest", KafkaCluster::ResetOffsetPolicy::Latest }, + { "largest", KafkaCluster::ResetOffsetPolicy::Latest }, + { "end", KafkaCluster::ResetOffsetPolicy::Latest }, + }; + + const ConfigurationMock* topic_config = config_.get_default_topic_configuration(); + if (!topic_config || !topic_config->has_key(KEY_NAME)) { + return KafkaCluster::ResetOffsetPolicy::Earliest; + } + else { + auto iter = MAPPINGS.find(topic_config->get(KEY_NAME)); + if (iter == MAPPINGS.end()) { + throw runtime_error("invalid auto.offset.reset value"); + } + return iter->second; + } +} + +bool ConsumerMock::get_partition_eof_enabled() const { + static const string KEY_NAME = "enable.partition.eof"; + return !config_.has_key(KEY_NAME) || config_.get(KEY_NAME) == "true"; +} + +bool ConsumerMock::get_auto_commit() const { + static const vector KEY_NAMES = { + "enable.auto.commit", + "auto.commit.enable" + }; + for (const string& key : KEY_NAMES) { + if (config_.has_key(key)) { + return config_.get(key) == "true"; + } + } + // By default, auto commit + return true; +} + +void ConsumerMock::on_assignment(const vector& topic_partitions) { + handle_rebalance(RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, topic_partitions); +} + +void ConsumerMock::on_revocation() { + // Fetch and all assigned topic partitions + vector topic_partitions = [&]() { + lock_guard _(mutex_); + vector output; + for (const auto& topic_partition_pair : assigned_partitions_) { + const TopicPartitionId& id = topic_partition_pair.first; + output.emplace_back(get<0>(id), get<1>(id)); + } + return output; + }(); + handle_rebalance(RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS, topic_partitions); +} + +void ConsumerMock::on_message(const string& topic_name, unsigned partition, uint64_t offset, + const KafkaMessageMock& message) { + auto id = make_tuple(topic_name, partition); + MessageAggregate aggregate = { topic_name, partition, offset, &message }; + + // We should only process this if we don't have this topic/partition assigned (assignment + // pending?) or the message offset comes after the next offset we have stored + lock_guard _(mutex_); + auto iter = assigned_partitions_.find(id); + if (iter == assigned_partitions_.end()) { + throw runtime_error("got message for unexpected partition " + to_string(partition)); + } + if (offset > iter->second.next_offset) { + throw runtime_error("got message with unexpected offset " + to_string(offset)); + } + else if (offset < iter->second.next_offset) { + return; + } + // This is the message we were waiting for + iter->second.next_offset++; + iter->second.messages.push(move(aggregate)); + if (!paused_topic_partitions_.count(id)) { + consumable_topic_partitions_.emplace(move(id)); + messages_condition_.notify_one(); + } +} + +void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type, + const vector& topic_partitions) { + auto rebalance_callback = config_.get_rebalance_callback(); + if (rebalance_callback) { + auto handle = to_rdkafka_handle(topic_partitions); + rebalance_callback(nullptr, type, handle.get(), config_.get_opaque()); + } +} + +} // mocking +} // cppkafka diff --git a/mocking/src/event_processor.cpp b/mocking/src/event_processor.cpp new file mode 100644 index 00000000..12d80e61 --- /dev/null +++ b/mocking/src/event_processor.cpp @@ -0,0 +1,68 @@ +#include + +using std::thread; +using std::lock_guard; +using std::unique_lock; +using std::mutex; +using std::move; + +using std::chrono::milliseconds; + +namespace cppkafka { +namespace mocking { + +EventProcessor::EventProcessor() { + processing_thread_ = thread(&EventProcessor::process_events, this); +} + +EventProcessor::~EventProcessor() { + { + lock_guard _(events_mutex_); + running_ = false; + new_events_condition_.notify_all(); + no_events_condition_.notify_all(); + } + processing_thread_.join(); +} + +void EventProcessor::add_event(EventPtr event) { + lock_guard _(events_mutex_); + events_.push(move(event)); + new_events_condition_.notify_one(); +} + +size_t EventProcessor::get_event_count() const { + lock_guard _(events_mutex_); + return events_.size(); +} + +bool EventProcessor::wait_until_empty(milliseconds timeout) { + unique_lock lock(events_mutex_); + if (running_ && !events_.empty()) { + no_events_condition_.wait_for(lock, timeout); + } + return events_.empty(); +} + +void EventProcessor::process_events() { + while (true) { + unique_lock lock(events_mutex_); + while (running_ && events_.empty()) { + new_events_condition_.wait(lock); + } + if (!running_) { + break; + } + if (events_.empty()) { + continue; + } + EventPtr event = move(events_.front()); + events_.pop(); + + lock.unlock(); + event->execute(); + } +} + +} // mocking +} // cppkafka diff --git a/mocking/src/events/event_base.cpp b/mocking/src/events/event_base.cpp new file mode 100644 index 00000000..bfc570fd --- /dev/null +++ b/mocking/src/events/event_base.cpp @@ -0,0 +1,17 @@ +#include +#include + +namespace cppkafka { +namespace mocking { + +EventBase::EventBase(ClusterPtr cluster) +: cluster_(move(cluster)) { + +} + +void EventBase::execute() { + execute_event(*cluster_); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/events/produce_message_event.cpp b/mocking/src/events/produce_message_event.cpp new file mode 100644 index 00000000..8977cfeb --- /dev/null +++ b/mocking/src/events/produce_message_event.cpp @@ -0,0 +1,26 @@ +#include +#include + +using std::string; +using std::move; + +namespace cppkafka { +namespace mocking { + +ProduceMessageEvent::ProduceMessageEvent(ClusterPtr cluster, MessageHandle message_handle) +: EventBase(move(cluster)), message_handle_(move(message_handle)) { + +} + +string ProduceMessageEvent::get_type() const { + return "produce message"; +} + +void ProduceMessageEvent::execute_event(KafkaCluster& cluster) { + const string& topic = message_handle_.get_topic().get_topic(); + const int partition = message_handle_.get_message().partition; + cluster.produce(topic, partition, message_handle_.make_message_mock()); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/handle_mock.cpp b/mocking/src/handle_mock.cpp new file mode 100644 index 00000000..ad876b15 --- /dev/null +++ b/mocking/src/handle_mock.cpp @@ -0,0 +1,63 @@ +#include +#include + +using std::runtime_error; +using std::move; + +namespace cppkafka { +namespace mocking { + +HandleMock::HandleMock(EventProcessorPtr processor) +: processor_(move(processor)) { + +} + +HandleMock::HandleMock(EventProcessorPtr processor, ClusterPtr cluster) +: processor_(move(processor)), cluster_(move(cluster)) { + +} + +void* HandleMock::get_opaque() const { + return opaque_; +} + +size_t HandleMock::get_event_count() const { + return processor_->get_event_count(); +} + +void HandleMock::set_cluster(ClusterPtr cluster) { + // Don't allow changing the cluster + if (cluster_) { + throw runtime_error("can't change the cluster"); + } + cluster_ = move(cluster); +} + +void HandleMock::set_opaque(void* opaque) { + opaque_ = opaque; +} + +KafkaCluster& HandleMock::get_cluster() { + if (!cluster_) { + throw runtime_error("cluster not set"); + } + return *cluster_; +} + +const KafkaCluster& HandleMock::get_cluster() const { + if (!cluster_) { + throw runtime_error("cluster not set"); + } + return *cluster_; +} + +void HandleMock::generate_event(EventPtr event) { + processor_->add_event(move(event)); +} + +EventProcessor& HandleMock::get_event_processor() { + return *processor_; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_cluster.cpp b/mocking/src/kafka_cluster.cpp new file mode 100644 index 00000000..6764b372 --- /dev/null +++ b/mocking/src/kafka_cluster.cpp @@ -0,0 +1,259 @@ +#include +#include +#include +#include +#include +#include + +using std::shared_ptr; +using std::make_shared; +using std::string; +using std::to_string; +using std::vector; +using std::invalid_argument; +using std::runtime_error; +using std::piecewise_construct; +using std::forward_as_tuple; +using std::move; +using std::tie; +using std::lock_guard; +using std::mutex; +using std::recursive_mutex; +using std::iota; + +namespace cppkafka { +namespace mocking { + +shared_ptr KafkaCluster::make_cluster(string url) { + shared_ptr output{ new KafkaCluster(move(url)) }; + detail::KafkaClusterRegistry::instance().add_cluster(output); + return output; +} + +KafkaCluster::KafkaCluster(string url) +: url_(move(url)), offset_manager_(make_shared()) { + +} + +KafkaCluster::~KafkaCluster() { + detail::KafkaClusterRegistry::instance().remove_cluster(url_); +} + +const string& KafkaCluster::get_url() const { + return url_; +} + +void KafkaCluster::create_topic(const string& name, unsigned partitions) { + lock_guard _(topics_mutex_); + topics_.emplace(piecewise_construct, forward_as_tuple(name), + forward_as_tuple(name, partitions)); +} + +bool KafkaCluster::topic_exists(const string& name) const { + lock_guard _(topics_mutex_); + return topics_.count(name) > 0; +} + +void KafkaCluster::produce(const string& topic, unsigned partition, KafkaMessageMock message) { + lock_guard _(topics_mutex_); + auto iter = topics_.find(topic); + if (iter == topics_.end()) { + throw invalid_argument("topic does not exist"); + } + iter->second.add_message(partition, move(message)); +} + +KafkaTopicMock& KafkaCluster::get_topic(const string& name) { + lock_guard _(topics_mutex_); + auto iter = topics_.find(name); + if (iter == topics_.end()) { + throw runtime_error("Topic " + name + " doesn't exist"); + } + return iter->second; +} + +const KafkaTopicMock& KafkaCluster::get_topic(const string& name) const { + return const_cast(*this).get_topic(name); +} + +void KafkaCluster::subscribe(const string& group_id, uint64_t consumer_id, + const vector& topics, AssignmentCallback assignment_callback, + RevocationCallback revocation_callback) { + lock_guard _(consumer_data_mutex_); + auto iter = consumer_data_.find(consumer_id); + // If it's already subscribed to something, unsubscribe from it + if (iter != consumer_data_.end()) { + do_unsubscribe(group_id, consumer_id); + } + ConsumerMetadata data = { + move(assignment_callback), + move(revocation_callback) + }; + iter = consumer_data_.emplace(consumer_id, move(data)).first; + + auto& group_data = group_topics_data_[group_id]; + for (const string& topic : topics) { + group_data[topic].emplace(consumer_id); + } + // First revoke any assignment that involve consumers in this group + generate_revocations(group_data); + // Now generate the assignments + generate_assignments(group_id, group_data); +} + +void KafkaCluster::unsubscribe(const string& group_id, uint64_t consumer_id) { + lock_guard _(consumer_data_mutex_); + do_unsubscribe(group_id, consumer_id); +} + +void KafkaCluster::assign(uint64_t consumer_id, const vector& topic_partitions, + ResetOffsetPolicy policy, const MessageCallback& message_callback) { + lock_guard _(consumer_data_mutex_); + auto iter = consumer_data_.find(consumer_id); + if (iter == consumer_data_.end()) { + iter = consumer_data_.emplace(consumer_id, ConsumerMetadata{}).first; + } + ConsumerMetadata& consumer = iter->second; + using namespace std::placeholders; + for (const TopicPartitionMock& topic_partition : topic_partitions) { + KafkaTopicMock& topic = get_topic(topic_partition.get_topic()); + auto& partition = topic.get_partition(topic_partition.get_partition()); + auto callback = bind(message_callback, topic_partition.get_topic(), + topic_partition.get_partition(), _1, _2); + partition.acquire([&]() { + int64_t start_offset; + int64_t end_offset; + tie(start_offset, end_offset) = partition.get_offset_bounds(); + int64_t next_offset = topic_partition.get_offset(); + if (next_offset == RD_KAFKA_OFFSET_INVALID) { + switch (policy) { + case ResetOffsetPolicy::Earliest: + next_offset = start_offset; + break; + case ResetOffsetPolicy::Latest: + next_offset = end_offset; + break; + } + } + if (start_offset >= next_offset && next_offset != end_offset && end_offset > 0) { + for (auto i = next_offset; i != end_offset; ++i) { + const KafkaMessageMock& message = partition.get_message(i); + callback(i, message); + } + } + const auto subscriber_id = partition.subscribe(move(callback)); + consumer.subscriptions[&topic].emplace(topic_partition.get_partition(), + subscriber_id); + }); + } +} + +void KafkaCluster::unassign(uint64_t consumer_id) { + lock_guard _(consumer_data_mutex_); + auto iter = consumer_data_.find(consumer_id); + if (iter == consumer_data_.end()) { + throw runtime_error("called unassign with unknown consumer id " + to_string(consumer_id)); + } + ConsumerMetadata& consumer = iter->second; + for (const auto& topic_subscription : consumer.subscriptions) { + KafkaTopicMock& topic = *topic_subscription.first; + for (const auto& partition_subscription : topic_subscription.second) { + auto& partition_mock = topic.get_partition(partition_subscription.first); + partition_mock.unsubscribe(partition_subscription.second); + } + } + consumer.subscriptions.clear(); +} + +void KafkaCluster::commit(const string& group_id, uint64_t consumer_id, + const vector& topic_partitions) { + offset_manager_->commit_offsets(group_id, topic_partitions); +} + +void KafkaCluster::generate_assignments(const string& group_id, + const TopicConsumersMap& topic_consumers) { + for (const auto& topic_consumers_pair : topic_consumers) { + const string& topic_name = topic_consumers_pair.first; + const ConsumerSet& consumers = topic_consumers_pair.second; + KafkaTopicMock& topic = get_topic(topic_name); + vector all_partitions(topic.get_partition_count()); + iota(all_partitions.begin(), all_partitions.end(), 0); + + size_t chunk_size = all_partitions.size() / consumers.size(); + size_t consumer_index = 0; + for (const uint64_t consumer_id : consumers) { + ConsumerMetadata& consumer = consumer_data_[consumer_id]; + + // Generate partition assignment + const size_t chunk_start = chunk_size * consumer_index; + // For the last one, add any remainders + if (consumer_index == consumers.size() - 1) { + chunk_size += all_partitions.size() % consumers.size(); + } + for (size_t i = 0; i < chunk_size; ++i) { + consumer.partitions_assigned.emplace_back(topic_name, chunk_start + i); + } + consumer_index++; + } + } + // Now do another pass and trigger the assignment callbacks + for (const auto& topic_consumers_pair : topic_consumers) { + for (const uint64_t consumer_id : topic_consumers_pair.second) { + ConsumerMetadata& consumer = consumer_data_[consumer_id]; + // Try to load the offsets for this consumer, and store the updated offsets and + // trigger the assignment callback + auto partitions_assigned = move(consumer.partitions_assigned); + consumer.partitions_assigned = offset_manager_->get_offsets(group_id, + move(partitions_assigned)); + consumer.assignment_callback(consumer.partitions_assigned); + } + } +} + +void KafkaCluster::generate_revocations(const TopicConsumersMap& topic_consumers) { + for (const auto& topic_consumers_pair : topic_consumers) { + const ConsumerSet& consumers = topic_consumers_pair.second; + for (const uint64_t consumer_id : consumers) { + ConsumerMetadata& consumer = consumer_data_[consumer_id]; + // Execute revocation callback and unsubscribe from the partition object + if (!consumer.partitions_assigned.empty()) { + consumer.revocation_callback(); + consumer.partitions_assigned.clear(); + } + consumer.subscriptions.clear(); + } + } +} + +void KafkaCluster::do_unsubscribe(const string& group_id, uint64_t consumer_id) { + auto iter = consumer_data_.find(consumer_id); + if (iter == consumer_data_.end()) { + return; + } + auto& group_data = group_topics_data_[group_id]; + // Revoke for all consumers + generate_revocations(group_data); + + auto topic_iter = group_data.begin(); + while (topic_iter != group_data.end()) { + topic_iter->second.erase(consumer_id); + if (topic_iter->second.empty()) { + topic_iter = group_data.erase(topic_iter); + } + else { + ++topic_iter; + } + } + consumer_data_.erase(consumer_id); + if (group_data.empty()) { + // If we ran out of consumers for this group, erase it + group_topics_data_.erase(group_id); + } + else { + // Otherwise re-generate the assignments + generate_assignments(group_id, group_data); + } +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_cluster_registry.cpp b/mocking/src/kafka_cluster_registry.cpp new file mode 100644 index 00000000..22f4eea0 --- /dev/null +++ b/mocking/src/kafka_cluster_registry.cpp @@ -0,0 +1,43 @@ +#include +#include + +using std::string; +using std::lock_guard; +using std::mutex; +using std::runtime_error; +using std::shared_ptr; +using std::weak_ptr; + +namespace cppkafka { +namespace mocking { +namespace detail { + +KafkaClusterRegistry& KafkaClusterRegistry::instance() { + static KafkaClusterRegistry registry; + return registry; +} + +void KafkaClusterRegistry::add_cluster(shared_ptr cluster) { + const string& url = cluster->get_url(); + lock_guard _(clusters_mutex_); + auto iter = clusters_.find(url); + if (iter != clusters_.end()) { + throw runtime_error("cluster already registered"); + } + clusters_.emplace(url, weak_ptr(cluster)); +} + +void KafkaClusterRegistry::remove_cluster(const string& url) { + lock_guard _(clusters_mutex_); + clusters_.erase(url); +} + +shared_ptr KafkaClusterRegistry::get_cluster(const string& name) const { + lock_guard _(clusters_mutex_); + auto iter = clusters_.find(name); + return iter != clusters_.end() ? iter->second.lock() : nullptr; +} + +} // detail +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_message_mock.cpp b/mocking/src/kafka_message_mock.cpp new file mode 100644 index 00000000..5fd100a8 --- /dev/null +++ b/mocking/src/kafka_message_mock.cpp @@ -0,0 +1,30 @@ +#include + +namespace cppkafka { +namespace mocking { + +KafkaMessageMock::KafkaMessageMock(Buffer key, Buffer payload, + rd_kafka_timestamp_type_t timestamp_type, int64_t timestamp) +: key_(move(key)), payload_(move(payload)), timestamp_type_(timestamp_type), + timestamp_(timestamp) { + +} + +const KafkaMessageMock::Buffer& KafkaMessageMock::get_key() const { + return key_; +} + +const KafkaMessageMock::Buffer& KafkaMessageMock::get_payload() const { + return payload_; +} + +rd_kafka_timestamp_type_t KafkaMessageMock::get_timestamp_type() const { + return timestamp_type_; +} + +int64_t KafkaMessageMock::get_timestamp() const { + return timestamp_; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_partition_mock.cpp b/mocking/src/kafka_partition_mock.cpp new file mode 100644 index 00000000..3079fe59 --- /dev/null +++ b/mocking/src/kafka_partition_mock.cpp @@ -0,0 +1,74 @@ +#include +#include +#include + +using std::vector; +using std::lock_guard; +using std::mutex; +using std::recursive_mutex; +using std::out_of_range; +using std::move; +using std::tuple; +using std::tie; +using std::make_tuple; + +namespace cppkafka { +namespace mocking { + +void KafkaPartitionMock::add_message(KafkaMessageMock message) { + const KafkaMessageMock* message_ptr; + uint64_t offset; + tie(message_ptr, offset) = [&] { + lock_guard _(mutex_); + messages_.emplace_back(move(message)); + return make_tuple(&messages_.back(), messages_.size() - 1); + }(); + + const vector callbacks = get_subscriber_callbacks(); + for (const MessageCallback& callback : callbacks) { + callback(offset, *message_ptr); + } +} + +const KafkaMessageMock& KafkaPartitionMock::get_message(uint64_t offset) const { + const uint64_t index = offset - base_offset_; + lock_guard _(mutex_); + if (index >= messages_.size()) { + throw out_of_range("invalid message index"); + } + return messages_[index]; +} + +size_t KafkaPartitionMock::get_message_count() const { + lock_guard _(mutex_); + return messages_.size(); +} + +KafkaPartitionMock::SubscriberId KafkaPartitionMock::subscribe(MessageCallback callback) { + lock_guard _(mutex_); + auto id = current_subscriber_id_++; + subscribers_.emplace(id, move(callback)); + return id; +} + +void KafkaPartitionMock::unsubscribe(SubscriberId id) { + lock_guard _(mutex_); + subscribers_.erase(id); +} + +tuple KafkaPartitionMock::get_offset_bounds() const { + lock_guard _(mutex_); + return make_tuple(base_offset_, base_offset_ + messages_.size()); +} + +vector KafkaPartitionMock::get_subscriber_callbacks() const { + lock_guard _(mutex_); + vector output; + for (const auto& subcriber_pair : subscribers_) { + output.emplace_back(subcriber_pair.second); + } + return output; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/kafka_topic_mock.cpp b/mocking/src/kafka_topic_mock.cpp new file mode 100644 index 00000000..86bd0031 --- /dev/null +++ b/mocking/src/kafka_topic_mock.cpp @@ -0,0 +1,48 @@ +#include +#include +#include + +using std::string; +using std::move; +using std::lock_guard; +using std::mutex; +using std::vector; +using std::out_of_range; +using std::runtime_error; + +namespace cppkafka { +namespace mocking { + +KafkaTopicMock::KafkaTopicMock(string name, unsigned partition_count) +: name_(move(name)), partitions_(partition_count) { + +} + +const string& KafkaTopicMock::get_name() const { + return name_; +} + +void KafkaTopicMock::add_message(unsigned partition, KafkaMessageMock message) { + if (partition >= partitions_.size()) { + throw out_of_range("invalid partition index"); + } + partitions_[partition].add_message(move(message)); +} + +KafkaPartitionMock& KafkaTopicMock::get_partition(unsigned partition) { + if (partition >= partitions_.size()) { + throw runtime_error("partition doesn't exist"); + } + return partitions_[partition]; +} + +const KafkaPartitionMock& KafkaTopicMock::get_partition(unsigned partition) const { + return const_cast(*this).get_partition(partition); +} + +size_t KafkaTopicMock::get_partition_count() const { + return partitions_.size(); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/message_handle.cpp b/mocking/src/message_handle.cpp new file mode 100644 index 00000000..8b393c97 --- /dev/null +++ b/mocking/src/message_handle.cpp @@ -0,0 +1,104 @@ +#include +#include +#include + +using std::unique_ptr; +using std::move; +using std::swap; + +namespace cppkafka { +namespace mocking { + +MessageHandlePrivateData::MessageHandlePrivateData(rd_kafka_timestamp_type_t timestamp_type, + int64_t timestamp) +: timestamp_type_(timestamp_type), timestamp_(timestamp) { + +} + +rd_kafka_timestamp_type_t MessageHandlePrivateData::get_timestamp_type() const { + return timestamp_type_; +} + +int64_t MessageHandlePrivateData::get_timestamp() const { + return timestamp_; +} + +MessageHandle* MessageHandlePrivateData::get_owner() const { + return owner_; +} + +void MessageHandlePrivateData::set_owner(MessageHandle* handle) { + owner_ = handle; +} + +void MessageHandlePrivateData::set_opaque(void* opaque) { + opaque_ = opaque; +} + +MessageHandle::MessageHandle(unique_ptr topic, int partition, int64_t offset, + void* key, size_t key_size, void* payload, size_t payload_size, + int error_code, MessageHandlePrivateData private_data, + PointerOwnership ownership) +: topic_(move(topic)), private_data_(private_data), ownership_(ownership) { + message_.rkt = reinterpret_cast(topic_.get()); + message_.partition = partition; + message_.payload = payload; + message_.len = payload_size; + message_.key = key; + message_.key_len = key_size; + message_.offset = offset; + set_private_data_pointer(); +} + +MessageHandle::MessageHandle(MessageHandle&& other) +: ownership_(PointerOwnership::Unowned) { + *this = move(other); +} + +MessageHandle& MessageHandle::operator=(MessageHandle&& other) { + swap(topic_, other.topic_); + swap(message_, other.message_); + swap(ownership_, other.ownership_); + swap(private_data_, other.private_data_); + set_private_data_pointer(); + other.set_private_data_pointer(); + return *this; +} + +MessageHandle::~MessageHandle() { + if (ownership_ == PointerOwnership::Owned) { + free(message_.payload); + free(message_.key); + } +} + +const TopicHandle& MessageHandle::get_topic() const { + return *topic_; +} + +rd_kafka_message_t& MessageHandle::get_message() { + return message_; +} + +const rd_kafka_message_t& MessageHandle::get_message() const { + return message_; +} + +KafkaMessageMock MessageHandle::make_message_mock() const { + auto key_ptr = reinterpret_cast(message_.key); + auto payload_ptr = reinterpret_cast(message_.payload); + return { + KafkaMessageMock::Buffer(key_ptr, key_ptr + message_.key_len), + KafkaMessageMock::Buffer(payload_ptr, payload_ptr + message_.len), + private_data_.get_timestamp_type(), + private_data_.get_timestamp() + }; +} + +void MessageHandle::set_private_data_pointer() { + private_data_.set_owner(this); + message_._private = reinterpret_cast(&private_data_); +} + +} // mocking +} // cppkafka diff --git a/mocking/src/offset_manager.cpp b/mocking/src/offset_manager.cpp new file mode 100644 index 00000000..e845abd3 --- /dev/null +++ b/mocking/src/offset_manager.cpp @@ -0,0 +1,46 @@ +#include +#include + +using std::string; +using std::vector; +using std::mutex; +using std::lock_guard; +using std::make_tuple; + +namespace cppkafka { +namespace mocking { + +void OffsetManager::commit_offsets(const string& group_id, + const vector& topic_partitions) { + lock_guard _(offsets_mutex_); + for (const TopicPartitionMock& topic_partition : topic_partitions) { + auto key = make_tuple(group_id, topic_partition.get_topic(), + topic_partition.get_partition()); + auto iter = offsets_.find(key); + if (iter == offsets_.end()) { + offsets_.emplace(key, topic_partition.get_offset()); + } + else { + iter->second = topic_partition.get_offset(); + } + } +} + +vector +OffsetManager::get_offsets(const string& group_id, + vector topic_partitions) const { + lock_guard _(offsets_mutex_); + for (TopicPartitionMock& topic_partition : topic_partitions) { + if (topic_partition.get_offset() == RD_KAFKA_OFFSET_INVALID) { + auto iter = offsets_.find(make_tuple(group_id, topic_partition.get_topic(), + topic_partition.get_partition())); + if (iter != offsets_.end()) { + topic_partition.set_offset(iter->second); + } + } + } + return topic_partitions; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/producer_mock.cpp b/mocking/src/producer_mock.cpp new file mode 100644 index 00000000..fad0982f --- /dev/null +++ b/mocking/src/producer_mock.cpp @@ -0,0 +1,32 @@ +#include +#include + +using std::move; + +using std::chrono::milliseconds; + +namespace cppkafka { +namespace mocking { + +ProducerMock::ProducerMock(ConfigurationMock config, EventProcessorPtr processor, + ClusterPtr cluster) +: HandleMock(move(processor), move(cluster)), config_(move(config)) { + +} + +void ProducerMock::produce(MessageHandle message_handle) { + generate_event(move(message_handle)); +} + +bool ProducerMock::flush(milliseconds timeout) { + // TODO: produce buffered events + return get_event_processor().wait_until_empty(timeout); +} + +size_t ProducerMock::poll(milliseconds timeout) { + // TODO: do something + return 0; +} + +} // mocking +} // cppkafka diff --git a/mocking/src/topic_partition_mock.cpp b/mocking/src/topic_partition_mock.cpp new file mode 100644 index 00000000..51f28746 --- /dev/null +++ b/mocking/src/topic_partition_mock.cpp @@ -0,0 +1,58 @@ +#include +#include + +using std::string; +using std::vector; +using std::tie; +using std::unique_ptr; + +namespace cppkafka { +namespace mocking { + +TopicPartitionMock::TopicPartitionMock(string topic, int partition, int64_t offset) +: topic_(move(topic)), partition_(partition), offset_(offset) { + +} + +const string& TopicPartitionMock::get_topic() const { + return topic_; +} + +int TopicPartitionMock::get_partition() const { + return partition_; +} + +int64_t TopicPartitionMock::get_offset() const { + return offset_; +} + +void TopicPartitionMock::set_offset(int64_t offset) { + offset_ = offset; +} + +TopicPartitionMockListPtr to_rdkafka_handle(const vector& topic_partitions){ + const size_t count = topic_partitions.size(); + TopicPartitionMockListPtr output{rd_kafka_topic_partition_list_new(count), + &rd_kafka_topic_partition_list_destroy}; + for (const TopicPartitionMock& topic_partition : topic_partitions) { + auto* ptr = rd_kafka_topic_partition_list_add(output.get(), + topic_partition.get_topic().data(), + topic_partition.get_partition()); + ptr->offset = topic_partition.get_offset(); + } + return output; +} + +vector +from_rdkafka_handle(const rd_kafka_topic_partition_list_t& topic_partitions){ + vector output; + for (int i = 0; i < topic_partitions.cnt; ++i) { + const auto& topic_partition = topic_partitions.elems[i]; + output.emplace_back(topic_partition.topic, topic_partition.partition, + topic_partition.offset); + } + return output; +} + +} // mocking +} // cppkafka diff --git a/src/consumer.cpp b/src/consumer.cpp index c9efe05a..88251d20 100644 --- a/src/consumer.cpp +++ b/src/consumer.cpp @@ -178,7 +178,10 @@ TopicPartitionList Consumer::get_assignment() const { } string Consumer::get_member_id() const { - return rd_kafka_memberid(get_handle()); + char* id = rd_kafka_memberid(get_handle()); + string output = id; + free(id); + return output; } const Consumer::AssignmentCallback& Consumer::get_assignment_callback() const { pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy