Skip to content

Commit e0289f0

Browse files
mfontaninimfontanini
committed
Fix several issues and make consumption work
1 parent 4665bf6 commit e0289f0

12 files changed

+138
-75
lines changed

‎mocking/include/cppkafka/mocking/configuration_mock.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ class ConfigurationMock {
7575

7676
struct Cloner {
7777
ConfigurationMock* operator()(const ConfigurationMock* ptr) const {
78-
return new ConfigurationMock(*ptr);
78+
return ptr ? new ConfigurationMock(*ptr) : nullptr;
7979
}
8080
};
8181

‎mocking/include/cppkafka/mocking/consumer_mock.h

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include <cppkafka/mocking/configuration_mock.h>
1717
#include <cppkafka/mocking/topic_partition_mock.h>
1818
#include <cppkafka/mocking/message_handle.h>
19+
#include <cppkafka/mocking/kafka_cluster.h>
1920

2021
namespace cppkafka {
2122
namespace mocking {
@@ -57,6 +58,8 @@ class ConsumerMock : public HandleMock {
5758
using TopicPartitionId = std::tuple<std::string, int>;
5859

5960
static TopicPartitionId make_id(const TopicPartitionMock& topic_partition);
61+
KafkaCluster::ResetOffsetPolicy get_offset_policy() const;
62+
bool get_partition_eof_enabled() const;
6063
void on_assignment(const std::vector<TopicPartitionMock>& topic_partitions);
6164
void on_revocation();
6265
void on_message(const std::string& topic_name, unsigned partition, uint64_t offset,
@@ -65,15 +68,14 @@ class ConsumerMock : public HandleMock {
6568
const std::vector<TopicPartitionMock>& topic_partitions);
6669

6770
ConfigurationMock config_;
68-
// TODO: initialize this and make it const
71+
std::string group_id_;
72+
const KafkaCluster::ResetOffsetPolicy offset_reset_policy_;
6973
bool emit_eofs_;
70-
const std::string group_id_;
7174
std::map<TopicPartitionId, TopicPartitionInfo> assigned_partitions_;
7275
std::set<TopicPartitionId> consumable_topic_partitions_;
7376
std::set<TopicPartitionId> paused_topic_partitions_;
7477
mutable std::mutex mutex_;
7578
std::condition_variable messages_condition_;
76-
void* opaque_;
7779
uint64_t consumer_id_;
7880
};
7981

‎mocking/include/cppkafka/mocking/kafka_cluster.h

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ class KafkaCluster {
2121
using MessageCallback = std::function<void(std::string topic, unsigned partition,
2222
uint64_t offset, const KafkaMessageMock&)>;
2323

24+
enum class ResetOffsetPolicy {
25+
Earliest = 1,
26+
Latest = 2
27+
};
28+
2429
static std::shared_ptr<KafkaCluster> make_cluster(std::string url);
2530

2631
KafkaCluster(const KafkaCluster&) = delete;
@@ -40,7 +45,7 @@ class KafkaCluster {
4045
RevocationCallback revocation_callback);
4146
void unsubscribe(const std::string& group_id, uint64_t consumer_id);
4247
void assign(uint64_t consumer_id, const std::vector<TopicPartitionMock>& topic_partitions,
43-
const MessageCallback& message_callback);
48+
ResetOffsetPolicy policy, const MessageCallback& message_callback);
4449
void unassign(uint64_t consumer_id);
4550
private:
4651
struct ConsumerMetadata {
@@ -68,7 +73,7 @@ class KafkaCluster {
6873
mutable std::mutex topics_mutex_;
6974
std::unordered_map<uint64_t, ConsumerMetadata> consumer_data_;
7075
std::unordered_map<std::string, TopicConsumersMap> group_topics_data_;
71-
mutable std::mutex consumer_data_mutex_;
76+
mutable std::recursive_mutex consumer_data_mutex_;
7277
};
7378

7479
} // mocking

‎mocking/include/cppkafka/mocking/kafka_cluster_registry.h

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,13 @@ namespace detail {
1111

1212
class KafkaClusterRegistry {
1313
public:
14-
using ClusterPtr = std::shared_ptr<KafkaCluster>;
15-
1614
static KafkaClusterRegistry& instance();
1715

18-
void add_cluster(ClusterPtr cluster);
19-
void remove_cluster(const KafkaCluster& cluster);
20-
ClusterPtr get_cluster(const std::string& name) const;
16+
void add_cluster(std::shared_ptr<KafkaCluster> cluster);
17+
void remove_cluster(const std::string& url);
18+
std::shared_ptr<KafkaCluster> get_cluster(const std::string& name) const;
2119
private:
22-
std::unordered_map<std::string, ClusterPtr> clusters_;
20+
std::unordered_map<std::string, std::weak_ptr<KafkaCluster>> clusters_;
2321
mutable std::mutex clusters_mutex_;
2422
};
2523

‎mocking/include/cppkafka/mocking/kafka_partition_mock.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class KafkaPartitionMock {
2323
SubscriberId subscribe(MessageCallback callback);
2424
void unsubscribe(SubscriberId id);
2525
// Returns interval [lowest offset, largest offset)
26-
std::tuple<uint64_t, uint64_t> get_offset_bounds() const;
26+
std::tuple<int64_t, int64_t> get_offset_bounds() const;
2727

2828
// Acquire this partition so that no messages can be produced while the callback is executed.
2929
template <typename Functor>

‎mocking/src/api.cpp

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ void rd_kafka_topic_conf_set_partitioner_cb(rd_kafka_topic_conf_t* conf,
115115
void rd_kafka_conf_set_default_topic_conf(rd_kafka_conf_t* conf,
116116
rd_kafka_topic_conf_t* tconf) {
117117
conf->get_handle().set_default_topic_configuration(tconf->get_handle());
118+
rd_kafka_topic_conf_destroy(tconf);
118119
}
119120

120121
void rd_kafka_conf_set_opaque(rd_kafka_conf_t* conf, void* opaque) {
@@ -198,7 +199,7 @@ rd_kafka_topic_partition_list_t* rd_kafka_topic_partition_list_new(int size) {
198199

199200
void rd_kafka_topic_partition_list_destroy(rd_kafka_topic_partition_list_t* toppar_list) {
200201
for (int i = 0; i < toppar_list->cnt; ++i) {
201-
delete toppar_list->elems[i].topic;
202+
delete[] toppar_list->elems[i].topic;
202203
}
203204
delete[] toppar_list->elems;
204205
delete toppar_list;
@@ -214,6 +215,7 @@ rd_kafka_topic_partition_list_add(rd_kafka_topic_partition_list_t* toppar_list,
214215
const size_t length = strlen(topic);
215216
output->topic = new char[length + 1];
216217
copy(topic, topic + length, output->topic);
218+
output->topic[length] = 0;
217219
output->partition = partition;
218220
output->offset = RD_KAFKA_OFFSET_INVALID;
219221
return output;
@@ -240,16 +242,17 @@ int rd_kafka_topic_partition_available(const rd_kafka_topic_t* rkt, int32_t part
240242

241243
// rd_kafka_t
242244

243-
rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf_ptr,
245+
rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t* conf_ptr,
244246
char *errstr, size_t errstr_size) {
245247
static const string BROKERS_OPTION = "metadata.broker.list";
246-
const auto& conf = conf_ptr->get_handle();
248+
auto& conf = conf_ptr->get_handle();
247249
HandleMock::ClusterPtr cluster;
248250
if (conf.has_key(BROKERS_OPTION)) {
249251
cluster = KafkaClusterRegistry::instance().get_cluster(conf.get(BROKERS_OPTION));
250252
}
251253
if (type == RD_KAFKA_PRODUCER) {
252-
return new rd_kafka_t(new ProducerMock(conf, make_shared<EventProcessor>(),
254+
unique_ptr<rd_kafka_conf_t> _(conf_ptr);
255+
return new rd_kafka_t(new ProducerMock(move(conf), make_shared<EventProcessor>(),
253256
move(cluster)));
254257
}
255258
else if (type == RD_KAFKA_CONSUMER) {
@@ -261,14 +264,15 @@ rd_kafka_t* rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf_ptr,
261264
}
262265
return nullptr;
263266
}
264-
return new rd_kafka_t(new ConsumerMock(conf, make_shared<EventProcessor>(),
267+
unique_ptr<rd_kafka_conf_t> _(conf_ptr);
268+
return new rd_kafka_t(new ConsumerMock(move(conf), make_shared<EventProcessor>(),
265269
move(cluster)));
266270
}
267271
return nullptr;
268272
}
269273

270274
void rd_kafka_destroy(rd_kafka_t* rk) {
271-
delete &rk->get_handle();
275+
delete rk;
272276
}
273277

274278
int rd_kafka_brokers_add(rd_kafka_t* rk, const char* brokerlist) {
@@ -340,9 +344,14 @@ rd_kafka_resp_err_t rd_kafka_subscription(rd_kafka_t* rk,
340344

341345
rd_kafka_resp_err_t rd_kafka_assign(rd_kafka_t* rk,
342346
const rd_kafka_topic_partition_list_t* partitions) {
343-
const vector<TopicPartitionMock> topic_partitions = from_rdkafka_handle(*partitions);
344347
auto& consumer = rk->get_handle<ConsumerMock>();
345-
consumer.assign(topic_partitions);
348+
if (partitions) {
349+
const vector<TopicPartitionMock> topic_partitions = from_rdkafka_handle(*partitions);
350+
consumer.assign(topic_partitions);
351+
}
352+
else {
353+
consumer.unassign();
354+
}
346355
return RD_KAFKA_RESP_ERR_NO_ERROR;
347356
}
348357

@@ -471,8 +480,8 @@ rd_kafka_resp_err_t rd_kafka_get_watermark_offsets(rd_kafka_t* rk, const char *t
471480
return RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION;
472481
}
473482
const auto& partition_object = topic_object.get_partition(partition);
474-
uint64_t lowest;
475-
uint64_t largest;
483+
int64_t lowest;
484+
int64_t largest;
476485
tie(lowest, largest) = partition_object.get_offset_bounds();
477486
*low = lowest;
478487
*high = largest;

‎mocking/src/consumer_mock.cpp

Lines changed: 49 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using std::atomic;
1010
using std::vector;
1111
using std::string;
12+
using std::unordered_map;
1213
using std::to_string;
1314
using std::move;
1415
using std::bind;
@@ -37,10 +38,12 @@ uint64_t ConsumerMock::make_consumer_id() {
3738
ConsumerMock::ConsumerMock(ConfigurationMock config, EventProcessorPtr processor,
3839
ClusterPtr cluster)
3940
: HandleMock(move(processor), move(cluster)), config_(move(config)),
41+
offset_reset_policy_(get_offset_policy()), emit_eofs_(get_partition_eof_enabled()),
4042
consumer_id_(make_consumer_id()) {
4143
if (!config_.has_key(CONFIG_GROUP_ID)) {
4244
throw runtime_error("Failed to find " + CONFIG_GROUP_ID + " in config");
4345
}
46+
group_id_ = config_.get(CONFIG_GROUP_ID);
4447
}
4548

4649
ConsumerMock::~ConsumerMock() {
@@ -73,7 +76,13 @@ void ConsumerMock::assign(const vector<TopicPartitionMock>& topic_partitions) {
7376
for (const TopicPartitionMock& topic_partition : topic_partitions) {
7477
const auto id = make_id(topic_partition);
7578
// We'll store the next offset from the one we've seen so far
76-
const uint64_t next_offset = topic_partition.get_offset() + 1;
79+
uint64_t next_offset;
80+
if (topic_partition.get_offset() == RD_KAFKA_OFFSET_INVALID) {
81+
next_offset = 0;
82+
}
83+
else {
84+
next_offset = topic_partition.get_offset() + 1;
85+
}
7786

7887
auto iter = assigned_partitions_.find(id);
7988
if (iter == assigned_partitions_.end()) {
@@ -93,14 +102,15 @@ void ConsumerMock::assign(const vector<TopicPartitionMock>& topic_partitions) {
93102
using namespace std::placeholders;
94103
// Now assign these partitions. This will atomically fetch all message we should fetch and
95104
// then subscribe us to the topic/partitions
96-
get_cluster().assign(consumer_id_, topic_partitions,
105+
get_cluster().assign(consumer_id_, topic_partitions, offset_reset_policy_,
97106
bind(&ConsumerMock::on_message, this, _1, _2, _3, _4));
98107
}
99108

100109
void ConsumerMock::unassign() {
101110
lock_guard<mutex> _(mutex_);
102-
get_cluster().unassign(consumer_id_);
103111
assigned_partitions_.clear();
112+
consumable_topic_partitions_.clear();
113+
get_cluster().unassign(consumer_id_);
104114
}
105115

106116
void ConsumerMock::pause_partitions(const vector<TopicPartitionMock>& topic_partitions) {
@@ -124,11 +134,10 @@ void ConsumerMock::resume_partitions(const vector<TopicPartitionMock>& topic_par
124134
}
125135
}
126136

127-
unique_ptr<MessageHandle> ConsumerMock::poll(std::chrono::milliseconds timeout) {
128-
auto wait_until = steady_clock::now() + timeout;
137+
unique_ptr<MessageHandle> ConsumerMock::poll(milliseconds timeout) {
129138
unique_lock<mutex> lock(mutex_);
130-
while(consumable_topic_partitions_.empty() && steady_clock::now() > wait_until) {
131-
messages_condition_.wait_until(lock, wait_until);
139+
if (consumable_topic_partitions_.empty()) {
140+
messages_condition_.wait_for(lock, timeout);
132141
}
133142
if (consumable_topic_partitions_.empty()) {
134143
return nullptr;
@@ -189,20 +198,48 @@ ConsumerMock::TopicPartitionId ConsumerMock::make_id(const TopicPartitionMock& t
189198
return make_tuple(topic_partition.get_topic(), topic_partition.get_partition());
190199
}
191200

201+
KafkaCluster::ResetOffsetPolicy ConsumerMock::get_offset_policy() const {
202+
static const string KEY_NAME = "auto.offset.reset";
203+
static unordered_map<string, KafkaCluster::ResetOffsetPolicy> MAPPINGS = {
204+
{ "smallest", KafkaCluster::ResetOffsetPolicy::Earliest },
205+
{ "earliest", KafkaCluster::ResetOffsetPolicy::Earliest },
206+
{ "beginning", KafkaCluster::ResetOffsetPolicy::Earliest },
207+
{ "latest", KafkaCluster::ResetOffsetPolicy::Latest },
208+
{ "largest", KafkaCluster::ResetOffsetPolicy::Latest },
209+
{ "end", KafkaCluster::ResetOffsetPolicy::Latest },
210+
};
211+
212+
const ConfigurationMock* topic_config = config_.get_default_topic_configuration();
213+
if (!topic_config || !topic_config->has_key(KEY_NAME)) {
214+
return KafkaCluster::ResetOffsetPolicy::Earliest;
215+
}
216+
else {
217+
auto iter = MAPPINGS.find(topic_config->get(KEY_NAME));
218+
if (iter == MAPPINGS.end()) {
219+
throw runtime_error("invalid auto.offset.reset value");
220+
}
221+
return iter->second;
222+
}
223+
}
224+
225+
bool ConsumerMock::get_partition_eof_enabled() const {
226+
static const string KEY_NAME = "enable.partition.eof";
227+
return !config_.has_key(KEY_NAME) || config_.get(KEY_NAME) == "true";
228+
}
229+
192230
void ConsumerMock::on_assignment(const vector<TopicPartitionMock>& topic_partitions) {
193231
handle_rebalance(RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, topic_partitions);
194232
}
195233

196234
void ConsumerMock::on_revocation() {
197-
// Fetch and reset all assigned topic partitions
235+
// Fetch and all assigned topic partitions
198236
vector<TopicPartitionMock> topic_partitions = [&]() {
199237
lock_guard<mutex> _(mutex_);
200238
vector<TopicPartitionMock> output;
201239
for (const auto& topic_partition_pair : assigned_partitions_) {
202240
const TopicPartitionId& id = topic_partition_pair.first;
203241
output.emplace_back(get<0>(id), get<1>(id));
204242
}
205-
assigned_partitions_.clear();
206243
return output;
207244
}();
208245
handle_rebalance(RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS, topic_partitions);
@@ -211,13 +248,13 @@ void ConsumerMock::on_revocation() {
211248
void ConsumerMock::on_message(const string& topic_name, unsigned partition, uint64_t offset,
212249
const KafkaMessageMock& message) {
213250
auto id = make_tuple(topic_name, partition);
251+
MessageAggregate aggregate = { topic_name, partition, offset, &message };
214252

215253
// We should only process this if we don't have this topic/partition assigned (assignment
216254
// pending?) or the message offset comes after the next offset we have stored
217255
lock_guard<mutex> _(mutex_);
218256
auto iter = assigned_partitions_.find(id);
219-
MessageAggregate aggregate = { topic_name, partition, offset, &message };
220-
if (iter != assigned_partitions_.end()) {
257+
if (iter == assigned_partitions_.end()) {
221258
throw runtime_error("got message for unexpected partition " + to_string(partition));
222259
}
223260
if (offset > iter->second.next_offset) {
@@ -240,7 +277,7 @@ void ConsumerMock::handle_rebalance(rd_kafka_resp_err_t type,
240277
auto rebalance_callback = config_.get_rebalance_callback();
241278
if (rebalance_callback) {
242279
auto handle = to_rdkafka_handle(topic_partitions);
243-
rebalance_callback(nullptr, type, handle.get(), opaque_);
280+
rebalance_callback(nullptr, type, handle.get(), config_.get_opaque());
244281
}
245282
}
246283

‎mocking/src/event_processor.cpp

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include <cppkafka/mocking/event_processor.h>
22

3+
using std::thread;
34
using std::lock_guard;
45
using std::unique_lock;
56
using std::mutex;
@@ -10,9 +11,8 @@ using std::chrono::milliseconds;
1011
namespace cppkafka {
1112
namespace mocking {
1213

13-
EventProcessor::EventProcessor()
14-
: processing_thread_(&EventProcessor::process_events, this) {
15-
14+
EventProcessor::EventProcessor() {
15+
processing_thread_ = thread(&EventProcessor::process_events, this);
1616
}
1717

1818
EventProcessor::~EventProcessor() {
@@ -50,10 +50,12 @@ void EventProcessor::process_events() {
5050
while (running_ && events_.empty()) {
5151
new_events_condition_.wait(lock);
5252
}
53-
5453
if (!running_) {
5554
break;
5655
}
56+
if (events_.empty()) {
57+
continue;
58+
}
5759
EventPtr event = move(events_.front());
5860
events_.pop();
5961

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy