EachBatch called on the same partition at the same time #222
Description
Environment Information
- OS [ubuntu 24.10]:
- Node Version [20.16.0]:
- confluent-kafka-javascript version [1.0.0]:
Summary
The following code produces messages to a given kafka topic with 2 partitions. When consuming using eachBatch, the eachBatch callback is being called twice at the time, on the same partition. Causes messages to be read out of order in a single partition
Reproduce
import { KafkaJS } from "@confluentinc/kafka-javascript";
import { config } from "../src/config";
const kafka = new KafkaJS.Kafka({
kafkaJS: {
brokers: config.kafkaBrokersAddress.split(","),
},
});
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const topicName = "some-name";
const consumerGroup = topicName;
(async () => {
const admin = kafka.admin();
await admin.connect();
await admin.createTopics({
topics: [
{
topic: topicName,
numPartitions: 2,
},
],
});
console.log("Partitions added successfully!");
await admin.disconnect();
const producer = kafka.producer();
await producer.connect();
for (let i = 0; i < 50; i++) {
const messages = [];
for (let q = 0; q < 1000; q++) {
messages.push({ headers: {}, value: (q + i * 1000).toString() + " " + "a".repeat(1000) });
}
await producer.send({
topic: topicName,
messages: messages,
});
}
await producer.disconnect();
const consumer = kafka.consumer({
kafkaJS: {
groupId: consumerGroup,
maxWaitTimeInMs: 5000,
fromBeginning: true,
},
});
await consumer.connect();
await consumer.subscribe({ topics: [topicName] });
await consumer.run({
partitionsConsumedConcurrently: 2,
eachBatch: async ({ batch }) => {
console.log("handling batch", {
partition: batch.partition,
topic: batch.topic,
});
await sleep(1000);
console.log("finished batch", { topic: batch.topic, partition: batch.partition });
},
});
await new Promise((resolve) => setTimeout(resolve, 30000));
await consumer.disconnect();
})();Output
handling batch {
timestamp: 2024-12-23T11:05:48.090Z,
partition: 1,
topic: 'some-name',
firstValue: '0',
lastValue: '0'
}
handling batch {
timestamp: 2024-12-23T11:05:48.091Z,
partition: 1,
topic: 'some-name',
firstValue: '1',
lastValue: '1'
}
finished batch {
timestamp: 2024-12-23T11:05:49.092Z,
topic: 'some-name',
partition: 1,
firstValue: '0',
lastValue: '0'
}
finished batch {
timestamp: 2024-12-23T11:05:49.093Z,
topic: 'some-name',
partition: 1,
firstValue: '1',
lastValue: '1'
}
handling batch {
timestamp: 2024-12-23T11:05:49.094Z,
partition: 1,
topic: 'some-name',
firstValue: '2',
lastValue: '3'
}
handling batch {
timestamp: 2024-12-23T11:05:49.095Z,
partition: 1,
topic: 'some-name',
firstValue: '7',
lastValue: '9'
}
finished batch {
timestamp: 2024-12-23T11:05:50.096Z,
topic: 'some-name',
partition: 1,
firstValue: '2',
lastValue: '3'
}
finished batch {
timestamp: 2024-12-23T11:05:50.097Z,
topic: 'some-name',
partition: 1,
firstValue: '7',
lastValue: '9'
}
...