From 36c9feaca95f2db77aa2060c69923a12e3336d40 Mon Sep 17 00:00:00 2001 From: Miquel Fontana Date: Mon, 28 Apr 2025 11:56:56 +0200 Subject: [PATCH 1/9] issues/57867 delayed queue for days to allow quorum queues with high expire --- .../Bridge/Amqp/Transport/Connection.php | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php index 0a6197810d4a..9c8317598011 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php @@ -80,6 +80,7 @@ class Connection 'flags', 'arguments', ]; + private const BASE_EXPIRATION = 24 * 60 * 60 * 1000; private AmqpFactory $amqpFactory; private mixed $autoSetupExchange; @@ -144,6 +145,11 @@ public function __construct( * * queue_name_pattern: Pattern to use to create the queues (Default: "delay_%exchange_name%_%routing_key%_%delay%") * * exchange_name: Name of the exchange to be used for the delayed/retried messages (Default: "delays") * * arguments: array of extra delay queue arguments (for example: ['x-queue-type' => 'classic', 'x-message-deduplication' => true,]) + * * daily_delay_queues: When true, delay queues will be created with names including the current date + * (e.g., 'delay_queue_2024_03_21'). These queues are automatically deleted by RabbitMQ after they + * expire (x-expires argument), the x-expires argument is set to 24 hours (24 * 60 * 60 * 1000) plus delay. This is useful for quorum queues. + * because quorum queues do not redeclare expire time. + * (Default: false) * * auto_setup: Enable or not the auto-setup of queues and exchanges (Default: true) * * * Connection tuning options (see http://www.rabbitmq.com/amqp-0-9-1-reference.html#connection.tune for details): @@ -390,11 +396,14 @@ private function createDelayQueue(int $delay, ?string $routingKey, bool $isRetry $queue = $this->amqpFactory->createQueue($this->channel()); $queue->setName($this->getRoutingKeyForDelay($delay, $routingKey, $isRetryAttempt)); $queue->setFlags(\AMQP_DURABLE); + $queueExpirationBase = ($this->connectionOptions['delay']['daily_delay_queues']?? false) ? + self::BASE_EXPIRATION : 0; $queue->setArguments(array_merge([ 'x-message-ttl' => $delay, - // delete the delay queue 10 seconds after the message expires - // publishing another message redeclares the queue which renews the lease - 'x-expires' => $delay + 10000, + // The delay queue will be automatically deleted by RabbitMQ after being empty for 10 seconds (x-expires). + // Each time a new message is published, the queue is redeclared for classic queues, which resets the expiry timer. + // For quorum queues, redeclaration is not allowed, so using daily_delay_queues=true is recommended to manage cleanup. + 'x-expires' => $queueExpirationBase + $delay + 10000, // message should be broadcast to all consumers during delay, but to only one queue during retry // empty name is default direct exchange 'x-dead-letter-exchange' => $isRetryAttempt ? '' : $this->exchangeOptions['name'], @@ -409,12 +418,13 @@ private function createDelayQueue(int $delay, ?string $routingKey, bool $isRetry private function getRoutingKeyForDelay(int $delay, ?string $finalRoutingKey, bool $isRetryAttempt): string { $action = $isRetryAttempt ? '_retry' : '_delay'; + $date = ($this->connectionOptions['delay']['daily_delay_queues']?? false) ? '_' . (new \DateTimeImmutable())->format('Y-m-d') : ''; return str_replace( - ['%delay%', '%exchange_name%', '%routing_key%'], - [$delay, $this->exchangeOptions['name'], $finalRoutingKey ?? ''], - $this->connectionOptions['delay']['queue_name_pattern'] - ).$action; + ['%delay%', '%exchange_name%', '%routing_key%'], + [$delay, $this->exchangeOptions['name'], $finalRoutingKey ?? ''], + $this->connectionOptions['delay']['queue_name_pattern'] + ).$action.$date; } /** From 8650b998ab1858212122221f6b2ffa58595ab4f1 Mon Sep 17 00:00:00 2001 From: Miquel Fontana Date: Mon, 28 Apr 2025 12:01:06 +0200 Subject: [PATCH 2/9] issues/57867 improve commend message --- .../Component/Messenger/Bridge/Amqp/Transport/Connection.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php index 9c8317598011..d5e0ff2853dc 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php @@ -400,9 +400,10 @@ private function createDelayQueue(int $delay, ?string $routingKey, bool $isRetry self::BASE_EXPIRATION : 0; $queue->setArguments(array_merge([ 'x-message-ttl' => $delay, - // The delay queue will be automatically deleted by RabbitMQ after being empty for 10 seconds (x-expires). - // Each time a new message is published, the queue is redeclared for classic queues, which resets the expiry timer. + // delete the delay queue 10 seconds after the message expires + // publishing another message redeclares the queue which renews the lease // For quorum queues, redeclaration is not allowed, so using daily_delay_queues=true is recommended to manage cleanup. + // It will create a new queue for each day, with x-expires set to 24 hours (24 * 60 * 60 * 1000) plus delay. 'x-expires' => $queueExpirationBase + $delay + 10000, // message should be broadcast to all consumers during delay, but to only one queue during retry // empty name is default direct exchange From e0929bb14571b1e459c8ccf1c4cf152a8b8454b1 Mon Sep 17 00:00:00 2001 From: Miquel Fontana Date: Tue, 29 Apr 2025 09:49:22 +0200 Subject: [PATCH 3/9] issues/57867 improve commend message --- .../Component/Messenger/Bridge/Amqp/Transport/Connection.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php index d5e0ff2853dc..12ca4c920275 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php @@ -146,7 +146,7 @@ public function __construct( * * exchange_name: Name of the exchange to be used for the delayed/retried messages (Default: "delays") * * arguments: array of extra delay queue arguments (for example: ['x-queue-type' => 'classic', 'x-message-deduplication' => true,]) * * daily_delay_queues: When true, delay queues will be created with names including the current date - * (e.g., 'delay_queue_2024_03_21'). These queues are automatically deleted by RabbitMQ after they + * (e.g., '%queue_name_pattern%_%current_date%'). These queues are automatically deleted by RabbitMQ after they * expire (x-expires argument), the x-expires argument is set to 24 hours (24 * 60 * 60 * 1000) plus delay. This is useful for quorum queues. * because quorum queues do not redeclare expire time. * (Default: false) From 884d79c06f30c39b3002a890fa29679f15cd589d Mon Sep 17 00:00:00 2001 From: Miquel Fontana Date: Tue, 29 Apr 2025 10:34:16 +0200 Subject: [PATCH 4/9] issues/57867 add tests --- .../Amqp/Tests/Transport/ConnectionTest.php | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php index f61c14cab066..a4e3b2b3f8c8 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php @@ -507,6 +507,22 @@ public function testItDelaysTheMessage() $connection->publish('{}', ['x-some-headers' => 'foo'], 5000); } + public function testItDelaysTheMessageWithDailyDelayQueues() + { + $delayExchange = $this->createMock(\AMQPExchange::class); + $date = (new \DateTimeImmutable())->format('Y-m-d'); + $delayExchange->expects($this->once()) + ->method('publish') + ->with('{}', "delay_messages__5000_delay_$date", \AMQP_NOPARAM, [ + 'headers' => ['x-some-headers' => 'foo'], + 'delivery_mode' => 2, + 'timestamp' => time(), + ]); + $connection = $this->createDelayOrRetryConnection($delayExchange, self::DEFAULT_EXCHANGE_NAME, "delay_messages__5000_delay_$date", true); + + $connection->publish('{}', ['x-some-headers' => 'foo'], 5000); + } + public function testItRetriesTheMessage() { $delayExchange = $this->createMock(\AMQPExchange::class); @@ -520,6 +536,20 @@ public function testItRetriesTheMessage() $connection->publish('{}', [], 5000, $amqpStamp); } + public function testItRetriesTheMessageWithDailyDelayQueues() + { + $delayExchange = $this->createMock(\AMQPExchange::class); + $date = (new \DateTimeImmutable())->format('Y-m-d'); + $delayExchange->expects($this->once()) + ->method('publish') + ->with('{}', "delay_messages__5000_retry_$date", \AMQP_NOPARAM); + $connection = $this->createDelayOrRetryConnection($delayExchange, '', "delay_messages__5000_retry_$date", true); + + $amqpEnvelope = $this->createMock(\AMQPEnvelope::class); + $amqpStamp = AmqpStamp::createFromAmqpEnvelope($amqpEnvelope, null, ''); + $connection->publish('{}', [], 5000, $amqpStamp); + } + public function testItDelaysTheMessageWithADifferentRoutingKeyAndTTLs() { $amqpConnection = $this->createMock(\AMQPConnection::class); @@ -849,7 +879,7 @@ public function testItWillRetryMaxThreeTimesWhenAMQPConnectionExceptionIsThrown( $connection->publish('body'); } - private function createDelayOrRetryConnection(\AMQPExchange $delayExchange, string $deadLetterExchangeName, string $delayQueueName): Connection + private function createDelayOrRetryConnection(\AMQPExchange $delayExchange, string $deadLetterExchangeName, string $delayQueueName, bool $dailyDelayQueues = false): Connection { $amqpConnection = $this->createMock(\AMQPConnection::class); $amqpChannel = $this->createMock(\AMQPChannel::class); @@ -861,19 +891,20 @@ private function createDelayOrRetryConnection(\AMQPExchange $delayExchange, stri $delayQueue = $this->createMock(\AMQPQueue::class); $factory->method('createQueue')->willReturn($this->createMock(\AMQPQueue::class), $delayQueue); $factory->method('createExchange')->willReturn($this->createMock(\AMQPExchange::class), $delayExchange); - + $baseExpire = $dailyDelayQueues ? 86400 * 1000 : 0; $delayQueue->expects($this->once())->method('setName')->with($delayQueueName); $delayQueue->expects($this->once())->method('setArguments')->with([ 'x-message-ttl' => 5000, - 'x-expires' => 5000 + 10000, + 'x-expires' => 5000 + 10000 + $baseExpire, 'x-dead-letter-exchange' => $deadLetterExchangeName, 'x-dead-letter-routing-key' => '', ]); $delayQueue->expects($this->once())->method('declareQueue'); $delayQueue->expects($this->once())->method('bind')->with('delays', $delayQueueName); + $options = $dailyDelayQueues ? ['delay' => ['daily_delay_queues' => true]] : []; - return Connection::fromDsn('amqp://localhost', [], $factory); + return Connection::fromDsn('amqp://localhost', $options, $factory); } } From fce316d402c49db813a91ad53132c6f527b0c4e4 Mon Sep 17 00:00:00 2001 From: Miquel Fontana Date: Tue, 29 Apr 2025 12:33:18 +0200 Subject: [PATCH 5/9] issues/57867 fix coding standard issues --- .../Messenger/Bridge/Amqp/Transport/Connection.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php index 12ca4c920275..3e12c92ee897 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php @@ -396,7 +396,7 @@ private function createDelayQueue(int $delay, ?string $routingKey, bool $isRetry $queue = $this->amqpFactory->createQueue($this->channel()); $queue->setName($this->getRoutingKeyForDelay($delay, $routingKey, $isRetryAttempt)); $queue->setFlags(\AMQP_DURABLE); - $queueExpirationBase = ($this->connectionOptions['delay']['daily_delay_queues']?? false) ? + $queueExpirationBase = ($this->connectionOptions['delay']['daily_delay_queues'] ?? false) ? self::BASE_EXPIRATION : 0; $queue->setArguments(array_merge([ 'x-message-ttl' => $delay, @@ -419,13 +419,13 @@ private function createDelayQueue(int $delay, ?string $routingKey, bool $isRetry private function getRoutingKeyForDelay(int $delay, ?string $finalRoutingKey, bool $isRetryAttempt): string { $action = $isRetryAttempt ? '_retry' : '_delay'; - $date = ($this->connectionOptions['delay']['daily_delay_queues']?? false) ? '_' . (new \DateTimeImmutable())->format('Y-m-d') : ''; + $date = ($this->connectionOptions['delay']['daily_delay_queues'] ?? false) ? '_' . (new \DateTimeImmutable())->format('Y-m-d') : ''; return str_replace( - ['%delay%', '%exchange_name%', '%routing_key%'], - [$delay, $this->exchangeOptions['name'], $finalRoutingKey ?? ''], - $this->connectionOptions['delay']['queue_name_pattern'] - ).$action.$date; + ['%delay%', '%exchange_name%', '%routing_key%'], + [$delay, $this->exchangeOptions['name'], $finalRoutingKey ?? ''], + $this->connectionOptions['delay']['queue_name_pattern'] + ).$action.$date; } /** From dc8802e9d8df819cac0a6b6952f21ff195792105 Mon Sep 17 00:00:00 2001 From: Miquel Fontana Date: Tue, 29 Apr 2025 12:34:29 +0200 Subject: [PATCH 6/9] issues/57867 fix coding standard issues --- .../Component/Messenger/Bridge/Amqp/Transport/Connection.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php index 3e12c92ee897..f47c2ddac06f 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php @@ -419,7 +419,7 @@ private function createDelayQueue(int $delay, ?string $routingKey, bool $isRetry private function getRoutingKeyForDelay(int $delay, ?string $finalRoutingKey, bool $isRetryAttempt): string { $action = $isRetryAttempt ? '_retry' : '_delay'; - $date = ($this->connectionOptions['delay']['daily_delay_queues'] ?? false) ? '_' . (new \DateTimeImmutable())->format('Y-m-d') : ''; + $date = ($this->connectionOptions['delay']['daily_delay_queues'] ?? false) ? '_'.(new \DateTimeImmutable())->format('Y-m-d') : ''; return str_replace( ['%delay%', '%exchange_name%', '%routing_key%'], From ff3c4b23fdec549c6d028bf4ce4150e161eea565 Mon Sep 17 00:00:00 2001 From: miquel-angel Date: Thu, 19 Jun 2025 12:39:35 +0200 Subject: [PATCH 7/9] Update src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php Co-authored-by: Oskar Stark --- .../Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php index a4e3b2b3f8c8..0ca79ecedb1a 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/ConnectionTest.php @@ -545,8 +545,7 @@ public function testItRetriesTheMessageWithDailyDelayQueues() ->with('{}', "delay_messages__5000_retry_$date", \AMQP_NOPARAM); $connection = $this->createDelayOrRetryConnection($delayExchange, '', "delay_messages__5000_retry_$date", true); - $amqpEnvelope = $this->createMock(\AMQPEnvelope::class); - $amqpStamp = AmqpStamp::createFromAmqpEnvelope($amqpEnvelope, null, ''); + $amqpStamp = AmqpStamp::createFromAmqpEnvelope($this->createMock(\AMQPEnvelope::class), null, ''); $connection->publish('{}', [], 5000, $amqpStamp); } From 82f948a4ed2483933b6445da941f9c294d487685 Mon Sep 17 00:00:00 2001 From: Miquel Fontana Date: Thu, 19 Jun 2025 12:43:09 +0200 Subject: [PATCH 8/9] issues/57867 remove not needed constant --- .../Component/Messenger/Bridge/Amqp/Transport/Connection.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php index f47c2ddac06f..7341a51ecb55 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php @@ -80,7 +80,6 @@ class Connection 'flags', 'arguments', ]; - private const BASE_EXPIRATION = 24 * 60 * 60 * 1000; private AmqpFactory $amqpFactory; private mixed $autoSetupExchange; @@ -397,7 +396,7 @@ private function createDelayQueue(int $delay, ?string $routingKey, bool $isRetry $queue->setName($this->getRoutingKeyForDelay($delay, $routingKey, $isRetryAttempt)); $queue->setFlags(\AMQP_DURABLE); $queueExpirationBase = ($this->connectionOptions['delay']['daily_delay_queues'] ?? false) ? - self::BASE_EXPIRATION : 0; + 24 * 60 * 60 * 1000 : 0; $queue->setArguments(array_merge([ 'x-message-ttl' => $delay, // delete the delay queue 10 seconds after the message expires From 43bd646db30215aecacb73cfda7299138fe10e53 Mon Sep 17 00:00:00 2001 From: Miquel Fontana Date: Mon, 28 Jul 2025 08:33:10 +0200 Subject: [PATCH 9/9] issues/57867 single line ternary and add in changelog new argument --- src/Symfony/Component/Messenger/Bridge/Amqp/CHANGELOG.md | 5 +++++ .../Component/Messenger/Bridge/Amqp/Transport/Connection.php | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/CHANGELOG.md b/src/Symfony/Component/Messenger/Bridge/Amqp/CHANGELOG.md index ff8061b69190..3d0b4773e758 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/CHANGELOG.md +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +7.4 +--- + + * Add option `delay[daily_delay_queues]` in the transport definition + 7.1 --- diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php index 7341a51ecb55..eef97cc52230 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/Connection.php @@ -395,8 +395,7 @@ private function createDelayQueue(int $delay, ?string $routingKey, bool $isRetry $queue = $this->amqpFactory->createQueue($this->channel()); $queue->setName($this->getRoutingKeyForDelay($delay, $routingKey, $isRetryAttempt)); $queue->setFlags(\AMQP_DURABLE); - $queueExpirationBase = ($this->connectionOptions['delay']['daily_delay_queues'] ?? false) ? - 24 * 60 * 60 * 1000 : 0; + $queueExpirationBase = ($this->connectionOptions['delay']['daily_delay_queues'] ?? false) ? 24 * 60 * 60 * 1000 : 0; $queue->setArguments(array_merge([ 'x-message-ttl' => $delay, // delete the delay queue 10 seconds after the message expires pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy