Skip to content

Commit 8d45013

Browse files
fabpotfabpot
committed
feature #38182 [HttpClient] Added RetryHttpClient (jderusse)
This PR was squashed before being merged into the 5.2-dev branch. Discussion ---------- [HttpClient] Added RetryHttpClient | Q | A | ------------- | --- | Branch? | master | Bug fix? | no | New feature? | yes | Deprecations? | no | Tickets | / | License | MIT | Doc PR | TODO This PR adds a new HttpClient decorator to automatically retry failed requests. When calling API, A very small % of requests are expected to timeout due to transient network issues. Some providers like AWS recommends retrying these requests and use a lower connection timeout so that the clients can fail fast and retry. I used the almost the same configuration as Messenger ```yaml framework: http_client: default_options: retry_failed: enabled: true // default false decider_service: null backoff_service: null http_codes: [423, 425, 429, 500, 502, 503, 504, 507, 510] max_retries: 3 delay: 1000 multiplier: 2 max_delay: 0 response_header: true scoped_clients: github: scope: 'https://api\.github\.com' retry_failed: max_delay: 2000 ``` Commits ------- 712ac59 [HttpClient] Added RetryHttpClient
2 parents 86c7113 + 712ac59 commit 8d45013

19 files changed

+662
-6
lines changed

‎src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ CHANGELOG
1010
`cache_clearer`, `filesystem` and `validator` services to private.
1111
* Added `TemplateAwareDataCollectorInterface` and `AbstractDataCollector` to simplify custom data collector creation and leverage autoconfiguration
1212
* Add `cache.adapter.redis_tag_aware` tag to use `RedisCacheAwareAdapter`
13+
* added `framework.http_client.retry_failing` configuration tree
1314

1415
5.1.0
1516
-----

‎src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
use Symfony\Bundle\FullStack;
1818
use Symfony\Component\Asset\Package;
1919
use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
20+
use Symfony\Component\Config\Definition\Builder\NodeBuilder;
2021
use Symfony\Component\Config\Definition\Builder\TreeBuilder;
2122
use Symfony\Component\Config\Definition\ConfigurationInterface;
2223
use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException;
@@ -1369,6 +1370,25 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
13691370
->info('HTTP Client configuration')
13701371
->{!class_exists(FullStack::class) && class_exists(HttpClient::class) ? 'canBeDisabled' : 'canBeEnabled'}()
13711372
->fixXmlConfig('scoped_client')
1373+
->beforeNormalization()
1374+
->always(function ($config) {
1375+
if (empty($config['scoped_clients']) || !\is_array($config['default_options']['retry_failed'] ?? null)) {
1376+
return $config;
1377+
}
1378+
1379+
foreach ($config['scoped_clients'] as &$scopedConfig) {
1380+
if (!isset($scopedConfig['retry_failed']) || true === $scopedConfig['retry_failed']) {
1381+
$scopedConfig['retry_failed'] = $config['default_options']['retry_failed'];
1382+
continue;
1383+
}
1384+
if (\is_array($scopedConfig['retry_failed'])) {
1385+
$scopedConfig['retry_failed'] = $scopedConfig['retry_failed'] + $config['default_options']['retry_failed'];
1386+
}
1387+
}
1388+
1389+
return $config;
1390+
})
1391+
->end()
13721392
->children()
13731393
->integerNode('max_host_connections')
13741394
->info('The maximum number of connections to a single host.')
@@ -1454,6 +1474,7 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
14541474
->variableNode('md5')->end()
14551475
->end()
14561476
->end()
1477+
->append($this->addHttpClientRetrySection())
14571478
->end()
14581479
->end()
14591480
->scalarNode('mock_response_factory')
@@ -1596,6 +1617,7 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
15961617
->variableNode('md5')->end()
15971618
->end()
15981619
->end()
1620+
->append($this->addHttpClientRetrySection())
15991621
->end()
16001622
->end()
16011623
->end()
@@ -1605,6 +1627,50 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
16051627
;
16061628
}
16071629

1630+
private function addHttpClientRetrySection()
1631+
{
1632+
$root = new NodeBuilder();
1633+
1634+
return $root
1635+
->arrayNode('retry_failed')
1636+
->fixXmlConfig('http_code')
1637+
->canBeEnabled()
1638+
->addDefaultsIfNotSet()
1639+
->beforeNormalization()
1640+
->always(function ($v) {
1641+
if (isset($v['backoff_service']) && (isset($v['delay']) || isset($v['multiplier']) || isset($v['max_delay']))) {
1642+
throw new \InvalidArgumentException('The "backoff_service" option cannot be used along with the "delay", "multiplier" or "max_delay" options.');
1643+
}
1644+
if (isset($v['decider_service']) && (isset($v['http_codes']))) {
1645+
throw new \InvalidArgumentException('The "decider_service" option cannot be used along with the "http_codes" options.');
1646+
}
1647+
1648+
return $v;
1649+
})
1650+
->end()
1651+
->children()
1652+
->scalarNode('backoff_service')->defaultNull()->info('service id to override the retry backoff')->end()
1653+
->scalarNode('decider_service')->defaultNull()->info('service id to override the retry decider')->end()
1654+
->arrayNode('http_codes')
1655+
->performNoDeepMerging()
1656+
->beforeNormalization()
1657+
->ifArray()
1658+
->then(function ($v) {
1659+
return array_filter(array_values($v));
1660+
})
1661+
->end()
1662+
->prototype('integer')->end()
1663+
->info('A list of HTTP status code that triggers a retry')
1664+
->defaultValue([423, 425, 429, 500, 502, 503, 504, 507, 510])
1665+
->end()
1666+
->integerNode('max_retries')->defaultValue(3)->min(0)->end()
1667+
->integerNode('delay')->defaultValue(1000)->min(0)->info('Time in ms to delay (or the initial value when multiplier is used)')->end()
1668+
->floatNode('multiplier')->defaultValue(2)->min(1)->info('If greater than 1, delay will grow exponentially for each retry: (delay * (multiple ^ retries))')->end()
1669+
->integerNode('max_delay')->defaultValue(0)->min(0)->info('Max time in ms that a retry should ever be delayed (0 = infinite)')->end()
1670+
->end()
1671+
;
1672+
}
1673+
16081674
private function addMailerSection(ArrayNodeDefinition $rootNode)
16091675
{
16101676
$rootNode

‎src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
use Symfony\Component\Form\FormTypeGuesserInterface;
6565
use Symfony\Component\Form\FormTypeInterface;
6666
use Symfony\Component\HttpClient\MockHttpClient;
67+
use Symfony\Component\HttpClient\RetryableHttpClient;
6768
use Symfony\Component\HttpClient\ScopingHttpClient;
6869
use Symfony\Component\HttpFoundation\Request;
6970
use Symfony\Component\HttpKernel\CacheClearer\CacheClearerInterface;
@@ -1991,7 +1992,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
19911992
{
19921993
$loader->load('http_client.php');
19931994

1994-
$container->getDefinition('http_client')->setArguments([$config['default_options'] ?? [], $config['max_host_connections'] ?? 6]);
1995+
$options = $config['default_options'] ?? [];
1996+
$retryOptions = $options['retry_failed'] ?? ['enabled' => false];
1997+
unset($options['retry_failed']);
1998+
$container->getDefinition('http_client')->setArguments([$options, $config['max_host_connections'] ?? 6]);
19951999

19962000
if (!$hasPsr18 = interface_exists(ClientInterface::class)) {
19972001
$container->removeDefinition('psr18.http_client');
@@ -2002,15 +2006,20 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
20022006
$container->removeDefinition(HttpClient::class);
20032007
}
20042008

2005-
$httpClientId = $this->isConfigEnabled($container, $profilerConfig) ? '.debug.http_client.inner' : 'http_client';
2009+
if ($this->isConfigEnabled($container, $retryOptions)) {
2010+
$this->registerHttpClientRetry($retryOptions, 'http_client', $container);
2011+
}
20062012

2013+
$httpClientId = $retryOptions['enabled'] ?? false ? 'http_client.retry.inner' : ($this->isConfigEnabled($container, $profilerConfig) ? '.debug.http_client.inner' : 'http_client');
20072014
foreach ($config['scoped_clients'] as $name => $scopeConfig) {
20082015
if ('http_client' === $name) {
20092016
throw new InvalidArgumentException(sprintf('Invalid scope name: "%s" is reserved.', $name));
20102017
}
20112018

20122019
$scope = $scopeConfig['scope'] ?? null;
20132020
unset($scopeConfig['scope']);
2021+
$retryOptions = $scopeConfig['retry_failed'] ?? ['enabled' => false];
2022+
unset($scopeConfig['retry_failed']);
20142023

20152024
if (null === $scope) {
20162025
$baseUri = $scopeConfig['base_uri'];
@@ -2028,6 +2037,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
20282037
;
20292038
}
20302039

2040+
if ($this->isConfigEnabled($container, $retryOptions)) {
2041+
$this->registerHttpClientRetry($retryOptions, $name, $container);
2042+
}
2043+
20312044
$container->registerAliasForArgument($name, HttpClientInterface::class);
20322045

20332046
if ($hasPsr18) {
@@ -2045,6 +2058,44 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
20452058
}
20462059
}
20472060

2061+
private function registerHttpClientRetry(array $retryOptions, string $name, ContainerBuilder $container)
2062+
{
2063+
if (!class_exists(RetryableHttpClient::class)) {
2064+
throw new LogicException('Retry failed request support cannot be enabled as version 5.2+ of the HTTP Client component is required.');
2065+
}
2066+
2067+
if (null !== $retryOptions['backoff_service']) {
2068+
$backoffReference = new Reference($retryOptions['backoff_service']);
2069+
} else {
2070+
$retryServiceId = $name.'.retry.exponential_backoff';
2071+
$retryDefinition = new ChildDefinition('http_client.retry.abstract_exponential_backoff');
2072+
$retryDefinition
2073+
->replaceArgument(0, $retryOptions['delay'])
2074+
->replaceArgument(1, $retryOptions['multiplier'])
2075+
->replaceArgument(2, $retryOptions['max_delay']);
2076+
$container->setDefinition($retryServiceId, $retryDefinition);
2077+
2078+
$backoffReference = new Reference($retryServiceId);
2079+
}
2080+
if (null !== $retryOptions['decider_service']) {
2081+
$deciderReference = new Reference($retryOptions['decider_service']);
2082+
} else {
2083+
$retryServiceId = $name.'.retry.decider';
2084+
$retryDefinition = new ChildDefinition('http_client.retry.abstract_httpstatuscode_decider');
2085+
$retryDefinition
2086+
->replaceArgument(0, $retryOptions['http_codes']);
2087+
$container->setDefinition($retryServiceId, $retryDefinition);
2088+
2089+
$deciderReference = new Reference($retryServiceId);
2090+
}
2091+
2092+
$container
2093+
->register($name.'.retry', RetryableHttpClient::class)
2094+
->setDecoratedService($name)
2095+
->setArguments([new Reference($name.'.retry.inner'), $deciderReference, $backoffReference, $retryOptions['max_retries'], new Reference('logger')])
2096+
->addTag('monolog.logger', ['channel' => 'http_client']);
2097+
}
2098+
20482099
private function registerMailerConfiguration(array $config, ContainerBuilder $container, PhpFileLoader $loader)
20492100
{
20502101
if (!class_exists(Mailer::class)) {

‎src/Symfony/Bundle/FrameworkBundle/Resources/config/http_client.php

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
use Symfony\Component\HttpClient\HttpClient;
1818
use Symfony\Component\HttpClient\HttplugClient;
1919
use Symfony\Component\HttpClient\Psr18Client;
20+
use Symfony\Component\HttpClient\Retry\ExponentialBackOff;
21+
use Symfony\Component\HttpClient\Retry\HttpStatusCodeDecider;
2022
use Symfony\Contracts\HttpClient\HttpClientInterface;
2123

2224
return static function (ContainerConfigurator $container) {
@@ -48,5 +50,19 @@
4850
service(ResponseFactoryInterface::class)->ignoreOnInvalid(),
4951
service(StreamFactoryInterface::class)->ignoreOnInvalid(),
5052
])
53+
54+
// retry
55+
->set('http_client.retry.abstract_exponential_backoff', ExponentialBackOff::class)
56+
->abstract()
57+
->args([
58+
abstract_arg('delay ms'),
59+
abstract_arg('multiplier'),
60+
abstract_arg('max delay ms'),
61+
])
62+
->set('http_client.retry.abstract_httpstatuscode_decider', HttpStatusCodeDecider::class)
63+
->abstract()
64+
->args([
65+
abstract_arg('http codes'),
66+
])
5167
;
5268
};

‎src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,7 @@
520520
<xsd:element name="resolve" type="http_resolve" minOccurs="0" maxOccurs="unbounded" />
521521
<xsd:element name="header" type="http_header" minOccurs="0" maxOccurs="unbounded" />
522522
<xsd:element name="peer-fingerprint" type="fingerprint" minOccurs="0" maxOccurs="unbounded" />
523+
<xsd:element name="retry-failed" type="http_client_retry_failed" minOccurs="0" maxOccurs="1" />
523524
</xsd:choice>
524525
<xsd:attribute name="max-redirects" type="xsd:integer" />
525526
<xsd:attribute name="http-version" type="xsd:string" />
@@ -536,7 +537,6 @@
536537
<xsd:attribute name="local-pk" type="xsd:string" />
537538
<xsd:attribute name="passphrase" type="xsd:string" />
538539
<xsd:attribute name="ciphers" type="xsd:string" />
539-
540540
</xsd:complexType>
541541

542542
<xsd:complexType name="http_client_scope_options" mixed="true">
@@ -545,6 +545,7 @@
545545
<xsd:element name="resolve" type="http_resolve" minOccurs="0" maxOccurs="unbounded" />
546546
<xsd:element name="header" type="http_header" minOccurs="0" maxOccurs="unbounded" />
547547
<xsd:element name="peer-fingerprint" type="fingerprint" minOccurs="0" maxOccurs="unbounded" />
548+
<xsd:element name="retry-failed" type="http_client_retry_failed" minOccurs="0" maxOccurs="1" />
548549
</xsd:choice>
549550
<xsd:attribute name="name" type="xsd:string" />
550551
<xsd:attribute name="scope" type="xsd:string" />
@@ -575,6 +576,20 @@
575576
</xsd:choice>
576577
</xsd:complexType>
577578

579+
<xsd:complexType name="http_client_retry_failed">
580+
<xsd:sequence>
581+
<xsd:element name="http-code" type="xsd:integer" minOccurs="0" maxOccurs="unbounded" />
582+
</xsd:sequence>
583+
<xsd:attribute name="enabled" type="xsd:boolean" />
584+
<xsd:attribute name="backoff-service" type="xsd:string" />
585+
<xsd:attribute name="decider-service" type="xsd:string" />
586+
<xsd:attribute name="max-retries" type="xsd:integer" />
587+
<xsd:attribute name="delay" type="xsd:integer" />
588+
<xsd:attribute name="multiplier" type="xsd:float" />
589+
<xsd:attribute name="max-delay" type="xsd:float" />
590+
<xsd:attribute name="response_header" type="xsd:boolean" />
591+
</xsd:complexType>
592+
578593
<xsd:complexType name="http_query" mixed="true">
579594
<xsd:attribute name="key" type="xsd:string" />
580595
</xsd:complexType>

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/php/http_client_retry.php

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
3+
$container->loadFromExtension('framework', [
4+
'http_client' => [
5+
'default_options' => [
6+
'retry_failed' => [
7+
'backoff_service' => null,
8+
'decider_service' => null,
9+
'http_codes' => [429, 500],
10+
'max_retries' => 2,
11+
'delay' => 100,
12+
'multiplier' => 2,
13+
'max_delay' => 0,
14+
]
15+
],
16+
'scoped_clients' => [
17+
'foo' => [
18+
'base_uri' => 'http://example.com',
19+
'retry_failed' => ['multiplier' => 4],
20+
],
21+
],
22+
],
23+
]);

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/xml/http_client_retry.xml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<container xmlns="http://symfony.com/schema/dic/services"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xmlns:framework="http://symfony.com/schema/dic/symfony"
5+
xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd
6+
http://symfony.com/schema/dic/symfony http://symfony.com/schema/dic/symfony/symfony-1.0.xsd">
7+
8+
<framework:config>
9+
<framework:http-client>
10+
<framework:default-options>
11+
<framework:retry-failed
12+
delay="100"
13+
max-delay="0"
14+
max-retries="2"
15+
multiplier="2">
16+
<framework:http-code>429</framework:http-code>
17+
<framework:http-code>500</framework:http-code>
18+
</framework:retry-failed>
19+
</framework:default-options>
20+
<framework:scoped-client name="foo" base-uri="http://example.com">
21+
<framework:retry-failed multiplier="4"/>
22+
</framework:scoped-client>
23+
</framework:http-client>
24+
</framework:config>
25+
</container>

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/yml/http_client_retry.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
framework:
2+
http_client:
3+
default_options:
4+
retry_failed:
5+
backoff_service: null
6+
decider_service: null
7+
http_codes: [429, 500]
8+
max_retries: 2
9+
delay: 100
10+
multiplier: 2
11+
max_delay: 0
12+
scoped_clients:
13+
foo:
14+
base_uri: http://example.com
15+
retry_failed:
16+
multiplier: 4

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTest.php

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,13 @@
3636
use Symfony\Component\DependencyInjection\ContainerBuilder;
3737
use Symfony\Component\DependencyInjection\ContainerInterface;
3838
use Symfony\Component\DependencyInjection\Definition;
39+
use Symfony\Component\DependencyInjection\Exception\LogicException;
3940
use Symfony\Component\DependencyInjection\Loader\ClosureLoader;
4041
use Symfony\Component\DependencyInjection\ParameterBag\EnvPlaceholderParameterBag;
4142
use Symfony\Component\DependencyInjection\Reference;
4243
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
4344
use Symfony\Component\HttpClient\MockHttpClient;
45+
use Symfony\Component\HttpClient\RetryableHttpClient;
4446
use Symfony\Component\HttpClient\ScopingHttpClient;
4547
use Symfony\Component\HttpKernel\DependencyInjection\LoggerPass;
4648
use Symfony\Component\Messenger\Transport\TransportFactory;
@@ -1482,6 +1484,23 @@ public function testHttpClientOverrideDefaultOptions()
14821484
$this->assertSame($expected, $container->getDefinition('foo')->getArgument(2));
14831485
}
14841486

1487+
public function testHttpClientRetry()
1488+
{
1489+
if (!class_exists(RetryableHttpClient::class)) {
1490+
$this->expectException(LogicException::class);
1491+
}
1492+
$container = $this->createContainerFromFile('http_client_retry');
1493+
1494+
$this->assertSame([429, 500], $container->getDefinition('http_client.retry.decider')->getArgument(0));
1495+
$this->assertSame(100, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(0));
1496+
$this->assertSame(2, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(1));
1497+
$this->assertSame(0, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(2));
1498+
$this->assertSame(2, $container->getDefinition('http_client.retry')->getArgument(3));
1499+
1500+
$this->assertSame(RetryableHttpClient::class, $container->getDefinition('foo.retry')->getClass());
1501+
$this->assertSame(4, $container->getDefinition('foo.retry.exponential_backoff')->getArgument(1));
1502+
}
1503+
14851504
public function testHttpClientWithQueryParameterKey()
14861505
{
14871506
$container = $this->createContainerFromFile('http_client_xml_key');

‎src/Symfony/Component/HttpClient/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ CHANGELOG
1010
* added `MockResponse::getRequestMethod()` and `getRequestUrl()` to allow inspecting which request has been sent
1111
* added `EventSourceHttpClient` a Server-Sent events stream implementing the [EventSource specification](https://www.w3.org/TR/eventsource/#eventsource)
1212
* added option "extra.curl" to allow setting additional curl options in `CurlHttpClient`
13+
* added `RetryableHttpClient` to automatically retry failed HTTP requests.
1314

1415
5.1.0
1516
-----

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy