bug #61242 [Console] [Table] Fix invalid UTF-8 due to text wrapping (schlndh)

nicolas-grekas · nicolas-grekas · commit 1b7d5c8d1e12 · 2025-07-30T11:35:42.000+02:00
This PR was merged into the 6.4 branch. Discussion ---------- [Console] [Table] Fix invalid UTF-8 due to text wrapping | Q | A | ------------- | --- | Branch? | 6.4 | Bug fix? | yes | New feature? | no | Deprecations? | no | Issues | Fix #58286 | License | MIT I changed the `OutputFormatter` to use string methods from `Helper` to avoid splitting multi-byte UTF-8 characters and causing the "Invalid UTF-8" exception as a result. Unfortunately, it turns out that `Helper::length` and `Helper::substr` are not mutually compatible for UTF-8, because `length` returns the length in graphemes, while `substr` worked with characters. I decided to fix that, which is technically a BC break (hopefully not too severe). I considered making a private fixed copy of the method, but that just seemed weird. Commits ------- 218ca62 [Console][Table] Fix invalid UTF-8 due to text wrapping
diff --git a/src/Symfony/Component/Console/Formatter/OutputFormatter.php b/src/Symfony/Component/Console/Formatter/OutputFormatter.php
@@ -12,6 +12,7 @@
 namespace Symfony\Component\Console\Formatter;
 
 use Symfony\Component\Console\Exception\InvalidArgumentException;
+use Symfony\Component\Console\Helper\Helper;
 
 use function Symfony\Component\String\b;
 
@@ -146,9 +147,11 @@ public function formatAndWrap(?string $message, int $width)
                 continue;
             }
 
+            // convert byte position to character position.
+            $pos = Helper::length(substr($message, 0, $pos));
             // add the text up to the next tag
-            $output .= $this->applyCurrentStyle(substr($message, $offset, $pos - $offset), $output, $width, $currentLineLength);
-            $offset = $pos + \strlen($text);
+            $output .= $this->applyCurrentStyle(Helper::substr($message, $offset, $pos - $offset), $output, $width, $currentLineLength);
+            $offset = $pos + Helper::length($text);
 
             // opening tag?
             if ($open = '/' !== $text[1]) {
@@ -169,7 +172,7 @@ public function formatAndWrap(?string $message, int $width)
             }
         }
 
-        $output .= $this->applyCurrentStyle(substr($message, $offset), $output, $width, $currentLineLength);
+        $output .= $this->applyCurrentStyle(Helper::substr($message, $offset), $output, $width, $currentLineLength);
 
         return strtr($output, ["\0" => '\\', '\\<' => '<', '\\>' => '>']);
     }
@@ -236,8 +239,8 @@ private function applyCurrentStyle(string $text, string $current, int $width, in
         }
 
         if ($currentLineLength) {
-            $prefix = substr($text, 0, $i = $width - $currentLineLength)."\n";
-            $text = substr($text, $i);
+            $prefix = Helper::substr($text, 0, $i = $width - $currentLineLength)."\n";
+            $text = Helper::substr($text, $i);
         } else {
             $prefix = '';
         }
@@ -253,7 +256,7 @@ private function applyCurrentStyle(string $text, string $current, int $width, in
         $lines = explode("\n", $text);
 
         foreach ($lines as $line) {
-            $currentLineLength += \strlen($line);
+            $currentLineLength += Helper::length($line);
             if ($width <= $currentLineLength) {
                 $currentLineLength = 0;
             }
diff --git a/src/Symfony/Component/Console/Helper/Helper.php b/src/Symfony/Component/Console/Helper/Helper.php
@@ -86,6 +86,10 @@ public static function substr(?string $string, int $from, ?int $length = null):
     {
         $string ??= '';
 
+        if (preg_match('//u', $string)) {
+            return (new UnicodeString($string))->slice($from, $length);
+        }
+
         if (false === $encoding = mb_detect_encoding($string, null, true)) {
             return substr($string, $from, $length);
         }
diff --git a/src/Symfony/Component/Console/Tests/Formatter/OutputFormatterTest.php b/src/Symfony/Component/Console/Tests/Formatter/OutputFormatterTest.php
@@ -365,6 +365,8 @@ public function testFormatAndWrap()
         $this->assertSame("Lore\nm \e[37;41mip\e[39;49m\n\e[37;41msum\e[39;49m \ndolo\nr \e[32msi\e[39m\n\e[32mt\e[39m am\net", $formatter->formatAndWrap('Lorem <error>ipsum</error> dolor <info>sit</info> amet', 4));
         $this->assertSame("Lorem \e[37;41mip\e[39;49m\n\e[37;41msum\e[39;49m dolo\nr \e[32msit\e[39m am\net", $formatter->formatAndWrap('Lorem <error>ipsum</error> dolor <info>sit</info> amet', 8));
         $this->assertSame("Lorem \e[37;41mipsum\e[39;49m dolor \e[32m\e[39m\n\e[32msit\e[39m, \e[37;41mamet\e[39;49m et \e[32mlauda\e[39m\n\e[32mntium\e[39m architecto", $formatter->formatAndWrap('Lorem <error>ipsum</error> dolor <info>sit</info>, <error>amet</error> et <info>laudantium</info> architecto', 18));
+        $this->assertSame("\e[37;41mnon-empty-array\e[39;49m\e[37;41m<mixed, mixed>\e[39;49m given.\n🪪\n  argument.type", $formatter->formatAndWrap("<error>non-empty-array<mixed, mixed></error> given.\n🪪  argument.type", 38));
+        $this->assertSame("Usuário <strong>{{user_name}}</strong> não é válid\no.", $formatter->formatAndWrap('Usuário <strong>{{user_name}}</strong> não é válido.', 50));
 
         $formatter = new OutputFormatter();
 
@@ -376,6 +378,8 @@ public function testFormatAndWrap()
         $this->assertSame("Â rèälly\nlöng tîtlè\nthät cöüld\nnèêd\nmúltîplê\nlínès", $formatter->formatAndWrap('Â rèälly löng tîtlè thät cöüld nèêd múltîplê línès', 10));
         $this->assertSame("Â rèälly\nlöng tîtlè\nthät cöüld\nnèêd\nmúltîplê\n línès", $formatter->formatAndWrap("Â rèälly löng tîtlè thät cöüld nèêd múltîplê\n línès", 10));
         $this->assertSame('', $formatter->formatAndWrap(null, 5));
+        $this->assertSame("non-empty-array<mixed, mixed> given.\n🪪\n  argument.type", $formatter->formatAndWrap("<error>non-empty-array<mixed, mixed></error> given.\n🪪  argument.type", 38));
+        $this->assertSame("Usuário <strong>{{user_name}}</strong> não é válid\no.", $formatter->formatAndWrap('Usuário <strong>{{user_name}}</strong> não é válido.', 50));
     }
 }
 

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@`
`12`	`12`	`namespace Symfony\Component\Console\Formatter;`
`13`	`13`
`14`	`14`	`use Symfony\Component\Console\Exception\InvalidArgumentException;`
	`15`	`+use Symfony\Component\Console\Helper\Helper;`
`15`	`16`
`16`	`17`	`use function Symfony\Component\String\b;`
`17`	`18`
`@@ -146,9 +147,11 @@ public function formatAndWrap(?string $message, int $width)`
`146`	`147`	`continue;`
`147`	`148`	`}`
`148`	`149`
	`150`	`+ // convert byte position to character position.`
	`151`	`+ $pos = Helper::length(substr($message, 0, $pos));`
`149`	`152`	`// add the text up to the next tag`
`150`		`- $output .= $this->applyCurrentStyle(substr($message, $offset, $pos - $offset), $output, $width, $currentLineLength);`
`151`		`- $offset = $pos + \strlen($text);`
	`153`	`+ $output .= $this->applyCurrentStyle(Helper::substr($message, $offset, $pos - $offset), $output, $width, $currentLineLength);`
	`154`	`+ $offset = $pos + Helper::length($text);`
`152`	`155`
`153`	`156`	`// opening tag?`
`154`	`157`	`if ($open = '/' !== $text[1]) {`
`@@ -169,7 +172,7 @@ public function formatAndWrap(?string $message, int $width)`
`169`	`172`	`}`
`170`	`173`	`}`
`171`	`174`
`172`		`- $output .= $this->applyCurrentStyle(substr($message, $offset), $output, $width, $currentLineLength);`
	`175`	`+ $output .= $this->applyCurrentStyle(Helper::substr($message, $offset), $output, $width, $currentLineLength);`
`173`	`176`
`174`	`177`	`return strtr($output, ["\0" => '\\', '\\<' => '<', '\\>' => '>']);`
`175`	`178`	`}`
`@@ -236,8 +239,8 @@ private function applyCurrentStyle(string $text, string $current, int $width, in`
`236`	`239`	`}`
`237`	`240`
`238`	`241`	`if ($currentLineLength) {`
`239`		`- $prefix = substr($text, 0, $i = $width - $currentLineLength)."\n";`
`240`		`- $text = substr($text, $i);`
	`242`	`+ $prefix = Helper::substr($text, 0, $i = $width - $currentLineLength)."\n";`
	`243`	`+ $text = Helper::substr($text, $i);`
`241`	`244`	`} else {`
`242`	`245`	`$prefix = '';`
`243`	`246`	`}`
`@@ -253,7 +256,7 @@ private function applyCurrentStyle(string $text, string $current, int $width, in`
`253`	`256`	`$lines = explode("\n", $text);`
`254`	`257`
`255`	`258`	`foreach ($lines as $line) {`
`256`		`- $currentLineLength += \strlen($line);`
	`259`	`+ $currentLineLength += Helper::length($line);`
`257`	`260`	`if ($width <= $currentLineLength) {`
`258`	`261`	`$currentLineLength = 0;`
`259`	`262`	`}`
Original file line number	Diff line number	Diff line change
`@@ -86,6 +86,10 @@ public static function substr(?string $string, int $from, ?int $length = null):`
`86`	`86`	`{`
`87`	`87`	`$string ??= '';`
`88`	`88`
	`89`	`+ if (preg_match('//u', $string)) {`
	`90`	`+ return (new UnicodeString($string))->slice($from, $length);`
	`91`	`+ }`
	`92`	`+`
`89`	`93`	`if (false === $encoding = mb_detect_encoding($string, null, true)) {`
`90`	`94`	`return substr($string, $from, $length);`
`91`	`95`	`}`
Original file line number	Diff line number	Diff line change
`@@ -365,6 +365,8 @@ public function testFormatAndWrap()`
`365`	`365`	`$this->assertSame("Lore\nm \e[37;41mip\e[39;49m\n\e[37;41msum\e[39;49m \ndolo\nr \e[32msi\e[39m\n\e[32mt\e[39m am\net", $formatter->formatAndWrap('Lorem <error>ipsum</error> dolor <info>sit</info> amet', 4));`
`366`	`366`	`$this->assertSame("Lorem \e[37;41mip\e[39;49m\n\e[37;41msum\e[39;49m dolo\nr \e[32msit\e[39m am\net", $formatter->formatAndWrap('Lorem <error>ipsum</error> dolor <info>sit</info> amet', 8));`
`367`	`367`	`$this->assertSame("Lorem \e[37;41mipsum\e[39;49m dolor \e[32m\e[39m\n\e[32msit\e[39m, \e[37;41mamet\e[39;49m et \e[32mlauda\e[39m\n\e[32mntium\e[39m architecto", $formatter->formatAndWrap('Lorem <error>ipsum</error> dolor <info>sit</info>, <error>amet</error> et <info>laudantium</info> architecto', 18));`
	`368`	`+ $this->assertSame("\e[37;41mnon-empty-array\e[39;49m\e[37;41m<mixed, mixed>\e[39;49m given.\n🪪\n argument.type", $formatter->formatAndWrap("<error>non-empty-array<mixed, mixed></error> given.\n🪪 argument.type", 38));`
	`369`	`+ $this->assertSame("Usuário <strong>{{user_name}}</strong> não é válid\no.", $formatter->formatAndWrap('Usuário <strong>{{user_name}}</strong> não é válido.', 50));`
`368`	`370`
`369`	`371`	`$formatter = new OutputFormatter();`
`370`	`372`
`@@ -376,6 +378,8 @@ public function testFormatAndWrap()`
`376`	`378`	`$this->assertSame("Â rèälly\nlöng tîtlè\nthät cöüld\nnèêd\nmúltîplê\nlínès", $formatter->formatAndWrap('Â rèälly löng tîtlè thät cöüld nèêd múltîplê línès', 10));`
`377`	`379`	`$this->assertSame("Â rèälly\nlöng tîtlè\nthät cöüld\nnèêd\nmúltîplê\n línès", $formatter->formatAndWrap("Â rèälly löng tîtlè thät cöüld nèêd múltîplê\n línès", 10));`
`378`	`380`	`$this->assertSame('', $formatter->formatAndWrap(null, 5));`
	`381`	`+ $this->assertSame("non-empty-array<mixed, mixed> given.\n🪪\n argument.type", $formatter->formatAndWrap("<error>non-empty-array<mixed, mixed></error> given.\n🪪 argument.type", 38));`
	`382`	`+ $this->assertSame("Usuário <strong>{{user_name}}</strong> não é válid\no.", $formatter->formatAndWrap('Usuário <strong>{{user_name}}</strong> não é válido.', 50));`
`379`	`383`	`}`
`380`	`384`	`}`
`381`	`385`