py/lexer: Support raw f-strings.

dpgeorge · dpgeorge · commit 3c8089d1b106 · 2024-06-06T17:34:28.000+10:00
Support for raw str/bytes already exists, and extending that to raw
f-strings is easy.  It also reduces code size because it eliminates an
error message.

Signed-off-by: Damien George &lt;damien@micropython.org&gt;
diff --git a/py/lexer.c b/py/lexer.c
@@ -661,21 +661,19 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
                 }
                 #if MICROPY_PY_FSTRINGS
                 if (is_char_following(lex, 'f')) {
-                    // raw-f-strings unsupported, immediately return (invalid) token.
-                    lex->tok_kind = MP_TOKEN_FSTRING_RAW;
-                    break;
+                    is_fstring = true;
+                    n_char = 2;
                 }
                 #endif
             }
             #if MICROPY_PY_FSTRINGS
             else if (is_char(lex, 'f')) {
+                is_fstring = true;
+                n_char = 1;
                 if (is_char_following(lex, 'r')) {
-                    // raw-f-strings unsupported, immediately return (invalid) token.
-                    lex->tok_kind = MP_TOKEN_FSTRING_RAW;
-                    break;
+                    is_raw = true;
+                    n_char = 2;
                 }
-                n_char = 1;
-                is_fstring = true;
             }
             #endif
 
diff --git a/py/lexer.h b/py/lexer.h
@@ -46,7 +46,6 @@ typedef enum _mp_token_kind_t {
     MP_TOKEN_LONELY_STRING_OPEN,
     #if MICROPY_PY_FSTRINGS
     MP_TOKEN_MALFORMED_FSTRING,
-    MP_TOKEN_FSTRING_RAW,
     #endif
 
     MP_TOKEN_NEWLINE,
diff --git a/py/parse.c b/py/parse.c
@@ -1351,9 +1351,6 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
         } else if (lex->tok_kind == MP_TOKEN_MALFORMED_FSTRING) {
             exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
                 MP_ERROR_TEXT("malformed f-string"));
-        } else if (lex->tok_kind == MP_TOKEN_FSTRING_RAW) {
-            exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
-                MP_ERROR_TEXT("raw f-strings are not supported"));
         #endif
         } else {
             exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
diff --git a/tests/basics/string_fstring.py b/tests/basics/string_fstring.py
@@ -75,3 +75,7 @@ def foo(a, b):
     f"cd---------------------------------"
     f"e{y}f---------------------------------"
 )
+
+# Raw f-strings.
+print(rf"\r\a\w {'f'} \s\t\r\i\n\g")
+print(fr"\r{x}")
diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp
@@ -1,6 +1,6 @@
 ----------------
 [   1] file_input_2(1) (n=10)
-         tok(6)
+         tok(5)
 [   4]   \(rule\|for_stmt\)(22) (n=4)
            id(i)
 [   4]     \(rule\|atom_paren\)(45) (n=1)
@@ -9,7 +9,7 @@
            NULL
 [   6]   \(rule\|expr_stmt\)(5) (n=2)
            id(a)
-           tok(16)
+           tok(15)
 [   7]   \(rule\|expr_stmt\)(5) (n=2)
            id(b)
            str(str)
diff --git a/tests/cpydiff/core_fstring_raw.py b/tests/cpydiff/core_fstring_raw.py

Original file line number	Diff line number	Diff line change
`@@ -661,21 +661,19 @@ void mp_lexer_to_next(mp_lexer_t *lex) {`
`661`	`661`	`}`
`662`	`662`	`#if MICROPY_PY_FSTRINGS`
`663`	`663`	`if (is_char_following(lex, 'f')) {`
`664`		`- // raw-f-strings unsupported, immediately return (invalid) token.`
`665`		`- lex->tok_kind = MP_TOKEN_FSTRING_RAW;`
`666`		`- break;`
	`664`	`+ is_fstring = true;`
	`665`	`+ n_char = 2;`
`667`	`666`	`}`
`668`	`667`	`#endif`
`669`	`668`	`}`
`670`	`669`	`#if MICROPY_PY_FSTRINGS`
`671`	`670`	`else if (is_char(lex, 'f')) {`
	`671`	`+ is_fstring = true;`
	`672`	`+ n_char = 1;`
`672`	`673`	`if (is_char_following(lex, 'r')) {`
`673`		`- // raw-f-strings unsupported, immediately return (invalid) token.`
`674`		`- lex->tok_kind = MP_TOKEN_FSTRING_RAW;`
`675`		`- break;`
	`674`	`+ is_raw = true;`
	`675`	`+ n_char = 2;`
`676`	`676`	`}`
`677`		`- n_char = 1;`
`678`		`- is_fstring = true;`
`679`	`677`	`}`
`680`	`678`	`#endif`
`681`	`679`
Original file line number	Diff line number	Diff line change
`@@ -75,3 +75,7 @@ def foo(a, b):`
`75`	`75`	`f"cd---------------------------------"`
`76`	`76`	`f"e{y}f---------------------------------"`
`77`	`77`	`)`
	`78`	`+`
	`79`	`+# Raw f-strings.`
	`80`	`+print(rf"\r\a\w {'f'} \s\t\r\i\n\g")`
	`81`	`+print(fr"\r{x}")`