Skip to content

Commit ed0ef34

Browse files
pablogsalpablogsal
committed
Initial implementation of PEP 701
1 parent 094cf39 commit ed0ef34

File tree

16 files changed

+3779
-2018
lines changed

16 files changed

+3779
-2018
lines changed

‎Doc/library/token-list.inc

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Grammar/Tokens

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,17 @@ ATEQUAL '@='
5353
RARROW '->'
5454
ELLIPSIS '...'
5555
COLONEQUAL ':='
56+
EXCLAMATION '!'
5657

5758
OP
5859
AWAIT
5960
ASYNC
6061
TYPE_IGNORE
6162
TYPE_COMMENT
6263
SOFT_KEYWORD
64+
FSTRING_START
65+
FSTRING_MIDDLE
66+
FSTRING_END
6367
ERRORTOKEN
6468

6569
# These aren't used by the C tokenizer but are needed for tokenize.py

‎Grammar/python.gram

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -807,7 +807,8 @@ atom[expr_ty]:
807807
| 'True' { _PyAST_Constant(Py_True, NULL, EXTRA) }
808808
| 'False' { _PyAST_Constant(Py_False, NULL, EXTRA) }
809809
| 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) }
810-
| &STRING strings
810+
# | gstring
811+
| &(STRING|FSTRING_START) strings
811812
| NUMBER
812813
| &'(' (tuple | group | genexp)
813814
| &'[' (list | listcomp)
@@ -877,7 +878,29 @@ lambda_param[arg_ty]: a=NAME { _PyAST_arg(a->v.Name.id, NULL, NULL, EXTRA) }
877878
# LITERALS
878879
# ========
879880

880-
strings[expr_ty] (memo): a=STRING+ { _PyPegen_concatenate_strings(p, a) }
881+
gstring_middle[expr_ty]:
882+
| gstring_replacement_field
883+
| t=FSTRING_MIDDLE { _PyPegen_constant_from_token2(p, t) }
884+
# There are some shenanigans with the gstring_format_spec: Don't try to put it in its own rule
885+
# or otherwise it will try to parse the first token with the regular tokenizer mode (due to the EXTRA).
886+
# TODO: (Ideally we need a way similar to 'memo' so the parser can set the tokenize mode on fstring/normal)
887+
gstring_replacement_field[expr_ty]:
888+
| expr_start='{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[
889+
conv_token="!" conv=NAME { _PyPegen_check_fstring_conversion(p, conv_token, conv) ? NULL : conv }
890+
] format=[
891+
':' spec=gstring_format_spec* { spec ? _PyAST_JoinedStr((asdl_expr_seq*)spec, EXTRA) : NULL }
892+
] &&'}' {
893+
_PyPegen_formatted_value(p, a, debug_expr, conversion, format, EXTRA)
894+
}
895+
gstring_format_spec[expr_ty]:
896+
| t=FSTRING_MIDDLE { _PyPegen_constant_from_token2(p, t) }
897+
| gstring_replacement_field
898+
gstring[expr_ty]:
899+
| a=FSTRING_START b=gstring_middle* c=FSTRING_END { deal_with_gstring2(p, a, (asdl_expr_seq*)b, c) }
900+
901+
string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_token(p, s) }
902+
strings[expr_ty] (memo): a[asdl_expr_seq*]=(gstring|string)+ { _PyPegen_concatenate_strings2(p, a, EXTRA) }
903+
# strings[expr_ty] (memo): a=STRING+ { _PyPegen_concatenate_strings(p, a) }
881904

882905
list[expr_ty]:
883906
| '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }

‎Include/internal/pycore_token.h

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -67,14 +67,18 @@ extern "C" {
6767
#define RARROW 51
6868
#define ELLIPSIS 52
6969
#define COLONEQUAL 53
70-
#define OP 54
71-
#define AWAIT 55
72-
#define ASYNC 56
73-
#define TYPE_IGNORE 57
74-
#define TYPE_COMMENT 58
75-
#define SOFT_KEYWORD 59
76-
#define ERRORTOKEN 60
77-
#define N_TOKENS 64
70+
#define EXCLAMATION 54
71+
#define OP 55
72+
#define AWAIT 56
73+
#define ASYNC 57
74+
#define TYPE_IGNORE 58
75+
#define TYPE_COMMENT 59
76+
#define SOFT_KEYWORD 60
77+
#define FSTRING_START 61
78+
#define FSTRING_MIDDLE 62
79+
#define FSTRING_END 63
80+
#define ERRORTOKEN 64
81+
#define N_TOKENS 68
7882
#define NT_OFFSET 256
7983

8084
/* Special definitions for cooperation with parser */

‎Lib/test/test_fstring.py

Lines changed: 76 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -329,13 +329,13 @@ def test_ast_line_numbers_multiline_fstring(self):
329329
self.assertEqual(t.body[1].lineno, 3)
330330
self.assertEqual(t.body[1].value.lineno, 3)
331331
self.assertEqual(t.body[1].value.values[0].lineno, 3)
332-
self.assertEqual(t.body[1].value.values[1].lineno, 3)
333-
self.assertEqual(t.body[1].value.values[2].lineno, 3)
332+
self.assertEqual(t.body[1].value.values[1].lineno, 4)
333+
self.assertEqual(t.body[1].value.values[2].lineno, 6)
334334
self.assertEqual(t.body[1].col_offset, 0)
335335
self.assertEqual(t.body[1].value.col_offset, 0)
336-
self.assertEqual(t.body[1].value.values[0].col_offset, 0)
337-
self.assertEqual(t.body[1].value.values[1].col_offset, 0)
338-
self.assertEqual(t.body[1].value.values[2].col_offset, 0)
336+
self.assertEqual(t.body[1].value.values[0].col_offset, 4)
337+
self.assertEqual(t.body[1].value.values[1].col_offset, 2)
338+
self.assertEqual(t.body[1].value.values[2].col_offset, 11)
339339
# NOTE: the following lineno information and col_offset is correct for
340340
# expressions within FormattedValues.
341341
binop = t.body[1].value.values[1].value
@@ -366,13 +366,13 @@ def test_ast_line_numbers_multiline_fstring(self):
366366
self.assertEqual(t.body[0].lineno, 2)
367367
self.assertEqual(t.body[0].value.lineno, 2)
368368
self.assertEqual(t.body[0].value.values[0].lineno, 2)
369-
self.assertEqual(t.body[0].value.values[1].lineno, 2)
370-
self.assertEqual(t.body[0].value.values[2].lineno, 2)
369+
self.assertEqual(t.body[0].value.values[1].lineno, 3)
370+
self.assertEqual(t.body[0].value.values[2].lineno, 3)
371371
self.assertEqual(t.body[0].col_offset, 0)
372372
self.assertEqual(t.body[0].value.col_offset, 4)
373-
self.assertEqual(t.body[0].value.values[0].col_offset, 4)
374-
self.assertEqual(t.body[0].value.values[1].col_offset, 4)
375-
self.assertEqual(t.body[0].value.values[2].col_offset, 4)
373+
self.assertEqual(t.body[0].value.values[0].col_offset, 8)
374+
self.assertEqual(t.body[0].value.values[1].col_offset, 10)
375+
self.assertEqual(t.body[0].value.values[2].col_offset, 17)
376376
# Check {blech}
377377
self.assertEqual(t.body[0].value.values[1].value.lineno, 3)
378378
self.assertEqual(t.body[0].value.values[1].value.end_lineno, 3)
@@ -387,6 +387,20 @@ def test_ast_line_numbers_with_parentheses(self):
387387
t = ast.parse(expr)
388388
self.assertEqual(type(t), ast.Module)
389389
self.assertEqual(len(t.body), 1)
390+
# check the joinedstr location
391+
joinedstr = t.body[0].value
392+
self.assertEqual(type(joinedstr), ast.JoinedStr)
393+
self.assertEqual(joinedstr.lineno, 3)
394+
self.assertEqual(joinedstr.end_lineno, 3)
395+
self.assertEqual(joinedstr.col_offset, 4)
396+
self.assertEqual(joinedstr.end_col_offset, 17)
397+
# check the formatted value location
398+
fv = t.body[0].value.values[1]
399+
self.assertEqual(type(fv), ast.FormattedValue)
400+
self.assertEqual(fv.lineno, 3)
401+
self.assertEqual(fv.end_lineno, 3)
402+
self.assertEqual(fv.col_offset, 7)
403+
self.assertEqual(fv.end_col_offset, 16)
390404
# check the test(t) location
391405
call = t.body[0].value.values[1].value
392406
self.assertEqual(type(call), ast.Call)
@@ -415,9 +429,9 @@ def test_ast_line_numbers_with_parentheses(self):
415429
# check the first wat
416430
self.assertEqual(type(wat1), ast.Constant)
417431
self.assertEqual(wat1.lineno, 4)
418-
self.assertEqual(wat1.end_lineno, 6)
419-
self.assertEqual(wat1.col_offset, 12)
420-
self.assertEqual(wat1.end_col_offset, 18)
432+
self.assertEqual(wat1.end_lineno, 5)
433+
self.assertEqual(wat1.col_offset, 14)
434+
self.assertEqual(wat1.end_col_offset, 26)
421435
# check the call
422436
call = middle.value
423437
self.assertEqual(type(call), ast.Call)
@@ -427,9 +441,9 @@ def test_ast_line_numbers_with_parentheses(self):
427441
self.assertEqual(call.end_col_offset, 31)
428442
# check the second wat
429443
self.assertEqual(type(wat2), ast.Constant)
430-
self.assertEqual(wat2.lineno, 4)
444+
self.assertEqual(wat2.lineno, 5)
431445
self.assertEqual(wat2.end_lineno, 6)
432-
self.assertEqual(wat2.col_offset, 12)
446+
self.assertEqual(wat2.col_offset, 32)
433447
self.assertEqual(wat2.end_col_offset, 18)
434448

435449
def test_docstring(self):
@@ -618,6 +632,7 @@ def test_format_specifier_expressions(self):
618632
self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa')
619633
self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', ' -0xa')
620634
self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa')
635+
self.assertEqual(f'result: {value:{width:{0}}.{precision:1}}', 'result: 12.35')
621636

622637
self.assertAllRaise(SyntaxError,
623638
"""f-string: invalid conversion character 'r{"': """
@@ -632,11 +647,6 @@ def test_format_specifier_expressions(self):
632647
"f'{4:{/5}}'",
633648
])
634649

635-
self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply",
636-
[# Can't nest format specifiers.
637-
"f'result: {value:{width:{0}}.{precision:1}}'",
638-
])
639-
640650
self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character',
641651
[# No expansion inside conversion or for
642652
# the : or ! itself.
@@ -848,6 +858,50 @@ def test_lambda(self):
848858
["f'{lambda x:x}'",
849859
])
850860

861+
def test_valid_prefixes(self):
862+
self.assertEqual(F'{1}', "1")
863+
self.assertEqual(FR'{2}', "2")
864+
self.assertEqual(fR'{3}', "3")
865+
866+
def test_roundtrip_raw_quotes(self):
867+
self.assertEqual(fr"\'", "\\'")
868+
self.assertEqual(fr'\"', '\\"')
869+
self.assertEqual(fr'\"\'', '\\"\\\'')
870+
self.assertEqual(fr'\'\"', '\\\'\\"')
871+
self.assertEqual(fr'\"\'\"', '\\"\\\'\\"')
872+
self.assertEqual(fr'\'\"\'', '\\\'\\"\\\'')
873+
self.assertEqual(fr'\"\'\"\'', '\\"\\\'\\"\\\'')
874+
875+
def test_fstring_backslash_before_double_bracket(self):
876+
self.assertEqual(f'\{{\}}', '\\{\\}')
877+
self.assertEqual(f'\{{', '\\{')
878+
self.assertEqual(f'\{{{1+1}', '\\{2')
879+
self.assertEqual(f'\}}{1+1}', '\\}2')
880+
self.assertEqual(f'{1+1}\}}', '2\\}')
881+
self.assertEqual(fr'\{{\}}', '\\{\\}')
882+
self.assertEqual(fr'\{{', '\\{')
883+
self.assertEqual(fr'\{{{1+1}', '\\{2')
884+
self.assertEqual(fr'\}}{1+1}', '\\}2')
885+
self.assertEqual(fr'{1+1}\}}', '2\\}')
886+
887+
def test_fstring_backslash_prefix_raw(self):
888+
self.assertEqual(f'\\', '\\')
889+
self.assertEqual(f'\\\\', '\\\\')
890+
self.assertEqual(fr'\\', r'\\')
891+
self.assertEqual(fr'\\\\', r'\\\\')
892+
self.assertEqual(rf'\\', r'\\')
893+
self.assertEqual(rf'\\\\', r'\\\\')
894+
self.assertEqual(Rf'\\', R'\\')
895+
self.assertEqual(Rf'\\\\', R'\\\\')
896+
self.assertEqual(fR'\\', R'\\')
897+
self.assertEqual(fR'\\\\', R'\\\\')
898+
self.assertEqual(FR'\\', R'\\')
899+
self.assertEqual(FR'\\\\', R'\\\\')
900+
901+
def test_fstring_format_spec_greedy_matching(self):
902+
self.assertEqual(f"{1:}}}", "1}")
903+
self.assertEqual(f"{1:>3{5}}}}", " 1}")
904+
851905
def test_yield(self):
852906
# Not terribly useful, but make sure the yield turns
853907
# a function into a generator
@@ -1314,6 +1368,7 @@ def __repr__(self):
13141368
self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y')
13151369
self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y')
13161370
self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y')
1371+
self.assertEqual(f"sadsd {1 + 1 = :{1 + 1:1d}f}", "sadsd 1 + 1 = 2.000000")
13171372

13181373
# These next lines contains tabs. Backslash escapes don't
13191374
# work in f-strings.
@@ -1324,6 +1379,7 @@ def __repr__(self):
13241379
#self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y')
13251380
#self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y')
13261381

1382+
13271383
def test_walrus(self):
13281384
x = 20
13291385
# This isn't an assignment expression, it's 'x', with a format

‎Lib/token.py

Lines changed: 16 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy