Skip to content

Commit d6aa8de

Browse files
dlechdlech
committed
py/runtime: Allow multiple *args in function call.
This is a partial implementation of PEP 448 to allow unmacking multiple star args in a function or method call. This is implemented by changing the emitted bytecodes so that both positoinal args and star args are stored as positoinal args. A bitmap is added to indicate if an argument at a given position is a positoinal argument or a star arg. In the bytecodes, this new bitmap takes the place of the old star arg. It is stored as a small int, so this means only the first N arguments can be star args where N is the number of bits in a small int. The runtime is modified to interpret this new bytecode format while still trying to perform as few memory reallocations as possible. Signed-off-by: David Lechner <david@pybricks.com>
1 parent 2e5e9da commit d6aa8de

File tree

6 files changed

+101
-67
lines changed

6 files changed

+101
-67
lines changed

‎py/compile.c

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2397,17 +2397,19 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar
23972397
int n_positional = n_positional_extra;
23982398
uint n_keyword = 0;
23992399
uint star_flags = 0;
2400-
mp_parse_node_struct_t *star_args_node = NULL;
2400+
mp_uint_t star_args = 0;
24012401
for (size_t i = 0; i < n_args; i++) {
24022402
if (MP_PARSE_NODE_IS_STRUCT(args[i])) {
24032403
mp_parse_node_struct_t *pns_arg = (mp_parse_node_struct_t *)args[i];
24042404
if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_star) {
2405-
if (star_flags & MP_EMIT_STAR_FLAG_SINGLE) {
2406-
compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("can't have multiple *x"));
2405+
if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) {
2406+
compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("* arg after **"));
24072407
return;
24082408
}
24092409
star_flags |= MP_EMIT_STAR_FLAG_SINGLE;
2410-
star_args_node = pns_arg;
2410+
star_args |= 1 << i;
2411+
compile_node(comp, pns_arg->nodes[0]);
2412+
n_positional++;
24112413
} else if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_dbl_star) {
24122414
star_flags |= MP_EMIT_STAR_FLAG_DOUBLE;
24132415
// double-star args are stored as kw arg with key of None
@@ -2438,27 +2440,22 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar
24382440
}
24392441
} else {
24402442
normal_argument:
2441-
if (star_flags) {
2442-
compile_syntax_error(comp, args[i], MP_ERROR_TEXT("non-keyword arg after */**"));
2443+
if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) {
2444+
compile_syntax_error(comp, args[i], MP_ERROR_TEXT("positional arg after **"));
24432445
return;
24442446
}
24452447
if (n_keyword > 0) {
2446-
compile_syntax_error(comp, args[i], MP_ERROR_TEXT("non-keyword arg after keyword arg"));
2448+
compile_syntax_error(comp, args[i], MP_ERROR_TEXT("positional arg after keyword arg"));
24472449
return;
24482450
}
24492451
compile_node(comp, args[i]);
24502452
n_positional++;
24512453
}
24522454
}
24532455

2454-
// compile the star/double-star arguments if we had them
2455-
// if we had one but not the other then we load "null" as a place holder
24562456
if (star_flags != 0) {
2457-
if (star_args_node == NULL) {
2458-
EMIT(load_null);
2459-
} else {
2460-
compile_node(comp, star_args_node->nodes[0]);
2461-
}
2457+
// one extra object that contains the star_args map
2458+
EMIT_ARG(load_const_small_int, star_args);
24622459
// FIXME: delete this and update interpreter
24632460
EMIT(load_null);
24642461
}

‎py/runtime.c

Lines changed: 64 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -692,9 +692,9 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
692692
}
693693
uint n_args = n_args_n_kw & 0xff;
694694
uint n_kw = (n_args_n_kw >> 8) & 0xff;
695-
mp_obj_t pos_seq = args[n_args + 2 * n_kw]; // may be MP_OBJ_NULL
695+
mp_uint_t star_args = mp_obj_get_int_truncated(args[n_args + 2 * n_kw]);
696696

697-
DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, seq=%p)\n", fun, self, n_args, n_kw, args, pos_seq);
697+
DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, map=%u)\n", fun, self, n_args, n_kw, args, star_args);
698698

699699
// We need to create the following array of objects:
700700
// args[0 .. n_args] unpacked(pos_seq) args[n_args .. n_args + 2 * n_kw] unpacked(kw_dict)
@@ -705,6 +705,20 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
705705
uint args2_alloc;
706706
uint args2_len = 0;
707707

708+
// Try to get a hint for unpacked * args length
709+
uint list_len = 0;
710+
711+
if (star_args != 0) {
712+
for (uint i = 0; i < n_args; i++) {
713+
if (star_args & (1 << i)) {
714+
mp_obj_t len = mp_obj_len_maybe(args[i]);
715+
if (len != MP_OBJ_NULL) {
716+
list_len += mp_obj_get_int(len);
717+
}
718+
}
719+
}
720+
}
721+
708722
// Try to get a hint for the size of the kw_dict
709723
uint kw_dict_len = 0;
710724

@@ -718,8 +732,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
718732

719733
// Extract the pos_seq sequence to the new args array.
720734
// Note that it can be arbitrary iterator.
721-
if (pos_seq == MP_OBJ_NULL) {
722-
// no sequence
735+
if (star_args == 0) {
736+
// no star args to unpack
723737

724738
// allocate memory for the new array of args
725739
args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len);
@@ -733,60 +747,69 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
733747
// copy the fixed pos args
734748
mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
735749
args2_len += n_args;
736-
737-
} else if (mp_obj_is_type(pos_seq, &mp_type_tuple) || mp_obj_is_type(pos_seq, &mp_type_list)) {
738-
// optimise the case of a tuple and list
739-
740-
// get the items
741-
size_t len;
742-
mp_obj_t *items;
743-
mp_obj_get_array(pos_seq, &len, &items);
744-
745-
// allocate memory for the new array of args
746-
args2_alloc = 1 + n_args + len + 2 * (n_kw + kw_dict_len);
747-
args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t));
748-
749-
// copy the self
750-
if (self != MP_OBJ_NULL) {
751-
args2[args2_len++] = self;
752-
}
753-
754-
// copy the fixed and variable position args
755-
mp_seq_cat(args2 + args2_len, args, n_args, items, len, mp_obj_t);
756-
args2_len += n_args + len;
757-
758750
} else {
759-
// generic iterator
751+
// at least one star arg to unpack
760752

761753
// allocate memory for the new array of args
762-
args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len) + 3;
754+
args2_alloc = 1 + n_args + list_len + 2 * (n_kw + kw_dict_len);
763755
args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t));
764756

765757
// copy the self
766758
if (self != MP_OBJ_NULL) {
767759
args2[args2_len++] = self;
768760
}
769761

770-
// copy the fixed position args
771-
mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
772-
args2_len += n_args;
773-
774-
// extract the variable position args from the iterator
775-
mp_obj_iter_buf_t iter_buf;
776-
mp_obj_t iterable = mp_getiter(pos_seq, &iter_buf);
777-
mp_obj_t item;
778-
while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) {
779-
if (args2_len >= args2_alloc) {
780-
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), args2_alloc * 2 * sizeof(mp_obj_t));
781-
args2_alloc *= 2;
762+
for (uint i = 0; i < n_args; i++) {
763+
mp_obj_t arg = args[i];
764+
if (star_args & (1 << i)) {
765+
// star arg
766+
if (mp_obj_is_type(arg, &mp_type_tuple) || mp_obj_is_type(arg, &mp_type_list)) {
767+
// optimise the case of a tuple and list
768+
769+
// get the items
770+
size_t len;
771+
mp_obj_t *items;
772+
mp_obj_get_array(arg, &len, &items);
773+
774+
// copy the items
775+
assert(args2_len + len <= args2_alloc);
776+
mp_seq_copy(args2 + args2_len, items, len, mp_obj_t);
777+
args2_len += len;
778+
} else {
779+
// generic iterator
780+
781+
// extract the variable position args from the iterator
782+
mp_obj_iter_buf_t iter_buf;
783+
mp_obj_t iterable = mp_getiter(arg, &iter_buf);
784+
mp_obj_t item;
785+
while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) {
786+
if (args2_len >= args2_alloc) {
787+
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t),
788+
args2_alloc * 2 * sizeof(mp_obj_t));
789+
args2_alloc *= 2;
790+
}
791+
args2[args2_len++] = item;
792+
}
793+
}
794+
} else {
795+
// normal argument
796+
assert(args2_len < args2_alloc);
797+
args2[args2_len++] = arg;
782798
}
783-
args2[args2_len++] = item;
784799
}
785800
}
786801

787802
// The size of the args2 array now is the number of positional args.
788803
uint pos_args_len = args2_len;
789804

805+
// ensure there is still enough room for kw args
806+
if (args2_len + 2 * (n_kw + kw_dict_len) > args2_alloc) {
807+
uint new_alloc = args2_len + 2 * (n_kw + kw_dict_len);
808+
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t),
809+
new_alloc * sizeof(mp_obj_t));
810+
args2_alloc = new_alloc;
811+
}
812+
790813
// Copy the kw args.
791814
for (uint i = 0; i < n_kw; i++) {
792815
mp_obj_t kw_key = args[n_args + i * 2];

‎tests/basics/fun_callstar.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@
33
def foo(a, b, c):
44
print(a, b, c)
55

6+
foo(*(), 1, 2, 3)
7+
foo(*(1,), 2, 3)
8+
foo(*(1, 2), 3)
69
foo(*(1, 2, 3))
710
foo(1, *(2, 3))
811
foo(1, 2, *(3,))
912
foo(1, 2, 3, *())
13+
foo(*(1,), 2, *(3,))
14+
foo(*(1, 2), *(3,))
15+
foo(*(1,), *(2, 3))
1016

1117
# Another sequence type
1218
foo(1, 2, *[100])
@@ -29,10 +35,16 @@ def foo(self, a, b, c):
2935
print(a, b, c)
3036

3137
a = A()
38+
a.foo(*(), 1, 2, 3)
39+
a.foo(*(1,), 2, 3)
40+
a.foo(*(1, 2), 3)
3241
a.foo(*(1, 2, 3))
3342
a.foo(1, *(2, 3))
3443
a.foo(1, 2, *(3,))
3544
a.foo(1, 2, 3, *())
45+
a.foo(*(1,), 2, *(3,))
46+
a.foo(*(1, 2), *(3,))
47+
a.foo(*(1,), *(2, 3))
3648

3749
# Another sequence type
3850
a.foo(1, 2, *[100])

‎tests/basics/fun_callstardblstar.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ def f(a, b, c, d):
66
f(*(1, 2), **{'c':3, 'd':4})
77
f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
88

9+
try:
10+
eval("f(**{'a': 1}, *(2, 3, 4))")
11+
except SyntaxError:
12+
print("SyntaxError")
13+
914
# test calling a method with *tuple and **dict
1015

1116
class A:
@@ -15,3 +20,8 @@ def f(self, a, b, c, d):
1520
a = A()
1621
a.f(*(1, 2), **{'c':3, 'd':4})
1722
a.f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
23+
24+
try:
25+
eval("a.f(**{'a': 1}, *(2, 3, 4))")
26+
except SyntaxError:
27+
print("SyntaxError")

‎tests/basics/python34.py

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,23 @@
66
print("SKIP")
77
raise SystemExit
88

9-
# from basics/fun_kwvarargs.py
10-
# test evaluation order of arguments (in 3.4 it's backwards, 3.5 it's fixed)
11-
def f4(*vargs, **kwargs):
12-
print(vargs, kwargs)
9+
1310
def print_ret(x):
1411
print(x)
1512
return x
16-
f4(*print_ret(['a', 'b']), kw_arg=print_ret(None))
1713

1814
# test evaluation order of dictionary key/value pair (in 3.4 it's backwards)
1915
{print_ret(1):print_ret(2)}
2016

17+
2118
# from basics/syntaxerror.py
2219
def test_syntax(code):
2320
try:
2421
exec(code)
2522
except SyntaxError:
2623
print("SyntaxError")
27-
test_syntax("f(*a, *b)") # can't have multiple * (in 3.5 we can)
28-
test_syntax("f(*a, b)") # can't have positional after *
24+
25+
2926
test_syntax("f(**a, b)") # can't have positional after **
3027
test_syntax("() = []") # can't assign to empty tuple (in 3.6 we can)
3128
test_syntax("del ()") # can't delete empty tuple (in 3.6 we can)

‎tests/basics/python34.py.exp

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
1-
None
2-
['a', 'b']
3-
('a', 'b') {'kw_arg': None}
41
2
52
1
63
SyntaxError
74
SyntaxError
85
SyntaxError
9-
SyntaxError
10-
SyntaxError
116
3.4
127
3 4
138
IndexError('foo',)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy