Skip to content

Commit 129849f

Browse files
dlechdlech
committed
py/runtime: Allow multiple *args in function call
This is a partial implementation of PEP 448 to allow unmacking multiple star args in a function or method call. This is implemented by changing the emitted bytecodes so that both positoinal args and star args are stored as positoinal args. A bitmap is added to indicate if an argument at a given position is a positoinal argument or a star arg. In the bytecodes, this new bitmap takes the place of the old star arg. It is stored as a small int, so this means only the first N arguments can be star args where N is the number of bits in a small int. The runtime is modified to interpret this new bytecode format while still trying to perform as few memory reallocations as possible.
1 parent 67060ab commit 129849f

File tree

7 files changed

+105
-69
lines changed

7 files changed

+105
-69
lines changed

‎py/compile.c

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2334,17 +2334,19 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar
23342334
int n_positional = n_positional_extra;
23352335
uint n_keyword = 0;
23362336
uint star_flags = 0;
2337-
mp_parse_node_struct_t *star_args_node = NULL;
2337+
mp_uint_t star_args = 0;
23382338
for (int i = 0; i < n_args; i++) {
23392339
if (MP_PARSE_NODE_IS_STRUCT(args[i])) {
23402340
mp_parse_node_struct_t *pns_arg = (mp_parse_node_struct_t *)args[i];
23412341
if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_star) {
2342-
if (star_flags & MP_EMIT_STAR_FLAG_SINGLE) {
2343-
compile_syntax_error(comp, (mp_parse_node_t)pns_arg, "can't have multiple *x");
2342+
if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) {
2343+
compile_syntax_error(comp, (mp_parse_node_t)pns_arg, "* arg after **");
23442344
return;
23452345
}
23462346
star_flags |= MP_EMIT_STAR_FLAG_SINGLE;
2347-
star_args_node = pns_arg;
2347+
star_args |= 1 << i;
2348+
compile_node(comp, pns_arg->nodes[0]);
2349+
n_positional++;
23482350
} else if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_dbl_star) {
23492351
star_flags |= MP_EMIT_STAR_FLAG_DOUBLE;
23502352
// double-star args are stored as kw arg with key of None
@@ -2369,27 +2371,22 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar
23692371
}
23702372
} else {
23712373
normal_argument:
2372-
if (star_flags) {
2373-
compile_syntax_error(comp, args[i], "non-keyword arg after */**");
2374+
if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) {
2375+
compile_syntax_error(comp, args[i], "positional arg after **");
23742376
return;
23752377
}
23762378
if (n_keyword > 0) {
2377-
compile_syntax_error(comp, args[i], "non-keyword arg after keyword arg");
2379+
compile_syntax_error(comp, args[i], "positional arg after keyword arg");
23782380
return;
23792381
}
23802382
compile_node(comp, args[i]);
23812383
n_positional++;
23822384
}
23832385
}
23842386

2385-
// compile the star/double-star arguments if we had them
2386-
// if we had one but not the other then we load "null" as a place holder
23872387
if (star_flags != 0) {
2388-
if (star_args_node == NULL) {
2389-
EMIT(load_null);
2390-
} else {
2391-
compile_node(comp, star_args_node->nodes[0]);
2392-
}
2388+
// one extra object that contains the star_args map
2389+
EMIT_ARG(load_const_small_int, star_args);
23932390
// FIXME: delete this and update interpreter
23942391
EMIT(load_null);
23952392
}

‎py/runtime.c

Lines changed: 64 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -674,9 +674,9 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
674674
}
675675
uint n_args = n_args_n_kw & 0xff;
676676
uint n_kw = (n_args_n_kw >> 8) & 0xff;
677-
mp_obj_t pos_seq = args[n_args + 2 * n_kw]; // may be MP_OBJ_NULL
677+
mp_uint_t star_args = mp_obj_get_int_truncated(args[n_args + 2 * n_kw]);
678678

679-
DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, seq=%p)\n", fun, self, n_args, n_kw, args, pos_seq);
679+
DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, map=%u)\n", fun, self, n_args, n_kw, args, star_args);
680680

681681
// We need to create the following array of objects:
682682
// args[0 .. n_args] unpacked(pos_seq) args[n_args .. n_args + 2 * n_kw] unpacked(kw_dict)
@@ -687,6 +687,20 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
687687
uint args2_alloc;
688688
uint args2_len = 0;
689689

690+
// Try to get a hint for unpacked * args length
691+
uint list_len = 0;
692+
693+
if (star_args != 0) {
694+
for (uint i = 0; i < n_args; i++) {
695+
if (star_args & (1 << i)) {
696+
mp_obj_t len = mp_obj_len_maybe(args[i]);
697+
if (len != MP_OBJ_NULL) {
698+
list_len += mp_obj_get_int(len);
699+
}
700+
}
701+
}
702+
}
703+
690704
// Try to get a hint for the size of the kw_dict
691705
uint kw_dict_len = 0;
692706

@@ -700,8 +714,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
700714

701715
// Extract the pos_seq sequence to the new args array.
702716
// Note that it can be arbitrary iterator.
703-
if (pos_seq == MP_OBJ_NULL) {
704-
// no sequence
717+
if (star_args == 0) {
718+
// no star args to unpack
705719

706720
// allocate memory for the new array of args
707721
args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len);
@@ -715,60 +729,69 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
715729
// copy the fixed pos args
716730
mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
717731
args2_len += n_args;
718-
719-
} else if (mp_obj_is_type(pos_seq, &mp_type_tuple) || mp_obj_is_type(pos_seq, &mp_type_list)) {
720-
// optimise the case of a tuple and list
721-
722-
// get the items
723-
size_t len;
724-
mp_obj_t *items;
725-
mp_obj_get_array(pos_seq, &len, &items);
726-
727-
// allocate memory for the new array of args
728-
args2_alloc = 1 + n_args + len + 2 * (n_kw + kw_dict_len);
729-
args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t));
730-
731-
// copy the self
732-
if (self != MP_OBJ_NULL) {
733-
args2[args2_len++] = self;
734-
}
735-
736-
// copy the fixed and variable position args
737-
mp_seq_cat(args2 + args2_len, args, n_args, items, len, mp_obj_t);
738-
args2_len += n_args + len;
739-
740732
} else {
741-
// generic iterator
733+
// at least one star arg to unpack
742734

743735
// allocate memory for the new array of args
744-
args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len) + 3;
736+
args2_alloc = 1 + n_args + list_len + 2 * (n_kw + kw_dict_len);
745737
args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t));
746738

747739
// copy the self
748740
if (self != MP_OBJ_NULL) {
749741
args2[args2_len++] = self;
750742
}
751743

752-
// copy the fixed position args
753-
mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
754-
args2_len += n_args;
755-
756-
// extract the variable position args from the iterator
757-
mp_obj_iter_buf_t iter_buf;
758-
mp_obj_t iterable = mp_getiter(pos_seq, &iter_buf);
759-
mp_obj_t item;
760-
while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) {
761-
if (args2_len >= args2_alloc) {
762-
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), args2_alloc * 2 * sizeof(mp_obj_t));
763-
args2_alloc *= 2;
744+
for (uint i = 0; i < n_args; i++) {
745+
mp_obj_t arg = args[i];
746+
if (star_args & (1 << i)) {
747+
// star arg
748+
if (mp_obj_is_type(arg, &mp_type_tuple) || mp_obj_is_type(arg, &mp_type_list)) {
749+
// optimise the case of a tuple and list
750+
751+
// get the items
752+
size_t len;
753+
mp_obj_t *items;
754+
mp_obj_get_array(arg, &len, &items);
755+
756+
// copy the items
757+
assert(args2_len + len <= args2_alloc);
758+
mp_seq_copy(args2 + args2_len, items, len, mp_obj_t);
759+
args2_len += len;
760+
} else {
761+
// generic iterator
762+
763+
// extract the variable position args from the iterator
764+
mp_obj_iter_buf_t iter_buf;
765+
mp_obj_t iterable = mp_getiter(arg, &iter_buf);
766+
mp_obj_t item;
767+
while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) {
768+
if (args2_len >= args2_alloc) {
769+
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t),
770+
args2_alloc * 2 * sizeof(mp_obj_t));
771+
args2_alloc *= 2;
772+
}
773+
args2[args2_len++] = item;
774+
}
775+
}
776+
} else {
777+
// normal argument
778+
assert(args2_len < args2_alloc);
779+
args2[args2_len++] = arg;
764780
}
765-
args2[args2_len++] = item;
766781
}
767782
}
768783

769784
// The size of the args2 array now is the number of positional args.
770785
uint pos_args_len = args2_len;
771786

787+
// ensure there is still enough room for kw args
788+
if (args2_len + 2 * (n_kw + kw_dict_len) > args2_alloc) {
789+
uint new_alloc = args2_len + 2 * (n_kw + kw_dict_len);
790+
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t),
791+
new_alloc * sizeof(mp_obj_t));
792+
args2_alloc = new_alloc;
793+
}
794+
772795
// Copy the kw args.
773796
for (uint i = 0; i < n_kw; i++) {
774797
mp_obj_t kw_key = args[n_args + i * 2];

‎tests/basics/fun_callstar.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@
33
def foo(a, b, c):
44
print(a, b, c)
55

6+
foo(*(), 1, 2, 3)
7+
foo(*(1,), 2, 3)
8+
foo(*(1, 2), 3)
69
foo(*(1, 2, 3))
710
foo(1, *(2, 3))
811
foo(1, 2, *(3,))
912
foo(1, 2, 3, *())
13+
foo(*(1,), 2, *(3,))
14+
foo(*(1, 2), *(3,))
15+
foo(*(1,), *(2, 3))
1016

1117
# Another sequence type
1218
foo(1, 2, *[100])
@@ -29,10 +35,16 @@ def foo(self, a, b, c):
2935
print(a, b, c)
3036

3137
a = A()
38+
a.foo(*(), 1, 2, 3)
39+
a.foo(*(1,), 2, 3)
40+
a.foo(*(1, 2), 3)
3241
a.foo(*(1, 2, 3))
3342
a.foo(1, *(2, 3))
3443
a.foo(1, 2, *(3,))
3544
a.foo(1, 2, 3, *())
45+
a.foo(*(1,), 2, *(3,))
46+
a.foo(*(1, 2), *(3,))
47+
a.foo(*(1,), *(2, 3))
3648

3749
# Another sequence type
3850
a.foo(1, 2, *[100])

‎tests/basics/fun_callstardblstar.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ def f(a, b, c, d):
66
f(*(1, 2), **{'c':3, 'd':4})
77
f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
88

9+
try:
10+
eval("f(**{'a': 1}, *(2, 3, 4))")
11+
except SyntaxError:
12+
print("SyntaxError")
13+
914
# test calling a method with *tuple and **dict
1015

1116
class A:
@@ -15,3 +20,8 @@ def f(self, a, b, c, d):
1520
a = A()
1621
a.f(*(1, 2), **{'c':3, 'd':4})
1722
a.f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
23+
24+
try:
25+
eval("a.f(**{'a': 1}, *(2, 3, 4))")
26+
except SyntaxError:
27+
print("SyntaxError")

‎tests/basics/python34.py

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,23 @@
66
print("SKIP")
77
raise SystemExit
88

9-
# from basics/fun_kwvarargs.py
10-
# test evaluation order of arguments (in 3.4 it's backwards, 3.5 it's fixed)
11-
def f4(*vargs, **kwargs):
12-
print(vargs, kwargs)
9+
1310
def print_ret(x):
1411
print(x)
1512
return x
16-
f4(*print_ret(['a', 'b']), kw_arg=print_ret(None))
1713

1814
# test evaluation order of dictionary key/value pair (in 3.4 it's backwards)
1915
{print_ret(1):print_ret(2)}
2016

17+
2118
# from basics/syntaxerror.py
2219
def test_syntax(code):
2320
try:
2421
exec(code)
2522
except SyntaxError:
2623
print("SyntaxError")
27-
test_syntax("f(*a, *b)") # can't have multiple * (in 3.5 we can)
28-
test_syntax("f(*a, b)") # can't have positional after *
24+
25+
2926
test_syntax("f(**a, b)") # can't have positional after **
3027
test_syntax("() = []") # can't assign to empty tuple (in 3.6 we can)
3128
test_syntax("del ()") # can't delete empty tuple (in 3.6 we can)

‎tests/basics/python34.py.exp

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
1-
None
2-
['a', 'b']
3-
('a', 'b') {'kw_arg': None}
41
2
52
1
63
SyntaxError
74
SyntaxError
85
SyntaxError
9-
SyntaxError
10-
SyntaxError
116
3.4
127
3 4
138
IndexError('foo',)

‎tests/cmdline/cmd_showbc.py.exp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,8 +204,9 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+):
204204
\\d\+ POP_TOP
205205
\\d\+ LOAD_FAST 0
206206
\\d\+ LOAD_DEREF 14
207+
\\d\+ LOAD_CONST_SMALL_INT 1
207208
\\d\+ LOAD_NULL
208-
\\d\+ CALL_FUNCTION_VAR_KW n=0 nkw=0
209+
\\d\+ CALL_FUNCTION_VAR_KW n=1 nkw=0
209210
\\d\+ POP_TOP
210211
\\d\+ LOAD_FAST 0
211212
\\d\+ LOAD_METHOD b
@@ -225,8 +226,9 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+):
225226
\\d\+ LOAD_FAST 0
226227
\\d\+ LOAD_METHOD b
227228
\\d\+ LOAD_FAST 1
229+
\\d\+ LOAD_CONST_SMALL_INT 1
228230
\\d\+ LOAD_NULL
229-
\\d\+ CALL_METHOD_VAR_KW n=0 nkw=0
231+
\\d\+ CALL_METHOD_VAR_KW n=1 nkw=0
230232
\\d\+ POP_TOP
231233
\\d\+ LOAD_FAST 0
232234
\\d\+ POP_JUMP_IF_FALSE \\d\+

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy