This document discusses dynamically creating functions in Python using metaprogramming techniques. It explores different approaches to programmatically defining functions with a variable number of arguments. The key challenges are correctly handling variable names, argument counts, closure variables, and memory allocation for the new function objects. After several attempts, the document arrives at a working solution that uses the types module to construct CodeType objects and FunctionType objects, ensuring all necessary function attributes like variable names and closure information are properly defined.

1. Коварный CodeType,
или от segfault'а
к работающему коду
Андрей Захаревич
Дмитрий Алимов

2. С чего все началось
# Plain functions
if inspect.isroutine(obj):
if inspect.ismethod(obj):
has_self = True
argspec = inspect.getargspec(obj)
# Class constructors
elif inspect.isclass(obj):
try:
argspec = inspect.getargspec(obj.__new__)
except (AttributeError, TypeError):
argspec = inspect.getargspec(obj.__init__)
has_self = True
# obj objects
elif hasattr(obj, '__call__'):
method = obj.__call__
argspec = inspect.getargspec(method)
has_self = inspect.ismethod(method)
return argspec.args[1:] if has_self else args

3. Проблема
● Моя функция принимает переменное число аргументов (*args)
● Число передаваемых аргументов я узнаю в рантайме
● Объект функции в это время уже создан
Что же делать?

4. Метапрограммирование!

5. Ленивый вариант
compile('def func(a, b): f(a, b)', 'func.py', 'exec')
Не спортивно

6. Идем на stackoverflow
import types
def create_function(name, args):
def y(): pass
y_code = types.CodeType(args,
y.func_code.co_nlocals,
y.func_code.co_stacksize,
y.func_code.co_flags,
y.func_code.co_code,
y.func_code.co_consts,
y.func_code.co_names,
y.func_code.co_varnames,
y.func_code.co_filename,
name,
y.func_code.co_firstlineno,
y.func_code.co_lnotab)
return types.FunctionType(y_code, y.func_globals, name)
myfunc = create_function('myfunc', 3)
myfunc(1, 2, 3, 4)
# TypeError: myfunc() takes exactly 3 arguments (4 given)

7. Кажется должно работать
def create_function(name, nargs):
def y(*args):
print args
y_code = types.CodeType(nargs,
y.func_code.co_nlocals,
# Параметры
)
return types.FunctionType(y_code, y.func_globals,
name)
myfunc = create_function('myfunc', 3)
myfunc(1, 2, 3)

8. Но нет
[1] 8514 segmentation fault python func.py

9. Еще попытка
class Meta(type):
def __new__(mcls, name, bases, attrs):
nargs = 3
varnames = tuple(['self'] + [str(i) for i in range(nargs)])
ret_code = types.CodeType(
nargs + 1,
attrs['ret_func'].func_code.co_nlocals,
attrs['ret_func'].func_code.co_stacksize,
attrs['ret_func'].func_code.co_flags,
attrs['ret_func'].func_code.co_code,
attrs['ret_func'].func_code.co_consts,
attrs['ret_func'].func_code.co_names,
varnames,
attrs['ret_func'].func_code.co_filename,
'call',
attrs['ret_func'].func_code.co_firstlineno,
attrs['ret_func'].func_code.co_lnotab)
attrs['call'] = types.FunctionType(
ret_code,
attrs['ret_func'].func_globals, 'call')
return super(Meta, mcls).__new__(mcls, name, bases, attrs)
class CustomCall(object):
__metaclass__ = Meta
def __call__(self, *args):
self.call(*args)
def ret_func(self):
print self.call.__name__
obj = CustomCall()
obj.call(1, 2, 3)
# Если
закомментировать, то
все упадет
obj(1, 2, 3)

10. Еще попытка
class Meta(type):
def __new__(mcls, name, bases, attrs):
nargs = 3
varnames = tuple(['self'] + [str(i) for i in range(nargs)])
ret_code = types.CodeType(
nargs + 1,
attrs['ret_func'].func_code.co_nlocals,
attrs['ret_func'].func_code.co_stacksize,
attrs['ret_func'].func_code.co_flags,
attrs['ret_func'].func_code.co_code,
attrs['ret_func'].func_code.co_consts,
attrs['ret_func'].func_code.co_names,
varnames,
attrs['ret_func'].func_code.co_filename,
'call',
attrs['ret_func'].func_code.co_firstlineno,
attrs['ret_func'].func_code.co_lnotab)
attrs['call'] = types.FunctionType(
ret_code,
attrs['ret_func'].func_globals, 'call')
return super(Meta, mcls).__new__(mcls, name, bases, attrs)
class CustomCall(object):
__metaclass__ = Meta
def __call__(self, *args):
self.call(*args)
def ret_func(self):
print self.call.__name__
obj = CustomCall()
obj.call(1, 2, 3)
# Если
закомментировать, то
все упадет
obj(1, 2, 3)

11. Заглянем в Disassembler
5D09E429 |. 8B7424 10 MOV ESI,DWORD PTR SS:[ESP+10] ; ESI = frame
5D09E42D |. 8B7D 14 MOV EDI,DWORD PTR SS:[EBP+14] ; EDI = *args
5D09E430 |. 8B5C24 18 MOV EBX,DWORD PTR SS:[ESP+18] ; EBX = argcount
5D09E434 |. 81C6 38010000 ADD ESI,0x138 ; ESI += 0x138 (f_localsplus)
5D09E43A |. 2BFE SUB EDI,ESI ; EDI -= ESI
5D09E43C |. 8D6424 00 LEA ESP,[ESP] ; NOP
5D09E440 |> 8B0437 /MOV EAX,DWORD PTR DS:[ESI+EDI] ; EAX = *(ESI + EDI)
5D09E443 |. FF00 |INC DWORD PTR DS:[EAX] ; Py_INCREF(*EAX)
5D09E445 |. 8B0E |MOV ECX,DWORD PTR DS:[ESI] ; ECX = *ESI
5D09E447 |. 8906 |MOV DWORD PTR DS:[ESI],EAX ; *ESI = EAX
5D09E449 |. 85C9 |TEST ECX,ECX ; test ECX:
5D09E44B |.- 74 11 |JZ SHORT 5D09E45E ; if 0: goto 5D09E45E
5D09E44D |. 8301 FF |ADD DWORD PTR DS:[ECX],-1 ; else: Py_XDECREF(*ECX)
5D09E450 |.- 75 0C |JNZ SHORT 5D09E45E ; if not 0: goto 5D09E45E
5D09E452 |. 8B41 04 |MOV EAX,DWORD PTR DS:[ECX+4] ; ECX->ob_type
5D09E455 |. 51 |PUSH ECX ; save fastlocals[i] to stack
5D09E456 |. 8B48 18 |MOV ECX,DWORD PTR DS:[EAX+18] ; ECX = ob_type->tp_dealloc
5D09E459 |. FFD1 |CALL ECX ; call ob_type->tp_dealloc()
5D09E45B |. 83C4 04 |ADD ESP,4 ; restore ESP
5D09E45E |> 83C6 04 |ADD ESI,4 ; ESI += 4
5D09E461 |. 83EB 01 |SUB EBX,1 ; EBX -= 1
5D09E464 |.- 75 DA JNZ SHORT 5D09E440 ;

12. 01eb3030 00000001 ob_refcnt
01eb3034 5886d830 python27!PyFrame_Type
01eb3038 00000002 ob_size
01eb303c 023b6b30 f_back
01eb3040 023a6b18 f_code
01eb3044 01e790c0 f_builtins
01eb3048 01e8aa50 f_globals
01eb304c 00000000 f_locals
01eb3050 01eb316c f_valuestack
01eb3054 01eb316c f_stacktop
01eb3058 00000000 f_trace
01eb305c 00000000 f_exc_type
01eb3060 00000000 f_exc_value
01eb3064 00000000 f_exc_traceback
01eb3068 01f72cf0 f_tstate
01eb306c ffffffff f_lasti
01eb3070 0000002f f_lineno
01eb3074 00000000 f_iblock
01eb3078 baadf00d f_blockstack[20] = {b_type,
01eb307c baadf00d b_handler
01eb3080 baadf00d b_level}
...
01eb3168 023bf550 f_localsplus // frame + 0x138
01eb316c 01f7d8a0
01eb3170 baadf00d
01eb3174 baadf00d
Что в памяти

13. PyObject *
PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
PyObject **args, int argcount, PyObject **kws, int kwcount,
PyObject **defs, int defcount, PyObject *closure) {
...
n = co->co_argcount;
for (i = 0; i < n; i++) {
x = args[i];
Py_INCREF(x);
SETLOCAL(i, x);
}
...
https://github.com/python/cpython/blob/2.7/Python/ceval.c
#define Py_INCREF(op) (
_Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA
((PyObject*)(op))->ob_refcnt++)
#define GETLOCAL(i) (fastlocals[i])
#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i);
GETLOCAL(i) = value;
Py_XDECREF(tmp); } while (0)

14. #define Py_XDECREF(op) do { if ((op) == NULL); else Py_DECREF(op); } while (0)
#define Py_DECREF(op)
do {
if (_Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA
--((PyObject*)(op))->ob_refcnt != 0)
_Py_CHECK_REFCNT(op)
else
_Py_Dealloc((PyObject *)(op));
} while (0)
#define _Py_Dealloc(op) (
_Py_INC_TPFREES(op) _Py_COUNT_ALLOCS_COMMA
(*Py_TYPE(op)->tp_dealloc)((PyObject *)(op)))
#endif /* !Py_TRACE_REFS */
https://github.com/python/cpython/blob/2.7/Include/object.h

15. typedef struct _frame {
PyObject_VAR_HEAD
struct _frame *f_back; /* previous frame, or NULL */
PyCodeObject *f_code; /* code segment */
PyObject *f_builtins; /* builtin symbol table (PyDictObject) */
PyObject *f_globals; /* global symbol table (PyDictObject) */
PyObject *f_locals; /* local symbol table (any mapping) */
PyObject **f_valuestack; /* points after the last local */
PyObject **f_stacktop;
PyObject *f_trace; /* Trace function */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
PyThreadState *f_tstate;
int f_lasti; /* Last instruction if called */
int f_lineno; /* Current line number */
int f_iblock; /* index in f_blockstack */
PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */
PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */
} PyFrameObject;
https://github.com/python/cpython/blob/2.7/Include/frameobject.h

16. ncells = PyTuple_GET_SIZE(code->co_cellvars);
nfrees = PyTuple_GET_SIZE(code->co_freevars);
extras = code->co_stacksize + code->co_nlocals + ncells + nfrees;
if (free_list == NULL) {
f = PyObject_GC_NewVar(PyFrameObject, &PyFrame_Type, extras);
...
https://github.com/python/cpython/blob/2.7/Objects/frameobject.c
co_cellvars – is a tuple with the names of local variables referenced by nested functions;
co_freevars – names of variables in the function, defined in an enclosing function scope;
Память для f_localsplus

17. Проблема найдена
ret_code = types.CodeType(
nargs + 1,
attrs['ret_func'].func_code.co_nlocals,
attrs['ret_func'].func_code.co_stacksize,
attrs['ret_func'].func_code.co_flags,
attrs['ret_func'].func_code.co_code,
attrs['ret_func'].func_code.co_consts,
attrs['ret_func'].func_code.co_names,
varnames,
attrs['ret_func'].func_code.co_filename,
'call',
attrs['ret_func'].func_code.co_firstlineno,
attrs['ret_func'].func_code.co_lnotab,
(), # freevars
() # cellvars
)

18. Попробуем применить
def create_func(fun, nargs):
def wrapper():
args = map(itemgetter(1),
sorted(((var_name, arg) for (var_name, arg)
in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
fun(*args)
varnames = tuple(str(i) for i in range(0, nargs))
ret_code = types.CodeType(nargs,
wrapper.func_code.co_nlocals + nargs,
# Параметры
)
return types.FunctionType(ret_code, wrapper.func_globals,
wrapper.func_code.co_name)
def test_func(*args):
print args
fun = create_func(test_func, 3)
fun(1, 2, 3)

19. Как-то не очень
Traceback (most recent call last):
File "func.py", line 38, in <module>
fun(1,2,3)
File "func.py", line 16, in wrapper
fun(*args)
SystemError: Objects/cellobject.c:24: bad argument to internal function

20. Заработало!
def create_func(fun, nargs):
def wrapper(fun):
args = map(itemgetter(1),
sorted(((var_name, arg) for (var_name, arg)
in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
fun(*args)
varnames = tuple(['fun'] + [str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs + 1,
wrapper.func_code.co_nlocals + nargs,
# Параметры
)
return partial(types.FunctionType(ret_code, wrapper.func_globals,
wrapper.func_code.co_name), fun)
def test_func(*args):
print args
fun = create_func(test_func, 3)
fun(1, 2, 3)

21. Что-то потеряли
>>> dir(wrapper.func_code)
['__class__', '__cmp__', '__delattr__', '__doc__', '__eq__', '__format__', '__ge__',
'__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__ne__', '__new__',
'__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__',
'__subclasshook__', 'co_argcount', 'co_cellvars', 'co_code', 'co_consts', 'co_filename',
'co_firstlineno', 'co_flags', 'co_freevars', 'co_lnotab', 'co_name', 'co_names', 'co_nlocals',
'co_stacksize', 'co_varnames']

22. Добавим параметр
ret_code = types.CodeType(nargs,
wrapper.func_code.co_nlocals + nargs,
# Параметры
wrapper.func_code.co_freevars)
return types.FunctionType(ret_code, wrapper.func_globals,
wrapper.func_code.co_name)
>>>Traceback (most recent call last):
File "func.py", line 39, in <module>
fun = create_func(test_func, 3)
File "func.py", line 33, in create_func
return types.FunctionType(ret_code, wrapper.func_globals,
wrapper.func_code.co_name)
TypeError: arg 5 (closure) must be tuple

23. def create_func(fun, nargs):
def wrapper():
args = map(itemgetter(1),
sorted(((var_name, arg) for (var_name, arg)
in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
fun(*args)
varnames = tuple([str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs,
wrapper.func_code.co_nlocals + nargs,
# Параметры
wrapper.func_code.co_freevars)
return types.FunctionType(ret_code, wrapper.func_globals,
wrapper.func_code.co_name, None,
wrapper.func_closure)
def test_func(*args):
print args
fun = create_func(test_func, 3)
fun(1, 2, 3)
И наконец, замыкания работают!

24. … почти
from operator import itemgetter
def wrapper():
print locals()
args = map(itemgetter(1),
sorted(((var_name, arg) for (var_name, arg)
in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
print locals()
fun(*args)
{'1': 2, '0': 1, '2': 3, 'fun': <function test_func at 0x10cd62398>}
{'1': 2, '0': [1, 2, 3], '2': 3, 'fun': <function test_func at 0x10cd62398>}

25. Спасибо за внимание!
Вопросы?
SPb Python Interest Group
https://telegram.me/spbpython