Extending Python with C (Part I – the Basics) June 2002 Brian Quinlan

Slide 2©2002 Brian Quinlan Why Mess with C? There is a preexisting library, available in C, that you would like to access from Python (e.g. pyexpat, math, zlib) Solving the problem in Python would be too CPU or memory inefficient (e.g. Numeric, PIL) Only way to create new fundament types (doesn’t seem too important to me; save it for another talk)

Slide 3©2002 Brian Quinlan Jumpin’ In Create our own extension module containing an eclectic set of functions We’ll call it: Vanpy Basic steps: –Do module initialization –Write functions –Write build script –Testing

Slide 4©2002 Brian Quinlan A module is just a dictionary foo.py magic_number = 5 def divide(x,y): return x / y >>> import foo >>> foo.__dict__ {'__doc__': None, 'magic_number': 5, 'divide':, …} >>> foo.__dict__['divide'](8,4) 2

Slide 5©2002 Brian Quinlan Module Initialization void initVanpy(void) { PyObject * module; module = Py_InitModule3("Vanpy", module_functions, module_doc); PyModule_AddStringConstant(module, "__version__", "0.0.1"); }

Slide 6©2002 Brian Quinlan A VERY Simple Function static PyObject * is_even(PyObject * self, PyObject * args) { long number; if (!PyArg_ParseTuple(args, "i", &number)) return NULL; return PyBuildValue("i", number % 2 == 0) } static PyMethodDef module_functions[] = { {"is_even", is_even, METH_VARARGS, is_even_doc}, {NULL, NULL} }

Slide 7©2002 Brian Quinlan A Simple Function static PyObject * classify_characters(PyObject * self, PyObject * args) { char * string; long length; long alpha, other = 0; if (!PyArg_ParseTuple(args, "s#", &string, &length)) return NULL; for (int i = 0; i < length; ++i) { isalnum(string[i]) ? ++alpha : ++other } return PyBuildValue("(iif)", alpha, other, ((float) alpha) / length) }

Slide 8©2002 Brian Quinlan Objects and Types typedef struct { struct _typeobject *ob_type; int ob_refcnt; /* The rest depends on the type */ } PyObject; typedef struct _typeobject { char *tp_name; /* Stuff to discuss in another talk */ }

Slide 9©2002 Brian Quinlan Reference Counting >>> a = “Hello” PyObject * a = Py_BuildValue(“s”, “Hello”) a 1Hello

Slide 10©2002 Brian Quinlan Reference Counting >>> a = “Hello” PyObject * a = Py_BuildValue(“s”, “Hello”) >>> b = a PyObject * b = a Py_INCREF(b) a 1Hello a 2 b

Slide 11©2002 Brian Quinlan Reference Counting >>> a = “Hello” PyObject * a = Py_BuildValue(“s”, “Hello”) >>> b = a PyObject * b = a Py_INCREF(b) >>> del a Py_DECREF(a) a 1Hello a 2 b b 1

Slide 12©2002 Brian Quinlan Reference Counting >>> a = “Hello” PyObject * a = Py_BuildValue(“s”, “Hello”) >>> b = a PyObject * b = a Py_INCREF(b) >>> del a Py_DECREF(a) >>> del b Py_DECREF(b) a 1Hello a 2 b b 1 0

Slide 13©2002 Brian Quinlan Reference Counting Rules When returning a PyObject from an extension function or method, always increment the object’s reference count USUALLY, when calling a Python API function, you need not increment the reference counts of PyObject arguments (only 2 important exceptions) USUALLY, when a PyObject is returned from a Python API function, you must decrement the object’s reference count when you are done with it (> 4 important exceptions)

Slide 14©2002 Brian Quinlan A “Realistic” Example static PyObject * password = NULL; static PyObject * set_password(PyObject * self, PyObject * args) { PyObject * new_password; if (!PyArg_ParseTuple(args, "O", &new_password)) return NULL; if (password != NULL) /* or Py_XDECREF(password) */ Py_DECREF(password); password = new_password; Py_INCREF(password); Py_INCREF(Py_None); return Py_None; }

Slide 15©2002 Brian Quinlan A “Realistic” Example static PyObject * get_password(PyObject * self, PyObject * args) { if (!PyArg_ParseTuple(args, "")) return NULL; if (password == NULL) { Py_INCREF(Py_None); return Py_None; } else { Py_INCREF(password); return password; }

Slide 16©2002 Brian Quinlan Exceptions Each Python thread has 3 exception variables associated with it: –The type of the exception (e.g. ValueError) –The value of the exception (e.g. “0 denominator”) –A traceback object (e.g. file “foo.py”, line 1, in “divide”) def divide(x,y): raise ValueError, “0 denominator” static PyObject * divide(PyObject * self, PyObject * args) { PyErr_SetString(PyExc_ValueError, “0 denominitor”); return NULL; }

Slide 17©2002 Brian Quinlan Passwords Revised static PyObject * set_password(PyObject * self, PyObject * args) { PyObject * new_password; if (!PyArg_ParseTuple(args, "O", &new_password)) return NULL; if (!PyString_Check(new_password) && !PyUnicode_Check(new_password)) { PyErr_Format(PyExc_TypeError, "expected string or " "Unicode, %80s found", new_password->ob_type->tp_name); return NULL; }...

Slide 18©2002 Brian Quinlan More Exceptions If you want to ignore an error, use PyErr_Clear to clear the current exception You can check to see if an exception has been set using PyErr_Occurred (this is not usually need since the API return value indicates failure) It is an error to return NULL without setting an exception

Slide 19©2002 Brian Quinlan Custom Exceptions I static PyObject * WeatherError; void initVanpy(void) { PyObject * module; PyObject * module_dict; module = Py_InitModule3("Vanpy", module_methods, module_doc); WeatherError = PyErr_NewException("Vanpy.WeatherError", NULL, NULL); module_dict = PyModule_GetDict(module); PyDict_SetItemString( module_dict, "WeatherError", WeatherError); }

Slide 20©2002 Brian Quinlan Custom Exceptions II static PyObject * goto_beach(PyObject * self, PyObject * args) { if (!PyArg_ParseTuple(args, "")) return NULL; if (rand() % 2) { PyObject * e = Py_BuildValue("(si)", "too cold", 5); if (e == NULL) return NULL; PyErr_SetObject(WeatherError, e); Py_DECREF(e); } else { PyErr_SetString(WeatherError, "too rainy"); } return NULL; }

Slide 21©2002 Brian Quinlan Building setup.py #!/usr/bin/env python from distutils.core import setup, Extension setup( name = "Vanpy", version = "0.0.1", description = "An eclectic set of functions", author = "Brian Quinlan", author_ = ext_modules = [ Extension("Vanpy", ["Vanpy.c"]) ] )

Slide 22©2002 Brian Quinlan Python API organization Exception handling Memory Management Threads Utilities –OS Utilities –Process Control –Importing Modules –Data Marshalling –Parsing Arguments & Building Values Abstract Object Support Concrete Object Support

Slide 23©2002 Brian Quinlan Abstract Object Support Applies to classes of objects, not specific types Divided into protocols: –Object –Number –Sequence –Mapping –Iterator –Buffer

Slide 24©2002 Brian Quinlan Object Protocol A sampling of functions in the object protocol: int PyObject_HasAttrString(PyObject *o, char *attr_name) int PyObject_Cmp(PyObject *o1, PyObject *o2, int *result) PyObject* PyObject_Repr(PyObject *o) PyObject* PyObject_Str(PyObject *o) int PyObject_IsInstance(PyObject *inst, PyObject *cls) PyObject* PyObject_CallFunction(PyObject *callable, char *format,...) PyObject* PyObject_CallMethod(PyObject *o, char *method, char *format,...) int PyObject_IsTrue(PyObject *o)

Slide 25©2002 Brian Quinlan Number Protocol A sampling of functions in the number protocol: int PyNumber_Check(PyObject *o) PyObject* PyNumber_Add(PyObject *o1, PyObject *o2) PyObject* PyNumber_Subtract(PyObject *o1, PyObject *o2) PyObject* PyNumber_Multiply(PyObject *o1, PyObject *o2) PyObject* PyNumber_Negative(PyObject *o) PyObject* PyNumber_Absolute(PyObject *o) PyObject* PyNumber_Lshift(PyObject *o1, PyObject *o2) PyObject* PyNumber_And(PyObject *o1, PyObject *o2) PyObject* PyNumber_Int(PyObject *o) PyObject* PyNumber_Long(PyObject *o) PyObject* PyNumber_Float(PyObject *o)

Slide 26©2002 Brian Quinlan Sequence Protocol A sampling of functions in the sequence protocol: int PySequence_Check(PyObject *o) int PySequence_Length(PyObject *o) PyObject* PySequence_Repeat(PyObject *o, int count) PyObject* PySequence_GetItem(PyObject *o, int i) PyObject* PySequence_GetSlice(PyObject *o, int i1, int i2) int PySequence_SetItem(PyObject *o, int i, PyObject *v) int PySequence_DelItem(PyObject *o, int i) PyObject* PySequence_Tuple(PyObject *o) int PySequence_Contains(PyObject *o, PyObject *value) int PySequence_Count(PyObject *o, PyObject *value)

Slide 27©2002 Brian Quinlan Concrete Object Support Applies to specific object types Different set of functions for every built-in Python type There any many Python object types: plain int, long int, float, complex, string, unicode, buffer, tuple, list, dictionary, file, instance, method, module, iterator, slice, weak reference, etc. Reference counting rules are less intuitive

Slide 28©2002 Brian Quinlan Int Objects This is the complete list of int functions: int PyInt_Check(PyObject* o) int PyInt_CheckExact(PyObject* o) PyObject* PyInt_FromLong(long ival) long PyInt_AsLong(PyObject *io) long PyInt_AS_LONG(PyObject *io) long PyInt_GetMax()

Slide 29©2002 Brian Quinlan String Objects A sampling of string functions: int PyString_Check(PyObject *o) int PyString_CheckExact(PyObject *o) PyObject* PyString_FromString(const char *v) PyObject* PyString_FromStringAndSize(const char *v, int len) PyObject* PyString_FromFormat(const char *format,...) PyObject* PyString_FromFormatV(const char *format, va_list vargs) int PyString_Size(PyObject *string) int PyString_GET_SIZE(PyObject *string) char* PyString_AsString(PyObject *string) char* PyString_AS_STRING(PyObject *string) void PyString_Concat(PyObject **string, PyObject *newpart)

Slide 30©2002 Brian Quinlan List Objects This is the complete list of list functions: int PyList_Check(PyObject *p) PyObject* PyList_New(int len) int PyList_Size(PyObject *list) int PyList_GET_SIZE(PyObject *list) PyObject* PyList_GetItem(PyObject *list, int index) PyObject* PyList_GET_ITEM(PyObject *list, int i) int PyList_SetItem(PyObject *list, int index, PyObject *item) void PyList_SET_ITEM(PyObject *list, int i, PyObject *o) int PyList_Insert(PyObject *list, int index, PyObject *item) int PyList_Append(PyObject *list, PyObject *item) PyObject* PyList_GetSlice(PyObject *list, int low, int high) int PyList_SetSlice(PyObject *list, int low, int high, PyObject *itemlist) int PyList_Sort(PyObject *list)

Slide 31©2002 Brian Quinlan Helpful Technologies SWIG/SIP - generate C/C++ interface code for Python Boost Python - an interface generator for wrapping C++ classes with a Python interface CXX - C++ wrapper around the Python API Pyrex - a language for writing Python extension modules Pyfort - a wrapper generator for interfacing Fortran with Python PyInline/Weave – embed C/C++/Perl code inside your Python script