Native Extensions
Served 3 Ways
Tejas Dinkar

Nilenso Software
about.me
•

Hi, I’m Tejas

•

Nilenso: Partner

•

twitter: tdinkar

•

github: gja
about.talk
•

Expect to see lots of code

•

Will have about 5 minutes for questions

•

Please laugh at my jokes!

•

Will cover C Extensions, FFI and SWIG
Native Extensions
•

Integrate with new libraries

•

Improve Performance of critical code

•

Write code that works across languages

•

Feel super 1337
Let’s talk about Python

•

Pythonista’s in the house?

•

Yes, I’m trolling you!

http://montgomeryq.blogspot.in/2011/05/random-illustration-tuesday-python-ruby.html
#include "Python.h"
#include "ruby.h"

!

static PyObject *python_ruby_eval(PyObject *self, PyObject *string)
{
VALUE val = rb_eval_string(PyString_AsString(string));
switch(TYPE(val)) {
case T_FIXNUM: return PyInt_FromLong(FIX2INT(val));
case T_STRING: return PyString_FromString(StringValuePtr(val));
default: return Py_None; // Can handle these cases later
}
}

!

static PyMethodDef module_functions[] = {
{ "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" },
{ NULL }
};

!

void initruby(void)
{
ruby_init();
Py_InitModule3("ruby", module_functions, "A ruby module for python.");
}
#include "Python.h"
#include "ruby.h"

!

static PyObject *python_ruby_eval(PyObject *self, PyObject *string)
{
VALUE val = rb_eval_string(PyString_AsString(string));
switch(TYPE(val)) {
case T_FIXNUM: return PyInt_FromLong(FIX2INT(val));
case T_STRING: return PyString_FromString(StringValuePtr(val));
default: return Py_None; // Can handle these cases later
}
}

!

static PyMethodDef module_functions[] = {
{ "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" },
{ NULL }
};

!

void initruby(void)
{
ruby_init();
Py_InitModule3("ruby", module_functions, "A ruby module for python.");
}
#include "Python.h"
#include "ruby.h"

!

static PyObject *python_ruby_eval(PyObject *self, PyObject *string)
{
VALUE val = rb_eval_string(PyString_AsString(string));
switch(TYPE(val)) {
case T_FIXNUM: return PyInt_FromLong(FIX2INT(val));
case T_STRING: return PyString_FromString(StringValuePtr(val));
default: return Py_None; // Can handle these cases later
}
}

!

static PyMethodDef module_functions[] = {
{ "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" },
{ NULL }
};

!

void initruby(void)
{
ruby_init();
Py_InitModule3("ruby", module_functions, "A ruby module for python.");
}
#include "Python.h"
#include "ruby.h"

!

static PyObject *python_ruby_eval(PyObject *self, PyObject *string)
{
VALUE val = rb_eval_string(PyString_AsString(string));
switch(TYPE(val)) {
case T_FIXNUM: return PyInt_FromLong(FIX2INT(val));
case T_STRING: return PyString_FromString(StringValuePtr(val));
default: return Py_None; // Can handle these cases later
}
}

!

static PyMethodDef module_functions[] = {
{ "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" },
{ NULL }
};

!

void initruby(void)
{
ruby_init();
Py_InitModule3("ruby", module_functions, "A ruby module for python.");
}
#include "Python.h"
#include "ruby.h"

!

static PyObject *python_ruby_eval(PyObject *self, PyObject *string)
{
VALUE val = rb_eval_string(PyString_AsString(string));
switch(TYPE(val)) {
case T_FIXNUM: return PyInt_FromLong(FIX2INT(val));
case T_STRING: return PyString_FromString(StringValuePtr(val));
default: return Py_None; // Can handle these cases later
}
}

!

static PyMethodDef module_functions[] = {
{ "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" },
{ NULL }
};

!

void initruby(void)
{
ruby_init();
Py_InitModule3("ruby", module_functions, "A ruby module for python.");
}
#include "Python.h"
#include "ruby.h"

!

static PyObject *python_ruby_eval(PyObject *self, PyObject *string)
{
VALUE val = rb_eval_string(PyString_AsString(string));
switch(TYPE(val)) {
case T_FIXNUM: return PyInt_FromLong(FIX2INT(val));
case T_STRING: return PyString_FromString(StringValuePtr(val));
default: return Py_None; // Can handle these cases later
}
}

!

static PyMethodDef module_functions[] = {
{ "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" },
{ NULL }
};

!

void initruby(void)
{
ruby_init();
Py_InitModule3("ruby", module_functions, "A ruby module for python.");
}
#include "Python.h"
#include "ruby.h"

!

static PyObject *python_ruby_eval(PyObject *self, PyObject *string)
{
VALUE val = rb_eval_string(PyString_AsString(string));
switch(TYPE(val)) {
case T_FIXNUM: return PyInt_FromLong(FIX2INT(val));
case T_STRING: return PyString_FromString(StringValuePtr(val));
default: return Py_None; // Can handle these cases later
}
}

!

static PyMethodDef module_functions[] = {
{ "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" },
{ NULL }
};

!

void initruby(void)
{
ruby_init();
Py_InitModule3("ruby", module_functions, "A ruby module for python.");
}
#include "Python.h"
#include "ruby.h"

!

static PyObject *python_ruby_eval(PyObject *self, PyObject *string)
{
VALUE val = rb_eval_string(PyString_AsString(string));
switch(TYPE(val)) {
case T_FIXNUM: return PyInt_FromLong(FIX2INT(val));
case T_STRING: return PyString_FromString(StringValuePtr(val));
default: return Py_None; // Can handle these cases later
}
}

!

static PyMethodDef module_functions[] = {
{ "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" },
{ NULL }
};

!

void initruby(void)
{
ruby_init();
Py_InitModule3("ruby", module_functions, "A ruby module for python.");
}
Require Files?
static PyObject *python_ruby_require(PyObject *self, PyObject *file)
{
rb_require(PyString_AsString(file));
return Py_True;
}
Congrats!
Common Fears

MEMORY ALLOCATION!?
Memory Management

•

Data_Wrap_Struct(klass, mark_cb, free_cb, *data)

•

Data_Get_Struct( VALUE, data_type, data* )
Common Fears
Portability

http://geekandpoke.typepad.com/geekandpoke/2008/05/the-history-of.html
string.c
void Init_String(void) {!
rb_cString = rb_define_class("String", rb_cObject);!
// ...!
rb_define_method(rb_cString, "eql?", rb_str_eql, 1);!
rb_define_method(rb_cString, "==", rb_str_equal, 1);!
// ...!
rb_define_method(rb_cString, "insert", rb_str_insert, 2);!
rb_define_method(rb_cString, "length", rb_str_length, 0);!
// ...!
}!
!
static VALUE rb_str_eql(VALUE self, VALUE str2)!
{!
if (self == str2) return Qtrue;!
if (!RB_TYPE_P(str2, T_STRING)) return Qfalse;!
return str_eql(self, str2);!
}!
string.c
void Init_String(void) {!
rb_cString = rb_define_class("String", rb_cObject);!
// ...!
rb_define_method(rb_cString, "eql?", rb_str_eql, 1);!
rb_define_method(rb_cString, "==", rb_str_equal, 1);!
// ...!
rb_define_method(rb_cString, "insert", rb_str_insert, 2);!
rb_define_method(rb_cString, "length", rb_str_length, 0);!
// ...!
}!
!
static VALUE rb_str_eql(VALUE str2, VALUE str2)!
{!
if (self == str2) return Qtrue;!
if (!RB_TYPE_P(str2, T_STRING)) return Qfalse;!
return str_eql(self, str2);!
}!
string.c
void Init_String(void) {!
rb_cString = rb_define_class("String", rb_cObject);!
// ...!
rb_define_method(rb_cString, "eql?", rb_str_eql, 1);!
rb_define_method(rb_cString, "==", rb_str_equal, 1);!
// ...!
rb_define_method(rb_cString, "insert", rb_str_insert, 2);!
rb_define_method(rb_cString, "length", rb_str_length, 0);!
// ...!
}!
!
static VALUE rb_str_eql(VALUE str2, VALUE str2)!
{!
if (self == str2) return Qtrue;!
if (!RB_TYPE_P(str2, T_STRING)) return Qfalse;!
return str_eql(self, str2);!
}!
C Extensions

Native Code

Ruby Aware!
Native Code

Ruby Code
Foreign Function Interface

Native Code

Native Aware!
Ruby Code

Ruby Code
Foreign Function Interface

•

A Ruby DSL

•

Works across all Ruby Implementations

•

Converts to and from C primitives for you
example
require 'ffi'!
!

module MyLib!
extend FFI::Library!
ffi_lib 'c'!
attach_function :puts, [:string], :int!
end!
!

MyLib.puts 'Hello, World using libc!'
another example
require 'ffi'!
!

module MyMathLib!
extend FFI::Library!
ffi_lib 'm'!
attach_function :pow, [:double, :double],!
:double!
end!
!

MyMathLib.pow(4, 5) # => 1024.0
Lots of built in types
Numbers!! ! ! ! ! Character!! ! ! ! ! Other!
:int! ! ! ! ! ! ! :char!! ! ! ! ! ! ! :pointer!
:short! ! ! ! ! ! :string!
:long!
:double!
:float!
Foreign Function Interface
•

Probably your best solution

•

It’s really easy

•

Do your modelling in Ruby

•

Still have to worry about GC

•

Sadly, no C++ without wrapping
Memory in FFI

def run_query_which_will_crash!
db_connection = MyFFIModule.database_connection("localhost")!
MyFFIModule.database_query(db_connection, "select * from users")!
end!
Memory in FFI
This will get GCed
def run_query_which_will_crash!
db_connection = MyFFIModule.database_connection("localhost")!
MyFFIModule.database_query(db_connection, "select * from users")!
end!
SWIG
•

Simplified Wrapper and Interface Generator

•

Annotate your C/C++ header files

•

It generates native extensions for languages
•

About 20 languages currently supported
SWIG
Ruby Code

Magic
Native Code

Python Code
The Magic
•

Takes an interface file

•

Auto generates code to make it work

•

For ruby, it’s a `regular’ C extension

•

For python, it’s a a .c and .py file

•

For Java it’s a JNI interface

•

Still need to do your own GC
The Rectangle
class Rectangle!
{!
int length;!
int breadth;!
!

public:!
Rectangle(int length, int breadth);!
int area();!
};
The Rectangle
#ifdef SWIG
%module shape
%{

SWIG Stuff Here

class Rectangle!
{!
int length;!
int breadth;!
!

public:!
Rectangle(int length, int breadth);!
int area();!
};
%}
require 'shapes'!
!

rectangle = shapes.Rectangle.new(10, 12)!
rectangle.area == 120!
Other Options

•

DL (Dynamic Load)

•

Fiddle(r)
TL;DR
•

Native Extensions are fun and easy to build

•

The three big tools

•

You want to pick FFI if you don’t maintain the lib

•

SWIG may be better if you are a maintainer
Thank You
super integration
wow

so extension

so native

ruby = win

wow

such easy

Many Questions?
no python

such performance

Gcrc talk

  • 1.
    Native Extensions Served 3Ways Tejas Dinkar Nilenso Software
  • 2.
    about.me • Hi, I’m Tejas • Nilenso:Partner • twitter: tdinkar • github: gja
  • 3.
    about.talk • Expect to seelots of code • Will have about 5 minutes for questions • Please laugh at my jokes! • Will cover C Extensions, FFI and SWIG
  • 4.
    Native Extensions • Integrate withnew libraries • Improve Performance of critical code • Write code that works across languages • Feel super 1337
  • 5.
    Let’s talk aboutPython • Pythonista’s in the house? • Yes, I’m trolling you! http://montgomeryq.blogspot.in/2011/05/random-illustration-tuesday-python-ruby.html
  • 7.
    #include "Python.h" #include "ruby.h" ! staticPyObject *python_ruby_eval(PyObject *self, PyObject *string) { VALUE val = rb_eval_string(PyString_AsString(string)); switch(TYPE(val)) { case T_FIXNUM: return PyInt_FromLong(FIX2INT(val)); case T_STRING: return PyString_FromString(StringValuePtr(val)); default: return Py_None; // Can handle these cases later } } ! static PyMethodDef module_functions[] = { { "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" }, { NULL } }; ! void initruby(void) { ruby_init(); Py_InitModule3("ruby", module_functions, "A ruby module for python."); }
  • 8.
    #include "Python.h" #include "ruby.h" ! staticPyObject *python_ruby_eval(PyObject *self, PyObject *string) { VALUE val = rb_eval_string(PyString_AsString(string)); switch(TYPE(val)) { case T_FIXNUM: return PyInt_FromLong(FIX2INT(val)); case T_STRING: return PyString_FromString(StringValuePtr(val)); default: return Py_None; // Can handle these cases later } } ! static PyMethodDef module_functions[] = { { "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" }, { NULL } }; ! void initruby(void) { ruby_init(); Py_InitModule3("ruby", module_functions, "A ruby module for python."); }
  • 9.
    #include "Python.h" #include "ruby.h" ! staticPyObject *python_ruby_eval(PyObject *self, PyObject *string) { VALUE val = rb_eval_string(PyString_AsString(string)); switch(TYPE(val)) { case T_FIXNUM: return PyInt_FromLong(FIX2INT(val)); case T_STRING: return PyString_FromString(StringValuePtr(val)); default: return Py_None; // Can handle these cases later } } ! static PyMethodDef module_functions[] = { { "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" }, { NULL } }; ! void initruby(void) { ruby_init(); Py_InitModule3("ruby", module_functions, "A ruby module for python."); }
  • 10.
    #include "Python.h" #include "ruby.h" ! staticPyObject *python_ruby_eval(PyObject *self, PyObject *string) { VALUE val = rb_eval_string(PyString_AsString(string)); switch(TYPE(val)) { case T_FIXNUM: return PyInt_FromLong(FIX2INT(val)); case T_STRING: return PyString_FromString(StringValuePtr(val)); default: return Py_None; // Can handle these cases later } } ! static PyMethodDef module_functions[] = { { "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" }, { NULL } }; ! void initruby(void) { ruby_init(); Py_InitModule3("ruby", module_functions, "A ruby module for python."); }
  • 11.
    #include "Python.h" #include "ruby.h" ! staticPyObject *python_ruby_eval(PyObject *self, PyObject *string) { VALUE val = rb_eval_string(PyString_AsString(string)); switch(TYPE(val)) { case T_FIXNUM: return PyInt_FromLong(FIX2INT(val)); case T_STRING: return PyString_FromString(StringValuePtr(val)); default: return Py_None; // Can handle these cases later } } ! static PyMethodDef module_functions[] = { { "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" }, { NULL } }; ! void initruby(void) { ruby_init(); Py_InitModule3("ruby", module_functions, "A ruby module for python."); }
  • 12.
    #include "Python.h" #include "ruby.h" ! staticPyObject *python_ruby_eval(PyObject *self, PyObject *string) { VALUE val = rb_eval_string(PyString_AsString(string)); switch(TYPE(val)) { case T_FIXNUM: return PyInt_FromLong(FIX2INT(val)); case T_STRING: return PyString_FromString(StringValuePtr(val)); default: return Py_None; // Can handle these cases later } } ! static PyMethodDef module_functions[] = { { "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" }, { NULL } }; ! void initruby(void) { ruby_init(); Py_InitModule3("ruby", module_functions, "A ruby module for python."); }
  • 13.
    #include "Python.h" #include "ruby.h" ! staticPyObject *python_ruby_eval(PyObject *self, PyObject *string) { VALUE val = rb_eval_string(PyString_AsString(string)); switch(TYPE(val)) { case T_FIXNUM: return PyInt_FromLong(FIX2INT(val)); case T_STRING: return PyString_FromString(StringValuePtr(val)); default: return Py_None; // Can handle these cases later } } ! static PyMethodDef module_functions[] = { { "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" }, { NULL } }; ! void initruby(void) { ruby_init(); Py_InitModule3("ruby", module_functions, "A ruby module for python."); }
  • 14.
    #include "Python.h" #include "ruby.h" ! staticPyObject *python_ruby_eval(PyObject *self, PyObject *string) { VALUE val = rb_eval_string(PyString_AsString(string)); switch(TYPE(val)) { case T_FIXNUM: return PyInt_FromLong(FIX2INT(val)); case T_STRING: return PyString_FromString(StringValuePtr(val)); default: return Py_None; // Can handle these cases later } } ! static PyMethodDef module_functions[] = { { "eval", python_ruby_eval, METH_O, "Evaluate Ruby Code" }, { NULL } }; ! void initruby(void) { ruby_init(); Py_InitModule3("ruby", module_functions, "A ruby module for python."); }
  • 15.
    Require Files? static PyObject*python_ruby_require(PyObject *self, PyObject *file) { rb_require(PyString_AsString(file)); return Py_True; }
  • 16.
  • 17.
  • 18.
    Memory Management • Data_Wrap_Struct(klass, mark_cb,free_cb, *data) • Data_Get_Struct( VALUE, data_type, data* )
  • 19.
  • 20.
  • 21.
    string.c void Init_String(void) {! rb_cString= rb_define_class("String", rb_cObject);! // ...! rb_define_method(rb_cString, "eql?", rb_str_eql, 1);! rb_define_method(rb_cString, "==", rb_str_equal, 1);! // ...! rb_define_method(rb_cString, "insert", rb_str_insert, 2);! rb_define_method(rb_cString, "length", rb_str_length, 0);! // ...! }! ! static VALUE rb_str_eql(VALUE self, VALUE str2)! {! if (self == str2) return Qtrue;! if (!RB_TYPE_P(str2, T_STRING)) return Qfalse;! return str_eql(self, str2);! }!
  • 22.
    string.c void Init_String(void) {! rb_cString= rb_define_class("String", rb_cObject);! // ...! rb_define_method(rb_cString, "eql?", rb_str_eql, 1);! rb_define_method(rb_cString, "==", rb_str_equal, 1);! // ...! rb_define_method(rb_cString, "insert", rb_str_insert, 2);! rb_define_method(rb_cString, "length", rb_str_length, 0);! // ...! }! ! static VALUE rb_str_eql(VALUE str2, VALUE str2)! {! if (self == str2) return Qtrue;! if (!RB_TYPE_P(str2, T_STRING)) return Qfalse;! return str_eql(self, str2);! }!
  • 23.
    string.c void Init_String(void) {! rb_cString= rb_define_class("String", rb_cObject);! // ...! rb_define_method(rb_cString, "eql?", rb_str_eql, 1);! rb_define_method(rb_cString, "==", rb_str_equal, 1);! // ...! rb_define_method(rb_cString, "insert", rb_str_insert, 2);! rb_define_method(rb_cString, "length", rb_str_length, 0);! // ...! }! ! static VALUE rb_str_eql(VALUE str2, VALUE str2)! {! if (self == str2) return Qtrue;! if (!RB_TYPE_P(str2, T_STRING)) return Qfalse;! return str_eql(self, str2);! }!
  • 24.
    C Extensions Native Code RubyAware! Native Code Ruby Code
  • 25.
    Foreign Function Interface NativeCode Native Aware! Ruby Code Ruby Code
  • 26.
    Foreign Function Interface • ARuby DSL • Works across all Ruby Implementations • Converts to and from C primitives for you
  • 27.
    example require 'ffi'! ! module MyLib! extendFFI::Library! ffi_lib 'c'! attach_function :puts, [:string], :int! end! ! MyLib.puts 'Hello, World using libc!'
  • 28.
    another example require 'ffi'! ! moduleMyMathLib! extend FFI::Library! ffi_lib 'm'! attach_function :pow, [:double, :double],! :double! end! ! MyMathLib.pow(4, 5) # => 1024.0
  • 29.
    Lots of builtin types Numbers!! ! ! ! ! Character!! ! ! ! ! Other! :int! ! ! ! ! ! ! :char!! ! ! ! ! ! ! :pointer! :short! ! ! ! ! ! :string! :long! :double! :float!
  • 30.
    Foreign Function Interface • Probablyyour best solution • It’s really easy • Do your modelling in Ruby • Still have to worry about GC • Sadly, no C++ without wrapping
  • 31.
    Memory in FFI defrun_query_which_will_crash! db_connection = MyFFIModule.database_connection("localhost")! MyFFIModule.database_query(db_connection, "select * from users")! end!
  • 32.
    Memory in FFI Thiswill get GCed def run_query_which_will_crash! db_connection = MyFFIModule.database_connection("localhost")! MyFFIModule.database_query(db_connection, "select * from users")! end!
  • 33.
    SWIG • Simplified Wrapper andInterface Generator • Annotate your C/C++ header files • It generates native extensions for languages • About 20 languages currently supported
  • 34.
  • 35.
    The Magic • Takes aninterface file • Auto generates code to make it work • For ruby, it’s a `regular’ C extension • For python, it’s a a .c and .py file • For Java it’s a JNI interface • Still need to do your own GC
  • 36.
    The Rectangle class Rectangle! {! intlength;! int breadth;! ! public:! Rectangle(int length, int breadth);! int area();! };
  • 37.
    The Rectangle #ifdef SWIG %moduleshape %{ SWIG Stuff Here class Rectangle! {! int length;! int breadth;! ! public:! Rectangle(int length, int breadth);! int area();! }; %}
  • 38.
    require 'shapes'! ! rectangle =shapes.Rectangle.new(10, 12)! rectangle.area == 120!
  • 39.
    Other Options • DL (DynamicLoad) • Fiddle(r)
  • 40.
    TL;DR • Native Extensions arefun and easy to build • The three big tools • You want to pick FFI if you don’t maintain the lib • SWIG may be better if you are a maintainer
  • 41.
    Thank You super integration wow soextension so native ruby = win wow such easy Many Questions? no python such performance