My EuroDjangoCon talk about how EveryBlock has used Mapnik and GeoDjango to create our own maps.

1. The Web map stack on Django
Paul Smith
http://www.pauladamsmith.com/ @paulsmith
EveryBlock
EuroDjangoCon ‘09

3. Data types

9. 11 metros
Boston Philadelphia
● ●
Charlotte San Francisco
● ●
Chicago San Jose
● ●
Los Angeles Seattle
● ●
Miami Washington, DC
● ●
New York … and growing
●

10. Open source
This summer

11. Why?

12. Control design

13. Prioritize visualizations

14. The Web map stack

15. The Web map stack

16. The Web map stack

17. The Web map stack

18. The Web map stack

19. GeoDjango + Mapnik
example app
“Your Political Footprint”

20. Mapnik overview

23. # models.py
from django.contrib.gis.db import models
class CongressionalDistrict(models.Model):
state = models.ForeignKey(State)
name = models.CharField(max_length=32) # ex. 1st, 25th, at-large
number = models.IntegerField() # 0 if at-large
district = models.MultiPolygonField(srid=4326)
objects = models.GeoManager()
def __unicode__(self):
return '%s %s' % (self.state.name, self.name)
class Footprint(models.Model):
location = models.CharField(max_length=200)
point = models.PointField(srid=4326)
cong_dist = models.ForeignKey(CongressionalDistrict)
objects = models.GeoManager()
def __unicode__(self):
return '%s in %s' % (self.location, self.cong_dist)

25. GET /tile/?bbox=-112.5,22.5,-90,45

26. # urls.py
from django.conf import settings
from django.conf.urls.defaults import *
from edc_demo.footprint import views
urlpatterns = patterns('',
(r'^footprint/', views.political_footprint),
(r'^tile/', views.map_tile)
)

27. # views.py
from mapnik import *
from django.http import HttpResponse, Http404
from django.conf import settings
from edc_demo.footprint.models import CongressionalDistrict
TILE_WIDTH = TILE_HEIGHT = 256
TILE_MIMETYPE = 'image/png'
LIGHT_GREY = '#C0CCC4'
PGIS_DB_CONN = dict(
host=settings.DATABASE_HOST,
dbname=settings.DATABASE_NAME,
user=settings.DATABASE_USER,
password=settings.DATABASE_PASSWORD)
def map_tile(request):
if request.GET.has_key('bbox'):
bbox = [float(x) for x in request.GET['bbox'].split(',')]
tile = Map(TILE_WIDTH, TILE_HEIGHT)
rule = Rule()
rule.symbols.append(LineSymbolizer(Color(LIGHT_GREY), 1.0))
style = Style()
style.rules.append(rule)
tile.append_style('cong_dist', style)
layer = Layer('cong_dists')
db_table = CongressionalDistrict._meta.db_table
layer.datasource = PostGIS(table=db_table, **PGIS_DB_CONN)
layer.styles.append('cong_dist')
tile.layers.append(layer)
tile.zoom_to_box(Envelope(*bbox))
img = Image(tile.width, tile.height)
render(tile, img)
img_bytes = img.tostring(TILE_MIMETYPE.split('/')[1])
return HttpResponse(img_bytes, mimetype=TILE_MIMETYPE)
else:
raise Http404()

28. # views.py cont'd
from django.shortcuts import render_to_response
from edc_demo.footprint.geocoder import geocode
def political_footprint(request):
context = {}
if request.GET.has_key('location'):
point = geocode(request.GET['location'])
cd = CongressionalDistrict.objects.get(district__contains=point)
footprint = Footprint.objects.create(
location = request.GET['location'],
point = point,
cong_dist = cd
)
context['footprint'] = footprint
context['cd_bbox'] = cong_dist.district.extent
return render_to_response('footprint.html', context)

29. // footprint.html
<script type=quot;text/javascriptquot;>
var map;
var TileLayerClass = OpenLayers.Class(OpenLayers.Layer.TMS, {
initialize: function(footprint_id) {
var name = quot;tilesquot;;
var url = quot;http://127.0.0.1:8000/tile/quot;;
var args = [];
args.push(name, url, {}, {});
OpenLayers.Layer.Grid.prototype.initialize.apply(this, args);
this.footprint_id = footprint_id;
},
getURL: function(bounds) {
var url = this.url + quot;?bbox=quot; + bounds.toBBOX();
if (this.footprint_id)
url += quot;&fp_id=quot; + this.footprint_id;
return url;
}
});
function onload() {
var options = {
minScale: 19660800,
numZoomLevels: 14,
units: quot;degreesquot;
};
map = new OpenLayers.Map(quot;mapquot;);
{% if not footprint %}
var bbox = new OpenLayers.Bounds(-126.298828, 17.578125, -64.775391, 57.128906);
var tileLayer = new TileLayerClass();
{% else %}
var bbox = new OpenLayers.Bounds({{ cd_bbox|join:quot;, quot; }});
var tileLayer = new TileLayerClass({{ footprint.id }});
{% endif %}
map.addLayer(tileLayer);
map.zoomToExtent(bbox);
}

30. # views.py
from edc_demo.footprint.models import Footprint
def map_tile(request):
if request.GET.has_key('bbox'):
bbox = [float(x) for x in request.GET['bbox'].split(',')]
tile = Map(TILE_WIDTH, TILE_HEIGHT)
rule = Rule()
rule.symbols.append(LineSymbolizer(Color(LIGHT_GREY), 1.0))
style = Style()
style.rules.append(rule)
if request.GET.has_key('fp_id'):
footprint = Footprint.objects.get(pk=request.GET['fp_id'])
rule = Rule()
rule.symbols.append(LineSymbolizer(Color(GREEN), 1.0))
rule.symbols.append(PolygonSymbolizer(Color(LIGHT_GREEN)))
rule.filter = Filter('[id] = ' + str(footprint.cong_dist.id))
style.rules.append(rule)
tile.append_style('cong_dist', style)
layer = Layer('cong_dists')
db_table = CongressionalDistrict._meta.db_table
layer.datasource = PostGIS(table=db_table, **PGIS_DB_CONN)
layer.styles.append('cong_dist')
tile.layers.append(layer)
if request.GET.has_key('fp_id'):
add_footprint_layer(tile, footprint)
tile.zoom_to_box(Envelope(*bbox))
img = Image(tile.width, tile.height)
render(tile, img)
img_bytes = img.tostring(TILE_MIMETYPE.split('/')[1])
return HttpResponse(img_bytes, mimetype=TILE_MIMETYPE)
else:
raise Http404()

31. # views.py cont'd
def add_footprint_layer(tile, footprint):
rule = Rule()
rule.symbols.append(
PointSymbolizer(
os.path.join(settings.STATIC_MEDIA_DIR, 'img', 'footprint.png'),
'png', 46, 46)
)
rule.filter = Filter('[id] = ' + str(footprint.id))
style = Style()
style.rules.append(rule)
tile.append_style('footprint', style)
layer = Layer('footprint')
layer.datasource = PostGIS(table=Footprint._meta.db_table, **PGIS_DB_CONN)
layer.styles.append('footprint')
tile.layers.append(layer)

33. Serving tiles

34. Zoom levels

35. Tile example
z: 5, x: 2384, y: 1352

36. TileCache
pro con
Cache population Python overhead
● ●
integrated with (rendering, serving)
request/response cycle
Flexible storage
●

37. Pre-render + custom nginx mod
pro con
Fast responses Render everything in
● ●
advance
Parallelizable, offline
●
rendering C module inflexibility
●
(esp. storage backends)

38. Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)

39. Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)

40. Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)

41. Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)

42. # nginx.conf
server {
server_name tile.example.com
root /var/www/maptiles;
expires max;
location ~* ^/[^/]+/w+/d+/d+,d+.(jpg|gif|png)$ {
tilecache;
}
}

43. // ngx_tilecache_mod.c
/*
* This struct holds the attributes that uniquely identify a map tile.
*/
typedef struct {
u_char *version;
u_char *name;
int x;
int y;
int z;
u_char *ext;
} tilecache_tile_t;
/*
* The following regex pattern matches the request URI for a tile and
* creates capture groups for the tile attributes. Example request URI:
*
* /1.0/main/8/654,23.png
*
* would map to the following attributes:
*
* version: 1.0
* name: main
* z: 8
* x: 654
* y: 23
* extension: png
*/
static ngx_str_t tile_request_pat = ngx_string(quot;^/([^/]+)/([^/]+)/([0-9]+)/([0-9]+),([0-9]+).([a-z]+)$quot;);

44. // ngx_tilecache_mod.c
u_char *
get_disk_key(u_char *s, u_char *name, int x, int y, int z, u_char *ext)
{
u_int a, b, c, d, e, f;
a = x / 100000;
b = (x / 1000) % 1000;
c = x % 1000;
d = y / 100000;
e = (y / 1000) % 1000;
f = y % 1000;
return ngx_sprintf(s, quot;/%s/%02d/%03d/%03d/%03d/%03d/%03d/%03d.%squot;,
name, z, a, b, c, d, e, f, ext);
}
static ngx_int_t
ngx_tilecache_handler(ngx_http_request_t *r)
{
// ... snip ...
sub_uri.data = ngx_pcalloc(r->pool, len + 1);
if (sub_uri.data == NULL) {
return NGX_ERROR;
}
get_disk_key(sub_uri.data, tile->name, tile->x, tile->y, tile->z, tile->ext);
sub_uri.len = ngx_strlen(sub_uri.data);
return ngx_http_internal_redirect(r, &sub_uri, &r->args);
}

45. Custom tile cache
technique responsibility
Far-future expiry Tile versions for cache
● ●
header expires max; invalidation

46. // everyblock.js
eb.TileLayer = OpenLayers.Class(OpenLayers.Layer.TMS, {
version: null, // see eb.TILE_VERSION
layername: null, // lower-cased: quot;mainquot;, quot;locatorquot;
type: null, // i.e., mime-type extension: quot;pngquot;, quot;jpgquot;, quot;gifquot;
initialize: function(name, url, options) {
var args = [];
args.push(name, url, {}, options);
OpenLayers.Layer.TMS.prototype.initialize.apply(this, args);
},
// Returns an object with the x, y, and z of a tile for a given bounds
getCoordinate: function(bounds) {
bounds = this.adjustBounds(bounds);
var res = this.map.getResolution();
var x = Math.round((bounds.left - this.tileOrigin.lon) / (res * this.tileSize.w));
var y = Math.round((bounds.bottom - this.tileOrigin.lat) / (res * this.tileSize.h));
var z = this.map.getZoom();
return {x: x, y: y, z: z};
},
getPath: function(x, y, z) {
return this.version + quot;/quot; + this.layername + quot;/quot; + z + quot;/quot; + x + quot;,quot; + y + quot;.quot; +
this.type;
},
getURL: function(bounds) {
var coord = this.getCoordinate(bounds);
var path = this.getPath(coord.x, coord.y, coord.z);
var url = this.url;
if (url instanceof Array)
url = this.selectUrl(path, url);
return url + path;
},
CLASS_NAME: quot;eb.TileLayerquot;
});

47. Clustering

50. # cluster.py
import math
from everyblock.maps.clustering.models import Bunch
def euclidean_distance(a, b):
return math.hypot(a[0] - b[0], a[1] - b[1])
def buffer_cluster(objects, radius, dist_fn=euclidean_distance):
bunches = []
buffer_ = radius
for key, point in objects.iteritems():
bunched = False
for bunch in bunches:
if dist_fn(point, bunch.center) <= buffer_:
bunch.add_obj(key, point)
bunched = True
break
if not bunched:
bunches.append(Bunch(key, point))
return bunches

51. # bunch.py
class Bunch(object):
def __init__(self, obj, point):
self.objects = []
self.points = []
self.center = (0, 0)
self.add_obj(obj, point)
def add_obj(self, obj, point):
self.objects.append(obj)
self.points.append(point)
self.update_center(point)
def update_center(self, point):
xs = [p[0] for p in self.points]
ys = [p[1] for p in self.points]
self.center = (sum(xs) * 1.0 / len(self.objects),
sum(ys) * 1.0 / len(self.objects))

52. # cluster_scale.py
from everyblock.maps import utils
from everyblock.maps.clustering import cluster
def cluster_by_scale(objs, radius, scale, extent=(-180, -90, 180, 90)):
resolution = utils.get_resolution(scale)
# Translate from lng/lat into coordinate system of the display.
objs = dict([(k, utils.px_from_lnglat(v, resolution, extent))
for k, v in objs.iteritems()])
bunches = []
for bunch in cluster.buffer_cluster(objs, radius):
# Translate back into lng/lat.
bunch.center = utils.lnglat_from_px(bunch.center,
resolution,
extent)
bunches.append(bunch)
return bunches

53. Sneak peek

54. Sneak peek

55. Sneak peek

56. Thank you
http://www.pauladamsmith.com/
@paulsmith
paulsmith@gmail.com
Further exploration:
“How to Lie with Maps”
Mark Monmonier