The Web map stack on Django
Paul Smith
http://www.pauladamsmith.com/ @paulsmith
EveryBlock
EuroDjangoCon ‘09

Data types

11 metros
Boston Philadelphia
● ●
Charlotte San Francisco
● ●
Chicago San Jose
● ●
Los Angeles Seattle
● ●
Miami Washington, DC
● ●
New York … and growing
●

Open source
This summer

Why?

Control design

Prioritize visualizations

The Web map stack

The Web map stack

The Web map stack

The Web map stack

The Web map stack

GeoDjango + Mapnik
example app
“Your Political Footprint”

Mapnik overview

# models.py
from django.contrib.gis.db import models
class CongressionalDistrict(models.Model):
state = models.ForeignKey(State)
name = models.CharField(max_length=32) # ex. 1st, 25th, at-large
number = models.IntegerField() # 0 if at-large
district = models.MultiPolygonField(srid=4326)
objects = models.GeoManager()
def __unicode__(self):
return '%s %s' % (self.state.name, self.name)
class Footprint(models.Model):
location = models.CharField(max_length=200)
point = models.PointField(srid=4326)
cong_dist = models.ForeignKey(CongressionalDistrict)
objects = models.GeoManager()
def __unicode__(self):
return '%s in %s' % (self.location, self.cong_dist)

GET /tile/?bbox=-112.5,22.5,-90,45

# urls.py
from django.conf import settings
from django.conf.urls.defaults import *
from edc_demo.footprint import views
urlpatterns = patterns('',
(r'^footprint/', views.political_footprint),
(r'^tile/', views.map_tile)
)

# views.py
from mapnik import *
from django.http import HttpResponse, Http404
from django.conf import settings
from edc_demo.footprint.models import CongressionalDistrict
TILE_WIDTH = TILE_HEIGHT = 256
TILE_MIMETYPE = 'image/png'
LIGHT_GREY = '#C0CCC4'
PGIS_DB_CONN = dict(
host=settings.DATABASE_HOST,
dbname=settings.DATABASE_NAME,
user=settings.DATABASE_USER,
password=settings.DATABASE_PASSWORD)
def map_tile(request):
if request.GET.has_key('bbox'):
bbox = [float(x) for x in request.GET['bbox'].split(',')]
tile = Map(TILE_WIDTH, TILE_HEIGHT)
rule = Rule()
rule.symbols.append(LineSymbolizer(Color(LIGHT_GREY), 1.0))
style = Style()
style.rules.append(rule)
tile.append_style('cong_dist', style)
layer = Layer('cong_dists')
db_table = CongressionalDistrict._meta.db_table
layer.datasource = PostGIS(table=db_table, **PGIS_DB_CONN)
layer.styles.append('cong_dist')
tile.layers.append(layer)
tile.zoom_to_box(Envelope(*bbox))
img = Image(tile.width, tile.height)
render(tile, img)
img_bytes = img.tostring(TILE_MIMETYPE.split('/')[1])
return HttpResponse(img_bytes, mimetype=TILE_MIMETYPE)
else:
raise Http404()

# views.py cont'd
from django.shortcuts import render_to_response
from edc_demo.footprint.geocoder import geocode
def political_footprint(request):
context = {}
if request.GET.has_key('location'):
point = geocode(request.GET['location'])
cd = CongressionalDistrict.objects.get(district__contains=point)
footprint = Footprint.objects.create(
location = request.GET['location'],
point = point,
cong_dist = cd
)
context['footprint'] = footprint
context['cd_bbox'] = cong_dist.district.extent
return render_to_response('footprint.html', context)

// footprint.html
<script type=\"text/javascript\">
var map;
var TileLayerClass = OpenLayers.Class(OpenLayers.Layer.TMS, {
initialize: function(footprint_id) {
var name = \"tiles\";
var url = \"http://127.0.0.1:8000/tile/\";
var args = [];
args.push(name, url, {}, {});
OpenLayers.Layer.Grid.prototype.initialize.apply(this, args);
this.footprint_id = footprint_id;
},
getURL: function(bounds) {
var url = this.url + \"?bbox=\" + bounds.toBBOX();
if (this.footprint_id)
url += \"&fp_id=\" + this.footprint_id;
return url;
}
});
function onload() {
var options = {
minScale: 19660800,
numZoomLevels: 14,
units: \"degrees\"
};
map = new OpenLayers.Map(\"map\");
{% if not footprint %}
var bbox = new OpenLayers.Bounds(-126.298828, 17.578125, -64.775391, 57.128906);
var tileLayer = new TileLayerClass();
{% else %}
var bbox = new OpenLayers.Bounds({{ cd_bbox|join:\", \" }});
var tileLayer = new TileLayerClass({{ footprint.id }});
{% endif %}
map.addLayer(tileLayer);
map.zoomToExtent(bbox);
}

# views.py
from edc_demo.footprint.models import Footprint
def map_tile(request):
if request.GET.has_key('bbox'):
bbox = [float(x) for x in request.GET['bbox'].split(',')]
tile = Map(TILE_WIDTH, TILE_HEIGHT)
rule = Rule()
rule.symbols.append(LineSymbolizer(Color(LIGHT_GREY), 1.0))
style = Style()
style.rules.append(rule)
if request.GET.has_key('fp_id'):
footprint = Footprint.objects.get(pk=request.GET['fp_id'])
rule = Rule()
rule.symbols.append(LineSymbolizer(Color(GREEN), 1.0))
rule.symbols.append(PolygonSymbolizer(Color(LIGHT_GREEN)))
rule.filter = Filter('[id] = ' + str(footprint.cong_dist.id))
style.rules.append(rule)
tile.append_style('cong_dist', style)
layer = Layer('cong_dists')
db_table = CongressionalDistrict._meta.db_table
layer.datasource = PostGIS(table=db_table, **PGIS_DB_CONN)
layer.styles.append('cong_dist')
tile.layers.append(layer)
if request.GET.has_key('fp_id'):
add_footprint_layer(tile, footprint)
tile.zoom_to_box(Envelope(*bbox))
img = Image(tile.width, tile.height)
render(tile, img)
img_bytes = img.tostring(TILE_MIMETYPE.split('/')[1])
return HttpResponse(img_bytes, mimetype=TILE_MIMETYPE)
else:
raise Http404()

# views.py cont'd
def add_footprint_layer(tile, footprint):
rule = Rule()
rule.symbols.append(
PointSymbolizer(
os.path.join(settings.STATIC_MEDIA_DIR, 'img', 'footprint.png'),
'png', 46, 46)
)
rule.filter = Filter('[id] = ' + str(footprint.id))
style = Style()
style.rules.append(rule)
tile.append_style('footprint', style)
layer = Layer('footprint')
layer.datasource = PostGIS(table=Footprint._meta.db_table, **PGIS_DB_CONN)
layer.styles.append('footprint')
tile.layers.append(layer)

Serving tiles

Zoom levels

Tile example
z: 5, x: 2384, y: 1352

TileCache
pro con
Cache population Python overhead
● ●
integrated with (rendering, serving)
request/response cycle
Flexible storage
●

Pre-render + custom nginx mod
pro con
Fast responses Render everything in
● ●
advance
Parallelizable, offline
●
rendering C module inflexibility
●
(esp. storage backends)

Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)

Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)

Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)

Tile rendering
for each zoom level z:
for each column x:
for each row y:
render tile (x, y, z)

# nginx.conf
server {
server_name tile.example.com
root /var/www/maptiles;
expires max;
location ~* ^/[^/]+/\\w+/\\d+/\\d+,\\d+\\.(jpg|gif|png)$ {
tilecache;
}
}

// ngx_tilecache_mod.c
/*
* This struct holds the attributes that uniquely identify a map tile.
*/
typedef struct {
u_char *version;
u_char *name;
int x;
int y;
int z;
u_char *ext;
} tilecache_tile_t;
/*
* The following regex pattern matches the request URI for a tile and
* creates capture groups for the tile attributes. Example request URI:
*
* /1.0/main/8/654,23.png
*
* would map to the following attributes:
*
* version: 1.0
* name: main
* z: 8
* x: 654
* y: 23
* extension: png
*/
static ngx_str_t tile_request_pat = ngx_string(\"^/([^/]+)/([^/]+)/([0-9]+)/([0-9]+),([0-9]+)\\\\.([a-z]+)$\");

// ngx_tilecache_mod.c
u_char *
get_disk_key(u_char *s, u_char *name, int x, int y, int z, u_char *ext)
{
u_int a, b, c, d, e, f;
a = x / 100000;
b = (x / 1000) % 1000;
c = x % 1000;
d = y / 100000;
e = (y / 1000) % 1000;
f = y % 1000;
return ngx_sprintf(s, \"/%s/%02d/%03d/%03d/%03d/%03d/%03d/%03d.%s\",
name, z, a, b, c, d, e, f, ext);
}
static ngx_int_t
ngx_tilecache_handler(ngx_http_request_t *r)
{
// ... snip ...
sub_uri.data = ngx_pcalloc(r->pool, len + 1);
if (sub_uri.data == NULL) {
return NGX_ERROR;
}
get_disk_key(sub_uri.data, tile->name, tile->x, tile->y, tile->z, tile->ext);
sub_uri.len = ngx_strlen(sub_uri.data);
return ngx_http_internal_redirect(r, &sub_uri, &r->args);
}

Custom tile cache
technique responsibility
Far-future expiry Tile versions for cache
● ●
header expires max; invalidation

// everyblock.js
eb.TileLayer = OpenLayers.Class(OpenLayers.Layer.TMS, {
version: null, // see eb.TILE_VERSION
layername: null, // lower-cased: \"main\", \"locator\"
type: null, // i.e., mime-type extension: \"png\", \"jpg\", \"gif\"
initialize: function(name, url, options) {
var args = [];
args.push(name, url, {}, options);
OpenLayers.Layer.TMS.prototype.initialize.apply(this, args);
},
// Returns an object with the x, y, and z of a tile for a given bounds
getCoordinate: function(bounds) {
bounds = this.adjustBounds(bounds);
var res = this.map.getResolution();
var x = Math.round((bounds.left - this.tileOrigin.lon) / (res * this.tileSize.w));
var y = Math.round((bounds.bottom - this.tileOrigin.lat) / (res * this.tileSize.h));
var z = this.map.getZoom();
return {x: x, y: y, z: z};
},
getPath: function(x, y, z) {
return this.version + \"/\" + this.layername + \"/\" + z + \"/\" + x + \",\" + y + \".\" +
this.type;
},
getURL: function(bounds) {
var coord = this.getCoordinate(bounds);
var path = this.getPath(coord.x, coord.y, coord.z);
var url = this.url;
if (url instanceof Array)
url = this.selectUrl(path, url);
return url + path;
},
CLASS_NAME: \"eb.TileLayer\"
});

Clustering

# cluster.py
import math
from everyblock.maps.clustering.models import Bunch
def euclidean_distance(a, b):
return math.hypot(a[0] - b[0], a[1] - b[1])
def buffer_cluster(objects, radius, dist_fn=euclidean_distance):
bunches = []
buffer_ = radius
for key, point in objects.iteritems():
bunched = False
for bunch in bunches:
if dist_fn(point, bunch.center) <= buffer_:
bunch.add_obj(key, point)
bunched = True
break
if not bunched:
bunches.append(Bunch(key, point))
return bunches

# bunch.py
class Bunch(object):
def __init__(self, obj, point):
self.objects = []
self.points = []
self.center = (0, 0)
self.add_obj(obj, point)
def add_obj(self, obj, point):
self.objects.append(obj)
self.points.append(point)
self.update_center(point)
def update_center(self, point):
xs = [p[0] for p in self.points]
ys = [p[1] for p in self.points]
self.center = (sum(xs) * 1.0 / len(self.objects),
sum(ys) * 1.0 / len(self.objects))

# cluster_scale.py
from everyblock.maps import utils
from everyblock.maps.clustering import cluster
def cluster_by_scale(objs, radius, scale, extent=(-180, -90, 180, 90)):
resolution = utils.get_resolution(scale)
# Translate from lng/lat into coordinate system of the display.
objs = dict([(k, utils.px_from_lnglat(v, resolution, extent))
for k, v in objs.iteritems()])
bunches = []
for bunch in cluster.buffer_cluster(objs, radius):
# Translate back into lng/lat.
bunch.center = utils.lnglat_from_px(bunch.center,
resolution,
extent)
bunches.append(bunch)
return bunches

Sneak peek

Sneak peek

Sneak peek

Thank you
http://www.pauladamsmith.com/
@paulsmith
paulsmith@gmail.com
Further exploration:
“How to Lie with Maps”
Mark Monmonier