Small Node.js proxy to turn a paginated JSON REST API into a CSV streaming download. Examples of code and patterns. Presented at the London Node User Group meetup, April 2014

1. Streaming downloads
proxy service with
Node.js
ismael celis @ismasan

2. bootic.net - Hosted e-commerce in South America

3. background
job Email
attachment
Previous setup

4. Previous setup

5. Previous setup
• Memory limitations
• Email deliverability
• Code bloat
• inflexible

6. New setup
• Monolithic micro
• Leverage existing API

7. New setup
• Monolithic micro
• Leverage existing API
curl -H “Authorization: Bearer xxx”
https://api.bootic.net/v1/orders.json?created_at:gte=2014-02-01&page=2

9. New setup

10. API -> CSV Stream
// pipe generated CSV onto the HTTP response
var writer = csv.createCsvStreamWriter(response)
// Turn a series of paginated requests
// to the backend API into a stream of data
var stream = apistream.instance(uri, token)
// Pipe data stream into CSV writer
stream.pipe(writer)

11. API -> CSV Stream
response.setHeader('Content-Type', ‘text/csv');
response.setHeader('Content-disposition', 'attachment;filename=' + name + '.csv');

12. API -> mappers -> CSV Stream
{
"code": "123EFCD",
"total": 80000,
"status": "shipped",
"date": "2014-02-03",
"items": [
{"product_title": "iPhone 5", "units": 2, "unit_price": 30000},
{"product_title": "Samsung Galaxy S4", "units": 1, "unit_price": 20000}
]
}
code, total, date, status, product, units, unit_price, total
2 123EFCD, 80000, 2014-02-03, shipped, iPhone 5, 2, 30000, 80000
3 123EFCD, 80000, 2014-02-03, shipped, Samsung Galaxy S4, 1, 20000, 80000

13. API -> mappers -> CSV Stream
var OrderMapper = csvmapper.define(function () {
this.scope('items', function () {
this
.map('id', '/id')
.map('order', '/code')
.map('status', '/status')
.map('discount', '/discount_total')
.map('shipping price', '/shipping_total')
.map('total', '/total')
.map('year', '/updated_on', year)
.map('month', '/updated_on', month)
.map('day', '/updated_on', day)
.map('payment method', '/payment_method_type')
.map('name', '/contact/name')
.map('email', '/contact/email')
.map('address', '/address', address)
.map('product', 'product_title')
.map('variant', 'variant_title')
.map('sku', 'variant_sku')
.map('unit price', 'unit_price')

14. API -> mappers -> CSV Stream
var writer = csv.createCsvStreamWriter(res);
var stream = apistream.instance(uri, token)
var mapper = new OrdersMapper()
// First line in CSV is the headers
writer.writeRecord(mapper.headers())
// mapper.eachRow() turns a single API resource into 1 or more CSV rows
stream.on('item', function (item) {
mapper.eachRow(item, function (row) {
writer.writeRecord(row)
})
})

15. API -> mappers -> CSV Stream
stream.on('item', function (item) {
mapper.eachRow(item, function (row) {
writer.writeRecord(row)
})
})

16. Paremeter definitions
c.net/v1/orders.json? created_at:gte=2014-02-01 & page=2

17. Paremeter definitions
var OrdersParams = params.define(function () {
this
.param('sort', 'updated_on:desc')
.param('per_page', 20)
.param('status', 'closed,pending,invalid,shipped')
})

18. Paremeter definitions
var params = new OrdersParams(request.query)
// Compose API url using sanitized / defaulted params
var uri = "https://api.com/orders?" + params.query;
var stream = apistream.instance(uri, token)

19. Secure CSV downloads

20. JSON Web Tokens
headers . claims . signature
http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html
eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk

21. JSON Web Tokens
headers {“typ":"JWT",
"alg":"HS256"}
claims
{
“shop_id":"acme",
"iat":1300819380,
"aud":"orders",
"filters": {"status": "shipped"}
}
signature
+ Base64
+ Base64
HMAC SHA-256 (headers + claims, secret) + Base64

22. Rails: Generate token (Ruby)
# controllers/downloads_controller.rb
def create
url = Rails.application.config.downloads_host
claims = params[:download_options]
# Add an issued_at timestamp
claims[:iat] = (Time.now.getutc.to_f * 1000).to_i
# Scope data on current account
claims[“shop_id"] = current_shop.id
# generate JWT
token = JWT.encode(claims, Rails.application.config.downloads_secret)
# Redirect to download URL. Browser will trigger download dialog
redirect_to “#{url}?jwt=#{token}"
end

23. Rails: Generate token (Ruby)
claims[:iat] = (Time.now.getutc.to_f * 1000).to_i
claims[“shop_id"] = current_shop.id
token = JWT.encode(claims, secret)
redirect_to "#{url}?jwt=#{token}"

24. Node: validate JWT
var TTL = 60000;
var tokenMiddleware = function(req, res, next){
try{
var decoded = jwt.decode(req.query.jwt, secret);
if(decoded.shop_id != req.param(‘shop_id') {
res.send(400, ‘JWT and query shop ids do not match');
return
}
var now = new Date(),
utc = getUtcCurrentDate();
if(utc - Number(decoded.iat) > TTL) {
res.send(401, "Web token has expired")
return
}
req.query = decoded
// all good, carry on
next()
} catch(e) {
res.send(401, 'Unauthorized or invalid web token');
}
}

25. Node: validate JWT
var decoded = jwt.decode(req.query.jwt, secret);
?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMD.dBjftJeZ4CVP-mB92K27uhbUJU

26. Node: validate JWT
if(decoded.shop_id != req.param(‘shop_id') {
res.send(400, ‘JWT and query shop ids do not match');
return
}

27. Node: validate JWT
var now = new Date(),
utc = getUtcCurrentDate();
if(utc - Number(decoded.iat) > TTL) {
res.send(401, "Web token has expired")
return
}

28. Node: validate JWT
req.query = decoded
// all good, carry on
next()

29. Node: HTTP handlers
app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...));
app.get('/:shop_id/contacts.csv', tokenMiddleware, handler.create('contacts', ...));
app.get('/:shop_id/products.csv', tokenMiddleware, handler.create('products', ...));

30. }
goo.gl/nolmRK
ismael celis @ismasan