Streaming downloads
proxy service with
Node.js
ismael celis @ismasan
bootic.net - Hosted e-commerce in South America
background
job Email
attachment
Previous setup
Previous setup
Previous setup
• Memory limitations
• Email deliverability
• Code bloat
• inflexible
New setup
• Monolithic micro
• Leverage existing API
New setup
• Monolithic micro
• Leverage existing API
curl -H “Authorization: Bearer xxx” 	
https://api.bootic.net/v1/order...
New setup
API -> CSV Stream
// pipe generated CSV onto the HTTP response

var writer = csv.createCsvStreamWriter(response)



// Tur...
API -> CSV Stream
response.setHeader('Content-Type', ‘text/csv');


response.setHeader('Content-disposition', 'attachment;...
API -> mappers -> CSV Stream
{

"code": "123EFCD",

"total": 80000,

"status": "shipped",

"date": "2014-02-03",

"items":...
API -> mappers -> CSV Stream
var OrderMapper = csvmapper.define(function () {



this.scope('items', function () {

this

...
API -> mappers -> CSV Stream
var writer = csv.createCsvStreamWriter(res);



var stream = apistream.instance(uri, token)

...
API -> mappers -> CSV Stream
stream.on('item', function (item) {

mapper.eachRow(item, function (row) {

writer.writeRecor...
Paremeter definitions
c.net/v1/orders.json? created_at:gte=2014-02-01 & page=2
Paremeter definitions
var OrdersParams = params.define(function () {

this

.param('sort', 'updated_on:desc')

.param('per_...
Paremeter definitions
var params = new OrdersParams(request.query)



// Compose API url using sanitized / defaulted params...
Secure CSV downloads
JSON Web Tokens
headers . claims . signature
http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html
eyJ0eXAiOiJ...
JSON Web Tokens
headers {“typ":"JWT",
"alg":"HS256"}
claims
{

“shop_id":"acme",

"iat":1300819380,

"aud":"orders",

"fil...
Rails: Generate token (Ruby)
# controllers/downloads_controller.rb

def create


url = Rails.application.config.downloads_...
Rails: Generate token (Ruby)
claims[:iat] = (Time.now.getutc.to_f * 1000).to_i



claims[“shop_id"] = current_shop.id



t...
Node: validate JWT
var TTL = 60000;



var tokenMiddleware = function(req, res, next){

try{

var decoded = jwt.decode(req...
Node: validate JWT
var decoded = jwt.decode(req.query.jwt, secret);
?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiO...
Node: validate JWT
if(decoded.shop_id != req.param(‘shop_id') {

res.send(400, ‘JWT and query shop ids do not match');

re...
Node: validate JWT
var now = new Date(),

utc = getUtcCurrentDate();



if(utc - Number(decoded.iat) > TTL) {

res.send(40...
Node: validate JWT
req.query = decoded


// all good, carry on

next()
Node: HTTP handlers
app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...));
app.get('/:shop_id/co...
}
goo.gl/nolmRK
ismael celis @ismasan
Node.js streaming csv downloads proxy
Upcoming SlideShare
Loading in...5
×

Node.js streaming csv downloads proxy

3,710

Published on

Small Node.js proxy to turn a paginated JSON REST API into a CSV streaming download. Examples of code and patterns.

Presented at the London Node User Group meetup, April 2014

Published in: Software, Technology
0 Comments
2 Likes
Statistics
Notes
  • Be the first to comment

No Downloads
Views
Total Views
3,710
On Slideshare
0
From Embeds
0
Number of Embeds
15
Actions
Shares
0
Downloads
13
Comments
0
Likes
2
Embeds 0
No embeds

No notes for slide

Node.js streaming csv downloads proxy

  1. 1. Streaming downloads proxy service with Node.js ismael celis @ismasan
  2. 2. bootic.net - Hosted e-commerce in South America
  3. 3. background job Email attachment Previous setup
  4. 4. Previous setup
  5. 5. Previous setup • Memory limitations • Email deliverability • Code bloat • inflexible
  6. 6. New setup • Monolithic micro • Leverage existing API
  7. 7. New setup • Monolithic micro • Leverage existing API curl -H “Authorization: Bearer xxx” https://api.bootic.net/v1/orders.json?created_at:gte=2014-02-01&page=2
  8. 8. New setup
  9. 9. API -> CSV Stream // pipe generated CSV onto the HTTP response
 var writer = csv.createCsvStreamWriter(response)
 
 // Turn a series of paginated requests 
 // to the backend API into a stream of data
 var stream = apistream.instance(uri, token)
 
 // Pipe data stream into CSV writer
 stream.pipe(writer)
  10. 10. API -> CSV Stream response.setHeader('Content-Type', ‘text/csv'); 
 response.setHeader('Content-disposition', 'attachment;filename=' + name + '.csv');
  11. 11. API -> mappers -> CSV Stream {
 "code": "123EFCD",
 "total": 80000,
 "status": "shipped",
 "date": "2014-02-03",
 "items": [
 {"product_title": "iPhone 5", "units": 2, "unit_price": 30000},
 {"product_title": "Samsung Galaxy S4", "units": 1, "unit_price": 20000}
 ]
 } code, total, date, status, product, units, unit_price, total
 2 123EFCD, 80000, 2014-02-03, shipped, iPhone 5, 2, 30000, 80000
 3 123EFCD, 80000, 2014-02-03, shipped, Samsung Galaxy S4, 1, 20000, 80000
  12. 12. API -> mappers -> CSV Stream var OrderMapper = csvmapper.define(function () {
 
 this.scope('items', function () {
 this
 .map('id', '/id')
 .map('order', '/code')
 .map('status', '/status')
 .map('discount', '/discount_total')
 .map('shipping price', '/shipping_total')
 .map('total', '/total')
 .map('year', '/updated_on', year)
 .map('month', '/updated_on', month)
 .map('day', '/updated_on', day)
 .map('payment method', '/payment_method_type')
 .map('name', '/contact/name')
 .map('email', '/contact/email')
 .map('address', '/address', address)
 .map('product', 'product_title')
 .map('variant', 'variant_title')
 .map('sku', 'variant_sku')
 .map('unit price', 'unit_price')

  13. 13. API -> mappers -> CSV Stream var writer = csv.createCsvStreamWriter(res);
 
 var stream = apistream.instance(uri, token)
 
 var mapper = new OrdersMapper()
 
 // First line in CSV is the headers
 writer.writeRecord(mapper.headers())
 
 // mapper.eachRow() turns a single API resource into 1 or more CSV rows
 stream.on('item', function (item) {
 mapper.eachRow(item, function (row) {
 writer.writeRecord(row)
 })
 })
  14. 14. API -> mappers -> CSV Stream stream.on('item', function (item) {
 mapper.eachRow(item, function (row) {
 writer.writeRecord(row)
 })
 })
  15. 15. Paremeter definitions c.net/v1/orders.json? created_at:gte=2014-02-01 & page=2
  16. 16. Paremeter definitions var OrdersParams = params.define(function () {
 this
 .param('sort', 'updated_on:desc')
 .param('per_page', 20)
 .param('status', 'closed,pending,invalid,shipped')
 })
  17. 17. Paremeter definitions var params = new OrdersParams(request.query)
 
 // Compose API url using sanitized / defaulted params
 var uri = "https://api.com/orders?" + params.query;
 
 var stream = apistream.instance(uri, token)
  18. 18. Secure CSV downloads
  19. 19. JSON Web Tokens headers . claims . signature http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk
  20. 20. JSON Web Tokens headers {“typ":"JWT", "alg":"HS256"} claims {
 “shop_id":"acme",
 "iat":1300819380,
 "aud":"orders",
 "filters": {"status": "shipped"}
 } signature + Base64 + Base64 HMAC SHA-256 (headers + claims, secret) + Base64
  21. 21. Rails: Generate token (Ruby) # controllers/downloads_controller.rb
 def create 
 url = Rails.application.config.downloads_host
 claims = params[:download_options]
 # Add an issued_at timestamp
 claims[:iat] = (Time.now.getutc.to_f * 1000).to_i
 # Scope data on current account
 claims[“shop_id"] = current_shop.id
 # generate JWT
 token = JWT.encode(claims, Rails.application.config.downloads_secret)
 
 # Redirect to download URL. Browser will trigger download dialog
 redirect_to “#{url}?jwt=#{token}" 
 end
  22. 22. Rails: Generate token (Ruby) claims[:iat] = (Time.now.getutc.to_f * 1000).to_i
 
 claims[“shop_id"] = current_shop.id
 
 token = JWT.encode(claims, secret)
 
 redirect_to "#{url}?jwt=#{token}"
  23. 23. Node: validate JWT var TTL = 60000;
 
 var tokenMiddleware = function(req, res, next){
 try{
 var decoded = jwt.decode(req.query.jwt, secret);
 
 if(decoded.shop_id != req.param(‘shop_id') {
 res.send(400, ‘JWT and query shop ids do not match');
 return
 }
 
 var now = new Date(),
 utc = getUtcCurrentDate();
 
 if(utc - Number(decoded.iat) > TTL) {
 res.send(401, "Web token has expired")
 return
 }
 
 req.query = decoded
 // all good, carry on
 next()
 
 } catch(e) {
 res.send(401, 'Unauthorized or invalid web token');
 }
 }
  24. 24. Node: validate JWT var decoded = jwt.decode(req.query.jwt, secret); ?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMD.dBjftJeZ4CVP-mB92K27uhbUJU
  25. 25. Node: validate JWT if(decoded.shop_id != req.param(‘shop_id') {
 res.send(400, ‘JWT and query shop ids do not match');
 return
 }

  26. 26. Node: validate JWT var now = new Date(),
 utc = getUtcCurrentDate();
 
 if(utc - Number(decoded.iat) > TTL) {
 res.send(401, "Web token has expired")
 return
 }
  27. 27. Node: validate JWT req.query = decoded 
 // all good, carry on
 next()
  28. 28. Node: HTTP handlers app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...)); app.get('/:shop_id/contacts.csv', tokenMiddleware, handler.create('contacts', ...)); app.get('/:shop_id/products.csv', tokenMiddleware, handler.create('products', ...));
  29. 29. } goo.gl/nolmRK ismael celis @ismasan
  1. A particular slide catching your eye?

    Clipping is a handy way to collect important slides you want to go back to later.

×