Your SlideShare is downloading. ×
0
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Node.js streaming csv downloads proxy
Upcoming SlideShare
Loading in...5
×

Thanks for flagging this SlideShare!

Oops! An error has occurred.

×
Saving this for later? Get the SlideShare app to save on your phone or tablet. Read anywhere, anytime – even offline.
Text the download link to your phone
Standard text messaging rates apply

Node.js streaming csv downloads proxy

3,321

Published on

Small Node.js proxy to turn a paginated JSON REST API into a CSV streaming download. Examples of code and patterns. …

Small Node.js proxy to turn a paginated JSON REST API into a CSV streaming download. Examples of code and patterns.

Presented at the London Node User Group meetup, April 2014

Published in: Software, Technology
0 Comments
2 Likes
Statistics
Notes
  • Be the first to comment

No Downloads
Views
Total Views
3,321
On Slideshare
0
From Embeds
0
Number of Embeds
15
Actions
Shares
0
Downloads
13
Comments
0
Likes
2
Embeds 0
No embeds

Report content
Flagged as inappropriate Flag as inappropriate
Flag as inappropriate

Select your reason for flagging this presentation as inappropriate.

Cancel
No notes for slide

Transcript

  • 1. Streaming downloads proxy service with Node.js ismael celis @ismasan
  • 2. bootic.net - Hosted e-commerce in South America
  • 3. background job Email attachment Previous setup
  • 4. Previous setup
  • 5. Previous setup • Memory limitations • Email deliverability • Code bloat • inflexible
  • 6. New setup • Monolithic micro • Leverage existing API
  • 7. New setup • Monolithic micro • Leverage existing API curl -H “Authorization: Bearer xxx” https://api.bootic.net/v1/orders.json?created_at:gte=2014-02-01&page=2
  • 8. New setup
  • 9. API -> CSV Stream // pipe generated CSV onto the HTTP response
 var writer = csv.createCsvStreamWriter(response)
 
 // Turn a series of paginated requests 
 // to the backend API into a stream of data
 var stream = apistream.instance(uri, token)
 
 // Pipe data stream into CSV writer
 stream.pipe(writer)
  • 10. API -> CSV Stream response.setHeader('Content-Type', ‘text/csv'); 
 response.setHeader('Content-disposition', 'attachment;filename=' + name + '.csv');
  • 11. API -> mappers -> CSV Stream {
 "code": "123EFCD",
 "total": 80000,
 "status": "shipped",
 "date": "2014-02-03",
 "items": [
 {"product_title": "iPhone 5", "units": 2, "unit_price": 30000},
 {"product_title": "Samsung Galaxy S4", "units": 1, "unit_price": 20000}
 ]
 } code, total, date, status, product, units, unit_price, total
 2 123EFCD, 80000, 2014-02-03, shipped, iPhone 5, 2, 30000, 80000
 3 123EFCD, 80000, 2014-02-03, shipped, Samsung Galaxy S4, 1, 20000, 80000
  • 12. API -> mappers -> CSV Stream var OrderMapper = csvmapper.define(function () {
 
 this.scope('items', function () {
 this
 .map('id', '/id')
 .map('order', '/code')
 .map('status', '/status')
 .map('discount', '/discount_total')
 .map('shipping price', '/shipping_total')
 .map('total', '/total')
 .map('year', '/updated_on', year)
 .map('month', '/updated_on', month)
 .map('day', '/updated_on', day)
 .map('payment method', '/payment_method_type')
 .map('name', '/contact/name')
 .map('email', '/contact/email')
 .map('address', '/address', address)
 .map('product', 'product_title')
 .map('variant', 'variant_title')
 .map('sku', 'variant_sku')
 .map('unit price', 'unit_price')

  • 13. API -> mappers -> CSV Stream var writer = csv.createCsvStreamWriter(res);
 
 var stream = apistream.instance(uri, token)
 
 var mapper = new OrdersMapper()
 
 // First line in CSV is the headers
 writer.writeRecord(mapper.headers())
 
 // mapper.eachRow() turns a single API resource into 1 or more CSV rows
 stream.on('item', function (item) {
 mapper.eachRow(item, function (row) {
 writer.writeRecord(row)
 })
 })
  • 14. API -> mappers -> CSV Stream stream.on('item', function (item) {
 mapper.eachRow(item, function (row) {
 writer.writeRecord(row)
 })
 })
  • 15. Paremeter definitions c.net/v1/orders.json? created_at:gte=2014-02-01 & page=2
  • 16. Paremeter definitions var OrdersParams = params.define(function () {
 this
 .param('sort', 'updated_on:desc')
 .param('per_page', 20)
 .param('status', 'closed,pending,invalid,shipped')
 })
  • 17. Paremeter definitions var params = new OrdersParams(request.query)
 
 // Compose API url using sanitized / defaulted params
 var uri = "https://api.com/orders?" + params.query;
 
 var stream = apistream.instance(uri, token)
  • 18. Secure CSV downloads
  • 19. JSON Web Tokens headers . claims . signature http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk
  • 20. JSON Web Tokens headers {“typ":"JWT", "alg":"HS256"} claims {
 “shop_id":"acme",
 "iat":1300819380,
 "aud":"orders",
 "filters": {"status": "shipped"}
 } signature + Base64 + Base64 HMAC SHA-256 (headers + claims, secret) + Base64
  • 21. Rails: Generate token (Ruby) # controllers/downloads_controller.rb
 def create 
 url = Rails.application.config.downloads_host
 claims = params[:download_options]
 # Add an issued_at timestamp
 claims[:iat] = (Time.now.getutc.to_f * 1000).to_i
 # Scope data on current account
 claims[“shop_id"] = current_shop.id
 # generate JWT
 token = JWT.encode(claims, Rails.application.config.downloads_secret)
 
 # Redirect to download URL. Browser will trigger download dialog
 redirect_to “#{url}?jwt=#{token}" 
 end
  • 22. Rails: Generate token (Ruby) claims[:iat] = (Time.now.getutc.to_f * 1000).to_i
 
 claims[“shop_id"] = current_shop.id
 
 token = JWT.encode(claims, secret)
 
 redirect_to "#{url}?jwt=#{token}"
  • 23. Node: validate JWT var TTL = 60000;
 
 var tokenMiddleware = function(req, res, next){
 try{
 var decoded = jwt.decode(req.query.jwt, secret);
 
 if(decoded.shop_id != req.param(‘shop_id') {
 res.send(400, ‘JWT and query shop ids do not match');
 return
 }
 
 var now = new Date(),
 utc = getUtcCurrentDate();
 
 if(utc - Number(decoded.iat) > TTL) {
 res.send(401, "Web token has expired")
 return
 }
 
 req.query = decoded
 // all good, carry on
 next()
 
 } catch(e) {
 res.send(401, 'Unauthorized or invalid web token');
 }
 }
  • 24. Node: validate JWT var decoded = jwt.decode(req.query.jwt, secret); ?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMD.dBjftJeZ4CVP-mB92K27uhbUJU
  • 25. Node: validate JWT if(decoded.shop_id != req.param(‘shop_id') {
 res.send(400, ‘JWT and query shop ids do not match');
 return
 }

  • 26. Node: validate JWT var now = new Date(),
 utc = getUtcCurrentDate();
 
 if(utc - Number(decoded.iat) > TTL) {
 res.send(401, "Web token has expired")
 return
 }
  • 27. Node: validate JWT req.query = decoded 
 // all good, carry on
 next()
  • 28. Node: HTTP handlers app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...)); app.get('/:shop_id/contacts.csv', tokenMiddleware, handler.create('contacts', ...)); app.get('/:shop_id/products.csv', tokenMiddleware, handler.create('products', ...));
  • 29. } goo.gl/nolmRK ismael celis @ismasan

×