SlideShare a Scribd company logo
1 of 30
Download to read offline
Streaming downloads
proxy service with
Node.js
ismael celis @ismasan
bootic.net - Hosted e-commerce in South America
background
job Email
attachment
Previous setup
Previous setup
Previous setup
• Memory limitations
• Email deliverability
• Code bloat
• inflexible
New setup
• Monolithic micro
• Leverage existing API
New setup
• Monolithic micro
• Leverage existing API
curl -H “Authorization: Bearer xxx” 	
https://api.bootic.net/v1/orders.json?created_at:gte=2014-02-01&page=2
New setup
API -> CSV Stream
// pipe generated CSV onto the HTTP response

var writer = csv.createCsvStreamWriter(response)



// Turn a series of paginated requests 

// to the backend API into a stream of data

var stream = apistream.instance(uri, token)



// Pipe data stream into CSV writer

stream.pipe(writer)
API -> CSV Stream
response.setHeader('Content-Type', ‘text/csv');


response.setHeader('Content-disposition', 'attachment;filename=' + name + '.csv');
API -> mappers -> CSV Stream
{

"code": "123EFCD",

"total": 80000,

"status": "shipped",

"date": "2014-02-03",

"items": [

{"product_title": "iPhone 5", "units": 2, "unit_price": 30000},

{"product_title": "Samsung Galaxy S4", "units": 1, "unit_price": 20000}

]

}
code, total, date, status, product, units, unit_price, total

2 123EFCD, 80000, 2014-02-03, shipped, iPhone 5, 2, 30000, 80000

3 123EFCD, 80000, 2014-02-03, shipped, Samsung Galaxy S4, 1, 20000, 80000
API -> mappers -> CSV Stream
var OrderMapper = csvmapper.define(function () {



this.scope('items', function () {

this

.map('id', '/id')

.map('order', '/code')

.map('status', '/status')

.map('discount', '/discount_total')

.map('shipping price', '/shipping_total')

.map('total', '/total')

.map('year', '/updated_on', year)

.map('month', '/updated_on', month)

.map('day', '/updated_on', day)

.map('payment method', '/payment_method_type')

.map('name', '/contact/name')

.map('email', '/contact/email')

.map('address', '/address', address)

.map('product', 'product_title')

.map('variant', 'variant_title')

.map('sku', 'variant_sku')

.map('unit price', 'unit_price')

API -> mappers -> CSV Stream
var writer = csv.createCsvStreamWriter(res);



var stream = apistream.instance(uri, token)



var mapper = new OrdersMapper()



// First line in CSV is the headers

writer.writeRecord(mapper.headers())



// mapper.eachRow() turns a single API resource into 1 or more CSV rows

stream.on('item', function (item) {

mapper.eachRow(item, function (row) {

writer.writeRecord(row)

})

})
API -> mappers -> CSV Stream
stream.on('item', function (item) {

mapper.eachRow(item, function (row) {

writer.writeRecord(row)

})

})
Paremeter definitions
c.net/v1/orders.json? created_at:gte=2014-02-01 & page=2
Paremeter definitions
var OrdersParams = params.define(function () {

this

.param('sort', 'updated_on:desc')

.param('per_page', 20)

.param('status', 'closed,pending,invalid,shipped')

})
Paremeter definitions
var params = new OrdersParams(request.query)



// Compose API url using sanitized / defaulted params

var uri = "https://api.com/orders?" + params.query;



var stream = apistream.instance(uri, token)
Secure CSV downloads
JSON Web Tokens
headers . claims . signature
http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html
eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk
JSON Web Tokens
headers {“typ":"JWT",
"alg":"HS256"}
claims
{

“shop_id":"acme",

"iat":1300819380,

"aud":"orders",

"filters": {"status": "shipped"}

}
signature
+ Base64
+ Base64
HMAC SHA-256 (headers + claims, secret) + Base64
Rails: Generate token (Ruby)
# controllers/downloads_controller.rb

def create


url = Rails.application.config.downloads_host

claims = params[:download_options]

# Add an issued_at timestamp

claims[:iat] = (Time.now.getutc.to_f * 1000).to_i

# Scope data on current account

claims[“shop_id"] = current_shop.id

# generate JWT

token = JWT.encode(claims, Rails.application.config.downloads_secret)



# Redirect to download URL. Browser will trigger download dialog

redirect_to “#{url}?jwt=#{token}"


end
Rails: Generate token (Ruby)
claims[:iat] = (Time.now.getutc.to_f * 1000).to_i



claims[“shop_id"] = current_shop.id



token = JWT.encode(claims, secret)



redirect_to "#{url}?jwt=#{token}"
Node: validate JWT
var TTL = 60000;



var tokenMiddleware = function(req, res, next){

try{

var decoded = jwt.decode(req.query.jwt, secret);



if(decoded.shop_id != req.param(‘shop_id') {

res.send(400, ‘JWT and query shop ids do not match');

return

}



var now = new Date(),

utc = getUtcCurrentDate();



if(utc - Number(decoded.iat) > TTL) {

res.send(401, "Web token has expired")

return

}



req.query = decoded

// all good, carry on

next()



} catch(e) {

res.send(401, 'Unauthorized or invalid web token');

}

}
Node: validate JWT
var decoded = jwt.decode(req.query.jwt, secret);
?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMD.dBjftJeZ4CVP-mB92K27uhbUJU
Node: validate JWT
if(decoded.shop_id != req.param(‘shop_id') {

res.send(400, ‘JWT and query shop ids do not match');

return

}

Node: validate JWT
var now = new Date(),

utc = getUtcCurrentDate();



if(utc - Number(decoded.iat) > TTL) {

res.send(401, "Web token has expired")

return

}
Node: validate JWT
req.query = decoded


// all good, carry on

next()
Node: HTTP handlers
app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...));
app.get('/:shop_id/contacts.csv', tokenMiddleware, handler.create('contacts', ...));
app.get('/:shop_id/products.csv', tokenMiddleware, handler.create('products', ...));
}
goo.gl/nolmRK
ismael celis @ismasan

More Related Content

What's hot

Ruby HTTP clients comparison
Ruby HTTP clients comparisonRuby HTTP clients comparison
Ruby HTTP clients comparisonHiroshi Nakamura
 
Bootstrapping multidc observability stack
Bootstrapping multidc observability stackBootstrapping multidc observability stack
Bootstrapping multidc observability stackBram Vogelaar
 
Bootstrapping multidc observability stack
Bootstrapping multidc observability stackBootstrapping multidc observability stack
Bootstrapping multidc observability stackBram Vogelaar
 
Puppet and the HashiStack
Puppet and the HashiStackPuppet and the HashiStack
Puppet and the HashiStackBram Vogelaar
 
Observability with Consul Connect
Observability with Consul ConnectObservability with Consul Connect
Observability with Consul ConnectBram Vogelaar
 
Static Typing in Vault
Static Typing in VaultStatic Typing in Vault
Static Typing in VaultGlynnForrest
 
Redis as a message queue
Redis as a message queueRedis as a message queue
Redis as a message queueBrandon Lamb
 
Securing Prometheus exporters using HashiCorp Vault
Securing Prometheus exporters using HashiCorp VaultSecuring Prometheus exporters using HashiCorp Vault
Securing Prometheus exporters using HashiCorp VaultBram Vogelaar
 
Redis & ZeroMQ: How to scale your application
Redis & ZeroMQ: How to scale your applicationRedis & ZeroMQ: How to scale your application
Redis & ZeroMQ: How to scale your applicationrjsmelo
 
Application Logging in the 21st century - 2014.key
Application Logging in the 21st century - 2014.keyApplication Logging in the 21st century - 2014.key
Application Logging in the 21st century - 2014.keyTim Bunce
 
Elasticsearch (R)Evolution — You Know, for Search… by Philipp Krenn at Big Da...
Elasticsearch (R)Evolution — You Know, for Search… by Philipp Krenn at Big Da...Elasticsearch (R)Evolution — You Know, for Search… by Philipp Krenn at Big Da...
Elasticsearch (R)Evolution — You Know, for Search… by Philipp Krenn at Big Da...Big Data Spain
 
Testing your infrastructure with litmus
Testing your infrastructure with litmusTesting your infrastructure with litmus
Testing your infrastructure with litmusBram Vogelaar
 
On UnQLite
On UnQLiteOn UnQLite
On UnQLitecharsbar
 
Mасштабирование микросервисов на Go, Matt Heath (Hailo)
Mасштабирование микросервисов на Go, Matt Heath (Hailo)Mасштабирование микросервисов на Go, Matt Heath (Hailo)
Mасштабирование микросервисов на Go, Matt Heath (Hailo)Ontico
 
ql.io: Consuming HTTP at Scale
ql.io: Consuming HTTP at Scale ql.io: Consuming HTTP at Scale
ql.io: Consuming HTTP at Scale Subbu Allamaraju
 
Nodejs Explained with Examples
Nodejs Explained with ExamplesNodejs Explained with Examples
Nodejs Explained with ExamplesGabriele Lana
 

What's hot (20)

Ruby HTTP clients comparison
Ruby HTTP clients comparisonRuby HTTP clients comparison
Ruby HTTP clients comparison
 
Bootstrapping multidc observability stack
Bootstrapping multidc observability stackBootstrapping multidc observability stack
Bootstrapping multidc observability stack
 
Bootstrapping multidc observability stack
Bootstrapping multidc observability stackBootstrapping multidc observability stack
Bootstrapping multidc observability stack
 
Puppet and the HashiStack
Puppet and the HashiStackPuppet and the HashiStack
Puppet and the HashiStack
 
Observability with Consul Connect
Observability with Consul ConnectObservability with Consul Connect
Observability with Consul Connect
 
Nginx-lua
Nginx-luaNginx-lua
Nginx-lua
 
Static Typing in Vault
Static Typing in VaultStatic Typing in Vault
Static Typing in Vault
 
Redis as a message queue
Redis as a message queueRedis as a message queue
Redis as a message queue
 
Securing Prometheus exporters using HashiCorp Vault
Securing Prometheus exporters using HashiCorp VaultSecuring Prometheus exporters using HashiCorp Vault
Securing Prometheus exporters using HashiCorp Vault
 
Ubic
UbicUbic
Ubic
 
Redis & ZeroMQ: How to scale your application
Redis & ZeroMQ: How to scale your applicationRedis & ZeroMQ: How to scale your application
Redis & ZeroMQ: How to scale your application
 
Application Logging in the 21st century - 2014.key
Application Logging in the 21st century - 2014.keyApplication Logging in the 21st century - 2014.key
Application Logging in the 21st century - 2014.key
 
Elasticsearch (R)Evolution — You Know, for Search… by Philipp Krenn at Big Da...
Elasticsearch (R)Evolution — You Know, for Search… by Philipp Krenn at Big Da...Elasticsearch (R)Evolution — You Know, for Search… by Philipp Krenn at Big Da...
Elasticsearch (R)Evolution — You Know, for Search… by Philipp Krenn at Big Da...
 
Testing your infrastructure with litmus
Testing your infrastructure with litmusTesting your infrastructure with litmus
Testing your infrastructure with litmus
 
On UnQLite
On UnQLiteOn UnQLite
On UnQLite
 
Mасштабирование микросервисов на Go, Matt Heath (Hailo)
Mасштабирование микросервисов на Go, Matt Heath (Hailo)Mасштабирование микросервисов на Go, Matt Heath (Hailo)
Mасштабирование микросервисов на Go, Matt Heath (Hailo)
 
ql.io: Consuming HTTP at Scale
ql.io: Consuming HTTP at Scale ql.io: Consuming HTTP at Scale
ql.io: Consuming HTTP at Scale
 
Tuning Solr for Logs
Tuning Solr for LogsTuning Solr for Logs
Tuning Solr for Logs
 
Nodejs Explained with Examples
Nodejs Explained with ExamplesNodejs Explained with Examples
Nodejs Explained with Examples
 
Beyond Phoenix
Beyond PhoenixBeyond Phoenix
Beyond Phoenix
 

Similar to Node.js streaming csv downloads proxy

Use Windows Azure Service Bus, BizTalk Services, Mobile Services, and BizTalk...
Use Windows Azure Service Bus, BizTalk Services, Mobile Services, and BizTalk...Use Windows Azure Service Bus, BizTalk Services, Mobile Services, and BizTalk...
Use Windows Azure Service Bus, BizTalk Services, Mobile Services, and BizTalk...BizTalk360
 
Rapid API development examples for Impress Application Server / Node.js (jsfw...
Rapid API development examples for Impress Application Server / Node.js (jsfw...Rapid API development examples for Impress Application Server / Node.js (jsfw...
Rapid API development examples for Impress Application Server / Node.js (jsfw...Timur Shemsedinov
 
API Days Australia - Automatic Testing of (RESTful) API Documentation
API Days Australia  - Automatic Testing of (RESTful) API DocumentationAPI Days Australia  - Automatic Testing of (RESTful) API Documentation
API Days Australia - Automatic Testing of (RESTful) API DocumentationRouven Weßling
 
API Days Paris - Automatic Testing of (RESTful) API Documentation
API Days Paris - Automatic Testing of (RESTful) API DocumentationAPI Days Paris - Automatic Testing of (RESTful) API Documentation
API Days Paris - Automatic Testing of (RESTful) API DocumentationRouven Weßling
 
Ppt on web development and this has all details
Ppt on web development and this has all detailsPpt on web development and this has all details
Ppt on web development and this has all detailsgogijoshiajmer
 
Exposing Salesforce REST Services Using Swagger
Exposing Salesforce REST Services Using SwaggerExposing Salesforce REST Services Using Swagger
Exposing Salesforce REST Services Using SwaggerSalesforce Developers
 
ASP.NET Overview - Alvin Lau
ASP.NET Overview - Alvin LauASP.NET Overview - Alvin Lau
ASP.NET Overview - Alvin LauSpiffy
 
Bonnes pratiques de développement avec Node js
Bonnes pratiques de développement avec Node jsBonnes pratiques de développement avec Node js
Bonnes pratiques de développement avec Node jsFrancois Zaninotto
 
UEMB200: Next Generation of Endpoint Management Architecture and Discovery Se...
UEMB200: Next Generation of Endpoint Management Architecture and Discovery Se...UEMB200: Next Generation of Endpoint Management Architecture and Discovery Se...
UEMB200: Next Generation of Endpoint Management Architecture and Discovery Se...Ivanti
 
nodejs_at_a_glance.ppt
nodejs_at_a_glance.pptnodejs_at_a_glance.ppt
nodejs_at_a_glance.pptWalaSidhom1
 
Introduction to the SharePoint Client Object Model and REST API
Introduction to the SharePoint Client Object Model and REST APIIntroduction to the SharePoint Client Object Model and REST API
Introduction to the SharePoint Client Object Model and REST APIRob Windsor
 
Nordic APIs - Automatic Testing of (RESTful) API Documentation
Nordic APIs - Automatic Testing of (RESTful) API DocumentationNordic APIs - Automatic Testing of (RESTful) API Documentation
Nordic APIs - Automatic Testing of (RESTful) API DocumentationRouven Weßling
 
Maximizer 2018 API training
Maximizer 2018 API trainingMaximizer 2018 API training
Maximizer 2018 API trainingMurylo Batista
 
Deploying your static web app to the Cloud
Deploying your static web app to the CloudDeploying your static web app to the Cloud
Deploying your static web app to the CloudChristoffer Noring
 
Share point hosted add ins munich
Share point hosted add ins munichShare point hosted add ins munich
Share point hosted add ins munichSonja Madsen
 
Integrating Force.com with Heroku
Integrating Force.com with HerokuIntegrating Force.com with Heroku
Integrating Force.com with HerokuPat Patterson
 
SharePoint Client Object Model (CSOM)
SharePoint Client Object Model (CSOM)SharePoint Client Object Model (CSOM)
SharePoint Client Object Model (CSOM)Kashif Imran
 

Similar to Node.js streaming csv downloads proxy (20)

Use Windows Azure Service Bus, BizTalk Services, Mobile Services, and BizTalk...
Use Windows Azure Service Bus, BizTalk Services, Mobile Services, and BizTalk...Use Windows Azure Service Bus, BizTalk Services, Mobile Services, and BizTalk...
Use Windows Azure Service Bus, BizTalk Services, Mobile Services, and BizTalk...
 
Rapid API development examples for Impress Application Server / Node.js (jsfw...
Rapid API development examples for Impress Application Server / Node.js (jsfw...Rapid API development examples for Impress Application Server / Node.js (jsfw...
Rapid API development examples for Impress Application Server / Node.js (jsfw...
 
API Days Australia - Automatic Testing of (RESTful) API Documentation
API Days Australia  - Automatic Testing of (RESTful) API DocumentationAPI Days Australia  - Automatic Testing of (RESTful) API Documentation
API Days Australia - Automatic Testing of (RESTful) API Documentation
 
API Days Paris - Automatic Testing of (RESTful) API Documentation
API Days Paris - Automatic Testing of (RESTful) API DocumentationAPI Days Paris - Automatic Testing of (RESTful) API Documentation
API Days Paris - Automatic Testing of (RESTful) API Documentation
 
Ppt on web development and this has all details
Ppt on web development and this has all detailsPpt on web development and this has all details
Ppt on web development and this has all details
 
ASP.NET WEB API
ASP.NET WEB APIASP.NET WEB API
ASP.NET WEB API
 
Exposing Salesforce REST Services Using Swagger
Exposing Salesforce REST Services Using SwaggerExposing Salesforce REST Services Using Swagger
Exposing Salesforce REST Services Using Swagger
 
ASP.NET Overview - Alvin Lau
ASP.NET Overview - Alvin LauASP.NET Overview - Alvin Lau
ASP.NET Overview - Alvin Lau
 
Bonnes pratiques de développement avec Node js
Bonnes pratiques de développement avec Node jsBonnes pratiques de développement avec Node js
Bonnes pratiques de développement avec Node js
 
UEMB200: Next Generation of Endpoint Management Architecture and Discovery Se...
UEMB200: Next Generation of Endpoint Management Architecture and Discovery Se...UEMB200: Next Generation of Endpoint Management Architecture and Discovery Se...
UEMB200: Next Generation of Endpoint Management Architecture and Discovery Se...
 
nodejs_at_a_glance.ppt
nodejs_at_a_glance.pptnodejs_at_a_glance.ppt
nodejs_at_a_glance.ppt
 
Reduxing like a pro
Reduxing like a proReduxing like a pro
Reduxing like a pro
 
Introduction to the SharePoint Client Object Model and REST API
Introduction to the SharePoint Client Object Model and REST APIIntroduction to the SharePoint Client Object Model and REST API
Introduction to the SharePoint Client Object Model and REST API
 
Nordic APIs - Automatic Testing of (RESTful) API Documentation
Nordic APIs - Automatic Testing of (RESTful) API DocumentationNordic APIs - Automatic Testing of (RESTful) API Documentation
Nordic APIs - Automatic Testing of (RESTful) API Documentation
 
Maximizer 2018 API training
Maximizer 2018 API trainingMaximizer 2018 API training
Maximizer 2018 API training
 
Deploying your static web app to the Cloud
Deploying your static web app to the CloudDeploying your static web app to the Cloud
Deploying your static web app to the Cloud
 
Share point hosted add ins munich
Share point hosted add ins munichShare point hosted add ins munich
Share point hosted add ins munich
 
Integrating Force.com with Heroku
Integrating Force.com with HerokuIntegrating Force.com with Heroku
Integrating Force.com with Heroku
 
Day7
Day7Day7
Day7
 
SharePoint Client Object Model (CSOM)
SharePoint Client Object Model (CSOM)SharePoint Client Object Model (CSOM)
SharePoint Client Object Model (CSOM)
 

Recently uploaded

WSO2CON 2024 - Cloud Native Middleware: Domain-Driven Design, Cell-Based Arch...
WSO2CON 2024 - Cloud Native Middleware: Domain-Driven Design, Cell-Based Arch...WSO2CON 2024 - Cloud Native Middleware: Domain-Driven Design, Cell-Based Arch...
WSO2CON 2024 - Cloud Native Middleware: Domain-Driven Design, Cell-Based Arch...WSO2
 
WSO2CON 2024 - IoT Needs CIAM: The Importance of Centralized IAM in a Growing...
WSO2CON 2024 - IoT Needs CIAM: The Importance of Centralized IAM in a Growing...WSO2CON 2024 - IoT Needs CIAM: The Importance of Centralized IAM in a Growing...
WSO2CON 2024 - IoT Needs CIAM: The Importance of Centralized IAM in a Growing...WSO2
 
WSO2CON 2024 - Not Just Microservices: Rightsize Your Services!
WSO2CON 2024 - Not Just Microservices: Rightsize Your Services!WSO2CON 2024 - Not Just Microservices: Rightsize Your Services!
WSO2CON 2024 - Not Just Microservices: Rightsize Your Services!WSO2
 
WSO2CON2024 - Why Should You Consider Ballerina for Your Next Integration
WSO2CON2024 - Why Should You Consider Ballerina for Your Next IntegrationWSO2CON2024 - Why Should You Consider Ballerina for Your Next Integration
WSO2CON2024 - Why Should You Consider Ballerina for Your Next IntegrationWSO2
 
WSO2CON 2024 - Does Open Source Still Matter?
WSO2CON 2024 - Does Open Source Still Matter?WSO2CON 2024 - Does Open Source Still Matter?
WSO2CON 2024 - Does Open Source Still Matter?WSO2
 
Effective Strategies for Wix's Scaling challenges - GeeCon
Effective Strategies for Wix's Scaling challenges - GeeConEffective Strategies for Wix's Scaling challenges - GeeCon
Effective Strategies for Wix's Scaling challenges - GeeConNatan Silnitsky
 
AzureNativeQumulo_HPC_Cloud_Native_Benchmarks.pdf
AzureNativeQumulo_HPC_Cloud_Native_Benchmarks.pdfAzureNativeQumulo_HPC_Cloud_Native_Benchmarks.pdf
AzureNativeQumulo_HPC_Cloud_Native_Benchmarks.pdfryanfarris8
 
What Goes Wrong with Language Definitions and How to Improve the Situation
What Goes Wrong with Language Definitions and How to Improve the SituationWhat Goes Wrong with Language Definitions and How to Improve the Situation
What Goes Wrong with Language Definitions and How to Improve the SituationJuha-Pekka Tolvanen
 
Large-scale Logging Made Easy: Meetup at Deutsche Bank 2024
Large-scale Logging Made Easy: Meetup at Deutsche Bank 2024Large-scale Logging Made Easy: Meetup at Deutsche Bank 2024
Large-scale Logging Made Easy: Meetup at Deutsche Bank 2024VictoriaMetrics
 
WSO2CON 2024 - OSU & WSO2: A Decade Journey in Integration & Innovation
WSO2CON 2024 - OSU & WSO2: A Decade Journey in Integration & InnovationWSO2CON 2024 - OSU & WSO2: A Decade Journey in Integration & Innovation
WSO2CON 2024 - OSU & WSO2: A Decade Journey in Integration & InnovationWSO2
 
WSO2Con2024 - From Code To Cloud: Fast Track Your Cloud Native Journey with C...
WSO2Con2024 - From Code To Cloud: Fast Track Your Cloud Native Journey with C...WSO2Con2024 - From Code To Cloud: Fast Track Your Cloud Native Journey with C...
WSO2Con2024 - From Code To Cloud: Fast Track Your Cloud Native Journey with C...WSO2
 
WSO2Con2024 - GitOps in Action: Navigating Application Deployment in the Plat...
WSO2Con2024 - GitOps in Action: Navigating Application Deployment in the Plat...WSO2Con2024 - GitOps in Action: Navigating Application Deployment in the Plat...
WSO2Con2024 - GitOps in Action: Navigating Application Deployment in the Plat...WSO2
 
WSO2Con2024 - Facilitating Broadband Switching Services for UK Telecoms Provi...
WSO2Con2024 - Facilitating Broadband Switching Services for UK Telecoms Provi...WSO2Con2024 - Facilitating Broadband Switching Services for UK Telecoms Provi...
WSO2Con2024 - Facilitating Broadband Switching Services for UK Telecoms Provi...WSO2
 
WSO2Con2024 - From Blueprint to Brilliance: WSO2's Guide to API-First Enginee...
WSO2Con2024 - From Blueprint to Brilliance: WSO2's Guide to API-First Enginee...WSO2Con2024 - From Blueprint to Brilliance: WSO2's Guide to API-First Enginee...
WSO2Con2024 - From Blueprint to Brilliance: WSO2's Guide to API-First Enginee...WSO2
 
WSO2CON 2024 - Unlocking the Identity: Embracing CIAM 2.0 for a Competitive A...
WSO2CON 2024 - Unlocking the Identity: Embracing CIAM 2.0 for a Competitive A...WSO2CON 2024 - Unlocking the Identity: Embracing CIAM 2.0 for a Competitive A...
WSO2CON 2024 - Unlocking the Identity: Embracing CIAM 2.0 for a Competitive A...WSO2
 
WSO2Con2024 - Unleashing the Financial Potential of 13 Million People
WSO2Con2024 - Unleashing the Financial Potential of 13 Million PeopleWSO2Con2024 - Unleashing the Financial Potential of 13 Million People
WSO2Con2024 - Unleashing the Financial Potential of 13 Million PeopleWSO2
 
WSO2CON 2024 - API Management Usage at La Poste and Its Impact on Business an...
WSO2CON 2024 - API Management Usage at La Poste and Its Impact on Business an...WSO2CON 2024 - API Management Usage at La Poste and Its Impact on Business an...
WSO2CON 2024 - API Management Usage at La Poste and Its Impact on Business an...WSO2
 
Artyushina_Guest lecture_YorkU CS May 2024.pptx
Artyushina_Guest lecture_YorkU CS May 2024.pptxArtyushina_Guest lecture_YorkU CS May 2024.pptx
Artyushina_Guest lecture_YorkU CS May 2024.pptxAnnaArtyushina1
 
WSO2Con204 - Hard Rock Presentation - Keynote
WSO2Con204 - Hard Rock Presentation - KeynoteWSO2Con204 - Hard Rock Presentation - Keynote
WSO2Con204 - Hard Rock Presentation - KeynoteWSO2
 
WSO2CON2024 - It's time to go Platformless
WSO2CON2024 - It's time to go PlatformlessWSO2CON2024 - It's time to go Platformless
WSO2CON2024 - It's time to go PlatformlessWSO2
 

Recently uploaded (20)

WSO2CON 2024 - Cloud Native Middleware: Domain-Driven Design, Cell-Based Arch...
WSO2CON 2024 - Cloud Native Middleware: Domain-Driven Design, Cell-Based Arch...WSO2CON 2024 - Cloud Native Middleware: Domain-Driven Design, Cell-Based Arch...
WSO2CON 2024 - Cloud Native Middleware: Domain-Driven Design, Cell-Based Arch...
 
WSO2CON 2024 - IoT Needs CIAM: The Importance of Centralized IAM in a Growing...
WSO2CON 2024 - IoT Needs CIAM: The Importance of Centralized IAM in a Growing...WSO2CON 2024 - IoT Needs CIAM: The Importance of Centralized IAM in a Growing...
WSO2CON 2024 - IoT Needs CIAM: The Importance of Centralized IAM in a Growing...
 
WSO2CON 2024 - Not Just Microservices: Rightsize Your Services!
WSO2CON 2024 - Not Just Microservices: Rightsize Your Services!WSO2CON 2024 - Not Just Microservices: Rightsize Your Services!
WSO2CON 2024 - Not Just Microservices: Rightsize Your Services!
 
WSO2CON2024 - Why Should You Consider Ballerina for Your Next Integration
WSO2CON2024 - Why Should You Consider Ballerina for Your Next IntegrationWSO2CON2024 - Why Should You Consider Ballerina for Your Next Integration
WSO2CON2024 - Why Should You Consider Ballerina for Your Next Integration
 
WSO2CON 2024 - Does Open Source Still Matter?
WSO2CON 2024 - Does Open Source Still Matter?WSO2CON 2024 - Does Open Source Still Matter?
WSO2CON 2024 - Does Open Source Still Matter?
 
Effective Strategies for Wix's Scaling challenges - GeeCon
Effective Strategies for Wix's Scaling challenges - GeeConEffective Strategies for Wix's Scaling challenges - GeeCon
Effective Strategies for Wix's Scaling challenges - GeeCon
 
AzureNativeQumulo_HPC_Cloud_Native_Benchmarks.pdf
AzureNativeQumulo_HPC_Cloud_Native_Benchmarks.pdfAzureNativeQumulo_HPC_Cloud_Native_Benchmarks.pdf
AzureNativeQumulo_HPC_Cloud_Native_Benchmarks.pdf
 
What Goes Wrong with Language Definitions and How to Improve the Situation
What Goes Wrong with Language Definitions and How to Improve the SituationWhat Goes Wrong with Language Definitions and How to Improve the Situation
What Goes Wrong with Language Definitions and How to Improve the Situation
 
Large-scale Logging Made Easy: Meetup at Deutsche Bank 2024
Large-scale Logging Made Easy: Meetup at Deutsche Bank 2024Large-scale Logging Made Easy: Meetup at Deutsche Bank 2024
Large-scale Logging Made Easy: Meetup at Deutsche Bank 2024
 
WSO2CON 2024 - OSU & WSO2: A Decade Journey in Integration & Innovation
WSO2CON 2024 - OSU & WSO2: A Decade Journey in Integration & InnovationWSO2CON 2024 - OSU & WSO2: A Decade Journey in Integration & Innovation
WSO2CON 2024 - OSU & WSO2: A Decade Journey in Integration & Innovation
 
WSO2Con2024 - From Code To Cloud: Fast Track Your Cloud Native Journey with C...
WSO2Con2024 - From Code To Cloud: Fast Track Your Cloud Native Journey with C...WSO2Con2024 - From Code To Cloud: Fast Track Your Cloud Native Journey with C...
WSO2Con2024 - From Code To Cloud: Fast Track Your Cloud Native Journey with C...
 
WSO2Con2024 - GitOps in Action: Navigating Application Deployment in the Plat...
WSO2Con2024 - GitOps in Action: Navigating Application Deployment in the Plat...WSO2Con2024 - GitOps in Action: Navigating Application Deployment in the Plat...
WSO2Con2024 - GitOps in Action: Navigating Application Deployment in the Plat...
 
WSO2Con2024 - Facilitating Broadband Switching Services for UK Telecoms Provi...
WSO2Con2024 - Facilitating Broadband Switching Services for UK Telecoms Provi...WSO2Con2024 - Facilitating Broadband Switching Services for UK Telecoms Provi...
WSO2Con2024 - Facilitating Broadband Switching Services for UK Telecoms Provi...
 
WSO2Con2024 - From Blueprint to Brilliance: WSO2's Guide to API-First Enginee...
WSO2Con2024 - From Blueprint to Brilliance: WSO2's Guide to API-First Enginee...WSO2Con2024 - From Blueprint to Brilliance: WSO2's Guide to API-First Enginee...
WSO2Con2024 - From Blueprint to Brilliance: WSO2's Guide to API-First Enginee...
 
WSO2CON 2024 - Unlocking the Identity: Embracing CIAM 2.0 for a Competitive A...
WSO2CON 2024 - Unlocking the Identity: Embracing CIAM 2.0 for a Competitive A...WSO2CON 2024 - Unlocking the Identity: Embracing CIAM 2.0 for a Competitive A...
WSO2CON 2024 - Unlocking the Identity: Embracing CIAM 2.0 for a Competitive A...
 
WSO2Con2024 - Unleashing the Financial Potential of 13 Million People
WSO2Con2024 - Unleashing the Financial Potential of 13 Million PeopleWSO2Con2024 - Unleashing the Financial Potential of 13 Million People
WSO2Con2024 - Unleashing the Financial Potential of 13 Million People
 
WSO2CON 2024 - API Management Usage at La Poste and Its Impact on Business an...
WSO2CON 2024 - API Management Usage at La Poste and Its Impact on Business an...WSO2CON 2024 - API Management Usage at La Poste and Its Impact on Business an...
WSO2CON 2024 - API Management Usage at La Poste and Its Impact on Business an...
 
Artyushina_Guest lecture_YorkU CS May 2024.pptx
Artyushina_Guest lecture_YorkU CS May 2024.pptxArtyushina_Guest lecture_YorkU CS May 2024.pptx
Artyushina_Guest lecture_YorkU CS May 2024.pptx
 
WSO2Con204 - Hard Rock Presentation - Keynote
WSO2Con204 - Hard Rock Presentation - KeynoteWSO2Con204 - Hard Rock Presentation - Keynote
WSO2Con204 - Hard Rock Presentation - Keynote
 
WSO2CON2024 - It's time to go Platformless
WSO2CON2024 - It's time to go PlatformlessWSO2CON2024 - It's time to go Platformless
WSO2CON2024 - It's time to go Platformless
 

Node.js streaming csv downloads proxy

  • 1. Streaming downloads proxy service with Node.js ismael celis @ismasan
  • 2. bootic.net - Hosted e-commerce in South America
  • 5. Previous setup • Memory limitations • Email deliverability • Code bloat • inflexible
  • 6. New setup • Monolithic micro • Leverage existing API
  • 7. New setup • Monolithic micro • Leverage existing API curl -H “Authorization: Bearer xxx” https://api.bootic.net/v1/orders.json?created_at:gte=2014-02-01&page=2
  • 8.
  • 10. API -> CSV Stream // pipe generated CSV onto the HTTP response
 var writer = csv.createCsvStreamWriter(response)
 
 // Turn a series of paginated requests 
 // to the backend API into a stream of data
 var stream = apistream.instance(uri, token)
 
 // Pipe data stream into CSV writer
 stream.pipe(writer)
  • 11. API -> CSV Stream response.setHeader('Content-Type', ‘text/csv'); 
 response.setHeader('Content-disposition', 'attachment;filename=' + name + '.csv');
  • 12. API -> mappers -> CSV Stream {
 "code": "123EFCD",
 "total": 80000,
 "status": "shipped",
 "date": "2014-02-03",
 "items": [
 {"product_title": "iPhone 5", "units": 2, "unit_price": 30000},
 {"product_title": "Samsung Galaxy S4", "units": 1, "unit_price": 20000}
 ]
 } code, total, date, status, product, units, unit_price, total
 2 123EFCD, 80000, 2014-02-03, shipped, iPhone 5, 2, 30000, 80000
 3 123EFCD, 80000, 2014-02-03, shipped, Samsung Galaxy S4, 1, 20000, 80000
  • 13. API -> mappers -> CSV Stream var OrderMapper = csvmapper.define(function () {
 
 this.scope('items', function () {
 this
 .map('id', '/id')
 .map('order', '/code')
 .map('status', '/status')
 .map('discount', '/discount_total')
 .map('shipping price', '/shipping_total')
 .map('total', '/total')
 .map('year', '/updated_on', year)
 .map('month', '/updated_on', month)
 .map('day', '/updated_on', day)
 .map('payment method', '/payment_method_type')
 .map('name', '/contact/name')
 .map('email', '/contact/email')
 .map('address', '/address', address)
 .map('product', 'product_title')
 .map('variant', 'variant_title')
 .map('sku', 'variant_sku')
 .map('unit price', 'unit_price')

  • 14. API -> mappers -> CSV Stream var writer = csv.createCsvStreamWriter(res);
 
 var stream = apistream.instance(uri, token)
 
 var mapper = new OrdersMapper()
 
 // First line in CSV is the headers
 writer.writeRecord(mapper.headers())
 
 // mapper.eachRow() turns a single API resource into 1 or more CSV rows
 stream.on('item', function (item) {
 mapper.eachRow(item, function (row) {
 writer.writeRecord(row)
 })
 })
  • 15. API -> mappers -> CSV Stream stream.on('item', function (item) {
 mapper.eachRow(item, function (row) {
 writer.writeRecord(row)
 })
 })
  • 17. Paremeter definitions var OrdersParams = params.define(function () {
 this
 .param('sort', 'updated_on:desc')
 .param('per_page', 20)
 .param('status', 'closed,pending,invalid,shipped')
 })
  • 18. Paremeter definitions var params = new OrdersParams(request.query)
 
 // Compose API url using sanitized / defaulted params
 var uri = "https://api.com/orders?" + params.query;
 
 var stream = apistream.instance(uri, token)
  • 20. JSON Web Tokens headers . claims . signature http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk
  • 21. JSON Web Tokens headers {“typ":"JWT", "alg":"HS256"} claims {
 “shop_id":"acme",
 "iat":1300819380,
 "aud":"orders",
 "filters": {"status": "shipped"}
 } signature + Base64 + Base64 HMAC SHA-256 (headers + claims, secret) + Base64
  • 22. Rails: Generate token (Ruby) # controllers/downloads_controller.rb
 def create 
 url = Rails.application.config.downloads_host
 claims = params[:download_options]
 # Add an issued_at timestamp
 claims[:iat] = (Time.now.getutc.to_f * 1000).to_i
 # Scope data on current account
 claims[“shop_id"] = current_shop.id
 # generate JWT
 token = JWT.encode(claims, Rails.application.config.downloads_secret)
 
 # Redirect to download URL. Browser will trigger download dialog
 redirect_to “#{url}?jwt=#{token}" 
 end
  • 23. Rails: Generate token (Ruby) claims[:iat] = (Time.now.getutc.to_f * 1000).to_i
 
 claims[“shop_id"] = current_shop.id
 
 token = JWT.encode(claims, secret)
 
 redirect_to "#{url}?jwt=#{token}"
  • 24. Node: validate JWT var TTL = 60000;
 
 var tokenMiddleware = function(req, res, next){
 try{
 var decoded = jwt.decode(req.query.jwt, secret);
 
 if(decoded.shop_id != req.param(‘shop_id') {
 res.send(400, ‘JWT and query shop ids do not match');
 return
 }
 
 var now = new Date(),
 utc = getUtcCurrentDate();
 
 if(utc - Number(decoded.iat) > TTL) {
 res.send(401, "Web token has expired")
 return
 }
 
 req.query = decoded
 // all good, carry on
 next()
 
 } catch(e) {
 res.send(401, 'Unauthorized or invalid web token');
 }
 }
  • 25. Node: validate JWT var decoded = jwt.decode(req.query.jwt, secret); ?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMD.dBjftJeZ4CVP-mB92K27uhbUJU
  • 26. Node: validate JWT if(decoded.shop_id != req.param(‘shop_id') {
 res.send(400, ‘JWT and query shop ids do not match');
 return
 }

  • 27. Node: validate JWT var now = new Date(),
 utc = getUtcCurrentDate();
 
 if(utc - Number(decoded.iat) > TTL) {
 res.send(401, "Web token has expired")
 return
 }
  • 28. Node: validate JWT req.query = decoded 
 // all good, carry on
 next()
  • 29. Node: HTTP handlers app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...)); app.get('/:shop_id/contacts.csv', tokenMiddleware, handler.create('contacts', ...)); app.get('/:shop_id/products.csv', tokenMiddleware, handler.create('products', ...));