SlideShare a Scribd company logo
1 of 52
Download to read offline
Making things
Work Together
   QCon NY June 20, 2012
                           1	
  
Subbu Allamaraju

@sallamar
github.com/s3u
                   2	
  
How to build an
interoperable
Programmable Web?
                    3	
  
Agreements


             4	
  
Discovery
     Linking, Identity
  Schemas, Media Types
         Formats
Interfaces (Uniform, RPC)
           HTTP
            TCP
                            5	
  
Server's point
   of view



             6	
  
Client's point of view   7	
  
Several hundred APIs
         Different styles
   Written by different teams
            Over years
Built under different constraints
    Built with different goals
                                    8	
  
Use case:

Find things from A
For each thing, find details from B
For each thing, find more details from C
Merge results


                                           9	
  
Use case:

Find products
Find dominant categories of products
Look up category info
Merge categories with products


                                       10	
  
Use case:

Get stuff from A
If A is down, try from B
Annotate stuff with other stuff from C
Ignore some things from the stuff
Join all
                                         11	
  
[Really important client]
Client
         Why don't you give me an API optimized
         for my use cases?




                                                     Producers	
  
                                                       Server
                 [Really important producer]
                 Thanks. Get a number and stand in
                 the line!
                                                           12	
  
[Really important client]
Client
         Why don't you give me an API optimized
         for my use cases?




                                                     Producers	
  
                 Who gets to decide the
                     right thing?




                                                       Server
                 [Really important producer]
                 Thanks. Get a number and stand in
                 the line!
                                                           13	
  
Discovery
       Linking, Identity
    Schemas, Media Types
           Formats
> Interfaces (Uniform, RPC)
             HTTP
              TCP
                              14	
  
Discovery
    Linking, Identity
> Schemas, Media Types
        Formats

          HTTP
           TCP
                         15	
  
>      Discovery
>   Linking, Identity

        Formats

         HTTP
          TCP
                        16	
  
Formats
 HTTP
  TCP
          17	
  
Latency
Smart
Client   Failure Recovery
 Code     Orchestration
                 IO
             Formats
               HTTP
                TCP
                            18	
  
19	
  
20	
  
https://github.com/ql-io/ql.io
         Platform Engineering

                                 21	
  
A domain specific language for
   HTTP client programming


                                 22	
  
23	
  
24	
  
http://ql.io/console
                       25	
  
Test console               Find script       Compile



 HTTP interface                     Request state




                                       HTTP client
               Evented
             orchestration
                                     Transformations




A runtime designed for IO workloads
          Runs on node.js
                                                         26	
  
1. Interop via HTTP
 Support the protocol – but
don't require style adherence	
  
                                    27	
  
create	
  table	
  mytable1	
  
	
  	
  on	
  select	
  	
  
	
  	
  get	
  from	
  'http://...';	
  
	
  
create	
  table	
  mytable2	
  
	
  	
  on	
  insert	
  
	
  	
  get	
  from	
  'http://...&do=create'	
  
	
  	
  	
  	
  
create	
  table	
  mytable3	
  
	
  	
  on	
  delete	
  
	
  	
  post	
  to	
  'http://...'	
  
	
  	
  using	
  headers	
  'X-­‐Do'	
  =	
  'Delete'	
  
	
  
                                                            28	
  
2. Interop using common
         formats
   Agnostic of domain
  specific type systems	
  
                              29	
  
/*	
  A	
  table	
  with	
  a	
  body	
  template	
  */	
  
create	
  table	
  bing.soap.search	
  
	
  	
  on	
  select	
  post	
  to	
  'http://api.bing.net/
soap.asmx'	
  
	
  	
  using	
  defaults	
  appid	
  =	
  'xxx'	
  
	
  	
  using	
  bodyTemplate	
  'bing.xml.mu'	
  	
  
	
  	
  type	
  'application/xml'	
  
	
  	
  resultset	
  
'soapenv:Envelope.soapenv:Body.SearchResponse.pa
rameters.Web.Results.WebResult';	
  




                                                         30	
  
<soapenv:Envelope	
  xmlns:soapenv="http://schemas.xmlsoap.org/
soap/envelope/">	
  
	
  	
  <soapenv:Header/>	
  
	
  	
  <soapenv:Body>	
  
	
  	
  	
  	
  <SearchRequest	
  xmlns="http://schemas.microsoft.com/
LiveSearch/2008/03/Search">	
  
	
  	
  	
  	
  	
  	
  <parameters>	
  
	
  	
  	
  	
  	
  	
  {{#params}}	
  
	
  	
  	
  	
  	
  	
  <Query>{{q}}</Query>	
  
	
  	
  	
  	
  	
  	
  <AppId>{{appid}}</AppId>	
  
	
  	
  	
  	
  	
  	
  {{/params}}	
  
	
  	
  	
  	
  	
  	
  <Sources>	
  
	
  	
  	
  	
  	
  	
  	
  	
  <SourceType>Web</SourceType>	
  
	
  	
  	
  	
  	
  	
  </Sources>	
  
	
  	
  	
  	
  	
  	
  </parameters>	
  
	
  	
  	
  	
  </SearchRequest>	
  
	
  	
  </soapenv:Body>	
  
</soapenv:Envelope>	
  


                                                                         31	
  
-­‐-­‐	
  Form	
  encoded	
  body	
  
create	
  table	
  mytable	
  
	
  	
  on	
  insert	
  post	
  to	
  'http://...'	
  
	
  	
  using	
  bodyTemplate	
  'somefile.txt'	
  
	
  	
  	
  	
  type	
  'application/x-­‐www-­‐form-­‐urlencoded'	
  
	
  
-­‐-­‐	
  Or	
  JSON	
  
create	
  table	
  mytable	
  
	
  	
  on	
  update	
  put	
  to	
  'http://...'	
  
	
  	
  using	
  bodyTemplate	
  'file.json'	
  type	
  
'application/json'	
  



                                                                        32	
  
create	
  table	
  {a	
  name}	
  
on	
  {select|insert|update|delete}	
  
	
  	
  	
  {get	
  from|post	
  to|	
  
	
  	
  	
  	
  put	
  to|delete|patch}	
  	
  
	
  	
  	
  {a	
  URI	
  template}	
  
using	
  headers	
  {name}={value}*	
  
using	
  bodyTemplate={a	
  file}	
  	
  
	
  	
  	
  	
  	
  	
  type={media	
  type}	
  

                                              33	
  
HTTP	
  method	
  

URI	
  template	
  
Headers	
  
Body	
  
            Media	
  type	
  

                                34	
  
-­‐-­‐	
  A	
  table	
  with	
  a	
  monkey	
  patch	
  
create	
  table	
  bitly.shorten	
  
	
  	
  on	
  insert	
  get	
  from	
  "..."	
  
	
  	
  	
  	
  using	
  patch	
  'bitly.js'	
  
	
  	
  on	
  select	
  get	
  from	
  "..."	
  
	
  	
  	
  	
  using	
  patch	
  'bitly.js'	
  
	
  
/*	
  bitly.js	
  A	
  Javascript	
  monkey	
  patch	
  */	
  
exports['patch	
  status']	
  =	
  function(options)	
  {	
  
	
  	
  	
  	
  return	
  args.body	
  ?	
  	
  
	
  	
  	
  	
  	
  	
  args.body.status_code	
  :	
  200;	
  	
  
};	
  


                                                                     35	
  
3. SQL inspired constructs
     CRUD operations,
      filtering, joins	
  
                             36	
  
-­‐-­‐	
  Get	
  the	
  complete	
  response	
  as	
  JSON	
  
select	
  *	
  from	
  google.geocode	
  where	
  
	
  	
  address	
  =	
  'Brooklyn,	
  NY'	
  
	
  
-­‐-­‐	
  Project	
  fields	
  
select	
  ProductID[0].Value,	
  	
  
	
  	
  DetailsURL,	
  StockPhotoURL	
  
	
  	
  from	
  eBay.FindProducts	
  	
  
	
  	
  where	
  QueryKeywords='mini	
  cooper'	
  




                                                                 37	
  
/*	
  Join	
  datasets	
  without	
  enforcing	
  identity	
  	
  
	
  	
  	
  across	
  systems	
  */	
  
	
  
select	
  e.ItemID	
  as	
  id,	
  e.Title	
  as	
  title,	
  	
  
	
  	
  e.ViewItemURLForNaturalSearch	
  as	
  url,	
  	
  
	
  	
  g.geometry.location	
  as	
  latlng	
  
	
  	
  from	
  details	
  	
  as	
  e,	
  google.geocode	
  as	
  g	
  
	
  	
  where	
  e.itemId	
  in	
  (select	
  itemId	
  from	
  	
  
	
  	
  finditems	
  where	
  keywords	
  =	
  "iPad")	
  
	
  	
  and	
  g.address	
  =	
  e.Location	
  	
  
	
  



                                                                           38	
  
4. Orchestration
Implicit sequential, scatter,
  parallel, forks and joins	
  
                                  39	
  
-­‐-­‐	
  Sequential	
  
minis	
  =	
  select	
  *	
  from	
  finditems	
  where	
  	
  
	
  	
  	
  	
  keywords	
  =	
  'mini	
  cooper'	
  limit	
  10;	
  
return	
  select	
  PictureURL	
  from	
  details	
  where	
  	
  
	
  	
  	
  	
  itemId	
  =	
  "{minis.itemId}";	
  
	
  
-­‐-­‐	
  Or	
  parallel	
  
keyword	
  =	
  "ql.io";	
  
web	
  =	
  select	
  *	
  from	
  bing.search	
  where	
  q	
  =	
  "{keyword}";	
  
tweets	
  =	
  select	
  id	
  as	
  id,	
  from_user_name	
  as	
  user_name,	
  
	
  	
  	
  	
  text	
  as	
  text	
  	
  from	
  twitter.search	
  where	
  q	
  =	
  "ql.io";	
  
	
  	
  
return	
  {	
  
	
  	
  "keyword":	
  "{keyword}",	
  
	
  	
  "web":	
  "{web}",	
  
	
  	
  "tweets":	
  "{tweets}"	
  
}	
  


                                                                                                  40	
  
prodid	
  =	
  select	
  ProductID[0].Value	
  from	
  eBay.FindProducts	
  	
  
	
  	
  	
  	
  where	
  QueryKeywords	
  =	
  'macbook	
  pro';	
  
	
  
-­‐-­‐	
  Scatters	
  "n"	
  requests	
  
details	
  =	
  select	
  *	
  from	
  eBay.ProductDetails	
  where	
  
	
  	
  	
  	
  ProductID	
  in	
  ('{prodid}')	
  and	
  	
  
	
  	
  	
  	
  ProductType	
  =	
  'Reference';	
  
	
  
-­‐-­‐	
  Scatters	
  "n"	
  requests	
  
reviews	
  =	
  select	
  *	
  from	
  eBay.ProductReviews	
  where	
  
	
  	
  	
  	
  ProductID	
  in	
  ('{prodid}')	
  and	
  	
  
	
  	
  	
  	
  ProductType	
  =	
  'Reference';	
  
	
  
-­‐-­‐	
  Gather,	
  merge,	
  and	
  join	
  
return	
  select	
  d.ProductID[0].Value	
  as	
  id,	
  	
  
	
  	
  	
  	
  d.Title	
  as	
  title,	
  d.ReviewCount	
  as	
  reviewCount,	
  
	
  	
  	
  	
  r.ReviewDetails.AverageRating	
  as	
  rating	
  
	
  	
  from	
  details	
  as	
  d,	
  reviews	
  as	
  r	
  
	
  	
  	
  	
  where	
  d.ProductID[0].Value	
  =	
  r.ProductID.Value	
  

                                                                                     41	
  
-­‐-­‐	
  An	
  API	
  that	
  takes	
  one	
  ID	
  at	
  a	
  time	
  
create	
  table	
  eBay.ProductDetails	
  
	
  	
  on	
  select	
  get	
  from	
  "...&...={^ProductID}"	
  
	
  	
  ...	
  	
  
	
  	
  resultset	
  'Product';	
  
	
  
	
  
-­‐-­‐	
  A	
  batch	
  API	
  –	
  takes	
  upto	
  20	
  IDs	
  
create	
  table	
  details	
  
	
  	
  on	
  select	
  get	
  from	
  "...&ItemID={20|itemId}	
  
	
  	
  ...	
  
	
  	
  resultset	
  'Item';	
  
	
  
	
                                                                         42	
  
43	
  
5. Failure Modes
Timeouts, back-off and
      recovery	
  
                         44	
  
-­‐-­‐	
  Timeout	
  on	
  idle	
  sockets	
  	
  
-­‐-­‐	
  Retry	
  once	
  for	
  idempotent	
  statements	
  
select	
  id,	
  img	
  from	
  recommendations	
  
	
  	
  timeout	
  100	
  
	
  
-­‐-­‐	
  On	
  three	
  successive	
  failures,	
  backoff	
  	
  
-­‐-­‐	
  for	
  100	
  –	
  2000	
  msec	
  
select	
  id,	
  img	
  from	
  recommendations	
  
	
  	
  timeout	
  100	
  minDelay	
  100	
  maxDelay	
  2000	
  
	
  




                                                                      45	
  
-­‐-­‐	
  Use	
  a	
  fallback	
  	
  
select	
  id	
  as	
  id,	
  item_url	
  as	
  img	
  	
  
	
  	
  from	
  recommendations	
  
	
  	
  timeout	
  100	
  	
  
	
  	
  ||	
  
	
  	
  select	
  ID	
  as	
  id,	
  deal_url	
  as	
  img	
  
	
  	
  	
  	
  from	
  deals;	
  




                                                                 46	
  
6. Interface
Consumer-specified	
  

                         47	
  
var	
  Engine	
  =	
  require('ql.io-­‐engine'),	
  
	
  	
  	
  	
  fs	
  =	
  require('fs');	
  
var	
  engine	
  =	
  new	
  Engine({	
  
	
  	
  	
  	
  tables	
  :	
  __dirname	
  +	
  '/../tables',	
  
	
  	
  	
  	
  config:	
  __dirname	
  +	
  '/../config/dev.json'	
  
});	
  
var	
  script	
  =	
  fs.readFileSync(__dirname	
  +	
  '/some.ql',	
  'UTF-­‐8');	
  
	
  
//	
  Exec	
  the	
  script	
  
engine.execute(script,	
  function(emitter)	
  {	
  
	
  	
  	
  	
  emitter.on('end',	
  function(err,	
  result)	
  {	
  
	
  	
  	
  	
  	
  	
  	
  	
  result.body.forEach(function(row)	
  {	
  
	
  	
  	
  	
  	
  	
  	
  	
  	
  	
  	
  	
  console.log(row);	
  
	
  	
  	
  	
  	
  	
  	
  	
  });	
  
	
  	
  	
  	
  });	
  
});	
  
	
  

                                                                                     48	
  
Push orchestration to the data center
	
  
•  Reduce HTTP requests
•  Reduce bandwidth use


                                        49	
  
/*	
  Shows	
  daily	
  deals	
  -­‐	
  use	
  siteId=0	
  for	
  US	
  and	
  siteId=3	
  
for	
  UK.	
  */	
  
resp	
  =	
  select	
  *	
  from	
  dailydeals	
  where	
  siteId="{siteId}";	
  
deals	
  =	
  "{resp.$..Item}";	
  
itemDetails	
  =	
  select	
  ItemID	
  as	
  itemId,	
  Title	
  as	
  title,	
  	
  
	
  	
  GalleryURL	
  as	
  pic,	
  Seller.UserID	
  as	
  sellerUserId,	
  	
  	
  	
  
	
  	
  Seller.FeedbackScore	
  as	
  rating,	
  	
  
	
  	
  Seller.PositiveFeedbackPercent	
  as	
  feedback,	
  
	
  	
  HitCount	
  as	
  hits	
  from	
  details	
  	
  
	
  	
  where	
  itemId	
  in	
  (deals.ItemID);	
  
	
  
return	
  itemDetails	
  via	
  route	
  '/deals/{siteId}'	
  using	
  
method	
  get;	
  




                                                                                          50	
  
Latency
         Failure Recovery
          Orchestration
          IO workloads
             Formats
Server         HTTP
                TCP
                            51	
  
code
   https://github.com/ql-io/ql.io
docs/demos
   http://ql.io
blog
   http://ql-io.github.com

                                    52	
  

More Related Content

What's hot

Jsp/Servlet
Jsp/ServletJsp/Servlet
Jsp/ServletSunil OS
 
A Dexterity Intro for Recovering Archetypes Addicts
A Dexterity Intro for Recovering Archetypes AddictsA Dexterity Intro for Recovering Archetypes Addicts
A Dexterity Intro for Recovering Archetypes AddictsDavid Glick
 
C&S APIs in IBM Notes and Domino
C&S APIs in IBM Notes and DominoC&S APIs in IBM Notes and Domino
C&S APIs in IBM Notes and DominoDave Delay
 
Learning Appcelerator® Alloy™
Learning Appcelerator® Alloy™Learning Appcelerator® Alloy™
Learning Appcelerator® Alloy™Ricardo Alcocer
 
Seam in action
Seam in actionSeam in action
Seam in actionsantigec
 
Python Code Camp for Professionals 3/4
Python Code Camp for Professionals 3/4Python Code Camp for Professionals 3/4
Python Code Camp for Professionals 3/4DEVCON
 
Python Code Camp for Professionals 4/4
Python Code Camp for Professionals 4/4Python Code Camp for Professionals 4/4
Python Code Camp for Professionals 4/4DEVCON
 
Jquery presentation
Jquery presentationJquery presentation
Jquery presentationguest5d87aa6
 
Mobile Augmented Reality Using junaio
Mobile Augmented Reality Using junaioMobile Augmented Reality Using junaio
Mobile Augmented Reality Using junaioMark Billinghurst
 
ERRest - The Next Steps
ERRest - The Next StepsERRest - The Next Steps
ERRest - The Next StepsWO Community
 
iPhone dev intro
iPhone dev introiPhone dev intro
iPhone dev introVonbo
 

What's hot (18)

Ejb examples
Ejb examplesEjb examples
Ejb examples
 
Jsp/Servlet
Jsp/ServletJsp/Servlet
Jsp/Servlet
 
A Dexterity Intro for Recovering Archetypes Addicts
A Dexterity Intro for Recovering Archetypes AddictsA Dexterity Intro for Recovering Archetypes Addicts
A Dexterity Intro for Recovering Archetypes Addicts
 
C&S APIs in IBM Notes and Domino
C&S APIs in IBM Notes and DominoC&S APIs in IBM Notes and Domino
C&S APIs in IBM Notes and Domino
 
Creating Alloy Widgets
Creating Alloy WidgetsCreating Alloy Widgets
Creating Alloy Widgets
 
Learning Appcelerator® Alloy™
Learning Appcelerator® Alloy™Learning Appcelerator® Alloy™
Learning Appcelerator® Alloy™
 
You suck at Memory Analysis
You suck at Memory AnalysisYou suck at Memory Analysis
You suck at Memory Analysis
 
Symfony2 meets propel 1.5
Symfony2 meets propel 1.5Symfony2 meets propel 1.5
Symfony2 meets propel 1.5
 
Play!ng with scala
Play!ng with scalaPlay!ng with scala
Play!ng with scala
 
Seam in action
Seam in actionSeam in action
Seam in action
 
Python Code Camp for Professionals 3/4
Python Code Camp for Professionals 3/4Python Code Camp for Professionals 3/4
Python Code Camp for Professionals 3/4
 
Python Code Camp for Professionals 4/4
Python Code Camp for Professionals 4/4Python Code Camp for Professionals 4/4
Python Code Camp for Professionals 4/4
 
ERRest
ERRestERRest
ERRest
 
Jquery presentation
Jquery presentationJquery presentation
Jquery presentation
 
Mobile Augmented Reality Using junaio
Mobile Augmented Reality Using junaioMobile Augmented Reality Using junaio
Mobile Augmented Reality Using junaio
 
ERRest - The Next Steps
ERRest - The Next StepsERRest - The Next Steps
ERRest - The Next Steps
 
Alfredo-PUMEX
Alfredo-PUMEXAlfredo-PUMEX
Alfredo-PUMEX
 
iPhone dev intro
iPhone dev introiPhone dev intro
iPhone dev intro
 

Similar to Making Things Work Together

How to create an Angular builder
How to create an Angular builderHow to create an Angular builder
How to create an Angular builderMaurizio Vitale
 
Optimizing Code Reusability for SharePoint using Linq to SharePoint & the MVP...
Optimizing Code Reusability for SharePoint using Linq to SharePoint & the MVP...Optimizing Code Reusability for SharePoint using Linq to SharePoint & the MVP...
Optimizing Code Reusability for SharePoint using Linq to SharePoint & the MVP...Sparkhound Inc.
 
ql.io: Consuming HTTP at Scale
ql.io: Consuming HTTP at Scale ql.io: Consuming HTTP at Scale
ql.io: Consuming HTTP at Scale Subbu Allamaraju
 
Real World Asp.Net WebApi Applications
Real World Asp.Net WebApi ApplicationsReal World Asp.Net WebApi Applications
Real World Asp.Net WebApi ApplicationsEffie Arditi
 
O365 Saturday - Deepdive SharePoint Client Side Rendering
O365 Saturday - Deepdive SharePoint Client Side RenderingO365 Saturday - Deepdive SharePoint Client Side Rendering
O365 Saturday - Deepdive SharePoint Client Side RenderingRiwut Libinuko
 
Parse cloud code
Parse cloud codeParse cloud code
Parse cloud code維佋 唐
 
OpenAPI development with Python
OpenAPI development with PythonOpenAPI development with Python
OpenAPI development with PythonTakuro Wada
 
OSCON 2012: ql.io and Node.js
OSCON 2012: ql.io and Node.jsOSCON 2012: ql.io and Node.js
OSCON 2012: ql.io and Node.jsJonathan LeBlanc
 
API Workshop: Deep dive into REST APIs
API Workshop: Deep dive into REST APIsAPI Workshop: Deep dive into REST APIs
API Workshop: Deep dive into REST APIsTom Johnson
 
Implementing a build manager in Ada
Implementing a build manager in AdaImplementing a build manager in Ada
Implementing a build manager in AdaStephane Carrez
 
Google App Engine in 40 minutes (the absolute essentials)
Google App Engine in 40 minutes (the absolute essentials)Google App Engine in 40 minutes (the absolute essentials)
Google App Engine in 40 minutes (the absolute essentials)Python Ireland
 
C# .NET Developer Portfolio
C# .NET Developer PortfolioC# .NET Developer Portfolio
C# .NET Developer Portfoliocummings49
 
SharePoint for the .NET Developer
SharePoint for the .NET DeveloperSharePoint for the .NET Developer
SharePoint for the .NET DeveloperJohn Calvert
 
High quality ap is with api platform
High quality ap is with api platformHigh quality ap is with api platform
High quality ap is with api platformNelson Kopliku
 
Use Eclipse technologies to build a modern embedded IDE
Use Eclipse technologies to build a modern embedded IDEUse Eclipse technologies to build a modern embedded IDE
Use Eclipse technologies to build a modern embedded IDEBenjamin Cabé
 

Similar to Making Things Work Together (20)

ql.io at NodePDX
ql.io at NodePDXql.io at NodePDX
ql.io at NodePDX
 
How to create an Angular builder
How to create an Angular builderHow to create an Angular builder
How to create an Angular builder
 
Optimizing Code Reusability for SharePoint using Linq to SharePoint & the MVP...
Optimizing Code Reusability for SharePoint using Linq to SharePoint & the MVP...Optimizing Code Reusability for SharePoint using Linq to SharePoint & the MVP...
Optimizing Code Reusability for SharePoint using Linq to SharePoint & the MVP...
 
ql.io: Consuming HTTP at Scale
ql.io: Consuming HTTP at Scale ql.io: Consuming HTTP at Scale
ql.io: Consuming HTTP at Scale
 
Bapi jco
Bapi jcoBapi jco
Bapi jco
 
Real World Asp.Net WebApi Applications
Real World Asp.Net WebApi ApplicationsReal World Asp.Net WebApi Applications
Real World Asp.Net WebApi Applications
 
O365 Saturday - Deepdive SharePoint Client Side Rendering
O365 Saturday - Deepdive SharePoint Client Side RenderingO365 Saturday - Deepdive SharePoint Client Side Rendering
O365 Saturday - Deepdive SharePoint Client Side Rendering
 
Parse cloud code
Parse cloud codeParse cloud code
Parse cloud code
 
OpenAPI development with Python
OpenAPI development with PythonOpenAPI development with Python
OpenAPI development with Python
 
OSCON 2012: ql.io and Node.js
OSCON 2012: ql.io and Node.jsOSCON 2012: ql.io and Node.js
OSCON 2012: ql.io and Node.js
 
API Workshop: Deep dive into REST APIs
API Workshop: Deep dive into REST APIsAPI Workshop: Deep dive into REST APIs
API Workshop: Deep dive into REST APIs
 
Implementing a build manager in Ada
Implementing a build manager in AdaImplementing a build manager in Ada
Implementing a build manager in Ada
 
Google App Engine in 40 minutes (the absolute essentials)
Google App Engine in 40 minutes (the absolute essentials)Google App Engine in 40 minutes (the absolute essentials)
Google App Engine in 40 minutes (the absolute essentials)
 
IOC + Javascript
IOC + JavascriptIOC + Javascript
IOC + Javascript
 
AEM 6.X (With Basics) Training Syllabus
AEM 6.X (With Basics) Training SyllabusAEM 6.X (With Basics) Training Syllabus
AEM 6.X (With Basics) Training Syllabus
 
Salesforce and sap integration
Salesforce and sap integrationSalesforce and sap integration
Salesforce and sap integration
 
C# .NET Developer Portfolio
C# .NET Developer PortfolioC# .NET Developer Portfolio
C# .NET Developer Portfolio
 
SharePoint for the .NET Developer
SharePoint for the .NET DeveloperSharePoint for the .NET Developer
SharePoint for the .NET Developer
 
High quality ap is with api platform
High quality ap is with api platformHigh quality ap is with api platform
High quality ap is with api platform
 
Use Eclipse technologies to build a modern embedded IDE
Use Eclipse technologies to build a modern embedded IDEUse Eclipse technologies to build a modern embedded IDE
Use Eclipse technologies to build a modern embedded IDE
 

More from Subbu Allamaraju

What Worked for Netflix May Not Work for You (OSCON-2018)
What Worked for Netflix May Not Work for You (OSCON-2018)What Worked for Netflix May Not Work for You (OSCON-2018)
What Worked for Netflix May Not Work for You (OSCON-2018)Subbu Allamaraju
 
Are We Ready for Serverless
Are We Ready for ServerlessAre We Ready for Serverless
Are We Ready for ServerlessSubbu Allamaraju
 
How to Sell Serverless to Your Colleagues
How to Sell Serverless to Your ColleaguesHow to Sell Serverless to Your Colleagues
How to Sell Serverless to Your ColleaguesSubbu Allamaraju
 
Turning Containers into Cattle
Turning Containers into CattleTurning Containers into Cattle
Turning Containers into CattleSubbu Allamaraju
 
Keystone at the Center of Our Universe
Keystone at the Center of Our UniverseKeystone at the Center of Our Universe
Keystone at the Center of Our UniverseSubbu Allamaraju
 
Journey and future of OpenStack eBay and PayPal
Journey and future of OpenStack eBay and PayPalJourney and future of OpenStack eBay and PayPal
Journey and future of OpenStack eBay and PayPalSubbu Allamaraju
 
RESTful Web Apps - Facts vs Fiction
RESTful Web Apps - Facts vs FictionRESTful Web Apps - Facts vs Fiction
RESTful Web Apps - Facts vs FictionSubbu Allamaraju
 

More from Subbu Allamaraju (15)

Five Rules
Five RulesFive Rules
Five Rules
 
Leading a Transformation
Leading a TransformationLeading a Transformation
Leading a Transformation
 
Taming the Rate of Change
Taming the Rate of ChangeTaming the Rate of Change
Taming the Rate of Change
 
What Worked for Netflix May Not Work for You (OSCON-2018)
What Worked for Netflix May Not Work for You (OSCON-2018)What Worked for Netflix May Not Work for You (OSCON-2018)
What Worked for Netflix May Not Work for You (OSCON-2018)
 
Are We Ready for Serverless
Are We Ready for ServerlessAre We Ready for Serverless
Are We Ready for Serverless
 
How to Sell Serverless to Your Colleagues
How to Sell Serverless to Your ColleaguesHow to Sell Serverless to Your Colleagues
How to Sell Serverless to Your Colleagues
 
Turning Containers into Cattle
Turning Containers into CattleTurning Containers into Cattle
Turning Containers into Cattle
 
Keystone at the Center of Our Universe
Keystone at the Center of Our UniverseKeystone at the Center of Our Universe
Keystone at the Center of Our Universe
 
Journey and future of OpenStack eBay and PayPal
Journey and future of OpenStack eBay and PayPalJourney and future of OpenStack eBay and PayPal
Journey and future of OpenStack eBay and PayPal
 
Engineering operations
Engineering operationsEngineering operations
Engineering operations
 
Open stack@ebay
Open stack@ebayOpen stack@ebay
Open stack@ebay
 
Measuring REST
Measuring RESTMeasuring REST
Measuring REST
 
REST: Theory vs Practice
REST: Theory vs PracticeREST: Theory vs Practice
REST: Theory vs Practice
 
RESTful Web Apps - Facts vs Fiction
RESTful Web Apps - Facts vs FictionRESTful Web Apps - Facts vs Fiction
RESTful Web Apps - Facts vs Fiction
 
Pragmatic Rest
Pragmatic RestPragmatic Rest
Pragmatic Rest
 

Recently uploaded

Basic Building Blocks of Internet of Things.
Basic Building Blocks of Internet of Things.Basic Building Blocks of Internet of Things.
Basic Building Blocks of Internet of Things.YounusS2
 
Machine Learning Model Validation (Aijun Zhang 2024).pdf
Machine Learning Model Validation (Aijun Zhang 2024).pdfMachine Learning Model Validation (Aijun Zhang 2024).pdf
Machine Learning Model Validation (Aijun Zhang 2024).pdfAijun Zhang
 
OpenShift Commons Paris - Choose Your Own Observability Adventure
OpenShift Commons Paris - Choose Your Own Observability AdventureOpenShift Commons Paris - Choose Your Own Observability Adventure
OpenShift Commons Paris - Choose Your Own Observability AdventureEric D. Schabell
 
How Accurate are Carbon Emissions Projections?
How Accurate are Carbon Emissions Projections?How Accurate are Carbon Emissions Projections?
How Accurate are Carbon Emissions Projections?IES VE
 
Secure your environment with UiPath and CyberArk technologies - Session 1
Secure your environment with UiPath and CyberArk technologies - Session 1Secure your environment with UiPath and CyberArk technologies - Session 1
Secure your environment with UiPath and CyberArk technologies - Session 1DianaGray10
 
Comparing Sidecar-less Service Mesh from Cilium and Istio
Comparing Sidecar-less Service Mesh from Cilium and IstioComparing Sidecar-less Service Mesh from Cilium and Istio
Comparing Sidecar-less Service Mesh from Cilium and IstioChristian Posta
 
NIST Cybersecurity Framework (CSF) 2.0 Workshop
NIST Cybersecurity Framework (CSF) 2.0 WorkshopNIST Cybersecurity Framework (CSF) 2.0 Workshop
NIST Cybersecurity Framework (CSF) 2.0 WorkshopBachir Benyammi
 
Bird eye's view on Camunda open source ecosystem
Bird eye's view on Camunda open source ecosystemBird eye's view on Camunda open source ecosystem
Bird eye's view on Camunda open source ecosystemAsko Soukka
 
Cybersecurity Workshop #1.pptx
Cybersecurity Workshop #1.pptxCybersecurity Workshop #1.pptx
Cybersecurity Workshop #1.pptxGDSC PJATK
 
activity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdf
activity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdf
activity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdfJamie (Taka) Wang
 
IaC & GitOps in a Nutshell - a FridayInANuthshell Episode.pdf
IaC & GitOps in a Nutshell - a FridayInANuthshell Episode.pdfIaC & GitOps in a Nutshell - a FridayInANuthshell Episode.pdf
IaC & GitOps in a Nutshell - a FridayInANuthshell Episode.pdfDaniel Santiago Silva Capera
 
Linked Data in Production: Moving Beyond Ontologies
Linked Data in Production: Moving Beyond OntologiesLinked Data in Production: Moving Beyond Ontologies
Linked Data in Production: Moving Beyond OntologiesDavid Newbury
 
VoIP Service and Marketing using Odoo and Asterisk PBX
VoIP Service and Marketing using Odoo and Asterisk PBXVoIP Service and Marketing using Odoo and Asterisk PBX
VoIP Service and Marketing using Odoo and Asterisk PBXTarek Kalaji
 
COMPUTER 10: Lesson 7 - File Storage and Online Collaboration
COMPUTER 10: Lesson 7 - File Storage and Online CollaborationCOMPUTER 10: Lesson 7 - File Storage and Online Collaboration
COMPUTER 10: Lesson 7 - File Storage and Online Collaborationbruanjhuli
 
Building AI-Driven Apps Using Semantic Kernel.pptx
Building AI-Driven Apps Using Semantic Kernel.pptxBuilding AI-Driven Apps Using Semantic Kernel.pptx
Building AI-Driven Apps Using Semantic Kernel.pptxUdaiappa Ramachandran
 
IESVE Software for Florida Code Compliance Using ASHRAE 90.1-2019
IESVE Software for Florida Code Compliance Using ASHRAE 90.1-2019IESVE Software for Florida Code Compliance Using ASHRAE 90.1-2019
IESVE Software for Florida Code Compliance Using ASHRAE 90.1-2019IES VE
 
Building Your Own AI Instance (TBLC AI )
Building Your Own AI Instance (TBLC AI )Building Your Own AI Instance (TBLC AI )
Building Your Own AI Instance (TBLC AI )Brian Pichman
 
Meet the new FSP 3000 M-Flex800™
Meet the new FSP 3000 M-Flex800™Meet the new FSP 3000 M-Flex800™
Meet the new FSP 3000 M-Flex800™Adtran
 
UWB Technology for Enhanced Indoor and Outdoor Positioning in Physiological M...
UWB Technology for Enhanced Indoor and Outdoor Positioning in Physiological M...UWB Technology for Enhanced Indoor and Outdoor Positioning in Physiological M...
UWB Technology for Enhanced Indoor and Outdoor Positioning in Physiological M...UbiTrack UK
 

Recently uploaded (20)

Basic Building Blocks of Internet of Things.
Basic Building Blocks of Internet of Things.Basic Building Blocks of Internet of Things.
Basic Building Blocks of Internet of Things.
 
Machine Learning Model Validation (Aijun Zhang 2024).pdf
Machine Learning Model Validation (Aijun Zhang 2024).pdfMachine Learning Model Validation (Aijun Zhang 2024).pdf
Machine Learning Model Validation (Aijun Zhang 2024).pdf
 
OpenShift Commons Paris - Choose Your Own Observability Adventure
OpenShift Commons Paris - Choose Your Own Observability AdventureOpenShift Commons Paris - Choose Your Own Observability Adventure
OpenShift Commons Paris - Choose Your Own Observability Adventure
 
How Accurate are Carbon Emissions Projections?
How Accurate are Carbon Emissions Projections?How Accurate are Carbon Emissions Projections?
How Accurate are Carbon Emissions Projections?
 
Secure your environment with UiPath and CyberArk technologies - Session 1
Secure your environment with UiPath and CyberArk technologies - Session 1Secure your environment with UiPath and CyberArk technologies - Session 1
Secure your environment with UiPath and CyberArk technologies - Session 1
 
Comparing Sidecar-less Service Mesh from Cilium and Istio
Comparing Sidecar-less Service Mesh from Cilium and IstioComparing Sidecar-less Service Mesh from Cilium and Istio
Comparing Sidecar-less Service Mesh from Cilium and Istio
 
NIST Cybersecurity Framework (CSF) 2.0 Workshop
NIST Cybersecurity Framework (CSF) 2.0 WorkshopNIST Cybersecurity Framework (CSF) 2.0 Workshop
NIST Cybersecurity Framework (CSF) 2.0 Workshop
 
Bird eye's view on Camunda open source ecosystem
Bird eye's view on Camunda open source ecosystemBird eye's view on Camunda open source ecosystem
Bird eye's view on Camunda open source ecosystem
 
Cybersecurity Workshop #1.pptx
Cybersecurity Workshop #1.pptxCybersecurity Workshop #1.pptx
Cybersecurity Workshop #1.pptx
 
activity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdf
activity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdf
activity_diagram_combine_v4_20190827.pdfactivity_diagram_combine_v4_20190827.pdf
 
IaC & GitOps in a Nutshell - a FridayInANuthshell Episode.pdf
IaC & GitOps in a Nutshell - a FridayInANuthshell Episode.pdfIaC & GitOps in a Nutshell - a FridayInANuthshell Episode.pdf
IaC & GitOps in a Nutshell - a FridayInANuthshell Episode.pdf
 
Linked Data in Production: Moving Beyond Ontologies
Linked Data in Production: Moving Beyond OntologiesLinked Data in Production: Moving Beyond Ontologies
Linked Data in Production: Moving Beyond Ontologies
 
VoIP Service and Marketing using Odoo and Asterisk PBX
VoIP Service and Marketing using Odoo and Asterisk PBXVoIP Service and Marketing using Odoo and Asterisk PBX
VoIP Service and Marketing using Odoo and Asterisk PBX
 
COMPUTER 10: Lesson 7 - File Storage and Online Collaboration
COMPUTER 10: Lesson 7 - File Storage and Online CollaborationCOMPUTER 10: Lesson 7 - File Storage and Online Collaboration
COMPUTER 10: Lesson 7 - File Storage and Online Collaboration
 
Building AI-Driven Apps Using Semantic Kernel.pptx
Building AI-Driven Apps Using Semantic Kernel.pptxBuilding AI-Driven Apps Using Semantic Kernel.pptx
Building AI-Driven Apps Using Semantic Kernel.pptx
 
IESVE Software for Florida Code Compliance Using ASHRAE 90.1-2019
IESVE Software for Florida Code Compliance Using ASHRAE 90.1-2019IESVE Software for Florida Code Compliance Using ASHRAE 90.1-2019
IESVE Software for Florida Code Compliance Using ASHRAE 90.1-2019
 
Building Your Own AI Instance (TBLC AI )
Building Your Own AI Instance (TBLC AI )Building Your Own AI Instance (TBLC AI )
Building Your Own AI Instance (TBLC AI )
 
Meet the new FSP 3000 M-Flex800™
Meet the new FSP 3000 M-Flex800™Meet the new FSP 3000 M-Flex800™
Meet the new FSP 3000 M-Flex800™
 
201610817 - edge part1
201610817 - edge part1201610817 - edge part1
201610817 - edge part1
 
UWB Technology for Enhanced Indoor and Outdoor Positioning in Physiological M...
UWB Technology for Enhanced Indoor and Outdoor Positioning in Physiological M...UWB Technology for Enhanced Indoor and Outdoor Positioning in Physiological M...
UWB Technology for Enhanced Indoor and Outdoor Positioning in Physiological M...
 

Making Things Work Together

  • 1. Making things Work Together QCon NY June 20, 2012 1  
  • 3. How to build an interoperable Programmable Web? 3  
  • 4. Agreements 4  
  • 5. Discovery Linking, Identity Schemas, Media Types Formats Interfaces (Uniform, RPC) HTTP TCP 5  
  • 6. Server's point of view 6  
  • 7. Client's point of view 7  
  • 8. Several hundred APIs Different styles Written by different teams Over years Built under different constraints Built with different goals 8  
  • 9. Use case: Find things from A For each thing, find details from B For each thing, find more details from C Merge results 9  
  • 10. Use case: Find products Find dominant categories of products Look up category info Merge categories with products 10  
  • 11. Use case: Get stuff from A If A is down, try from B Annotate stuff with other stuff from C Ignore some things from the stuff Join all 11  
  • 12. [Really important client] Client Why don't you give me an API optimized for my use cases? Producers   Server [Really important producer] Thanks. Get a number and stand in the line! 12  
  • 13. [Really important client] Client Why don't you give me an API optimized for my use cases? Producers   Who gets to decide the right thing? Server [Really important producer] Thanks. Get a number and stand in the line! 13  
  • 14. Discovery Linking, Identity Schemas, Media Types Formats > Interfaces (Uniform, RPC) HTTP TCP 14  
  • 15. Discovery Linking, Identity > Schemas, Media Types Formats HTTP TCP 15  
  • 16. > Discovery > Linking, Identity Formats HTTP TCP 16  
  • 17. Formats HTTP TCP 17  
  • 18. Latency Smart Client Failure Recovery Code Orchestration IO Formats HTTP TCP 18  
  • 19. 19  
  • 20. 20  
  • 21. https://github.com/ql-io/ql.io Platform Engineering 21  
  • 22. A domain specific language for HTTP client programming 22  
  • 23. 23  
  • 24. 24  
  • 26. Test console Find script Compile HTTP interface Request state HTTP client Evented orchestration Transformations A runtime designed for IO workloads Runs on node.js 26  
  • 27. 1. Interop via HTTP Support the protocol – but don't require style adherence   27  
  • 28. create  table  mytable1      on  select        get  from  'http://...';     create  table  mytable2      on  insert      get  from  'http://...&do=create'           create  table  mytable3      on  delete      post  to  'http://...'      using  headers  'X-­‐Do'  =  'Delete'     28  
  • 29. 2. Interop using common formats Agnostic of domain specific type systems   29  
  • 30. /*  A  table  with  a  body  template  */   create  table  bing.soap.search      on  select  post  to  'http://api.bing.net/ soap.asmx'      using  defaults  appid  =  'xxx'      using  bodyTemplate  'bing.xml.mu'        type  'application/xml'      resultset   'soapenv:Envelope.soapenv:Body.SearchResponse.pa rameters.Web.Results.WebResult';   30  
  • 31. <soapenv:Envelope  xmlns:soapenv="http://schemas.xmlsoap.org/ soap/envelope/">      <soapenv:Header/>      <soapenv:Body>          <SearchRequest  xmlns="http://schemas.microsoft.com/ LiveSearch/2008/03/Search">              <parameters>              {{#params}}              <Query>{{q}}</Query>              <AppId>{{appid}}</AppId>              {{/params}}              <Sources>                  <SourceType>Web</SourceType>              </Sources>              </parameters>          </SearchRequest>      </soapenv:Body>   </soapenv:Envelope>   31  
  • 32. -­‐-­‐  Form  encoded  body   create  table  mytable      on  insert  post  to  'http://...'      using  bodyTemplate  'somefile.txt'          type  'application/x-­‐www-­‐form-­‐urlencoded'     -­‐-­‐  Or  JSON   create  table  mytable      on  update  put  to  'http://...'      using  bodyTemplate  'file.json'  type   'application/json'   32  
  • 33. create  table  {a  name}   on  {select|insert|update|delete}        {get  from|post  to|          put  to|delete|patch}          {a  URI  template}   using  headers  {name}={value}*   using  bodyTemplate={a  file}                type={media  type}   33  
  • 34. HTTP  method   URI  template   Headers   Body   Media  type   34  
  • 35. -­‐-­‐  A  table  with  a  monkey  patch   create  table  bitly.shorten      on  insert  get  from  "..."          using  patch  'bitly.js'      on  select  get  from  "..."          using  patch  'bitly.js'     /*  bitly.js  A  Javascript  monkey  patch  */   exports['patch  status']  =  function(options)  {          return  args.body  ?                args.body.status_code  :  200;     };   35  
  • 36. 3. SQL inspired constructs CRUD operations, filtering, joins   36  
  • 37. -­‐-­‐  Get  the  complete  response  as  JSON   select  *  from  google.geocode  where      address  =  'Brooklyn,  NY'     -­‐-­‐  Project  fields   select  ProductID[0].Value,        DetailsURL,  StockPhotoURL      from  eBay.FindProducts        where  QueryKeywords='mini  cooper'   37  
  • 38. /*  Join  datasets  without  enforcing  identity          across  systems  */     select  e.ItemID  as  id,  e.Title  as  title,        e.ViewItemURLForNaturalSearch  as  url,        g.geometry.location  as  latlng      from  details    as  e,  google.geocode  as  g      where  e.itemId  in  (select  itemId  from        finditems  where  keywords  =  "iPad")      and  g.address  =  e.Location       38  
  • 39. 4. Orchestration Implicit sequential, scatter, parallel, forks and joins   39  
  • 40. -­‐-­‐  Sequential   minis  =  select  *  from  finditems  where            keywords  =  'mini  cooper'  limit  10;   return  select  PictureURL  from  details  where            itemId  =  "{minis.itemId}";     -­‐-­‐  Or  parallel   keyword  =  "ql.io";   web  =  select  *  from  bing.search  where  q  =  "{keyword}";   tweets  =  select  id  as  id,  from_user_name  as  user_name,          text  as  text    from  twitter.search  where  q  =  "ql.io";       return  {      "keyword":  "{keyword}",      "web":  "{web}",      "tweets":  "{tweets}"   }   40  
  • 41. prodid  =  select  ProductID[0].Value  from  eBay.FindProducts            where  QueryKeywords  =  'macbook  pro';     -­‐-­‐  Scatters  "n"  requests   details  =  select  *  from  eBay.ProductDetails  where          ProductID  in  ('{prodid}')  and            ProductType  =  'Reference';     -­‐-­‐  Scatters  "n"  requests   reviews  =  select  *  from  eBay.ProductReviews  where          ProductID  in  ('{prodid}')  and            ProductType  =  'Reference';     -­‐-­‐  Gather,  merge,  and  join   return  select  d.ProductID[0].Value  as  id,            d.Title  as  title,  d.ReviewCount  as  reviewCount,          r.ReviewDetails.AverageRating  as  rating      from  details  as  d,  reviews  as  r          where  d.ProductID[0].Value  =  r.ProductID.Value   41  
  • 42. -­‐-­‐  An  API  that  takes  one  ID  at  a  time   create  table  eBay.ProductDetails      on  select  get  from  "...&...={^ProductID}"      ...        resultset  'Product';       -­‐-­‐  A  batch  API  –  takes  upto  20  IDs   create  table  details      on  select  get  from  "...&ItemID={20|itemId}      ...      resultset  'Item';       42  
  • 43. 43  
  • 44. 5. Failure Modes Timeouts, back-off and recovery   44  
  • 45. -­‐-­‐  Timeout  on  idle  sockets     -­‐-­‐  Retry  once  for  idempotent  statements   select  id,  img  from  recommendations      timeout  100     -­‐-­‐  On  three  successive  failures,  backoff     -­‐-­‐  for  100  –  2000  msec   select  id,  img  from  recommendations      timeout  100  minDelay  100  maxDelay  2000     45  
  • 46. -­‐-­‐  Use  a  fallback     select  id  as  id,  item_url  as  img        from  recommendations      timeout  100        ||      select  ID  as  id,  deal_url  as  img          from  deals;   46  
  • 48. var  Engine  =  require('ql.io-­‐engine'),          fs  =  require('fs');   var  engine  =  new  Engine({          tables  :  __dirname  +  '/../tables',          config:  __dirname  +  '/../config/dev.json'   });   var  script  =  fs.readFileSync(__dirname  +  '/some.ql',  'UTF-­‐8');     //  Exec  the  script   engine.execute(script,  function(emitter)  {          emitter.on('end',  function(err,  result)  {                  result.body.forEach(function(row)  {                          console.log(row);                  });          });   });     48  
  • 49. Push orchestration to the data center   •  Reduce HTTP requests •  Reduce bandwidth use 49  
  • 50. /*  Shows  daily  deals  -­‐  use  siteId=0  for  US  and  siteId=3   for  UK.  */   resp  =  select  *  from  dailydeals  where  siteId="{siteId}";   deals  =  "{resp.$..Item}";   itemDetails  =  select  ItemID  as  itemId,  Title  as  title,        GalleryURL  as  pic,  Seller.UserID  as  sellerUserId,            Seller.FeedbackScore  as  rating,        Seller.PositiveFeedbackPercent  as  feedback,      HitCount  as  hits  from  details        where  itemId  in  (deals.ItemID);     return  itemDetails  via  route  '/deals/{siteId}'  using   method  get;   50  
  • 51. Latency Failure Recovery Orchestration IO workloads Formats Server HTTP TCP 51  
  • 52. code https://github.com/ql-io/ql.io docs/demos http://ql.io blog http://ql-io.github.com 52