Node js mongodriver

21,730 views
21,632 views

Published on

Hamburg presentation of the Node.js driver building a simple location based app using express.js and mongodb.

Published in: Technology
2 Comments
20 Likes
Statistics
Notes
No Downloads
Views
Total views
21,730
On SlideShare
0
From Embeds
0
Number of Embeds
4
Actions
Shares
0
Downloads
251
Comments
2
Likes
20
Embeds 0
No embeds

No notes for slide
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • \n
  • Node js mongodriver

    1. 1. An introduction to thenode.js mongo driver Christian Amor Kvalheim
    2. 2. • Asynchronous Javascript platform built on top of V8• Single-threaded• Package manager• Awesome
    3. 3. Hello World Servervar http = require(http);http.createServer( function (req, res) { res.writeHead(200, {Content-Type: text/plain}); res.end(Hello Worldn); }).listen(8124, "127.0.0.1");
    4. 4. Hello World Server Load the http libraryvar http = require(http);http.createServer( function (req, res) { res.writeHead(200, {Content-Type: text/plain}); res.end(Hello Worldn); }).listen(8124, "127.0.0.1");
    5. 5. Hello World Server Load the http libraryvar http = require(http); Set up the connectionhttp.createServer( handler function (req, res) { res.writeHead(200, {Content-Type: text/plain}); res.end(Hello Worldn); }).listen(8124, "127.0.0.1");
    6. 6. How to get started• Install node.js (from source or package)• Install npm (node package manager)• Create an empty directory• npm install mongodb• npm install express
    7. 7. Let’s do some code• Using Express and Mongo• Textmate basic.js
    8. 8. Picking it apartvar db = new Db(node-mongo-examples, new Server(host, port, {}),{native_parser:false});db.open(function(err, db) { ... ..... app.listen(8124);var app = express.createServer();app.get(/, function(req, res){ res.send(Hello World);});
    9. 9. Picking it apart Creates a db using the js bson parservar db = new Db(node-mongo-examples, new Server(host, port, {}),{native_parser:false});db.open(function(err, db) { ... ..... app.listen(8124);var app = express.createServer();app.get(/, function(req, res){ res.send(Hello World);});
    10. 10. Picking it apart Creates a db using the js bson parservar db = new Db(node-mongo-examples, new Server(host, port, {}),{native_parser:false}); Opens the connection to the dbdb.open(function(err, db) { ... ..... app.listen(8124);var app = express.createServer();app.get(/, function(req, res){ res.send(Hello World);});
    11. 11. Picking it apart Creates a db using the js bson parservar db = new Db(node-mongo-examples, new Server(host, port, {}),{native_parser:false}); Opens the connection to the dbdb.open(function(err, db) { ... ..... app.listen(8124); The server is now running with access to the mongo server connectionvar app = express.createServer();app.get(/, function(req, res){ res.send(Hello World);});
    12. 12. Let’s add some meat• CRUD operations• Textmate basic_2.js
    13. 13. Picking it apart// Create methodapp.post(/location, function(req, res) { geoCodeDecorateObject(req.body.address, {description:req.body.description},function(err, object) { db.collection(locations, function(err, collection) { // Insert doc collection.insert(object, {safe:true}, function(err, result) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_2.jade, {locals: {locations:items}}); }) }); }); });});
    14. 14. Picking it apartGeo Encode Address// Create methodapp.post(/location, function(req, res) { geoCodeDecorateObject(req.body.address, {description:req.body.description},function(err, object) { db.collection(locations, function(err, collection) { // Insert doc collection.insert(object, {safe:true}, function(err, result) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_2.jade, {locals: {locations:items}}); }) }); }); });});
    15. 15. Picking it apart Geo Encode Address // Create method app.post(/location, function(req, res) { geoCodeDecorateObject(req.body.address, {description:req.body.description}, function(err, object) { db.collection(locations, function(err, collection) { // Insert doc collection.insert(object, {safe:true}, function(err, result) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_2.jade, {locals: {locations:items}}); })Insert record, safe ensures that we trap any }); }); }); errors by doing a error check against }); mongodb
    16. 16. Picking it apart // Create method app.post(/location, function(req, res) { geoCodeDecorateObject(req.body.address, {description:req.body.description}, function(err, object) { db.collection(locations, function(err, collection) { // Insert doc collection.insert(object, {safe:true}, function(err, result) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_2.jade, {locals: {locations:items}}); }) }); }); }); find all records from the locations });collection. Find returns a cursor that allows for stepping through doucments
    17. 17. Safe or not• Mongo Insert/Update/Delete are async• 2nd call to lastError required to check for the success of the operation• safe option ensures the second error call• you can also run the driver in strict mode
    18. 18. Let’s add some meat• CRUD operations• Textmate basic_3.js
    19. 19. Picking it apart// Delete methodapp.del(/location, function(req, res) { var id = ObjectID.createFromHexString(req.body.id); db.collection(locations, function(err, collection) { collection.remove({_id:id}, {safe:true}, function(err, numberOfDeletedRecords) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_3.jade, {locals: {locations:items}}); }) }) });});
    20. 20. Picking it apart Convert hex string to// Delete method objectIDapp.del(/location, function(req, res) { var id = ObjectID.createFromHexString(req.body.id); db.collection(locations, function(err, collection) { collection.remove({_id:id}, {safe:true}, function(err, numberOfDeletedRecords) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_3.jade, {locals: {locations:items}}); }) }) });});
    21. 21. Picking it apart Convert hex string to// Delete method objectIDapp.del(/location, function(req, res) { var id = ObjectID.createFromHexString(req.body.id); db.collection(locations, function(err, collection) { collection.remove({_id:id}, {safe:true}, function(err, numberOfDeletedRecords) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_3.jade, {locals: {locations:items}}); }) }) Remove the document using the specific id });}); passed in, using safe.
    22. 22. Let’s add some meat• CRUD operations• Textmate basic_4.js
    23. 23. Picking it apart// Get methodapp.get(/location, function(req, res) { var id = ObjectID.createFromHexString(req.body.id); db.collection(locations, function(err, collection) { collection.findOne({_id:id}, function(err, item) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_4.jade, {locals: {locations:items, location:item}}); }) }) });});
    24. 24. Picking it apart Convert hex string to// Get methodapp.get(/location, function(req, res) { objectID var id = ObjectID.createFromHexString(req.body.id); db.collection(locations, function(err, collection) { collection.findOne({_id:id}, function(err, item) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_4.jade, {locals: {locations:items, location:item}}); }) }) });});
    25. 25. Picking it apart Convert hex string to// Get methodapp.get(/location, function(req, res) { objectID var id = ObjectID.createFromHexString(req.body.id); db.collection(locations, function(err, collection) { collection.findOne({_id:id}, function(err, item) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_4.jade, {locals: {locations:items, location:item}}); }) }) Locate one document by id });});
    26. 26. Let’s add some meat• CRUD operations• Textmate basic_5.js
    27. 27. Picking it apart// Update methodapp.put(/location, function(req, res) { var id = ObjectID.createFromHexString(req.body.id); db.collection(locations, function(err, collection) { collection.findOne({_id:id}, function(err, object) { object.description = req.body.description; object.address = req.body.address; geoCodeDecorateObject(req.body.address, object, function(err, object) { collection.update({_id:object._id}, object, {safe:true}, function(err,numberOfUpdatedObjects) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_5.jade, {locals: {locations:items}}); }) }) })
    28. 28. Picking it apart// Update methodapp.put(/location, function(req, res) { Locate object we wish var id = ObjectID.createFromHexString(req.body.id); to modify db.collection(locations, function(err, collection) { collection.findOne({_id:id}, function(err, object) { object.description = req.body.description; object.address = req.body.address; geoCodeDecorateObject(req.body.address, object, function(err, object) { collection.update({_id:object._id}, object, {safe:true}, function(err,numberOfUpdatedObjects) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_5.jade, {locals: {locations:items}}); }) }) })
    29. 29. Picking it apart// Update methodapp.put(/location, function(req, res) { Locate object we wish var id = ObjectID.createFromHexString(req.body.id); to modify db.collection(locations, function(err, collection) { collection.findOne({_id:id}, function(err, object) { object.description = req.body.description; object.address = req.body.address; geoCodeDecorateObject(req.body.address, object, function(err, object) { collection.update({_id:object._id}, object, {safe:true}, function(err,numberOfUpdatedObjects) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_5.jade, {locals: {locations:items}}); }) }) }) Update document using the id to select what document to update
    30. 30. Let’s add some meat• Final with search• Textmate basic_6.js
    31. 31. Picking it apartdb.open(function(err, db) { if(err) throw err // !!! CHANGE db.ensureIndex("locations", {loc:"2d"}, function(err, result) { if(err) throw err app.listen(8124); })});
    32. 32. Picking it apartdb.open(function(err, db) { if(err) throw err // !!! CHANGE db.ensureIndex("locations", {loc:"2d"}, function(err, result) { if(err) throw err app.listen(8124); })}); Ensures there is a 2d geospatial index on the attribute loc on any document in the locations collection
    33. 33. Picking it apartvar geoCodeDecorateObject = function(address, object, callback) { var googleGeoCodeApi = {host: maps.googleapis.com, port: 80,path: /maps/api/geocode/json?sensor=false&address= +escape(address), method: GET }; var clientReq = http.get(googleGeoCodeApi, function(clientRes) { var data = []; clientRes.on(data, function(chunk) { data.push(chunk.toString()); }); clientRes.on(end, function() { var googleObject = JSON.parse(data.join()); object.address = address; object.geodata = googleObject.results.pop(); // !!! CHANGE object.loc = {long:object.geodata.geometry.location.lng,lat:object.geodata.geometry.location.lat}; callback(null, object); });
    34. 34. Picking it apartvar geoCodeDecorateObject = function(address, object, callback) { var googleGeoCodeApi = {host: maps.googleapis.com, port: 80,path: /maps/api/geocode/json?sensor=false&address= +escape(address), method: GET }; var clientReq = http.get(googleGeoCodeApi, function(clientRes) { var data = []; clientRes.on(data, function(chunk) { data.push(chunk.toString()); We are adding a loc }); clientRes.on(end, function() { attribute to all var googleObject = JSON.parse(data.join()); documents with object.address = address; object.geodata = googleObject.results.pop(); longtidude and // !!! CHANGE latitude in the right object.loc = {long:object.geodata.geometry.location.lng,lat:object.geodata.geometry.location.lat}; order for mongodb to callback(null, object); }); search
    35. 35. Picking it apart// !!! CHANGE// Search optionapp.post(/search, function(req, res) { geoCodeDecorateObject(req.body.address, {}, function(err, object) { // Unpack geo object var long = object.geodata.geometry.location.lng; var lat = object.geodata.geometry.location.lat db.collection(locations, function(err, collection) { collection.find({loc : {$near: [long, lat], $maxDistance:parseFloat(req.body.distance)}}).toArray(function(err, geoItems) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_6.jade, {locals: {locations:items,results:geoItems}}); }) }); }); });});
    36. 36. Picking it apart Encode address for// !!! CHANGE// Search option searchingapp.post(/search, function(req, res) { geoCodeDecorateObject(req.body.address, {}, function(err, object) { // Unpack geo object var long = object.geodata.geometry.location.lng; var lat = object.geodata.geometry.location.lat db.collection(locations, function(err, collection) { collection.find({loc : {$near: [long, lat], $maxDistance:parseFloat(req.body.distance)}}).toArray(function(err, geoItems) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_6.jade, {locals: {locations:items,results:geoItems}}); }) }); }); });});
    37. 37. Picking it apart Encode address for// !!! CHANGE// Search option searchingapp.post(/search, function(req, res) { geoCodeDecorateObject(req.body.address, {}, function(err, object) { // Unpack geo object var long = object.geodata.geometry.location.lng; var lat = object.geodata.geometry.location.lat db.collection(locations, function(err, collection) { collection.find({loc : {$near: [long, lat], $maxDistance:parseFloat(req.body.distance)}}).toArray(function(err, geoItems) { // Fetch all docs for rendering of list collection.find({}).toArray(function(err, items) { res.render(./basic_6.jade, {locals: {locations:items,results:geoItems}}); }) }); Search for all items $near our address at }); }); $maxDistance});
    38. 38. Other features• Cursors• Grid FS
    39. 39. Cursors• toArray Function• each Function• streamRecords Function
    40. 40. toArraycollection.find().toArray(function(err, documents){ test.deepEqual([1, 2, 3], documents[0].b); // Lets close the db test.done();});
    41. 41. toArray Fetches all docs in one go for the query. Use with cautioncollection.find().toArray(function(err, documents){ test.deepEqual([1, 2, 3], documents[0].b); // Lets close the db test.done();});
    42. 42. eachcollection.find({}, {sort:[[age, 1]]}).each(function(err, item){ if(item != null) { // Do work } else { // Finished }});
    43. 43. each Fetches docs in batches and allows iteration over resultscollection.find({}, {sort:[[age, 1]]}).each(function(err, item){ if(item != null) { // Do work } else { // Finished }});
    44. 44. each Fetches docs in batches and allows iteration over resultscollection.find({}, {sort:[[age, 1]]}).each(function(err, item){ if(item != null) { // Do work } else { // Finished }}); Returns null when no more results available
    45. 45. streamRecords var stream = collection.find({}, {limit : 3}).streamRecords();stream.on(end, function() { // No more results in the pipe});stream.on(data,function(data){ // Item});
    46. 46. streamRecords var stream = collection.find({}, {limit : 3}).streamRecords();stream.on(end, function() { // No more results in the pipe});stream.on(data,function(data){ // Item}); When an item is ready the event ‘data’ is triggered
    47. 47. streamRecords var stream = collection.find({}, {limit : 3}).streamRecords(); When no more itemsstream.on(end, function() { are available // No more results in the pipe});stream.on(data,function(data){ // Item}); When an item is ready the event ‘data’ is triggered
    48. 48. streamRecords var stream = collection.find({}, {limit : 3}).streamRecords(); When no more itemsstream.on(end, function() { are available // No more results in the pipe}); REFERED Pstream.on(data,function(data){ // Item}); When an item is ready the event ‘data’ is triggered
    49. 49. Grid FS• Write and read a file• Stream a file
    50. 50. Write a filevar gridStore = new GridStore(client, test_gs_writing_file, w);gridStore.open(function(err, gridStore) {gridStore.writeFile(./test_gs_weird_bug.png, function(err, gridStore) { GridStore.read(client, test_gs_writing_file, function(err, fileData) { });
    51. 51. Write a filevar gridStore = new GridStore(client, test_gs_writing_file, w);gridStore.open(function(err, gridStore) {gridStore.writeFile(./test_gs_weird_bug.png, function(err, gridStore) { Writes the file in chunks to mongodb (avoids freezing the eventloop in node.js) GridStore.read(client, test_gs_writing_file, function(err, fileData) { });
    52. 52. Write a filevar gridStore = new GridStore(client, test_gs_writing_file, w);gridStore.open(function(err, gridStore) {gridStore.writeFile(./test_gs_weird_bug.png, function(err, gridStore) { Writes the file in chunks to mongodb (avoids freezing the eventloop in node.js) GridStore.read(client, test_gs_writing_file, function(err, fileData) { }); Read the whole file. Careful with the memory
    53. 53. Stream a filevar gridStore = new GridStore(client, "test_gs_read_stream", "r");gridStore.open(function(err, gs) { var stream = gs.stream(true); stream.on("data", function(chunk) { // Received a chunk of data }); stream.on("end", function() { // Finished streaming });});
    54. 54. Stream a filevar gridStore = new GridStore(client, "test_gs_read_stream", "r");gridStore.open(function(err, gs) { var stream = gs.stream(true); Receive a chunk and do something stream.on("data", function(chunk) { // Received a chunk of data }); stream.on("end", function() { // Finished streaming });});
    55. 55. Stream a filevar gridStore = new GridStore(client, "test_gs_read_stream", "r");gridStore.open(function(err, gs) { var stream = gs.stream(true); Receive a chunk and do something stream.on("data", function(chunk) { // Received a chunk of data }); Finshed stream the file stream.on("end", function() { // Finished streaming });});
    56. 56. Stream a filevar gridStore = new GridStore(client, "test_gs_read_stream", "r");gridStore.open(function(err, gs) { var stream = gs.stream(true); Receive a chunk and do something stream.on("data", function(chunk) { // Received a chunk of data }); Finshed stream the file stream.on("end", function() { // Finished streaming PR EFE RED });});
    57. 57. Typical starting problemsfor(var i = 0; i < 100; i++) { collection.insert({i:i});}collection.count(function(err, count) { // count is > 0 <= 100});
    58. 58. Typical starting problemsfor(var i = 0; i < 100; i++) { collection.insert({i:i});} All executed in parallel, non- deterministic end of executioncollection.count(function(err, count) { // count is > 0 <= 100});
    59. 59. Typical starting problemsfor(var i = 0; i < 100; i++) { collection.insert({i:i});} All executed in parallel, non- deterministic end of executioncollection.count(function(err, count) { // count is > 0 <= 100}); All inserts MAY have been written to disk so passes most of the time. But with Pooling NO WAY
    60. 60. Typical starting problemsfor(var i = 0; i < 100; i++) { collection.insert({i:i}, {safe:true}, function(err, result) { // Do something });}??????????????????????????????collection.count(function(err, count) { // count is > 0 <= 100});
    61. 61. Typical starting problemsfor(var i = 0; i < 100; i++) { collection.insert({i:i}, {safe:true}, function(err, result) { // Do something });} How do we handle the callbacks??????????????????????????????collection.count(function(err, count) { // count is > 0 <= 100});
    62. 62. Typical starting problemsfor(var i = 0; i < 100; i++) { collection.insert({i:i}, {safe:true}, function(err, result) { // Do something });} How do we handle the callbacks??????????????????????????????collection.count(function(err, count) { // count is > 0 <= 100}); How do we get to this code ??????
    63. 63. Typical starting problemsfor(var i = 0; i < 100; i++) { collection.insert({i:i}, {safe:true}, function(err, result) { // Do something });} How do we handle the callbacks??????????????????????????????collection.count(function(err, count) { // count is > 0 <= 100}); How do we get to this code ?????? Many possible solutions
    64. 64. My Choosen Solution https://github.com/creationix/step Step( function insert() { var group = this.group(); for(var i = 0; i < 100; i++) { collection.insert({i:i}, {safe:true}, group()); } }, function finished() { collection.count(function(err, count) { // Count is 100 :) }); })
    65. 65. My Choosen Solution https://github.com/creationix/step Step( function insert() { var group = this.group(); for(var i = 0; i < 100; i++) { collection.insert({i:i}, {safe:true}, group()); } }, Ensures all the inserts are finished before calling the next function function finished() { collection.count(function(err, count) { // Count is 100 :) }); })
    66. 66. My Choosen Solution https://github.com/creationix/step Step( function insert() { var group = this.group(); for(var i = 0; i < 100; i++) { collection.insert({i:i}, {safe:true}, group()); } }, Ensures all the inserts are finished before calling the next function function finished() { collection.count(function(err, count) { // Count is 100 :) }); }) When all inserts are done :)
    67. 67. DONE Code onhttps://github.com/christkv/mongodb-hamburg

    ×