MongoDB's oplog is possibly its most underrated feature. The oplog is vital as the basis on which replication is built, but its value doesn't stop there. Unlike the MySQL binlog, which is poorly documented and not directly exposed to MySQL clients, the oplog is a well-documented, structured format for changes that is query-able through the same mechanisms as your data. This allows many types of powerful, application-driven streaming or transformation. At Stripe, we've used the MongoDB oplog to create PostgresSQL, HBase, and ElasticSearch mirrors of our data. We've built a simple real-time trigger mechanism for detecting new data. And we've even used it to recover data. In this talk, we'll show you how we use the MongoDB oplog, and how you can build powerful reactive streaming data applications on top of it.
32. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
33. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
34. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
35. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
36.
37.
38.
39.
40.
41.
42. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
43. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
44. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
45. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
51. cursor.each do |op|
case op['op']
when 'i'
puts op['o']['_id']
else
# ¯_(ツ)_/¯
end
end
52. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') +
') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
else
# ¯_(ツ)_/¯
end
end
53.
54.
55.
56. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') +
') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end
57.
58.
59.
60. query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id="
query += op['o2']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end
61. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') + ') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id=" + op['o2']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end
74. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == nil
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end
75. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == false
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end
76. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == false
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end