1. MongoDB and
the Oplog
EVAN BRODER @ebroder

4. MongoDB and
the Oplog
EVAN BRODER @ebroder

5. AGENDA
INTRO TO THE OPLOG
EXAMPLE APPLICATIONS

6. INTRO
TO THE OPLOG

7. PRIMARY
SECONDARIES
APPLICATION

8. APPLICATION
save
{_id: 1, a: 2}

9. THINGS I’VE DONE:
- save {_id: 1, a: 2}

10. APPLICATION
update where
{a: 2},
{$set: {a: 3}}

11. THINGS I’VE DONE:
- save {_id: 1, a: 2}
- update {_id: 1},
{$set: {a: 3}}

12. THINGS I’VE DONE:
- save {_id: 1, a: 2}
- update {_id: 1},
{$set: {a: 3}}
- insert…
- delete…
- delete…
- save…
- update…

13. THINGS I’VE
DONE:
save
…
THINGS I’VE
DONE:

14. THINGS I’VE
DONE:
save
…
THINGS I’VE
DONE:
save
…

23. TRIGGERS

24. GOAL:
EVENT PROCESSING

28. GOAL:
DETECT INSERTIONS

30. oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end

31. oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end

32. oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end

33. oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end

34. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

35. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

36. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

37. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

44. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end

45. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']

46. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

47. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

48. DATA
TRANSFORMATIONS

49. GOAL:
MONGODB TO POSTGRESQL

50. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

51. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

52. cursor.each do |op|
puts op['o']['_id']
end

53. cursor.each do |op|
case op['op']
when 'i'
puts op['o']['_id']
else
# ¯_(ツ)_/¯
end
end

54. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') +
') VALUES (' +
op['o'].values.map(&:inspect).join(', ')
else
# ¯_(ツ)_/¯
end
end

58. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') +
') VALUES (' +
op['o'].values.map(&:inspect).join(', ')
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end

64. query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id="
query += op['o2']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end

65. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') + ') VALUES (' +
op['o'].values.map(&:inspect).join(', ')
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id=" + op['o2']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end

66. github.com/stripe/mosql

67. github.com/stripe/zerowing

68. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') + ') VALUES (' +
op['o'].values.map(&:inspect).join(', ')
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id=" + op['o2']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end

69. DISASTER
RECOVERY

70. task = collection.find_one({'finished' => nil}
# do something with task…
collection.update({'_id' => task.id},
{'$set' => {'finished' => Time.now.to_i}})

71. loop do
collection.remove(
{'finished' => {'$lt' => Time.now.to_i - 30}})
sleep(10)
end

72. evan@caron:~$ mongo
MongoDB shell version: 2.4.10
connecting to: test
normal:PRIMARY> null < (Date.now() / 1000) - 30
true

75. > db.getReplicationInfo()
{
"logSizeMB" : 48964.3541015625,
"usedMB" : 46116.4,
"timeDiff" : 316550,
"timeDiffHours" : 87.93,
"tFirst" : "Thu Apr 11 2013 07:24:29 GMT+0000 (UTC)",
"tLast" : "Sun Apr 14 2013 23:20:19 GMT+0000 (UTC)",
"now" : "Sat May 24 2014 07:52:35 GMT+0000 (UTC)"
}

76. > db.getReplicationInfo()
{
"logSizeMB" : 48964.3541015625,
"usedMB" : 46116.4,
"timeDiff" : 316550,
"timeDiffHours" : 87.93,
"tFirst" : "Thu Apr 11 2013 07:24:29 GMT+0000 (UTC)",
"tLast" : "Sun Apr 14 2013 23:20:19 GMT+0000 (UTC)",
"now" : "Sat May 24 2014 07:52:35 GMT+0000 (UTC)"
}

77. THINGS I’VE
DONE:
insert
delete
…
THINGS I’VE
DONE:

78. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == nil
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end

79. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == false
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end

80. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == false
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end

81. THINGS I’VE
DONE:
save
…
THINGS I’VE
DONE:
save
…

86. QUESTIONS?