More Related Content Similar to MongoDB GridFS Stores and Retrieves Files Similar to MongoDB GridFS Stores and Retrieves Files (20) More from Takahiro Inoue (20) MongoDB GridFS Stores and Retrieves Files11. {
"_id" : <unspecified>, //file ID
"length" : data_number, // bytes
"chunkSize" : data_number, // :256KB
"uploadDate" : data_date, //file
"md5" : data_string //"filemd5"
}
12. {
"filename" : data_string,
"contentType" : data_string,
"aliases" : data_array of data_string,
"metadata" : data_object,
}
13. {
"_id" : <unspecified>, //chunk ID
"files_id" : <unspecified>, //file _id
"n" : chunk_number, //0 chunk
0,1,2... chunk
"data" : data_binary, // BSON
}
15. ➜ ~ mongofiles --help
usage: mongofiles [options] command [gridfs filename]
command:
one of (list|search|put|get)
list - file
search - file
put - file
get - file
delete - file
16. ➜ ~ mongofiles list
connected to: 127.0.0.1
➜ ~ mongofiles put ByeForNow.mp3
connected to: 127.0.0.1
added file: { _id: ObjectId('4df17f8d9d47ba5c0247e72e'),
filename: "ByeForNow.mp3", chunkSize: 262144, uploadDate:
new Date(1307672462538), md5:
"9ee9472200a2e18bf376ce622c3b0055", length: 11183104 }
done!
➜ ~ mongofiles list -v // -v
Fri Jun 10 11:21:05 creating new connection to:127.0.0.1
Fri Jun 10 11:21:05 BackgroundJob starting:
connected to: 127.0.0.1
ByeForNow.mp3 11183104
17. ➜ ~ mongofiles put Maria.mp3
connected to: 127.0.0.1
added file: { _id: ObjectId('4df181fc5e354129e833193f'),
filename: "Maria.mp3", chunkSize: 262144, uploadDate: new
Date(1307673086293), md5:
"9d4f424fa1843711e196e502d8a00183", length: 12225353 }
done!
➜ ~ mongofiles list
connected to: 127.0.0.1
ByeForNow.mp3 11183104
Maria.mp3 12225353
➜ ~ mongofiles list M
connected to: 127.0.0.1
Maria.mp3 12225353
➜ ~ mongofiles search .mp3
connected to: 127.0.0.1
ByeForNow.mp3 11183104
Maria.mp3 12225353
18. ➜ ~ mkdir tmp // get
➜ ~ cd tmp
➜ mongofiles get Maria.mp3
connected to: 127.0.0.1
done write to: Maria.mp3
➜ ls
Maria.mp3
➜ mongofiles get Bye
ERROR: file not found
➜ mongofiles get ByeForNow.mp3
done write to: ByeForNow.mp3
➜ md5 Maria.mp3
MD5 (Maria.mp3) = 9d4f424fa1843711e196e502d8a00183
➜ md5 ../Maria.mp3
MD5 (../Maria.mp3) = 9d4f424fa1843711e196e502d8a00183
19. > db.fs.files.find().forEach(printjson)
{
"_id" : ObjectId("4df17f8d9d47ba5c0247e72e"),
"filename" : "ByeForNow.mp3",
"chunkSize" : 262144,
"uploadDate" : ISODate("2011-06-10T02:21:02.538Z"),
"md5" : "9ee9472200a2e18bf376ce622c3b0055",
"length" : 11183104
}
{
"_id" : ObjectId("4df181fc5e354129e833193f"),
"filename" : "Maria.mp3",
"chunkSize" : 262144,
"uploadDate" : ISODate("2011-06-10T02:31:26.293Z"),
"md5" : "9d4f424fa1843711e196e502d8a00183",
"length" : 12225353
}
20. > db.fs.chunks.findOne(
{n:0,files_id:ObjectId("4df181fc5e354129e833193f")})
{
"_id" : ObjectId("4df181fcd40994ca1de28d09"),
"files_id" : ObjectId("4df181fc5e354129e833193f"),
"n" : 0,
"data" : BinData(0,"SUQzAwAAAABQdlRSQ0sAAAACAAAAM1RJVDIA
AAAHAAAAg32DioNBVUZJRAAAAGYAAGh0dHA6Ly93d3cuY2RkYi5jb20vaWQ
zL3RhZ2luZm8xLmh0bWwAM0NEM00xMTBRMjA3NTYwOTc4VjY4MTBCQTlBNj
UzN0JCQUQ1QTgyOUE4NTRCRkQ2QTdBRjNQOAAAAAAAAAAAAEdFT0IAACNgA
AAAYXBwbGljYX...)
}
21. > db.fs.chunks.ensureIndex({files_id: 1});
> db.runCommand({ shardcollection : "test.fs.chunks", key :
{ files_id : 1 }})
{ "collectionsharded" : "test.fs.chunks", "ok" : 1 }
26. { loc : [ 50 , 30 ] }
{ loc : { x : 50 , y : 30 } }
{ loc : { foo : 50 , y : 30 } }
{ loc : { long : 40.739037, lat: 73.992964 } }
28. db.places.find( { loc : [50,50] } )
db.places.find( { loc : { $near : [50,50] } } )
db.places.find( { loc : { $near : [50,50] } } ).limit(20)
db.places.find( { loc : { $near : [50,50] ,
$maxDistance : 5 } } ).limit(20)
29. //
> box = [[40.73083, -73.99756], [40.741404, -73.988135]]
> db.places.find({"loc" : {"$within" : {"$box" : box}}})
//
> center = [50, 50]
> radius = 10
> db.places.find({"loc" : {"$within" : {"$center" : [center, radius]}}})
// v1.9
> polygonA = [ [ 10, 20 ], [ 10, 40 ], [ 30, 40 ], [ 30, 20 ] ]
> polygonB = { a : { x : 10, y : 20 }, b : { x : 15, y : 25 }, c : { x :
20, y : 20 } }
> db.places.find({ "loc" : { "$within" : { "$polygon" : polygonA } } })
> db.places.find({ "loc" : { "$within" : { "$polygon" : polygonB } } })
30. > db.runCommand( { geoNear : "places" , near : [ 50 , 50 ], num : 10,
query : { type : "museum" } } );
{
"ns" : "test.places",
"near" : "1100110000001111110000001111110000001111110000001111",
"results" : [
{
"dis" : 69.29646421910687,
"obj" : {
"_id" : ObjectId("4b8bd6b93b83c574d8760280"),
"y" : [
1,
1
],
"category" : "Coffee"
}
},
{
"dis" : 69.29646421910687,
"obj" : {
"_id" : ObjectId("4b8bd6b03b83c574d876027f"),
1
...}
33. > db.points.insert({ pos : { long : 30, lat : 30 } })
> db.points.insert({ pos : { long : -10, lat : -20 } })
> db.points.ensureIndex({ pos : "2d" })
> db.points.find({ pos: { $nearSphere: [0,0], $maxDistance : 0.4 } })
{ "_id" : ObjectId("4df11e47b8e84370f84afdd3"), "pos" : { "long" : -10,
"lat" : -20 } }
34. > var earthRadius = 6378 // km
> var range = 3000 // km
> distances = db.runCommand({ geoNear : "points", near : [0, 0], spherical :
true, maxDistance : range / earthRadius }).results
[
{
"dis" : 0.3886630122897946, //
"obj" : {
"_id" : ObjectId("4df11e47b8e84370f84afdd3"),
"pos" : {
"long" : -10,
"lat" : -20
}
}
}
]
> pointDistance = distances[0].dis * earthRadius
2478.89269238431 // km
46. ~ mkdir -p shard/shard00
~ mkdir -p shard/shard01
~ mkdir -p shard/shard02
~ mkdir -p shard/config
~ mongod --shardsvr --port 27017 --dbpath shard/shard00
~ mongod --shardsvr --port 27018 --dbpath shard/shard01
~ mongod --shardsvr --port 27019 --dbpath shard/shard02
~ mongod --configsvr --port 27020 --dbpath shard/config
~ mongos --configdb localhost:27020 --port 27021
47. ➜ ~ mongo localhost:27021 // mongos
MongoDB shell version: 1.8.0
connecting to: localhost:27021/test
> show dbs
config 0.1875GB // sharding
> db.adminCommand( { addshard: "localhost:27017", name: "shard00" } )
{ "shardAdded" : "shard00", "ok" : 1 }
> db.adminCommand( { addshard: "localhost:27018", name: "shard01" } )
{ "shardAdded" : "shard01", "ok" : 1 }
> db.adminCommand( { addshard: "localhost:27019", name: "shard02" } )
{ "shardAdded" : "shard02", "ok" : 1 }
48. // mongos
> db.adminCommand( { enablesharding : "test" } )
{ "ok" : 1 }
> db.adminCommand( { moveprimary : "test", to : "shard02" } );
{ "primary " : "shard02:localhost:27019", "ok" : 1 }
> db.adminCommand( { shardcollection : "test.myshard", key : { n : 1 } } )
{ "collectionsharded" : "test.myshard", "ok" : 1 }
49. // mongos
> db.adminCommand({split : "test.myshard", middle : { n: 0 } } )
{ "ok" : 1 }
> db.adminCommand({split : "test.myshard", middle : { n: 1 } } )
{ "ok" : 1 }
> db.adminCommand({split : "test.myshard", middle : { n: 2 } } )
{ "ok" : 1 }
50. > db.printShardingStatus() //
--- Sharding Status ---
sharding version: { "_id" : 1, "version" : 3 }
shards:
{ "_id" : "shard00", "host" : "localhost:27017" }
{ "_id" : "shard01", "host" : "localhost:27018" }
{ "_id" : "shard02", "host" : "localhost:27019" }
databases:
{ "_id" : "admin", "partitioned" : false, "primary" : "config" }
{ "_id" : "test", "partitioned" : true, "primary" : "shard02" }
test.myshard chunks:
shard02 4
{ "n" : { $minKey : 1 } } -->> { "n" : 0 } on : shard02 { "t" : 1000, "i" : 1 }
{ "n" : 0 } -->> { "n" : 1 } on : shard02 { "t" : 1000, "i" : 3 }
{ "n" : 1 } -->> { "n" : 2 } on : shard02 { "t" : 1000, "i" : 5 }
{ "n" : 2 } -->> { "n" : { $maxKey : 1 } } on : shard02 { "t" : 1000, "i" : 6 }
51. // mongos
> db.adminCommand({moveChunk: "test.myshard", find: { n: 0 }, to: "shard00" });
{ "millis" : 1051, "ok" : 1 }
> db.adminCommand({moveChunk: "test.myshard", find: { n: 1 }, to: "shard01" });
{ "millis" : 1046, "ok" : 1 }
> db.adminCommand({moveChunk :"test.myshard", find: { n: 2 }, to: "shard02" });
{ "ok" : 0, "errmsg" : "that chunk is already on that shard" }
52. > db.printShardingStatus()
--- Sharding Status ---
sharding version: { "_id" : 1, "version" : 3 }
shards:
{ "_id" : "shard00", "host" : "localhost:27017" }
{ "_id" : "shard01", "host" : "localhost:27018" }
{ "_id" : "shard02", "host" : "localhost:27019" }
databases:
{ "_id" : "admin", "partitioned" : false, "primary" : "config" }
{ "_id" : "test", "partitioned" : true, "primary" : "shard02" }
test.myshard chunks:
shard02 2
shard00 1
shard01 1
{ "n" : { $minKey : 1 } } -->> { "n" : 0 } on : shard02 { "t" : 3000, "i" : 1 }
{ "n" : 0 } -->> { "n" : 1 } on : shard00 { "t" : 2000, "i" : 0 }
{ "n" : 1 } -->> { "n" : 2 } on : shard01 { "t" : 3000, "i" : 0 }
{ "n" : 2 } -->> { "n" : { $maxKey : 1 } } on : shard02 { "t" : 1000, "i" : 6 }
53. // mongos
> for(var m=0; m<100; m++){ db.myshard.insert({n: m % 3}) } // n =0,1,2,0,1,...
> printShardingSizes() // Shard
...
test.myshard chunks:
{ "n" : { $minKey : 1 } } -->> { "n" : 0 } on : shard02 { "estimate" :
false, "size" : 0, "numObjects" : 0 }
{ "n" : 0 } -->> { "n" : 1 } on : shard00 { "estimate" : false, "size" :
1224, "numObjects" : 34 }
{ "n" : 1 } -->> { "n" : 2 } on : shard01 { "estimate" : false, "size" :
1188, "numObjects" : 33 }
{ "n" : 2 } -->> { "n" : { $maxKey : 1 } } on : shard02 { "estimate" :
false, "size" : 1188, "numObjects" : 33 }
54. ➜ ~ mongo localhost:27017
> db.myshard.count()
34
> db.myshard.distinct("n")
[ 0 ]
➜ ~ mongo localhost:27018
> db.myshard.count()
33
> db.myshard.distinct("n")
[ 1 ]
➜ ~ mongo localhost:27019
> db.myshard.count()
33
> db.myshard.distinct("n")
[ 2 ]
57. > db.adminCommand( { addshard: "set00/delta1:27017,delta2:27017", name:
"shard00" } )
{ "shardAdded" : "shard00", "ok" : 1 }
> db.adminCommand( { addshard: "set01/delta3:27018,delta4:27018", name:
"shard01" } )
{ "shardAdded" : "shard01", "ok" : 1 }
> db.adminCommand( { addshard: "set02/delta5:27019,delta6:27019", name:
"shard02" } )
{ "shardAdded" : "shard02", "ok" : 1 }
67. db.mycoll.mapReduce(
map : <map >,
reduce : <reduce >
[, finalize : <finalize >]
[, query : < >]
[, sort : <
reduce >]
[, out : < >]
[, scope : <object where fields go into javascript global scope >]
);
68. db.people.insert({userId:1, age:24, type:"A"})
db.people.insert({userId:2, age:47, type:"B"})
db.people.insert({userId:3, age:34, type:"C"})
db.people.insert({userId:4, age:4, type:"B"})
db.people.insert({userId:5, age:14, type:"A"})
db.people.insert({userId:6, age:29, type:"C"})
db.people.insert({userId:7, age:39, type:"C"})
db.people.insert({userId:8, age:42, type:"B"})
db.people.insert({userId:9, age:56, type:"A"})
db.people.insert({userId:10, age:12, type:"A"})
70. r = function(key, values) {
var result = { age: 0, num: 0 };
values.forEach( function(value){
result.age += value.age; // age
result.num += value.num; //
} );
return result;
}
71. r = function(key, values) {
var num = 0;
var sum = 0;
values.forEach( function(value){
sum += value.age; // age
num += value.num; //
} );
return sum/num;
}
72. m = function() {
emit(this.type, { age: this.age, num: 1 } );
}
r = function(key, values) {
var result = { age: 0, num: 0 };
values.forEach( function(value){
result.age += value.age; // age
result.num += value.num; //
} );
return result;
}
73. m = function() {
emit(this.type, { age: this.age, num: 1 } );
}
r = function(key, values) {
var result = { age: 0, num: 0 };
values.forEach( function(value){
result.age += value.age; //
result.num += value.num; //
} );
return result;
}
74. f = function( key, result ){
result.avg = result.age / result.num;
return result;
}
75. > res = db.people.mapReduce( m , r ,
{ finalize : f , out : "mr_out", verbose: true });
{
"result" : "mr_out", //
"timeMillis" : 18, //
"timing" : {
"mapTime" : NumberLong(1),
"emitLoop" : 12,
"total" : 18
},
"counts" : {
"input" : 10, // 10
"emit" : 10, // 10 emit
"output" : 3 // 3
},
"ok" : 1,
}
76. > db.mr_out.find()
{ "_id" : "A",
"value" : { "age" : 106, "num" : 4, "avg" : 26.5 } }
{ "_id" : "B",
"value" : { "age" : 93, "num" : 3, "avg" : 31 } }
{ "_id" : "C",
"value" : { "age" : 102, "num" : 3, "avg" : 34 } }
77. > res = db.people.mapReduce( m , r ,
{ finalize : f , out : "mr_out", query: { age: { $gte: 20 } } });
{
"result" : "mr_out",
"timeMillis" : 35,
"counts" : {
"input" : 7, // 3
"emit" : 7,
"output" : 3
},
"ok" : 1,
}
78. > db.mr_out.find() //
{ "_id" : "A",
"value" : { "age" : 80, "num" : 2, "avg" : 40 } }
{ "_id" : "B",
"value" : { "age" : 89, "num" : 2, "avg" : 44.5 } }
{ "_id" : "C",
"value" : { "age" : 102, "num" : 3, "avg" : 34 } }
81. db.items.insert({ date: "2011-06-01", item: "apple", price: 100 })
db.items.insert({ date: "2011-06-01", item: "banana", price: 200 })
db.items.insert({ date: "2011-06-01", item: "apple" , price :100})
db.items.insert({ date: "2011-06-02", item: "orange", price: 50 })
db.items.insert({ date: "2011-06-02", item: "orange", price: 50 })
db.items.insert({ date: "2011-06-02", item: "apple", price: 100 })
82. m = function() { emit(this.item, this.price); }
r = function(key,values) {
var result = 0;
values.forEach( function(value){ result += value } );
return result;
}
> res = db.items.mapReduce( m, r, { query: {date: "2011-06-01"} ,
out: {replace: "mr_out2"}} );
{
"result" : "mr_out2",
"timeMillis" : 10,
"counts" : {
"input" : 3,
"emit" : 3,
"output" : 2
},
"ok" : 1,
}
>db.mr_out2.find()
{ "_id" : "apple", "value" : 200 }
{ "_id" : "banana", "value" : 200 }
83. > res = db.items.mapReduce( m, r,
{ query: {date: "2011-06-02"} , out: {reduce: "mr_out2"}} );
{
"result" : "mr_out2",
"timeMillis" : 23,
"counts" : {
"input" : 3,
"emit" : 3,
"output" : 3
},
"ok" : 1,
}
> db.mr_out2.find()
{ "_id" : "apple", "value" : 300 } //
{ "_id" : "banana", "value" : 200 } //
{ "_id" : "orange", "value" : 100 } //