Skip to content

Commit

Permalink
WriteableTable.bulkAdd()
Browse files Browse the repository at this point in the history 
In response to issue #90, a bulk add method. Does support
hook('creating') if used. This method takes special care of not keeping
any closure reference to the added objects since there are many users
that have had memory consumption and performance issues when adding lots
of items.

I would in the long term also want to create a generic bulk()  method
that would take an array of operations that could be a mix of
add/put/update/deletes, but that will require a bit more complexity.
Starting with this method to get some feedback on whether it improves
performance as it is intended to do.
  • Loading branch information
@dfahlander
dfahlander committed Nov 6, 2015
1 parent b057cc6 commit 62c487a
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 29 deletions.
93 changes: 87 additions & 6 deletions src/Dexie.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1030,7 +1030,83 @@
}

derive(WriteableTable).from(Table).extend(function () {

function BulkErrorHandler(errorList, resolve, hookCtx) {
return function(ev) {
if (ev.stopPropagation) ev.stopPropagation();
if (ev.preventDefault) ev.preventDefault();
errorList.push(ev.target.error);
if (hookCtx && hookCtx.onerror) hookCtx.onerror(ev.target.error);
if (resolve) resolve(errorList); // Only done in last request.
}
}

function BulkSuccessHandler(errorList, resolve, hookCtx) {
return hookCtx ? function(ev) {
hookCtx.onsuccess && hookCtx.onsuccess(ev.target.result);
if (resolve) resolve(errorList);
} : function() {
resolve(errorList);
};
}

return {
bulkAdd: function(objects) {
var self = this,
creatingHook = this.hook.creating.fire;
return this._idbstore(READWRITE, function (resolve, reject, idbstore, trans) {
if (!idbstore.keyPath) throw new Error("bulkAdd() only support inbound keys");
if (objects.length === 0) return resolve([]); // Caller provided empty list.
var req,
errorList = [],
errorHandler,
successHandler;
if (creatingHook !== nop) {
//
// There are subscribers to hook('creating')
// Must behave as documented.
//
var keyPath = idbstore.keyPath,
hookCtx = { onerror: null, onsuccess: null };
errorHandler = BulkErrorHandler(errorList, null, hookCtx);
successHandler = BulkSuccessHandler(errorList, null, hookCtx);
for (var i = 0, l = objects.length; i < l; ++i) {
var obj = objects[i],
effectiveKey = getByKeyPath(obj, keyPath),
keyToUse = creatingHook.call(hookCtx, effectiveKey, obj, trans);
if (effectiveKey === undefined && keyToUse !== undefined) {
obj = deepClone(obj);
setByKeyPath(obj, keyPath, keyToUse);
}

req = idbstore.add(obj);
if (i < l - 1) {
req.onerror = errorHandler;
if (hookCtx.onsuccess)
req.onsuccess = successHandler;
// Reset event listeners for next iteration.
hookCtx.onerror = null;
hookCtx.onsuccess = null;
}
}
req.onerror = BulkErrorHandler(errorList, resolve, hookCtx);
req.onsuccess = BulkSuccessHandler(errorList, resolve, hookCtx);
} else {
//
// Standard Bulk (no 'creating' hook to care about)
//
errorHandler = BulkErrorHandler(errorList);
for (var i = 0, l = objects.length; i < l; ++i) {
req = idbstore.add(objects[i]);
req.onerror = errorHandler;
}
// Only need to catch success or error on the last operation
// according to the IDB spec.
req.onerror = BulkErrorHandler(errorList, resolve);
req.onsuccess = BulkSuccessHandler(errorList, resolve);
}
});
},
add: function (obj, key) {
/// <summary>
/// Add an object to the database. In case an object with same primary key already exists, the object will not be added.
Expand All @@ -1040,7 +1116,7 @@
var self = this,
creatingHook = this.hook.creating.fire;
return this._idbstore(READWRITE, function (resolve, reject, idbstore, trans) {
var thisCtx = {};
var thisCtx = {onsuccess:null, onerror:null};
if (creatingHook !== nop) {
var effectiveKey = key || (idbstore.keyPath ? getByKeyPath(obj, idbstore.keyPath) : undefined);
var keyToUse = creatingHook.call(thisCtx, effectiveKey, obj, trans); // Allow subscribers to when("creating") to generate the key.
Expand Down Expand Up @@ -2060,7 +2136,12 @@

function modifyItem(item, cursor, advance) {
currentKey = cursor.primaryKey;
var thisContext = { primKey: cursor.primaryKey, value: item };
var thisContext = {
primKey: cursor.primaryKey,
value: item,
onsuccess: null,
onerror: null
};
if (modifyer.call(thisContext, item) !== false) { // If a callback explicitely returns false, do not perform the update!
var bDelete = !thisContext.hasOwnProperty("value");
var req = (bDelete ? cursor.delete() : cursor.update(thisContext.value));
Expand Down Expand Up @@ -2697,8 +2778,8 @@
if (res !== undefined) arguments[0] = res;
var onsuccess = this.onsuccess, // In case event listener has set this.onsuccess
onerror = this.onerror; // In case event listener has set this.onerror
delete this.onsuccess;
delete this.onerror;
this.onsuccess = null;
this.onerror = null;
var res2 = f2.apply(this, arguments);
if (onsuccess) this.onsuccess = this.onsuccess ? callBoth(onsuccess, this.onsuccess) : onsuccess;
if (onerror) this.onerror = this.onerror ? callBoth(onerror, this.onerror) : onerror;
Expand All @@ -2713,8 +2794,8 @@
if (res !== undefined) extend(arguments[0], res); // If f1 returns new modifications, extend caller's modifications with the result before calling next in chain.
var onsuccess = this.onsuccess, // In case event listener has set this.onsuccess
onerror = this.onerror; // In case event listener has set this.onerror
delete this.onsuccess;
delete this.onerror;
this.onsuccess = null;
this.onerror = null;
var res2 = f2.apply(this, arguments);
if (onsuccess) this.onsuccess = this.onsuccess ? callBoth(onsuccess, this.onsuccess) : onsuccess;
if (onerror) this.onerror = this.onerror ? callBoth(onerror, this.onerror) : onerror;
Expand Down
46 changes: 23 additions & 23 deletions test/tests-performance.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,36 +24,36 @@
});

var tick;
db.delete().then(function () {

function randomString(count) {
return function () {
var ms = [];
for (var i = 0; i < count; ++i) {
ms.push(String.fromCharCode(32 + Math.floor(Math.random() * 96)));
}
return ms.join('');
function randomString(count) {
return function () {
var ms = [];
for (var i = 0; i < count; ++i) {
ms.push(String.fromCharCode(32 + Math.floor(Math.random() * 96)));
}
return ms.join('');
}
}
db.delete().then(function () {
return db.open();
}).then(function(){

var i;
var bulkArray = new Array(10000);
for (i = 1; i <= 10000; ++i) {
bulkArray[i - 1] = {
from: "from" + i + "@test.com",
to: "to" + i + "@test.com",
subject: "subject" + i,
message: "message" + i,
shortStr: randomString(2)()
};
}

db.open();

tick = Date.now();

// Create 10,000 emails
ok(true, "Creating 10,000 emails");
return db.transaction("rw", db.emails, function() {
for (var i = 1; i <= 10000; ++i) {
db.emails.add({
from: "from" + i + "@test.com",
to: "to" + i + "@test.com",
subject: "subject" + i,
message: "message" + i,
shortStr: randomString(2)()
});
}
});

return db.emails.bulkAdd(bulkArray);
}).then(function () {
ok(true, "Time taken: " + (Date.now() - tick));

Expand Down
19 changes: 19 additions & 0 deletions test/tests-table.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,25 @@
ok(false, "Error: " + e);
}).finally(start);
});
asyncTest("bulkAdd", function() {
db.transaction("rw", db.users, function() {
var newUsers = [
{ first: "Åke1", last: "Persbrant1", username: "aper1", email: ["[email protected]"] },
{ first: "Åke2", last: "Persbrant2", username: "aper2", email: ["[email protected]"] },
{ first: "Åke2", last: "Persbrant2", username: "aper2", email: ["[email protected]"] }, // Should fail
{ first: "Åke3", last: "Persbrant3", username: "aper3", email: ["[email protected]"] }
];
db.users.bulkAdd(newUsers).then(function(errors) {
equal(errors.length, 1, "One error due to a duplicate username");
});

db.users.where("username").startsWith("aper").count(function(count) {
equal(count, 3);
});
}).catch(function (e) {
ok(false, "Error: " + e);
}).finally(start);
});
asyncTest("delete", function () {
// Without transaction
db.users.get(idOfFirstUser, function (user) {
Expand Down

0 comments on commit 62c487a

Please sign in to comment.